X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fintel%2Fcompiler%2Fbrw_eu_compact.c;h=d7c352236429bc0dc70093952c6ae01675b45ff8;hb=8d8a3815ef698348857cd9812eaa0dc76f5b19b5;hp=51dce9efbddc4eab0a763d0fd3c495cfd1fded80;hpb=2393cc7f00ee1b94c07e29fb82fc14add89e1238;p=mesa.git diff --git a/src/intel/compiler/brw_eu_compact.c b/src/intel/compiler/brw_eu_compact.c index 51dce9efbdd..d7c35223642 100644 --- a/src/intel/compiler/brw_eu_compact.c +++ b/src/intel/compiler/brw_eu_compact.c @@ -1,5 +1,5 @@ /* - * Copyright © 2012 Intel Corporation + * Copyright © 2012-2018 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -70,6 +70,12 @@ * while JIP (must be negative) * * Gen 8 adds support for compacting 3-src instructions. + * + * Gen12 reduces the number of bits that available to compacted immediates from + * 13 to 12, but improves the compaction of floating-point immediates by + * allowing the high bits to be encoded (the sign, 8-bit exponent, and the + * three most significant bits of the mantissa), rather than the lowest bits of + * the mantissa. */ #include "brw_eu.h" @@ -672,6 +678,149 @@ static const uint32_t gen11_datatype_table[32] = { 0b001001011001001001000, }; +static const uint32_t gen12_control_index_table[32] = { + 0b000000000000000000100, /* (16|M0) */ + 0b000000000000000000011, /* (8|M0) */ + 0b000000010000000000000, /* (W) (1|M0) */ + 0b000000010000000000100, /* (W) (16|M0) */ + 0b000000010000000000011, /* (W) (8|M0) */ + 0b010000000000000000100, /* (16|M0) (ge)f0.0 */ + 0b000000000000000100100, /* (16|M16) */ + 0b010100000000000000100, /* (16|M0) (lt)f0.0 */ + 0b000000000000000000000, /* (1|M0) */ + 0b000010000000000000100, /* (16|M0) (sat) */ + 0b000000000000000010011, /* (8|M8) */ + 0b001100000000000000100, /* (16|M0) (gt)f0.0 */ + 0b000100000000000000100, /* (16|M0) (eq)f0.0 */ + 0b000100010000000000100, /* (W) (16|M0) (eq)f0.0 */ + 0b001000000000000000100, /* (16|M0) (ne)f0.0 */ + 0b000000000000100000100, /* (f0.0) (16|M0) */ + 0b010100000000000000011, /* (8|M0) (lt)f0.0 */ + 0b000000000000110000100, /* (f1.0) (16|M0) */ + 0b000000010000000000001, /* (W) (2|M0) */ + 0b000000000000101000100, /* (f0.1) (16|M0) */ + 0b000000000000111000100, /* (f1.1) (16|M0) */ + 0b010000010000000000100, /* (W) (16|M0) (ge)f0.0 */ + 0b000000000000000100011, /* (8|M16) */ + 0b000000000000000110011, /* (8|M24) */ + 0b010100010000000000100, /* (W) (16|M0) (lt)f0.0 */ + 0b010000000000000000011, /* (8|M0) (ge)f0.0 */ + 0b000100010000000000000, /* (W) (1|M0) (eq)f0.0 */ + 0b000010000000000000011, /* (8|M0) (sat) */ + 0b010100000000010000100, /* (16|M0) (lt)f1.0 */ + 0b000100000000000000011, /* (8|M0) (eq)f0.0 */ + 0b000001000000000000011, /* (8|M0) {AccWrEn} */ + 0b000000010000000100100, /* (W) (16|M16) */ +}; + +static const uint32_t gen12_datatype_table[32] = { + 0b11010110100101010100, /* grf<1>:f grf:f grf:f */ + 0b00000110100101010100, /* grf<1>:f grf:f arf:ub */ + 0b00000010101101010100, /* grf<1>:f imm:f arf:ub */ + 0b01010110110101010100, /* grf<1>:f grf:f imm:f */ + 0b11010100100101010100, /* arf<1>:f grf:f grf:f */ + 0b11010010100101010100, /* grf<1>:f arf:f grf:f */ + 0b01010100110101010100, /* arf<1>:f grf:f imm:f */ + 0b00000000100000000000, /* arf<1>:ub arf:ub arf:ub */ + 0b11010000100101010100, /* arf<1>:f arf:f grf:f */ + 0b00101110110011001100, /* grf<1>:d grf:d imm:w */ + 0b10110110100011001100, /* grf<1>:d grf:d grf:d */ + 0b01010010110101010100, /* grf<1>:f arf:f imm:f */ + 0b10010110100001000100, /* grf<1>:ud grf:ud grf:ud */ + 0b01010000110101010100, /* arf<1>:f arf:f imm:f */ + 0b00110110110011001100, /* grf<1>:d grf:d imm:d */ + 0b00010110110001000100, /* grf<1>:ud grf:ud imm:ud */ + 0b00000111000101010100, /* grf<2>:f grf:f arf:ub */ + 0b00101100110011001100, /* arf<1>:d grf:d imm:w */ + 0b00000000100000100010, /* arf<1>:uw arf:uw arf:ub */ + 0b00000010100001000100, /* grf<1>:ud arf:ud arf:ub */ + 0b00100110110000101010, /* grf<1>:w grf:uw imm:uv */ + 0b00001110110000100010, /* grf<1>:uw grf:uw imm:uw */ + 0b10010111000001000100, /* grf<2>:ud grf:ud grf:ud */ + 0b00000110100101001100, /* grf<1>:d grf:f arf:ub */ + 0b10001100100011001100, /* arf<1>:d grf:d grf:uw */ + 0b00000110100001010100, /* grf<1>:f grf:ud arf:ub */ + 0b00101110110001001100, /* grf<1>:d grf:ud imm:w */ + 0b00000010100000100010, /* grf<1>:uw arf:uw arf:ub */ + 0b00000110100000110100, /* grf<1>:f grf:uw arf:ub */ + 0b00000110100000010100, /* grf<1>:f grf:ub arf:ub */ + 0b00000110100011010100, /* grf<1>:f grf:d arf:ub */ + 0b00000010100101010100, /* grf<1>:f arf:f arf:ub */ +}; + +static const uint16_t gen12_subreg_table[32] = { + 0b000000000000000, /* .0 .0 .0 */ + 0b100000000000000, /* .0 .0 .16 */ + 0b001000000000000, /* .0 .0 .4 */ + 0b011000000000000, /* .0 .0 .12 */ + 0b000000010000000, /* .0 .4 .0 */ + 0b010000000000000, /* .0 .0 .8 */ + 0b101000000000000, /* .0 .0 .20 */ + 0b000000000001000, /* .8 .0 .0 */ + 0b000000100000000, /* .0 .8 .0 */ + 0b110000000000000, /* .0 .0 .24 */ + 0b111000000000000, /* .0 .0 .28 */ + 0b000001000000000, /* .0 .16 .0 */ + 0b000000000000100, /* .4 .0 .0 */ + 0b000001100000000, /* .0 .24 .0 */ + 0b000001010000000, /* .0 .20 .0 */ + 0b000000110000000, /* .0 .12 .0 */ + 0b000001110000000, /* .0 .28 .0 */ + 0b000000000011100, /* .28 .0 .0 */ + 0b000000000010000, /* .16 .0 .0 */ + 0b000000000001100, /* .12 .0 .0 */ + 0b000000000011000, /* .24 .0 .0 */ + 0b000000000010100, /* .20 .0 .0 */ + 0b000000000000010, /* .2 .0 .0 */ + 0b000000101000000, /* .0 .10 .0 */ + 0b000000001000000, /* .0 .2 .0 */ + 0b000000010000100, /* .4 .4 .0 */ + 0b000000001011100, /* .28 .2 .0 */ + 0b000000001000010, /* .2 .2 .0 */ + 0b000000110001100, /* .12 .12 .0 */ + 0b000000000100000, /* .0 .1 .0 */ + 0b000000001100000, /* .0 .3 .0 */ + 0b110001100000000, /* .0 .24 .24 */ +}; + +static const uint16_t gen12_src0_index_table[16] = { + 0b010001100100, /* r<8;8,1> */ + 0b000000000000, /* r<0;1,0> */ + 0b010001100110, /* -r<8;8,1> */ + 0b010001100101, /* (abs)r<8;8,1> */ + 0b000000000010, /* -r<0;1,0> */ + 0b001000000000, /* r<2;1,0> */ + 0b001001000000, /* r<2;4,0> */ + 0b001101000000, /* r<4;4,0> */ + 0b001000100100, /* r<2;2,1> */ + 0b001100000000, /* r<4;1,0> */ + 0b001000100110, /* -r<2;2,1> */ + 0b001101000100, /* r<4;4,1> */ + 0b010001100111, /* -(abs)r<8;8,1> */ + 0b000100000000, /* r<1;1,0> */ + 0b000000000001, /* (abs)r<0;1,0> */ + 0b111100010000, /* r[a]<1,0> */ +}; + +static const uint16_t gen12_src1_index_table[16] = { + 0b000100011001, /* r<8;8,1> */ + 0b000000000000, /* r<0;1,0> */ + 0b100100011001, /* -r<8;8,1> */ + 0b100000000000, /* -r<0;1,0> */ + 0b010100011001, /* (abs)r<8;8,1> */ + 0b100011010000, /* -r<4;4,0> */ + 0b000010000000, /* r<2;1,0> */ + 0b000010001001, /* r<2;2,1> */ + 0b100010001001, /* -r<2;2,1> */ + 0b000011010000, /* r<4;4,0> */ + 0b000011010001, /* r<4;4,1> */ + 0b000011000000, /* r<4;1,0> */ + 0b110100011001, /* -(abs)r<8;8,1> */ + 0b010000000000, /* (abs)r<0;1,0> */ + 0b110000000000, /* -(abs)r<0;1,0> */ + 0b100011010001, /* -r<4;4,1> */ +}; + /* This is actually the control index table for Cherryview (26 bits), but the * only difference from Broadwell (24 bits) is that it has two extra 0-bits at * the start. @@ -699,32 +848,160 @@ static const uint64_t gen8_3src_source_index_table[4] = { 0b0000001110010011100100111001000001111000000100000, }; -static const uint32_t *control_index_table; -static const uint32_t *datatype_table; -static const uint16_t *subreg_table; -static const uint16_t *src_index_table; +static const uint64_t gen12_3src_control_index_table[32] = { + 0b000001001010010101000000000000000100, /* (16|M0) grf<1>:f :f :f :f */ + 0b000001001010010101000000000000000011, /* (8|M0) grf<1>:f :f :f :f */ + 0b000001001000010101000000000000000011, /* (8|M0) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000000011, /* (W) (8|M0) grf<1>:f :f :f :f */ + 0b000001001000010101000010000000000011, /* (W) (8|M0) arf<1>:f :f :f :f */ + 0b000001001000010101000000000000010011, /* (8|M8) arf<1>:f :f :f :f */ + 0b000001001010010101000000000000010011, /* (8|M8) grf<1>:f :f :f :f */ + 0b000001001000010101000010000000010011, /* (W) (8|M8) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000010011, /* (W) (8|M8) grf<1>:f :f :f :f */ + 0b000001001010010101000010000000000100, /* (W) (16|M0) grf<1>:f :f :f :f */ + 0b000001001000010101000000000000000100, /* (16|M0) arf<1>:f :f :f :f */ + 0b000001001010010101010000000000000100, /* (16|M0) (sat)grf<1>:f :f :f :f */ + 0b000001001010010101000000000000100100, /* (16|M16) grf<1>:f :f :f :f */ + 0b000001001000010101000010000000000100, /* (W) (16|M0) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000000000, /* (W) (1|M0) grf<1>:f :f :f :f */ + 0b000001001010010101010000000000000011, /* (8|M0) (sat)grf<1>:f :f :f :f */ + 0b000001001000010101000010000000110011, /* (W) (8|M24) arf<1>:f :f :f :f */ + 0b000001001000010101000010000000100011, /* (W) (8|M16) arf<1>:f :f :f :f */ + 0b000001001010010101000010000000110011, /* (W) (8|M24) grf<1>:f :f :f :f */ + 0b000001001010010101000010000000100011, /* (W) (8|M16) grf<1>:f :f :f :f */ + 0b000001001000010101000000000000100011, /* (8|M16) arf<1>:f :f :f :f */ + 0b000001001000010101000000000000110011, /* (8|M24) arf<1>:f :f :f :f */ + 0b000001001010010101000000000000100011, /* (8|M16) grf<1>:f :f :f :f */ + 0b000001001010010101000000000000110011, /* (8|M24) grf<1>:f :f :f :f */ + 0b000001001000010101010000000000000100, /* (16|M0) (sat)arf<1>:f :f :f :f */ + 0b000001001010010101010010000000000100, /* (W) (16|M0) (sat)grf<1>:f :f :f :f */ + 0b000001001010010101000010000000100100, /* (W) (16|M16) grf<1>:f :f :f :f */ + 0b000001001010010001000010000000000000, /* (W) (1|M0) grf<1>:ud :ud :ud :ud */ + 0b000001001000010101000000000000100100, /* (16|M16) arf<1>:f :f :f :f */ + 0b000001001010010101010000000000100100, /* (16|M16) (sat)grf<1>:f :f :f :f */ + 0b000001001010010101000010000000000010, /* (W) (4|M0) grf<1>:f :f :f :f */ + 0b000001001000010101010000000000000011, /* (8|M0) (sat)arf<1>:f :f :f :f */ +}; + +static const uint32_t gen12_3src_source_index_table[32] = { + 0b100101100001100000000, /* grf<0;0> grf<8;1> grf<0> */ + 0b100101100001001000010, /* arf<4;1> grf<8;1> grf<0> */ + 0b101101100001101000011, /* grf<8;1> grf<8;1> grf<1> */ + 0b100101100001101000011, /* grf<8;1> grf<8;1> grf<0> */ + 0b101100000000101000011, /* grf<8;1> grf<0;0> grf<1> */ + 0b101101100001101001011, /* -grf<8;1> grf<8;1> grf<1> */ + 0b101001100001101000011, /* grf<8;1> arf<8;1> grf<1> */ + 0b100001100001100000000, /* grf<0;0> arf<8;1> grf<0> */ + 0b101101100001100000000, /* grf<0;0> grf<8;1> grf<1> */ + 0b101101100101101000011, /* grf<8;1> grf<8;1> -grf<1> */ + 0b101101110001101000011, /* grf<8;1> -grf<8;1> grf<1> */ + 0b101100000000100000000, /* grf<0;0> grf<0;0> grf<1> */ + 0b100001100001101000011, /* grf<8;1> arf<8;1> grf<0> */ + 0b100101110001100000000, /* grf<0;0> -grf<8;1> grf<0> */ + 0b100101110001101000011, /* grf<8;1> -grf<8;1> grf<0> */ + 0b100101100001101001011, /* -grf<8;1> grf<8;1> grf<0> */ + 0b100100000000101000011, /* grf<8;1> grf<0;0> grf<0> */ + 0b100101100001100001000, /* -grf<0;0> grf<8;1> grf<0> */ + 0b100100000000100000000, /* grf<0;0> grf<0;0> grf<0> */ + 0b101101110001100000000, /* grf<0;0> -grf<8;1> grf<1> */ + 0b100101100101100000000, /* grf<0;0> grf<8;1> -grf<0> */ + 0b101001100001100000000, /* grf<0;0> arf<8;1> grf<1> */ + 0b100101100101101000011, /* grf<8;1> grf<8;1> -grf<0> */ + 0b101101100101101001011, /* -grf<8;1> grf<8;1> -grf<1> */ + 0b101001100001101001011, /* -grf<8;1> arf<8;1> grf<1> */ + 0b101101110001101001011, /* -grf<8;1> -grf<8;1> grf<1> */ + 0b101100010000101000011, /* grf<8;1> -grf<0;0> grf<1> */ + 0b101100000100101000011, /* grf<8;1> grf<0;0> -grf<1> */ + 0b101101100001100001000, /* -grf<0;0> grf<8;1> grf<1> */ + 0b101101100101100000000, /* grf<0;0> grf<8;1> -grf<1> */ + 0b100100000100101000011, /* grf<8;1> grf<0;0> -grf<0> */ + 0b101001100101101000011, /* grf<8;1> arf<8;1> -grf<1> */ +}; + +static const uint32_t gen12_3src_subreg_table[32] = { + 0b00000000000000000000, /* .0 .0 .0 .0 */ + 0b00100000000000000000, /* .0 .0 .0 .4 */ + 0b00000000000110000000, /* .0 .12 .0 .0 */ + 0b10100000000000000000, /* .0 .0 .0 .20 */ + 0b10000000001110000000, /* .0 .28 .0 .16 */ + 0b01100000000000000000, /* .0 .0 .0 .12 */ + 0b01000000000000000000, /* .0 .0 .0 .8 */ + 0b00000010000000000000, /* .0 .0 .8 .0 */ + 0b00000001000000000000, /* .0 .0 .4 .0 */ + 0b11000000000000000000, /* .0 .0 .0 .24 */ + 0b10000000000000000000, /* .0 .0 .0 .16 */ + 0b11100000000000000000, /* .0 .0 .0 .28 */ + 0b00000110000000000000, /* .0 .0 .24 .0 */ + 0b00000000000010000000, /* .0 .4 .0 .0 */ + 0b00000100000000000000, /* .0 .0 .16 .0 */ + 0b00000011000000000000, /* .0 .0 .12 .0 */ + 0b00000101000000000000, /* .0 .0 .20 .0 */ + 0b00000111000000000000, /* .0 .0 .28 .0 */ + 0b00000000000100000000, /* .0 .8 .0 .0 */ + 0b00000000001000000000, /* .0 .16 .0 .0 */ + 0b00000000001100000000, /* .0 .24 .0 .0 */ + 0b00000000001010000000, /* .0 .20 .0 .0 */ + 0b00000000001110000000, /* .0 .28 .0 .0 */ + 0b11000000001110000000, /* .0 .28 .0 .24 */ + 0b00100000000100000000, /* .0 .8 .0 .4 */ + 0b00100000000110000000, /* .0 .12 .0 .4 */ + 0b01000000000110000000, /* .0 .12 .0 .8 */ + 0b10000000001100000000, /* .0 .24 .0 .16 */ + 0b10000000001010000000, /* .0 .20 .0 .16 */ + 0b01100000000010000000, /* .0 .4 .0 .12 */ + 0b10100000001110000000, /* .0 .28 .0 .20 */ + 0b01000000000010000000, /* .0 .4 .0 .8 */ +}; + +struct compaction_state { + const struct gen_device_info *devinfo; + const uint32_t *control_index_table; + const uint32_t *datatype_table; + const uint16_t *subreg_table; + const uint16_t *src0_index_table; + const uint16_t *src1_index_table; +}; + +static void compaction_state_init(struct compaction_state *c, + const struct gen_device_info *devinfo); static bool -set_control_index(const struct gen_device_info *devinfo, +set_control_index(const struct compaction_state *c, brw_compact_inst *dst, const brw_inst *src) { - uint32_t uncompacted = devinfo->gen >= 8 /* 17b/G45; 19b/IVB+ */ - ? (brw_inst_bits(src, 33, 31) << 16) | /* 3b */ - (brw_inst_bits(src, 23, 12) << 4) | /* 12b */ - (brw_inst_bits(src, 10, 9) << 2) | /* 2b */ - (brw_inst_bits(src, 34, 34) << 1) | /* 1b */ - (brw_inst_bits(src, 8, 8)) /* 1b */ - : (brw_inst_bits(src, 31, 31) << 16) | /* 1b */ - (brw_inst_bits(src, 23, 8)); /* 16b */ - - /* On gen7, the flag register and subregister numbers are integrated into - * the control index. - */ - if (devinfo->gen == 7) - uncompacted |= brw_inst_bits(src, 90, 89) << 17; /* 2b */ + const struct gen_device_info *devinfo = c->devinfo; + uint32_t uncompacted; /* 17b/G45; 19b/IVB+; 21b/TGL+ */ + + if (devinfo->gen >= 12) { + uncompacted = (brw_inst_bits(src, 95, 92) << 17) | /* 4b */ + (brw_inst_bits(src, 34, 34) << 16) | /* 1b */ + (brw_inst_bits(src, 33, 33) << 15) | /* 1b */ + (brw_inst_bits(src, 32, 32) << 14) | /* 1b */ + (brw_inst_bits(src, 31, 31) << 13) | /* 1b */ + (brw_inst_bits(src, 28, 28) << 12) | /* 1b */ + (brw_inst_bits(src, 27, 24) << 8) | /* 4b */ + (brw_inst_bits(src, 23, 22) << 6) | /* 2b */ + (brw_inst_bits(src, 21, 19) << 3) | /* 3b */ + (brw_inst_bits(src, 18, 16)); /* 3b */ + } else if (devinfo->gen >= 8) { + uncompacted = (brw_inst_bits(src, 33, 31) << 16) | /* 3b */ + (brw_inst_bits(src, 23, 12) << 4) | /* 12b */ + (brw_inst_bits(src, 10, 9) << 2) | /* 2b */ + (brw_inst_bits(src, 34, 34) << 1) | /* 1b */ + (brw_inst_bits(src, 8, 8)); /* 1b */ + } else { + uncompacted = (brw_inst_bits(src, 31, 31) << 16) | /* 1b */ + (brw_inst_bits(src, 23, 8)); /* 16b */ + + /* On gen7, the flag register and subregister numbers are integrated into + * the control index. + */ + if (devinfo->gen == 7) + uncompacted |= brw_inst_bits(src, 90, 89) << 17; /* 2b */ + } for (int i = 0; i < 32; i++) { - if (control_index_table[i] == uncompacted) { + if (c->control_index_table[i] == uncompacted) { brw_compact_inst_set_control_index(devinfo, dst, i); return true; } @@ -734,18 +1011,40 @@ set_control_index(const struct gen_device_info *devinfo, } static bool -set_datatype_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, - const brw_inst *src) +set_datatype_index(const struct compaction_state *c, brw_compact_inst *dst, + const brw_inst *src, bool is_immediate) { - uint32_t uncompacted = devinfo->gen >= 8 /* 18b/G45+; 21b/BDW+ */ - ? (brw_inst_bits(src, 63, 61) << 18) | /* 3b */ - (brw_inst_bits(src, 94, 89) << 12) | /* 6b */ - (brw_inst_bits(src, 46, 35)) /* 12b */ - : (brw_inst_bits(src, 63, 61) << 15) | /* 3b */ - (brw_inst_bits(src, 46, 32)); /* 15b */ + const struct gen_device_info *devinfo = c->devinfo; + uint32_t uncompacted; /* 18b/G45+; 21b/BDW+; 20b/TGL+ */ + + if (devinfo->gen >= 12) { + uncompacted = (brw_inst_bits(src, 91, 88) << 15) | /* 4b */ + (brw_inst_bits(src, 66, 66) << 14) | /* 1b */ + (brw_inst_bits(src, 50, 50) << 13) | /* 1b */ + (brw_inst_bits(src, 49, 48) << 11) | /* 2b */ + (brw_inst_bits(src, 47, 47) << 10) | /* 1b */ + (brw_inst_bits(src, 46, 46) << 9) | /* 1b */ + (brw_inst_bits(src, 43, 40) << 5) | /* 4b */ + (brw_inst_bits(src, 39, 36) << 1) | /* 4b */ + (brw_inst_bits(src, 35, 35)); /* 1b */ + + /* Src1.RegFile overlaps with the immediate, so ignore it if an immediate + * is present + */ + if (!is_immediate) { + uncompacted |= brw_inst_bits(src, 98, 98) << 19; /* 1b */ + } + } else if (devinfo->gen >= 8) { + uncompacted = (brw_inst_bits(src, 63, 61) << 18) | /* 3b */ + (brw_inst_bits(src, 94, 89) << 12) | /* 6b */ + (brw_inst_bits(src, 46, 35)); /* 12b */ + } else { + uncompacted = (brw_inst_bits(src, 63, 61) << 15) | /* 3b */ + (brw_inst_bits(src, 46, 32)); /* 15b */ + } for (int i = 0; i < 32; i++) { - if (datatype_table[i] == uncompacted) { + if (c->datatype_table[i] == uncompacted) { brw_compact_inst_set_datatype_index(devinfo, dst, i); return true; } @@ -755,18 +1054,28 @@ set_datatype_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, } static bool -set_subreg_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, +set_subreg_index(const struct compaction_state *c, brw_compact_inst *dst, const brw_inst *src, bool is_immediate) { - uint16_t uncompacted = /* 15b */ - (brw_inst_bits(src, 52, 48) << 0) | /* 5b */ - (brw_inst_bits(src, 68, 64) << 5); /* 5b */ + const struct gen_device_info *devinfo = c->devinfo; + uint16_t uncompacted; /* 15b */ - if (!is_immediate) - uncompacted |= brw_inst_bits(src, 100, 96) << 10; /* 5b */ + if (devinfo->gen >= 12) { + uncompacted = (brw_inst_bits(src, 55, 51) << 0) | /* 5b */ + (brw_inst_bits(src, 71, 67) << 5); /* 5b */ + + if (!is_immediate) + uncompacted |= brw_inst_bits(src, 103, 99) << 10; /* 5b */ + } else { + uncompacted = (brw_inst_bits(src, 52, 48) << 0) | /* 5b */ + (brw_inst_bits(src, 68, 64) << 5); /* 5b */ + + if (!is_immediate) + uncompacted |= brw_inst_bits(src, 100, 96) << 10; /* 5b */ + } for (int i = 0; i < 32; i++) { - if (subreg_table[i] == uncompacted) { + if (c->subreg_table[i] == uncompacted) { brw_compact_inst_set_subreg_index(devinfo, dst, i); return true; } @@ -776,12 +1085,28 @@ set_subreg_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, } static bool -get_src_index(uint16_t uncompacted, - uint16_t *compacted) +set_src0_index(const struct compaction_state *c, brw_compact_inst *dst, + const brw_inst *src) { - for (int i = 0; i < 32; i++) { - if (src_index_table[i] == uncompacted) { - *compacted = i; + const struct gen_device_info *devinfo = c->devinfo; + uint16_t uncompacted; /* 12b */ + int table_len; + + if (devinfo->gen >= 12) { + table_len = ARRAY_SIZE(gen12_src0_index_table); + uncompacted = (brw_inst_bits(src, 87, 84) << 8) | /* 4b */ + (brw_inst_bits(src, 83, 81) << 5) | /* 3b */ + (brw_inst_bits(src, 80, 80) << 4) | /* 1b */ + (brw_inst_bits(src, 65, 64) << 2) | /* 2b */ + (brw_inst_bits(src, 45, 44)); /* 2b */ + } else { + table_len = ARRAY_SIZE(gen8_src_index_table); + uncompacted = brw_inst_bits(src, 88, 77); /* 12b */ + } + + for (int i = 0; i < table_len; i++) { + if (c->src0_index_table[i] == uncompacted) { + brw_compact_inst_set_src0_index(devinfo, dst, i); return true; } } @@ -790,38 +1115,44 @@ get_src_index(uint16_t uncompacted, } static bool -set_src0_index(const struct gen_device_info *devinfo, - brw_compact_inst *dst, const brw_inst *src) -{ - uint16_t compacted; - uint16_t uncompacted = brw_inst_bits(src, 88, 77); /* 12b */ - - if (!get_src_index(uncompacted, &compacted)) - return false; - - brw_compact_inst_set_src0_index(devinfo, dst, compacted); - - return true; -} - -static bool -set_src1_index(const struct gen_device_info *devinfo, brw_compact_inst *dst, - const brw_inst *src, bool is_immediate) +set_src1_index(const struct compaction_state *c, brw_compact_inst *dst, + const brw_inst *src, bool is_immediate, unsigned imm) { - uint16_t compacted; - + const struct gen_device_info *devinfo = c->devinfo; if (is_immediate) { - compacted = (brw_inst_imm_ud(devinfo, src) >> 8) & 0x1f; + if (devinfo->gen >= 12) { + /* src1 index takes the low 4 bits of the 12-bit compacted value */ + brw_compact_inst_set_src1_index(devinfo, dst, imm & 0xf); + } else { + /* src1 index takes the high 5 bits of the 13-bit compacted value */ + brw_compact_inst_set_src1_index(devinfo, dst, imm >> 8); + } + return true; } else { - uint16_t uncompacted = brw_inst_bits(src, 120, 109); /* 12b */ + uint16_t uncompacted; /* 12b */ + int table_len; + + if (devinfo->gen >= 12) { + table_len = ARRAY_SIZE(gen12_src0_index_table); + uncompacted = (brw_inst_bits(src, 121, 120) << 10) | /* 2b */ + (brw_inst_bits(src, 119, 116) << 6) | /* 4b */ + (brw_inst_bits(src, 115, 113) << 3) | /* 3b */ + (brw_inst_bits(src, 112, 112) << 2) | /* 1b */ + (brw_inst_bits(src, 97, 96)); /* 2b */ + } else { + table_len = ARRAY_SIZE(gen8_src_index_table); + uncompacted = brw_inst_bits(src, 120, 109); /* 12b */ + } - if (!get_src_index(uncompacted, &compacted)) - return false; + for (int i = 0; i < table_len; i++) { + if (c->src1_index_table[i] == uncompacted) { + brw_compact_inst_set_src1_index(devinfo, dst, i); + return true; + } + } } - brw_compact_inst_set_src1_index(devinfo, dst, compacted); - - return true; + return false; } static bool @@ -830,17 +1161,48 @@ set_3src_control_index(const struct gen_device_info *devinfo, { assert(devinfo->gen >= 8); - uint32_t uncompacted = /* 24b/BDW; 26b/CHV */ - (brw_inst_bits(src, 34, 32) << 21) | /* 3b */ - (brw_inst_bits(src, 28, 8)); /* 21b */ + if (devinfo->gen >= 12) { + uint64_t uncompacted = /* 36b/TGL+ */ + (brw_inst_bits(src, 95, 92) << 32) | /* 4b */ + (brw_inst_bits(src, 90, 88) << 29) | /* 3b */ + (brw_inst_bits(src, 82, 80) << 26) | /* 3b */ + (brw_inst_bits(src, 50, 50) << 25) | /* 1b */ + (brw_inst_bits(src, 48, 48) << 24) | /* 1b */ + (brw_inst_bits(src, 42, 40) << 21) | /* 3b */ + (brw_inst_bits(src, 39, 39) << 20) | /* 1b */ + (brw_inst_bits(src, 38, 36) << 17) | /* 3b */ + (brw_inst_bits(src, 34, 34) << 16) | /* 1b */ + (brw_inst_bits(src, 33, 33) << 15) | /* 1b */ + (brw_inst_bits(src, 32, 32) << 14) | /* 1b */ + (brw_inst_bits(src, 31, 31) << 13) | /* 1b */ + (brw_inst_bits(src, 28, 28) << 12) | /* 1b */ + (brw_inst_bits(src, 27, 24) << 8) | /* 4b */ + (brw_inst_bits(src, 23, 23) << 7) | /* 1b */ + (brw_inst_bits(src, 22, 22) << 6) | /* 1b */ + (brw_inst_bits(src, 21, 19) << 3) | /* 3b */ + (brw_inst_bits(src, 18, 16)); /* 3b */ + + for (unsigned i = 0; i < ARRAY_SIZE(gen12_3src_control_index_table); i++) { + if (gen12_3src_control_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_control_index(devinfo, dst, i); + return true; + } + } + } else { + uint32_t uncompacted = /* 24b/BDW; 26b/CHV/SKL+ */ + (brw_inst_bits(src, 34, 32) << 21) | /* 3b */ + (brw_inst_bits(src, 28, 8)); /* 21b */ - if (devinfo->gen >= 9 || devinfo->is_cherryview) - uncompacted |= brw_inst_bits(src, 36, 35) << 24; /* 2b */ + if (devinfo->gen >= 9 || devinfo->is_cherryview) { + uncompacted |= + brw_inst_bits(src, 36, 35) << 24; /* 2b */ + } - for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_control_index_table); i++) { - if (gen8_3src_control_index_table[i] == uncompacted) { - brw_compact_inst_set_3src_control_index(devinfo, dst, i); - return true; + for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_control_index_table); i++) { + if (gen8_3src_control_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_control_index(devinfo, dst, i); + return true; + } } } @@ -853,27 +1215,75 @@ set_3src_source_index(const struct gen_device_info *devinfo, { assert(devinfo->gen >= 8); - uint64_t uncompacted = /* 46b/BDW; 49b/CHV */ - (brw_inst_bits(src, 83, 83) << 43) | /* 1b */ - (brw_inst_bits(src, 114, 107) << 35) | /* 8b */ - (brw_inst_bits(src, 93, 86) << 27) | /* 8b */ - (brw_inst_bits(src, 72, 65) << 19) | /* 8b */ - (brw_inst_bits(src, 55, 37)); /* 19b */ - - if (devinfo->gen >= 9 || devinfo->is_cherryview) { - uncompacted |= - (brw_inst_bits(src, 126, 125) << 47) | /* 2b */ - (brw_inst_bits(src, 105, 104) << 45) | /* 2b */ - (brw_inst_bits(src, 84, 84) << 44); /* 1b */ + if (devinfo->gen >= 12) { + uint32_t uncompacted = /* 21b/TGL+ */ + (brw_inst_bits(src, 114, 114) << 20) | /* 1b */ + (brw_inst_bits(src, 113, 112) << 18) | /* 2b */ + (brw_inst_bits(src, 98, 98) << 17) | /* 1b */ + (brw_inst_bits(src, 97, 96) << 15) | /* 2b */ + (brw_inst_bits(src, 91, 91) << 14) | /* 1b */ + (brw_inst_bits(src, 87, 86) << 12) | /* 2b */ + (brw_inst_bits(src, 85, 84) << 10) | /* 2b */ + (brw_inst_bits(src, 83, 83) << 9) | /* 1b */ + (brw_inst_bits(src, 66, 66) << 8) | /* 1b */ + (brw_inst_bits(src, 65, 64) << 6) | /* 2b */ + (brw_inst_bits(src, 47, 47) << 5) | /* 1b */ + (brw_inst_bits(src, 46, 46) << 4) | /* 1b */ + (brw_inst_bits(src, 45, 44) << 2) | /* 2b */ + (brw_inst_bits(src, 43, 43) << 1) | /* 1b */ + (brw_inst_bits(src, 35, 35)); /* 1b */ + + for (unsigned i = 0; i < ARRAY_SIZE(gen12_3src_source_index_table); i++) { + if (gen12_3src_source_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_source_index(devinfo, dst, i); + return true; + } + } } else { - uncompacted |= - (brw_inst_bits(src, 125, 125) << 45) | /* 1b */ - (brw_inst_bits(src, 104, 104) << 44); /* 1b */ + uint64_t uncompacted = /* 46b/BDW; 49b/CHV/SKL+ */ + (brw_inst_bits(src, 83, 83) << 43) | /* 1b */ + (brw_inst_bits(src, 114, 107) << 35) | /* 8b */ + (brw_inst_bits(src, 93, 86) << 27) | /* 8b */ + (brw_inst_bits(src, 72, 65) << 19) | /* 8b */ + (brw_inst_bits(src, 55, 37)); /* 19b */ + + if (devinfo->gen >= 9 || devinfo->is_cherryview) { + uncompacted |= + (brw_inst_bits(src, 126, 125) << 47) | /* 2b */ + (brw_inst_bits(src, 105, 104) << 45) | /* 2b */ + (brw_inst_bits(src, 84, 84) << 44); /* 1b */ + } else { + uncompacted |= + (brw_inst_bits(src, 125, 125) << 45) | /* 1b */ + (brw_inst_bits(src, 104, 104) << 44); /* 1b */ + } + + for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_source_index_table); i++) { + if (gen8_3src_source_index_table[i] == uncompacted) { + brw_compact_inst_set_3src_source_index(devinfo, dst, i); + return true; + } + } } - for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_source_index_table); i++) { - if (gen8_3src_source_index_table[i] == uncompacted) { - brw_compact_inst_set_3src_source_index(devinfo, dst, i); + return false; +} + +static bool +set_3src_subreg_index(const struct gen_device_info *devinfo, + brw_compact_inst *dst, const brw_inst *src) +{ + assert(devinfo->gen >= 12); + + uint32_t uncompacted = /* 20b/TGL+ */ + (brw_inst_bits(src, 119, 115) << 15) | /* 5b */ + (brw_inst_bits(src, 103, 99) << 10) | /* 5b */ + (brw_inst_bits(src, 71, 67) << 5) | /* 5b */ + (brw_inst_bits(src, 55, 51)); /* 5b */ + + for (unsigned i = 0; i < ARRAY_SIZE(gen12_3src_subreg_table); i++) { + if (gen12_3src_subreg_table[i] == uncompacted) { + brw_compact_inst_set_3src_subreg_index(devinfo, dst, i); return true; } } @@ -899,7 +1309,10 @@ has_unmapped_bits(const struct gen_device_info *devinfo, const brw_inst *src) * - Imm64[27:31] (bits 91-95 on Gen7, bit 95 on Gen8) * - UIP[31] (bit 95 on Gen8) */ - if (devinfo->gen >= 8) { + if (devinfo->gen >= 12) { + assert(!brw_inst_bits(src, 7, 7)); + return false; + } else if (devinfo->gen >= 8) { assert(!brw_inst_bits(src, 7, 7)); return brw_inst_bits(src, 95, 95) || brw_inst_bits(src, 47, 47) || @@ -920,7 +1333,9 @@ has_3src_unmapped_bits(const struct gen_device_info *devinfo, * fields of the compacted instruction. All of them seem to be reserved * bits currently. */ - if (devinfo->gen >= 9 || devinfo->is_cherryview) { + if (devinfo->gen >= 12) { + assert(!brw_inst_bits(src, 7, 7)); + } else if (devinfo->gen >= 9 || devinfo->is_cherryview) { assert(!brw_inst_bits(src, 127, 127) && !brw_inst_bits(src, 7, 7)); } else { @@ -928,8 +1343,11 @@ has_3src_unmapped_bits(const struct gen_device_info *devinfo, assert(!brw_inst_bits(src, 127, 126) && !brw_inst_bits(src, 105, 105) && !brw_inst_bits(src, 84, 84) && - !brw_inst_bits(src, 36, 35) && !brw_inst_bits(src, 7, 7)); + + /* Src1Type and Src2Type, used for mixed-precision floating point */ + if (brw_inst_bits(src, 36, 35)) + return true; } return false; @@ -949,7 +1367,7 @@ brw_try_compact_3src_instruction(const struct gen_device_info *devinfo, #define compact_a16(field) \ brw_compact_inst_set_3src_##field(devinfo, dst, brw_inst_3src_a16_##field(devinfo, src)) - compact(opcode); + compact(hw_opcode); if (!set_3src_control_index(devinfo, dst, src)) return false; @@ -957,19 +1375,31 @@ brw_try_compact_3src_instruction(const struct gen_device_info *devinfo, if (!set_3src_source_index(devinfo, dst, src)) return false; - compact(dst_reg_nr); - compact_a16(src0_rep_ctrl); + if (devinfo->gen >= 12) { + if (!set_3src_subreg_index(devinfo, dst, src)) + return false; + + compact(swsb); + compact(debug_control); + compact(dst_reg_nr); + compact(src0_reg_nr); + compact(src1_reg_nr); + compact(src2_reg_nr); + } else { + compact(dst_reg_nr); + compact_a16(src0_rep_ctrl); + compact(debug_control); + compact(saturate); + compact_a16(src1_rep_ctrl); + compact_a16(src2_rep_ctrl); + compact(src0_reg_nr); + compact(src1_reg_nr); + compact(src2_reg_nr); + compact_a16(src0_subreg_nr); + compact_a16(src1_subreg_nr); + compact_a16(src2_subreg_nr); + } brw_compact_inst_set_3src_cmpt_control(devinfo, dst, true); - compact(debug_control); - compact(saturate); - compact_a16(src1_rep_ctrl); - compact_a16(src2_rep_ctrl); - compact(src0_reg_nr); - compact(src1_reg_nr); - compact(src2_reg_nr); - compact_a16(src0_subreg_nr); - compact_a16(src1_subreg_nr); - compact_a16(src2_subreg_nr); #undef compact #undef compact_a16 @@ -977,20 +1407,143 @@ brw_try_compact_3src_instruction(const struct gen_device_info *devinfo, return true; } -/* Compacted instructions have 12-bits for immediate sources, and a 13th bit - * that's replicated through the high 20 bits. +/* On SNB through ICL, compacted instructions have 12-bits for immediate + * sources, and a 13th bit that's replicated through the high 20 bits. * * Effectively this means we get 12-bit integers, 0.0f, and some limited uses * of packed vectors as compactable immediates. + * + * On TGL+, the high 12-bits of floating-point values (:f and :hf) are encoded + * rather than the low 12-bits. For signed integer the 12th bit is replicated, + * while for unsigned integers it is not. + * + * Returns the compacted immediate, or -1 if immediate cannot be compacted */ +static int +compact_immediate(const struct gen_device_info *devinfo, + enum brw_reg_type type, unsigned imm) +{ + if (devinfo->gen >= 12) { + /* 16-bit immediates need to be replicated through the 32-bit immediate + * field + */ + switch (type) { + case BRW_REGISTER_TYPE_W: + case BRW_REGISTER_TYPE_UW: + case BRW_REGISTER_TYPE_HF: + if ((imm >> 16) != (imm & 0xffff)) + return -1; + break; + default: + break; + } + + switch (type) { + case BRW_REGISTER_TYPE_F: + /* We get the high 12-bits as-is; rest must be zero */ + if ((imm & 0xfffff) == 0) + return (imm >> 20) & 0xfff; + break; + case BRW_REGISTER_TYPE_HF: + /* We get the high 12-bits as-is; rest must be zero */ + if ((imm & 0xf) == 0) + return (imm >> 4) & 0xfff; + break; + case BRW_REGISTER_TYPE_UD: + case BRW_REGISTER_TYPE_VF: + case BRW_REGISTER_TYPE_UV: + case BRW_REGISTER_TYPE_V: + /* We get the low 12-bits as-is; rest must be zero */ + if ((imm & 0xfffff000) == 0) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_UW: + /* We get the low 12-bits as-is; rest must be zero */ + if ((imm & 0xf000) == 0) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_D: + /* We get the low 11-bits as-is; 12th is replicated */ + if (((int)imm >> 11) == 0 || ((int)imm >> 11) == -1) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_W: + /* We get the low 11-bits as-is; 12th is replicated */ + if (((short)imm >> 11) == 0 || ((short)imm >> 11) == -1) + return imm & 0xfff; + break; + case BRW_REGISTER_TYPE_NF: + case BRW_REGISTER_TYPE_DF: + case BRW_REGISTER_TYPE_Q: + case BRW_REGISTER_TYPE_UQ: + case BRW_REGISTER_TYPE_B: + case BRW_REGISTER_TYPE_UB: + return -1; + } + } else { + /* We get the low 12 bits as-is; 13th is replicated */ + if (((int)imm >> 12) == 0 || ((int)imm >> 12 == -1)) { + return imm & 0x1fff; + } + } + + return -1; +} + +static int +uncompact_immediate(const struct gen_device_info *devinfo, + enum brw_reg_type type, unsigned compact_imm) +{ + if (devinfo->gen >= 12) { + switch (type) { + case BRW_REGISTER_TYPE_F: + return compact_imm << 20; + case BRW_REGISTER_TYPE_HF: + return (compact_imm << 20) | (compact_imm << 4); + case BRW_REGISTER_TYPE_UD: + case BRW_REGISTER_TYPE_VF: + case BRW_REGISTER_TYPE_UV: + case BRW_REGISTER_TYPE_V: + return compact_imm; + case BRW_REGISTER_TYPE_UW: + /* Replicate */ + return compact_imm << 16 | compact_imm; + case BRW_REGISTER_TYPE_D: + /* Extend the 12th bit into the high 20 bits */ + return (int)(compact_imm << 20) >> 20; + case BRW_REGISTER_TYPE_W: + /* Extend the 12th bit into the high 4 bits and replicate */ + return ( (int)(compact_imm << 20) >> 4) | + ((short)(compact_imm << 4) >> 4); + case BRW_REGISTER_TYPE_NF: + case BRW_REGISTER_TYPE_DF: + case BRW_REGISTER_TYPE_Q: + case BRW_REGISTER_TYPE_UQ: + case BRW_REGISTER_TYPE_B: + case BRW_REGISTER_TYPE_UB: + unreachable("not reached"); + } + } else { + /* Replicate the 13th bit into the high 19 bits */ + return (int)(compact_imm << 19) >> 19; + } + + unreachable("not reached"); +} + static bool -is_compactable_immediate(unsigned imm) +has_immediate(const struct gen_device_info *devinfo, const brw_inst *inst, + enum brw_reg_type *type) { - /* We get the low 12 bits as-is. */ - imm &= ~0xfff; + if (brw_inst_src0_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) { + *type = brw_inst_src0_type(devinfo, inst); + return *type != INVALID_REG_TYPE; + } else if (brw_inst_src1_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) { + *type = brw_inst_src1_type(devinfo, inst); + return *type != INVALID_REG_TYPE; + } - /* We get one bit replicated through the top 20 bits. */ - return imm == 0 || imm == 0xfffff000; + return false; } /** @@ -1036,8 +1589,7 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) (brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_DF || brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_UQ || brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_Q))) { - enum brw_reg_file file = brw_inst_src1_reg_file(devinfo, &inst); - brw_inst_set_src1_file_type(devinfo, &inst, file, BRW_REGISTER_TYPE_UD); + brw_inst_set_src1_reg_hw_type(devinfo, &inst, 0); } /* Compacted instructions only have 12-bits (plus 1 for the other 20) @@ -1050,8 +1602,12 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) * to do this there. * * If we see a 0.0:F, change the type to VF so that it can be compacted. + * + * Compaction of floating-point immediates is improved on Gen12, thus + * removing the need for this. */ - if (brw_inst_imm_ud(devinfo, &inst) == 0x0 && + if (devinfo->gen < 12 && + brw_inst_imm_ud(devinfo, &inst) == 0x0 && brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_F && brw_inst_dst_type(devinfo, &inst) == BRW_REGISTER_TYPE_F && brw_inst_dst_hstride(devinfo, &inst) == BRW_HORIZONTAL_STRIDE_1) { @@ -1061,8 +1617,12 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) /* There are no mappings for dst:d | i:d, so if the immediate is suitable * set the types to :UD so the instruction can be compacted. + * + * FINISHME: Use dst:f | imm:f on Gen12 */ - if (is_compactable_immediate(brw_inst_imm_ud(devinfo, &inst)) && + if (devinfo->gen < 12 && + compact_immediate(devinfo, BRW_REGISTER_TYPE_D, + brw_inst_imm_ud(devinfo, &inst)) != -1 && brw_inst_cond_modifier(devinfo, &inst) == BRW_CONDITIONAL_NONE && brw_inst_src0_type(devinfo, &inst) == BRW_REGISTER_TYPE_D && brw_inst_dst_type(devinfo, &inst) == BRW_REGISTER_TYPE_D) { @@ -1082,10 +1642,11 @@ precompact(const struct gen_device_info *devinfo, brw_inst inst) * It doesn't modify dst unless src is compactable, which is relied on by * brw_compact_instructions(). */ -bool -brw_try_compact_instruction(const struct gen_device_info *devinfo, - brw_compact_inst *dst, const brw_inst *src) +static bool +try_compact_instruction(const struct compaction_state *c, + brw_compact_inst *dst, const brw_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; brw_compact_inst temp; assert(brw_inst_cmpt_control(devinfo, src) == 0); @@ -1104,13 +1665,20 @@ brw_try_compact_instruction(const struct gen_device_info *devinfo, } } - bool is_immediate = - brw_inst_src0_reg_file(devinfo, src) == BRW_IMMEDIATE_VALUE || - brw_inst_src1_reg_file(devinfo, src) == BRW_IMMEDIATE_VALUE; - if (is_immediate && - (devinfo->gen < 6 || - !is_compactable_immediate(brw_inst_imm_ud(devinfo, src)))) { - return false; + enum brw_reg_type type; + bool is_immediate = has_immediate(devinfo, src, &type); + + unsigned compacted_imm = 0; + + if (is_immediate) { + /* Instructions with immediates cannot be compacted on Gen < 6 */ + if (devinfo->gen < 6) + return false; + + compacted_imm = compact_immediate(devinfo, type, + brw_inst_imm_ud(devinfo, src)); + if (compacted_imm == -1) + return false; } if (has_unmapped_bits(devinfo, src)) @@ -1120,63 +1688,96 @@ brw_try_compact_instruction(const struct gen_device_info *devinfo, #define compact(field) \ brw_compact_inst_set_##field(devinfo, &temp, brw_inst_##field(devinfo, src)) +#define compact_reg(field) \ + brw_compact_inst_set_##field##_reg_nr(devinfo, &temp, \ + brw_inst_##field##_da_reg_nr(devinfo, src)) - compact(opcode); + compact(hw_opcode); compact(debug_control); - if (!set_control_index(devinfo, &temp, src)) + if (!set_control_index(c, &temp, src)) return false; - if (!set_datatype_index(devinfo, &temp, src)) + if (!set_datatype_index(c, &temp, src, is_immediate)) return false; - if (!set_subreg_index(devinfo, &temp, src, is_immediate)) + if (!set_subreg_index(c, &temp, src, is_immediate)) + return false; + if (!set_src0_index(c, &temp, src)) + return false; + if (!set_src1_index(c, &temp, src, is_immediate, compacted_imm)) return false; - if (devinfo->gen >= 6) { - compact(acc_wr_control); - } else { - compact(mask_control_ex); - } - - compact(cond_modifier); + if (devinfo->gen >= 12) { + compact(swsb); + compact_reg(dst); + compact_reg(src0); - if (devinfo->gen <= 6) - compact(flag_subreg_nr); + if (is_immediate) { + /* src1 reg takes the high 8 bits (of the 12-bit compacted value) */ + brw_compact_inst_set_src1_reg_nr(devinfo, &temp, compacted_imm >> 4); + } else { + compact_reg(src1); + } + } else { + if (devinfo->gen >= 6) { + compact(acc_wr_control); + } else { + compact(mask_control_ex); + } - brw_compact_inst_set_cmpt_control(devinfo, &temp, true); + if (devinfo->gen <= 6) + compact(flag_subreg_nr); - if (!set_src0_index(devinfo, &temp, src)) - return false; - if (!set_src1_index(devinfo, &temp, src, is_immediate)) - return false; + compact(cond_modifier); - brw_compact_inst_set_dst_reg_nr(devinfo, &temp, - brw_inst_dst_da_reg_nr(devinfo, src)); - brw_compact_inst_set_src0_reg_nr(devinfo, &temp, - brw_inst_src0_da_reg_nr(devinfo, src)); + compact_reg(dst); + compact_reg(src0); - if (is_immediate) { - brw_compact_inst_set_src1_reg_nr(devinfo, &temp, - brw_inst_imm_ud(devinfo, src) & 0xff); - } else { - brw_compact_inst_set_src1_reg_nr(devinfo, &temp, - brw_inst_src1_da_reg_nr(devinfo, src)); + if (is_immediate) { + /* src1 reg takes the low 8 bits (of the 13-bit compacted value) */ + brw_compact_inst_set_src1_reg_nr(devinfo, &temp, compacted_imm & 0xff); + } else { + compact_reg(src1); + } } + brw_compact_inst_set_cmpt_control(devinfo, &temp, true); #undef compact +#undef compact_reg *dst = temp; return true; } +bool +brw_try_compact_instruction(const struct gen_device_info *devinfo, + brw_compact_inst *dst, const brw_inst *src) +{ + struct compaction_state c; + compaction_state_init(&c, devinfo); + return try_compact_instruction(&c, dst, src); +} + static void -set_uncompacted_control(const struct gen_device_info *devinfo, brw_inst *dst, +set_uncompacted_control(const struct compaction_state *c, brw_inst *dst, brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; uint32_t uncompacted = - control_index_table[brw_compact_inst_control_index(devinfo, src)]; - - if (devinfo->gen >= 8) { + c->control_index_table[brw_compact_inst_control_index(devinfo, src)]; + + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 95, 92, (uncompacted >> 17)); + brw_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); + brw_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); + brw_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); + brw_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); + brw_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); + brw_inst_set_bits(dst, 23, 22, (uncompacted >> 6) & 0x3); + brw_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); + brw_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); + } else if (devinfo->gen >= 8) { brw_inst_set_bits(dst, 33, 31, (uncompacted >> 16)); brw_inst_set_bits(dst, 23, 12, (uncompacted >> 4) & 0xfff); brw_inst_set_bits(dst, 10, 9, (uncompacted >> 2) & 0x3); @@ -1192,13 +1793,25 @@ set_uncompacted_control(const struct gen_device_info *devinfo, brw_inst *dst, } static void -set_uncompacted_datatype(const struct gen_device_info *devinfo, brw_inst *dst, +set_uncompacted_datatype(const struct compaction_state *c, brw_inst *dst, brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; uint32_t uncompacted = - datatype_table[brw_compact_inst_datatype_index(devinfo, src)]; - - if (devinfo->gen >= 8) { + c->datatype_table[brw_compact_inst_datatype_index(devinfo, src)]; + + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 98, 98, (uncompacted >> 19)); + brw_inst_set_bits(dst, 91, 88, (uncompacted >> 15) & 0xf); + brw_inst_set_bits(dst, 66, 66, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 50, 50, (uncompacted >> 13) & 0x1); + brw_inst_set_bits(dst, 49, 48, (uncompacted >> 11) & 0x3); + brw_inst_set_bits(dst, 47, 47, (uncompacted >> 10) & 0x1); + brw_inst_set_bits(dst, 46, 46, (uncompacted >> 9) & 0x1); + brw_inst_set_bits(dst, 43, 40, (uncompacted >> 5) & 0xf); + brw_inst_set_bits(dst, 39, 36, (uncompacted >> 1) & 0xf); + brw_inst_set_bits(dst, 35, 35, (uncompacted >> 0) & 0x1); + } else if (devinfo->gen >= 8) { brw_inst_set_bits(dst, 63, 61, (uncompacted >> 18)); brw_inst_set_bits(dst, 94, 89, (uncompacted >> 12) & 0x3f); brw_inst_set_bits(dst, 46, 35, (uncompacted >> 0) & 0xfff); @@ -1209,57 +1822,101 @@ set_uncompacted_datatype(const struct gen_device_info *devinfo, brw_inst *dst, } static void -set_uncompacted_subreg(const struct gen_device_info *devinfo, brw_inst *dst, +set_uncompacted_subreg(const struct compaction_state *c, brw_inst *dst, brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; uint16_t uncompacted = - subreg_table[brw_compact_inst_subreg_index(devinfo, src)]; + c->subreg_table[brw_compact_inst_subreg_index(devinfo, src)]; - brw_inst_set_bits(dst, 100, 96, (uncompacted >> 10)); - brw_inst_set_bits(dst, 68, 64, (uncompacted >> 5) & 0x1f); - brw_inst_set_bits(dst, 52, 48, (uncompacted >> 0) & 0x1f); + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 103, 99, (uncompacted >> 10)); + brw_inst_set_bits(dst, 71, 67, (uncompacted >> 5) & 0x1f); + brw_inst_set_bits(dst, 55, 51, (uncompacted >> 0) & 0x1f); + } else { + brw_inst_set_bits(dst, 100, 96, (uncompacted >> 10)); + brw_inst_set_bits(dst, 68, 64, (uncompacted >> 5) & 0x1f); + brw_inst_set_bits(dst, 52, 48, (uncompacted >> 0) & 0x1f); + } } static void -set_uncompacted_src0(const struct gen_device_info *devinfo, brw_inst *dst, +set_uncompacted_src0(const struct compaction_state *c, brw_inst *dst, brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; uint32_t compacted = brw_compact_inst_src0_index(devinfo, src); - uint16_t uncompacted = src_index_table[compacted]; - - brw_inst_set_bits(dst, 88, 77, uncompacted); + uint16_t uncompacted = c->src0_index_table[compacted]; + + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 87, 84, (uncompacted >> 8)); + brw_inst_set_bits(dst, 83, 81, (uncompacted >> 5) & 0x7); + brw_inst_set_bits(dst, 80, 80, (uncompacted >> 4) & 0x1); + brw_inst_set_bits(dst, 65, 64, (uncompacted >> 2) & 0x3); + brw_inst_set_bits(dst, 45, 44, (uncompacted >> 0) & 0x3); + } else { + brw_inst_set_bits(dst, 88, 77, uncompacted); + } } static void -set_uncompacted_src1(const struct gen_device_info *devinfo, brw_inst *dst, - brw_compact_inst *src, bool is_immediate) +set_uncompacted_src1(const struct compaction_state *c, brw_inst *dst, + brw_compact_inst *src) { - if (is_immediate) { - signed high5 = brw_compact_inst_src1_index(devinfo, src); - /* Replicate top bit of src1_index into high 20 bits of the immediate. */ - brw_inst_set_imm_ud(devinfo, dst, (high5 << 27) >> 19); + const struct gen_device_info *devinfo = c->devinfo; + uint16_t uncompacted = + c->src1_index_table[brw_compact_inst_src1_index(devinfo, src)]; + + if (devinfo->gen >= 12) { + brw_inst_set_bits(dst, 121, 120, (uncompacted >> 10)); + brw_inst_set_bits(dst, 119, 116, (uncompacted >> 6) & 0xf); + brw_inst_set_bits(dst, 115, 113, (uncompacted >> 3) & 0x7); + brw_inst_set_bits(dst, 112, 112, (uncompacted >> 2) & 0x1); + brw_inst_set_bits(dst, 97, 96, (uncompacted >> 0) & 0x3); } else { - uint16_t uncompacted = - src_index_table[brw_compact_inst_src1_index(devinfo, src)]; - brw_inst_set_bits(dst, 120, 109, uncompacted); } } static void -set_uncompacted_3src_control_index(const struct gen_device_info *devinfo, +set_uncompacted_3src_control_index(const struct compaction_state *c, brw_inst *dst, brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; assert(devinfo->gen >= 8); - uint32_t compacted = brw_compact_inst_3src_control_index(devinfo, src); - uint32_t uncompacted = gen8_3src_control_index_table[compacted]; + if (devinfo->gen >= 12) { + uint64_t compacted = brw_compact_inst_3src_control_index(devinfo, src); + uint64_t uncompacted = gen12_3src_control_index_table[compacted]; + + brw_inst_set_bits(dst, 95, 92, (uncompacted >> 32)); + brw_inst_set_bits(dst, 90, 88, (uncompacted >> 29) & 0x7); + brw_inst_set_bits(dst, 82, 80, (uncompacted >> 26) & 0x7); + brw_inst_set_bits(dst, 50, 50, (uncompacted >> 25) & 0x1); + brw_inst_set_bits(dst, 48, 48, (uncompacted >> 24) & 0x1); + brw_inst_set_bits(dst, 42, 40, (uncompacted >> 21) & 0x7); + brw_inst_set_bits(dst, 39, 39, (uncompacted >> 20) & 0x1); + brw_inst_set_bits(dst, 38, 36, (uncompacted >> 17) & 0x7); + brw_inst_set_bits(dst, 34, 34, (uncompacted >> 16) & 0x1); + brw_inst_set_bits(dst, 33, 33, (uncompacted >> 15) & 0x1); + brw_inst_set_bits(dst, 32, 32, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 31, 31, (uncompacted >> 13) & 0x1); + brw_inst_set_bits(dst, 28, 28, (uncompacted >> 12) & 0x1); + brw_inst_set_bits(dst, 27, 24, (uncompacted >> 8) & 0xf); + brw_inst_set_bits(dst, 23, 23, (uncompacted >> 7) & 0x1); + brw_inst_set_bits(dst, 22, 22, (uncompacted >> 6) & 0x1); + brw_inst_set_bits(dst, 21, 19, (uncompacted >> 3) & 0x7); + brw_inst_set_bits(dst, 18, 16, (uncompacted >> 0) & 0x7); + } else { + uint32_t compacted = brw_compact_inst_3src_control_index(devinfo, src); + uint32_t uncompacted = gen8_3src_control_index_table[compacted]; - brw_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7); - brw_inst_set_bits(dst, 28, 8, (uncompacted >> 0) & 0x1fffff); + brw_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7); + brw_inst_set_bits(dst, 28, 8, (uncompacted >> 0) & 0x1fffff); - if (devinfo->gen >= 9 || devinfo->is_cherryview) - brw_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3); + if (devinfo->gen >= 9 || devinfo->is_cherryview) + brw_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3); + } } static void @@ -1269,28 +1926,65 @@ set_uncompacted_3src_source_index(const struct gen_device_info *devinfo, assert(devinfo->gen >= 8); uint32_t compacted = brw_compact_inst_3src_source_index(devinfo, src); - uint64_t uncompacted = gen8_3src_source_index_table[compacted]; - - brw_inst_set_bits(dst, 83, 83, (uncompacted >> 43) & 0x1); - brw_inst_set_bits(dst, 114, 107, (uncompacted >> 35) & 0xff); - brw_inst_set_bits(dst, 93, 86, (uncompacted >> 27) & 0xff); - brw_inst_set_bits(dst, 72, 65, (uncompacted >> 19) & 0xff); - brw_inst_set_bits(dst, 55, 37, (uncompacted >> 0) & 0x7ffff); - - if (devinfo->gen >= 9 || devinfo->is_cherryview) { - brw_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3); - brw_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3); - brw_inst_set_bits(dst, 84, 84, (uncompacted >> 44) & 0x1); + + if (devinfo->gen >= 12) { + uint32_t uncompacted = gen12_3src_source_index_table[compacted]; + + brw_inst_set_bits(dst, 114, 114, (uncompacted >> 20)); + brw_inst_set_bits(dst, 113, 112, (uncompacted >> 18) & 0x3); + brw_inst_set_bits(dst, 98, 98, (uncompacted >> 17) & 0x1); + brw_inst_set_bits(dst, 97, 96, (uncompacted >> 15) & 0x3); + brw_inst_set_bits(dst, 91, 91, (uncompacted >> 14) & 0x1); + brw_inst_set_bits(dst, 87, 86, (uncompacted >> 12) & 0x3); + brw_inst_set_bits(dst, 85, 84, (uncompacted >> 10) & 0x3); + brw_inst_set_bits(dst, 83, 83, (uncompacted >> 9) & 0x1); + brw_inst_set_bits(dst, 66, 66, (uncompacted >> 8) & 0x1); + brw_inst_set_bits(dst, 65, 64, (uncompacted >> 6) & 0x3); + brw_inst_set_bits(dst, 47, 47, (uncompacted >> 5) & 0x1); + brw_inst_set_bits(dst, 46, 46, (uncompacted >> 4) & 0x1); + brw_inst_set_bits(dst, 45, 44, (uncompacted >> 2) & 0x3); + brw_inst_set_bits(dst, 43, 43, (uncompacted >> 1) & 0x1); + brw_inst_set_bits(dst, 35, 35, (uncompacted >> 0) & 0x1); } else { - brw_inst_set_bits(dst, 125, 125, (uncompacted >> 45) & 0x1); - brw_inst_set_bits(dst, 104, 104, (uncompacted >> 44) & 0x1); + uint64_t uncompacted = gen8_3src_source_index_table[compacted]; + + brw_inst_set_bits(dst, 83, 83, (uncompacted >> 43) & 0x1); + brw_inst_set_bits(dst, 114, 107, (uncompacted >> 35) & 0xff); + brw_inst_set_bits(dst, 93, 86, (uncompacted >> 27) & 0xff); + brw_inst_set_bits(dst, 72, 65, (uncompacted >> 19) & 0xff); + brw_inst_set_bits(dst, 55, 37, (uncompacted >> 0) & 0x7ffff); + + if (devinfo->gen >= 9 || devinfo->is_cherryview) { + brw_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3); + brw_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3); + brw_inst_set_bits(dst, 84, 84, (uncompacted >> 44) & 0x1); + } else { + brw_inst_set_bits(dst, 125, 125, (uncompacted >> 45) & 0x1); + brw_inst_set_bits(dst, 104, 104, (uncompacted >> 44) & 0x1); + } } } static void -brw_uncompact_3src_instruction(const struct gen_device_info *devinfo, +set_uncompacted_3src_subreg_index(const struct gen_device_info *devinfo, + brw_inst *dst, brw_compact_inst *src) +{ + assert(devinfo->gen >= 12); + + uint32_t compacted = brw_compact_inst_3src_subreg_index(devinfo, src); + uint32_t uncompacted = gen12_3src_subreg_table[compacted]; + + brw_inst_set_bits(dst, 119, 115, (uncompacted >> 15)); + brw_inst_set_bits(dst, 103, 99, (uncompacted >> 10) & 0x1f); + brw_inst_set_bits(dst, 71, 67, (uncompacted >> 5) & 0x1f); + brw_inst_set_bits(dst, 55, 51, (uncompacted >> 0) & 0x1f); +} + +static void +brw_uncompact_3src_instruction(const struct compaction_state *c, brw_inst *dst, brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; assert(devinfo->gen >= 8); #define uncompact(field) \ @@ -1298,85 +1992,112 @@ brw_uncompact_3src_instruction(const struct gen_device_info *devinfo, #define uncompact_a16(field) \ brw_inst_set_3src_a16_##field(devinfo, dst, brw_compact_inst_3src_##field(devinfo, src)) - uncompact(opcode); + uncompact(hw_opcode); - set_uncompacted_3src_control_index(devinfo, dst, src); - set_uncompacted_3src_source_index(devinfo, dst, src); + if (devinfo->gen >= 12) { + set_uncompacted_3src_control_index(c, dst, src); + set_uncompacted_3src_source_index(devinfo, dst, src); + set_uncompacted_3src_subreg_index(devinfo, dst, src); - uncompact(dst_reg_nr); - uncompact_a16(src0_rep_ctrl); + uncompact(debug_control); + uncompact(swsb); + uncompact(dst_reg_nr); + uncompact(src0_reg_nr); + uncompact(src1_reg_nr); + uncompact(src2_reg_nr); + } else { + set_uncompacted_3src_control_index(c, dst, src); + set_uncompacted_3src_source_index(devinfo, dst, src); + + uncompact(dst_reg_nr); + uncompact_a16(src0_rep_ctrl); + uncompact(debug_control); + uncompact(saturate); + uncompact_a16(src1_rep_ctrl); + uncompact_a16(src2_rep_ctrl); + uncompact(src0_reg_nr); + uncompact(src1_reg_nr); + uncompact(src2_reg_nr); + uncompact_a16(src0_subreg_nr); + uncompact_a16(src1_subreg_nr); + uncompact_a16(src2_subreg_nr); + } brw_inst_set_3src_cmpt_control(devinfo, dst, false); - uncompact(debug_control); - uncompact(saturate); - uncompact_a16(src1_rep_ctrl); - uncompact_a16(src2_rep_ctrl); - uncompact(src0_reg_nr); - uncompact(src1_reg_nr); - uncompact(src2_reg_nr); - uncompact_a16(src0_subreg_nr); - uncompact_a16(src1_subreg_nr); - uncompact_a16(src2_subreg_nr); #undef uncompact #undef uncompact_a16 } -void -brw_uncompact_instruction(const struct gen_device_info *devinfo, brw_inst *dst, - brw_compact_inst *src) +static void +uncompact_instruction(const struct compaction_state *c, brw_inst *dst, + brw_compact_inst *src) { + const struct gen_device_info *devinfo = c->devinfo; memset(dst, 0, sizeof(*dst)); if (devinfo->gen >= 8 && - is_3src(devinfo, brw_compact_inst_3src_opcode(devinfo, src))) { - brw_uncompact_3src_instruction(devinfo, dst, src); + is_3src(devinfo, brw_opcode_decode( + devinfo, brw_compact_inst_3src_hw_opcode(devinfo, src)))) { + brw_uncompact_3src_instruction(c, dst, src); return; } #define uncompact(field) \ brw_inst_set_##field(devinfo, dst, brw_compact_inst_##field(devinfo, src)) +#define uncompact_reg(field) \ + brw_inst_set_##field##_da_reg_nr(devinfo, dst, \ + brw_compact_inst_##field##_reg_nr(devinfo, src)) - uncompact(opcode); + uncompact(hw_opcode); uncompact(debug_control); - set_uncompacted_control(devinfo, dst, src); - set_uncompacted_datatype(devinfo, dst, src); - - /* src0/1 register file fields are in the datatype table. */ - bool is_immediate = brw_inst_src0_reg_file(devinfo, dst) == BRW_IMMEDIATE_VALUE || - brw_inst_src1_reg_file(devinfo, dst) == BRW_IMMEDIATE_VALUE; + set_uncompacted_control(c, dst, src); + set_uncompacted_datatype(c, dst, src); + set_uncompacted_subreg(c, dst, src); + set_uncompacted_src0(c, dst, src); - set_uncompacted_subreg(devinfo, dst, src); - - if (devinfo->gen >= 6) { - uncompact(acc_wr_control); + enum brw_reg_type type; + if (has_immediate(devinfo, dst, &type)) { + unsigned imm = uncompact_immediate(devinfo, type, + brw_compact_inst_imm(devinfo, src)); + brw_inst_set_imm_ud(devinfo, dst, imm); } else { - uncompact(mask_control_ex); + set_uncompacted_src1(c, dst, src); + uncompact_reg(src1); } - uncompact(cond_modifier); - - if (devinfo->gen <= 6) - uncompact(flag_subreg_nr); + if (devinfo->gen >= 12) { + uncompact(swsb); + uncompact_reg(dst); + uncompact_reg(src0); + } else { + if (devinfo->gen >= 6) { + uncompact(acc_wr_control); + } else { + uncompact(mask_control_ex); + } - set_uncompacted_src0(devinfo, dst, src); - set_uncompacted_src1(devinfo, dst, src, is_immediate); + uncompact(cond_modifier); - brw_inst_set_dst_da_reg_nr(devinfo, dst, - brw_compact_inst_dst_reg_nr(devinfo, src)); - brw_inst_set_src0_da_reg_nr(devinfo, dst, - brw_compact_inst_src0_reg_nr(devinfo, src)); + if (devinfo->gen <= 6) + uncompact(flag_subreg_nr); - if (is_immediate) { - brw_inst_set_imm_ud(devinfo, dst, - brw_inst_imm_ud(devinfo, dst) | - brw_compact_inst_src1_reg_nr(devinfo, src)); - } else { - brw_inst_set_src1_da_reg_nr(devinfo, dst, - brw_compact_inst_src1_reg_nr(devinfo, src)); + uncompact_reg(dst); + uncompact_reg(src0); } + brw_inst_set_cmpt_control(devinfo, dst, false); #undef uncompact +#undef uncompact_reg +} + +void +brw_uncompact_instruction(const struct gen_device_info *devinfo, brw_inst *dst, + brw_compact_inst *src) +{ + struct compaction_state c; + compaction_state_init(&c, devinfo); + uncompact_instruction(&c, dst, src); } void brw_debug_compact_uncompact(const struct gen_device_info *devinfo, @@ -1387,10 +2108,10 @@ void brw_debug_compact_uncompact(const struct gen_device_info *devinfo, devinfo->gen); fprintf(stderr, " before: "); - brw_disassemble_inst(stderr, devinfo, orig, true); + brw_disassemble_inst(stderr, devinfo, orig, true, 0, NULL); fprintf(stderr, " after: "); - brw_disassemble_inst(stderr, devinfo, uncompacted, false); + brw_disassemble_inst(stderr, devinfo, uncompacted, false, 0, NULL); uint32_t *before_bits = (uint32_t *)orig; uint32_t *after_bits = (uint32_t *)uncompacted; @@ -1466,8 +2187,9 @@ update_gen4_jump_count(const struct gen_device_info *devinfo, brw_inst *insn, brw_inst_set_gen4_jump_count(devinfo, insn, jump_count_compacted >> shift); } -void -brw_init_compaction_tables(const struct gen_device_info *devinfo) +static void +compaction_state_init(struct compaction_state *c, + const struct gen_device_info *devinfo) { assert(g45_control_index_table[ARRAY_SIZE(g45_control_index_table) - 1] != 0); assert(g45_datatype_table[ARRAY_SIZE(g45_datatype_table) - 1] != 0); @@ -1486,40 +2208,58 @@ brw_init_compaction_tables(const struct gen_device_info *devinfo) assert(gen8_subreg_table[ARRAY_SIZE(gen8_subreg_table) - 1] != 0); assert(gen8_src_index_table[ARRAY_SIZE(gen8_src_index_table) - 1] != 0); assert(gen11_datatype_table[ARRAY_SIZE(gen11_datatype_table) - 1] != 0); + assert(gen12_control_index_table[ARRAY_SIZE(gen12_control_index_table) - 1] != 0); + assert(gen12_datatype_table[ARRAY_SIZE(gen12_datatype_table) - 1] != 0); + assert(gen12_subreg_table[ARRAY_SIZE(gen12_subreg_table) - 1] != 0); + assert(gen12_src0_index_table[ARRAY_SIZE(gen12_src0_index_table) - 1] != 0); + assert(gen12_src1_index_table[ARRAY_SIZE(gen12_src1_index_table) - 1] != 0); + c->devinfo = devinfo; switch (devinfo->gen) { + case 12: + c->control_index_table = gen12_control_index_table;; + c->datatype_table = gen12_datatype_table; + c->subreg_table = gen12_subreg_table; + c->src0_index_table = gen12_src0_index_table; + c->src1_index_table = gen12_src1_index_table; + break; case 11: - control_index_table = gen8_control_index_table; - datatype_table = gen11_datatype_table; - subreg_table = gen8_subreg_table; - src_index_table = gen8_src_index_table; + c->control_index_table = gen8_control_index_table; + c->datatype_table = gen11_datatype_table; + c->subreg_table = gen8_subreg_table; + c->src0_index_table = gen8_src_index_table; + c->src1_index_table = gen8_src_index_table; break; case 10: case 9: case 8: - control_index_table = gen8_control_index_table; - datatype_table = gen8_datatype_table; - subreg_table = gen8_subreg_table; - src_index_table = gen8_src_index_table; + c->control_index_table = gen8_control_index_table; + c->datatype_table = gen8_datatype_table; + c->subreg_table = gen8_subreg_table; + c->src0_index_table = gen8_src_index_table; + c->src1_index_table = gen8_src_index_table; break; case 7: - control_index_table = gen7_control_index_table; - datatype_table = gen7_datatype_table; - subreg_table = gen7_subreg_table; - src_index_table = gen7_src_index_table; + c->control_index_table = gen7_control_index_table; + c->datatype_table = gen7_datatype_table; + c->subreg_table = gen7_subreg_table; + c->src0_index_table = gen7_src_index_table; + c->src1_index_table = gen7_src_index_table; break; case 6: - control_index_table = gen6_control_index_table; - datatype_table = gen6_datatype_table; - subreg_table = gen6_subreg_table; - src_index_table = gen6_src_index_table; + c->control_index_table = gen6_control_index_table; + c->datatype_table = gen6_datatype_table; + c->subreg_table = gen6_subreg_table; + c->src0_index_table = gen6_src_index_table; + c->src1_index_table = gen6_src_index_table; break; case 5: case 4: - control_index_table = g45_control_index_table; - datatype_table = g45_datatype_table; - subreg_table = g45_subreg_table; - src_index_table = g45_src_index_table; + c->control_index_table = g45_control_index_table; + c->datatype_table = g45_datatype_table; + c->subreg_table = g45_subreg_table; + c->src0_index_table = g45_src_index_table; + c->src1_index_table = g45_src_index_table; break; default: unreachable("unknown generation"); @@ -1548,6 +2288,9 @@ brw_compact_instructions(struct brw_codegen *p, int start_offset, if (devinfo->gen == 4 && !devinfo->is_g4x) return; + struct compaction_state c; + compaction_state_init(&c, devinfo); + int offset = 0; int compacted_count = 0; for (int src_offset = 0; src_offset < p->next_insn_offset - start_offset; @@ -1561,12 +2304,12 @@ brw_compact_instructions(struct brw_codegen *p, int start_offset, brw_inst inst = precompact(devinfo, *src); brw_inst saved = inst; - if (brw_try_compact_instruction(devinfo, dst, &inst)) { + if (try_compact_instruction(&c, dst, &inst)) { compacted_count++; if (INTEL_DEBUG) { brw_inst uncompacted; - brw_uncompact_instruction(devinfo, &uncompacted, dst); + uncompact_instruction(&c, &uncompacted, dst); if (memcmp(&saved, &uncompacted, sizeof(uncompacted))) { brw_debug_compact_uncompact(devinfo, &saved, &uncompacted); } @@ -1578,7 +2321,8 @@ brw_compact_instructions(struct brw_codegen *p, int start_offset, if ((offset & sizeof(brw_compact_inst)) != 0 && devinfo->is_g4x){ brw_compact_inst *align = store + offset; memset(align, 0, sizeof(*align)); - brw_compact_inst_set_opcode(devinfo, align, BRW_OPCODE_NENOP); + brw_compact_inst_set_hw_opcode( + devinfo, align, brw_opcode_encode(devinfo, BRW_OPCODE_NENOP)); brw_compact_inst_set_cmpt_control(devinfo, align, true); offset += sizeof(brw_compact_inst); compacted_count--; @@ -1632,15 +2376,14 @@ brw_compact_instructions(struct brw_codegen *p, int start_offset, if (devinfo->gen >= 7) { if (brw_inst_cmpt_control(devinfo, insn)) { brw_inst uncompacted; - brw_uncompact_instruction(devinfo, &uncompacted, - (brw_compact_inst *)insn); + uncompact_instruction(&c, &uncompacted, + (brw_compact_inst *)insn); update_uip_jip(devinfo, &uncompacted, this_old_ip, compacted_counts); - bool ret = brw_try_compact_instruction(devinfo, - (brw_compact_inst *)insn, - &uncompacted); + bool ret = try_compact_instruction(&c, (brw_compact_inst *)insn, + &uncompacted); assert(ret); (void)ret; } else { update_uip_jip(devinfo, insn, this_old_ip, compacted_counts); @@ -1682,6 +2425,9 @@ brw_compact_instructions(struct brw_codegen *p, int start_offset, brw_inst_set_imm_ud(devinfo, insn, jump_compacted << shift); } break; + + default: + break; } } @@ -1693,12 +2439,22 @@ brw_compact_instructions(struct brw_codegen *p, int start_offset, if (p->next_insn_offset & sizeof(brw_compact_inst)) { brw_compact_inst *align = store + offset; memset(align, 0, sizeof(*align)); - brw_compact_inst_set_opcode(devinfo, align, BRW_OPCODE_NOP); + brw_compact_inst_set_hw_opcode( + devinfo, align, brw_opcode_encode(devinfo, BRW_OPCODE_NOP)); brw_compact_inst_set_cmpt_control(devinfo, align, true); p->next_insn_offset += sizeof(brw_compact_inst); } p->nr_insn = p->next_insn_offset / sizeof(brw_inst); + for (int i = 0; i < p->num_relocs; i++) { + if (p->relocs[i].offset < (uint32_t)start_offset) + continue; + + assert(p->relocs[i].offset % 16 == 0); + unsigned idx = (p->relocs[i].offset - start_offset) / 16; + p->relocs[i].offset -= compacted_counts[idx] * 8; + } + /* Update the instruction offsets for each group. */ if (disasm) { int offset = 0;