src/mesa/drivers/dri/i965/brw_eu_compact.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /** @file brw_eu_compact.c
  25  *
  26  * Instruction compaction is a feature of G45 and newer hardware that allows
  27  * for a smaller instruction encoding.
  28  *
  29  * The instruction cache is on the order of 32KB, and many programs generate
  30  * far more instructions than that.  The instruction cache is built to barely
  31  * keep up with instruction dispatch ability in cache hit cases -- L1
  32  * instruction cache misses that still hit in the next level could limit
  33  * throughput by around 50%.
  34  *
  35  * The idea of instruction compaction is that most instructions use a tiny
  36  * subset of the GPU functionality, so we can encode what would be a 16 byte
  37  * instruction in 8 bytes using some lookup tables for various fields.
  38  *
  39  *
  40  * Instruction compaction capabilities vary subtly by generation.
  41  *
  42  * G45's support for instruction compaction is very limited. Jump counts on
  43  * this generation are in units of 16-byte uncompacted instructions. As such,
  44  * all jump targets must be 16-byte aligned. Also, all instructions must be
  45  * naturally aligned, i.e. uncompacted instructions must be 16-byte aligned.
  46  * A G45-only instruction, NENOP, must be used to provide padding to align
  47  * uncompacted instructions.
  48  *
  49  * Gen5 removes these restrictions and changes jump counts to be in units of
  50  * 8-byte compacted instructions, allowing jump targets to be only 8-byte
  51  * aligned. Uncompacted instructions can also be placed on 8-byte boundaries.
  52  *
  53  * Gen6 adds the ability to compact instructions with a limited range of
  54  * immediate values. Compactable immediates have 12 unrestricted bits, and a
  55  * 13th bit that's replicated through the high 20 bits, to create the 32-bit
  56  * value of DW3 in the uncompacted instruction word.
  57  *
  58  * On Gen7 we can compact some control flow instructions with a small positive
  59  * immediate in the low bits of DW3, like ENDIF with the JIP field. Other
  60  * control flow instructions with UIP cannot be compacted, because of the
  61  * replicated 13th bit. No control flow instructions can be compacted on Gen6
  62  * since the jump count field is not in DW3.
  63  *
  64  *    break    JIP/UIP
  65  *    cont     JIP/UIP
  66  *    halt     JIP/UIP
  67  *    if       JIP/UIP
  68  *    else     JIP (plus UIP on BDW+)
  69  *    endif    JIP
  70  *    while    JIP (must be negative)
  71  *
  72  * Gen 8 adds support for compacting 3-src instructions.
  73  */
  74
  75 #include "brw_context.h"
  76 #include "brw_eu.h"
  77 #include "intel_asm_annotation.h"
  78 #include "util/u_atomic.h" /* for p_atomic_cmpxchg */
  79
  80 static const uint32_t g45_control_index_table[32] = {
  81    0b00000000000000000,
  82    0b01000000000000000,
  83    0b00110000000000000,
  84    0b00000000000000010,
  85    0b00100000000000000,
  86    0b00010000000000000,
  87    0b01000000000100000,
  88    0b01000000100000000,
  89    0b01010000000100000,
  90    0b00000000100000010,
  91    0b11000000000000000,
  92    0b00001000100000010,
  93    0b01001000100000000,
  94    0b00000000100000000,
  95    0b11000000000100000,
  96    0b00001000100000000,
  97    0b10110000000000000,
  98    0b11010000000100000,
  99    0b00110000100000000,
 100    0b00100000100000000,
 101    0b01000000000001000,
 102    0b01000000000000100,
 103    0b00111100000000000,
 104    0b00101011000000000,
 105    0b00110000000010000,
 106    0b00010000100000000,
 107    0b01000000000100100,
 108    0b01000000000101000,
 109    0b00110000000000110,
 110    0b00000000000001010,
 111    0b01010000000101000,
 112    0b01010000000100100
 113 };
 114
 115 static const uint32_t g45_datatype_table[32] = {
 116    0b001000000000100001,
 117    0b001011010110101101,
 118    0b001000001000110001,
 119    0b001111011110111101,
 120    0b001011010110101100,
 121    0b001000000110101101,
 122    0b001000000000100000,
 123    0b010100010110110001,
 124    0b001100011000101101,
 125    0b001000000000100010,
 126    0b001000001000110110,
 127    0b010000001000110001,
 128    0b001000001000110010,
 129    0b011000001000110010,
 130    0b001111011110111100,
 131    0b001000000100101000,
 132    0b010100011000110001,
 133    0b001010010100101001,
 134    0b001000001000101001,
 135    0b010000001000110110,
 136    0b101000001000110001,
 137    0b001011011000101101,
 138    0b001000000100001001,
 139    0b001011011000101100,
 140    0b110100011000110001,
 141    0b001000001110111101,
 142    0b110000001000110001,
 143    0b011000000100101010,
 144    0b101000001000101001,
 145    0b001011010110001100,
 146    0b001000000110100001,
 147    0b001010010100001000
 148 };
 149
 150 static const uint16_t g45_subreg_table[32] = {
 151    0b000000000000000,
 152    0b000000010000000,
 153    0b000001000000000,
 154    0b000100000000000,
 155    0b000000000100000,
 156    0b100000000000000,
 157    0b000000000010000,
 158    0b001100000000000,
 159    0b001010000000000,
 160    0b000000100000000,
 161    0b001000000000000,
 162    0b000000000001000,
 163    0b000000001000000,
 164    0b000000000000001,
 165    0b000010000000000,
 166    0b000000010100000,
 167    0b000000000000111,
 168    0b000001000100000,
 169    0b011000000000000,
 170    0b000000110000000,
 171    0b000000000000010,
 172    0b000000000000100,
 173    0b000000001100000,
 174    0b000100000000010,
 175    0b001110011000110,
 176    0b001110100001000,
 177    0b000110011000110,
 178    0b000001000011000,
 179    0b000110010000100,
 180    0b001100000000110,
 181    0b000000010000110,
 182    0b000001000110000
 183 };
 184
 185 static const uint16_t g45_src_index_table[32] = {
 186    0b000000000000,
 187    0b010001101000,
 188    0b010110001000,
 189    0b011010010000,
 190    0b001101001000,
 191    0b010110001010,
 192    0b010101110000,
 193    0b011001111000,
 194    0b001000101000,
 195    0b000000101000,
 196    0b010001010000,
 197    0b111101101100,
 198    0b010110001100,
 199    0b010001101100,
 200    0b011010010100,
 201    0b010001001100,
 202    0b001100101000,
 203    0b000000000010,
 204    0b111101001100,
 205    0b011001101000,
 206    0b010101001000,
 207    0b000000000100,
 208    0b000000101100,
 209    0b010001101010,
 210    0b000000111000,
 211    0b010101011000,
 212    0b000100100000,
 213    0b010110000000,
 214    0b010000000100,
 215    0b010000111000,
 216    0b000101100000,
 217    0b111101110100
 218 };
 219
 220 static const uint32_t gen6_control_index_table[32] = {
 221    0b00000000000000000,
 222    0b01000000000000000,
 223    0b00110000000000000,
 224    0b00000000100000000,
 225    0b00010000000000000,
 226    0b00001000100000000,
 227    0b00000000100000010,
 228    0b00000000000000010,
 229    0b01000000100000000,
 230    0b01010000000000000,
 231    0b10110000000000000,
 232    0b00100000000000000,
 233    0b11010000000000000,
 234    0b11000000000000000,
 235    0b01001000100000000,
 236    0b01000000000001000,
 237    0b01000000000000100,
 238    0b00000000000001000,
 239    0b00000000000000100,
 240    0b00111000100000000,
 241    0b00001000100000010,
 242    0b00110000100000000,
 243    0b00110000000000001,
 244    0b00100000000000001,
 245    0b00110000000000010,
 246    0b00110000000000101,
 247    0b00110000000001001,
 248    0b00110000000010000,
 249    0b00110000000000011,
 250    0b00110000000000100,
 251    0b00110000100001000,
 252    0b00100000000001001
 253 };
 254
 255 static const uint32_t gen6_datatype_table[32] = {
 256    0b001001110000000000,
 257    0b001000110000100000,
 258    0b001001110000000001,
 259    0b001000000001100000,
 260    0b001010110100101001,
 261    0b001000000110101101,
 262    0b001100011000101100,
 263    0b001011110110101101,
 264    0b001000000111101100,
 265    0b001000000001100001,
 266    0b001000110010100101,
 267    0b001000000001000001,
 268    0b001000001000110001,
 269    0b001000001000101001,
 270    0b001000000000100000,
 271    0b001000001000110010,
 272    0b001010010100101001,
 273    0b001011010010100101,
 274    0b001000000110100101,
 275    0b001100011000101001,
 276    0b001011011000101100,
 277    0b001011010110100101,
 278    0b001011110110100101,
 279    0b001111011110111101,
 280    0b001111011110111100,
 281    0b001111011110111101,
 282    0b001111011110011101,
 283    0b001111011110111110,
 284    0b001000000000100001,
 285    0b001000000000100010,
 286    0b001001111111011101,
 287    0b001000001110111110,
 288 };
 289
 290 static const uint16_t gen6_subreg_table[32] = {
 291    0b000000000000000,
 292    0b000000000000100,
 293    0b000000110000000,
 294    0b111000000000000,
 295    0b011110000001000,
 296    0b000010000000000,
 297    0b000000000010000,
 298    0b000110000001100,
 299    0b001000000000000,
 300    0b000001000000000,
 301    0b000001010010100,
 302    0b000000001010110,
 303    0b010000000000000,
 304    0b110000000000000,
 305    0b000100000000000,
 306    0b000000010000000,
 307    0b000000000001000,
 308    0b100000000000000,
 309    0b000001010000000,
 310    0b001010000000000,
 311    0b001100000000000,
 312    0b000000001010100,
 313    0b101101010010100,
 314    0b010100000000000,
 315    0b000000010001111,
 316    0b011000000000000,
 317    0b111110000000000,
 318    0b101000000000000,
 319    0b000000000001111,
 320    0b000100010001111,
 321    0b001000010001111,
 322    0b000110000000000,
 323 };
 324
 325 static const uint16_t gen6_src_index_table[32] = {
 326    0b000000000000,
 327    0b010110001000,
 328    0b010001101000,
 329    0b001000101000,
 330    0b011010010000,
 331    0b000100100000,
 332    0b010001101100,
 333    0b010101110000,
 334    0b011001111000,
 335    0b001100101000,
 336    0b010110001100,
 337    0b001000100000,
 338    0b010110001010,
 339    0b000000000010,
 340    0b010101010000,
 341    0b010101101000,
 342    0b111101001100,
 343    0b111100101100,
 344    0b011001110000,
 345    0b010110001001,
 346    0b010101011000,
 347    0b001101001000,
 348    0b010000101100,
 349    0b010000000000,
 350    0b001101110000,
 351    0b001100010000,
 352    0b001100000000,
 353    0b010001101010,
 354    0b001101111000,
 355    0b000001110000,
 356    0b001100100000,
 357    0b001101010000,
 358 };
 359
 360 static const uint32_t gen7_control_index_table[32] = {
 361    0b0000000000000000010,
 362    0b0000100000000000000,
 363    0b0000100000000000001,
 364    0b0000100000000000010,
 365    0b0000100000000000011,
 366    0b0000100000000000100,
 367    0b0000100000000000101,
 368    0b0000100000000000111,
 369    0b0000100000000001000,
 370    0b0000100000000001001,
 371    0b0000100000000001101,
 372    0b0000110000000000000,
 373    0b0000110000000000001,
 374    0b0000110000000000010,
 375    0b0000110000000000011,
 376    0b0000110000000000100,
 377    0b0000110000000000101,
 378    0b0000110000000000111,
 379    0b0000110000000001001,
 380    0b0000110000000001101,
 381    0b0000110000000010000,
 382    0b0000110000100000000,
 383    0b0001000000000000000,
 384    0b0001000000000000010,
 385    0b0001000000000000100,
 386    0b0001000000100000000,
 387    0b0010110000000000000,
 388    0b0010110000000010000,
 389    0b0011000000000000000,
 390    0b0011000000100000000,
 391    0b0101000000000000000,
 392    0b0101000000100000000
 393 };
 394
 395 static const uint32_t gen7_datatype_table[32] = {
 396    0b001000000000000001,
 397    0b001000000000100000,
 398    0b001000000000100001,
 399    0b001000000001100001,
 400    0b001000000010111101,
 401    0b001000001011111101,
 402    0b001000001110100001,
 403    0b001000001110100101,
 404    0b001000001110111101,
 405    0b001000010000100001,
 406    0b001000110000100000,
 407    0b001000110000100001,
 408    0b001001010010100101,
 409    0b001001110010100100,
 410    0b001001110010100101,
 411    0b001111001110111101,
 412    0b001111011110011101,
 413    0b001111011110111100,
 414    0b001111011110111101,
 415    0b001111111110111100,
 416    0b000000001000001100,
 417    0b001000000000111101,
 418    0b001000000010100101,
 419    0b001000010000100000,
 420    0b001001010010100100,
 421    0b001001110010000100,
 422    0b001010010100001001,
 423    0b001101111110111101,
 424    0b001111111110111101,
 425    0b001011110110101100,
 426    0b001010010100101000,
 427    0b001010110100101000
 428 };
 429
 430 static const uint16_t gen7_subreg_table[32] = {
 431    0b000000000000000,
 432    0b000000000000001,
 433    0b000000000001000,
 434    0b000000000001111,
 435    0b000000000010000,
 436    0b000000010000000,
 437    0b000000100000000,
 438    0b000000110000000,
 439    0b000001000000000,
 440    0b000001000010000,
 441    0b000010100000000,
 442    0b001000000000000,
 443    0b001000000000001,
 444    0b001000010000001,
 445    0b001000010000010,
 446    0b001000010000011,
 447    0b001000010000100,
 448    0b001000010000111,
 449    0b001000010001000,
 450    0b001000010001110,
 451    0b001000010001111,
 452    0b001000110000000,
 453    0b001000111101000,
 454    0b010000000000000,
 455    0b010000110000000,
 456    0b011000000000000,
 457    0b011110010000111,
 458    0b100000000000000,
 459    0b101000000000000,
 460    0b110000000000000,
 461    0b111000000000000,
 462    0b111000000011100
 463 };
 464
 465 static const uint16_t gen7_src_index_table[32] = {
 466    0b000000000000,
 467    0b000000000010,
 468    0b000000010000,
 469    0b000000010010,
 470    0b000000011000,
 471    0b000000100000,
 472    0b000000101000,
 473    0b000001001000,
 474    0b000001010000,
 475    0b000001110000,
 476    0b000001111000,
 477    0b001100000000,
 478    0b001100000010,
 479    0b001100001000,
 480    0b001100010000,
 481    0b001100010010,
 482    0b001100100000,
 483    0b001100101000,
 484    0b001100111000,
 485    0b001101000000,
 486    0b001101000010,
 487    0b001101001000,
 488    0b001101010000,
 489    0b001101100000,
 490    0b001101101000,
 491    0b001101110000,
 492    0b001101110001,
 493    0b001101111000,
 494    0b010001101000,
 495    0b010001101001,
 496    0b010001101010,
 497    0b010110001000
 498 };
 499
 500 static const uint32_t gen8_control_index_table[32] = {
 501    0b0000000000000000010,
 502    0b0000100000000000000,
 503    0b0000100000000000001,
 504    0b0000100000000000010,
 505    0b0000100000000000011,
 506    0b0000100000000000100,
 507    0b0000100000000000101,
 508    0b0000100000000000111,
 509    0b0000100000000001000,
 510    0b0000100000000001001,
 511    0b0000100000000001101,
 512    0b0000110000000000000,
 513    0b0000110000000000001,
 514    0b0000110000000000010,
 515    0b0000110000000000011,
 516    0b0000110000000000100,
 517    0b0000110000000000101,
 518    0b0000110000000000111,
 519    0b0000110000000001001,
 520    0b0000110000000001101,
 521    0b0000110000000010000,
 522    0b0000110000100000000,
 523    0b0001000000000000000,
 524    0b0001000000000000010,
 525    0b0001000000000000100,
 526    0b0001000000100000000,
 527    0b0010110000000000000,
 528    0b0010110000000010000,
 529    0b0011000000000000000,
 530    0b0011000000100000000,
 531    0b0101000000000000000,
 532    0b0101000000100000000
 533 };
 534
 535 static const uint32_t gen8_datatype_table[32] = {
 536    0b001000000000000000001,
 537    0b001000000000001000000,
 538    0b001000000000001000001,
 539    0b001000000000011000001,
 540    0b001000000000101011101,
 541    0b001000000010111011101,
 542    0b001000000011101000001,
 543    0b001000000011101000101,
 544    0b001000000011101011101,
 545    0b001000001000001000001,
 546    0b001000011000001000000,
 547    0b001000011000001000001,
 548    0b001000101000101000101,
 549    0b001000111000101000100,
 550    0b001000111000101000101,
 551    0b001011100011101011101,
 552    0b001011101011100011101,
 553    0b001011101011101011100,
 554    0b001011101011101011101,
 555    0b001011111011101011100,
 556    0b000000000010000001100,
 557    0b001000000000001011101,
 558    0b001000000000101000101,
 559    0b001000001000001000000,
 560    0b001000101000101000100,
 561    0b001000111000100000100,
 562    0b001001001001000001001,
 563    0b001010111011101011101,
 564    0b001011111011101011101,
 565    0b001001111001101001100,
 566    0b001001001001001001000,
 567    0b001001011001001001000
 568 };
 569
 570 static const uint16_t gen8_subreg_table[32] = {
 571    0b000000000000000,
 572    0b000000000000001,
 573    0b000000000001000,
 574    0b000000000001111,
 575    0b000000000010000,
 576    0b000000010000000,
 577    0b000000100000000,
 578    0b000000110000000,
 579    0b000001000000000,
 580    0b000001000010000,
 581    0b000001010000000,
 582    0b001000000000000,
 583    0b001000000000001,
 584    0b001000010000001,
 585    0b001000010000010,
 586    0b001000010000011,
 587    0b001000010000100,
 588    0b001000010000111,
 589    0b001000010001000,
 590    0b001000010001110,
 591    0b001000010001111,
 592    0b001000110000000,
 593    0b001000111101000,
 594    0b010000000000000,
 595    0b010000110000000,
 596    0b011000000000000,
 597    0b011110010000111,
 598    0b100000000000000,
 599    0b101000000000000,
 600    0b110000000000000,
 601    0b111000000000000,
 602    0b111000000011100
 603 };
 604
 605 static const uint16_t gen8_src_index_table[32] = {
 606    0b000000000000,
 607    0b000000000010,
 608    0b000000010000,
 609    0b000000010010,
 610    0b000000011000,
 611    0b000000100000,
 612    0b000000101000,
 613    0b000001001000,
 614    0b000001010000,
 615    0b000001110000,
 616    0b000001111000,
 617    0b001100000000,
 618    0b001100000010,
 619    0b001100001000,
 620    0b001100010000,
 621    0b001100010010,
 622    0b001100100000,
 623    0b001100101000,
 624    0b001100111000,
 625    0b001101000000,
 626    0b001101000010,
 627    0b001101001000,
 628    0b001101010000,
 629    0b001101100000,
 630    0b001101101000,
 631    0b001101110000,
 632    0b001101110001,
 633    0b001101111000,
 634    0b010001101000,
 635    0b010001101001,
 636    0b010001101010,
 637    0b010110001000
 638 };
 639
 640 /* This is actually the control index table for Cherryview (26 bits), but the
 641  * only difference from Broadwell (24 bits) is that it has two extra 0-bits at
 642  * the start.
 643  *
 644  * The low 24 bits have the same mappings on both hardware.
 645  */
 646 static const uint32_t gen8_3src_control_index_table[4] = {
 647    0b00100000000110000000000001,
 648    0b00000000000110000000000001,
 649    0b00000000001000000000000001,
 650    0b00000000001000000000100001
 651 };
 652
 653 /* This is actually the control index table for Cherryview (49 bits), but the
 654  * only difference from Broadwell (46 bits) is that it has three extra 0-bits
 655  * at the start.
 656  *
 657  * The low 44 bits have the same mappings on both hardware, and since the high
 658  * three bits on Broadwell are zero, we can reuse Cherryview's table.
 659  */
 660 static const uint64_t gen8_3src_source_index_table[4] = {
 661    0b0000001110010011100100111001000001111000000000000,
 662    0b0000001110010011100100111001000001111000000000010,
 663    0b0000001110010011100100111001000001111000000001000,
 664    0b0000001110010011100100111001000001111000000100000
 665 };
 666
 667 static const uint32_t *control_index_table;
 668 static const uint32_t *datatype_table;
 669 static const uint16_t *subreg_table;
 670 static const uint16_t *src_index_table;
 671
 672 static bool
 673 set_control_index(const struct brw_device_info *devinfo,
 674                   brw_compact_inst *dst, brw_inst *src)
 675 {
 676    uint32_t uncompacted = devinfo->gen >= 8  /* 17b/G45; 19b/IVB+ */
 677       ? (brw_inst_bits(src, 33, 31) << 16) | /*  3b */
 678         (brw_inst_bits(src, 23, 12) <<  4) | /* 12b */
 679         (brw_inst_bits(src, 10,  9) <<  2) | /*  2b */
 680         (brw_inst_bits(src, 34, 34) <<  1) | /*  1b */
 681         (brw_inst_bits(src,  8,  8))         /*  1b */
 682       : (brw_inst_bits(src, 31, 31) << 16) | /*  1b */
 683         (brw_inst_bits(src, 23,  8));        /* 16b */
 684
 685    /* On gen7, the flag register and subregister numbers are integrated into
 686     * the control index.
 687     */
 688    if (devinfo->gen == 7)
 689       uncompacted |= brw_inst_bits(src, 90, 89) << 17; /* 2b */
 690
 691    for (int i = 0; i < 32; i++) {
 692       if (control_index_table[i] == uncompacted) {
 693          brw_compact_inst_set_control_index(devinfo, dst, i);
 694          return true;
 695       }
 696    }
 697
 698    return false;
 699 }
 700
 701 static bool
 702 set_datatype_index(const struct brw_device_info *devinfo, brw_compact_inst *dst,
 703                    brw_inst *src)
 704 {
 705    uint32_t uncompacted = devinfo->gen >= 8  /* 18b/G45+; 21b/BDW+ */
 706       ? (brw_inst_bits(src, 63, 61) << 18) | /*  3b */
 707         (brw_inst_bits(src, 94, 89) << 12) | /*  6b */
 708         (brw_inst_bits(src, 46, 35))         /* 12b */
 709       : (brw_inst_bits(src, 63, 61) << 15) | /*  3b */
 710         (brw_inst_bits(src, 46, 32));        /* 15b */
 711
 712    for (int i = 0; i < 32; i++) {
 713       if (datatype_table[i] == uncompacted) {
 714          brw_compact_inst_set_datatype_index(devinfo, dst, i);
 715          return true;
 716       }
 717    }
 718
 719    return false;
 720 }
 721
 722 static bool
 723 set_subreg_index(const struct brw_device_info *devinfo, brw_compact_inst *dst,
 724                  brw_inst *src, bool is_immediate)
 725 {
 726    uint16_t uncompacted =                 /* 15b */
 727       (brw_inst_bits(src, 52, 48) << 0) | /*  5b */
 728       (brw_inst_bits(src, 68, 64) << 5);  /*  5b */
 729
 730    if (!is_immediate)
 731       uncompacted |= brw_inst_bits(src, 100, 96) << 10; /* 5b */
 732
 733    for (int i = 0; i < 32; i++) {
 734       if (subreg_table[i] == uncompacted) {
 735          brw_compact_inst_set_subreg_index(devinfo, dst, i);
 736          return true;
 737       }
 738    }
 739
 740    return false;
 741 }
 742
 743 static bool
 744 get_src_index(uint16_t uncompacted,
 745               uint16_t *compacted)
 746 {
 747    for (int i = 0; i < 32; i++) {
 748       if (src_index_table[i] == uncompacted) {
 749          *compacted = i;
 750          return true;
 751       }
 752    }
 753
 754    return false;
 755 }
 756
 757 static bool
 758 set_src0_index(const struct brw_device_info *devinfo,
 759                brw_compact_inst *dst, brw_inst *src)
 760 {
 761    uint16_t compacted;
 762    uint16_t uncompacted = brw_inst_bits(src, 88, 77); /* 12b */
 763
 764    if (!get_src_index(uncompacted, &compacted))
 765       return false;
 766
 767    brw_compact_inst_set_src0_index(devinfo, dst, compacted);
 768
 769    return true;
 770 }
 771
 772 static bool
 773 set_src1_index(const struct brw_device_info *devinfo, brw_compact_inst *dst,
 774                brw_inst *src, bool is_immediate)
 775 {
 776    uint16_t compacted;
 777
 778    if (is_immediate) {
 779       compacted = (brw_inst_imm_ud(devinfo, src) >> 8) & 0x1f;
 780    } else {
 781       uint16_t uncompacted = brw_inst_bits(src, 120, 109); /* 12b */
 782
 783       if (!get_src_index(uncompacted, &compacted))
 784          return false;
 785    }
 786
 787    brw_compact_inst_set_src1_index(devinfo, dst, compacted);
 788
 789    return true;
 790 }
 791
 792 static bool
 793 set_3src_control_index(const struct brw_device_info *devinfo,
 794                        brw_compact_inst *dst, brw_inst *src)
 795 {
 796    assert(devinfo->gen >= 8);
 797
 798    uint32_t uncompacted =                  /* 24b/BDW; 26b/CHV */
 799       (brw_inst_bits(src, 34, 32) << 21) | /*  3b */
 800       (brw_inst_bits(src, 28,  8));        /* 21b */
 801
 802    if (devinfo->gen >= 9 || devinfo->is_cherryview)
 803       uncompacted |= brw_inst_bits(src, 36, 35) << 24; /* 2b */
 804
 805    for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_control_index_table); i++) {
 806       if (gen8_3src_control_index_table[i] == uncompacted) {
 807          brw_compact_inst_set_3src_control_index(devinfo, dst, i);
 808          return true;
 809       }
 810    }
 811
 812    return false;
 813 }
 814
 815 static bool
 816 set_3src_source_index(const struct brw_device_info *devinfo,
 817                       brw_compact_inst *dst, brw_inst *src)
 818 {
 819    assert(devinfo->gen >= 8);
 820
 821    uint64_t uncompacted =                    /* 46b/BDW; 49b/CHV */
 822       (brw_inst_bits(src,  83,  83) << 43) | /*  1b */
 823       (brw_inst_bits(src, 114, 107) << 35) | /*  8b */
 824       (brw_inst_bits(src,  93,  86) << 27) | /*  8b */
 825       (brw_inst_bits(src,  72,  65) << 19) | /*  8b */
 826       (brw_inst_bits(src,  55,  37));        /* 19b */
 827
 828    if (devinfo->gen >= 9 || devinfo->is_cherryview) {
 829       uncompacted |=
 830          (brw_inst_bits(src, 126, 125) << 47) | /* 2b */
 831          (brw_inst_bits(src, 105, 104) << 45) | /* 2b */
 832          (brw_inst_bits(src,  84,  84) << 44);  /* 1b */
 833    } else {
 834       uncompacted |=
 835          (brw_inst_bits(src, 125, 125) << 45) | /* 1b */
 836          (brw_inst_bits(src, 104, 104) << 44);  /* 1b */
 837    }
 838
 839    for (unsigned i = 0; i < ARRAY_SIZE(gen8_3src_source_index_table); i++) {
 840       if (gen8_3src_source_index_table[i] == uncompacted) {
 841          brw_compact_inst_set_3src_source_index(devinfo, dst, i);
 842          return true;
 843       }
 844    }
 845
 846    return false;
 847 }
 848
 849 static bool
 850 has_unmapped_bits(const struct brw_device_info *devinfo, brw_inst *src)
 851 {
 852    /* EOT can only be mapped on a send if the src1 is an immediate */
 853    if ((brw_inst_opcode(devinfo, src) == BRW_OPCODE_SENDC ||
 854         brw_inst_opcode(devinfo, src) == BRW_OPCODE_SEND) &&
 855        brw_inst_eot(devinfo, src))
 856       return true;
 857
 858    /* Check for instruction bits that don't map to any of the fields of the
 859     * compacted instruction.  The instruction cannot be compacted if any of
 860     * them are set.  They overlap with:
 861     *  - NibCtrl (bit 47 on Gen7, bit 11 on Gen8)
 862     *  - Dst.AddrImm[9] (bit 47 on Gen8)
 863     *  - Src0.AddrImm[9] (bit 95 on Gen8)
 864     *  - Imm64[27:31] (bits 91-95 on Gen7, bit 95 on Gen8)
 865     *  - UIP[31] (bit 95 on Gen8)
 866     */
 867    if (devinfo->gen >= 8) {
 868       assert(!brw_inst_bits(src, 7,  7));
 869       return brw_inst_bits(src, 95, 95) ||
 870              brw_inst_bits(src, 47, 47) ||
 871              brw_inst_bits(src, 11, 11);
 872    } else {
 873       assert(!brw_inst_bits(src, 7,  7) &&
 874              !(devinfo->gen < 7 && brw_inst_bits(src, 90, 90)));
 875       return brw_inst_bits(src, 95, 91) ||
 876              brw_inst_bits(src, 47, 47);
 877    }
 878 }
 879
 880 static bool
 881 has_3src_unmapped_bits(const struct brw_device_info *devinfo, brw_inst *src)
 882 {
 883    /* Check for three-source instruction bits that don't map to any of the
 884     * fields of the compacted instruction.  All of them seem to be reserved
 885     * bits currently.
 886     */
 887    if (devinfo->gen >= 9 || devinfo->is_cherryview) {
 888       assert(!brw_inst_bits(src, 127, 127) &&
 889              !brw_inst_bits(src, 7,  7));
 890    } else {
 891       assert(devinfo->gen >= 8);
 892       assert(!brw_inst_bits(src, 127, 126) &&
 893              !brw_inst_bits(src, 105, 105) &&
 894              !brw_inst_bits(src, 84, 84) &&
 895              !brw_inst_bits(src, 36, 35) &&
 896              !brw_inst_bits(src, 7,  7));
 897    }
 898
 899    return false;
 900 }
 901
 902 static bool
 903 brw_try_compact_3src_instruction(const struct brw_device_info *devinfo,
 904                                  brw_compact_inst *dst, brw_inst *src)
 905 {
 906    assert(devinfo->gen >= 8);
 907
 908    if (has_3src_unmapped_bits(devinfo, src))
 909       return false;
 910
 911 #define compact(field) \
 912    brw_compact_inst_set_3src_##field(devinfo, dst, brw_inst_3src_##field(devinfo, src))
 913
 914    compact(opcode);
 915
 916    if (!set_3src_control_index(devinfo, dst, src))
 917       return false;
 918
 919    if (!set_3src_source_index(devinfo, dst, src))
 920       return false;
 921
 922    compact(dst_reg_nr);
 923    compact(src0_rep_ctrl);
 924    brw_compact_inst_set_3src_cmpt_control(devinfo, dst, true);
 925    compact(debug_control);
 926    compact(saturate);
 927    compact(src1_rep_ctrl);
 928    compact(src2_rep_ctrl);
 929    compact(src0_reg_nr);
 930    compact(src1_reg_nr);
 931    compact(src2_reg_nr);
 932    compact(src0_subreg_nr);
 933    compact(src1_subreg_nr);
 934    compact(src2_subreg_nr);
 935
 936 #undef compact
 937
 938    return true;
 939 }
 940
 941 /* Compacted instructions have 12-bits for immediate sources, and a 13th bit
 942  * that's replicated through the high 20 bits.
 943  *
 944  * Effectively this means we get 12-bit integers, 0.0f, and some limited uses
 945  * of packed vectors as compactable immediates.
 946  */
 947 static bool
 948 is_compactable_immediate(unsigned imm)
 949 {
 950    /* We get the low 12 bits as-is. */
 951    imm &= ~0xfff;
 952
 953    /* We get one bit replicated through the top 20 bits. */
 954    return imm == 0 || imm == 0xfffff000;
 955 }
 956
 957 /* Returns whether an opcode takes three sources. */
 958 static bool
 959 is_3src(uint32_t op)
 960 {
 961    return opcode_descs[op].nsrc == 3;
 962 }
 963
 964 /**
 965  * Tries to compact instruction src into dst.
 966  *
 967  * It doesn't modify dst unless src is compactable, which is relied on by
 968  * brw_compact_instructions().
 969  */
 970 bool
 971 brw_try_compact_instruction(const struct brw_device_info *devinfo,
 972                             brw_compact_inst *dst, brw_inst *src)
 973 {
 974    brw_compact_inst temp;
 975
 976    assert(brw_inst_cmpt_control(devinfo, src) == 0);
 977
 978    if (is_3src(brw_inst_opcode(devinfo, src))) {
 979       if (devinfo->gen >= 8) {
 980          memset(&temp, 0, sizeof(temp));
 981          if (brw_try_compact_3src_instruction(devinfo, &temp, src)) {
 982             *dst = temp;
 983             return true;
 984          } else {
 985             return false;
 986          }
 987       } else {
 988          return false;
 989       }
 990    }
 991
 992    bool is_immediate =
 993       brw_inst_src0_reg_file(devinfo, src) == BRW_IMMEDIATE_VALUE ||
 994       brw_inst_src1_reg_file(devinfo, src) == BRW_IMMEDIATE_VALUE;
 995    if (is_immediate &&
 996        (devinfo->gen < 6 ||
 997         !is_compactable_immediate(brw_inst_imm_ud(devinfo, src)))) {
 998       return false;
 999    }
1000
1001    if (has_unmapped_bits(devinfo, src))
1002       return false;
1003
1004    memset(&temp, 0, sizeof(temp));
1005
1006 #define compact(field) \
1007    brw_compact_inst_set_##field(devinfo, &temp, brw_inst_##field(devinfo, src))
1008
1009    compact(opcode);
1010    compact(debug_control);
1011
1012    if (!set_control_index(devinfo, &temp, src))
1013       return false;
1014    if (!set_datatype_index(devinfo, &temp, src))
1015       return false;
1016    if (!set_subreg_index(devinfo, &temp, src, is_immediate))
1017       return false;
1018
1019    if (devinfo->gen >= 6) {
1020       compact(acc_wr_control);
1021    } else {
1022       compact(mask_control_ex);
1023    }
1024
1025    compact(cond_modifier);
1026
1027    if (devinfo->gen <= 6)
1028       compact(flag_subreg_nr);
1029
1030    brw_compact_inst_set_cmpt_control(devinfo, &temp, true);
1031
1032    if (!set_src0_index(devinfo, &temp, src))
1033       return false;
1034    if (!set_src1_index(devinfo, &temp, src, is_immediate))
1035       return false;
1036
1037    brw_compact_inst_set_dst_reg_nr(devinfo, &temp,
1038                                    brw_inst_dst_da_reg_nr(devinfo, src));
1039    brw_compact_inst_set_src0_reg_nr(devinfo, &temp,
1040                                     brw_inst_src0_da_reg_nr(devinfo, src));
1041
1042    if (is_immediate) {
1043       brw_compact_inst_set_src1_reg_nr(devinfo, &temp,
1044                                        brw_inst_imm_ud(devinfo, src) & 0xff);
1045    } else {
1046       brw_compact_inst_set_src1_reg_nr(devinfo, &temp,
1047                                        brw_inst_src1_da_reg_nr(devinfo, src));
1048    }
1049
1050 #undef compact
1051
1052    *dst = temp;
1053
1054    return true;
1055 }
1056
1057 static void
1058 set_uncompacted_control(const struct brw_device_info *devinfo, brw_inst *dst,
1059                         brw_compact_inst *src)
1060 {
1061    uint32_t uncompacted =
1062       control_index_table[brw_compact_inst_control_index(devinfo, src)];
1063
1064    if (devinfo->gen >= 8) {
1065       brw_inst_set_bits(dst, 33, 31, (uncompacted >> 16));
1066       brw_inst_set_bits(dst, 23, 12, (uncompacted >>  4) & 0xfff);
1067       brw_inst_set_bits(dst, 10,  9, (uncompacted >>  2) & 0x3);
1068       brw_inst_set_bits(dst, 34, 34, (uncompacted >>  1) & 0x1);
1069       brw_inst_set_bits(dst,  8,  8, (uncompacted >>  0) & 0x1);
1070    } else {
1071       brw_inst_set_bits(dst, 31, 31, (uncompacted >> 16) & 0x1);
1072       brw_inst_set_bits(dst, 23,  8, (uncompacted & 0xffff));
1073
1074       if (devinfo->gen == 7)
1075          brw_inst_set_bits(dst, 90, 89, uncompacted >> 17);
1076    }
1077 }
1078
1079 static void
1080 set_uncompacted_datatype(const struct brw_device_info *devinfo, brw_inst *dst,
1081                          brw_compact_inst *src)
1082 {
1083    uint32_t uncompacted =
1084       datatype_table[brw_compact_inst_datatype_index(devinfo, src)];
1085
1086    if (devinfo->gen >= 8) {
1087       brw_inst_set_bits(dst, 63, 61, (uncompacted >> 18));
1088       brw_inst_set_bits(dst, 94, 89, (uncompacted >> 12) & 0x3f);
1089       brw_inst_set_bits(dst, 46, 35, (uncompacted >>  0) & 0xfff);
1090    } else {
1091       brw_inst_set_bits(dst, 63, 61, (uncompacted >> 15));
1092       brw_inst_set_bits(dst, 46, 32, (uncompacted & 0x7fff));
1093    }
1094 }
1095
1096 static void
1097 set_uncompacted_subreg(const struct brw_device_info *devinfo, brw_inst *dst,
1098                        brw_compact_inst *src)
1099 {
1100    uint16_t uncompacted =
1101       subreg_table[brw_compact_inst_subreg_index(devinfo, src)];
1102
1103    brw_inst_set_bits(dst, 100, 96, (uncompacted >> 10));
1104    brw_inst_set_bits(dst,  68, 64, (uncompacted >>  5) & 0x1f);
1105    brw_inst_set_bits(dst,  52, 48, (uncompacted >>  0) & 0x1f);
1106 }
1107
1108 static void
1109 set_uncompacted_src0(const struct brw_device_info *devinfo, brw_inst *dst,
1110                      brw_compact_inst *src)
1111 {
1112    uint32_t compacted = brw_compact_inst_src0_index(devinfo, src);
1113    uint16_t uncompacted = src_index_table[compacted];
1114
1115    brw_inst_set_bits(dst, 88, 77, uncompacted);
1116 }
1117
1118 static void
1119 set_uncompacted_src1(const struct brw_device_info *devinfo, brw_inst *dst,
1120                      brw_compact_inst *src, bool is_immediate)
1121 {
1122    if (is_immediate) {
1123       signed high5 = brw_compact_inst_src1_index(devinfo, src);
1124       /* Replicate top bit of src1_index into high 20 bits of the immediate. */
1125       brw_inst_set_imm_ud(devinfo, dst, (high5 << 27) >> 19);
1126    } else {
1127       uint16_t uncompacted =
1128          src_index_table[brw_compact_inst_src1_index(devinfo, src)];
1129
1130       brw_inst_set_bits(dst, 120, 109, uncompacted);
1131    }
1132 }
1133
1134 static void
1135 set_uncompacted_3src_control_index(const struct brw_device_info *devinfo,
1136                                    brw_inst *dst, brw_compact_inst *src)
1137 {
1138    assert(devinfo->gen >= 8);
1139
1140    uint32_t compacted = brw_compact_inst_3src_control_index(devinfo, src);
1141    uint32_t uncompacted = gen8_3src_control_index_table[compacted];
1142
1143    brw_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7);
1144    brw_inst_set_bits(dst, 28,  8, (uncompacted >>  0) & 0x1fffff);
1145
1146    if (devinfo->gen >= 9 || devinfo->is_cherryview)
1147       brw_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3);
1148 }
1149
1150 static void
1151 set_uncompacted_3src_source_index(const struct brw_device_info *devinfo,
1152                                   brw_inst *dst, brw_compact_inst *src)
1153 {
1154    assert(devinfo->gen >= 8);
1155
1156    uint32_t compacted = brw_compact_inst_3src_source_index(devinfo, src);
1157    uint64_t uncompacted = gen8_3src_source_index_table[compacted];
1158
1159    brw_inst_set_bits(dst,  83,  83, (uncompacted >> 43) & 0x1);
1160    brw_inst_set_bits(dst, 114, 107, (uncompacted >> 35) & 0xff);
1161    brw_inst_set_bits(dst,  93,  86, (uncompacted >> 27) & 0xff);
1162    brw_inst_set_bits(dst,  72,  65, (uncompacted >> 19) & 0xff);
1163    brw_inst_set_bits(dst,  55,  37, (uncompacted >>  0) & 0x7ffff);
1164
1165    if (devinfo->gen >= 9 || devinfo->is_cherryview) {
1166       brw_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3);
1167       brw_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3);
1168       brw_inst_set_bits(dst,  84,  84, (uncompacted >> 44) & 0x1);
1169    } else {
1170       brw_inst_set_bits(dst, 125, 125, (uncompacted >> 45) & 0x1);
1171       brw_inst_set_bits(dst, 104, 104, (uncompacted >> 44) & 0x1);
1172    }
1173 }
1174
1175 static void
1176 brw_uncompact_3src_instruction(const struct brw_device_info *devinfo,
1177                                brw_inst *dst, brw_compact_inst *src)
1178 {
1179    assert(devinfo->gen >= 8);
1180
1181 #define uncompact(field) \
1182    brw_inst_set_3src_##field(devinfo, dst, brw_compact_inst_3src_##field(devinfo, src))
1183
1184    uncompact(opcode);
1185
1186    set_uncompacted_3src_control_index(devinfo, dst, src);
1187    set_uncompacted_3src_source_index(devinfo, dst, src);
1188
1189    uncompact(dst_reg_nr);
1190    uncompact(src0_rep_ctrl);
1191    brw_inst_set_3src_cmpt_control(devinfo, dst, false);
1192    uncompact(debug_control);
1193    uncompact(saturate);
1194    uncompact(src1_rep_ctrl);
1195    uncompact(src2_rep_ctrl);
1196    uncompact(src0_reg_nr);
1197    uncompact(src1_reg_nr);
1198    uncompact(src2_reg_nr);
1199    uncompact(src0_subreg_nr);
1200    uncompact(src1_subreg_nr);
1201    uncompact(src2_subreg_nr);
1202
1203 #undef uncompact
1204 }
1205
1206 void
1207 brw_uncompact_instruction(const struct brw_device_info *devinfo, brw_inst *dst,
1208                           brw_compact_inst *src)
1209 {
1210    memset(dst, 0, sizeof(*dst));
1211
1212    if (devinfo->gen >= 8 && is_3src(brw_compact_inst_3src_opcode(devinfo, src))) {
1213       brw_uncompact_3src_instruction(devinfo, dst, src);
1214       return;
1215    }
1216
1217 #define uncompact(field) \
1218    brw_inst_set_##field(devinfo, dst, brw_compact_inst_##field(devinfo, src))
1219
1220    uncompact(opcode);
1221    uncompact(debug_control);
1222
1223    set_uncompacted_control(devinfo, dst, src);
1224    set_uncompacted_datatype(devinfo, dst, src);
1225
1226    /* src0/1 register file fields are in the datatype table. */
1227    bool is_immediate = brw_inst_src0_reg_file(devinfo, dst) == BRW_IMMEDIATE_VALUE ||
1228                        brw_inst_src1_reg_file(devinfo, dst) == BRW_IMMEDIATE_VALUE;
1229
1230    set_uncompacted_subreg(devinfo, dst, src);
1231
1232    if (devinfo->gen >= 6) {
1233       uncompact(acc_wr_control);
1234    } else {
1235       uncompact(mask_control_ex);
1236    }
1237
1238    uncompact(cond_modifier);
1239
1240    if (devinfo->gen <= 6)
1241       uncompact(flag_subreg_nr);
1242
1243    set_uncompacted_src0(devinfo, dst, src);
1244    set_uncompacted_src1(devinfo, dst, src, is_immediate);
1245
1246    brw_inst_set_dst_da_reg_nr(devinfo, dst,
1247                               brw_compact_inst_dst_reg_nr(devinfo, src));
1248    brw_inst_set_src0_da_reg_nr(devinfo, dst,
1249                                brw_compact_inst_src0_reg_nr(devinfo, src));
1250
1251    if (is_immediate) {
1252       brw_inst_set_imm_ud(devinfo, dst,
1253                           brw_inst_imm_ud(devinfo, dst) |
1254                           brw_compact_inst_src1_reg_nr(devinfo, src));
1255    } else {
1256       brw_inst_set_src1_da_reg_nr(devinfo, dst,
1257                                   brw_compact_inst_src1_reg_nr(devinfo, src));
1258    }
1259
1260 #undef uncompact
1261 }
1262
1263 void brw_debug_compact_uncompact(const struct brw_device_info *devinfo,
1264                                  brw_inst *orig,
1265                                  brw_inst *uncompacted)
1266 {
1267    fprintf(stderr, "Instruction compact/uncompact changed (gen%d):\n",
1268            devinfo->gen);
1269
1270    fprintf(stderr, "  before: ");
1271    brw_disassemble_inst(stderr, devinfo, orig, true);
1272
1273    fprintf(stderr, "  after:  ");
1274    brw_disassemble_inst(stderr, devinfo, uncompacted, false);
1275
1276    uint32_t *before_bits = (uint32_t *)orig;
1277    uint32_t *after_bits = (uint32_t *)uncompacted;
1278    fprintf(stderr, "  changed bits:\n");
1279    for (int i = 0; i < 128; i++) {
1280       uint32_t before = before_bits[i / 32] & (1 << (i & 31));
1281       uint32_t after = after_bits[i / 32] & (1 << (i & 31));
1282
1283       if (before != after) {
1284          fprintf(stderr, "  bit %d, %s to %s\n", i,
1285                  before ? "set" : "unset",
1286                  after ? "set" : "unset");
1287       }
1288    }
1289 }
1290
1291 static int
1292 compacted_between(int old_ip, int old_target_ip, int *compacted_counts)
1293 {
1294    int this_compacted_count = compacted_counts[old_ip];
1295    int target_compacted_count = compacted_counts[old_target_ip];
1296    return target_compacted_count - this_compacted_count;
1297 }
1298
1299 static void
1300 update_uip_jip(const struct brw_device_info *devinfo, brw_inst *insn,
1301                int this_old_ip, int *compacted_counts)
1302 {
1303    /* JIP and UIP are in units of:
1304     *    - bytes on Gen8+; and
1305     *    - compacted instructions on Gen6+.
1306     */
1307    int shift = devinfo->gen >= 8 ? 3 : 0;
1308
1309    int32_t jip_compacted = brw_inst_jip(devinfo, insn) >> shift;
1310    jip_compacted -= compacted_between(this_old_ip,
1311                                       this_old_ip + (jip_compacted / 2),
1312                                       compacted_counts);
1313    brw_inst_set_jip(devinfo, insn, jip_compacted << shift);
1314
1315    if (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_ENDIF ||
1316        brw_inst_opcode(devinfo, insn) == BRW_OPCODE_WHILE ||
1317        (brw_inst_opcode(devinfo, insn) == BRW_OPCODE_ELSE && devinfo->gen <= 7))
1318       return;
1319
1320    int32_t uip_compacted = brw_inst_uip(devinfo, insn) >> shift;
1321    uip_compacted -= compacted_between(this_old_ip,
1322                                       this_old_ip + (uip_compacted / 2),
1323                                       compacted_counts);
1324    brw_inst_set_uip(devinfo, insn, uip_compacted << shift);
1325 }
1326
1327 static void
1328 update_gen4_jump_count(const struct brw_device_info *devinfo, brw_inst *insn,
1329                        int this_old_ip, int *compacted_counts)
1330 {
1331    assert(devinfo->gen == 5 || devinfo->is_g4x);
1332
1333    /* Jump Count is in units of:
1334     *    - uncompacted instructions on G45; and
1335     *    - compacted instructions on Gen5.
1336     */
1337    int shift = devinfo->is_g4x ? 1 : 0;
1338
1339    int jump_count_compacted = brw_inst_gen4_jump_count(devinfo, insn) << shift;
1340
1341    int target_old_ip = this_old_ip + (jump_count_compacted / 2);
1342
1343    int this_compacted_count = compacted_counts[this_old_ip];
1344    int target_compacted_count = compacted_counts[target_old_ip];
1345
1346    jump_count_compacted -= (target_compacted_count - this_compacted_count);
1347    brw_inst_set_gen4_jump_count(devinfo, insn, jump_count_compacted >> shift);
1348 }
1349
1350 void
1351 brw_init_compaction_tables(const struct brw_device_info *devinfo)
1352 {
1353    static bool initialized;
1354    if (initialized || p_atomic_cmpxchg(&initialized, false, true) != false)
1355       return;
1356
1357    assert(g45_control_index_table[ARRAY_SIZE(g45_control_index_table) - 1] != 0);
1358    assert(g45_datatype_table[ARRAY_SIZE(g45_datatype_table) - 1] != 0);
1359    assert(g45_subreg_table[ARRAY_SIZE(g45_subreg_table) - 1] != 0);
1360    assert(g45_src_index_table[ARRAY_SIZE(g45_src_index_table) - 1] != 0);
1361    assert(gen6_control_index_table[ARRAY_SIZE(gen6_control_index_table) - 1] != 0);
1362    assert(gen6_datatype_table[ARRAY_SIZE(gen6_datatype_table) - 1] != 0);
1363    assert(gen6_subreg_table[ARRAY_SIZE(gen6_subreg_table) - 1] != 0);
1364    assert(gen6_src_index_table[ARRAY_SIZE(gen6_src_index_table) - 1] != 0);
1365    assert(gen7_control_index_table[ARRAY_SIZE(gen7_control_index_table) - 1] != 0);
1366    assert(gen7_datatype_table[ARRAY_SIZE(gen7_datatype_table) - 1] != 0);
1367    assert(gen7_subreg_table[ARRAY_SIZE(gen7_subreg_table) - 1] != 0);
1368    assert(gen7_src_index_table[ARRAY_SIZE(gen7_src_index_table) - 1] != 0);
1369    assert(gen8_control_index_table[ARRAY_SIZE(gen8_control_index_table) - 1] != 0);
1370    assert(gen8_datatype_table[ARRAY_SIZE(gen8_datatype_table) - 1] != 0);
1371    assert(gen8_subreg_table[ARRAY_SIZE(gen8_subreg_table) - 1] != 0);
1372    assert(gen8_src_index_table[ARRAY_SIZE(gen8_src_index_table) - 1] != 0);
1373
1374    switch (devinfo->gen) {
1375    case 9:
1376    case 8:
1377       control_index_table = gen8_control_index_table;
1378       datatype_table = gen8_datatype_table;
1379       subreg_table = gen8_subreg_table;
1380       src_index_table = gen8_src_index_table;
1381       break;
1382    case 7:
1383       control_index_table = gen7_control_index_table;
1384       datatype_table = gen7_datatype_table;
1385       subreg_table = gen7_subreg_table;
1386       src_index_table = gen7_src_index_table;
1387       break;
1388    case 6:
1389       control_index_table = gen6_control_index_table;
1390       datatype_table = gen6_datatype_table;
1391       subreg_table = gen6_subreg_table;
1392       src_index_table = gen6_src_index_table;
1393       break;
1394    case 5:
1395    case 4:
1396       control_index_table = g45_control_index_table;
1397       datatype_table = g45_datatype_table;
1398       subreg_table = g45_subreg_table;
1399       src_index_table = g45_src_index_table;
1400       break;
1401    default:
1402       unreachable("unknown generation");
1403    }
1404 }
1405
1406 void
1407 brw_compact_instructions(struct brw_codegen *p, int start_offset,
1408                          int num_annotations, struct annotation *annotation)
1409 {
1410    if (unlikely(INTEL_DEBUG & DEBUG_NO_COMPACTION))
1411       return;
1412
1413    const struct brw_device_info *devinfo = p->devinfo;
1414    void *store = p->store + start_offset / 16;
1415    /* For an instruction at byte offset 16*i before compaction, this is the
1416     * number of compacted instructions minus the number of padding NOP/NENOPs
1417     * that preceded it.
1418     */
1419    int compacted_counts[(p->next_insn_offset - start_offset) / sizeof(brw_inst)];
1420    /* For an instruction at byte offset 8*i after compaction, this was its IP
1421     * (in 16-byte units) before compaction.
1422     */
1423    int old_ip[(p->next_insn_offset - start_offset) / sizeof(brw_compact_inst)];
1424
1425    if (devinfo->gen == 4 && !devinfo->is_g4x)
1426       return;
1427
1428    int offset = 0;
1429    int compacted_count = 0;
1430    for (int src_offset = 0; src_offset < p->next_insn_offset - start_offset;
1431         src_offset += sizeof(brw_inst)) {
1432       brw_inst *src = store + src_offset;
1433       void *dst = store + offset;
1434
1435       old_ip[offset / sizeof(brw_compact_inst)] = src_offset / sizeof(brw_inst);
1436       compacted_counts[src_offset / sizeof(brw_inst)] = compacted_count;
1437
1438       brw_inst saved = *src;
1439
1440       if (brw_try_compact_instruction(devinfo, dst, src)) {
1441          compacted_count++;
1442
1443          if (INTEL_DEBUG) {
1444             brw_inst uncompacted;
1445             brw_uncompact_instruction(devinfo, &uncompacted, dst);
1446             if (memcmp(&saved, &uncompacted, sizeof(uncompacted))) {
1447                brw_debug_compact_uncompact(devinfo, &saved, &uncompacted);
1448             }
1449          }
1450
1451          offset += sizeof(brw_compact_inst);
1452       } else {
1453          /* All uncompacted instructions need to be aligned on G45. */
1454          if ((offset & sizeof(brw_compact_inst)) != 0 && devinfo->is_g4x){
1455             brw_compact_inst *align = store + offset;
1456             memset(align, 0, sizeof(*align));
1457             brw_compact_inst_set_opcode(devinfo, align, BRW_OPCODE_NENOP);
1458             brw_compact_inst_set_cmpt_control(devinfo, align, true);
1459             offset += sizeof(brw_compact_inst);
1460             compacted_count--;
1461             compacted_counts[src_offset / sizeof(brw_inst)] = compacted_count;
1462             old_ip[offset / sizeof(brw_compact_inst)] = src_offset / sizeof(brw_inst);
1463
1464             dst = store + offset;
1465          }
1466
1467          /* If we didn't compact this intruction, we need to move it down into
1468           * place.
1469           */
1470          if (offset != src_offset) {
1471             memmove(dst, src, sizeof(brw_inst));
1472          }
1473          offset += sizeof(brw_inst);
1474       }
1475    }
1476
1477    /* Fix up control flow offsets. */
1478    p->next_insn_offset = start_offset + offset;
1479    for (offset = 0; offset < p->next_insn_offset - start_offset;
1480         offset = next_offset(devinfo, store, offset)) {
1481       brw_inst *insn = store + offset;
1482       int this_old_ip = old_ip[offset / sizeof(brw_compact_inst)];
1483       int this_compacted_count = compacted_counts[this_old_ip];
1484
1485       switch (brw_inst_opcode(devinfo, insn)) {
1486       case BRW_OPCODE_BREAK:
1487       case BRW_OPCODE_CONTINUE:
1488       case BRW_OPCODE_HALT:
1489          if (devinfo->gen >= 6) {
1490             update_uip_jip(devinfo, insn, this_old_ip, compacted_counts);
1491          } else {
1492             update_gen4_jump_count(devinfo, insn, this_old_ip,
1493                                    compacted_counts);
1494          }
1495          break;
1496
1497       case BRW_OPCODE_IF:
1498       case BRW_OPCODE_IFF:
1499       case BRW_OPCODE_ELSE:
1500       case BRW_OPCODE_ENDIF:
1501       case BRW_OPCODE_WHILE:
1502          if (devinfo->gen >= 7) {
1503             if (brw_inst_cmpt_control(devinfo, insn)) {
1504                brw_inst uncompacted;
1505                brw_uncompact_instruction(devinfo, &uncompacted,
1506                                          (brw_compact_inst *)insn);
1507
1508                update_uip_jip(devinfo, &uncompacted, this_old_ip,
1509                               compacted_counts);
1510
1511                bool ret = brw_try_compact_instruction(devinfo,
1512                                                       (brw_compact_inst *)insn,
1513                                                       &uncompacted);
1514                assert(ret); (void)ret;
1515             } else {
1516                update_uip_jip(devinfo, insn, this_old_ip, compacted_counts);
1517             }
1518          } else if (devinfo->gen == 6) {
1519             assert(!brw_inst_cmpt_control(devinfo, insn));
1520
1521             /* Jump Count is in units of compacted instructions on Gen6. */
1522             int jump_count_compacted = brw_inst_gen6_jump_count(devinfo, insn);
1523
1524             int target_old_ip = this_old_ip + (jump_count_compacted / 2);
1525             int target_compacted_count = compacted_counts[target_old_ip];
1526             jump_count_compacted -= (target_compacted_count - this_compacted_count);
1527             brw_inst_set_gen6_jump_count(devinfo, insn, jump_count_compacted);
1528          } else {
1529             update_gen4_jump_count(devinfo, insn, this_old_ip,
1530                                    compacted_counts);
1531          }
1532          break;
1533
1534       case BRW_OPCODE_ADD:
1535          /* Add instructions modifying the IP register use an immediate src1,
1536           * and Gens that use this cannot compact instructions with immediate
1537           * operands.
1538           */
1539          if (brw_inst_cmpt_control(devinfo, insn))
1540             break;
1541
1542          if (brw_inst_dst_reg_file(devinfo, insn) == BRW_ARCHITECTURE_REGISTER_FILE &&
1543              brw_inst_dst_da_reg_nr(devinfo, insn) == BRW_ARF_IP) {
1544             assert(brw_inst_src1_reg_file(devinfo, insn) == BRW_IMMEDIATE_VALUE);
1545
1546             int shift = 3;
1547             int jump_compacted = brw_inst_imm_d(devinfo, insn) >> shift;
1548
1549             int target_old_ip = this_old_ip + (jump_compacted / 2);
1550             int target_compacted_count = compacted_counts[target_old_ip];
1551             jump_compacted -= (target_compacted_count - this_compacted_count);
1552             brw_inst_set_imm_ud(devinfo, insn, jump_compacted << shift);
1553          }
1554          break;
1555       }
1556    }
1557
1558    /* p->nr_insn is counting the number of uncompacted instructions still, so
1559     * divide.  We do want to be sure there's a valid instruction in any
1560     * alignment padding, so that the next compression pass (for the FS 8/16
1561     * compile passes) parses correctly.
1562     */
1563    if (p->next_insn_offset & sizeof(brw_compact_inst)) {
1564       brw_compact_inst *align = store + offset;
1565       memset(align, 0, sizeof(*align));
1566       brw_compact_inst_set_opcode(devinfo, align, BRW_OPCODE_NOP);
1567       brw_compact_inst_set_cmpt_control(devinfo, align, true);
1568       p->next_insn_offset += sizeof(brw_compact_inst);
1569    }
1570    p->nr_insn = p->next_insn_offset / sizeof(brw_inst);
1571
1572    /* Update the instruction offsets for each annotation. */
1573    if (annotation) {
1574       for (int offset = 0, i = 0; i < num_annotations; i++) {
1575          while (start_offset + old_ip[offset / sizeof(brw_compact_inst)] *
1576                 sizeof(brw_inst) != annotation[i].offset) {
1577             assert(start_offset + old_ip[offset / sizeof(brw_compact_inst)] *
1578                    sizeof(brw_inst) < annotation[i].offset);
1579             offset = next_offset(devinfo, store, offset);
1580          }
1581
1582          annotation[i].offset = start_offset + offset;
1583
1584          offset = next_offset(devinfo, store, offset);
1585       }
1586
1587       annotation[num_annotations].offset = p->next_insn_offset;
1588    }
1589 }