src/broadcom/compiler/v3d_compiler.h

   1 /*
   2  * Copyright © 2016 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #ifndef V3D_COMPILER_H
  25 #define V3D_COMPILER_H
  26
  27 #include <assert.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <stdbool.h>
  31 #include <stdint.h>
  32 #include <string.h>
  33
  34 #include "util/macros.h"
  35 #include "common/v3d_debug.h"
  36 #include "common/v3d_device_info.h"
  37 #include "common/v3d_limits.h"
  38 #include "compiler/nir/nir.h"
  39 #include "util/list.h"
  40 #include "util/u_math.h"
  41
  42 #include "qpu/qpu_instr.h"
  43 #include "pipe/p_state.h"
  44
  45 struct nir_builder;
  46
  47 struct v3d_fs_inputs {
  48         /**
  49          * Array of the meanings of the VPM inputs this shader needs.
  50          *
  51          * It doesn't include those that aren't part of the VPM, like
  52          * point/line coordinates.
  53          */
  54         struct v3d_varying_slot *input_slots;
  55         uint32_t num_inputs;
  56 };
  57
  58 enum qfile {
  59         /** An unused source or destination register. */
  60         QFILE_NULL,
  61
  62         /** A physical register, such as the W coordinate payload. */
  63         QFILE_REG,
  64         /** One of the regsiters for fixed function interactions. */
  65         QFILE_MAGIC,
  66
  67         /**
  68          *  A virtual register, that will be allocated to actual accumulator
  69          * or physical registers later.
  70          */
  71         QFILE_TEMP,
  72         QFILE_UNIF,
  73         QFILE_TLB,
  74         QFILE_TLBU,
  75
  76         /**
  77          * VPM reads use this with an index value to say what part of the VPM
  78          * is being read.
  79          */
  80         QFILE_VPM,
  81
  82         /**
  83          * Stores an immediate value in the index field that will be used
  84          * directly by qpu_load_imm().
  85          */
  86         QFILE_LOAD_IMM,
  87
  88         /**
  89          * Stores an immediate value in the index field that can be turned
  90          * into a small immediate field by qpu_encode_small_immediate().
  91          */
  92         QFILE_SMALL_IMM,
  93 };
  94
  95 /**
  96  * A reference to a QPU register or a virtual temp register.
  97  */
  98 struct qreg {
  99         enum qfile file;
 100         uint32_t index;
 101 };
 102
 103 static inline struct qreg vir_reg(enum qfile file, uint32_t index)
 104 {
 105         return (struct qreg){file, index};
 106 }
 107
 108 static inline struct qreg vir_nop_reg(void)
 109 {
 110         return (struct qreg){QFILE_NULL, 0};
 111 }
 112
 113 /**
 114  * A reference to an actual register at the QPU level, for register
 115  * allocation.
 116  */
 117 struct qpu_reg {
 118         bool magic;
 119         bool smimm;
 120         int index;
 121 };
 122
 123 struct qinst {
 124         /** Entry in qblock->instructions */
 125         struct list_head link;
 126
 127         /**
 128          * The instruction being wrapped.  Its condition codes, pack flags,
 129          * signals, etc. will all be used, with just the register references
 130          * being replaced by the contents of qinst->dst and qinst->src[].
 131          */
 132         struct v3d_qpu_instr qpu;
 133
 134         /* Pre-register-allocation references to src/dst registers */
 135         struct qreg dst;
 136         struct qreg src[3];
 137         bool is_last_thrsw;
 138
 139         /* If the instruction reads a uniform (other than through src[i].file
 140          * == QFILE_UNIF), that uniform's index in c->uniform_contents.  ~0
 141          * otherwise.
 142          */
 143         int uniform;
 144 };
 145
 146 enum quniform_contents {
 147         /**
 148          * Indicates that a constant 32-bit value is copied from the program's
 149          * uniform contents.
 150          */
 151         QUNIFORM_CONSTANT,
 152         /**
 153          * Indicates that the program's uniform contents are used as an index
 154          * into the GL uniform storage.
 155          */
 156         QUNIFORM_UNIFORM,
 157
 158         /** @{
 159          * Scaling factors from clip coordinates to relative to the viewport
 160          * center.
 161          *
 162          * This is used by the coordinate and vertex shaders to produce the
 163          * 32-bit entry consisting of 2 16-bit fields with 12.4 signed fixed
 164          * point offsets from the viewport ccenter.
 165          */
 166         QUNIFORM_VIEWPORT_X_SCALE,
 167         QUNIFORM_VIEWPORT_Y_SCALE,
 168         /** @} */
 169
 170         QUNIFORM_VIEWPORT_Z_OFFSET,
 171         QUNIFORM_VIEWPORT_Z_SCALE,
 172
 173         QUNIFORM_USER_CLIP_PLANE,
 174
 175         /**
 176          * A reference to a V3D 3.x texture config parameter 0 uniform.
 177          *
 178          * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
 179          * defines texture type, miplevels, and such.  It will be found as a
 180          * parameter to the first QOP_TEX_[STRB] instruction in a sequence.
 181          */
 182         QUNIFORM_TEXTURE_CONFIG_P0_0,
 183         QUNIFORM_TEXTURE_CONFIG_P0_1,
 184         QUNIFORM_TEXTURE_CONFIG_P0_2,
 185         QUNIFORM_TEXTURE_CONFIG_P0_3,
 186         QUNIFORM_TEXTURE_CONFIG_P0_4,
 187         QUNIFORM_TEXTURE_CONFIG_P0_5,
 188         QUNIFORM_TEXTURE_CONFIG_P0_6,
 189         QUNIFORM_TEXTURE_CONFIG_P0_7,
 190         QUNIFORM_TEXTURE_CONFIG_P0_8,
 191         QUNIFORM_TEXTURE_CONFIG_P0_9,
 192         QUNIFORM_TEXTURE_CONFIG_P0_10,
 193         QUNIFORM_TEXTURE_CONFIG_P0_11,
 194         QUNIFORM_TEXTURE_CONFIG_P0_12,
 195         QUNIFORM_TEXTURE_CONFIG_P0_13,
 196         QUNIFORM_TEXTURE_CONFIG_P0_14,
 197         QUNIFORM_TEXTURE_CONFIG_P0_15,
 198         QUNIFORM_TEXTURE_CONFIG_P0_16,
 199         QUNIFORM_TEXTURE_CONFIG_P0_17,
 200         QUNIFORM_TEXTURE_CONFIG_P0_18,
 201         QUNIFORM_TEXTURE_CONFIG_P0_19,
 202         QUNIFORM_TEXTURE_CONFIG_P0_20,
 203         QUNIFORM_TEXTURE_CONFIG_P0_21,
 204         QUNIFORM_TEXTURE_CONFIG_P0_22,
 205         QUNIFORM_TEXTURE_CONFIG_P0_23,
 206         QUNIFORM_TEXTURE_CONFIG_P0_24,
 207         QUNIFORM_TEXTURE_CONFIG_P0_25,
 208         QUNIFORM_TEXTURE_CONFIG_P0_26,
 209         QUNIFORM_TEXTURE_CONFIG_P0_27,
 210         QUNIFORM_TEXTURE_CONFIG_P0_28,
 211         QUNIFORM_TEXTURE_CONFIG_P0_29,
 212         QUNIFORM_TEXTURE_CONFIG_P0_30,
 213         QUNIFORM_TEXTURE_CONFIG_P0_31,
 214         QUNIFORM_TEXTURE_CONFIG_P0_32,
 215
 216         /**
 217          * A reference to a V3D 3.x texture config parameter 1 uniform.
 218          *
 219          * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
 220          * has the pointer to the indirect texture state.  Our data[] field
 221          * will have a packed p1 value, but the address field will be just
 222          * which texture unit's texture should be referenced.
 223          */
 224         QUNIFORM_TEXTURE_CONFIG_P1,
 225
 226         /* A V3D 4.x texture config parameter.  The high 8 bits will be
 227          * which texture or sampler is being sampled, and the driver must
 228          * replace the address field with the appropriate address.
 229          */
 230         QUNIFORM_TMU_CONFIG_P0,
 231         QUNIFORM_TMU_CONFIG_P1,
 232
 233         QUNIFORM_IMAGE_TMU_CONFIG_P0,
 234
 235         QUNIFORM_TEXTURE_FIRST_LEVEL,
 236
 237         QUNIFORM_TEXTURE_WIDTH,
 238         QUNIFORM_TEXTURE_HEIGHT,
 239         QUNIFORM_TEXTURE_DEPTH,
 240         QUNIFORM_TEXTURE_ARRAY_SIZE,
 241         QUNIFORM_TEXTURE_LEVELS,
 242
 243         QUNIFORM_UBO_ADDR,
 244
 245         QUNIFORM_TEXRECT_SCALE_X,
 246         QUNIFORM_TEXRECT_SCALE_Y,
 247
 248         /* Returns the base offset of the SSBO given by the data value. */
 249         QUNIFORM_SSBO_OFFSET,
 250
 251         /* Returns the size of the SSBO given by the data value. */
 252         QUNIFORM_GET_BUFFER_SIZE,
 253
 254         /* Sizes (in pixels) of a shader image given by the data value. */
 255         QUNIFORM_IMAGE_WIDTH,
 256         QUNIFORM_IMAGE_HEIGHT,
 257         QUNIFORM_IMAGE_DEPTH,
 258         QUNIFORM_IMAGE_ARRAY_SIZE,
 259
 260         QUNIFORM_ALPHA_REF,
 261
 262         /* Number of workgroups passed to glDispatchCompute in the dimension
 263          * selected by the data value.
 264          */
 265         QUNIFORM_NUM_WORK_GROUPS,
 266
 267         /**
 268          * Returns the the offset of the scratch buffer for register spilling.
 269          */
 270         QUNIFORM_SPILL_OFFSET,
 271         QUNIFORM_SPILL_SIZE_PER_THREAD,
 272
 273         /**
 274          * Returns the offset of the shared memory for compute shaders.
 275          *
 276          * This will be accessed using TMU general memory operations, so the
 277          * L2T cache will effectively be the shared memory area.
 278          */
 279         QUNIFORM_SHARED_OFFSET,
 280 };
 281
 282 static inline uint32_t v3d_tmu_config_data_create(uint32_t unit, uint32_t value)
 283 {
 284         return unit << 24 | value;
 285 }
 286
 287 static inline uint32_t v3d_tmu_config_data_get_unit(uint32_t data)
 288 {
 289         return data >> 24;
 290 }
 291
 292 static inline uint32_t v3d_tmu_config_data_get_value(uint32_t data)
 293 {
 294         return data & 0xffffff;
 295 }
 296
 297 struct v3d_varying_slot {
 298         uint8_t slot_and_component;
 299 };
 300
 301 static inline struct v3d_varying_slot
 302 v3d_slot_from_slot_and_component(uint8_t slot, uint8_t component)
 303 {
 304         assert(slot < 255 / 4);
 305         return (struct v3d_varying_slot){ (slot << 2) + component };
 306 }
 307
 308 static inline uint8_t v3d_slot_get_slot(struct v3d_varying_slot slot)
 309 {
 310         return slot.slot_and_component >> 2;
 311 }
 312
 313 static inline uint8_t v3d_slot_get_component(struct v3d_varying_slot slot)
 314 {
 315         return slot.slot_and_component & 3;
 316 }
 317
 318 struct v3d_ubo_range {
 319         /**
 320          * offset in bytes from the start of the ubo where this range is
 321          * uploaded.
 322          *
 323          * Only set once used is set.
 324          */
 325         uint32_t dst_offset;
 326
 327         /**
 328          * offset in bytes from the start of the gallium uniforms where the
 329          * data comes from.
 330          */
 331         uint32_t src_offset;
 332
 333         /** size in bytes of this ubo range */
 334         uint32_t size;
 335 };
 336
 337 struct v3d_key {
 338         void *shader_state;
 339         struct {
 340                 uint8_t swizzle[4];
 341                 uint8_t return_size;
 342                 uint8_t return_channels;
 343                 bool clamp_s:1;
 344                 bool clamp_t:1;
 345                 bool clamp_r:1;
 346         } tex[V3D_MAX_TEXTURE_SAMPLERS];
 347         uint8_t ucp_enables;
 348 };
 349
 350 struct v3d_fs_key {
 351         struct v3d_key base;
 352         bool depth_enabled;
 353         bool is_points;
 354         bool is_lines;
 355         bool alpha_test;
 356         bool point_coord_upper_left;
 357         bool light_twoside;
 358         bool msaa;
 359         bool sample_coverage;
 360         bool sample_alpha_to_coverage;
 361         bool sample_alpha_to_one;
 362         bool clamp_color;
 363         bool shade_model_flat;
 364         /* Mask of which color render targets are present. */
 365         uint8_t cbufs;
 366         uint8_t swap_color_rb;
 367         /* Mask of which render targets need to be written as 32-bit floats */
 368         uint8_t f32_color_rb;
 369         /* Masks of which render targets need to be written as ints/uints.
 370          * Used by gallium to work around lost information in TGSI.
 371          */
 372         uint8_t int_color_rb;
 373         uint8_t uint_color_rb;
 374         uint8_t alpha_test_func;
 375         uint8_t logicop_func;
 376         uint32_t point_sprite_mask;
 377
 378         struct pipe_rt_blend_state blend;
 379 };
 380
 381 struct v3d_vs_key {
 382         struct v3d_key base;
 383
 384         struct v3d_varying_slot fs_inputs[V3D_MAX_FS_INPUTS];
 385         uint8_t num_fs_inputs;
 386
 387         bool is_coord;
 388         bool per_vertex_point_size;
 389         bool clamp_color;
 390 };
 391
 392 /** A basic block of VIR intructions. */
 393 struct qblock {
 394         struct list_head link;
 395
 396         struct list_head instructions;
 397
 398         struct set *predecessors;
 399         struct qblock *successors[2];
 400
 401         int index;
 402
 403         /* Instruction IPs for the first and last instruction of the block.
 404          * Set by qpu_schedule.c.
 405          */
 406         uint32_t start_qpu_ip;
 407         uint32_t end_qpu_ip;
 408
 409         /* Instruction IP for the branch instruction of the block.  Set by
 410          * qpu_schedule.c.
 411          */
 412         uint32_t branch_qpu_ip;
 413
 414         /** Offset within the uniform stream at the start of the block. */
 415         uint32_t start_uniform;
 416         /** Offset within the uniform stream of the branch instruction */
 417         uint32_t branch_uniform;
 418
 419         /** @{ used by v3d_vir_live_variables.c */
 420         BITSET_WORD *def;
 421         BITSET_WORD *defin;
 422         BITSET_WORD *defout;
 423         BITSET_WORD *use;
 424         BITSET_WORD *live_in;
 425         BITSET_WORD *live_out;
 426         int start_ip, end_ip;
 427         /** @} */
 428 };
 429
 430 /** Which util/list.h add mode we should use when inserting an instruction. */
 431 enum vir_cursor_mode {
 432         vir_cursor_add,
 433         vir_cursor_addtail,
 434 };
 435
 436 /**
 437  * Tracking structure for where new instructions should be inserted.  Create
 438  * with one of the vir_after_inst()-style helper functions.
 439  *
 440  * This does not protect against removal of the block or instruction, so we
 441  * have an assert in instruction removal to try to catch it.
 442  */
 443 struct vir_cursor {
 444         enum vir_cursor_mode mode;
 445         struct list_head *link;
 446 };
 447
 448 static inline struct vir_cursor
 449 vir_before_inst(struct qinst *inst)
 450 {
 451         return (struct vir_cursor){ vir_cursor_addtail, &inst->link };
 452 }
 453
 454 static inline struct vir_cursor
 455 vir_after_inst(struct qinst *inst)
 456 {
 457         return (struct vir_cursor){ vir_cursor_add, &inst->link };
 458 }
 459
 460 static inline struct vir_cursor
 461 vir_before_block(struct qblock *block)
 462 {
 463         return (struct vir_cursor){ vir_cursor_add, &block->instructions };
 464 }
 465
 466 static inline struct vir_cursor
 467 vir_after_block(struct qblock *block)
 468 {
 469         return (struct vir_cursor){ vir_cursor_addtail, &block->instructions };
 470 }
 471
 472 /**
 473  * Compiler state saved across compiler invocations, for any expensive global
 474  * setup.
 475  */
 476 struct v3d_compiler {
 477         const struct v3d_device_info *devinfo;
 478         struct ra_regs *regs;
 479         unsigned int reg_class_any[3];
 480         unsigned int reg_class_r5[3];
 481         unsigned int reg_class_phys[3];
 482         unsigned int reg_class_phys_or_acc[3];
 483 };
 484
 485 struct v3d_compile {
 486         const struct v3d_device_info *devinfo;
 487         nir_shader *s;
 488         nir_function_impl *impl;
 489         struct exec_list *cf_node_list;
 490         const struct v3d_compiler *compiler;
 491
 492         void (*debug_output)(const char *msg,
 493                              void *debug_output_data);
 494         void *debug_output_data;
 495
 496         /**
 497          * Mapping from nir_register * or nir_ssa_def * to array of struct
 498          * qreg for the values.
 499          */
 500         struct hash_table *def_ht;
 501
 502         /* For each temp, the instruction generating its value. */
 503         struct qinst **defs;
 504         uint32_t defs_array_size;
 505
 506         /**
 507          * Inputs to the shader, arranged by TGSI declaration order.
 508          *
 509          * Not all fragment shader QFILE_VARY reads are present in this array.
 510          */
 511         struct qreg *inputs;
 512         struct qreg *outputs;
 513         bool msaa_per_sample_output;
 514         struct qreg color_reads[V3D_MAX_SAMPLES];
 515         struct qreg sample_colors[V3D_MAX_SAMPLES];
 516         uint32_t inputs_array_size;
 517         uint32_t outputs_array_size;
 518         uint32_t uniforms_array_size;
 519
 520         /* Booleans for whether the corresponding QFILE_VARY[i] is
 521          * flat-shaded.  This includes gl_FragColor flat-shading, which is
 522          * customized based on the shademodel_flat shader key.
 523          */
 524         uint32_t flat_shade_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
 525
 526         uint32_t noperspective_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
 527
 528         uint32_t centroid_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
 529
 530         bool uses_center_w;
 531         bool writes_z;
 532
 533         struct v3d_ubo_range *ubo_ranges;
 534         bool *ubo_range_used;
 535         uint32_t ubo_ranges_array_size;
 536         /** Number of uniform areas tracked in ubo_ranges. */
 537         uint32_t num_ubo_ranges;
 538         uint32_t next_ubo_dst_offset;
 539
 540         /* State for whether we're executing on each channel currently.  0 if
 541          * yes, otherwise a block number + 1 that the channel jumped to.
 542          */
 543         struct qreg execute;
 544         bool in_control_flow;
 545
 546         struct qreg line_x, point_x, point_y;
 547
 548         /**
 549          * Instance ID, which comes in before the vertex attribute payload if
 550          * the shader record requests it.
 551          */
 552         struct qreg iid;
 553
 554         /**
 555          * Vertex ID, which comes in before the vertex attribute payload
 556          * (after Instance ID) if the shader record requests it.
 557          */
 558         struct qreg vid;
 559
 560         /* Fragment shader payload regs. */
 561         struct qreg payload_w, payload_w_centroid, payload_z;
 562
 563         struct qreg cs_payload[2];
 564         struct qreg cs_shared_offset;
 565         int local_invocation_index_bits;
 566
 567         uint8_t vattr_sizes[V3D_MAX_VS_INPUTS / 4];
 568         uint32_t vpm_output_size;
 569
 570         /* Size in bytes of registers that have been spilled. This is how much
 571          * space needs to be available in the spill BO per thread per QPU.
 572          */
 573         uint32_t spill_size;
 574         /* Shader-db stats */
 575         uint32_t spills, fills, loops;
 576         /**
 577          * Register spilling's per-thread base address, shared between each
 578          * spill/fill's addressing calculations.
 579          */
 580         struct qreg spill_base;
 581         /* Bit vector of which temps may be spilled */
 582         BITSET_WORD *spillable;
 583
 584         /**
 585          * Array of the VARYING_SLOT_* of all FS QFILE_VARY reads.
 586          *
 587          * This includes those that aren't part of the VPM varyings, like
 588          * point/line coordinates.
 589          */
 590         struct v3d_varying_slot input_slots[V3D_MAX_FS_INPUTS];
 591
 592         /**
 593          * An entry per outputs[] in the VS indicating what the VARYING_SLOT_*
 594          * of the output is.  Used to emit from the VS in the order that the
 595          * FS needs.
 596          */
 597         struct v3d_varying_slot *output_slots;
 598
 599         struct pipe_shader_state *shader_state;
 600         struct v3d_key *key;
 601         struct v3d_fs_key *fs_key;
 602         struct v3d_vs_key *vs_key;
 603
 604         /* Live ranges of temps. */
 605         int *temp_start, *temp_end;
 606         bool live_intervals_valid;
 607
 608         uint32_t *uniform_data;
 609         enum quniform_contents *uniform_contents;
 610         uint32_t uniform_array_size;
 611         uint32_t num_uniforms;
 612         uint32_t output_position_index;
 613         nir_variable *output_color_var[4];
 614         uint32_t output_sample_mask_index;
 615
 616         struct qreg undef;
 617         uint32_t num_temps;
 618
 619         struct vir_cursor cursor;
 620         struct list_head blocks;
 621         int next_block_index;
 622         struct qblock *cur_block;
 623         struct qblock *loop_cont_block;
 624         struct qblock *loop_break_block;
 625
 626         uint64_t *qpu_insts;
 627         uint32_t qpu_inst_count;
 628         uint32_t qpu_inst_size;
 629
 630         /* For the FS, the number of varying inputs not counting the
 631          * point/line varyings payload
 632          */
 633         uint32_t num_inputs;
 634
 635         /**
 636          * Number of inputs from num_inputs remaining to be queued to the read
 637          * FIFO in the VS/CS.
 638          */
 639         uint32_t num_inputs_remaining;
 640
 641         /* Number of inputs currently in the read FIFO for the VS/CS */
 642         uint32_t num_inputs_in_fifo;
 643
 644         /** Next offset in the VPM to read from in the VS/CS */
 645         uint32_t vpm_read_offset;
 646
 647         uint32_t program_id;
 648         uint32_t variant_id;
 649
 650         /* Set to compile program in in 1x, 2x, or 4x threaded mode, where
 651          * SIG_THREAD_SWITCH is used to hide texturing latency at the cost of
 652          * limiting ourselves to the part of the physical reg space.
 653          *
 654          * On V3D 3.x, 2x or 4x divide the physical reg space by 2x or 4x.  On
 655          * V3D 4.x, all shaders are 2x threaded, and 4x only divides the
 656          * physical reg space in half.
 657          */
 658         uint8_t threads;
 659         struct qinst *last_thrsw;
 660         bool last_thrsw_at_top_level;
 661
 662         bool failed;
 663 };
 664
 665 struct v3d_uniform_list {
 666         enum quniform_contents *contents;
 667         uint32_t *data;
 668         uint32_t count;
 669 };
 670
 671 struct v3d_prog_data {
 672         struct v3d_uniform_list uniforms;
 673
 674         struct v3d_ubo_range *ubo_ranges;
 675         uint32_t num_ubo_ranges;
 676         uint32_t ubo_size;
 677         uint32_t spill_size;
 678
 679         uint8_t threads;
 680
 681         /* For threads > 1, whether the program should be dispatched in the
 682          * after-final-THRSW state.
 683          */
 684         bool single_seg;
 685 };
 686
 687 struct v3d_vs_prog_data {
 688         struct v3d_prog_data base;
 689
 690         bool uses_iid, uses_vid;
 691
 692         /* Number of components read from each vertex attribute. */
 693         uint8_t vattr_sizes[V3D_MAX_VS_INPUTS / 4];
 694
 695         /* Total number of components read, for the shader state record. */
 696         uint32_t vpm_input_size;
 697
 698         /* Total number of components written, for the shader state record. */
 699         uint32_t vpm_output_size;
 700
 701         /* Set if there should be separate VPM segments for input and output.
 702          * If unset, vpm_input_size will be 0.
 703          */
 704         bool separate_segments;
 705
 706         /* Value to be programmed in VCM_CACHE_SIZE. */
 707         uint8_t vcm_cache_size;
 708 };
 709
 710 struct v3d_fs_prog_data {
 711         struct v3d_prog_data base;
 712
 713         struct v3d_varying_slot input_slots[V3D_MAX_FS_INPUTS];
 714
 715         /* Array of flat shade flags.
 716          *
 717          * Each entry is only 24 bits (high 8 bits 0), to match the hardware
 718          * packet layout.
 719          */
 720         uint32_t flat_shade_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
 721
 722         uint32_t noperspective_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
 723
 724         uint32_t centroid_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
 725
 726         uint8_t num_inputs;
 727         bool writes_z;
 728         bool disable_ez;
 729         bool uses_center_w;
 730 };
 731
 732 static inline bool
 733 vir_has_uniform(struct qinst *inst)
 734 {
 735         return inst->uniform != ~0;
 736 }
 737
 738 /* Special nir_load_input intrinsic index for loading the current TLB
 739  * destination color.
 740  */
 741 #define V3D_NIR_TLB_COLOR_READ_INPUT            2000000000
 742
 743 #define V3D_NIR_MS_MASK_OUTPUT                  2000000000
 744
 745 extern const nir_shader_compiler_options v3d_nir_options;
 746
 747 const struct v3d_compiler *v3d_compiler_init(const struct v3d_device_info *devinfo);
 748 void v3d_compiler_free(const struct v3d_compiler *compiler);
 749 void v3d_optimize_nir(struct nir_shader *s);
 750
 751 uint64_t *v3d_compile(const struct v3d_compiler *compiler,
 752                       struct v3d_key *key,
 753                       struct v3d_prog_data **prog_data,
 754                       nir_shader *s,
 755                       void (*debug_output)(const char *msg,
 756                                            void *debug_output_data),
 757                       void *debug_output_data,
 758                       int program_id, int variant_id,
 759                       uint32_t *final_assembly_size);
 760
 761 void v3d_nir_to_vir(struct v3d_compile *c);
 762
 763 void vir_compile_destroy(struct v3d_compile *c);
 764 const char *vir_get_stage_name(struct v3d_compile *c);
 765 struct qblock *vir_new_block(struct v3d_compile *c);
 766 void vir_set_emit_block(struct v3d_compile *c, struct qblock *block);
 767 void vir_link_blocks(struct qblock *predecessor, struct qblock *successor);
 768 struct qblock *vir_entry_block(struct v3d_compile *c);
 769 struct qblock *vir_exit_block(struct v3d_compile *c);
 770 struct qinst *vir_add_inst(enum v3d_qpu_add_op op, struct qreg dst,
 771                            struct qreg src0, struct qreg src1);
 772 struct qinst *vir_mul_inst(enum v3d_qpu_mul_op op, struct qreg dst,
 773                            struct qreg src0, struct qreg src1);
 774 struct qinst *vir_branch_inst(struct v3d_compile *c,
 775                               enum v3d_qpu_branch_cond cond);
 776 void vir_remove_instruction(struct v3d_compile *c, struct qinst *qinst);
 777 uint32_t vir_get_uniform_index(struct v3d_compile *c,
 778                                enum quniform_contents contents,
 779                                uint32_t data);
 780 struct qreg vir_uniform(struct v3d_compile *c,
 781                         enum quniform_contents contents,
 782                         uint32_t data);
 783 void vir_schedule_instructions(struct v3d_compile *c);
 784 struct v3d_qpu_instr v3d_qpu_nop(void);
 785
 786 struct qreg vir_emit_def(struct v3d_compile *c, struct qinst *inst);
 787 struct qinst *vir_emit_nondef(struct v3d_compile *c, struct qinst *inst);
 788 void vir_set_cond(struct qinst *inst, enum v3d_qpu_cond cond);
 789 void vir_set_pf(struct qinst *inst, enum v3d_qpu_pf pf);
 790 void vir_set_uf(struct qinst *inst, enum v3d_qpu_uf uf);
 791 void vir_set_unpack(struct qinst *inst, int src,
 792                     enum v3d_qpu_input_unpack unpack);
 793
 794 struct qreg vir_get_temp(struct v3d_compile *c);
 795 void vir_emit_last_thrsw(struct v3d_compile *c);
 796 void vir_calculate_live_intervals(struct v3d_compile *c);
 797 int vir_get_nsrc(struct qinst *inst);
 798 bool vir_has_side_effects(struct v3d_compile *c, struct qinst *inst);
 799 bool vir_get_add_op(struct qinst *inst, enum v3d_qpu_add_op *op);
 800 bool vir_get_mul_op(struct qinst *inst, enum v3d_qpu_mul_op *op);
 801 bool vir_is_raw_mov(struct qinst *inst);
 802 bool vir_is_tex(struct qinst *inst);
 803 bool vir_is_add(struct qinst *inst);
 804 bool vir_is_mul(struct qinst *inst);
 805 bool vir_writes_r3(const struct v3d_device_info *devinfo, struct qinst *inst);
 806 bool vir_writes_r4(const struct v3d_device_info *devinfo, struct qinst *inst);
 807 struct qreg vir_follow_movs(struct v3d_compile *c, struct qreg reg);
 808 uint8_t vir_channels_written(struct qinst *inst);
 809 struct qreg ntq_get_src(struct v3d_compile *c, nir_src src, int i);
 810 void ntq_store_dest(struct v3d_compile *c, nir_dest *dest, int chan,
 811                     struct qreg result);
 812 void vir_emit_thrsw(struct v3d_compile *c);
 813
 814 void vir_dump(struct v3d_compile *c);
 815 void vir_dump_inst(struct v3d_compile *c, struct qinst *inst);
 816 void vir_dump_uniform(enum quniform_contents contents, uint32_t data);
 817
 818 void vir_validate(struct v3d_compile *c);
 819
 820 void vir_optimize(struct v3d_compile *c);
 821 bool vir_opt_algebraic(struct v3d_compile *c);
 822 bool vir_opt_constant_folding(struct v3d_compile *c);
 823 bool vir_opt_copy_propagate(struct v3d_compile *c);
 824 bool vir_opt_dead_code(struct v3d_compile *c);
 825 bool vir_opt_peephole_sf(struct v3d_compile *c);
 826 bool vir_opt_small_immediates(struct v3d_compile *c);
 827 bool vir_opt_vpm(struct v3d_compile *c);
 828 void v3d_nir_lower_blend(nir_shader *s, struct v3d_compile *c);
 829 void v3d_nir_lower_io(nir_shader *s, struct v3d_compile *c);
 830 void v3d_nir_lower_txf_ms(nir_shader *s, struct v3d_compile *c);
 831 void v3d_nir_lower_image_load_store(nir_shader *s);
 832 void vir_lower_uniforms(struct v3d_compile *c);
 833
 834 void v3d33_vir_vpm_read_setup(struct v3d_compile *c, int num_components);
 835 void v3d33_vir_vpm_write_setup(struct v3d_compile *c);
 836 void v3d33_vir_emit_tex(struct v3d_compile *c, nir_tex_instr *instr);
 837 void v3d40_vir_emit_tex(struct v3d_compile *c, nir_tex_instr *instr);
 838 void v3d40_vir_emit_image_load_store(struct v3d_compile *c,
 839                                      nir_intrinsic_instr *instr);
 840
 841 void v3d_vir_to_qpu(struct v3d_compile *c, struct qpu_reg *temp_registers);
 842 uint32_t v3d_qpu_schedule_instructions(struct v3d_compile *c);
 843 void qpu_validate(struct v3d_compile *c);
 844 struct qpu_reg *v3d_register_allocate(struct v3d_compile *c, bool *spilled);
 845 bool vir_init_reg_sets(struct v3d_compiler *compiler);
 846
 847 bool v3d_gl_format_is_return_32(GLenum format);
 848
 849 static inline bool
 850 quniform_contents_is_texture_p0(enum quniform_contents contents)
 851 {
 852         return (contents >= QUNIFORM_TEXTURE_CONFIG_P0_0 &&
 853                 contents < (QUNIFORM_TEXTURE_CONFIG_P0_0 +
 854                             V3D_MAX_TEXTURE_SAMPLERS));
 855 }
 856
 857 static inline bool
 858 vir_in_nonuniform_control_flow(struct v3d_compile *c)
 859 {
 860         return c->execute.file != QFILE_NULL;
 861 }
 862
 863 static inline struct qreg
 864 vir_uniform_ui(struct v3d_compile *c, uint32_t ui)
 865 {
 866         return vir_uniform(c, QUNIFORM_CONSTANT, ui);
 867 }
 868
 869 static inline struct qreg
 870 vir_uniform_f(struct v3d_compile *c, float f)
 871 {
 872         return vir_uniform(c, QUNIFORM_CONSTANT, fui(f));
 873 }
 874
 875 #define VIR_ALU0(name, vir_inst, op)                                     \
 876 static inline struct qreg                                                \
 877 vir_##name(struct v3d_compile *c)                                        \
 878 {                                                                        \
 879         return vir_emit_def(c, vir_inst(op, c->undef,                    \
 880                                         c->undef, c->undef));            \
 881 }                                                                        \
 882 static inline struct qinst *                                             \
 883 vir_##name##_dest(struct v3d_compile *c, struct qreg dest)               \
 884 {                                                                        \
 885         return vir_emit_nondef(c, vir_inst(op, dest,                     \
 886                                            c->undef, c->undef));         \
 887 }
 888
 889 #define VIR_ALU1(name, vir_inst, op)                                     \
 890 static inline struct qreg                                                \
 891 vir_##name(struct v3d_compile *c, struct qreg a)                         \
 892 {                                                                        \
 893         return vir_emit_def(c, vir_inst(op, c->undef,                    \
 894                                         a, c->undef));                   \
 895 }                                                                        \
 896 static inline struct qinst *                                             \
 897 vir_##name##_dest(struct v3d_compile *c, struct qreg dest,               \
 898                   struct qreg a)                                         \
 899 {                                                                        \
 900         return vir_emit_nondef(c, vir_inst(op, dest, a,          \
 901                                            c->undef));                   \
 902 }
 903
 904 #define VIR_ALU2(name, vir_inst, op)                                       \
 905 static inline struct qreg                                                \
 906 vir_##name(struct v3d_compile *c, struct qreg a, struct qreg b)          \
 907 {                                                                        \
 908         return vir_emit_def(c, vir_inst(op, c->undef, a, b));    \
 909 }                                                                        \
 910 static inline struct qinst *                                             \
 911 vir_##name##_dest(struct v3d_compile *c, struct qreg dest,               \
 912                   struct qreg a, struct qreg b)                          \
 913 {                                                                        \
 914         return vir_emit_nondef(c, vir_inst(op, dest, a, b));     \
 915 }
 916
 917 #define VIR_NODST_0(name, vir_inst, op)                                 \
 918 static inline struct qinst *                                            \
 919 vir_##name(struct v3d_compile *c)                                       \
 920 {                                                                       \
 921         return vir_emit_nondef(c, vir_inst(op, c->undef,                \
 922                                            c->undef, c->undef));        \
 923 }
 924
 925 #define VIR_NODST_1(name, vir_inst, op)                                               \
 926 static inline struct qinst *                                            \
 927 vir_##name(struct v3d_compile *c, struct qreg a)                        \
 928 {                                                                       \
 929         return vir_emit_nondef(c, vir_inst(op, c->undef,        \
 930                                            a, c->undef));               \
 931 }
 932
 933 #define VIR_NODST_2(name, vir_inst, op)                                               \
 934 static inline struct qinst *                                            \
 935 vir_##name(struct v3d_compile *c, struct qreg a, struct qreg b)         \
 936 {                                                                       \
 937         return vir_emit_nondef(c, vir_inst(op, c->undef,                \
 938                                            a, b));                      \
 939 }
 940
 941 #define VIR_SFU(name)                                                      \
 942 static inline struct qreg                                                \
 943 vir_##name(struct v3d_compile *c, struct qreg a)                         \
 944 {                                                                        \
 945         if (c->devinfo->ver >= 41) {                                     \
 946                 return vir_emit_def(c, vir_add_inst(V3D_QPU_A_##name,    \
 947                                                     c->undef,            \
 948                                                     a, c->undef));       \
 949         } else {                                                         \
 950                 vir_FMOV_dest(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_##name), a); \
 951                 return vir_FMOV(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4)); \
 952         }                                                                \
 953 }                                                                        \
 954 static inline struct qinst *                                             \
 955 vir_##name##_dest(struct v3d_compile *c, struct qreg dest,               \
 956                   struct qreg a)                                         \
 957 {                                                                        \
 958         if (c->devinfo->ver >= 41) {                                     \
 959                 return vir_emit_nondef(c, vir_add_inst(V3D_QPU_A_##name, \
 960                                                        dest,             \
 961                                                        a, c->undef));    \
 962         } else {                                                         \
 963                 vir_FMOV_dest(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_##name), a); \
 964                 return vir_FMOV_dest(c, dest, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4)); \
 965         }                                                                \
 966 }
 967
 968 #define VIR_A_ALU2(name) VIR_ALU2(name, vir_add_inst, V3D_QPU_A_##name)
 969 #define VIR_M_ALU2(name) VIR_ALU2(name, vir_mul_inst, V3D_QPU_M_##name)
 970 #define VIR_A_ALU1(name) VIR_ALU1(name, vir_add_inst, V3D_QPU_A_##name)
 971 #define VIR_M_ALU1(name) VIR_ALU1(name, vir_mul_inst, V3D_QPU_M_##name)
 972 #define VIR_A_ALU0(name) VIR_ALU0(name, vir_add_inst, V3D_QPU_A_##name)
 973 #define VIR_M_ALU0(name) VIR_ALU0(name, vir_mul_inst, V3D_QPU_M_##name)
 974 #define VIR_A_NODST_2(name) VIR_NODST_2(name, vir_add_inst, V3D_QPU_A_##name)
 975 #define VIR_M_NODST_2(name) VIR_NODST_2(name, vir_mul_inst, V3D_QPU_M_##name)
 976 #define VIR_A_NODST_1(name) VIR_NODST_1(name, vir_add_inst, V3D_QPU_A_##name)
 977 #define VIR_M_NODST_1(name) VIR_NODST_1(name, vir_mul_inst, V3D_QPU_M_##name)
 978 #define VIR_A_NODST_0(name) VIR_NODST_0(name, vir_add_inst, V3D_QPU_A_##name)
 979
 980 VIR_A_ALU2(FADD)
 981 VIR_A_ALU2(VFPACK)
 982 VIR_A_ALU2(FSUB)
 983 VIR_A_ALU2(FMIN)
 984 VIR_A_ALU2(FMAX)
 985
 986 VIR_A_ALU2(ADD)
 987 VIR_A_ALU2(SUB)
 988 VIR_A_ALU2(SHL)
 989 VIR_A_ALU2(SHR)
 990 VIR_A_ALU2(ASR)
 991 VIR_A_ALU2(ROR)
 992 VIR_A_ALU2(MIN)
 993 VIR_A_ALU2(MAX)
 994 VIR_A_ALU2(UMIN)
 995 VIR_A_ALU2(UMAX)
 996 VIR_A_ALU2(AND)
 997 VIR_A_ALU2(OR)
 998 VIR_A_ALU2(XOR)
 999 VIR_A_ALU2(VADD)
1000 VIR_A_ALU2(VSUB)
1001 VIR_A_NODST_2(STVPMV)
1002 VIR_A_ALU1(NOT)
1003 VIR_A_ALU1(NEG)
1004 VIR_A_ALU1(FLAPUSH)
1005 VIR_A_ALU1(FLBPUSH)
1006 VIR_A_ALU1(FLPOP)
1007 VIR_A_ALU1(SETMSF)
1008 VIR_A_ALU1(SETREVF)
1009 VIR_A_ALU0(TIDX)
1010 VIR_A_ALU0(EIDX)
1011 VIR_A_ALU1(LDVPMV_IN)
1012 VIR_A_ALU1(LDVPMV_OUT)
1013 VIR_A_ALU0(TMUWT)
1014
1015 VIR_A_ALU0(FXCD)
1016 VIR_A_ALU0(XCD)
1017 VIR_A_ALU0(FYCD)
1018 VIR_A_ALU0(YCD)
1019 VIR_A_ALU0(MSF)
1020 VIR_A_ALU0(REVF)
1021 VIR_A_ALU0(BARRIERID)
1022 VIR_A_NODST_1(VPMSETUP)
1023 VIR_A_NODST_0(VPMWT)
1024 VIR_A_ALU2(FCMP)
1025 VIR_A_ALU2(VFMAX)
1026
1027 VIR_A_ALU1(FROUND)
1028 VIR_A_ALU1(FTOIN)
1029 VIR_A_ALU1(FTRUNC)
1030 VIR_A_ALU1(FTOIZ)
1031 VIR_A_ALU1(FFLOOR)
1032 VIR_A_ALU1(FTOUZ)
1033 VIR_A_ALU1(FCEIL)
1034 VIR_A_ALU1(FTOC)
1035
1036 VIR_A_ALU1(FDX)
1037 VIR_A_ALU1(FDY)
1038
1039 VIR_A_ALU1(ITOF)
1040 VIR_A_ALU1(CLZ)
1041 VIR_A_ALU1(UTOF)
1042
1043 VIR_M_ALU2(UMUL24)
1044 VIR_M_ALU2(FMUL)
1045 VIR_M_ALU2(SMUL24)
1046 VIR_M_NODST_2(MULTOP)
1047
1048 VIR_M_ALU1(MOV)
1049 VIR_M_ALU1(FMOV)
1050
1051 VIR_SFU(RECIP)
1052 VIR_SFU(RSQRT)
1053 VIR_SFU(EXP)
1054 VIR_SFU(LOG)
1055 VIR_SFU(SIN)
1056 VIR_SFU(RSQRT2)
1057
1058 static inline struct qinst *
1059 vir_MOV_cond(struct v3d_compile *c, enum v3d_qpu_cond cond,
1060              struct qreg dest, struct qreg src)
1061 {
1062         struct qinst *mov = vir_MOV_dest(c, dest, src);
1063         vir_set_cond(mov, cond);
1064         return mov;
1065 }
1066
1067 static inline struct qreg
1068 vir_SEL(struct v3d_compile *c, enum v3d_qpu_cond cond,
1069         struct qreg src0, struct qreg src1)
1070 {
1071         struct qreg t = vir_get_temp(c);
1072         vir_MOV_dest(c, t, src1);
1073         vir_MOV_cond(c, cond, t, src0);
1074         return t;
1075 }
1076
1077 static inline struct qinst *
1078 vir_NOP(struct v3d_compile *c)
1079 {
1080         return vir_emit_nondef(c, vir_add_inst(V3D_QPU_A_NOP,
1081                                                c->undef, c->undef, c->undef));
1082 }
1083
1084 static inline struct qreg
1085 vir_LDTMU(struct v3d_compile *c)
1086 {
1087         if (c->devinfo->ver >= 41) {
1088                 struct qinst *ldtmu = vir_add_inst(V3D_QPU_A_NOP, c->undef,
1089                                                    c->undef, c->undef);
1090                 ldtmu->qpu.sig.ldtmu = true;
1091
1092                 return vir_emit_def(c, ldtmu);
1093         } else {
1094                 vir_NOP(c)->qpu.sig.ldtmu = true;
1095                 return vir_MOV(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4));
1096         }
1097 }
1098
1099 static inline struct qreg
1100 vir_UMUL(struct v3d_compile *c, struct qreg src0, struct qreg src1)
1101 {
1102         vir_MULTOP(c, src0, src1);
1103         return vir_UMUL24(c, src0, src1);
1104 }
1105
1106 /*
1107 static inline struct qreg
1108 vir_LOAD_IMM(struct v3d_compile *c, uint32_t val)
1109 {
1110         return vir_emit_def(c, vir_inst(QOP_LOAD_IMM, c->undef,
1111                                         vir_reg(QFILE_LOAD_IMM, val), c->undef));
1112 }
1113
1114 static inline struct qreg
1115 vir_LOAD_IMM_U2(struct v3d_compile *c, uint32_t val)
1116 {
1117         return vir_emit_def(c, vir_inst(QOP_LOAD_IMM_U2, c->undef,
1118                                         vir_reg(QFILE_LOAD_IMM, val),
1119                                         c->undef));
1120 }
1121 static inline struct qreg
1122 vir_LOAD_IMM_I2(struct v3d_compile *c, uint32_t val)
1123 {
1124         return vir_emit_def(c, vir_inst(QOP_LOAD_IMM_I2, c->undef,
1125                                         vir_reg(QFILE_LOAD_IMM, val),
1126                                         c->undef));
1127 }
1128 */
1129
1130 static inline struct qinst *
1131 vir_BRANCH(struct v3d_compile *c, enum v3d_qpu_branch_cond cond)
1132 {
1133         /* The actual uniform_data value will be set at scheduling time */
1134         return vir_emit_nondef(c, vir_branch_inst(c, cond));
1135 }
1136
1137 #define vir_for_each_block(block, c)                                    \
1138         list_for_each_entry(struct qblock, block, &c->blocks, link)
1139
1140 #define vir_for_each_block_rev(block, c)                                \
1141         list_for_each_entry_rev(struct qblock, block, &c->blocks, link)
1142
1143 /* Loop over the non-NULL members of the successors array. */
1144 #define vir_for_each_successor(succ, block)                             \
1145         for (struct qblock *succ = block->successors[0];                \
1146              succ != NULL;                                              \
1147              succ = (succ == block->successors[1] ? NULL :              \
1148                      block->successors[1]))
1149
1150 #define vir_for_each_inst(inst, block)                                  \
1151         list_for_each_entry(struct qinst, inst, &block->instructions, link)
1152
1153 #define vir_for_each_inst_rev(inst, block)                                  \
1154         list_for_each_entry_rev(struct qinst, inst, &block->instructions, link)
1155
1156 #define vir_for_each_inst_safe(inst, block)                             \
1157         list_for_each_entry_safe(struct qinst, inst, &block->instructions, link)
1158
1159 #define vir_for_each_inst_inorder(inst, c)                              \
1160         vir_for_each_block(_block, c)                                   \
1161                 vir_for_each_inst(inst, _block)
1162
1163 #define vir_for_each_inst_inorder_safe(inst, c)                         \
1164         vir_for_each_block(_block, c)                                   \
1165                 vir_for_each_inst_safe(inst, _block)
1166
1167 #endif /* V3D_COMPILER_H */