src/broadcom/compiler/v3d_compiler.h

   1 /*
   2  * Copyright © 2016 Broadcom
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #ifndef V3D_COMPILER_H
  25 #define V3D_COMPILER_H
  26
  27 #include <assert.h>
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <stdbool.h>
  31 #include <stdint.h>
  32 #include <string.h>
  33
  34 #include "util/macros.h"
  35 #include "common/v3d_debug.h"
  36 #include "common/v3d_device_info.h"
  37 #include "compiler/nir/nir.h"
  38 #include "util/list.h"
  39 #include "util/u_math.h"
  40
  41 #include "qpu/qpu_instr.h"
  42 #include "pipe/p_state.h"
  43
  44 #define V3D_MAX_TEXTURE_SAMPLERS 32
  45 #define V3D_MAX_SAMPLES 4
  46 #define V3D_MAX_FS_INPUTS 64
  47 #define V3D_MAX_VS_INPUTS 64
  48
  49 struct nir_builder;
  50
  51 struct v3d_fs_inputs {
  52         /**
  53          * Array of the meanings of the VPM inputs this shader needs.
  54          *
  55          * It doesn't include those that aren't part of the VPM, like
  56          * point/line coordinates.
  57          */
  58         struct v3d_varying_slot *input_slots;
  59         uint32_t num_inputs;
  60 };
  61
  62 enum qfile {
  63         /** An unused source or destination register. */
  64         QFILE_NULL,
  65
  66         /** A physical register, such as the W coordinate payload. */
  67         QFILE_REG,
  68         /** One of the regsiters for fixed function interactions. */
  69         QFILE_MAGIC,
  70
  71         /**
  72          *  A virtual register, that will be allocated to actual accumulator
  73          * or physical registers later.
  74          */
  75         QFILE_TEMP,
  76         QFILE_UNIF,
  77         QFILE_TLB,
  78         QFILE_TLBU,
  79
  80         /**
  81          * VPM reads use this with an index value to say what part of the VPM
  82          * is being read.
  83          */
  84         QFILE_VPM,
  85
  86         /**
  87          * Stores an immediate value in the index field that will be used
  88          * directly by qpu_load_imm().
  89          */
  90         QFILE_LOAD_IMM,
  91
  92         /**
  93          * Stores an immediate value in the index field that can be turned
  94          * into a small immediate field by qpu_encode_small_immediate().
  95          */
  96         QFILE_SMALL_IMM,
  97 };
  98
  99 /**
 100  * A reference to a QPU register or a virtual temp register.
 101  */
 102 struct qreg {
 103         enum qfile file;
 104         uint32_t index;
 105 };
 106
 107 static inline struct qreg vir_reg(enum qfile file, uint32_t index)
 108 {
 109         return (struct qreg){file, index};
 110 }
 111
 112 /**
 113  * A reference to an actual register at the QPU level, for register
 114  * allocation.
 115  */
 116 struct qpu_reg {
 117         bool magic;
 118         int index;
 119 };
 120
 121 struct qinst {
 122         /** Entry in qblock->instructions */
 123         struct list_head link;
 124
 125         /**
 126          * The instruction being wrapped.  Its condition codes, pack flags,
 127          * signals, etc. will all be used, with just the register references
 128          * being replaced by the contents of qinst->dst and qinst->src[].
 129          */
 130         struct v3d_qpu_instr qpu;
 131
 132         /* Pre-register-allocation references to src/dst registers */
 133         struct qreg dst;
 134         struct qreg src[3];
 135         bool cond_is_exec_mask;
 136         bool has_implicit_uniform;
 137         bool is_last_thrsw;
 138
 139         /* After vir_to_qpu.c: If instr reads a uniform, which uniform from
 140          * the uncompiled stream it is.
 141          */
 142         int uniform;
 143 };
 144
 145 enum quniform_contents {
 146         /**
 147          * Indicates that a constant 32-bit value is copied from the program's
 148          * uniform contents.
 149          */
 150         QUNIFORM_CONSTANT,
 151         /**
 152          * Indicates that the program's uniform contents are used as an index
 153          * into the GL uniform storage.
 154          */
 155         QUNIFORM_UNIFORM,
 156
 157         /** @{
 158          * Scaling factors from clip coordinates to relative to the viewport
 159          * center.
 160          *
 161          * This is used by the coordinate and vertex shaders to produce the
 162          * 32-bit entry consisting of 2 16-bit fields with 12.4 signed fixed
 163          * point offsets from the viewport ccenter.
 164          */
 165         QUNIFORM_VIEWPORT_X_SCALE,
 166         QUNIFORM_VIEWPORT_Y_SCALE,
 167         /** @} */
 168
 169         QUNIFORM_VIEWPORT_Z_OFFSET,
 170         QUNIFORM_VIEWPORT_Z_SCALE,
 171
 172         QUNIFORM_USER_CLIP_PLANE,
 173
 174         /**
 175          * A reference to a V3D 3.x texture config parameter 0 uniform.
 176          *
 177          * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
 178          * defines texture type, miplevels, and such.  It will be found as a
 179          * parameter to the first QOP_TEX_[STRB] instruction in a sequence.
 180          */
 181         QUNIFORM_TEXTURE_CONFIG_P0_0,
 182         QUNIFORM_TEXTURE_CONFIG_P0_1,
 183         QUNIFORM_TEXTURE_CONFIG_P0_2,
 184         QUNIFORM_TEXTURE_CONFIG_P0_3,
 185         QUNIFORM_TEXTURE_CONFIG_P0_4,
 186         QUNIFORM_TEXTURE_CONFIG_P0_5,
 187         QUNIFORM_TEXTURE_CONFIG_P0_6,
 188         QUNIFORM_TEXTURE_CONFIG_P0_7,
 189         QUNIFORM_TEXTURE_CONFIG_P0_8,
 190         QUNIFORM_TEXTURE_CONFIG_P0_9,
 191         QUNIFORM_TEXTURE_CONFIG_P0_10,
 192         QUNIFORM_TEXTURE_CONFIG_P0_11,
 193         QUNIFORM_TEXTURE_CONFIG_P0_12,
 194         QUNIFORM_TEXTURE_CONFIG_P0_13,
 195         QUNIFORM_TEXTURE_CONFIG_P0_14,
 196         QUNIFORM_TEXTURE_CONFIG_P0_15,
 197         QUNIFORM_TEXTURE_CONFIG_P0_16,
 198         QUNIFORM_TEXTURE_CONFIG_P0_17,
 199         QUNIFORM_TEXTURE_CONFIG_P0_18,
 200         QUNIFORM_TEXTURE_CONFIG_P0_19,
 201         QUNIFORM_TEXTURE_CONFIG_P0_20,
 202         QUNIFORM_TEXTURE_CONFIG_P0_21,
 203         QUNIFORM_TEXTURE_CONFIG_P0_22,
 204         QUNIFORM_TEXTURE_CONFIG_P0_23,
 205         QUNIFORM_TEXTURE_CONFIG_P0_24,
 206         QUNIFORM_TEXTURE_CONFIG_P0_25,
 207         QUNIFORM_TEXTURE_CONFIG_P0_26,
 208         QUNIFORM_TEXTURE_CONFIG_P0_27,
 209         QUNIFORM_TEXTURE_CONFIG_P0_28,
 210         QUNIFORM_TEXTURE_CONFIG_P0_29,
 211         QUNIFORM_TEXTURE_CONFIG_P0_30,
 212         QUNIFORM_TEXTURE_CONFIG_P0_31,
 213         QUNIFORM_TEXTURE_CONFIG_P0_32,
 214
 215         /**
 216          * A reference to a V3D 3.x texture config parameter 1 uniform.
 217          *
 218          * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
 219          * has the pointer to the indirect texture state.  Our data[] field
 220          * will have a packed p1 value, but the address field will be just
 221          * which texture unit's texture should be referenced.
 222          */
 223         QUNIFORM_TEXTURE_CONFIG_P1,
 224
 225         /* A a V3D 4.x texture config parameter.  The high 8 bits will be
 226          * which texture or sampler is being sampled, and the driver must
 227          * replace the address field with the appropriate address.
 228          */
 229         QUNIFORM_TMU_CONFIG_P0,
 230         QUNIFORM_TMU_CONFIG_P1,
 231
 232         QUNIFORM_TEXTURE_FIRST_LEVEL,
 233
 234         QUNIFORM_TEXTURE_WIDTH,
 235         QUNIFORM_TEXTURE_HEIGHT,
 236         QUNIFORM_TEXTURE_DEPTH,
 237         QUNIFORM_TEXTURE_ARRAY_SIZE,
 238         QUNIFORM_TEXTURE_LEVELS,
 239
 240         QUNIFORM_UBO_ADDR,
 241
 242         QUNIFORM_TEXRECT_SCALE_X,
 243         QUNIFORM_TEXRECT_SCALE_Y,
 244
 245         QUNIFORM_TEXTURE_BORDER_COLOR,
 246
 247         QUNIFORM_ALPHA_REF,
 248         QUNIFORM_SAMPLE_MASK,
 249
 250         /**
 251          * Returns the the offset of the scratch buffer for register spilling.
 252          */
 253         QUNIFORM_SPILL_OFFSET,
 254         QUNIFORM_SPILL_SIZE_PER_THREAD,
 255 };
 256
 257 struct v3d_varying_slot {
 258         uint8_t slot_and_component;
 259 };
 260
 261 static inline struct v3d_varying_slot
 262 v3d_slot_from_slot_and_component(uint8_t slot, uint8_t component)
 263 {
 264         assert(slot < 255 / 4);
 265         return (struct v3d_varying_slot){ (slot << 2) + component };
 266 }
 267
 268 static inline uint8_t v3d_slot_get_slot(struct v3d_varying_slot slot)
 269 {
 270         return slot.slot_and_component >> 2;
 271 }
 272
 273 static inline uint8_t v3d_slot_get_component(struct v3d_varying_slot slot)
 274 {
 275         return slot.slot_and_component & 3;
 276 }
 277
 278 struct v3d_ubo_range {
 279         /**
 280          * offset in bytes from the start of the ubo where this range is
 281          * uploaded.
 282          *
 283          * Only set once used is set.
 284          */
 285         uint32_t dst_offset;
 286
 287         /**
 288          * offset in bytes from the start of the gallium uniforms where the
 289          * data comes from.
 290          */
 291         uint32_t src_offset;
 292
 293         /** size in bytes of this ubo range */
 294         uint32_t size;
 295 };
 296
 297 struct v3d_key {
 298         void *shader_state;
 299         struct {
 300                 uint8_t swizzle[4];
 301                 uint8_t return_size;
 302                 uint8_t return_channels;
 303                 unsigned compare_mode:1;
 304                 unsigned compare_func:3;
 305                 bool clamp_s:1;
 306                 bool clamp_t:1;
 307                 bool clamp_r:1;
 308         } tex[V3D_MAX_TEXTURE_SAMPLERS];
 309         uint8_t ucp_enables;
 310 };
 311
 312 struct v3d_fs_key {
 313         struct v3d_key base;
 314         bool depth_enabled;
 315         bool is_points;
 316         bool is_lines;
 317         bool alpha_test;
 318         bool point_coord_upper_left;
 319         bool light_twoside;
 320         bool msaa;
 321         bool sample_coverage;
 322         bool sample_alpha_to_coverage;
 323         bool sample_alpha_to_one;
 324         bool clamp_color;
 325         bool shade_model_flat;
 326         uint8_t nr_cbufs;
 327         uint8_t swap_color_rb;
 328         /* Mask of which render targets need to be written as 32-bit floats */
 329         uint8_t f32_color_rb;
 330         /* Masks of which render targets need to be written as ints/uints.
 331          * Used by gallium to work around lost information in TGSI.
 332          */
 333         uint8_t int_color_rb;
 334         uint8_t uint_color_rb;
 335         uint8_t alpha_test_func;
 336         uint8_t logicop_func;
 337         uint32_t point_sprite_mask;
 338
 339         struct pipe_rt_blend_state blend;
 340 };
 341
 342 struct v3d_vs_key {
 343         struct v3d_key base;
 344
 345         struct v3d_varying_slot fs_inputs[V3D_MAX_FS_INPUTS];
 346         uint8_t num_fs_inputs;
 347
 348         bool is_coord;
 349         bool per_vertex_point_size;
 350         bool clamp_color;
 351 };
 352
 353 /** A basic block of VIR intructions. */
 354 struct qblock {
 355         struct list_head link;
 356
 357         struct list_head instructions;
 358
 359         struct set *predecessors;
 360         struct qblock *successors[2];
 361
 362         int index;
 363
 364         /* Instruction IPs for the first and last instruction of the block.
 365          * Set by qpu_schedule.c.
 366          */
 367         uint32_t start_qpu_ip;
 368         uint32_t end_qpu_ip;
 369
 370         /* Instruction IP for the branch instruction of the block.  Set by
 371          * qpu_schedule.c.
 372          */
 373         uint32_t branch_qpu_ip;
 374
 375         /** Offset within the uniform stream at the start of the block. */
 376         uint32_t start_uniform;
 377         /** Offset within the uniform stream of the branch instruction */
 378         uint32_t branch_uniform;
 379
 380         /** @{ used by v3d_vir_live_variables.c */
 381         BITSET_WORD *def;
 382         BITSET_WORD *use;
 383         BITSET_WORD *live_in;
 384         BITSET_WORD *live_out;
 385         int start_ip, end_ip;
 386         /** @} */
 387 };
 388
 389 /** Which util/list.h add mode we should use when inserting an instruction. */
 390 enum vir_cursor_mode {
 391         vir_cursor_add,
 392         vir_cursor_addtail,
 393 };
 394
 395 /**
 396  * Tracking structure for where new instructions should be inserted.  Create
 397  * with one of the vir_after_inst()-style helper functions.
 398  *
 399  * This does not protect against removal of the block or instruction, so we
 400  * have an assert in instruction removal to try to catch it.
 401  */
 402 struct vir_cursor {
 403         enum vir_cursor_mode mode;
 404         struct list_head *link;
 405 };
 406
 407 static inline struct vir_cursor
 408 vir_before_inst(struct qinst *inst)
 409 {
 410         return (struct vir_cursor){ vir_cursor_addtail, &inst->link };
 411 }
 412
 413 static inline struct vir_cursor
 414 vir_after_inst(struct qinst *inst)
 415 {
 416         return (struct vir_cursor){ vir_cursor_add, &inst->link };
 417 }
 418
 419 static inline struct vir_cursor
 420 vir_before_block(struct qblock *block)
 421 {
 422         return (struct vir_cursor){ vir_cursor_add, &block->instructions };
 423 }
 424
 425 static inline struct vir_cursor
 426 vir_after_block(struct qblock *block)
 427 {
 428         return (struct vir_cursor){ vir_cursor_addtail, &block->instructions };
 429 }
 430
 431 /**
 432  * Compiler state saved across compiler invocations, for any expensive global
 433  * setup.
 434  */
 435 struct v3d_compiler {
 436         const struct v3d_device_info *devinfo;
 437         struct ra_regs *regs;
 438         unsigned int reg_class_phys[3];
 439         unsigned int reg_class_phys_or_acc[3];
 440 };
 441
 442 struct v3d_compile {
 443         const struct v3d_device_info *devinfo;
 444         nir_shader *s;
 445         nir_function_impl *impl;
 446         struct exec_list *cf_node_list;
 447         const struct v3d_compiler *compiler;
 448
 449         /**
 450          * Mapping from nir_register * or nir_ssa_def * to array of struct
 451          * qreg for the values.
 452          */
 453         struct hash_table *def_ht;
 454
 455         /* For each temp, the instruction generating its value. */
 456         struct qinst **defs;
 457         uint32_t defs_array_size;
 458
 459         /**
 460          * Inputs to the shader, arranged by TGSI declaration order.
 461          *
 462          * Not all fragment shader QFILE_VARY reads are present in this array.
 463          */
 464         struct qreg *inputs;
 465         struct qreg *outputs;
 466         bool msaa_per_sample_output;
 467         struct qreg color_reads[V3D_MAX_SAMPLES];
 468         struct qreg sample_colors[V3D_MAX_SAMPLES];
 469         uint32_t inputs_array_size;
 470         uint32_t outputs_array_size;
 471         uint32_t uniforms_array_size;
 472
 473         /* Booleans for whether the corresponding QFILE_VARY[i] is
 474          * flat-shaded.  This includes gl_FragColor flat-shading, which is
 475          * customized based on the shademodel_flat shader key.
 476          */
 477         uint32_t flat_shade_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
 478
 479         uint32_t centroid_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
 480
 481         bool uses_center_w;
 482
 483         struct v3d_ubo_range *ubo_ranges;
 484         bool *ubo_range_used;
 485         uint32_t ubo_ranges_array_size;
 486         /** Number of uniform areas tracked in ubo_ranges. */
 487         uint32_t num_ubo_ranges;
 488         uint32_t next_ubo_dst_offset;
 489
 490         /* State for whether we're executing on each channel currently.  0 if
 491          * yes, otherwise a block number + 1 that the channel jumped to.
 492          */
 493         struct qreg execute;
 494
 495         struct qreg line_x, point_x, point_y;
 496
 497         /**
 498          * Instance ID, which comes in before the vertex attribute payload if
 499          * the shader record requests it.
 500          */
 501         struct qreg iid;
 502
 503         /**
 504          * Vertex ID, which comes in before the vertex attribute payload
 505          * (after Instance ID) if the shader record requests it.
 506          */
 507         struct qreg vid;
 508
 509         /* Fragment shader payload regs. */
 510         struct qreg payload_w, payload_w_centroid, payload_z;
 511
 512         uint8_t vattr_sizes[V3D_MAX_VS_INPUTS];
 513         uint32_t num_vpm_writes;
 514
 515         /* Size in bytes of registers that have been spilled. This is how much
 516          * space needs to be available in the spill BO per thread per QPU.
 517          */
 518         uint32_t spill_size;
 519         /* Shader-db stats for register spilling. */
 520         uint32_t spills, fills;
 521         /**
 522          * Register spilling's per-thread base address, shared between each
 523          * spill/fill's addressing calculations.
 524          */
 525         struct qreg spill_base;
 526         /* Bit vector of which temps may be spilled */
 527         BITSET_WORD *spillable;
 528
 529         /**
 530          * Array of the VARYING_SLOT_* of all FS QFILE_VARY reads.
 531          *
 532          * This includes those that aren't part of the VPM varyings, like
 533          * point/line coordinates.
 534          */
 535         struct v3d_varying_slot input_slots[V3D_MAX_FS_INPUTS];
 536
 537         /**
 538          * An entry per outputs[] in the VS indicating what the VARYING_SLOT_*
 539          * of the output is.  Used to emit from the VS in the order that the
 540          * FS needs.
 541          */
 542         struct v3d_varying_slot *output_slots;
 543
 544         struct pipe_shader_state *shader_state;
 545         struct v3d_key *key;
 546         struct v3d_fs_key *fs_key;
 547         struct v3d_vs_key *vs_key;
 548
 549         /* Live ranges of temps. */
 550         int *temp_start, *temp_end;
 551         bool live_intervals_valid;
 552
 553         uint32_t *uniform_data;
 554         enum quniform_contents *uniform_contents;
 555         uint32_t uniform_array_size;
 556         uint32_t num_uniforms;
 557         uint32_t num_outputs;
 558         uint32_t output_position_index;
 559         nir_variable *output_color_var[4];
 560         uint32_t output_point_size_index;
 561         uint32_t output_sample_mask_index;
 562
 563         struct qreg undef;
 564         uint32_t num_temps;
 565
 566         struct vir_cursor cursor;
 567         struct list_head blocks;
 568         int next_block_index;
 569         struct qblock *cur_block;
 570         struct qblock *loop_cont_block;
 571         struct qblock *loop_break_block;
 572
 573         uint64_t *qpu_insts;
 574         uint32_t qpu_inst_count;
 575         uint32_t qpu_inst_size;
 576
 577         /* For the FS, the number of varying inputs not counting the
 578          * point/line varyings payload
 579          */
 580         uint32_t num_inputs;
 581
 582         /**
 583          * Number of inputs from num_inputs remaining to be queued to the read
 584          * FIFO in the VS/CS.
 585          */
 586         uint32_t num_inputs_remaining;
 587
 588         /* Number of inputs currently in the read FIFO for the VS/CS */
 589         uint32_t num_inputs_in_fifo;
 590
 591         /** Next offset in the VPM to read from in the VS/CS */
 592         uint32_t vpm_read_offset;
 593
 594         uint32_t program_id;
 595         uint32_t variant_id;
 596
 597         /* Set to compile program in in 1x, 2x, or 4x threaded mode, where
 598          * SIG_THREAD_SWITCH is used to hide texturing latency at the cost of
 599          * limiting ourselves to the part of the physical reg space.
 600          *
 601          * On V3D 3.x, 2x or 4x divide the physical reg space by 2x or 4x.  On
 602          * V3D 4.x, all shaders are 2x threaded, and 4x only divides the
 603          * physical reg space in half.
 604          */
 605         uint8_t threads;
 606         struct qinst *last_thrsw;
 607         bool last_thrsw_at_top_level;
 608
 609         bool failed;
 610 };
 611
 612 struct v3d_uniform_list {
 613         enum quniform_contents *contents;
 614         uint32_t *data;
 615         uint32_t count;
 616 };
 617
 618 struct v3d_prog_data {
 619         struct v3d_uniform_list uniforms;
 620
 621         struct v3d_ubo_range *ubo_ranges;
 622         uint32_t num_ubo_ranges;
 623         uint32_t ubo_size;
 624         uint32_t spill_size;
 625
 626         uint8_t num_inputs;
 627         uint8_t threads;
 628
 629         /* For threads > 1, whether the program should be dispatched in the
 630          * after-final-THRSW state.
 631          */
 632         bool single_seg;
 633 };
 634
 635 struct v3d_vs_prog_data {
 636         struct v3d_prog_data base;
 637
 638         bool uses_iid, uses_vid;
 639
 640         /* Number of components read from each vertex attribute. */
 641         uint8_t vattr_sizes[32];
 642
 643         /* Total number of components read, for the shader state record. */
 644         uint32_t vpm_input_size;
 645
 646         /* Total number of components written, for the shader state record. */
 647         uint32_t vpm_output_size;
 648 };
 649
 650 struct v3d_fs_prog_data {
 651         struct v3d_prog_data base;
 652
 653         struct v3d_varying_slot input_slots[V3D_MAX_FS_INPUTS];
 654
 655         /* Array of flat shade flags.
 656          *
 657          * Each entry is only 24 bits (high 8 bits 0), to match the hardware
 658          * packet layout.
 659          */
 660         uint32_t flat_shade_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
 661
 662         uint32_t centroid_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
 663
 664         bool writes_z;
 665         bool discard;
 666         bool uses_center_w;
 667 };
 668
 669 /* Special nir_load_input intrinsic index for loading the current TLB
 670  * destination color.
 671  */
 672 #define V3D_NIR_TLB_COLOR_READ_INPUT            2000000000
 673
 674 #define V3D_NIR_MS_MASK_OUTPUT                  2000000000
 675
 676 extern const nir_shader_compiler_options v3d_nir_options;
 677
 678 const struct v3d_compiler *v3d_compiler_init(const struct v3d_device_info *devinfo);
 679 void v3d_compiler_free(const struct v3d_compiler *compiler);
 680 void v3d_optimize_nir(struct nir_shader *s);
 681
 682 uint64_t *v3d_compile_vs(const struct v3d_compiler *compiler,
 683                          struct v3d_vs_key *key,
 684                          struct v3d_vs_prog_data *prog_data,
 685                          nir_shader *s,
 686                          int program_id, int variant_id,
 687                          uint32_t *final_assembly_size);
 688
 689 uint64_t *v3d_compile_fs(const struct v3d_compiler *compiler,
 690                          struct v3d_fs_key *key,
 691                          struct v3d_fs_prog_data *prog_data,
 692                          nir_shader *s,
 693                          int program_id, int variant_id,
 694                          uint32_t *final_assembly_size);
 695
 696 void v3d_nir_to_vir(struct v3d_compile *c);
 697
 698 void vir_compile_destroy(struct v3d_compile *c);
 699 const char *vir_get_stage_name(struct v3d_compile *c);
 700 struct qblock *vir_new_block(struct v3d_compile *c);
 701 void vir_set_emit_block(struct v3d_compile *c, struct qblock *block);
 702 void vir_link_blocks(struct qblock *predecessor, struct qblock *successor);
 703 struct qblock *vir_entry_block(struct v3d_compile *c);
 704 struct qblock *vir_exit_block(struct v3d_compile *c);
 705 struct qinst *vir_add_inst(enum v3d_qpu_add_op op, struct qreg dst,
 706                            struct qreg src0, struct qreg src1);
 707 struct qinst *vir_mul_inst(enum v3d_qpu_mul_op op, struct qreg dst,
 708                            struct qreg src0, struct qreg src1);
 709 struct qinst *vir_branch_inst(enum v3d_qpu_branch_cond cond, struct qreg src0);
 710 void vir_remove_instruction(struct v3d_compile *c, struct qinst *qinst);
 711 struct qreg vir_uniform(struct v3d_compile *c,
 712                         enum quniform_contents contents,
 713                         uint32_t data);
 714 void vir_schedule_instructions(struct v3d_compile *c);
 715 struct v3d_qpu_instr v3d_qpu_nop(void);
 716
 717 struct qreg vir_emit_def(struct v3d_compile *c, struct qinst *inst);
 718 struct qinst *vir_emit_nondef(struct v3d_compile *c, struct qinst *inst);
 719 void vir_set_cond(struct qinst *inst, enum v3d_qpu_cond cond);
 720 void vir_set_pf(struct qinst *inst, enum v3d_qpu_pf pf);
 721 void vir_set_unpack(struct qinst *inst, int src,
 722                     enum v3d_qpu_input_unpack unpack);
 723
 724 struct qreg vir_get_temp(struct v3d_compile *c);
 725 void vir_emit_last_thrsw(struct v3d_compile *c);
 726 void vir_calculate_live_intervals(struct v3d_compile *c);
 727 bool vir_has_implicit_uniform(struct qinst *inst);
 728 int vir_get_implicit_uniform_src(struct qinst *inst);
 729 int vir_get_non_sideband_nsrc(struct qinst *inst);
 730 int vir_get_nsrc(struct qinst *inst);
 731 bool vir_has_side_effects(struct v3d_compile *c, struct qinst *inst);
 732 bool vir_get_add_op(struct qinst *inst, enum v3d_qpu_add_op *op);
 733 bool vir_get_mul_op(struct qinst *inst, enum v3d_qpu_mul_op *op);
 734 bool vir_is_raw_mov(struct qinst *inst);
 735 bool vir_is_tex(struct qinst *inst);
 736 bool vir_is_add(struct qinst *inst);
 737 bool vir_is_mul(struct qinst *inst);
 738 bool vir_is_float_input(struct qinst *inst);
 739 bool vir_depends_on_flags(struct qinst *inst);
 740 bool vir_writes_r3(const struct v3d_device_info *devinfo, struct qinst *inst);
 741 bool vir_writes_r4(const struct v3d_device_info *devinfo, struct qinst *inst);
 742 struct qreg vir_follow_movs(struct v3d_compile *c, struct qreg reg);
 743 uint8_t vir_channels_written(struct qinst *inst);
 744 struct qreg ntq_get_src(struct v3d_compile *c, nir_src src, int i);
 745 void ntq_store_dest(struct v3d_compile *c, nir_dest *dest, int chan,
 746                     struct qreg result);
 747 void vir_emit_thrsw(struct v3d_compile *c);
 748
 749 void vir_dump(struct v3d_compile *c);
 750 void vir_dump_inst(struct v3d_compile *c, struct qinst *inst);
 751
 752 void vir_validate(struct v3d_compile *c);
 753
 754 void vir_optimize(struct v3d_compile *c);
 755 bool vir_opt_algebraic(struct v3d_compile *c);
 756 bool vir_opt_constant_folding(struct v3d_compile *c);
 757 bool vir_opt_copy_propagate(struct v3d_compile *c);
 758 bool vir_opt_dead_code(struct v3d_compile *c);
 759 bool vir_opt_peephole_sf(struct v3d_compile *c);
 760 bool vir_opt_small_immediates(struct v3d_compile *c);
 761 bool vir_opt_vpm(struct v3d_compile *c);
 762 void v3d_nir_lower_blend(nir_shader *s, struct v3d_compile *c);
 763 void v3d_nir_lower_io(nir_shader *s, struct v3d_compile *c);
 764 void v3d_nir_lower_txf_ms(nir_shader *s, struct v3d_compile *c);
 765 void vir_lower_uniforms(struct v3d_compile *c);
 766
 767 void v3d33_vir_vpm_read_setup(struct v3d_compile *c, int num_components);
 768 void v3d33_vir_vpm_write_setup(struct v3d_compile *c);
 769 void v3d33_vir_emit_tex(struct v3d_compile *c, nir_tex_instr *instr);
 770 void v3d40_vir_emit_tex(struct v3d_compile *c, nir_tex_instr *instr);
 771
 772 void v3d_vir_to_qpu(struct v3d_compile *c, struct qpu_reg *temp_registers);
 773 uint32_t v3d_qpu_schedule_instructions(struct v3d_compile *c);
 774 void qpu_validate(struct v3d_compile *c);
 775 struct qpu_reg *v3d_register_allocate(struct v3d_compile *c, bool *spilled);
 776 bool vir_init_reg_sets(struct v3d_compiler *compiler);
 777
 778 void vir_PF(struct v3d_compile *c, struct qreg src, enum v3d_qpu_pf pf);
 779
 780 static inline bool
 781 quniform_contents_is_texture_p0(enum quniform_contents contents)
 782 {
 783         return (contents >= QUNIFORM_TEXTURE_CONFIG_P0_0 &&
 784                 contents < (QUNIFORM_TEXTURE_CONFIG_P0_0 +
 785                             V3D_MAX_TEXTURE_SAMPLERS));
 786 }
 787
 788 static inline struct qreg
 789 vir_uniform_ui(struct v3d_compile *c, uint32_t ui)
 790 {
 791         return vir_uniform(c, QUNIFORM_CONSTANT, ui);
 792 }
 793
 794 static inline struct qreg
 795 vir_uniform_f(struct v3d_compile *c, float f)
 796 {
 797         return vir_uniform(c, QUNIFORM_CONSTANT, fui(f));
 798 }
 799
 800 #define VIR_ALU0(name, vir_inst, op)                                     \
 801 static inline struct qreg                                                \
 802 vir_##name(struct v3d_compile *c)                                        \
 803 {                                                                        \
 804         return vir_emit_def(c, vir_inst(op, c->undef,                    \
 805                                         c->undef, c->undef));            \
 806 }                                                                        \
 807 static inline struct qinst *                                             \
 808 vir_##name##_dest(struct v3d_compile *c, struct qreg dest)               \
 809 {                                                                        \
 810         return vir_emit_nondef(c, vir_inst(op, dest,                     \
 811                                            c->undef, c->undef));         \
 812 }
 813
 814 #define VIR_ALU1(name, vir_inst, op)                                     \
 815 static inline struct qreg                                                \
 816 vir_##name(struct v3d_compile *c, struct qreg a)                         \
 817 {                                                                        \
 818         return vir_emit_def(c, vir_inst(op, c->undef,                    \
 819                                         a, c->undef));                   \
 820 }                                                                        \
 821 static inline struct qinst *                                             \
 822 vir_##name##_dest(struct v3d_compile *c, struct qreg dest,               \
 823                   struct qreg a)                                         \
 824 {                                                                        \
 825         return vir_emit_nondef(c, vir_inst(op, dest, a,          \
 826                                            c->undef));                   \
 827 }
 828
 829 #define VIR_ALU2(name, vir_inst, op)                                       \
 830 static inline struct qreg                                                \
 831 vir_##name(struct v3d_compile *c, struct qreg a, struct qreg b)          \
 832 {                                                                        \
 833         return vir_emit_def(c, vir_inst(op, c->undef, a, b));    \
 834 }                                                                        \
 835 static inline struct qinst *                                             \
 836 vir_##name##_dest(struct v3d_compile *c, struct qreg dest,               \
 837                   struct qreg a, struct qreg b)                          \
 838 {                                                                        \
 839         return vir_emit_nondef(c, vir_inst(op, dest, a, b));     \
 840 }
 841
 842 #define VIR_NODST_0(name, vir_inst, op)                                 \
 843 static inline struct qinst *                                            \
 844 vir_##name(struct v3d_compile *c)                                       \
 845 {                                                                       \
 846         return vir_emit_nondef(c, vir_inst(op, c->undef,                \
 847                                            c->undef, c->undef));        \
 848 }
 849
 850 #define VIR_NODST_1(name, vir_inst, op)                                               \
 851 static inline struct qinst *                                            \
 852 vir_##name(struct v3d_compile *c, struct qreg a)                        \
 853 {                                                                       \
 854         return vir_emit_nondef(c, vir_inst(op, c->undef,        \
 855                                            a, c->undef));               \
 856 }
 857
 858 #define VIR_NODST_2(name, vir_inst, op)                                               \
 859 static inline struct qinst *                                            \
 860 vir_##name(struct v3d_compile *c, struct qreg a, struct qreg b)         \
 861 {                                                                       \
 862         return vir_emit_nondef(c, vir_inst(op, c->undef,                \
 863                                            a, b));                      \
 864 }
 865
 866 #define VIR_A_ALU2(name) VIR_ALU2(name, vir_add_inst, V3D_QPU_A_##name)
 867 #define VIR_M_ALU2(name) VIR_ALU2(name, vir_mul_inst, V3D_QPU_M_##name)
 868 #define VIR_A_ALU1(name) VIR_ALU1(name, vir_add_inst, V3D_QPU_A_##name)
 869 #define VIR_M_ALU1(name) VIR_ALU1(name, vir_mul_inst, V3D_QPU_M_##name)
 870 #define VIR_A_ALU0(name) VIR_ALU0(name, vir_add_inst, V3D_QPU_A_##name)
 871 #define VIR_M_ALU0(name) VIR_ALU0(name, vir_mul_inst, V3D_QPU_M_##name)
 872 #define VIR_A_NODST_2(name) VIR_NODST_2(name, vir_add_inst, V3D_QPU_A_##name)
 873 #define VIR_M_NODST_2(name) VIR_NODST_2(name, vir_mul_inst, V3D_QPU_M_##name)
 874 #define VIR_A_NODST_1(name) VIR_NODST_1(name, vir_add_inst, V3D_QPU_A_##name)
 875 #define VIR_M_NODST_1(name) VIR_NODST_1(name, vir_mul_inst, V3D_QPU_M_##name)
 876 #define VIR_A_NODST_0(name) VIR_NODST_0(name, vir_add_inst, V3D_QPU_A_##name)
 877
 878 VIR_A_ALU2(FADD)
 879 VIR_A_ALU2(VFPACK)
 880 VIR_A_ALU2(FSUB)
 881 VIR_A_ALU2(FMIN)
 882 VIR_A_ALU2(FMAX)
 883
 884 VIR_A_ALU2(ADD)
 885 VIR_A_ALU2(SUB)
 886 VIR_A_ALU2(SHL)
 887 VIR_A_ALU2(SHR)
 888 VIR_A_ALU2(ASR)
 889 VIR_A_ALU2(ROR)
 890 VIR_A_ALU2(MIN)
 891 VIR_A_ALU2(MAX)
 892 VIR_A_ALU2(UMIN)
 893 VIR_A_ALU2(UMAX)
 894 VIR_A_ALU2(AND)
 895 VIR_A_ALU2(OR)
 896 VIR_A_ALU2(XOR)
 897 VIR_A_ALU2(VADD)
 898 VIR_A_ALU2(VSUB)
 899 VIR_A_ALU2(STVPMV)
 900 VIR_A_ALU1(NOT)
 901 VIR_A_ALU1(NEG)
 902 VIR_A_ALU1(FLAPUSH)
 903 VIR_A_ALU1(FLBPUSH)
 904 VIR_A_ALU1(FLBPOP)
 905 VIR_A_ALU1(SETMSF)
 906 VIR_A_ALU1(SETREVF)
 907 VIR_A_ALU0(TIDX)
 908 VIR_A_ALU0(EIDX)
 909 VIR_A_ALU1(LDVPMV_IN)
 910 VIR_A_ALU1(LDVPMV_OUT)
 911
 912 VIR_A_ALU0(FXCD)
 913 VIR_A_ALU0(XCD)
 914 VIR_A_ALU0(FYCD)
 915 VIR_A_ALU0(YCD)
 916 VIR_A_ALU0(MSF)
 917 VIR_A_ALU0(REVF)
 918 VIR_A_NODST_1(VPMSETUP)
 919 VIR_A_NODST_0(VPMWT)
 920 VIR_A_ALU2(FCMP)
 921 VIR_A_ALU2(VFMAX)
 922
 923 VIR_A_ALU1(FROUND)
 924 VIR_A_ALU1(FTOIN)
 925 VIR_A_ALU1(FTRUNC)
 926 VIR_A_ALU1(FTOIZ)
 927 VIR_A_ALU1(FFLOOR)
 928 VIR_A_ALU1(FTOUZ)
 929 VIR_A_ALU1(FCEIL)
 930 VIR_A_ALU1(FTOC)
 931
 932 VIR_A_ALU1(FDX)
 933 VIR_A_ALU1(FDY)
 934
 935 VIR_A_ALU1(ITOF)
 936 VIR_A_ALU1(CLZ)
 937 VIR_A_ALU1(UTOF)
 938
 939 VIR_M_ALU2(UMUL24)
 940 VIR_M_ALU2(FMUL)
 941 VIR_M_ALU2(SMUL24)
 942 VIR_M_NODST_2(MULTOP)
 943
 944 VIR_M_ALU1(MOV)
 945 VIR_M_ALU1(FMOV)
 946
 947 static inline struct qinst *
 948 vir_MOV_cond(struct v3d_compile *c, enum v3d_qpu_cond cond,
 949              struct qreg dest, struct qreg src)
 950 {
 951         struct qinst *mov = vir_MOV_dest(c, dest, src);
 952         vir_set_cond(mov, cond);
 953         return mov;
 954 }
 955
 956 static inline struct qreg
 957 vir_SEL(struct v3d_compile *c, enum v3d_qpu_cond cond,
 958         struct qreg src0, struct qreg src1)
 959 {
 960         struct qreg t = vir_get_temp(c);
 961         vir_MOV_dest(c, t, src1);
 962         vir_MOV_cond(c, cond, t, src0);
 963         return t;
 964 }
 965
 966 static inline struct qinst *
 967 vir_NOP(struct v3d_compile *c)
 968 {
 969         return vir_emit_nondef(c, vir_add_inst(V3D_QPU_A_NOP,
 970                                                c->undef, c->undef, c->undef));
 971 }
 972
 973 static inline struct qreg
 974 vir_LDTMU(struct v3d_compile *c)
 975 {
 976         if (c->devinfo->ver >= 41) {
 977                 struct qinst *ldtmu = vir_add_inst(V3D_QPU_A_NOP, c->undef,
 978                                                    c->undef, c->undef);
 979                 ldtmu->qpu.sig.ldtmu = true;
 980
 981                 return vir_emit_def(c, ldtmu);
 982         } else {
 983                 vir_NOP(c)->qpu.sig.ldtmu = true;
 984                 return vir_MOV(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4));
 985         }
 986 }
 987
 988 static inline struct qreg
 989 vir_UMUL(struct v3d_compile *c, struct qreg src0, struct qreg src1)
 990 {
 991         vir_MULTOP(c, src0, src1);
 992         return vir_UMUL24(c, src0, src1);
 993 }
 994
 995 /*
 996 static inline struct qreg
 997 vir_LOAD_IMM(struct v3d_compile *c, uint32_t val)
 998 {
 999         return vir_emit_def(c, vir_inst(QOP_LOAD_IMM, c->undef,
1000                                         vir_reg(QFILE_LOAD_IMM, val), c->undef));
1001 }
1002
1003 static inline struct qreg
1004 vir_LOAD_IMM_U2(struct v3d_compile *c, uint32_t val)
1005 {
1006         return vir_emit_def(c, vir_inst(QOP_LOAD_IMM_U2, c->undef,
1007                                         vir_reg(QFILE_LOAD_IMM, val),
1008                                         c->undef));
1009 }
1010 static inline struct qreg
1011 vir_LOAD_IMM_I2(struct v3d_compile *c, uint32_t val)
1012 {
1013         return vir_emit_def(c, vir_inst(QOP_LOAD_IMM_I2, c->undef,
1014                                         vir_reg(QFILE_LOAD_IMM, val),
1015                                         c->undef));
1016 }
1017 */
1018
1019 static inline struct qinst *
1020 vir_BRANCH(struct v3d_compile *c, enum v3d_qpu_cond cond)
1021 {
1022         /* The actual uniform_data value will be set at scheduling time */
1023         return vir_emit_nondef(c, vir_branch_inst(cond, vir_uniform_ui(c, 0)));
1024 }
1025
1026 #define vir_for_each_block(block, c)                                    \
1027         list_for_each_entry(struct qblock, block, &c->blocks, link)
1028
1029 #define vir_for_each_block_rev(block, c)                                \
1030         list_for_each_entry_rev(struct qblock, block, &c->blocks, link)
1031
1032 /* Loop over the non-NULL members of the successors array. */
1033 #define vir_for_each_successor(succ, block)                             \
1034         for (struct qblock *succ = block->successors[0];                \
1035              succ != NULL;                                              \
1036              succ = (succ == block->successors[1] ? NULL :              \
1037                      block->successors[1]))
1038
1039 #define vir_for_each_inst(inst, block)                                  \
1040         list_for_each_entry(struct qinst, inst, &block->instructions, link)
1041
1042 #define vir_for_each_inst_rev(inst, block)                                  \
1043         list_for_each_entry_rev(struct qinst, inst, &block->instructions, link)
1044
1045 #define vir_for_each_inst_safe(inst, block)                             \
1046         list_for_each_entry_safe(struct qinst, inst, &block->instructions, link)
1047
1048 #define vir_for_each_inst_inorder(inst, c)                              \
1049         vir_for_each_block(_block, c)                                   \
1050                 vir_for_each_inst(inst, _block)
1051
1052 #endif /* V3D_COMPILER_H */