src/gallium/auxiliary/tgsi/tgsi_ureg.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009-2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE, INC AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 #include "pipe/p_screen.h"
  30 #include "pipe/p_context.h"
  31 #include "pipe/p_state.h"
  32 #include "tgsi/tgsi_ureg.h"
  33 #include "tgsi/tgsi_build.h"
  34 #include "tgsi/tgsi_info.h"
  35 #include "tgsi/tgsi_dump.h"
  36 #include "tgsi/tgsi_sanity.h"
  37 #include "util/u_debug.h"
  38 #include "util/u_memory.h"
  39 #include "util/u_math.h"
  40 #include "util/u_bitmask.h"
  41
  42 union tgsi_any_token {
  43    struct tgsi_header header;
  44    struct tgsi_processor processor;
  45    struct tgsi_token token;
  46    struct tgsi_property prop;
  47    struct tgsi_property_data prop_data;
  48    struct tgsi_declaration decl;
  49    struct tgsi_declaration_range decl_range;
  50    struct tgsi_declaration_dimension decl_dim;
  51    struct tgsi_declaration_interp decl_interp;
  52    struct tgsi_declaration_semantic decl_semantic;
  53    struct tgsi_declaration_sampler_view decl_sampler_view;
  54    struct tgsi_declaration_array array;
  55    struct tgsi_immediate imm;
  56    union  tgsi_immediate_data imm_data;
  57    struct tgsi_instruction insn;
  58    struct tgsi_instruction_predicate insn_predicate;
  59    struct tgsi_instruction_label insn_label;
  60    struct tgsi_instruction_texture insn_texture;
  61    struct tgsi_texture_offset insn_texture_offset;
  62    struct tgsi_src_register src;
  63    struct tgsi_ind_register ind;
  64    struct tgsi_dimension dim;
  65    struct tgsi_dst_register dst;
  66    unsigned value;
  67 };
  68
  69
  70 struct ureg_tokens {
  71    union tgsi_any_token *tokens;
  72    unsigned size;
  73    unsigned order;
  74    unsigned count;
  75 };
  76
  77 #define UREG_MAX_INPUT PIPE_MAX_SHADER_INPUTS
  78 #define UREG_MAX_SYSTEM_VALUE PIPE_MAX_ATTRIBS
  79 #define UREG_MAX_OUTPUT PIPE_MAX_SHADER_OUTPUTS
  80 #define UREG_MAX_CONSTANT_RANGE 32
  81 #define UREG_MAX_IMMEDIATE 4096
  82 #define UREG_MAX_ADDR 3
  83 #define UREG_MAX_PRED 1
  84 #define UREG_MAX_ARRAY_TEMPS 256
  85
  86 struct const_decl {
  87    struct {
  88       unsigned first;
  89       unsigned last;
  90    } constant_range[UREG_MAX_CONSTANT_RANGE];
  91    unsigned nr_constant_ranges;
  92 };
  93
  94 #define DOMAIN_DECL 0
  95 #define DOMAIN_INSN 1
  96
  97 struct ureg_program
  98 {
  99    unsigned processor;
 100    bool supports_any_inout_decl_range;
 101
 102    struct {
 103       unsigned semantic_name;
 104       unsigned semantic_index;
 105       unsigned interp;
 106       unsigned char cylindrical_wrap;
 107       unsigned interp_location;
 108       unsigned first;
 109       unsigned last;
 110       unsigned array_id;
 111    } input[UREG_MAX_INPUT];
 112    unsigned nr_inputs, nr_input_regs;
 113
 114    unsigned vs_inputs[PIPE_MAX_ATTRIBS/32];
 115
 116    struct {
 117       unsigned index;
 118       unsigned semantic_name;
 119       unsigned semantic_index;
 120    } system_value[UREG_MAX_SYSTEM_VALUE];
 121    unsigned nr_system_values;
 122
 123    struct {
 124       unsigned semantic_name;
 125       unsigned semantic_index;
 126       unsigned usage_mask; /* = TGSI_WRITEMASK_* */
 127       unsigned first;
 128       unsigned last;
 129       unsigned array_id;
 130    } output[UREG_MAX_OUTPUT];
 131    unsigned nr_outputs, nr_output_regs;
 132
 133    struct {
 134       union {
 135          float f[4];
 136          unsigned u[4];
 137          int i[4];
 138       } value;
 139       unsigned nr;
 140       unsigned type;
 141    } immediate[UREG_MAX_IMMEDIATE];
 142    unsigned nr_immediates;
 143
 144    struct ureg_src sampler[PIPE_MAX_SAMPLERS];
 145    unsigned nr_samplers;
 146
 147    struct {
 148       unsigned index;
 149       unsigned target;
 150       unsigned return_type_x;
 151       unsigned return_type_y;
 152       unsigned return_type_z;
 153       unsigned return_type_w;
 154    } sampler_view[PIPE_MAX_SHADER_SAMPLER_VIEWS];
 155    unsigned nr_sampler_views;
 156
 157    struct util_bitmask *free_temps;
 158    struct util_bitmask *local_temps;
 159    struct util_bitmask *decl_temps;
 160    unsigned nr_temps;
 161
 162    unsigned array_temps[UREG_MAX_ARRAY_TEMPS];
 163    unsigned nr_array_temps;
 164
 165    struct const_decl const_decls;
 166    struct const_decl const_decls2D[PIPE_MAX_CONSTANT_BUFFERS];
 167
 168    unsigned properties[TGSI_PROPERTY_COUNT];
 169
 170    unsigned nr_addrs;
 171    unsigned nr_preds;
 172    unsigned nr_instructions;
 173
 174    struct ureg_tokens domain[2];
 175 };
 176
 177 static union tgsi_any_token error_tokens[32];
 178
 179 static void tokens_error( struct ureg_tokens *tokens )
 180 {
 181    if (tokens->tokens && tokens->tokens != error_tokens)
 182       FREE(tokens->tokens);
 183
 184    tokens->tokens = error_tokens;
 185    tokens->size = Elements(error_tokens);
 186    tokens->count = 0;
 187 }
 188
 189
 190 static void tokens_expand( struct ureg_tokens *tokens,
 191                            unsigned count )
 192 {
 193    unsigned old_size = tokens->size * sizeof(unsigned);
 194
 195    if (tokens->tokens == error_tokens) {
 196       return;
 197    }
 198
 199    while (tokens->count + count > tokens->size) {
 200       tokens->size = (1 << ++tokens->order);
 201    }
 202
 203    tokens->tokens = REALLOC(tokens->tokens,
 204                             old_size,
 205                             tokens->size * sizeof(unsigned));
 206    if (tokens->tokens == NULL) {
 207       tokens_error(tokens);
 208    }
 209 }
 210
 211 static void set_bad( struct ureg_program *ureg )
 212 {
 213    tokens_error(&ureg->domain[0]);
 214 }
 215
 216
 217
 218 static union tgsi_any_token *get_tokens( struct ureg_program *ureg,
 219                                          unsigned domain,
 220                                          unsigned count )
 221 {
 222    struct ureg_tokens *tokens = &ureg->domain[domain];
 223    union tgsi_any_token *result;
 224
 225    if (tokens->count + count > tokens->size)
 226       tokens_expand(tokens, count);
 227
 228    result = &tokens->tokens[tokens->count];
 229    tokens->count += count;
 230    return result;
 231 }
 232
 233
 234 static union tgsi_any_token *retrieve_token( struct ureg_program *ureg,
 235                                             unsigned domain,
 236                                             unsigned nr )
 237 {
 238    if (ureg->domain[domain].tokens == error_tokens)
 239       return &error_tokens[0];
 240
 241    return &ureg->domain[domain].tokens[nr];
 242 }
 243
 244 void
 245 ureg_property(struct ureg_program *ureg, unsigned name, unsigned value)
 246 {
 247    assert(name < Elements(ureg->properties));
 248    ureg->properties[name] = value;
 249 }
 250
 251 struct ureg_src
 252 ureg_DECL_fs_input_cyl_centroid(struct ureg_program *ureg,
 253                        unsigned semantic_name,
 254                        unsigned semantic_index,
 255                        unsigned interp_mode,
 256                        unsigned cylindrical_wrap,
 257                        unsigned interp_location,
 258                        unsigned array_id,
 259                        unsigned array_size)
 260 {
 261    unsigned i;
 262
 263    for (i = 0; i < ureg->nr_inputs; i++) {
 264       if (ureg->input[i].semantic_name == semantic_name &&
 265           ureg->input[i].semantic_index == semantic_index) {
 266          assert(ureg->input[i].interp == interp_mode);
 267          assert(ureg->input[i].cylindrical_wrap == cylindrical_wrap);
 268          assert(ureg->input[i].interp_location == interp_location);
 269          assert(ureg->input[i].array_id == array_id);
 270          goto out;
 271       }
 272    }
 273
 274    if (ureg->nr_inputs < UREG_MAX_INPUT) {
 275       assert(array_size >= 1);
 276       ureg->input[i].semantic_name = semantic_name;
 277       ureg->input[i].semantic_index = semantic_index;
 278       ureg->input[i].interp = interp_mode;
 279       ureg->input[i].cylindrical_wrap = cylindrical_wrap;
 280       ureg->input[i].interp_location = interp_location;
 281       ureg->input[i].first = ureg->nr_input_regs;
 282       ureg->input[i].last = ureg->nr_input_regs + array_size - 1;
 283       ureg->input[i].array_id = array_id;
 284       ureg->nr_input_regs += array_size;
 285       ureg->nr_inputs++;
 286    } else {
 287       set_bad(ureg);
 288    }
 289
 290 out:
 291    return ureg_src_array_register(TGSI_FILE_INPUT, ureg->input[i].first,
 292                                   array_id);
 293 }
 294
 295
 296 struct ureg_src
 297 ureg_DECL_vs_input( struct ureg_program *ureg,
 298                     unsigned index )
 299 {
 300    assert(ureg->processor == TGSI_PROCESSOR_VERTEX);
 301    assert(index / 32 < ARRAY_SIZE(ureg->vs_inputs));
 302
 303    ureg->vs_inputs[index/32] |= 1 << (index % 32);
 304    return ureg_src_register( TGSI_FILE_INPUT, index );
 305 }
 306
 307
 308 struct ureg_src
 309 ureg_DECL_input(struct ureg_program *ureg,
 310                 unsigned semantic_name,
 311                 unsigned semantic_index,
 312                 unsigned array_id,
 313                 unsigned array_size)
 314 {
 315    return ureg_DECL_fs_input_cyl_centroid(ureg, semantic_name, semantic_index,
 316                                           0, 0, 0, array_id, array_size);
 317 }
 318
 319
 320 struct ureg_src
 321 ureg_DECL_system_value(struct ureg_program *ureg,
 322                        unsigned index,
 323                        unsigned semantic_name,
 324                        unsigned semantic_index)
 325 {
 326    if (ureg->nr_system_values < UREG_MAX_SYSTEM_VALUE) {
 327       ureg->system_value[ureg->nr_system_values].index = index;
 328       ureg->system_value[ureg->nr_system_values].semantic_name = semantic_name;
 329       ureg->system_value[ureg->nr_system_values].semantic_index = semantic_index;
 330       ureg->nr_system_values++;
 331    } else {
 332       set_bad(ureg);
 333    }
 334
 335    return ureg_src_register(TGSI_FILE_SYSTEM_VALUE, index);
 336 }
 337
 338
 339 struct ureg_dst
 340 ureg_DECL_output_masked(struct ureg_program *ureg,
 341                         unsigned name,
 342                         unsigned index,
 343                         unsigned usage_mask,
 344                         unsigned array_id,
 345                         unsigned array_size)
 346 {
 347    unsigned i;
 348
 349    assert(usage_mask != 0);
 350
 351    for (i = 0; i < ureg->nr_outputs; i++) {
 352       if (ureg->output[i].semantic_name == name &&
 353           ureg->output[i].semantic_index == index) {
 354          assert(ureg->output[i].array_id == array_id);
 355          ureg->output[i].usage_mask |= usage_mask;
 356          goto out;
 357       }
 358    }
 359
 360    if (ureg->nr_outputs < UREG_MAX_OUTPUT) {
 361       ureg->output[i].semantic_name = name;
 362       ureg->output[i].semantic_index = index;
 363       ureg->output[i].usage_mask = usage_mask;
 364       ureg->output[i].first = ureg->nr_output_regs;
 365       ureg->output[i].last = ureg->nr_output_regs + array_size - 1;
 366       ureg->output[i].array_id = array_id;
 367       ureg->nr_output_regs += array_size;
 368       ureg->nr_outputs++;
 369    }
 370    else {
 371       set_bad( ureg );
 372    }
 373
 374 out:
 375    return ureg_dst_array_register(TGSI_FILE_OUTPUT, ureg->output[i].first,
 376                                   array_id);
 377 }
 378
 379
 380 struct ureg_dst
 381 ureg_DECL_output(struct ureg_program *ureg,
 382                  unsigned name,
 383                  unsigned index)
 384 {
 385    return ureg_DECL_output_masked(ureg, name, index, TGSI_WRITEMASK_XYZW,
 386                                   0, 1);
 387 }
 388
 389 struct ureg_dst
 390 ureg_DECL_output_array(struct ureg_program *ureg,
 391                        unsigned semantic_name,
 392                        unsigned semantic_index,
 393                        unsigned array_id,
 394                        unsigned array_size)
 395 {
 396    return ureg_DECL_output_masked(ureg, semantic_name, semantic_index,
 397                                   TGSI_WRITEMASK_XYZW,
 398                                   array_id, array_size);
 399 }
 400
 401
 402 /* Returns a new constant register.  Keep track of which have been
 403  * referred to so that we can emit decls later.
 404  *
 405  * Constant operands declared with this function must be addressed
 406  * with a two-dimensional index.
 407  *
 408  * There is nothing in this code to bind this constant to any tracked
 409  * value or manage any constant_buffer contents -- that's the
 410  * resposibility of the calling code.
 411  */
 412 void
 413 ureg_DECL_constant2D(struct ureg_program *ureg,
 414                      unsigned first,
 415                      unsigned last,
 416                      unsigned index2D)
 417 {
 418    struct const_decl *decl = &ureg->const_decls2D[index2D];
 419
 420    assert(index2D < PIPE_MAX_CONSTANT_BUFFERS);
 421
 422    if (decl->nr_constant_ranges < UREG_MAX_CONSTANT_RANGE) {
 423       uint i = decl->nr_constant_ranges++;
 424
 425       decl->constant_range[i].first = first;
 426       decl->constant_range[i].last = last;
 427    }
 428 }
 429
 430
 431 /* A one-dimensional, depricated version of ureg_DECL_constant2D().
 432  *
 433  * Constant operands declared with this function must be addressed
 434  * with a one-dimensional index.
 435  */
 436 struct ureg_src
 437 ureg_DECL_constant(struct ureg_program *ureg,
 438                    unsigned index)
 439 {
 440    struct const_decl *decl = &ureg->const_decls;
 441    unsigned minconst = index, maxconst = index;
 442    unsigned i;
 443
 444    /* Inside existing range?
 445     */
 446    for (i = 0; i < decl->nr_constant_ranges; i++) {
 447       if (decl->constant_range[i].first <= index &&
 448           decl->constant_range[i].last >= index) {
 449          goto out;
 450       }
 451    }
 452
 453    /* Extend existing range?
 454     */
 455    for (i = 0; i < decl->nr_constant_ranges; i++) {
 456       if (decl->constant_range[i].last == index - 1) {
 457          decl->constant_range[i].last = index;
 458          goto out;
 459       }
 460
 461       if (decl->constant_range[i].first == index + 1) {
 462          decl->constant_range[i].first = index;
 463          goto out;
 464       }
 465
 466       minconst = MIN2(minconst, decl->constant_range[i].first);
 467       maxconst = MAX2(maxconst, decl->constant_range[i].last);
 468    }
 469
 470    /* Create new range?
 471     */
 472    if (decl->nr_constant_ranges < UREG_MAX_CONSTANT_RANGE) {
 473       i = decl->nr_constant_ranges++;
 474       decl->constant_range[i].first = index;
 475       decl->constant_range[i].last = index;
 476       goto out;
 477    }
 478
 479    /* Collapse all ranges down to one:
 480     */
 481    i = 0;
 482    decl->constant_range[0].first = minconst;
 483    decl->constant_range[0].last = maxconst;
 484    decl->nr_constant_ranges = 1;
 485
 486 out:
 487    assert(i < decl->nr_constant_ranges);
 488    assert(decl->constant_range[i].first <= index);
 489    assert(decl->constant_range[i].last >= index);
 490    return ureg_src_register(TGSI_FILE_CONSTANT, index);
 491 }
 492
 493 static struct ureg_dst alloc_temporary( struct ureg_program *ureg,
 494                                         boolean local )
 495 {
 496    unsigned i;
 497
 498    /* Look for a released temporary.
 499     */
 500    for (i = util_bitmask_get_first_index(ureg->free_temps);
 501         i != UTIL_BITMASK_INVALID_INDEX;
 502         i = util_bitmask_get_next_index(ureg->free_temps, i + 1)) {
 503       if (util_bitmask_get(ureg->local_temps, i) == local)
 504          break;
 505    }
 506
 507    /* Or allocate a new one.
 508     */
 509    if (i == UTIL_BITMASK_INVALID_INDEX) {
 510       i = ureg->nr_temps++;
 511
 512       if (local)
 513          util_bitmask_set(ureg->local_temps, i);
 514
 515       /* Start a new declaration when the local flag changes */
 516       if (!i || util_bitmask_get(ureg->local_temps, i - 1) != local)
 517          util_bitmask_set(ureg->decl_temps, i);
 518    }
 519
 520    util_bitmask_clear(ureg->free_temps, i);
 521
 522    return ureg_dst_register( TGSI_FILE_TEMPORARY, i );
 523 }
 524
 525 struct ureg_dst ureg_DECL_temporary( struct ureg_program *ureg )
 526 {
 527    return alloc_temporary(ureg, FALSE);
 528 }
 529
 530 struct ureg_dst ureg_DECL_local_temporary( struct ureg_program *ureg )
 531 {
 532    return alloc_temporary(ureg, TRUE);
 533 }
 534
 535 struct ureg_dst ureg_DECL_array_temporary( struct ureg_program *ureg,
 536                                            unsigned size,
 537                                            boolean local )
 538 {
 539    unsigned i = ureg->nr_temps;
 540    struct ureg_dst dst = ureg_dst_register( TGSI_FILE_TEMPORARY, i );
 541
 542    if (local)
 543       util_bitmask_set(ureg->local_temps, i);
 544
 545    /* Always start a new declaration at the start */
 546    util_bitmask_set(ureg->decl_temps, i);
 547
 548    ureg->nr_temps += size;
 549
 550    /* and also at the end of the array */
 551    util_bitmask_set(ureg->decl_temps, ureg->nr_temps);
 552
 553    if (ureg->nr_array_temps < UREG_MAX_ARRAY_TEMPS) {
 554       ureg->array_temps[ureg->nr_array_temps++] = i;
 555       dst.ArrayID = ureg->nr_array_temps;
 556    }
 557
 558    return dst;
 559 }
 560
 561 void ureg_release_temporary( struct ureg_program *ureg,
 562                              struct ureg_dst tmp )
 563 {
 564    if(tmp.File == TGSI_FILE_TEMPORARY)
 565       util_bitmask_set(ureg->free_temps, tmp.Index);
 566 }
 567
 568
 569 /* Allocate a new address register.
 570  */
 571 struct ureg_dst ureg_DECL_address( struct ureg_program *ureg )
 572 {
 573    if (ureg->nr_addrs < UREG_MAX_ADDR)
 574       return ureg_dst_register( TGSI_FILE_ADDRESS, ureg->nr_addrs++ );
 575
 576    assert( 0 );
 577    return ureg_dst_register( TGSI_FILE_ADDRESS, 0 );
 578 }
 579
 580 /* Allocate a new predicate register.
 581  */
 582 struct ureg_dst
 583 ureg_DECL_predicate(struct ureg_program *ureg)
 584 {
 585    if (ureg->nr_preds < UREG_MAX_PRED) {
 586       return ureg_dst_register(TGSI_FILE_PREDICATE, ureg->nr_preds++);
 587    }
 588
 589    assert(0);
 590    return ureg_dst_register(TGSI_FILE_PREDICATE, 0);
 591 }
 592
 593 /* Allocate a new sampler.
 594  */
 595 struct ureg_src ureg_DECL_sampler( struct ureg_program *ureg,
 596                                    unsigned nr )
 597 {
 598    unsigned i;
 599
 600    for (i = 0; i < ureg->nr_samplers; i++)
 601       if (ureg->sampler[i].Index == nr)
 602          return ureg->sampler[i];
 603
 604    if (i < PIPE_MAX_SAMPLERS) {
 605       ureg->sampler[i] = ureg_src_register( TGSI_FILE_SAMPLER, nr );
 606       ureg->nr_samplers++;
 607       return ureg->sampler[i];
 608    }
 609
 610    assert( 0 );
 611    return ureg->sampler[0];
 612 }
 613
 614 /*
 615  * Allocate a new shader sampler view.
 616  */
 617 struct ureg_src
 618 ureg_DECL_sampler_view(struct ureg_program *ureg,
 619                        unsigned index,
 620                        unsigned target,
 621                        unsigned return_type_x,
 622                        unsigned return_type_y,
 623                        unsigned return_type_z,
 624                        unsigned return_type_w)
 625 {
 626    struct ureg_src reg = ureg_src_register(TGSI_FILE_SAMPLER_VIEW, index);
 627    uint i;
 628
 629    for (i = 0; i < ureg->nr_sampler_views; i++) {
 630       if (ureg->sampler_view[i].index == index) {
 631          return reg;
 632       }
 633    }
 634
 635    if (i < PIPE_MAX_SHADER_SAMPLER_VIEWS) {
 636       ureg->sampler_view[i].index = index;
 637       ureg->sampler_view[i].target = target;
 638       ureg->sampler_view[i].return_type_x = return_type_x;
 639       ureg->sampler_view[i].return_type_y = return_type_y;
 640       ureg->sampler_view[i].return_type_z = return_type_z;
 641       ureg->sampler_view[i].return_type_w = return_type_w;
 642       ureg->nr_sampler_views++;
 643       return reg;
 644    }
 645
 646    assert(0);
 647    return reg;
 648 }
 649
 650 static int
 651 match_or_expand_immediate64( const unsigned *v,
 652                              int type,
 653                              unsigned nr,
 654                              unsigned *v2,
 655                              unsigned *pnr2,
 656                              unsigned *swizzle )
 657 {
 658    unsigned nr2 = *pnr2;
 659    unsigned i, j;
 660    *swizzle = 0;
 661
 662    for (i = 0; i < nr; i += 2) {
 663       boolean found = FALSE;
 664
 665       for (j = 0; j < nr2 && !found; j += 2) {
 666          if (v[i] == v2[j] && v[i + 1] == v2[j + 1]) {
 667             *swizzle |= (j << (i * 2)) | ((j + 1) << ((i + 1) * 2));
 668             found = TRUE;
 669          }
 670       }
 671       if (!found) {
 672          if ((nr2) >= 4) {
 673             return FALSE;
 674          }
 675
 676          v2[nr2] = v[i];
 677          v2[nr2 + 1] = v[i + 1];
 678
 679          *swizzle |= (nr2 << (i * 2)) | ((nr2 + 1) << ((i + 1) * 2));
 680          nr2 += 2;
 681       }
 682    }
 683
 684    /* Actually expand immediate only when fully succeeded.
 685     */
 686    *pnr2 = nr2;
 687    return TRUE;
 688 }
 689
 690 static int
 691 match_or_expand_immediate( const unsigned *v,
 692                            int type,
 693                            unsigned nr,
 694                            unsigned *v2,
 695                            unsigned *pnr2,
 696                            unsigned *swizzle )
 697 {
 698    unsigned nr2 = *pnr2;
 699    unsigned i, j;
 700
 701    if (type == TGSI_IMM_FLOAT64)
 702       return match_or_expand_immediate64(v, type, nr, v2, pnr2, swizzle);
 703
 704    *swizzle = 0;
 705
 706    for (i = 0; i < nr; i++) {
 707       boolean found = FALSE;
 708
 709       for (j = 0; j < nr2 && !found; j++) {
 710          if (v[i] == v2[j]) {
 711             *swizzle |= j << (i * 2);
 712             found = TRUE;
 713          }
 714       }
 715
 716       if (!found) {
 717          if (nr2 >= 4) {
 718             return FALSE;
 719          }
 720
 721          v2[nr2] = v[i];
 722          *swizzle |= nr2 << (i * 2);
 723          nr2++;
 724       }
 725    }
 726
 727    /* Actually expand immediate only when fully succeeded.
 728     */
 729    *pnr2 = nr2;
 730    return TRUE;
 731 }
 732
 733
 734 static struct ureg_src
 735 decl_immediate( struct ureg_program *ureg,
 736                 const unsigned *v,
 737                 unsigned nr,
 738                 unsigned type )
 739 {
 740    unsigned i, j;
 741    unsigned swizzle = 0;
 742
 743    /* Could do a first pass where we examine all existing immediates
 744     * without expanding.
 745     */
 746
 747    for (i = 0; i < ureg->nr_immediates; i++) {
 748       if (ureg->immediate[i].type != type) {
 749          continue;
 750       }
 751       if (match_or_expand_immediate(v,
 752                                     type,
 753                                     nr,
 754                                     ureg->immediate[i].value.u,
 755                                     &ureg->immediate[i].nr,
 756                                     &swizzle)) {
 757          goto out;
 758       }
 759    }
 760
 761    if (ureg->nr_immediates < UREG_MAX_IMMEDIATE) {
 762       i = ureg->nr_immediates++;
 763       ureg->immediate[i].type = type;
 764       if (match_or_expand_immediate(v,
 765                                     type,
 766                                     nr,
 767                                     ureg->immediate[i].value.u,
 768                                     &ureg->immediate[i].nr,
 769                                     &swizzle)) {
 770          goto out;
 771       }
 772    }
 773
 774    set_bad(ureg);
 775
 776 out:
 777    /* Make sure that all referenced elements are from this immediate.
 778     * Has the effect of making size-one immediates into scalars.
 779     */
 780    if (type == TGSI_IMM_FLOAT64) {
 781       for (j = nr; j < 4; j+=2) {
 782          swizzle |= (swizzle & 0xf) << (j * 2);
 783       }
 784    } else {
 785       for (j = nr; j < 4; j++) {
 786          swizzle |= (swizzle & 0x3) << (j * 2);
 787       }
 788    }
 789    return ureg_swizzle(ureg_src_register(TGSI_FILE_IMMEDIATE, i),
 790                        (swizzle >> 0) & 0x3,
 791                        (swizzle >> 2) & 0x3,
 792                        (swizzle >> 4) & 0x3,
 793                        (swizzle >> 6) & 0x3);
 794 }
 795
 796
 797 struct ureg_src
 798 ureg_DECL_immediate( struct ureg_program *ureg,
 799                      const float *v,
 800                      unsigned nr )
 801 {
 802    union {
 803       float f[4];
 804       unsigned u[4];
 805    } fu;
 806    unsigned int i;
 807
 808    for (i = 0; i < nr; i++) {
 809       fu.f[i] = v[i];
 810    }
 811
 812    return decl_immediate(ureg, fu.u, nr, TGSI_IMM_FLOAT32);
 813 }
 814
 815 struct ureg_src
 816 ureg_DECL_immediate_f64( struct ureg_program *ureg,
 817                          const double *v,
 818                          unsigned nr )
 819 {
 820    union {
 821       unsigned u[4];
 822       double d[2];
 823    } fu;
 824    unsigned int i;
 825
 826    assert((nr / 2) < 3);
 827    for (i = 0; i < nr / 2; i++) {
 828       fu.d[i] = v[i];
 829    }
 830
 831    return decl_immediate(ureg, fu.u, nr, TGSI_IMM_FLOAT64);
 832 }
 833
 834 struct ureg_src
 835 ureg_DECL_immediate_uint( struct ureg_program *ureg,
 836                           const unsigned *v,
 837                           unsigned nr )
 838 {
 839    return decl_immediate(ureg, v, nr, TGSI_IMM_UINT32);
 840 }
 841
 842
 843 struct ureg_src
 844 ureg_DECL_immediate_block_uint( struct ureg_program *ureg,
 845                                 const unsigned *v,
 846                                 unsigned nr )
 847 {
 848    uint index;
 849    uint i;
 850
 851    if (ureg->nr_immediates + (nr + 3) / 4 > UREG_MAX_IMMEDIATE) {
 852       set_bad(ureg);
 853       return ureg_src_register(TGSI_FILE_IMMEDIATE, 0);
 854    }
 855
 856    index = ureg->nr_immediates;
 857    ureg->nr_immediates += (nr + 3) / 4;
 858
 859    for (i = index; i < ureg->nr_immediates; i++) {
 860       ureg->immediate[i].type = TGSI_IMM_UINT32;
 861       ureg->immediate[i].nr = nr > 4 ? 4 : nr;
 862       memcpy(ureg->immediate[i].value.u,
 863              &v[(i - index) * 4],
 864              ureg->immediate[i].nr * sizeof(uint));
 865       nr -= 4;
 866    }
 867
 868    return ureg_src_register(TGSI_FILE_IMMEDIATE, index);
 869 }
 870
 871
 872 struct ureg_src
 873 ureg_DECL_immediate_int( struct ureg_program *ureg,
 874                          const int *v,
 875                          unsigned nr )
 876 {
 877    return decl_immediate(ureg, (const unsigned *)v, nr, TGSI_IMM_INT32);
 878 }
 879
 880
 881 void
 882 ureg_emit_src( struct ureg_program *ureg,
 883                struct ureg_src src )
 884 {
 885    unsigned size = 1 + (src.Indirect ? 1 : 0) +
 886                    (src.Dimension ? (src.DimIndirect ? 2 : 1) : 0);
 887
 888    union tgsi_any_token *out = get_tokens( ureg, DOMAIN_INSN, size );
 889    unsigned n = 0;
 890
 891    assert(src.File != TGSI_FILE_NULL);
 892    assert(src.File < TGSI_FILE_COUNT);
 893
 894    out[n].value = 0;
 895    out[n].src.File = src.File;
 896    out[n].src.SwizzleX = src.SwizzleX;
 897    out[n].src.SwizzleY = src.SwizzleY;
 898    out[n].src.SwizzleZ = src.SwizzleZ;
 899    out[n].src.SwizzleW = src.SwizzleW;
 900    out[n].src.Index = src.Index;
 901    out[n].src.Negate = src.Negate;
 902    out[0].src.Absolute = src.Absolute;
 903    n++;
 904
 905    if (src.Indirect) {
 906       out[0].src.Indirect = 1;
 907       out[n].value = 0;
 908       out[n].ind.File = src.IndirectFile;
 909       out[n].ind.Swizzle = src.IndirectSwizzle;
 910       out[n].ind.Index = src.IndirectIndex;
 911       if (!ureg->supports_any_inout_decl_range &&
 912           (src.File == TGSI_FILE_INPUT || src.File == TGSI_FILE_OUTPUT))
 913          out[n].ind.ArrayID = 0;
 914       else
 915          out[n].ind.ArrayID = src.ArrayID;
 916       n++;
 917    }
 918
 919    if (src.Dimension) {
 920       out[0].src.Dimension = 1;
 921       out[n].dim.Dimension = 0;
 922       out[n].dim.Padding = 0;
 923       if (src.DimIndirect) {
 924          out[n].dim.Indirect = 1;
 925          out[n].dim.Index = src.DimensionIndex;
 926          n++;
 927          out[n].value = 0;
 928          out[n].ind.File = src.DimIndFile;
 929          out[n].ind.Swizzle = src.DimIndSwizzle;
 930          out[n].ind.Index = src.DimIndIndex;
 931          if (!ureg->supports_any_inout_decl_range &&
 932              (src.File == TGSI_FILE_INPUT || src.File == TGSI_FILE_OUTPUT))
 933             out[n].ind.ArrayID = 0;
 934          else
 935             out[n].ind.ArrayID = src.ArrayID;
 936       } else {
 937          out[n].dim.Indirect = 0;
 938          out[n].dim.Index = src.DimensionIndex;
 939       }
 940       n++;
 941    }
 942
 943    assert(n == size);
 944 }
 945
 946
 947 void
 948 ureg_emit_dst( struct ureg_program *ureg,
 949                struct ureg_dst dst )
 950 {
 951    unsigned size = 1 + (dst.Indirect ? 1 : 0) +
 952                    (dst.Dimension ? (dst.DimIndirect ? 2 : 1) : 0);
 953
 954    union tgsi_any_token *out = get_tokens( ureg, DOMAIN_INSN, size );
 955    unsigned n = 0;
 956
 957    assert(dst.File != TGSI_FILE_NULL);
 958    assert(dst.File != TGSI_FILE_CONSTANT);
 959    assert(dst.File != TGSI_FILE_INPUT);
 960    assert(dst.File != TGSI_FILE_SAMPLER);
 961    assert(dst.File != TGSI_FILE_SAMPLER_VIEW);
 962    assert(dst.File != TGSI_FILE_IMMEDIATE);
 963    assert(dst.File < TGSI_FILE_COUNT);
 964
 965    out[n].value = 0;
 966    out[n].dst.File = dst.File;
 967    out[n].dst.WriteMask = dst.WriteMask;
 968    out[n].dst.Indirect = dst.Indirect;
 969    out[n].dst.Index = dst.Index;
 970    n++;
 971
 972    if (dst.Indirect) {
 973       out[n].value = 0;
 974       out[n].ind.File = dst.IndirectFile;
 975       out[n].ind.Swizzle = dst.IndirectSwizzle;
 976       out[n].ind.Index = dst.IndirectIndex;
 977       if (!ureg->supports_any_inout_decl_range &&
 978           (dst.File == TGSI_FILE_INPUT || dst.File == TGSI_FILE_OUTPUT))
 979          out[n].ind.ArrayID = 0;
 980       else
 981          out[n].ind.ArrayID = dst.ArrayID;
 982       n++;
 983    }
 984
 985    if (dst.Dimension) {
 986       out[0].dst.Dimension = 1;
 987       out[n].dim.Dimension = 0;
 988       out[n].dim.Padding = 0;
 989       if (dst.DimIndirect) {
 990          out[n].dim.Indirect = 1;
 991          out[n].dim.Index = dst.DimensionIndex;
 992          n++;
 993          out[n].value = 0;
 994          out[n].ind.File = dst.DimIndFile;
 995          out[n].ind.Swizzle = dst.DimIndSwizzle;
 996          out[n].ind.Index = dst.DimIndIndex;
 997          if (!ureg->supports_any_inout_decl_range &&
 998              (dst.File == TGSI_FILE_INPUT || dst.File == TGSI_FILE_OUTPUT))
 999             out[n].ind.ArrayID = 0;
1000          else
1001             out[n].ind.ArrayID = dst.ArrayID;
1002       } else {
1003          out[n].dim.Indirect = 0;
1004          out[n].dim.Index = dst.DimensionIndex;
1005       }
1006       n++;
1007    }
1008
1009    assert(n == size);
1010 }
1011
1012
1013 static void validate( unsigned opcode,
1014                       unsigned nr_dst,
1015                       unsigned nr_src )
1016 {
1017 #ifdef DEBUG
1018    const struct tgsi_opcode_info *info = tgsi_get_opcode_info( opcode );
1019    assert(info);
1020    if(info) {
1021       assert(nr_dst == info->num_dst);
1022       assert(nr_src == info->num_src);
1023    }
1024 #endif
1025 }
1026
1027 struct ureg_emit_insn_result
1028 ureg_emit_insn(struct ureg_program *ureg,
1029                unsigned opcode,
1030                boolean saturate,
1031                boolean predicate,
1032                boolean pred_negate,
1033                unsigned pred_swizzle_x,
1034                unsigned pred_swizzle_y,
1035                unsigned pred_swizzle_z,
1036                unsigned pred_swizzle_w,
1037                unsigned num_dst,
1038                unsigned num_src )
1039 {
1040    union tgsi_any_token *out;
1041    uint count = predicate ? 2 : 1;
1042    struct ureg_emit_insn_result result;
1043
1044    validate( opcode, num_dst, num_src );
1045
1046    out = get_tokens( ureg, DOMAIN_INSN, count );
1047    out[0].insn = tgsi_default_instruction();
1048    out[0].insn.Opcode = opcode;
1049    out[0].insn.Saturate = saturate;
1050    out[0].insn.NumDstRegs = num_dst;
1051    out[0].insn.NumSrcRegs = num_src;
1052
1053    result.insn_token = ureg->domain[DOMAIN_INSN].count - count;
1054    result.extended_token = result.insn_token;
1055
1056    if (predicate) {
1057       out[0].insn.Predicate = 1;
1058       out[1].insn_predicate = tgsi_default_instruction_predicate();
1059       out[1].insn_predicate.Negate = pred_negate;
1060       out[1].insn_predicate.SwizzleX = pred_swizzle_x;
1061       out[1].insn_predicate.SwizzleY = pred_swizzle_y;
1062       out[1].insn_predicate.SwizzleZ = pred_swizzle_z;
1063       out[1].insn_predicate.SwizzleW = pred_swizzle_w;
1064    }
1065
1066    ureg->nr_instructions++;
1067
1068    return result;
1069 }
1070
1071
1072 /**
1073  * Emit a label token.
1074  * \param label_token returns a token number indicating where the label
1075  * needs to be patched later.  Later, this value should be passed to the
1076  * ureg_fixup_label() function.
1077  */
1078 void
1079 ureg_emit_label(struct ureg_program *ureg,
1080                 unsigned extended_token,
1081                 unsigned *label_token )
1082 {
1083    union tgsi_any_token *out, *insn;
1084
1085    if(!label_token)
1086       return;
1087
1088    out = get_tokens( ureg, DOMAIN_INSN, 1 );
1089    out[0].value = 0;
1090
1091    insn = retrieve_token( ureg, DOMAIN_INSN, extended_token );
1092    insn->insn.Label = 1;
1093
1094    *label_token = ureg->domain[DOMAIN_INSN].count - 1;
1095 }
1096
1097 /* Will return a number which can be used in a label to point to the
1098  * next instruction to be emitted.
1099  */
1100 unsigned
1101 ureg_get_instruction_number( struct ureg_program *ureg )
1102 {
1103    return ureg->nr_instructions;
1104 }
1105
1106 /* Patch a given label (expressed as a token number) to point to a
1107  * given instruction (expressed as an instruction number).
1108  */
1109 void
1110 ureg_fixup_label(struct ureg_program *ureg,
1111                  unsigned label_token,
1112                  unsigned instruction_number )
1113 {
1114    union tgsi_any_token *out = retrieve_token( ureg, DOMAIN_INSN, label_token );
1115
1116    out->insn_label.Label = instruction_number;
1117 }
1118
1119
1120 void
1121 ureg_emit_texture(struct ureg_program *ureg,
1122                   unsigned extended_token,
1123                   unsigned target, unsigned num_offsets)
1124 {
1125    union tgsi_any_token *out, *insn;
1126
1127    out = get_tokens( ureg, DOMAIN_INSN, 1 );
1128    insn = retrieve_token( ureg, DOMAIN_INSN, extended_token );
1129
1130    insn->insn.Texture = 1;
1131
1132    out[0].value = 0;
1133    out[0].insn_texture.Texture = target;
1134    out[0].insn_texture.NumOffsets = num_offsets;
1135 }
1136
1137 void
1138 ureg_emit_texture_offset(struct ureg_program *ureg,
1139                          const struct tgsi_texture_offset *offset)
1140 {
1141    union tgsi_any_token *out;
1142
1143    out = get_tokens( ureg, DOMAIN_INSN, 1);
1144
1145    out[0].value = 0;
1146    out[0].insn_texture_offset = *offset;
1147
1148 }
1149
1150
1151 void
1152 ureg_fixup_insn_size(struct ureg_program *ureg,
1153                      unsigned insn )
1154 {
1155    union tgsi_any_token *out = retrieve_token( ureg, DOMAIN_INSN, insn );
1156
1157    assert(out->insn.Type == TGSI_TOKEN_TYPE_INSTRUCTION);
1158    out->insn.NrTokens = ureg->domain[DOMAIN_INSN].count - insn - 1;
1159 }
1160
1161
1162 void
1163 ureg_insn(struct ureg_program *ureg,
1164           unsigned opcode,
1165           const struct ureg_dst *dst,
1166           unsigned nr_dst,
1167           const struct ureg_src *src,
1168           unsigned nr_src )
1169 {
1170    struct ureg_emit_insn_result insn;
1171    unsigned i;
1172    boolean saturate;
1173    boolean predicate;
1174    boolean negate = FALSE;
1175    unsigned swizzle[4] = { 0 };
1176
1177    if (nr_dst && ureg_dst_is_empty(dst[0])) {
1178       return;
1179    }
1180
1181    saturate = nr_dst ? dst[0].Saturate : FALSE;
1182    predicate = nr_dst ? dst[0].Predicate : FALSE;
1183    if (predicate) {
1184       negate = dst[0].PredNegate;
1185       swizzle[0] = dst[0].PredSwizzleX;
1186       swizzle[1] = dst[0].PredSwizzleY;
1187       swizzle[2] = dst[0].PredSwizzleZ;
1188       swizzle[3] = dst[0].PredSwizzleW;
1189    }
1190
1191    insn = ureg_emit_insn(ureg,
1192                          opcode,
1193                          saturate,
1194                          predicate,
1195                          negate,
1196                          swizzle[0],
1197                          swizzle[1],
1198                          swizzle[2],
1199                          swizzle[3],
1200                          nr_dst,
1201                          nr_src);
1202
1203    for (i = 0; i < nr_dst; i++)
1204       ureg_emit_dst( ureg, dst[i] );
1205
1206    for (i = 0; i < nr_src; i++)
1207       ureg_emit_src( ureg, src[i] );
1208
1209    ureg_fixup_insn_size( ureg, insn.insn_token );
1210 }
1211
1212 void
1213 ureg_tex_insn(struct ureg_program *ureg,
1214               unsigned opcode,
1215               const struct ureg_dst *dst,
1216               unsigned nr_dst,
1217               unsigned target,
1218               const struct tgsi_texture_offset *texoffsets,
1219               unsigned nr_offset,
1220               const struct ureg_src *src,
1221               unsigned nr_src )
1222 {
1223    struct ureg_emit_insn_result insn;
1224    unsigned i;
1225    boolean saturate;
1226    boolean predicate;
1227    boolean negate = FALSE;
1228    unsigned swizzle[4] = { 0 };
1229
1230    if (nr_dst && ureg_dst_is_empty(dst[0])) {
1231       return;
1232    }
1233
1234    saturate = nr_dst ? dst[0].Saturate : FALSE;
1235    predicate = nr_dst ? dst[0].Predicate : FALSE;
1236    if (predicate) {
1237       negate = dst[0].PredNegate;
1238       swizzle[0] = dst[0].PredSwizzleX;
1239       swizzle[1] = dst[0].PredSwizzleY;
1240       swizzle[2] = dst[0].PredSwizzleZ;
1241       swizzle[3] = dst[0].PredSwizzleW;
1242    }
1243
1244    insn = ureg_emit_insn(ureg,
1245                          opcode,
1246                          saturate,
1247                          predicate,
1248                          negate,
1249                          swizzle[0],
1250                          swizzle[1],
1251                          swizzle[2],
1252                          swizzle[3],
1253                          nr_dst,
1254                          nr_src);
1255
1256    ureg_emit_texture( ureg, insn.extended_token, target, nr_offset );
1257
1258    for (i = 0; i < nr_offset; i++)
1259       ureg_emit_texture_offset( ureg, &texoffsets[i]);
1260
1261    for (i = 0; i < nr_dst; i++)
1262       ureg_emit_dst( ureg, dst[i] );
1263
1264    for (i = 0; i < nr_src; i++)
1265       ureg_emit_src( ureg, src[i] );
1266
1267    ureg_fixup_insn_size( ureg, insn.insn_token );
1268 }
1269
1270
1271 void
1272 ureg_label_insn(struct ureg_program *ureg,
1273                 unsigned opcode,
1274                 const struct ureg_src *src,
1275                 unsigned nr_src,
1276                 unsigned *label_token )
1277 {
1278    struct ureg_emit_insn_result insn;
1279    unsigned i;
1280
1281    insn = ureg_emit_insn(ureg,
1282                          opcode,
1283                          FALSE,
1284                          FALSE,
1285                          FALSE,
1286                          TGSI_SWIZZLE_X,
1287                          TGSI_SWIZZLE_Y,
1288                          TGSI_SWIZZLE_Z,
1289                          TGSI_SWIZZLE_W,
1290                          0,
1291                          nr_src);
1292
1293    ureg_emit_label( ureg, insn.extended_token, label_token );
1294
1295    for (i = 0; i < nr_src; i++)
1296       ureg_emit_src( ureg, src[i] );
1297
1298    ureg_fixup_insn_size( ureg, insn.insn_token );
1299 }
1300
1301
1302 static void
1303 emit_decl_semantic(struct ureg_program *ureg,
1304                    unsigned file,
1305                    unsigned first,
1306                    unsigned last,
1307                    unsigned semantic_name,
1308                    unsigned semantic_index,
1309                    unsigned usage_mask,
1310                    unsigned array_id)
1311 {
1312    union tgsi_any_token *out = get_tokens(ureg, DOMAIN_DECL, array_id ? 4 : 3);
1313
1314    out[0].value = 0;
1315    out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
1316    out[0].decl.NrTokens = 3;
1317    out[0].decl.File = file;
1318    out[0].decl.UsageMask = usage_mask;
1319    out[0].decl.Semantic = 1;
1320    out[0].decl.Array = array_id != 0;
1321
1322    out[1].value = 0;
1323    out[1].decl_range.First = first;
1324    out[1].decl_range.Last = last;
1325
1326    out[2].value = 0;
1327    out[2].decl_semantic.Name = semantic_name;
1328    out[2].decl_semantic.Index = semantic_index;
1329
1330    if (array_id) {
1331       out[3].value = 0;
1332       out[3].array.ArrayID = array_id;
1333    }
1334 }
1335
1336
1337 static void
1338 emit_decl_fs(struct ureg_program *ureg,
1339              unsigned file,
1340              unsigned first,
1341              unsigned last,
1342              unsigned semantic_name,
1343              unsigned semantic_index,
1344              unsigned interpolate,
1345              unsigned cylindrical_wrap,
1346              unsigned interpolate_location,
1347              unsigned array_id)
1348 {
1349    union tgsi_any_token *out = get_tokens(ureg, DOMAIN_DECL,
1350                                           array_id ? 5 : 4);
1351
1352    out[0].value = 0;
1353    out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
1354    out[0].decl.NrTokens = 4;
1355    out[0].decl.File = file;
1356    out[0].decl.UsageMask = TGSI_WRITEMASK_XYZW; /* FIXME! */
1357    out[0].decl.Interpolate = 1;
1358    out[0].decl.Semantic = 1;
1359    out[0].decl.Array = array_id != 0;
1360
1361    out[1].value = 0;
1362    out[1].decl_range.First = first;
1363    out[1].decl_range.Last = last;
1364
1365    out[2].value = 0;
1366    out[2].decl_interp.Interpolate = interpolate;
1367    out[2].decl_interp.CylindricalWrap = cylindrical_wrap;
1368    out[2].decl_interp.Location = interpolate_location;
1369
1370    out[3].value = 0;
1371    out[3].decl_semantic.Name = semantic_name;
1372    out[3].decl_semantic.Index = semantic_index;
1373
1374    if (array_id) {
1375       out[4].value = 0;
1376       out[4].array.ArrayID = array_id;
1377    }
1378 }
1379
1380 static void
1381 emit_decl_temps( struct ureg_program *ureg,
1382                  unsigned first, unsigned last,
1383                  boolean local,
1384                  unsigned arrayid )
1385 {
1386    union tgsi_any_token *out = get_tokens( ureg, DOMAIN_DECL,
1387                                            arrayid ? 3 : 2 );
1388
1389    out[0].value = 0;
1390    out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
1391    out[0].decl.NrTokens = 2;
1392    out[0].decl.File = TGSI_FILE_TEMPORARY;
1393    out[0].decl.UsageMask = TGSI_WRITEMASK_XYZW;
1394    out[0].decl.Local = local;
1395
1396    out[1].value = 0;
1397    out[1].decl_range.First = first;
1398    out[1].decl_range.Last = last;
1399
1400    if (arrayid) {
1401       out[0].decl.Array = 1;
1402       out[2].value = 0;
1403       out[2].array.ArrayID = arrayid;
1404    }
1405 }
1406
1407 static void emit_decl_range( struct ureg_program *ureg,
1408                              unsigned file,
1409                              unsigned first,
1410                              unsigned count )
1411 {
1412    union tgsi_any_token *out = get_tokens( ureg, DOMAIN_DECL, 2 );
1413
1414    out[0].value = 0;
1415    out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
1416    out[0].decl.NrTokens = 2;
1417    out[0].decl.File = file;
1418    out[0].decl.UsageMask = TGSI_WRITEMASK_XYZW;
1419    out[0].decl.Semantic = 0;
1420
1421    out[1].value = 0;
1422    out[1].decl_range.First = first;
1423    out[1].decl_range.Last = first + count - 1;
1424 }
1425
1426 static void
1427 emit_decl_range2D(struct ureg_program *ureg,
1428                   unsigned file,
1429                   unsigned first,
1430                   unsigned last,
1431                   unsigned index2D)
1432 {
1433    union tgsi_any_token *out = get_tokens(ureg, DOMAIN_DECL, 3);
1434
1435    out[0].value = 0;
1436    out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
1437    out[0].decl.NrTokens = 3;
1438    out[0].decl.File = file;
1439    out[0].decl.UsageMask = TGSI_WRITEMASK_XYZW;
1440    out[0].decl.Dimension = 1;
1441
1442    out[1].value = 0;
1443    out[1].decl_range.First = first;
1444    out[1].decl_range.Last = last;
1445
1446    out[2].value = 0;
1447    out[2].decl_dim.Index2D = index2D;
1448 }
1449
1450 static void
1451 emit_decl_sampler_view(struct ureg_program *ureg,
1452                        unsigned index,
1453                        unsigned target,
1454                        unsigned return_type_x,
1455                        unsigned return_type_y,
1456                        unsigned return_type_z,
1457                        unsigned return_type_w )
1458 {
1459    union tgsi_any_token *out = get_tokens(ureg, DOMAIN_DECL, 3);
1460
1461    out[0].value = 0;
1462    out[0].decl.Type = TGSI_TOKEN_TYPE_DECLARATION;
1463    out[0].decl.NrTokens = 3;
1464    out[0].decl.File = TGSI_FILE_SAMPLER_VIEW;
1465    out[0].decl.UsageMask = 0xf;
1466
1467    out[1].value = 0;
1468    out[1].decl_range.First = index;
1469    out[1].decl_range.Last = index;
1470
1471    out[2].value = 0;
1472    out[2].decl_sampler_view.Resource    = target;
1473    out[2].decl_sampler_view.ReturnTypeX = return_type_x;
1474    out[2].decl_sampler_view.ReturnTypeY = return_type_y;
1475    out[2].decl_sampler_view.ReturnTypeZ = return_type_z;
1476    out[2].decl_sampler_view.ReturnTypeW = return_type_w;
1477 }
1478
1479 static void
1480 emit_immediate( struct ureg_program *ureg,
1481                 const unsigned *v,
1482                 unsigned type )
1483 {
1484    union tgsi_any_token *out = get_tokens( ureg, DOMAIN_DECL, 5 );
1485
1486    out[0].value = 0;
1487    out[0].imm.Type = TGSI_TOKEN_TYPE_IMMEDIATE;
1488    out[0].imm.NrTokens = 5;
1489    out[0].imm.DataType = type;
1490    out[0].imm.Padding = 0;
1491
1492    out[1].imm_data.Uint = v[0];
1493    out[2].imm_data.Uint = v[1];
1494    out[3].imm_data.Uint = v[2];
1495    out[4].imm_data.Uint = v[3];
1496 }
1497
1498 static void
1499 emit_property(struct ureg_program *ureg,
1500               unsigned name,
1501               unsigned data)
1502 {
1503    union tgsi_any_token *out = get_tokens(ureg, DOMAIN_DECL, 2);
1504
1505    out[0].value = 0;
1506    out[0].prop.Type = TGSI_TOKEN_TYPE_PROPERTY;
1507    out[0].prop.NrTokens = 2;
1508    out[0].prop.PropertyName = name;
1509
1510    out[1].prop_data.Data = data;
1511 }
1512
1513
1514 static void emit_decls( struct ureg_program *ureg )
1515 {
1516    unsigned i,j;
1517
1518    for (i = 0; i < Elements(ureg->properties); i++)
1519       if (ureg->properties[i] != ~0)
1520          emit_property(ureg, i, ureg->properties[i]);
1521
1522    if (ureg->processor == TGSI_PROCESSOR_VERTEX) {
1523       for (i = 0; i < PIPE_MAX_ATTRIBS; i++) {
1524          if (ureg->vs_inputs[i/32] & (1 << (i%32))) {
1525             emit_decl_range( ureg, TGSI_FILE_INPUT, i, 1 );
1526          }
1527       }
1528    } else if (ureg->processor == TGSI_PROCESSOR_FRAGMENT) {
1529       if (ureg->supports_any_inout_decl_range) {
1530          for (i = 0; i < ureg->nr_inputs; i++) {
1531             emit_decl_fs(ureg,
1532                          TGSI_FILE_INPUT,
1533                          ureg->input[i].first,
1534                          ureg->input[i].last,
1535                          ureg->input[i].semantic_name,
1536                          ureg->input[i].semantic_index,
1537                          ureg->input[i].interp,
1538                          ureg->input[i].cylindrical_wrap,
1539                          ureg->input[i].interp_location,
1540                          ureg->input[i].array_id);
1541          }
1542       }
1543       else {
1544          for (i = 0; i < ureg->nr_inputs; i++) {
1545             for (j = ureg->input[i].first; j <= ureg->input[i].last; j++) {
1546                emit_decl_fs(ureg,
1547                             TGSI_FILE_INPUT,
1548                             j, j,
1549                             ureg->input[i].semantic_name,
1550                             ureg->input[i].semantic_index +
1551                             (j - ureg->input[i].first),
1552                             ureg->input[i].interp,
1553                             ureg->input[i].cylindrical_wrap,
1554                             ureg->input[i].interp_location, 0);
1555             }
1556          }
1557       }
1558    } else {
1559       if (ureg->supports_any_inout_decl_range) {
1560          for (i = 0; i < ureg->nr_inputs; i++) {
1561             emit_decl_semantic(ureg,
1562                                TGSI_FILE_INPUT,
1563                                ureg->input[i].first,
1564                                ureg->input[i].last,
1565                                ureg->input[i].semantic_name,
1566                                ureg->input[i].semantic_index,
1567                                TGSI_WRITEMASK_XYZW,
1568                                ureg->input[i].array_id);
1569          }
1570       }
1571       else {
1572          for (i = 0; i < ureg->nr_inputs; i++) {
1573             for (j = ureg->input[i].first; j <= ureg->input[i].last; j++) {
1574                emit_decl_semantic(ureg,
1575                                   TGSI_FILE_INPUT,
1576                                   j, j,
1577                                   ureg->input[i].semantic_name,
1578                                   ureg->input[i].semantic_index +
1579                                   (j - ureg->input[i].first),
1580                                   TGSI_WRITEMASK_XYZW, 0);
1581             }
1582          }
1583       }
1584    }
1585
1586    for (i = 0; i < ureg->nr_system_values; i++) {
1587       emit_decl_semantic(ureg,
1588                          TGSI_FILE_SYSTEM_VALUE,
1589                          ureg->system_value[i].index,
1590                          ureg->system_value[i].index,
1591                          ureg->system_value[i].semantic_name,
1592                          ureg->system_value[i].semantic_index,
1593                          TGSI_WRITEMASK_XYZW, 0);
1594    }
1595
1596    if (ureg->supports_any_inout_decl_range) {
1597       for (i = 0; i < ureg->nr_outputs; i++) {
1598          emit_decl_semantic(ureg,
1599                             TGSI_FILE_OUTPUT,
1600                             ureg->output[i].first,
1601                             ureg->output[i].last,
1602                             ureg->output[i].semantic_name,
1603                             ureg->output[i].semantic_index,
1604                             ureg->output[i].usage_mask,
1605                             ureg->output[i].array_id);
1606       }
1607    }
1608    else {
1609       for (i = 0; i < ureg->nr_outputs; i++) {
1610          for (j = ureg->output[i].first; j <= ureg->output[i].last; j++) {
1611             emit_decl_semantic(ureg,
1612                                TGSI_FILE_OUTPUT,
1613                                j, j,
1614                                ureg->output[i].semantic_name,
1615                                ureg->output[i].semantic_index +
1616                                (j - ureg->output[i].first),
1617                                ureg->output[i].usage_mask, 0);
1618          }
1619       }
1620    }
1621
1622    for (i = 0; i < ureg->nr_samplers; i++) {
1623       emit_decl_range( ureg,
1624                        TGSI_FILE_SAMPLER,
1625                        ureg->sampler[i].Index, 1 );
1626    }
1627
1628    for (i = 0; i < ureg->nr_sampler_views; i++) {
1629       emit_decl_sampler_view(ureg,
1630                              ureg->sampler_view[i].index,
1631                              ureg->sampler_view[i].target,
1632                              ureg->sampler_view[i].return_type_x,
1633                              ureg->sampler_view[i].return_type_y,
1634                              ureg->sampler_view[i].return_type_z,
1635                              ureg->sampler_view[i].return_type_w);
1636    }
1637
1638    if (ureg->const_decls.nr_constant_ranges) {
1639       for (i = 0; i < ureg->const_decls.nr_constant_ranges; i++) {
1640          emit_decl_range(ureg,
1641                          TGSI_FILE_CONSTANT,
1642                          ureg->const_decls.constant_range[i].first,
1643                          ureg->const_decls.constant_range[i].last - ureg->const_decls.constant_range[i].first + 1);
1644       }
1645    }
1646
1647    for (i = 0; i < PIPE_MAX_CONSTANT_BUFFERS; i++) {
1648       struct const_decl *decl = &ureg->const_decls2D[i];
1649
1650       if (decl->nr_constant_ranges) {
1651          uint j;
1652
1653          for (j = 0; j < decl->nr_constant_ranges; j++) {
1654             emit_decl_range2D(ureg,
1655                               TGSI_FILE_CONSTANT,
1656                               decl->constant_range[j].first,
1657                               decl->constant_range[j].last,
1658                               i);
1659          }
1660       }
1661    }
1662
1663    if (ureg->nr_temps) {
1664       unsigned array = 0;
1665       for (i = 0; i < ureg->nr_temps;) {
1666          boolean local = util_bitmask_get(ureg->local_temps, i);
1667          unsigned first = i;
1668          i = util_bitmask_get_next_index(ureg->decl_temps, i + 1);
1669          if (i == UTIL_BITMASK_INVALID_INDEX)
1670             i = ureg->nr_temps;
1671
1672          if (array < ureg->nr_array_temps && ureg->array_temps[array] == first)
1673             emit_decl_temps( ureg, first, i - 1, local, ++array );
1674          else
1675             emit_decl_temps( ureg, first, i - 1, local, 0 );
1676       }
1677    }
1678
1679    if (ureg->nr_addrs) {
1680       emit_decl_range( ureg,
1681                        TGSI_FILE_ADDRESS,
1682                        0, ureg->nr_addrs );
1683    }
1684
1685    if (ureg->nr_preds) {
1686       emit_decl_range(ureg,
1687                       TGSI_FILE_PREDICATE,
1688                       0,
1689                       ureg->nr_preds);
1690    }
1691
1692    for (i = 0; i < ureg->nr_immediates; i++) {
1693       emit_immediate( ureg,
1694                       ureg->immediate[i].value.u,
1695                       ureg->immediate[i].type );
1696    }
1697 }
1698
1699 /* Append the instruction tokens onto the declarations to build a
1700  * contiguous stream suitable to send to the driver.
1701  */
1702 static void copy_instructions( struct ureg_program *ureg )
1703 {
1704    unsigned nr_tokens = ureg->domain[DOMAIN_INSN].count;
1705    union tgsi_any_token *out = get_tokens( ureg,
1706                                            DOMAIN_DECL,
1707                                            nr_tokens );
1708
1709    memcpy(out,
1710           ureg->domain[DOMAIN_INSN].tokens,
1711           nr_tokens * sizeof out[0] );
1712 }
1713
1714
1715 static void
1716 fixup_header_size(struct ureg_program *ureg)
1717 {
1718    union tgsi_any_token *out = retrieve_token( ureg, DOMAIN_DECL, 0 );
1719
1720    out->header.BodySize = ureg->domain[DOMAIN_DECL].count - 2;
1721 }
1722
1723
1724 static void
1725 emit_header( struct ureg_program *ureg )
1726 {
1727    union tgsi_any_token *out = get_tokens( ureg, DOMAIN_DECL, 2 );
1728
1729    out[0].header.HeaderSize = 2;
1730    out[0].header.BodySize = 0;
1731
1732    out[1].processor.Processor = ureg->processor;
1733    out[1].processor.Padding = 0;
1734 }
1735
1736
1737 const struct tgsi_token *ureg_finalize( struct ureg_program *ureg )
1738 {
1739    const struct tgsi_token *tokens;
1740
1741    emit_header( ureg );
1742    emit_decls( ureg );
1743    copy_instructions( ureg );
1744    fixup_header_size( ureg );
1745
1746    if (ureg->domain[0].tokens == error_tokens ||
1747        ureg->domain[1].tokens == error_tokens) {
1748       debug_printf("%s: error in generated shader\n", __FUNCTION__);
1749       assert(0);
1750       return NULL;
1751    }
1752
1753    tokens = &ureg->domain[DOMAIN_DECL].tokens[0].token;
1754
1755    if (0) {
1756       debug_printf("%s: emitted shader %d tokens:\n", __FUNCTION__,
1757                    ureg->domain[DOMAIN_DECL].count);
1758       tgsi_dump( tokens, 0 );
1759    }
1760
1761 #if DEBUG
1762    if (tokens && !tgsi_sanity_check(tokens)) {
1763       debug_printf("tgsi_ureg.c, sanity check failed on generated tokens:\n");
1764       tgsi_dump(tokens, 0);
1765       assert(0);
1766    }
1767 #endif
1768
1769
1770    return tokens;
1771 }
1772
1773
1774 void *ureg_create_shader( struct ureg_program *ureg,
1775                           struct pipe_context *pipe,
1776                           const struct pipe_stream_output_info *so )
1777 {
1778    struct pipe_shader_state state;
1779
1780    state.tokens = ureg_finalize(ureg);
1781    if(!state.tokens)
1782       return NULL;
1783
1784    if (so)
1785       state.stream_output = *so;
1786    else
1787       memset(&state.stream_output, 0, sizeof(state.stream_output));
1788
1789    switch (ureg->processor) {
1790    case TGSI_PROCESSOR_VERTEX:
1791       return pipe->create_vs_state(pipe, &state);
1792    case TGSI_PROCESSOR_TESS_CTRL:
1793       return pipe->create_tcs_state(pipe, &state);
1794    case TGSI_PROCESSOR_TESS_EVAL:
1795       return pipe->create_tes_state(pipe, &state);
1796    case TGSI_PROCESSOR_GEOMETRY:
1797       return pipe->create_gs_state(pipe, &state);
1798    case TGSI_PROCESSOR_FRAGMENT:
1799       return pipe->create_fs_state(pipe, &state);
1800    default:
1801       return NULL;
1802    }
1803 }
1804
1805
1806 const struct tgsi_token *ureg_get_tokens( struct ureg_program *ureg,
1807                                           unsigned *nr_tokens )
1808 {
1809    const struct tgsi_token *tokens;
1810
1811    ureg_finalize(ureg);
1812
1813    tokens = &ureg->domain[DOMAIN_DECL].tokens[0].token;
1814
1815    if (nr_tokens)
1816       *nr_tokens = ureg->domain[DOMAIN_DECL].size;
1817
1818    ureg->domain[DOMAIN_DECL].tokens = 0;
1819    ureg->domain[DOMAIN_DECL].size = 0;
1820    ureg->domain[DOMAIN_DECL].order = 0;
1821    ureg->domain[DOMAIN_DECL].count = 0;
1822
1823    return tokens;
1824 }
1825
1826
1827 void ureg_free_tokens( const struct tgsi_token *tokens )
1828 {
1829    FREE((struct tgsi_token *)tokens);
1830 }
1831
1832
1833 static INLINE unsigned
1834 pipe_shader_from_tgsi_processor(unsigned processor)
1835 {
1836    switch (processor) {
1837    case TGSI_PROCESSOR_VERTEX:
1838       return PIPE_SHADER_VERTEX;
1839    case TGSI_PROCESSOR_TESS_CTRL:
1840       return PIPE_SHADER_TESS_CTRL;
1841    case TGSI_PROCESSOR_TESS_EVAL:
1842       return PIPE_SHADER_TESS_EVAL;
1843    case TGSI_PROCESSOR_GEOMETRY:
1844       return PIPE_SHADER_GEOMETRY;
1845    case TGSI_PROCESSOR_FRAGMENT:
1846       return PIPE_SHADER_FRAGMENT;
1847    case TGSI_PROCESSOR_COMPUTE:
1848       return PIPE_SHADER_COMPUTE;
1849    default:
1850       assert(0);
1851       return PIPE_SHADER_VERTEX;
1852    }
1853 }
1854
1855
1856 struct ureg_program *
1857 ureg_create(unsigned processor)
1858 {
1859    return ureg_create_with_screen(processor, NULL);
1860 }
1861
1862
1863 struct ureg_program *
1864 ureg_create_with_screen(unsigned processor, struct pipe_screen *screen)
1865 {
1866    int i;
1867    struct ureg_program *ureg = CALLOC_STRUCT( ureg_program );
1868    if (ureg == NULL)
1869       goto no_ureg;
1870
1871    ureg->processor = processor;
1872    ureg->supports_any_inout_decl_range =
1873       screen &&
1874       screen->get_shader_param(screen,
1875                                pipe_shader_from_tgsi_processor(processor),
1876                                PIPE_SHADER_CAP_TGSI_ANY_INOUT_DECL_RANGE) != 0;
1877
1878    for (i = 0; i < Elements(ureg->properties); i++)
1879       ureg->properties[i] = ~0;
1880
1881    ureg->free_temps = util_bitmask_create();
1882    if (ureg->free_temps == NULL)
1883       goto no_free_temps;
1884
1885    ureg->local_temps = util_bitmask_create();
1886    if (ureg->local_temps == NULL)
1887       goto no_local_temps;
1888
1889    ureg->decl_temps = util_bitmask_create();
1890    if (ureg->decl_temps == NULL)
1891       goto no_decl_temps;
1892
1893    return ureg;
1894
1895 no_decl_temps:
1896    util_bitmask_destroy(ureg->local_temps);
1897 no_local_temps:
1898    util_bitmask_destroy(ureg->free_temps);
1899 no_free_temps:
1900    FREE(ureg);
1901 no_ureg:
1902    return NULL;
1903 }
1904
1905
1906 unsigned
1907 ureg_get_nr_outputs( const struct ureg_program *ureg )
1908 {
1909    if (!ureg)
1910       return 0;
1911    return ureg->nr_outputs;
1912 }
1913
1914
1915 void ureg_destroy( struct ureg_program *ureg )
1916 {
1917    unsigned i;
1918
1919    for (i = 0; i < Elements(ureg->domain); i++) {
1920       if (ureg->domain[i].tokens &&
1921           ureg->domain[i].tokens != error_tokens)
1922          FREE(ureg->domain[i].tokens);
1923    }
1924
1925    util_bitmask_destroy(ureg->free_temps);
1926    util_bitmask_destroy(ureg->local_temps);
1927    util_bitmask_destroy(ureg->decl_temps);
1928
1929    FREE(ureg);
1930 }