projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
r600g: use tables with ISA info v3
[mesa.git] / src / gallium / drivers / r600 / r600_asm.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #include "r600_sq.h"
24 #include "r600_opcodes.h"
25 #include "r600_formats.h"
26 #include "r600_shader.h"
27 #include "r600d.h"
28
29 #include <errno.h>
30 #include <byteswap.h>
31 #include "util/u_memory.h"
32 #include "pipe/p_shader_tokens.h"
33
34 #define NUM_OF_CYCLES 3
35 #define NUM_OF_COMPONENTS 4
36
37 static inline unsigned int r600_bytecode_get_num_operands(
38 struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
39 {
40 return r600_isa_alu(alu->op)->src_count;
41 }
42
43 int r700_bytecode_alu_build(struct r600_bytecode *bc,
44 struct r600_bytecode_alu *alu, unsigned id);
45
46 static struct r600_bytecode_cf *r600_bytecode_cf(void)
47 {
48 struct r600_bytecode_cf *cf = CALLOC_STRUCT(r600_bytecode_cf);
49
50 if (cf == NULL)
51 return NULL;
52 LIST_INITHEAD(&cf->list);
53 LIST_INITHEAD(&cf->alu);
54 LIST_INITHEAD(&cf->vtx);
55 LIST_INITHEAD(&cf->tex);
56 return cf;
57 }
58
59 static struct r600_bytecode_alu *r600_bytecode_alu(void)
60 {
61 struct r600_bytecode_alu *alu = CALLOC_STRUCT(r600_bytecode_alu);
62
63 if (alu == NULL)
64 return NULL;
65 LIST_INITHEAD(&alu->list);
66 return alu;
67 }
68
69 static struct r600_bytecode_vtx *r600_bytecode_vtx(void)
70 {
71 struct r600_bytecode_vtx *vtx = CALLOC_STRUCT(r600_bytecode_vtx);
72
73 if (vtx == NULL)
74 return NULL;
75 LIST_INITHEAD(&vtx->list);
76 return vtx;
77 }
78
79 static struct r600_bytecode_tex *r600_bytecode_tex(void)
80 {
81 struct r600_bytecode_tex *tex = CALLOC_STRUCT(r600_bytecode_tex);
82
83 if (tex == NULL)
84 return NULL;
85 LIST_INITHEAD(&tex->list);
86 return tex;
87 }
88
89 void r600_bytecode_init(struct r600_bytecode *bc,
90 enum chip_class chip_class,
91 enum radeon_family family,
92 enum r600_msaa_texture_mode msaa_texture_mode)
93 {
94 if ((chip_class == R600) &&
95 (family != CHIP_RV670 && family != CHIP_RS780 && family != CHIP_RS880)) {
96 bc->ar_handling = AR_HANDLE_RV6XX;
97 bc->r6xx_nop_after_rel_dst = 1;
98 } else {
99 bc->ar_handling = AR_HANDLE_NORMAL;
100 bc->r6xx_nop_after_rel_dst = 0;
101 }
102
103 LIST_INITHEAD(&bc->cf);
104 bc->chip_class = chip_class;
105 bc->msaa_texture_mode = msaa_texture_mode;
106 }
107
108 static int r600_bytecode_add_cf(struct r600_bytecode *bc)
109 {
110 struct r600_bytecode_cf *cf = r600_bytecode_cf();
111
112 if (cf == NULL)
113 return -ENOMEM;
114 LIST_ADDTAIL(&cf->list, &bc->cf);
115 if (bc->cf_last) {
116 cf->id = bc->cf_last->id + 2;
117 if (bc->cf_last->eg_alu_extended) {
118 /* take into account extended alu size */
119 cf->id += 2;
120 bc->ndw += 2;
121 }
122 }
123 bc->cf_last = cf;
124 bc->ncf++;
125 bc->ndw += 2;
126 bc->force_add_cf = 0;
127 bc->ar_loaded = 0;
128 return 0;
129 }
130
131 int r600_bytecode_add_output(struct r600_bytecode *bc,
132 const struct r600_bytecode_output *output)
133 {
134 int r;
135
136 if (output->gpr >= bc->ngpr)
137 bc->ngpr = output->gpr + 1;
138
139 if (bc->cf_last && (bc->cf_last->op == output->op ||
140 (bc->cf_last->op == CF_OP_EXPORT &&
141 output->op == CF_OP_EXPORT_DONE)) &&
142 output->type == bc->cf_last->output.type &&
143 output->elem_size == bc->cf_last->output.elem_size &&
144 output->swizzle_x == bc->cf_last->output.swizzle_x &&
145 output->swizzle_y == bc->cf_last->output.swizzle_y &&
146 output->swizzle_z == bc->cf_last->output.swizzle_z &&
147 output->swizzle_w == bc->cf_last->output.swizzle_w &&
148 (output->burst_count + bc->cf_last->output.burst_count) <= 16) {
149
150 if ((output->gpr + output->burst_count) == bc->cf_last->output.gpr &&
151 (output->array_base + output->burst_count) == bc->cf_last->output.array_base) {
152
153 bc->cf_last->output.end_of_program |= output->end_of_program;
154 bc->cf_last->op = bc->cf_last->output.op = output->op;
155 bc->cf_last->output.gpr = output->gpr;
156 bc->cf_last->output.array_base = output->array_base;
157 bc->cf_last->output.burst_count += output->burst_count;
158 return 0;
159
160 } else if (output->gpr == (bc->cf_last->output.gpr + bc->cf_last->output.burst_count) &&
161 output->array_base == (bc->cf_last->output.array_base + bc->cf_last->output.burst_count)) {
162
163 bc->cf_last->output.end_of_program |= output->end_of_program;
164 bc->cf_last->op = bc->cf_last->output.op = output->op;
165 bc->cf_last->output.burst_count += output->burst_count;
166 return 0;
167 }
168 }
169
170 r = r600_bytecode_add_cf(bc);
171 if (r)
172 return r;
173 bc->cf_last->op = output->op;
174 memcpy(&bc->cf_last->output, output, sizeof(struct r600_bytecode_output));
175 return 0;
176 }
177
178 /* alu instructions that can ony exits once per group */
179 static int is_alu_once_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
180 {
181 return r600_isa_alu(alu->op)->flags & (AF_KILL | AF_PRED);
182 }
183
184 static int is_alu_reduction_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
185 {
186 return (r600_isa_alu(alu->op)->flags & AF_REPL) &&
187 (r600_isa_alu_slots(bc->isa->hw_class, alu->op) == AF_4V);
188 }
189
190 static int is_alu_mova_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
191 {
192 return r600_isa_alu(alu->op)->flags & AF_MOVA;
193 }
194
195 static int alu_uses_rel(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
196 {
197 unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
198 unsigned src;
199
200 if (alu->dst.rel) {
201 return 1;
202 }
203
204 for (src = 0; src < num_src; ++src) {
205 if (alu->src[src].rel) {
206 return 1;
207 }
208 }
209 return 0;
210 }
211
212 static int is_alu_vec_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
213 {
214 unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
215 return !(slots & AF_S);
216 }
217
218 static int is_alu_trans_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
219 {
220 unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
221 return !(slots & AF_V);
222 }
223
224 /* alu instructions that can execute on any unit */
225 static int is_alu_any_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
226 {
227 unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
228 return slots == AF_VS;
229 }
230
231 static int is_nop_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
232 {
233 return alu->op == ALU_OP0_NOP;
234 }
235
236 static int assign_alu_units(struct r600_bytecode *bc, struct r600_bytecode_alu *alu_first,
237 struct r600_bytecode_alu *assignment[5])
238 {
239 struct r600_bytecode_alu *alu;
240 unsigned i, chan, trans;
241 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
242
243 for (i = 0; i < max_slots; i++)
244 assignment[i] = NULL;
245
246 for (alu = alu_first; alu; alu = LIST_ENTRY(struct r600_bytecode_alu, alu->list.next, list)) {
247 chan = alu->dst.chan;
248 if (max_slots == 4)
249 trans = 0;
250 else if (is_alu_trans_unit_inst(bc, alu))
251 trans = 1;
252 else if (is_alu_vec_unit_inst(bc, alu))
253 trans = 0;
254 else if (assignment[chan])
255 trans = 1; /* Assume ALU_INST_PREFER_VECTOR. */
256 else
257 trans = 0;
258
259 if (trans) {
260 if (assignment[4]) {
261 assert(0); /* ALU.Trans has already been allocated. */
262 return -1;
263 }
264 assignment[4] = alu;
265 } else {
266 if (assignment[chan]) {
267 assert(0); /* ALU.chan has already been allocated. */
268 return -1;
269 }
270 assignment[chan] = alu;
271 }
272
273 if (alu->last)
274 break;
275 }
276 return 0;
277 }
278
279 struct alu_bank_swizzle {
280 int hw_gpr[NUM_OF_CYCLES][NUM_OF_COMPONENTS];
281 int hw_cfile_addr[4];
282 int hw_cfile_elem[4];
283 };
284
285 static const unsigned cycle_for_bank_swizzle_vec[][3] = {
286 [SQ_ALU_VEC_012] = { 0, 1, 2 },
287 [SQ_ALU_VEC_021] = { 0, 2, 1 },
288 [SQ_ALU_VEC_120] = { 1, 2, 0 },
289 [SQ_ALU_VEC_102] = { 1, 0, 2 },
290 [SQ_ALU_VEC_201] = { 2, 0, 1 },
291 [SQ_ALU_VEC_210] = { 2, 1, 0 }
292 };
293
294 static const unsigned cycle_for_bank_swizzle_scl[][3] = {
295 [SQ_ALU_SCL_210] = { 2, 1, 0 },
296 [SQ_ALU_SCL_122] = { 1, 2, 2 },
297 [SQ_ALU_SCL_212] = { 2, 1, 2 },
298 [SQ_ALU_SCL_221] = { 2, 2, 1 }
299 };
300
301 static void init_bank_swizzle(struct alu_bank_swizzle *bs)
302 {
303 int i, cycle, component;
304 /* set up gpr use */
305 for (cycle = 0; cycle < NUM_OF_CYCLES; cycle++)
306 for (component = 0; component < NUM_OF_COMPONENTS; component++)
307 bs->hw_gpr[cycle][component] = -1;
308 for (i = 0; i < 4; i++)
309 bs->hw_cfile_addr[i] = -1;
310 for (i = 0; i < 4; i++)
311 bs->hw_cfile_elem[i] = -1;
312 }
313
314 static int reserve_gpr(struct alu_bank_swizzle *bs, unsigned sel, unsigned chan, unsigned cycle)
315 {
316 if (bs->hw_gpr[cycle][chan] == -1)
317 bs->hw_gpr[cycle][chan] = sel;
318 else if (bs->hw_gpr[cycle][chan] != (int)sel) {
319 /* Another scalar operation has already used the GPR read port for the channel. */
320 return -1;
321 }
322 return 0;
323 }
324
325 static int reserve_cfile(struct r600_bytecode *bc, struct alu_bank_swizzle *bs, unsigned sel, unsigned chan)
326 {
327 int res, num_res = 4;
328 if (bc->chip_class >= R700) {
329 num_res = 2;
330 chan /= 2;
331 }
332 for (res = 0; res < num_res; ++res) {
333 if (bs->hw_cfile_addr[res] == -1) {
334 bs->hw_cfile_addr[res] = sel;
335 bs->hw_cfile_elem[res] = chan;
336 return 0;
337 } else if (bs->hw_cfile_addr[res] == sel &&
338 bs->hw_cfile_elem[res] == chan)
339 return 0; /* Read for this scalar element already reserved, nothing to do here. */
340 }
341 /* All cfile read ports are used, cannot reference vector element. */
342 return -1;
343 }
344
345 static int is_gpr(unsigned sel)
346 {
347 return (sel >= 0 && sel <= 127);
348 }
349
350 /* CB constants start at 512, and get translated to a kcache index when ALU
351 * clauses are constructed. Note that we handle kcache constants the same way
352 * as (the now gone) cfile constants, is that really required? */
353 static int is_cfile(unsigned sel)
354 {
355 return (sel > 255 && sel < 512) ||
356 (sel > 511 && sel < 4607) || /* Kcache before translation. */
357 (sel > 127 && sel < 192); /* Kcache after translation. */
358 }
359
360 static int is_const(int sel)
361 {
362 return is_cfile(sel) ||
363 (sel >= V_SQ_ALU_SRC_0 &&
364 sel <= V_SQ_ALU_SRC_LITERAL);
365 }
366
367 static int check_vector(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
368 struct alu_bank_swizzle *bs, int bank_swizzle)
369 {
370 int r, src, num_src, sel, elem, cycle;
371
372 num_src = r600_bytecode_get_num_operands(bc, alu);
373 for (src = 0; src < num_src; src++) {
374 sel = alu->src[src].sel;
375 elem = alu->src[src].chan;
376 if (is_gpr(sel)) {
377 cycle = cycle_for_bank_swizzle_vec[bank_swizzle][src];
378 if (src == 1 && sel == alu->src[0].sel && elem == alu->src[0].chan)
379 /* Nothing to do; special-case optimization,
380 * second source uses first source’s reservation. */
381 continue;
382 else {
383 r = reserve_gpr(bs, sel, elem, cycle);
384 if (r)
385 return r;
386 }
387 } else if (is_cfile(sel)) {
388 r = reserve_cfile(bc, bs, (alu->src[src].kc_bank<<16) + sel, elem);
389 if (r)
390 return r;
391 }
392 /* No restrictions on PV, PS, literal or special constants. */
393 }
394 return 0;
395 }
396
397 static int check_scalar(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
398 struct alu_bank_swizzle *bs, int bank_swizzle)
399 {
400 int r, src, num_src, const_count, sel, elem, cycle;
401
402 num_src = r600_bytecode_get_num_operands(bc, alu);
403 for (const_count = 0, src = 0; src < num_src; ++src) {
404 sel = alu->src[src].sel;
405 elem = alu->src[src].chan;
406 if (is_const(sel)) { /* Any constant, including literal and inline constants. */
407 if (const_count >= 2)
408 /* More than two references to a constant in
409 * transcendental operation. */
410 return -1;
411 else
412 const_count++;
413 }
414 if (is_cfile(sel)) {
415 r = reserve_cfile(bc, bs, (alu->src[src].kc_bank<<16) + sel, elem);
416 if (r)
417 return r;
418 }
419 }
420 for (src = 0; src < num_src; ++src) {
421 sel = alu->src[src].sel;
422 elem = alu->src[src].chan;
423 if (is_gpr(sel)) {
424 cycle = cycle_for_bank_swizzle_scl[bank_swizzle][src];
425 if (cycle < const_count)
426 /* Cycle for GPR load conflicts with
427 * constant load in transcendental operation. */
428 return -1;
429 r = reserve_gpr(bs, sel, elem, cycle);
430 if (r)
431 return r;
432 }
433 /* PV PS restrictions */
434 if (const_count && (sel == 254 || sel == 255)) {
435 cycle = cycle_for_bank_swizzle_scl[bank_swizzle][src];
436 if (cycle < const_count)
437 return -1;
438 }
439 }
440 return 0;
441 }
442
443 static int check_and_set_bank_swizzle(struct r600_bytecode *bc,
444 struct r600_bytecode_alu *slots[5])
445 {
446 struct alu_bank_swizzle bs;
447 int bank_swizzle[5];
448 int i, r = 0, forced = 1;
449 boolean scalar_only = bc->chip_class == CAYMAN ? false : true;
450 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
451
452 for (i = 0; i < max_slots; i++) {
453 if (slots[i]) {
454 if (slots[i]->bank_swizzle_force) {
455 slots[i]->bank_swizzle = slots[i]->bank_swizzle_force;
456 } else {
457 forced = 0;
458 }
459 }
460
461 if (i < 4 && slots[i])
462 scalar_only = false;
463 }
464 if (forced)
465 return 0;
466
467 /* Just check every possible combination of bank swizzle.
468 * Not very efficent, but works on the first try in most of the cases. */
469 for (i = 0; i < 4; i++)
470 if (!slots[i] || !slots[i]->bank_swizzle_force)
471 bank_swizzle[i] = SQ_ALU_VEC_012;
472 else
473 bank_swizzle[i] = slots[i]->bank_swizzle;
474
475 bank_swizzle[4] = SQ_ALU_SCL_210;
476 while(bank_swizzle[4] <= SQ_ALU_SCL_221) {
477
478 if (max_slots == 4) {
479 for (i = 0; i < max_slots; i++) {
480 if (bank_swizzle[i] == SQ_ALU_VEC_210)
481 return -1;
482 }
483 }
484 init_bank_swizzle(&bs);
485 if (scalar_only == false) {
486 for (i = 0; i < 4; i++) {
487 if (slots[i]) {
488 r = check_vector(bc, slots[i], &bs, bank_swizzle[i]);
489 if (r)
490 break;
491 }
492 }
493 } else
494 r = 0;
495
496 if (!r && slots[4] && max_slots == 5) {
497 r = check_scalar(bc, slots[4], &bs, bank_swizzle[4]);
498 }
499 if (!r) {
500 for (i = 0; i < max_slots; i++) {
501 if (slots[i])
502 slots[i]->bank_swizzle = bank_swizzle[i];
503 }
504 return 0;
505 }
506
507 if (scalar_only) {
508 bank_swizzle[4]++;
509 } else {
510 for (i = 0; i < max_slots; i++) {
511 if (!slots[i] || !slots[i]->bank_swizzle_force) {
512 bank_swizzle[i]++;
513 if (bank_swizzle[i] <= SQ_ALU_VEC_210)
514 break;
515 else
516 bank_swizzle[i] = SQ_ALU_VEC_012;
517 }
518 }
519 }
520 }
521
522 /* Couldn't find a working swizzle. */
523 return -1;
524 }
525
526 static int replace_gpr_with_pv_ps(struct r600_bytecode *bc,
527 struct r600_bytecode_alu *slots[5], struct r600_bytecode_alu *alu_prev)
528 {
529 struct r600_bytecode_alu *prev[5];
530 int gpr[5], chan[5];
531 int i, j, r, src, num_src;
532 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
533
534 r = assign_alu_units(bc, alu_prev, prev);
535 if (r)
536 return r;
537
538 for (i = 0; i < max_slots; ++i) {
539 if (prev[i] && (prev[i]->dst.write || prev[i]->is_op3) && !prev[i]->dst.rel) {
540 gpr[i] = prev[i]->dst.sel;
541 /* cube writes more than PV.X */
542 if (is_alu_reduction_inst(bc, prev[i]))
543 chan[i] = 0;
544 else
545 chan[i] = prev[i]->dst.chan;
546 } else
547 gpr[i] = -1;
548 }
549
550 for (i = 0; i < max_slots; ++i) {
551 struct r600_bytecode_alu *alu = slots[i];
552 if(!alu)
553 continue;
554
555 num_src = r600_bytecode_get_num_operands(bc, alu);
556 for (src = 0; src < num_src; ++src) {
557 if (!is_gpr(alu->src[src].sel) || alu->src[src].rel)
558 continue;
559
560 if (bc->chip_class < CAYMAN) {
561 if (alu->src[src].sel == gpr[4] &&
562 alu->src[src].chan == chan[4] &&
563 alu_prev->pred_sel == alu->pred_sel) {
564 alu->src[src].sel = V_SQ_ALU_SRC_PS;
565 alu->src[src].chan = 0;
566 continue;
567 }
568 }
569
570 for (j = 0; j < 4; ++j) {
571 if (alu->src[src].sel == gpr[j] &&
572 alu->src[src].chan == j &&
573 alu_prev->pred_sel == alu->pred_sel) {
574 alu->src[src].sel = V_SQ_ALU_SRC_PV;
575 alu->src[src].chan = chan[j];
576 break;
577 }
578 }
579 }
580 }
581
582 return 0;
583 }
584
585 void r600_bytecode_special_constants(uint32_t value, unsigned *sel, unsigned *neg)
586 {
587 switch(value) {
588 case 0:
589 *sel = V_SQ_ALU_SRC_0;
590 break;
591 case 1:
592 *sel = V_SQ_ALU_SRC_1_INT;
593 break;
594 case -1:
595 *sel = V_SQ_ALU_SRC_M_1_INT;
596 break;
597 case 0x3F800000: /* 1.0f */
598 *sel = V_SQ_ALU_SRC_1;
599 break;
600 case 0x3F000000: /* 0.5f */
601 *sel = V_SQ_ALU_SRC_0_5;
602 break;
603 case 0xBF800000: /* -1.0f */
604 *sel = V_SQ_ALU_SRC_1;
605 *neg ^= 1;
606 break;
607 case 0xBF000000: /* -0.5f */
608 *sel = V_SQ_ALU_SRC_0_5;
609 *neg ^= 1;
610 break;
611 default:
612 *sel = V_SQ_ALU_SRC_LITERAL;
613 break;
614 }
615 }
616
617 /* compute how many literal are needed */
618 static int r600_bytecode_alu_nliterals(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
619 uint32_t literal[4], unsigned *nliteral)
620 {
621 unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
622 unsigned i, j;
623
624 for (i = 0; i < num_src; ++i) {
625 if (alu->src[i].sel == V_SQ_ALU_SRC_LITERAL) {
626 uint32_t value = alu->src[i].value;
627 unsigned found = 0;
628 for (j = 0; j < *nliteral; ++j) {
629 if (literal[j] == value) {
630 found = 1;
631 break;
632 }
633 }
634 if (!found) {
635 if (*nliteral >= 4)
636 return -EINVAL;
637 literal[(*nliteral)++] = value;
638 }
639 }
640 }
641 return 0;
642 }
643
644 static void r600_bytecode_alu_adjust_literals(struct r600_bytecode *bc,
645 struct r600_bytecode_alu *alu,
646 uint32_t literal[4], unsigned nliteral)
647 {
648 unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
649 unsigned i, j;
650
651 for (i = 0; i < num_src; ++i) {
652 if (alu->src[i].sel == V_SQ_ALU_SRC_LITERAL) {
653 uint32_t value = alu->src[i].value;
654 for (j = 0; j < nliteral; ++j) {
655 if (literal[j] == value) {
656 alu->src[i].chan = j;
657 break;
658 }
659 }
660 }
661 }
662 }
663
664 static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu *slots[5],
665 struct r600_bytecode_alu *alu_prev)
666 {
667 struct r600_bytecode_alu *prev[5];
668 struct r600_bytecode_alu *result[5] = { NULL };
669
670 uint32_t literal[4], prev_literal[4];
671 unsigned nliteral = 0, prev_nliteral = 0;
672
673 int i, j, r, src, num_src;
674 int num_once_inst = 0;
675 int have_mova = 0, have_rel = 0;
676 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
677
678 r = assign_alu_units(bc, alu_prev, prev);
679 if (r)
680 return r;
681
682 for (i = 0; i < max_slots; ++i) {
683 if (prev[i]) {
684 if (prev[i]->pred_sel)
685 return 0;
686 if (is_alu_once_inst(bc, prev[i]))
687 return 0;
688 }
689 if (slots[i]) {
690 if (slots[i]->pred_sel)
691 return 0;
692 if (is_alu_once_inst(bc, slots[i]))
693 return 0;
694 }
695 }
696
697 for (i = 0; i < max_slots; ++i) {
698 struct r600_bytecode_alu *alu;
699
700 if (num_once_inst > 0)
701 return 0;
702
703 /* check number of literals */
704 if (prev[i]) {
705 if (r600_bytecode_alu_nliterals(bc, prev[i], literal, &nliteral))
706 return 0;
707 if (r600_bytecode_alu_nliterals(bc, prev[i], prev_literal, &prev_nliteral))
708 return 0;
709 if (is_alu_mova_inst(bc, prev[i])) {
710 if (have_rel)
711 return 0;
712 have_mova = 1;
713 }
714
715 if (alu_uses_rel(bc, prev[i])) {
716 if (have_mova) {
717 return 0;
718 }
719 have_rel = 1;
720 }
721
722 num_once_inst += is_alu_once_inst(bc, prev[i]);
723 }
724 if (slots[i] && r600_bytecode_alu_nliterals(bc, slots[i], literal, &nliteral))
725 return 0;
726
727 /* Let's check used slots. */
728 if (prev[i] && !slots[i]) {
729 result[i] = prev[i];
730 continue;
731 } else if (prev[i] && slots[i]) {
732 if (max_slots == 5 && result[4] == NULL && prev[4] == NULL && slots[4] == NULL) {
733 /* Trans unit is still free try to use it. */
734 if (is_alu_any_unit_inst(bc, slots[i])) {
735 result[i] = prev[i];
736 result[4] = slots[i];
737 } else if (is_alu_any_unit_inst(bc, prev[i])) {
738 if (slots[i]->dst.sel == prev[i]->dst.sel &&
739 (slots[i]->dst.write == 1 || slots[i]->is_op3) &&
740 (prev[i]->dst.write == 1 || prev[i]->is_op3))
741 return 0;
742
743 result[i] = slots[i];
744 result[4] = prev[i];
745 } else
746 return 0;
747 } else
748 return 0;
749 } else if(!slots[i]) {
750 continue;
751 } else {
752 if (max_slots == 5 && slots[i] && prev[4] &&
753 slots[i]->dst.sel == prev[4]->dst.sel &&
754 slots[i]->dst.chan == prev[4]->dst.chan &&
755 (slots[i]->dst.write == 1 || slots[i]->is_op3) &&
756 (prev[4]->dst.write == 1 || prev[4]->is_op3))
757 return 0;
758
759 result[i] = slots[i];
760 }
761
762 alu = slots[i];
763 num_once_inst += is_alu_once_inst(bc, alu);
764
765 /* don't reschedule NOPs */
766 if (is_nop_inst(bc, alu))
767 return 0;
768
769 if (is_alu_mova_inst(bc, alu)) {
770 if (have_rel) {
771 return 0;
772 }
773 have_mova = 1;
774 }
775
776 if (alu_uses_rel(bc, alu)) {
777 if (have_mova) {
778 return 0;
779 }
780 have_rel = 1;
781 }
782
783 /* Let's check source gprs */
784 num_src = r600_bytecode_get_num_operands(bc, alu);
785 for (src = 0; src < num_src; ++src) {
786
787 /* Constants don't matter. */
788 if (!is_gpr(alu->src[src].sel))
789 continue;
790
791 for (j = 0; j < max_slots; ++j) {
792 if (!prev[j] || !(prev[j]->dst.write || prev[j]->is_op3))
793 continue;
794
795 /* If it's relative then we can't determin which gpr is really used. */
796 if (prev[j]->dst.chan == alu->src[src].chan &&
797 (prev[j]->dst.sel == alu->src[src].sel ||
798 prev[j]->dst.rel || alu->src[src].rel))
799 return 0;
800 }
801 }
802 }
803
804 /* more than one PRED_ or KILL_ ? */
805 if (num_once_inst > 1)
806 return 0;
807
808 /* check if the result can still be swizzlet */
809 r = check_and_set_bank_swizzle(bc, result);
810 if (r)
811 return 0;
812
813 /* looks like everything worked out right, apply the changes */
814
815 /* undo adding previus literals */
816 bc->cf_last->ndw -= align(prev_nliteral, 2);
817
818 /* sort instructions */
819 for (i = 0; i < max_slots; ++i) {
820 slots[i] = result[i];
821 if (result[i]) {
822 LIST_DEL(&result[i]->list);
823 result[i]->last = 0;
824 LIST_ADDTAIL(&result[i]->list, &bc->cf_last->alu);
825 }
826 }
827
828 /* determine new last instruction */
829 LIST_ENTRY(struct r600_bytecode_alu, bc->cf_last->alu.prev, list)->last = 1;
830
831 /* determine new first instruction */
832 for (i = 0; i < max_slots; ++i) {
833 if (result[i]) {
834 bc->cf_last->curr_bs_head = result[i];
835 break;
836 }
837 }
838
839 bc->cf_last->prev_bs_head = bc->cf_last->prev2_bs_head;
840 bc->cf_last->prev2_bs_head = NULL;
841
842 return 0;
843 }
844
845 /* we'll keep kcache sets sorted by bank & addr */
846 static int r600_bytecode_alloc_kcache_line(struct r600_bytecode *bc,
847 struct r600_bytecode_kcache *kcache,
848 unsigned bank, unsigned line)
849 {
850 int i, kcache_banks = bc->chip_class >= EVERGREEN ? 4 : 2;
851
852 for (i = 0; i < kcache_banks; i++) {
853 if (kcache[i].mode) {
854 int d;
855
856 if (kcache[i].bank < bank)
857 continue;
858
859 if ((kcache[i].bank == bank && kcache[i].addr > line+1) ||
860 kcache[i].bank > bank) {
861 /* try to insert new line */
862 if (kcache[kcache_banks-1].mode) {
863 /* all sets are in use */
864 return -ENOMEM;
865 }
866
867 memmove(&kcache[i+1],&kcache[i], (kcache_banks-i-1)*sizeof(struct r600_bytecode_kcache));
868 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_1;
869 kcache[i].bank = bank;
870 kcache[i].addr = line;
871 return 0;
872 }
873
874 d = line - kcache[i].addr;
875
876 if (d == -1) {
877 kcache[i].addr--;
878 if (kcache[i].mode == V_SQ_CF_KCACHE_LOCK_2) {
879 /* we are prepending the line to the current set,
880 * discarding the existing second line,
881 * so we'll have to insert line+2 after it */
882 line += 2;
883 continue;
884 } else if (kcache[i].mode == V_SQ_CF_KCACHE_LOCK_1) {
885 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_2;
886 return 0;
887 } else {
888 /* V_SQ_CF_KCACHE_LOCK_LOOP_INDEX is not supported */
889 return -ENOMEM;
890 }
891 } else if (d == 1) {
892 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_2;
893 return 0;
894 } else if (d == 0)
895 return 0;
896 } else { /* free kcache set - use it */
897 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_1;
898 kcache[i].bank = bank;
899 kcache[i].addr = line;
900 return 0;
901 }
902 }
903 return -ENOMEM;
904 }
905
906 static int r600_bytecode_alloc_inst_kcache_lines(struct r600_bytecode *bc,
907 struct r600_bytecode_kcache *kcache,
908 struct r600_bytecode_alu *alu)
909 {
910 int i, r;
911
912 for (i = 0; i < 3; i++) {
913 unsigned bank, line, sel = alu->src[i].sel;
914
915 if (sel < 512)
916 continue;
917
918 bank = alu->src[i].kc_bank;
919 line = (sel-512)>>4;
920
921 if ((r = r600_bytecode_alloc_kcache_line(bc, kcache, bank, line)))
922 return r;
923 }
924 return 0;
925 }
926
927 static int r600_bytecode_assign_kcache_banks(struct r600_bytecode *bc,
928 struct r600_bytecode_alu *alu,
929 struct r600_bytecode_kcache * kcache)
930 {
931 int i, j;
932
933 /* Alter the src operands to refer to the kcache. */
934 for (i = 0; i < 3; ++i) {
935 static const unsigned int base[] = {128, 160, 256, 288};
936 unsigned int line, sel = alu->src[i].sel, found = 0;
937
938 if (sel < 512)
939 continue;
940
941 sel -= 512;
942 line = sel>>4;
943
944 for (j = 0; j < 4 && !found; ++j) {
945 switch (kcache[j].mode) {
946 case V_SQ_CF_KCACHE_NOP:
947 case V_SQ_CF_KCACHE_LOCK_LOOP_INDEX:
948 R600_ERR("unexpected kcache line mode\n");
949 return -ENOMEM;
950 default:
951 if (kcache[j].bank == alu->src[i].kc_bank &&
952 kcache[j].addr <= line &&
953 line < kcache[j].addr + kcache[j].mode) {
954 alu->src[i].sel = sel - (kcache[j].addr<<4);
955 alu->src[i].sel += base[j];
956 found=1;
957 }
958 }
959 }
960 }
961 return 0;
962 }
963
964 static int r600_bytecode_alloc_kcache_lines(struct r600_bytecode *bc,
965 struct r600_bytecode_alu *alu,
966 unsigned type)
967 {
968 struct r600_bytecode_kcache kcache_sets[4];
969 struct r600_bytecode_kcache *kcache = kcache_sets;
970 int r;
971
972 memcpy(kcache, bc->cf_last->kcache, 4 * sizeof(struct r600_bytecode_kcache));
973
974 if ((r = r600_bytecode_alloc_inst_kcache_lines(bc, kcache, alu))) {
975 /* can't alloc, need to start new clause */
976 if ((r = r600_bytecode_add_cf(bc))) {
977 return r;
978 }
979 bc->cf_last->op = type;
980
981 /* retry with the new clause */
982 kcache = bc->cf_last->kcache;
983 if ((r = r600_bytecode_alloc_inst_kcache_lines(bc, kcache, alu))) {
984 /* can't alloc again- should never happen */
985 return r;
986 }
987 } else {
988 /* update kcache sets */
989 memcpy(bc->cf_last->kcache, kcache, 4 * sizeof(struct r600_bytecode_kcache));
990 }
991
992 /* if we actually used more than 2 kcache sets - use ALU_EXTENDED on eg+ */
993 if (kcache[2].mode != V_SQ_CF_KCACHE_NOP) {
994 if (bc->chip_class < EVERGREEN)
995 return -ENOMEM;
996 bc->cf_last->eg_alu_extended = 1;
997 }
998
999 return 0;
1000 }
1001
1002 static int insert_nop_r6xx(struct r600_bytecode *bc)
1003 {
1004 struct r600_bytecode_alu alu;
1005 int r, i;
1006
1007 for (i = 0; i < 4; i++) {
1008 memset(&alu, 0, sizeof(alu));
1009 alu.op = ALU_OP0_NOP;
1010 alu.src[0].chan = i;
1011 alu.dst.chan = i;
1012 alu.last = (i == 3);
1013 r = r600_bytecode_add_alu(bc, &alu);
1014 if (r)
1015 return r;
1016 }
1017 return 0;
1018 }
1019
1020 /* load AR register from gpr (bc->ar_reg) with MOVA_INT */
1021 static int load_ar_r6xx(struct r600_bytecode *bc)
1022 {
1023 struct r600_bytecode_alu alu;
1024 int r;
1025
1026 if (bc->ar_loaded)
1027 return 0;
1028
1029 /* hack to avoid making MOVA the last instruction in the clause */
1030 if ((bc->cf_last->ndw>>1) >= 110)
1031 bc->force_add_cf = 1;
1032
1033 memset(&alu, 0, sizeof(alu));
1034 alu.op = ALU_OP1_MOVA_GPR_INT;
1035 alu.src[0].sel = bc->ar_reg;
1036 alu.src[0].chan = bc->ar_chan;
1037 alu.last = 1;
1038 alu.index_mode = INDEX_MODE_LOOP;
1039 r = r600_bytecode_add_alu(bc, &alu);
1040 if (r)
1041 return r;
1042
1043 /* no requirement to set uses waterfall on MOVA_GPR_INT */
1044 bc->ar_loaded = 1;
1045 return 0;
1046 }
1047
1048 /* load AR register from gpr (bc->ar_reg) with MOVA_INT */
1049 static int load_ar(struct r600_bytecode *bc)
1050 {
1051 struct r600_bytecode_alu alu;
1052 int r;
1053
1054 if (bc->ar_handling)
1055 return load_ar_r6xx(bc);
1056
1057 if (bc->ar_loaded)
1058 return 0;
1059
1060 /* hack to avoid making MOVA the last instruction in the clause */
1061 if ((bc->cf_last->ndw>>1) >= 110)
1062 bc->force_add_cf = 1;
1063
1064 memset(&alu, 0, sizeof(alu));
1065 alu.op = ALU_OP1_MOVA_INT;
1066 alu.src[0].sel = bc->ar_reg;
1067 alu.src[0].chan = bc->ar_chan;
1068 alu.last = 1;
1069 r = r600_bytecode_add_alu(bc, &alu);
1070 if (r)
1071 return r;
1072
1073 bc->cf_last->r6xx_uses_waterfall = 1;
1074 bc->ar_loaded = 1;
1075 return 0;
1076 }
1077
1078 int r600_bytecode_add_alu_type(struct r600_bytecode *bc,
1079 const struct r600_bytecode_alu *alu, unsigned type)
1080 {
1081 struct r600_bytecode_alu *nalu = r600_bytecode_alu();
1082 struct r600_bytecode_alu *lalu;
1083 int i, r;
1084
1085 if (nalu == NULL)
1086 return -ENOMEM;
1087 memcpy(nalu, alu, sizeof(struct r600_bytecode_alu));
1088
1089 if (bc->cf_last != NULL && bc->cf_last->op != type) {
1090 /* check if we could add it anyway */
1091 if (bc->cf_last->op == CF_OP_ALU &&
1092 type == CF_OP_ALU_PUSH_BEFORE) {
1093 LIST_FOR_EACH_ENTRY(lalu, &bc->cf_last->alu, list) {
1094 if (lalu->execute_mask) {
1095 bc->force_add_cf = 1;
1096 break;
1097 }
1098 }
1099 } else
1100 bc->force_add_cf = 1;
1101 }
1102
1103 /* cf can contains only alu or only vtx or only tex */
1104 if (bc->cf_last == NULL || bc->force_add_cf) {
1105 r = r600_bytecode_add_cf(bc);
1106 if (r) {
1107 free(nalu);
1108 return r;
1109 }
1110 }
1111 bc->cf_last->op = type;
1112
1113 /* Check AR usage and load it if required */
1114 for (i = 0; i < 3; i++)
1115 if (nalu->src[i].rel && !bc->ar_loaded)
1116 load_ar(bc);
1117
1118 if (nalu->dst.rel && !bc->ar_loaded)
1119 load_ar(bc);
1120
1121 /* Setup the kcache for this ALU instruction. This will start a new
1122 * ALU clause if needed. */
1123 if ((r = r600_bytecode_alloc_kcache_lines(bc, nalu, type))) {
1124 free(nalu);
1125 return r;
1126 }
1127
1128 if (!bc->cf_last->curr_bs_head) {
1129 bc->cf_last->curr_bs_head = nalu;
1130 }
1131 /* number of gpr == the last gpr used in any alu */
1132 for (i = 0; i < 3; i++) {
1133 if (nalu->src[i].sel >= bc->ngpr && nalu->src[i].sel < 128) {
1134 bc->ngpr = nalu->src[i].sel + 1;
1135 }
1136 if (nalu->src[i].sel == V_SQ_ALU_SRC_LITERAL)
1137 r600_bytecode_special_constants(nalu->src[i].value,
1138 &nalu->src[i].sel, &nalu->src[i].neg);
1139 }
1140 if (nalu->dst.sel >= bc->ngpr) {
1141 bc->ngpr = nalu->dst.sel + 1;
1142 }
1143 LIST_ADDTAIL(&nalu->list, &bc->cf_last->alu);
1144 /* each alu use 2 dwords */
1145 bc->cf_last->ndw += 2;
1146 bc->ndw += 2;
1147
1148 /* process cur ALU instructions for bank swizzle */
1149 if (nalu->last) {
1150 uint32_t literal[4];
1151 unsigned nliteral;
1152 struct r600_bytecode_alu *slots[5];
1153 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
1154 r = assign_alu_units(bc, bc->cf_last->curr_bs_head, slots);
1155 if (r)
1156 return r;
1157
1158 if (bc->cf_last->prev_bs_head) {
1159 r = merge_inst_groups(bc, slots, bc->cf_last->prev_bs_head);
1160 if (r)
1161 return r;
1162 }
1163
1164 if (bc->cf_last->prev_bs_head) {
1165 r = replace_gpr_with_pv_ps(bc, slots, bc->cf_last->prev_bs_head);
1166 if (r)
1167 return r;
1168 }
1169
1170 r = check_and_set_bank_swizzle(bc, slots);
1171 if (r)
1172 return r;
1173
1174 for (i = 0, nliteral = 0; i < max_slots; i++) {
1175 if (slots[i]) {
1176 r = r600_bytecode_alu_nliterals(bc, slots[i], literal, &nliteral);
1177 if (r)
1178 return r;
1179 }
1180 }
1181 bc->cf_last->ndw += align(nliteral, 2);
1182
1183 /* at most 128 slots, one add alu can add 5 slots + 4 constants(2 slots)
1184 * worst case */
1185 if ((bc->cf_last->ndw >> 1) >= 120) {
1186 bc->force_add_cf = 1;
1187 }
1188
1189 bc->cf_last->prev2_bs_head = bc->cf_last->prev_bs_head;
1190 bc->cf_last->prev_bs_head = bc->cf_last->curr_bs_head;
1191 bc->cf_last->curr_bs_head = NULL;
1192 }
1193
1194 if (nalu->dst.rel && bc->r6xx_nop_after_rel_dst)
1195 insert_nop_r6xx(bc);
1196
1197 return 0;
1198 }
1199
1200 int r600_bytecode_add_alu(struct r600_bytecode *bc, const struct r600_bytecode_alu *alu)
1201 {
1202 return r600_bytecode_add_alu_type(bc, alu, CF_OP_ALU);
1203 }
1204
1205 static unsigned r600_bytecode_num_tex_and_vtx_instructions(const struct r600_bytecode *bc)
1206 {
1207 switch (bc->chip_class) {
1208 case R600:
1209 return 8;
1210
1211 case R700:
1212 case EVERGREEN:
1213 case CAYMAN:
1214 return 16;
1215
1216 default:
1217 R600_ERR("Unknown chip class %d.\n", bc->chip_class);
1218 return 8;
1219 }
1220 }
1221
1222 static inline boolean last_inst_was_not_vtx_fetch(struct r600_bytecode *bc)
1223 {
1224 return !((r600_isa_cf(bc->cf_last->op)->flags & CF_FETCH) &&
1225 (bc->chip_class == CAYMAN ||
1226 bc->cf_last->op != CF_OP_TEX));
1227 }
1228
1229 int r600_bytecode_add_vtx(struct r600_bytecode *bc, const struct r600_bytecode_vtx *vtx)
1230 {
1231 struct r600_bytecode_vtx *nvtx = r600_bytecode_vtx();
1232 int r;
1233
1234 if (nvtx == NULL)
1235 return -ENOMEM;
1236 memcpy(nvtx, vtx, sizeof(struct r600_bytecode_vtx));
1237
1238 /* cf can contains only alu or only vtx or only tex */
1239 if (bc->cf_last == NULL ||
1240 last_inst_was_not_vtx_fetch(bc) ||
1241 bc->force_add_cf) {
1242 r = r600_bytecode_add_cf(bc);
1243 if (r) {
1244 free(nvtx);
1245 return r;
1246 }
1247 switch (bc->chip_class) {
1248 case R600:
1249 case R700:
1250 case EVERGREEN:
1251 bc->cf_last->op = CF_OP_VTX;
1252 break;
1253 case CAYMAN:
1254 bc->cf_last->op = CF_OP_TEX;
1255 break;
1256 default:
1257 R600_ERR("Unknown chip class %d.\n", bc->chip_class);
1258 free(nvtx);
1259 return -EINVAL;
1260 }
1261 }
1262 LIST_ADDTAIL(&nvtx->list, &bc->cf_last->vtx);
1263 /* each fetch use 4 dwords */
1264 bc->cf_last->ndw += 4;
1265 bc->ndw += 4;
1266 if ((bc->cf_last->ndw / 4) >= r600_bytecode_num_tex_and_vtx_instructions(bc))
1267 bc->force_add_cf = 1;
1268
1269 bc->ngpr = MAX2(bc->ngpr, vtx->src_gpr + 1);
1270 bc->ngpr = MAX2(bc->ngpr, vtx->dst_gpr + 1);
1271
1272 return 0;
1273 }
1274
1275 int r600_bytecode_add_tex(struct r600_bytecode *bc, const struct r600_bytecode_tex *tex)
1276 {
1277 struct r600_bytecode_tex *ntex = r600_bytecode_tex();
1278 int r;
1279
1280 if (ntex == NULL)
1281 return -ENOMEM;
1282 memcpy(ntex, tex, sizeof(struct r600_bytecode_tex));
1283
1284 /* we can't fetch data und use it as texture lookup address in the same TEX clause */
1285 if (bc->cf_last != NULL &&
1286 bc->cf_last->op == CF_OP_TEX) {
1287 struct r600_bytecode_tex *ttex;
1288 LIST_FOR_EACH_ENTRY(ttex, &bc->cf_last->tex, list) {
1289 if (ttex->dst_gpr == ntex->src_gpr) {
1290 bc->force_add_cf = 1;
1291 break;
1292 }
1293 }
1294 /* slight hack to make gradients always go into same cf */
1295 if (ntex->op == FETCH_OP_SET_GRADIENTS_H)
1296 bc->force_add_cf = 1;
1297 }
1298
1299 /* cf can contains only alu or only vtx or only tex */
1300 if (bc->cf_last == NULL ||
1301 bc->cf_last->op != CF_OP_TEX ||
1302 bc->force_add_cf) {
1303 r = r600_bytecode_add_cf(bc);
1304 if (r) {
1305 free(ntex);
1306 return r;
1307 }
1308 bc->cf_last->op = CF_OP_TEX;
1309 }
1310 if (ntex->src_gpr >= bc->ngpr) {
1311 bc->ngpr = ntex->src_gpr + 1;
1312 }
1313 if (ntex->dst_gpr >= bc->ngpr) {
1314 bc->ngpr = ntex->dst_gpr + 1;
1315 }
1316 LIST_ADDTAIL(&ntex->list, &bc->cf_last->tex);
1317 /* each texture fetch use 4 dwords */
1318 bc->cf_last->ndw += 4;
1319 bc->ndw += 4;
1320 if ((bc->cf_last->ndw / 4) >= r600_bytecode_num_tex_and_vtx_instructions(bc))
1321 bc->force_add_cf = 1;
1322 return 0;
1323 }
1324
1325 int r600_bytecode_add_cfinst(struct r600_bytecode *bc, unsigned op)
1326 {
1327 int r;
1328 r = r600_bytecode_add_cf(bc);
1329 if (r)
1330 return r;
1331
1332 bc->cf_last->cond = V_SQ_CF_COND_ACTIVE;
1333 bc->cf_last->op = op;
1334 return 0;
1335 }
1336
1337 int cm_bytecode_add_cf_end(struct r600_bytecode *bc)
1338 {
1339 return r600_bytecode_add_cfinst(bc, CF_OP_CF_END);
1340 }
1341
1342 /* common to all 3 families */
1343 static int r600_bytecode_vtx_build(struct r600_bytecode *bc, struct r600_bytecode_vtx *vtx, unsigned id)
1344 {
1345 bc->bytecode[id] = S_SQ_VTX_WORD0_BUFFER_ID(vtx->buffer_id) |
1346 S_SQ_VTX_WORD0_FETCH_TYPE(vtx->fetch_type) |
1347 S_SQ_VTX_WORD0_SRC_GPR(vtx->src_gpr) |
1348 S_SQ_VTX_WORD0_SRC_SEL_X(vtx->src_sel_x);
1349 if (bc->chip_class < CAYMAN)
1350 bc->bytecode[id] |= S_SQ_VTX_WORD0_MEGA_FETCH_COUNT(vtx->mega_fetch_count);
1351 id++;
1352 bc->bytecode[id++] = S_SQ_VTX_WORD1_DST_SEL_X(vtx->dst_sel_x) |
1353 S_SQ_VTX_WORD1_DST_SEL_Y(vtx->dst_sel_y) |
1354 S_SQ_VTX_WORD1_DST_SEL_Z(vtx->dst_sel_z) |
1355 S_SQ_VTX_WORD1_DST_SEL_W(vtx->dst_sel_w) |
1356 S_SQ_VTX_WORD1_USE_CONST_FIELDS(vtx->use_const_fields) |
1357 S_SQ_VTX_WORD1_DATA_FORMAT(vtx->data_format) |
1358 S_SQ_VTX_WORD1_NUM_FORMAT_ALL(vtx->num_format_all) |
1359 S_SQ_VTX_WORD1_FORMAT_COMP_ALL(vtx->format_comp_all) |
1360 S_SQ_VTX_WORD1_SRF_MODE_ALL(vtx->srf_mode_all) |
1361 S_SQ_VTX_WORD1_GPR_DST_GPR(vtx->dst_gpr);
1362 bc->bytecode[id] = S_SQ_VTX_WORD2_OFFSET(vtx->offset)|
1363 S_SQ_VTX_WORD2_ENDIAN_SWAP(vtx->endian);
1364 if (bc->chip_class < CAYMAN)
1365 bc->bytecode[id] |= S_SQ_VTX_WORD2_MEGA_FETCH(1);
1366 id++;
1367 bc->bytecode[id++] = 0;
1368 return 0;
1369 }
1370
1371 /* common to all 3 families */
1372 static int r600_bytecode_tex_build(struct r600_bytecode *bc, struct r600_bytecode_tex *tex, unsigned id)
1373 {
1374 bc->bytecode[id++] = S_SQ_TEX_WORD0_TEX_INST(
1375 r600_isa_fetch_opcode(bc->isa->hw_class, tex->op)) |
1376 EG_S_SQ_TEX_WORD0_INST_MOD(tex->inst_mod) |
1377 S_SQ_TEX_WORD0_RESOURCE_ID(tex->resource_id) |
1378 S_SQ_TEX_WORD0_SRC_GPR(tex->src_gpr) |
1379 S_SQ_TEX_WORD0_SRC_REL(tex->src_rel);
1380 bc->bytecode[id++] = S_SQ_TEX_WORD1_DST_GPR(tex->dst_gpr) |
1381 S_SQ_TEX_WORD1_DST_REL(tex->dst_rel) |
1382 S_SQ_TEX_WORD1_DST_SEL_X(tex->dst_sel_x) |
1383 S_SQ_TEX_WORD1_DST_SEL_Y(tex->dst_sel_y) |
1384 S_SQ_TEX_WORD1_DST_SEL_Z(tex->dst_sel_z) |
1385 S_SQ_TEX_WORD1_DST_SEL_W(tex->dst_sel_w) |
1386 S_SQ_TEX_WORD1_LOD_BIAS(tex->lod_bias) |
1387 S_SQ_TEX_WORD1_COORD_TYPE_X(tex->coord_type_x) |
1388 S_SQ_TEX_WORD1_COORD_TYPE_Y(tex->coord_type_y) |
1389 S_SQ_TEX_WORD1_COORD_TYPE_Z(tex->coord_type_z) |
1390 S_SQ_TEX_WORD1_COORD_TYPE_W(tex->coord_type_w);
1391 bc->bytecode[id++] = S_SQ_TEX_WORD2_OFFSET_X(tex->offset_x) |
1392 S_SQ_TEX_WORD2_OFFSET_Y(tex->offset_y) |
1393 S_SQ_TEX_WORD2_OFFSET_Z(tex->offset_z) |
1394 S_SQ_TEX_WORD2_SAMPLER_ID(tex->sampler_id) |
1395 S_SQ_TEX_WORD2_SRC_SEL_X(tex->src_sel_x) |
1396 S_SQ_TEX_WORD2_SRC_SEL_Y(tex->src_sel_y) |
1397 S_SQ_TEX_WORD2_SRC_SEL_Z(tex->src_sel_z) |
1398 S_SQ_TEX_WORD2_SRC_SEL_W(tex->src_sel_w);
1399 bc->bytecode[id++] = 0;
1400 return 0;
1401 }
1402
1403 /* r600 only, r700/eg bits in r700_asm.c */
1404 static int r600_bytecode_alu_build(struct r600_bytecode *bc, struct r600_bytecode_alu *alu, unsigned id)
1405 {
1406 unsigned opcode = r600_isa_alu_opcode(bc->isa->hw_class, alu->op);
1407
1408 /* don't replace gpr by pv or ps for destination register */
1409 bc->bytecode[id++] = S_SQ_ALU_WORD0_SRC0_SEL(alu->src[0].sel) |
1410 S_SQ_ALU_WORD0_SRC0_REL(alu->src[0].rel) |
1411 S_SQ_ALU_WORD0_SRC0_CHAN(alu->src[0].chan) |
1412 S_SQ_ALU_WORD0_SRC0_NEG(alu->src[0].neg) |
1413 S_SQ_ALU_WORD0_SRC1_SEL(alu->src[1].sel) |
1414 S_SQ_ALU_WORD0_SRC1_REL(alu->src[1].rel) |
1415 S_SQ_ALU_WORD0_SRC1_CHAN(alu->src[1].chan) |
1416 S_SQ_ALU_WORD0_SRC1_NEG(alu->src[1].neg) |
1417 S_SQ_ALU_WORD0_INDEX_MODE(alu->index_mode) |
1418 S_SQ_ALU_WORD0_PRED_SEL(alu->pred_sel) |
1419 S_SQ_ALU_WORD0_LAST(alu->last);
1420
1421 if (alu->is_op3) {
1422 bc->bytecode[id++] = S_SQ_ALU_WORD1_DST_GPR(alu->dst.sel) |
1423 S_SQ_ALU_WORD1_DST_CHAN(alu->dst.chan) |
1424 S_SQ_ALU_WORD1_DST_REL(alu->dst.rel) |
1425 S_SQ_ALU_WORD1_CLAMP(alu->dst.clamp) |
1426 S_SQ_ALU_WORD1_OP3_SRC2_SEL(alu->src[2].sel) |
1427 S_SQ_ALU_WORD1_OP3_SRC2_REL(alu->src[2].rel) |
1428 S_SQ_ALU_WORD1_OP3_SRC2_CHAN(alu->src[2].chan) |
1429 S_SQ_ALU_WORD1_OP3_SRC2_NEG(alu->src[2].neg) |
1430 S_SQ_ALU_WORD1_OP3_ALU_INST(opcode) |
1431 S_SQ_ALU_WORD1_BANK_SWIZZLE(alu->bank_swizzle);
1432 } else {
1433 bc->bytecode[id++] = S_SQ_ALU_WORD1_DST_GPR(alu->dst.sel) |
1434 S_SQ_ALU_WORD1_DST_CHAN(alu->dst.chan) |
1435 S_SQ_ALU_WORD1_DST_REL(alu->dst.rel) |
1436 S_SQ_ALU_WORD1_CLAMP(alu->dst.clamp) |
1437 S_SQ_ALU_WORD1_OP2_SRC0_ABS(alu->src[0].abs) |
1438 S_SQ_ALU_WORD1_OP2_SRC1_ABS(alu->src[1].abs) |
1439 S_SQ_ALU_WORD1_OP2_WRITE_MASK(alu->dst.write) |
1440 S_SQ_ALU_WORD1_OP2_OMOD(alu->omod) |
1441 S_SQ_ALU_WORD1_OP2_ALU_INST(opcode) |
1442 S_SQ_ALU_WORD1_BANK_SWIZZLE(alu->bank_swizzle) |
1443 S_SQ_ALU_WORD1_OP2_UPDATE_EXECUTE_MASK(alu->execute_mask) |
1444 S_SQ_ALU_WORD1_OP2_UPDATE_PRED(alu->update_pred);
1445 }
1446 return 0;
1447 }
1448
1449 static void r600_bytecode_cf_vtx_build(uint32_t *bytecode, const struct r600_bytecode_cf *cf)
1450 {
1451 *bytecode++ = S_SQ_CF_WORD0_ADDR(cf->addr >> 1);
1452 *bytecode++ = S_SQ_CF_WORD1_CF_INST(r600_isa_cf_opcode(ISA_CC_R600, cf->op)) |
1453 S_SQ_CF_WORD1_BARRIER(1) |
1454 S_SQ_CF_WORD1_COUNT((cf->ndw / 4) - 1);
1455 }
1456
1457 /* common for r600/r700 - eg in eg_asm.c */
1458 static int r600_bytecode_cf_build(struct r600_bytecode *bc, struct r600_bytecode_cf *cf)
1459 {
1460 unsigned id = cf->id;
1461 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
1462 unsigned opcode = r600_isa_cf_opcode(bc->isa->hw_class, cf->op);
1463
1464 if (cfop->flags & CF_ALU) {
1465 bc->bytecode[id++] = S_SQ_CF_ALU_WORD0_ADDR(cf->addr >> 1) |
1466 S_SQ_CF_ALU_WORD0_KCACHE_MODE0(cf->kcache[0].mode) |
1467 S_SQ_CF_ALU_WORD0_KCACHE_BANK0(cf->kcache[0].bank) |
1468 S_SQ_CF_ALU_WORD0_KCACHE_BANK1(cf->kcache[1].bank);
1469
1470 bc->bytecode[id++] = S_SQ_CF_ALU_WORD1_CF_INST(opcode) |
1471 S_SQ_CF_ALU_WORD1_KCACHE_MODE1(cf->kcache[1].mode) |
1472 S_SQ_CF_ALU_WORD1_KCACHE_ADDR0(cf->kcache[0].addr) |
1473 S_SQ_CF_ALU_WORD1_KCACHE_ADDR1(cf->kcache[1].addr) |
1474 S_SQ_CF_ALU_WORD1_BARRIER(1) |
1475 S_SQ_CF_ALU_WORD1_USES_WATERFALL(bc->chip_class == R600 ? cf->r6xx_uses_waterfall : 0) |
1476 S_SQ_CF_ALU_WORD1_COUNT((cf->ndw / 2) - 1);
1477 } else if (cfop->flags & CF_FETCH) {
1478 if (bc->chip_class == R700)
1479 r700_bytecode_cf_vtx_build(&bc->bytecode[id], cf);
1480 else
1481 r600_bytecode_cf_vtx_build(&bc->bytecode[id], cf);
1482 } else if (cfop->flags & CF_EXP) {
1483 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD0_RW_GPR(cf->output.gpr) |
1484 S_SQ_CF_ALLOC_EXPORT_WORD0_ELEM_SIZE(cf->output.elem_size) |
1485 S_SQ_CF_ALLOC_EXPORT_WORD0_ARRAY_BASE(cf->output.array_base) |
1486 S_SQ_CF_ALLOC_EXPORT_WORD0_TYPE(cf->output.type);
1487 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD1_BURST_COUNT(cf->output.burst_count - 1) |
1488 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_X(cf->output.swizzle_x) |
1489 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Y(cf->output.swizzle_y) |
1490 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Z(cf->output.swizzle_z) |
1491 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_W(cf->output.swizzle_w) |
1492 S_SQ_CF_ALLOC_EXPORT_WORD1_BARRIER(cf->output.barrier) |
1493 S_SQ_CF_ALLOC_EXPORT_WORD1_CF_INST(opcode) |
1494 S_SQ_CF_ALLOC_EXPORT_WORD1_END_OF_PROGRAM(cf->output.end_of_program);
1495 } else if (cfop->flags & CF_STRM) {
1496 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD0_RW_GPR(cf->output.gpr) |
1497 S_SQ_CF_ALLOC_EXPORT_WORD0_ELEM_SIZE(cf->output.elem_size) |
1498 S_SQ_CF_ALLOC_EXPORT_WORD0_ARRAY_BASE(cf->output.array_base) |
1499 S_SQ_CF_ALLOC_EXPORT_WORD0_TYPE(cf->output.type);
1500 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD1_BURST_COUNT(cf->output.burst_count - 1) |
1501 S_SQ_CF_ALLOC_EXPORT_WORD1_BARRIER(cf->output.barrier) |
1502 S_SQ_CF_ALLOC_EXPORT_WORD1_CF_INST(opcode) |
1503 S_SQ_CF_ALLOC_EXPORT_WORD1_END_OF_PROGRAM(cf->output.end_of_program) |
1504 S_SQ_CF_ALLOC_EXPORT_WORD1_BUF_ARRAY_SIZE(cf->output.array_size) |
1505 S_SQ_CF_ALLOC_EXPORT_WORD1_BUF_COMP_MASK(cf->output.comp_mask);
1506 } else {
1507 bc->bytecode[id++] = S_SQ_CF_WORD0_ADDR(cf->cf_addr >> 1);
1508 bc->bytecode[id++] = S_SQ_CF_WORD1_CF_INST(opcode) |
1509 S_SQ_CF_WORD1_BARRIER(1) |
1510 S_SQ_CF_WORD1_COND(cf->cond) |
1511 S_SQ_CF_WORD1_POP_COUNT(cf->pop_count);
1512 }
1513 return 0;
1514 }
1515
1516 int r600_bytecode_build(struct r600_bytecode *bc)
1517 {
1518 struct r600_bytecode_cf *cf;
1519 struct r600_bytecode_alu *alu;
1520 struct r600_bytecode_vtx *vtx;
1521 struct r600_bytecode_tex *tex;
1522 uint32_t literal[4];
1523 unsigned nliteral;
1524 unsigned addr;
1525 int i, r;
1526
1527 if (bc->callstack[0].max > 0)
1528 bc->nstack = ((bc->callstack[0].max + 3) >> 2) + 2;
1529 if (bc->type == TGSI_PROCESSOR_VERTEX && !bc->nstack) {
1530 bc->nstack = 1;
1531 }
1532
1533 /* first path compute addr of each CF block */
1534 /* addr start after all the CF instructions */
1535 addr = bc->cf_last->id + 2;
1536 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
1537 if (r600_isa_cf(cf->op)->flags & CF_FETCH) {
1538 addr += 3;
1539 addr &= 0xFFFFFFFCUL;
1540 }
1541 cf->addr = addr;
1542 addr += cf->ndw;
1543 bc->ndw = cf->addr + cf->ndw;
1544 }
1545 free(bc->bytecode);
1546 bc->bytecode = calloc(1, bc->ndw * 4);
1547 if (bc->bytecode == NULL)
1548 return -ENOMEM;
1549 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
1550 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
1551 addr = cf->addr;
1552 if (bc->chip_class >= EVERGREEN)
1553 r = eg_bytecode_cf_build(bc, cf);
1554 else
1555 r = r600_bytecode_cf_build(bc, cf);
1556 if (r)
1557 return r;
1558 if (cfop->flags & CF_ALU) {
1559 nliteral = 0;
1560 memset(literal, 0, sizeof(literal));
1561 LIST_FOR_EACH_ENTRY(alu, &cf->alu, list) {
1562 r = r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
1563 if (r)
1564 return r;
1565 r600_bytecode_alu_adjust_literals(bc, alu, literal, nliteral);
1566 r600_bytecode_assign_kcache_banks(bc, alu, cf->kcache);
1567
1568 switch(bc->chip_class) {
1569 case R600:
1570 r = r600_bytecode_alu_build(bc, alu, addr);
1571 break;
1572 case R700:
1573 case EVERGREEN: /* eg alu is same encoding as r700 */
1574 case CAYMAN:
1575 r = r700_bytecode_alu_build(bc, alu, addr);
1576 break;
1577 default:
1578 R600_ERR("unknown chip class %d.\n", bc->chip_class);
1579 return -EINVAL;
1580 }
1581 if (r)
1582 return r;
1583 addr += 2;
1584 if (alu->last) {
1585 for (i = 0; i < align(nliteral, 2); ++i) {
1586 bc->bytecode[addr++] = literal[i];
1587 }
1588 nliteral = 0;
1589 memset(literal, 0, sizeof(literal));
1590 }
1591 }
1592 } else if (cf->op == CF_OP_VTX) {
1593 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
1594 r = r600_bytecode_vtx_build(bc, vtx, addr);
1595 if (r)
1596 return r;
1597 addr += 4;
1598 }
1599 } else if (cf->op == CF_OP_TEX) {
1600 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
1601 assert(bc->chip_class >= EVERGREEN);
1602 r = r600_bytecode_vtx_build(bc, vtx, addr);
1603 if (r)
1604 return r;
1605 addr += 4;
1606 }
1607 LIST_FOR_EACH_ENTRY(tex, &cf->tex, list) {
1608 r = r600_bytecode_tex_build(bc, tex, addr);
1609 if (r)
1610 return r;
1611 addr += 4;
1612 }
1613 }
1614 }
1615 return 0;
1616 }
1617
1618 void r600_bytecode_clear(struct r600_bytecode *bc)
1619 {
1620 struct r600_bytecode_cf *cf = NULL, *next_cf;
1621
1622 free(bc->bytecode);
1623 bc->bytecode = NULL;
1624
1625 LIST_FOR_EACH_ENTRY_SAFE(cf, next_cf, &bc->cf, list) {
1626 struct r600_bytecode_alu *alu = NULL, *next_alu;
1627 struct r600_bytecode_tex *tex = NULL, *next_tex;
1628 struct r600_bytecode_tex *vtx = NULL, *next_vtx;
1629
1630 LIST_FOR_EACH_ENTRY_SAFE(alu, next_alu, &cf->alu, list) {
1631 free(alu);
1632 }
1633
1634 LIST_INITHEAD(&cf->alu);
1635
1636 LIST_FOR_EACH_ENTRY_SAFE(tex, next_tex, &cf->tex, list) {
1637 free(tex);
1638 }
1639
1640 LIST_INITHEAD(&cf->tex);
1641
1642 LIST_FOR_EACH_ENTRY_SAFE(vtx, next_vtx, &cf->vtx, list) {
1643 free(vtx);
1644 }
1645
1646 LIST_INITHEAD(&cf->vtx);
1647
1648 free(cf);
1649 }
1650
1651 LIST_INITHEAD(&cf->list);
1652 }
1653
1654 void r600_bytecode_dump(struct r600_bytecode *bc)
1655 {
1656 struct r600_bytecode_cf *cf = NULL;
1657 struct r600_bytecode_alu *alu = NULL;
1658 struct r600_bytecode_vtx *vtx = NULL;
1659 struct r600_bytecode_tex *tex = NULL;
1660
1661 unsigned i, id;
1662 uint32_t literal[4];
1663 unsigned nliteral;
1664 char chip = '6';
1665
1666 switch (bc->chip_class) {
1667 case R700:
1668 chip = '7';
1669 break;
1670 case EVERGREEN:
1671 chip = 'E';
1672 break;
1673 case CAYMAN:
1674 chip = 'C';
1675 break;
1676 case R600:
1677 default:
1678 chip = '6';
1679 break;
1680 }
1681 fprintf(stderr, "bytecode %d dw -- %d gprs ---------------------\n", bc->ndw, bc->ngpr);
1682 fprintf(stderr, " %c\n", chip);
1683
1684 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
1685 id = cf->id;
1686 if (cf->op == CF_NATIVE) {
1687 fprintf(stderr, "%04d %08X CF NATIVE\n", id, bc->bytecode[id]);
1688 fprintf(stderr, "%04d %08X CF NATIVE\n", id + 1, bc->bytecode[id + 1]);
1689 } else {
1690 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
1691 if (cfop->flags & CF_ALU) {
1692 if (cf->eg_alu_extended) {
1693 fprintf(stderr, "%04d %08X ALU_EXT0 ", id, bc->bytecode[id]);
1694 fprintf(stderr, "KCACHE_BANK2:%X ", cf->kcache[2].bank);
1695 fprintf(stderr, "KCACHE_BANK3:%X ", cf->kcache[3].bank);
1696 fprintf(stderr, "KCACHE_MODE2:%X\n", cf->kcache[2].mode);
1697 id++;
1698 fprintf(stderr, "%04d %08X ALU_EXT1 ", id, bc->bytecode[id]);
1699 fprintf(stderr, "KCACHE_MODE3:%X ", cf->kcache[3].mode);
1700 fprintf(stderr, "KCACHE_ADDR2:%X ", cf->kcache[2].addr);
1701 fprintf(stderr, "KCACHE_ADDR3:%X\n", cf->kcache[3].addr);
1702 id++;
1703 }
1704
1705 fprintf(stderr, "%04d %08X ALU ", id, bc->bytecode[id]);
1706 fprintf(stderr, "ADDR:%d ", cf->addr);
1707 fprintf(stderr, "KCACHE_MODE0:%X ", cf->kcache[0].mode);
1708 fprintf(stderr, "KCACHE_BANK0:%X ", cf->kcache[0].bank);
1709 fprintf(stderr, "KCACHE_BANK1:%X\n", cf->kcache[1].bank);
1710 id++;
1711 fprintf(stderr, "%04d %08X ALU ", id, bc->bytecode[id]);
1712 fprintf(stderr, "INST: %s ", cfop->name);
1713 fprintf(stderr, "KCACHE_MODE1:%X ", cf->kcache[1].mode);
1714 fprintf(stderr, "KCACHE_ADDR0:%X ", cf->kcache[0].addr);
1715 fprintf(stderr, "KCACHE_ADDR1:%X ", cf->kcache[1].addr);
1716 fprintf(stderr, "COUNT:%d\n", cf->ndw / 2);
1717 } else if (cfop->flags & CF_FETCH) {
1718 fprintf(stderr, "%04d %08X TEX/VTX ", id, bc->bytecode[id]);
1719 fprintf(stderr, "ADDR:%d\n", cf->addr);
1720 id++;
1721 fprintf(stderr, "%04d %08X TEX/VTX ", id, bc->bytecode[id]);
1722 fprintf(stderr, "INST: %s ", cfop->name);
1723 fprintf(stderr, "COUNT:%d\n", cf->ndw / 4);
1724 } else if (cfop->flags & CF_EXP) {
1725 fprintf(stderr, "%04d %08X %s ", id, bc->bytecode[id],
1726 cfop->name);
1727 fprintf(stderr, "GPR:%X ", cf->output.gpr);
1728 fprintf(stderr, "ELEM_SIZE:%X ", cf->output.elem_size);
1729 fprintf(stderr, "ARRAY_BASE:%X ", cf->output.array_base);
1730 fprintf(stderr, "TYPE:%X\n", cf->output.type);
1731 id++;
1732 fprintf(stderr, "%04d %08X %s ", id, bc->bytecode[id],
1733 cfop->name);
1734 fprintf(stderr, "SWIZ_X:%X ", cf->output.swizzle_x);
1735 fprintf(stderr, "SWIZ_Y:%X ", cf->output.swizzle_y);
1736 fprintf(stderr, "SWIZ_Z:%X ", cf->output.swizzle_z);
1737 fprintf(stderr, "SWIZ_W:%X ", cf->output.swizzle_w);
1738 fprintf(stderr, "BARRIER:%X ", cf->output.barrier);
1739 fprintf(stderr, "INST: %s ", r600_isa_cf(cf->op)->name);
1740 fprintf(stderr, "BURST_COUNT:%d ", cf->output.burst_count);
1741 fprintf(stderr, "EOP:%X\n", cf->output.end_of_program);
1742 } else if (cfop->flags & CF_STRM) {
1743 fprintf(stderr, "%04d %08X EXPORT %s ", id, bc->bytecode[id],
1744 cfop->name);
1745 fprintf(stderr, "GPR:%X ", cf->output.gpr);
1746 fprintf(stderr, "ELEM_SIZE:%i ", cf->output.elem_size);
1747 fprintf(stderr, "ARRAY_BASE:%i ", cf->output.array_base);
1748 fprintf(stderr, "TYPE:%X\n", cf->output.type);
1749 id++;
1750 fprintf(stderr, "%04d %08X EXPORT %s ", id, bc->bytecode[id],
1751 cfop->name);
1752 fprintf(stderr, "ARRAY_SIZE:%i ", cf->output.array_size);
1753 fprintf(stderr, "COMP_MASK:%X ", cf->output.comp_mask);
1754 fprintf(stderr, "BARRIER:%X ", cf->output.barrier);
1755 fprintf(stderr, "INST: %s ", cfop->name);
1756 fprintf(stderr, "BURST_COUNT:%d ", cf->output.burst_count);
1757 fprintf(stderr, "EOP:%X\n", cf->output.end_of_program);
1758
1759 } else {
1760 fprintf(stderr, "%04d %08X CF ", id, bc->bytecode[id]);
1761 fprintf(stderr, "ADDR:%d\n", cf->cf_addr);
1762 id++;
1763 fprintf(stderr, "%04d %08X CF ", id, bc->bytecode[id]);
1764 fprintf(stderr, "INST: %s ", cfop->name);
1765 fprintf(stderr, "COND:%X ", cf->cond);
1766 fprintf(stderr, "POP_COUNT:%X\n", cf->pop_count);
1767 }
1768 }
1769
1770 id = cf->addr;
1771 nliteral = 0;
1772 LIST_FOR_EACH_ENTRY(alu, &cf->alu, list) {
1773 r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
1774
1775 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1776 fprintf(stderr, " SRC0(SEL:%d ", alu->src[0].sel);
1777 fprintf(stderr, "REL:%d ", alu->src[0].rel);
1778 fprintf(stderr, "CHAN:%d ", alu->src[0].chan);
1779 fprintf(stderr, "NEG:%d) ", alu->src[0].neg);
1780 fprintf(stderr, "SRC1(SEL:%d ", alu->src[1].sel);
1781 fprintf(stderr, "REL:%d ", alu->src[1].rel);
1782 fprintf(stderr, "CHAN:%d ", alu->src[1].chan);
1783 fprintf(stderr, "NEG:%d ", alu->src[1].neg);
1784 fprintf(stderr, "IM:%d) ", alu->index_mode);
1785 fprintf(stderr, "PRED_SEL:%d ", alu->pred_sel);
1786 fprintf(stderr, "LAST:%d)\n", alu->last);
1787 id++;
1788 fprintf(stderr, "%04d %08X %c ", id, bc->bytecode[id], alu->last ? '*' : ' ');
1789 fprintf(stderr, "INST: %s ", r600_isa_alu(alu->op)->name);
1790 fprintf(stderr, "DST(SEL:%d ", alu->dst.sel);
1791 fprintf(stderr, "CHAN:%d ", alu->dst.chan);
1792 fprintf(stderr, "REL:%d ", alu->dst.rel);
1793 fprintf(stderr, "CLAMP:%d) ", alu->dst.clamp);
1794 fprintf(stderr, "BANK_SWIZZLE:%d ", alu->bank_swizzle);
1795 if (alu->is_op3) {
1796 fprintf(stderr, "SRC2(SEL:%d ", alu->src[2].sel);
1797 fprintf(stderr, "REL:%d ", alu->src[2].rel);
1798 fprintf(stderr, "CHAN:%d ", alu->src[2].chan);
1799 fprintf(stderr, "NEG:%d)\n", alu->src[2].neg);
1800 } else {
1801 fprintf(stderr, "SRC0_ABS:%d ", alu->src[0].abs);
1802 fprintf(stderr, "SRC1_ABS:%d ", alu->src[1].abs);
1803 fprintf(stderr, "WRITE_MASK:%d ", alu->dst.write);
1804 fprintf(stderr, "OMOD:%d ", alu->omod);
1805 fprintf(stderr, "EXECUTE_MASK:%d ", alu->execute_mask);
1806 fprintf(stderr, "UPDATE_PRED:%d\n", alu->update_pred);
1807 }
1808
1809 id++;
1810 if (alu->last) {
1811 for (i = 0; i < nliteral; i++, id++) {
1812 float *f = (float*)(bc->bytecode + id);
1813 fprintf(stderr, "%04d %08X\t%f (%d)\n", id, bc->bytecode[id], *f,
1814 *(bc->bytecode + id));
1815 }
1816 id += nliteral & 1;
1817 nliteral = 0;
1818 }
1819 }
1820
1821 LIST_FOR_EACH_ENTRY(tex, &cf->tex, list) {
1822 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1823 fprintf(stderr, "INST: %s ", r600_isa_fetch(tex->op)->name);
1824 fprintf(stderr, "RESOURCE_ID:%d ", tex->resource_id);
1825 fprintf(stderr, "SRC(GPR:%d ", tex->src_gpr);
1826 fprintf(stderr, "REL:%d)\n", tex->src_rel);
1827 id++;
1828 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1829 fprintf(stderr, "DST(GPR:%d ", tex->dst_gpr);
1830 fprintf(stderr, "REL:%d ", tex->dst_rel);
1831 fprintf(stderr, "SEL_X:%d ", tex->dst_sel_x);
1832 fprintf(stderr, "SEL_Y:%d ", tex->dst_sel_y);
1833 fprintf(stderr, "SEL_Z:%d ", tex->dst_sel_z);
1834 fprintf(stderr, "SEL_W:%d) ", tex->dst_sel_w);
1835 fprintf(stderr, "LOD_BIAS:%d ", tex->lod_bias);
1836 fprintf(stderr, "COORD_TYPE_X:%d ", tex->coord_type_x);
1837 fprintf(stderr, "COORD_TYPE_Y:%d ", tex->coord_type_y);
1838 fprintf(stderr, "COORD_TYPE_Z:%d ", tex->coord_type_z);
1839 fprintf(stderr, "COORD_TYPE_W:%d\n", tex->coord_type_w);
1840 id++;
1841 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1842 fprintf(stderr, "OFFSET_X:%d ", tex->offset_x);
1843 fprintf(stderr, "OFFSET_Y:%d ", tex->offset_y);
1844 fprintf(stderr, "OFFSET_Z:%d ", tex->offset_z);
1845 fprintf(stderr, "SAMPLER_ID:%d ", tex->sampler_id);
1846 fprintf(stderr, "SRC(SEL_X:%d ", tex->src_sel_x);
1847 fprintf(stderr, "SEL_Y:%d ", tex->src_sel_y);
1848 fprintf(stderr, "SEL_Z:%d ", tex->src_sel_z);
1849 fprintf(stderr, "SEL_W:%d)\n", tex->src_sel_w);
1850 id++;
1851 fprintf(stderr, "%04d %08X \n", id, bc->bytecode[id]);
1852 id++;
1853 }
1854
1855 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
1856 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1857 fprintf(stderr, "INST: %s ", r600_isa_fetch(vtx->op)->name);
1858 fprintf(stderr, "FETCH_TYPE:%d ", vtx->fetch_type);
1859 fprintf(stderr, "BUFFER_ID:%d\n", vtx->buffer_id);
1860 id++;
1861 /* This assumes that no semantic fetches exist */
1862 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1863 fprintf(stderr, "SRC(GPR:%d ", vtx->src_gpr);
1864 fprintf(stderr, "SEL_X:%d) ", vtx->src_sel_x);
1865 if (bc->chip_class < CAYMAN)
1866 fprintf(stderr, "MEGA_FETCH_COUNT:%d ", vtx->mega_fetch_count);
1867 else
1868 fprintf(stderr, "SEL_Y:%d) ", 0);
1869 fprintf(stderr, "DST(GPR:%d ", vtx->dst_gpr);
1870 fprintf(stderr, "SEL_X:%d ", vtx->dst_sel_x);
1871 fprintf(stderr, "SEL_Y:%d ", vtx->dst_sel_y);
1872 fprintf(stderr, "SEL_Z:%d ", vtx->dst_sel_z);
1873 fprintf(stderr, "SEL_W:%d) ", vtx->dst_sel_w);
1874 fprintf(stderr, "USE_CONST_FIELDS:%d ", vtx->use_const_fields);
1875 fprintf(stderr, "FORMAT(DATA:%d ", vtx->data_format);
1876 fprintf(stderr, "NUM:%d ", vtx->num_format_all);
1877 fprintf(stderr, "COMP:%d ", vtx->format_comp_all);
1878 fprintf(stderr, "MODE:%d)\n", vtx->srf_mode_all);
1879 id++;
1880 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
1881 fprintf(stderr, "ENDIAN:%d ", vtx->endian);
1882 fprintf(stderr, "OFFSET:%d\n", vtx->offset);
1883 /* XXX */
1884 id++;
1885 fprintf(stderr, "%04d %08X \n", id, bc->bytecode[id]);
1886 id++;
1887 }
1888 }
1889
1890 fprintf(stderr, "--------------------------------------\n");
1891 }
1892
1893 void r600_vertex_data_type(enum pipe_format pformat,
1894 unsigned *format,
1895 unsigned *num_format, unsigned *format_comp, unsigned *endian)
1896 {
1897 const struct util_format_description *desc;
1898 unsigned i;
1899
1900 *format = 0;
1901 *num_format = 0;
1902 *format_comp = 0;
1903 *endian = ENDIAN_NONE;
1904
1905 desc = util_format_description(pformat);
1906 if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN) {
1907 goto out_unknown;
1908 }
1909
1910 /* Find the first non-VOID channel. */
1911 for (i = 0; i < 4; i++) {
1912 if (desc->channel[i].type != UTIL_FORMAT_TYPE_VOID) {
1913 break;
1914 }
1915 }
1916
1917 *endian = r600_endian_swap(desc->channel[i].size);
1918
1919 switch (desc->channel[i].type) {
1920 /* Half-floats, floats, ints */
1921 case UTIL_FORMAT_TYPE_FLOAT:
1922 switch (desc->channel[i].size) {
1923 case 16:
1924 switch (desc->nr_channels) {
1925 case 1:
1926 *format = FMT_16_FLOAT;
1927 break;
1928 case 2:
1929 *format = FMT_16_16_FLOAT;
1930 break;
1931 case 3:
1932 case 4:
1933 *format = FMT_16_16_16_16_FLOAT;
1934 break;
1935 }
1936 break;
1937 case 32:
1938 switch (desc->nr_channels) {
1939 case 1:
1940 *format = FMT_32_FLOAT;
1941 break;
1942 case 2:
1943 *format = FMT_32_32_FLOAT;
1944 break;
1945 case 3:
1946 *format = FMT_32_32_32_FLOAT;
1947 break;
1948 case 4:
1949 *format = FMT_32_32_32_32_FLOAT;
1950 break;
1951 }
1952 break;
1953 default:
1954 goto out_unknown;
1955 }
1956 break;
1957 /* Unsigned ints */
1958 case UTIL_FORMAT_TYPE_UNSIGNED:
1959 /* Signed ints */
1960 case UTIL_FORMAT_TYPE_SIGNED:
1961 switch (desc->channel[i].size) {
1962 case 8:
1963 switch (desc->nr_channels) {
1964 case 1:
1965 *format = FMT_8;
1966 break;
1967 case 2:
1968 *format = FMT_8_8;
1969 break;
1970 case 3:
1971 case 4:
1972 *format = FMT_8_8_8_8;
1973 break;
1974 }
1975 break;
1976 case 10:
1977 if (desc->nr_channels != 4)
1978 goto out_unknown;
1979
1980 *format = FMT_2_10_10_10;
1981 break;
1982 case 16:
1983 switch (desc->nr_channels) {
1984 case 1:
1985 *format = FMT_16;
1986 break;
1987 case 2:
1988 *format = FMT_16_16;
1989 break;
1990 case 3:
1991 case 4:
1992 *format = FMT_16_16_16_16;
1993 break;
1994 }
1995 break;
1996 case 32:
1997 switch (desc->nr_channels) {
1998 case 1:
1999 *format = FMT_32;
2000 break;
2001 case 2:
2002 *format = FMT_32_32;
2003 break;
2004 case 3:
2005 *format = FMT_32_32_32;
2006 break;
2007 case 4:
2008 *format = FMT_32_32_32_32;
2009 break;
2010 }
2011 break;
2012 default:
2013 goto out_unknown;
2014 }
2015 break;
2016 default:
2017 goto out_unknown;
2018 }
2019
2020 if (desc->channel[i].type == UTIL_FORMAT_TYPE_SIGNED) {
2021 *format_comp = 1;
2022 }
2023
2024 *num_format = 0;
2025 if (desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED ||
2026 desc->channel[i].type == UTIL_FORMAT_TYPE_SIGNED) {
2027 if (!desc->channel[i].normalized) {
2028 if (desc->channel[i].pure_integer)
2029 *num_format = 1;
2030 else
2031 *num_format = 2;
2032 }
2033 }
2034 return;
2035 out_unknown:
2036 R600_ERR("unsupported vertex format %s\n", util_format_name(pformat));
2037 }
2038
2039 void *r600_create_vertex_fetch_shader(struct pipe_context *ctx,
2040 unsigned count,
2041 const struct pipe_vertex_element *elements)
2042 {
2043 struct r600_context *rctx = (struct r600_context *)ctx;
2044 static int dump_shaders = -1;
2045 struct r600_bytecode bc;
2046 struct r600_bytecode_vtx vtx;
2047 const struct util_format_description *desc;
2048 unsigned fetch_resource_start = rctx->chip_class >= EVERGREEN ? 0 : 160;
2049 unsigned format, num_format, format_comp, endian;
2050 uint32_t *bytecode;
2051 int i, j, r, fs_size;
2052 struct r600_fetch_shader *shader;
2053
2054 assert(count < 32);
2055
2056 memset(&bc, 0, sizeof(bc));
2057 r600_bytecode_init(&bc, rctx->chip_class, rctx->family,
2058 rctx->screen->msaa_texture_support);
2059
2060 bc.isa = rctx->isa;
2061
2062 for (i = 0; i < count; i++) {
2063 if (elements[i].instance_divisor > 1) {
2064 if (rctx->chip_class == CAYMAN) {
2065 for (j = 0; j < 4; j++) {
2066 struct r600_bytecode_alu alu;
2067 memset(&alu, 0, sizeof(alu));
2068 alu.op = ALU_OP2_MULHI_UINT;
2069 alu.src[0].sel = 0;
2070 alu.src[0].chan = 3;
2071 alu.src[1].sel = V_SQ_ALU_SRC_LITERAL;
2072 alu.src[1].value = (1ll << 32) / elements[i].instance_divisor + 1;
2073 alu.dst.sel = i + 1;
2074 alu.dst.chan = j;
2075 alu.dst.write = j == 3;
2076 alu.last = j == 3;
2077 if ((r = r600_bytecode_add_alu(&bc, &alu))) {
2078 r600_bytecode_clear(&bc);
2079 return NULL;
2080 }
2081 }
2082 } else {
2083 struct r600_bytecode_alu alu;
2084 memset(&alu, 0, sizeof(alu));
2085 alu.op = ALU_OP2_MULHI_UINT;
2086 alu.src[0].sel = 0;
2087 alu.src[0].chan = 3;
2088 alu.src[1].sel = V_SQ_ALU_SRC_LITERAL;
2089 alu.src[1].value = (1ll << 32) / elements[i].instance_divisor + 1;
2090 alu.dst.sel = i + 1;
2091 alu.dst.chan = 3;
2092 alu.dst.write = 1;
2093 alu.last = 1;
2094 if ((r = r600_bytecode_add_alu(&bc, &alu))) {
2095 r600_bytecode_clear(&bc);
2096 return NULL;
2097 }
2098 }
2099 }
2100 }
2101
2102 for (i = 0; i < count; i++) {
2103 r600_vertex_data_type(elements[i].src_format,
2104 &format, &num_format, &format_comp, &endian);
2105
2106 desc = util_format_description(elements[i].src_format);
2107 if (desc == NULL) {
2108 r600_bytecode_clear(&bc);
2109 R600_ERR("unknown format %d\n", elements[i].src_format);
2110 return NULL;
2111 }
2112
2113 if (elements[i].src_offset > 65535) {
2114 r600_bytecode_clear(&bc);
2115 R600_ERR("too big src_offset: %u\n", elements[i].src_offset);
2116 return NULL;
2117 }
2118
2119 memset(&vtx, 0, sizeof(vtx));
2120 vtx.buffer_id = elements[i].vertex_buffer_index + fetch_resource_start;
2121 vtx.fetch_type = elements[i].instance_divisor ? 1 : 0;
2122 vtx.src_gpr = elements[i].instance_divisor > 1 ? i + 1 : 0;
2123 vtx.src_sel_x = elements[i].instance_divisor ? 3 : 0;
2124 vtx.mega_fetch_count = 0x1F;
2125 vtx.dst_gpr = i + 1;
2126 vtx.dst_sel_x = desc->swizzle[0];
2127 vtx.dst_sel_y = desc->swizzle[1];
2128 vtx.dst_sel_z = desc->swizzle[2];
2129 vtx.dst_sel_w = desc->swizzle[3];
2130 vtx.data_format = format;
2131 vtx.num_format_all = num_format;
2132 vtx.format_comp_all = format_comp;
2133 vtx.srf_mode_all = 1;
2134 vtx.offset = elements[i].src_offset;
2135 vtx.endian = endian;
2136
2137 if ((r = r600_bytecode_add_vtx(&bc, &vtx))) {
2138 r600_bytecode_clear(&bc);
2139 return NULL;
2140 }
2141 }
2142
2143 r600_bytecode_add_cfinst(&bc, CF_OP_RET);
2144
2145 if ((r = r600_bytecode_build(&bc))) {
2146 r600_bytecode_clear(&bc);
2147 return NULL;
2148 }
2149
2150 if (dump_shaders == -1)
2151 dump_shaders = debug_get_bool_option("R600_DUMP_SHADERS", FALSE);
2152
2153 if (dump_shaders) {
2154 fprintf(stderr, "--------------------------------------------------------------\n");
2155 r600_bytecode_dump(&bc);
2156 fprintf(stderr, "______________________________________________________________\n");
2157 }
2158
2159 fs_size = bc.ndw*4;
2160
2161 /* Allocate the CSO. */
2162 shader = CALLOC_STRUCT(r600_fetch_shader);
2163 if (!shader) {
2164 r600_bytecode_clear(&bc);
2165 return NULL;
2166 }
2167
2168 u_suballocator_alloc(rctx->allocator_fetch_shader, fs_size, &shader->offset,
2169 (struct pipe_resource**)&shader->buffer);
2170 if (!shader->buffer) {
2171 r600_bytecode_clear(&bc);
2172 FREE(shader);
2173 return NULL;
2174 }
2175
2176 bytecode = r600_buffer_mmap_sync_with_rings(rctx, shader->buffer, PIPE_TRANSFER_WRITE | PIPE_TRANSFER_UNSYNCHRONIZED);
2177 bytecode += shader->offset / 4;
2178
2179 if (R600_BIG_ENDIAN) {
2180 for (i = 0; i < fs_size / 4; ++i) {
2181 bytecode[i] = bswap_32(bc.bytecode[i]);
2182 }
2183 } else {
2184 memcpy(bytecode, bc.bytecode, fs_size);
2185 }
2186 rctx->ws->buffer_unmap(shader->buffer->cs_buf);
2187
2188 r600_bytecode_clear(&bc);
2189 return shader;
2190 }
2191
2192 void r600_bytecode_alu_read(struct r600_bytecode *bc,
2193 struct r600_bytecode_alu *alu, uint32_t word0, uint32_t word1)
2194 {
2195 /* WORD0 */
2196 alu->src[0].sel = G_SQ_ALU_WORD0_SRC0_SEL(word0);
2197 alu->src[0].rel = G_SQ_ALU_WORD0_SRC0_REL(word0);
2198 alu->src[0].chan = G_SQ_ALU_WORD0_SRC0_CHAN(word0);
2199 alu->src[0].neg = G_SQ_ALU_WORD0_SRC0_NEG(word0);
2200 alu->src[1].sel = G_SQ_ALU_WORD0_SRC1_SEL(word0);
2201 alu->src[1].rel = G_SQ_ALU_WORD0_SRC1_REL(word0);
2202 alu->src[1].chan = G_SQ_ALU_WORD0_SRC1_CHAN(word0);
2203 alu->src[1].neg = G_SQ_ALU_WORD0_SRC1_NEG(word0);
2204 alu->index_mode = G_SQ_ALU_WORD0_INDEX_MODE(word0);
2205 alu->pred_sel = G_SQ_ALU_WORD0_PRED_SEL(word0);
2206 alu->last = G_SQ_ALU_WORD0_LAST(word0);
2207
2208 /* WORD1 */
2209 alu->bank_swizzle = G_SQ_ALU_WORD1_BANK_SWIZZLE(word1);
2210 if (alu->bank_swizzle)
2211 alu->bank_swizzle_force = alu->bank_swizzle;
2212 alu->dst.sel = G_SQ_ALU_WORD1_DST_GPR(word1);
2213 alu->dst.rel = G_SQ_ALU_WORD1_DST_REL(word1);
2214 alu->dst.chan = G_SQ_ALU_WORD1_DST_CHAN(word1);
2215 alu->dst.clamp = G_SQ_ALU_WORD1_CLAMP(word1);
2216 if (G_SQ_ALU_WORD1_ENCODING(word1)) /*ALU_DWORD1_OP3*/
2217 {
2218 alu->is_op3 = 1;
2219 alu->src[2].sel = G_SQ_ALU_WORD1_OP3_SRC2_SEL(word1);
2220 alu->src[2].rel = G_SQ_ALU_WORD1_OP3_SRC2_REL(word1);
2221 alu->src[2].chan = G_SQ_ALU_WORD1_OP3_SRC2_CHAN(word1);
2222 alu->src[2].neg = G_SQ_ALU_WORD1_OP3_SRC2_NEG(word1);
2223 alu->op = r600_isa_alu_by_opcode(bc->isa,
2224 G_SQ_ALU_WORD1_OP3_ALU_INST(word1), /* is_op3 = */ 1);
2225
2226 }
2227 else /*ALU_DWORD1_OP2*/
2228 {
2229 alu->src[0].abs = G_SQ_ALU_WORD1_OP2_SRC0_ABS(word1);
2230 alu->src[1].abs = G_SQ_ALU_WORD1_OP2_SRC1_ABS(word1);
2231 alu->op = r600_isa_alu_by_opcode(bc->isa,
2232 G_SQ_ALU_WORD1_OP2_ALU_INST(word1), /* is_op3 = */ 0);
2233 alu->omod = G_SQ_ALU_WORD1_OP2_OMOD(word1);
2234 alu->dst.write = G_SQ_ALU_WORD1_OP2_WRITE_MASK(word1);
2235 alu->update_pred = G_SQ_ALU_WORD1_OP2_UPDATE_PRED(word1);
2236 alu->execute_mask =
2237 G_SQ_ALU_WORD1_OP2_UPDATE_EXECUTE_MASK(word1);
2238 }
2239 }
2240
2241 void r600_bytecode_export_read(struct r600_bytecode *bc,
2242 struct r600_bytecode_output *output, uint32_t word0, uint32_t word1)
2243 {
2244 output->array_base = G_SQ_CF_ALLOC_EXPORT_WORD0_ARRAY_BASE(word0);
2245 output->type = G_SQ_CF_ALLOC_EXPORT_WORD0_TYPE(word0);
2246 output->gpr = G_SQ_CF_ALLOC_EXPORT_WORD0_RW_GPR(word0);
2247 output->elem_size = G_SQ_CF_ALLOC_EXPORT_WORD0_ELEM_SIZE(word0);
2248
2249 output->swizzle_x = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_X(word1);
2250 output->swizzle_y = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Y(word1);
2251 output->swizzle_z = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Z(word1);
2252 output->swizzle_w = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_W(word1);
2253 output->burst_count = G_SQ_CF_ALLOC_EXPORT_WORD1_BURST_COUNT(word1);
2254 output->end_of_program = G_SQ_CF_ALLOC_EXPORT_WORD1_END_OF_PROGRAM(word1);
2255 output->op = r600_isa_cf_by_opcode(bc->isa,
2256 G_SQ_CF_ALLOC_EXPORT_WORD1_CF_INST(word1), 0);
2257 output->barrier = G_SQ_CF_ALLOC_EXPORT_WORD1_BARRIER(word1);
2258 output->array_size = G_SQ_CF_ALLOC_EXPORT_WORD1_BUF_ARRAY_SIZE(word1);
2259 output->comp_mask = G_SQ_CF_ALLOC_EXPORT_WORD1_BUF_COMP_MASK(word1);
2260 }