projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
r600g: fix lockup when hyperz & alpha test are enabled together. v3
[mesa.git] / src / gallium / drivers / r600 / r600_asm.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #include "r600_sq.h"
24 #include "r600_opcodes.h"
25 #include "r600_formats.h"
26 #include "r600_shader.h"
27 #include "r600d.h"
28
29 #include <errno.h>
30 #include <byteswap.h>
31 #include "util/u_memory.h"
32 #include "pipe/p_shader_tokens.h"
33
34 #define NUM_OF_CYCLES 3
35 #define NUM_OF_COMPONENTS 4
36
37 static inline unsigned int r600_bytecode_get_num_operands(
38 struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
39 {
40 return r600_isa_alu(alu->op)->src_count;
41 }
42
43 int r700_bytecode_alu_build(struct r600_bytecode *bc,
44 struct r600_bytecode_alu *alu, unsigned id);
45
46 static struct r600_bytecode_cf *r600_bytecode_cf(void)
47 {
48 struct r600_bytecode_cf *cf = CALLOC_STRUCT(r600_bytecode_cf);
49
50 if (cf == NULL)
51 return NULL;
52 LIST_INITHEAD(&cf->list);
53 LIST_INITHEAD(&cf->alu);
54 LIST_INITHEAD(&cf->vtx);
55 LIST_INITHEAD(&cf->tex);
56 return cf;
57 }
58
59 static struct r600_bytecode_alu *r600_bytecode_alu(void)
60 {
61 struct r600_bytecode_alu *alu = CALLOC_STRUCT(r600_bytecode_alu);
62
63 if (alu == NULL)
64 return NULL;
65 LIST_INITHEAD(&alu->list);
66 return alu;
67 }
68
69 static struct r600_bytecode_vtx *r600_bytecode_vtx(void)
70 {
71 struct r600_bytecode_vtx *vtx = CALLOC_STRUCT(r600_bytecode_vtx);
72
73 if (vtx == NULL)
74 return NULL;
75 LIST_INITHEAD(&vtx->list);
76 return vtx;
77 }
78
79 static struct r600_bytecode_tex *r600_bytecode_tex(void)
80 {
81 struct r600_bytecode_tex *tex = CALLOC_STRUCT(r600_bytecode_tex);
82
83 if (tex == NULL)
84 return NULL;
85 LIST_INITHEAD(&tex->list);
86 return tex;
87 }
88
89 void r600_bytecode_init(struct r600_bytecode *bc,
90 enum chip_class chip_class,
91 enum radeon_family family,
92 enum r600_msaa_texture_mode msaa_texture_mode)
93 {
94 if ((chip_class == R600) &&
95 (family != CHIP_RV670 && family != CHIP_RS780 && family != CHIP_RS880)) {
96 bc->ar_handling = AR_HANDLE_RV6XX;
97 bc->r6xx_nop_after_rel_dst = 1;
98 } else {
99 bc->ar_handling = AR_HANDLE_NORMAL;
100 bc->r6xx_nop_after_rel_dst = 0;
101 }
102
103 LIST_INITHEAD(&bc->cf);
104 bc->chip_class = chip_class;
105 bc->msaa_texture_mode = msaa_texture_mode;
106 }
107
108 static int r600_bytecode_add_cf(struct r600_bytecode *bc)
109 {
110 struct r600_bytecode_cf *cf = r600_bytecode_cf();
111
112 if (cf == NULL)
113 return -ENOMEM;
114 LIST_ADDTAIL(&cf->list, &bc->cf);
115 if (bc->cf_last) {
116 cf->id = bc->cf_last->id + 2;
117 if (bc->cf_last->eg_alu_extended) {
118 /* take into account extended alu size */
119 cf->id += 2;
120 bc->ndw += 2;
121 }
122 }
123 bc->cf_last = cf;
124 bc->ncf++;
125 bc->ndw += 2;
126 bc->force_add_cf = 0;
127 bc->ar_loaded = 0;
128 return 0;
129 }
130
131 int r600_bytecode_add_output(struct r600_bytecode *bc,
132 const struct r600_bytecode_output *output)
133 {
134 int r;
135
136 if (output->gpr >= bc->ngpr)
137 bc->ngpr = output->gpr + 1;
138
139 if (bc->cf_last && (bc->cf_last->op == output->op ||
140 (bc->cf_last->op == CF_OP_EXPORT &&
141 output->op == CF_OP_EXPORT_DONE)) &&
142 output->type == bc->cf_last->output.type &&
143 output->elem_size == bc->cf_last->output.elem_size &&
144 output->swizzle_x == bc->cf_last->output.swizzle_x &&
145 output->swizzle_y == bc->cf_last->output.swizzle_y &&
146 output->swizzle_z == bc->cf_last->output.swizzle_z &&
147 output->swizzle_w == bc->cf_last->output.swizzle_w &&
148 (output->burst_count + bc->cf_last->output.burst_count) <= 16) {
149
150 if ((output->gpr + output->burst_count) == bc->cf_last->output.gpr &&
151 (output->array_base + output->burst_count) == bc->cf_last->output.array_base) {
152
153 bc->cf_last->output.end_of_program |= output->end_of_program;
154 bc->cf_last->op = bc->cf_last->output.op = output->op;
155 bc->cf_last->output.gpr = output->gpr;
156 bc->cf_last->output.array_base = output->array_base;
157 bc->cf_last->output.burst_count += output->burst_count;
158 return 0;
159
160 } else if (output->gpr == (bc->cf_last->output.gpr + bc->cf_last->output.burst_count) &&
161 output->array_base == (bc->cf_last->output.array_base + bc->cf_last->output.burst_count)) {
162
163 bc->cf_last->output.end_of_program |= output->end_of_program;
164 bc->cf_last->op = bc->cf_last->output.op = output->op;
165 bc->cf_last->output.burst_count += output->burst_count;
166 return 0;
167 }
168 }
169
170 r = r600_bytecode_add_cf(bc);
171 if (r)
172 return r;
173 bc->cf_last->op = output->op;
174 memcpy(&bc->cf_last->output, output, sizeof(struct r600_bytecode_output));
175 return 0;
176 }
177
178 /* alu instructions that can ony exits once per group */
179 static int is_alu_once_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
180 {
181 return r600_isa_alu(alu->op)->flags & (AF_KILL | AF_PRED);
182 }
183
184 static int is_alu_reduction_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
185 {
186 return (r600_isa_alu(alu->op)->flags & AF_REPL) &&
187 (r600_isa_alu_slots(bc->isa->hw_class, alu->op) == AF_4V);
188 }
189
190 static int is_alu_mova_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
191 {
192 return r600_isa_alu(alu->op)->flags & AF_MOVA;
193 }
194
195 static int alu_uses_rel(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
196 {
197 unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
198 unsigned src;
199
200 if (alu->dst.rel) {
201 return 1;
202 }
203
204 for (src = 0; src < num_src; ++src) {
205 if (alu->src[src].rel) {
206 return 1;
207 }
208 }
209 return 0;
210 }
211
212 static int is_alu_vec_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
213 {
214 unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
215 return !(slots & AF_S);
216 }
217
218 static int is_alu_trans_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
219 {
220 unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
221 return !(slots & AF_V);
222 }
223
224 /* alu instructions that can execute on any unit */
225 static int is_alu_any_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
226 {
227 unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
228 return slots == AF_VS;
229 }
230
231 static int is_nop_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
232 {
233 return alu->op == ALU_OP0_NOP;
234 }
235
236 static int assign_alu_units(struct r600_bytecode *bc, struct r600_bytecode_alu *alu_first,
237 struct r600_bytecode_alu *assignment[5])
238 {
239 struct r600_bytecode_alu *alu;
240 unsigned i, chan, trans;
241 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
242
243 for (i = 0; i < max_slots; i++)
244 assignment[i] = NULL;
245
246 for (alu = alu_first; alu; alu = LIST_ENTRY(struct r600_bytecode_alu, alu->list.next, list)) {
247 chan = alu->dst.chan;
248 if (max_slots == 4)
249 trans = 0;
250 else if (is_alu_trans_unit_inst(bc, alu))
251 trans = 1;
252 else if (is_alu_vec_unit_inst(bc, alu))
253 trans = 0;
254 else if (assignment[chan])
255 trans = 1; /* Assume ALU_INST_PREFER_VECTOR. */
256 else
257 trans = 0;
258
259 if (trans) {
260 if (assignment[4]) {
261 assert(0); /* ALU.Trans has already been allocated. */
262 return -1;
263 }
264 assignment[4] = alu;
265 } else {
266 if (assignment[chan]) {
267 assert(0); /* ALU.chan has already been allocated. */
268 return -1;
269 }
270 assignment[chan] = alu;
271 }
272
273 if (alu->last)
274 break;
275 }
276 return 0;
277 }
278
279 struct alu_bank_swizzle {
280 int hw_gpr[NUM_OF_CYCLES][NUM_OF_COMPONENTS];
281 int hw_cfile_addr[4];
282 int hw_cfile_elem[4];
283 };
284
285 static const unsigned cycle_for_bank_swizzle_vec[][3] = {
286 [SQ_ALU_VEC_012] = { 0, 1, 2 },
287 [SQ_ALU_VEC_021] = { 0, 2, 1 },
288 [SQ_ALU_VEC_120] = { 1, 2, 0 },
289 [SQ_ALU_VEC_102] = { 1, 0, 2 },
290 [SQ_ALU_VEC_201] = { 2, 0, 1 },
291 [SQ_ALU_VEC_210] = { 2, 1, 0 }
292 };
293
294 static const unsigned cycle_for_bank_swizzle_scl[][3] = {
295 [SQ_ALU_SCL_210] = { 2, 1, 0 },
296 [SQ_ALU_SCL_122] = { 1, 2, 2 },
297 [SQ_ALU_SCL_212] = { 2, 1, 2 },
298 [SQ_ALU_SCL_221] = { 2, 2, 1 }
299 };
300
301 static void init_bank_swizzle(struct alu_bank_swizzle *bs)
302 {
303 int i, cycle, component;
304 /* set up gpr use */
305 for (cycle = 0; cycle < NUM_OF_CYCLES; cycle++)
306 for (component = 0; component < NUM_OF_COMPONENTS; component++)
307 bs->hw_gpr[cycle][component] = -1;
308 for (i = 0; i < 4; i++)
309 bs->hw_cfile_addr[i] = -1;
310 for (i = 0; i < 4; i++)
311 bs->hw_cfile_elem[i] = -1;
312 }
313
314 static int reserve_gpr(struct alu_bank_swizzle *bs, unsigned sel, unsigned chan, unsigned cycle)
315 {
316 if (bs->hw_gpr[cycle][chan] == -1)
317 bs->hw_gpr[cycle][chan] = sel;
318 else if (bs->hw_gpr[cycle][chan] != (int)sel) {
319 /* Another scalar operation has already used the GPR read port for the channel. */
320 return -1;
321 }
322 return 0;
323 }
324
325 static int reserve_cfile(struct r600_bytecode *bc, struct alu_bank_swizzle *bs, unsigned sel, unsigned chan)
326 {
327 int res, num_res = 4;
328 if (bc->chip_class >= R700) {
329 num_res = 2;
330 chan /= 2;
331 }
332 for (res = 0; res < num_res; ++res) {
333 if (bs->hw_cfile_addr[res] == -1) {
334 bs->hw_cfile_addr[res] = sel;
335 bs->hw_cfile_elem[res] = chan;
336 return 0;
337 } else if (bs->hw_cfile_addr[res] == sel &&
338 bs->hw_cfile_elem[res] == chan)
339 return 0; /* Read for this scalar element already reserved, nothing to do here. */
340 }
341 /* All cfile read ports are used, cannot reference vector element. */
342 return -1;
343 }
344
345 static int is_gpr(unsigned sel)
346 {
347 return (sel >= 0 && sel <= 127);
348 }
349
350 /* CB constants start at 512, and get translated to a kcache index when ALU
351 * clauses are constructed. Note that we handle kcache constants the same way
352 * as (the now gone) cfile constants, is that really required? */
353 static int is_cfile(unsigned sel)
354 {
355 return (sel > 255 && sel < 512) ||
356 (sel > 511 && sel < 4607) || /* Kcache before translation. */
357 (sel > 127 && sel < 192); /* Kcache after translation. */
358 }
359
360 static int is_const(int sel)
361 {
362 return is_cfile(sel) ||
363 (sel >= V_SQ_ALU_SRC_0 &&
364 sel <= V_SQ_ALU_SRC_LITERAL);
365 }
366
367 static int check_vector(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
368 struct alu_bank_swizzle *bs, int bank_swizzle)
369 {
370 int r, src, num_src, sel, elem, cycle;
371
372 num_src = r600_bytecode_get_num_operands(bc, alu);
373 for (src = 0; src < num_src; src++) {
374 sel = alu->src[src].sel;
375 elem = alu->src[src].chan;
376 if (is_gpr(sel)) {
377 cycle = cycle_for_bank_swizzle_vec[bank_swizzle][src];
378 if (src == 1 && sel == alu->src[0].sel && elem == alu->src[0].chan)
379 /* Nothing to do; special-case optimization,
380 * second source uses first source’s reservation. */
381 continue;
382 else {
383 r = reserve_gpr(bs, sel, elem, cycle);
384 if (r)
385 return r;
386 }
387 } else if (is_cfile(sel)) {
388 r = reserve_cfile(bc, bs, (alu->src[src].kc_bank<<16) + sel, elem);
389 if (r)
390 return r;
391 }
392 /* No restrictions on PV, PS, literal or special constants. */
393 }
394 return 0;
395 }
396
397 static int check_scalar(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
398 struct alu_bank_swizzle *bs, int bank_swizzle)
399 {
400 int r, src, num_src, const_count, sel, elem, cycle;
401
402 num_src = r600_bytecode_get_num_operands(bc, alu);
403 for (const_count = 0, src = 0; src < num_src; ++src) {
404 sel = alu->src[src].sel;
405 elem = alu->src[src].chan;
406 if (is_const(sel)) { /* Any constant, including literal and inline constants. */
407 if (const_count >= 2)
408 /* More than two references to a constant in
409 * transcendental operation. */
410 return -1;
411 else
412 const_count++;
413 }
414 if (is_cfile(sel)) {
415 r = reserve_cfile(bc, bs, (alu->src[src].kc_bank<<16) + sel, elem);
416 if (r)
417 return r;
418 }
419 }
420 for (src = 0; src < num_src; ++src) {
421 sel = alu->src[src].sel;
422 elem = alu->src[src].chan;
423 if (is_gpr(sel)) {
424 cycle = cycle_for_bank_swizzle_scl[bank_swizzle][src];
425 if (cycle < const_count)
426 /* Cycle for GPR load conflicts with
427 * constant load in transcendental operation. */
428 return -1;
429 r = reserve_gpr(bs, sel, elem, cycle);
430 if (r)
431 return r;
432 }
433 /* PV PS restrictions */
434 if (const_count && (sel == 254 || sel == 255)) {
435 cycle = cycle_for_bank_swizzle_scl[bank_swizzle][src];
436 if (cycle < const_count)
437 return -1;
438 }
439 }
440 return 0;
441 }
442
443 static int check_and_set_bank_swizzle(struct r600_bytecode *bc,
444 struct r600_bytecode_alu *slots[5])
445 {
446 struct alu_bank_swizzle bs;
447 int bank_swizzle[5];
448 int i, r = 0, forced = 1;
449 boolean scalar_only = bc->chip_class == CAYMAN ? false : true;
450 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
451
452 for (i = 0; i < max_slots; i++) {
453 if (slots[i]) {
454 if (slots[i]->bank_swizzle_force) {
455 slots[i]->bank_swizzle = slots[i]->bank_swizzle_force;
456 } else {
457 forced = 0;
458 }
459 }
460
461 if (i < 4 && slots[i])
462 scalar_only = false;
463 }
464 if (forced)
465 return 0;
466
467 /* Just check every possible combination of bank swizzle.
468 * Not very efficent, but works on the first try in most of the cases. */
469 for (i = 0; i < 4; i++)
470 if (!slots[i] || !slots[i]->bank_swizzle_force)
471 bank_swizzle[i] = SQ_ALU_VEC_012;
472 else
473 bank_swizzle[i] = slots[i]->bank_swizzle;
474
475 bank_swizzle[4] = SQ_ALU_SCL_210;
476 while(bank_swizzle[4] <= SQ_ALU_SCL_221) {
477
478 if (max_slots == 4) {
479 for (i = 0; i < max_slots; i++) {
480 if (bank_swizzle[i] == SQ_ALU_VEC_210)
481 return -1;
482 }
483 }
484 init_bank_swizzle(&bs);
485 if (scalar_only == false) {
486 for (i = 0; i < 4; i++) {
487 if (slots[i]) {
488 r = check_vector(bc, slots[i], &bs, bank_swizzle[i]);
489 if (r)
490 break;
491 }
492 }
493 } else
494 r = 0;
495
496 if (!r && slots[4] && max_slots == 5) {
497 r = check_scalar(bc, slots[4], &bs, bank_swizzle[4]);
498 }
499 if (!r) {
500 for (i = 0; i < max_slots; i++) {
501 if (slots[i])
502 slots[i]->bank_swizzle = bank_swizzle[i];
503 }
504 return 0;
505 }
506
507 if (scalar_only) {
508 bank_swizzle[4]++;
509 } else {
510 for (i = 0; i < max_slots; i++) {
511 if (!slots[i] || !slots[i]->bank_swizzle_force) {
512 bank_swizzle[i]++;
513 if (bank_swizzle[i] <= SQ_ALU_VEC_210)
514 break;
515 else
516 bank_swizzle[i] = SQ_ALU_VEC_012;
517 }
518 }
519 }
520 }
521
522 /* Couldn't find a working swizzle. */
523 return -1;
524 }
525
526 static int replace_gpr_with_pv_ps(struct r600_bytecode *bc,
527 struct r600_bytecode_alu *slots[5], struct r600_bytecode_alu *alu_prev)
528 {
529 struct r600_bytecode_alu *prev[5];
530 int gpr[5], chan[5];
531 int i, j, r, src, num_src;
532 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
533
534 r = assign_alu_units(bc, alu_prev, prev);
535 if (r)
536 return r;
537
538 for (i = 0; i < max_slots; ++i) {
539 if (prev[i] && (prev[i]->dst.write || prev[i]->is_op3) && !prev[i]->dst.rel) {
540 gpr[i] = prev[i]->dst.sel;
541 /* cube writes more than PV.X */
542 if (is_alu_reduction_inst(bc, prev[i]))
543 chan[i] = 0;
544 else
545 chan[i] = prev[i]->dst.chan;
546 } else
547 gpr[i] = -1;
548 }
549
550 for (i = 0; i < max_slots; ++i) {
551 struct r600_bytecode_alu *alu = slots[i];
552 if(!alu)
553 continue;
554
555 num_src = r600_bytecode_get_num_operands(bc, alu);
556 for (src = 0; src < num_src; ++src) {
557 if (!is_gpr(alu->src[src].sel) || alu->src[src].rel)
558 continue;
559
560 if (bc->chip_class < CAYMAN) {
561 if (alu->src[src].sel == gpr[4] &&
562 alu->src[src].chan == chan[4] &&
563 alu_prev->pred_sel == alu->pred_sel) {
564 alu->src[src].sel = V_SQ_ALU_SRC_PS;
565 alu->src[src].chan = 0;
566 continue;
567 }
568 }
569
570 for (j = 0; j < 4; ++j) {
571 if (alu->src[src].sel == gpr[j] &&
572 alu->src[src].chan == j &&
573 alu_prev->pred_sel == alu->pred_sel) {
574 alu->src[src].sel = V_SQ_ALU_SRC_PV;
575 alu->src[src].chan = chan[j];
576 break;
577 }
578 }
579 }
580 }
581
582 return 0;
583 }
584
585 void r600_bytecode_special_constants(uint32_t value, unsigned *sel, unsigned *neg)
586 {
587 switch(value) {
588 case 0:
589 *sel = V_SQ_ALU_SRC_0;
590 break;
591 case 1:
592 *sel = V_SQ_ALU_SRC_1_INT;
593 break;
594 case -1:
595 *sel = V_SQ_ALU_SRC_M_1_INT;
596 break;
597 case 0x3F800000: /* 1.0f */
598 *sel = V_SQ_ALU_SRC_1;
599 break;
600 case 0x3F000000: /* 0.5f */
601 *sel = V_SQ_ALU_SRC_0_5;
602 break;
603 case 0xBF800000: /* -1.0f */
604 *sel = V_SQ_ALU_SRC_1;
605 *neg ^= 1;
606 break;
607 case 0xBF000000: /* -0.5f */
608 *sel = V_SQ_ALU_SRC_0_5;
609 *neg ^= 1;
610 break;
611 default:
612 *sel = V_SQ_ALU_SRC_LITERAL;
613 break;
614 }
615 }
616
617 /* compute how many literal are needed */
618 static int r600_bytecode_alu_nliterals(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
619 uint32_t literal[4], unsigned *nliteral)
620 {
621 unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
622 unsigned i, j;
623
624 for (i = 0; i < num_src; ++i) {
625 if (alu->src[i].sel == V_SQ_ALU_SRC_LITERAL) {
626 uint32_t value = alu->src[i].value;
627 unsigned found = 0;
628 for (j = 0; j < *nliteral; ++j) {
629 if (literal[j] == value) {
630 found = 1;
631 break;
632 }
633 }
634 if (!found) {
635 if (*nliteral >= 4)
636 return -EINVAL;
637 literal[(*nliteral)++] = value;
638 }
639 }
640 }
641 return 0;
642 }
643
644 static void r600_bytecode_alu_adjust_literals(struct r600_bytecode *bc,
645 struct r600_bytecode_alu *alu,
646 uint32_t literal[4], unsigned nliteral)
647 {
648 unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
649 unsigned i, j;
650
651 for (i = 0; i < num_src; ++i) {
652 if (alu->src[i].sel == V_SQ_ALU_SRC_LITERAL) {
653 uint32_t value = alu->src[i].value;
654 for (j = 0; j < nliteral; ++j) {
655 if (literal[j] == value) {
656 alu->src[i].chan = j;
657 break;
658 }
659 }
660 }
661 }
662 }
663
664 static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu *slots[5],
665 struct r600_bytecode_alu *alu_prev)
666 {
667 struct r600_bytecode_alu *prev[5];
668 struct r600_bytecode_alu *result[5] = { NULL };
669
670 uint32_t literal[4], prev_literal[4];
671 unsigned nliteral = 0, prev_nliteral = 0;
672
673 int i, j, r, src, num_src;
674 int num_once_inst = 0;
675 int have_mova = 0, have_rel = 0;
676 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
677
678 r = assign_alu_units(bc, alu_prev, prev);
679 if (r)
680 return r;
681
682 for (i = 0; i < max_slots; ++i) {
683 if (prev[i]) {
684 if (prev[i]->pred_sel)
685 return 0;
686 if (is_alu_once_inst(bc, prev[i]))
687 return 0;
688 }
689 if (slots[i]) {
690 if (slots[i]->pred_sel)
691 return 0;
692 if (is_alu_once_inst(bc, slots[i]))
693 return 0;
694 }
695 }
696
697 for (i = 0; i < max_slots; ++i) {
698 struct r600_bytecode_alu *alu;
699
700 if (num_once_inst > 0)
701 return 0;
702
703 /* check number of literals */
704 if (prev[i]) {
705 if (r600_bytecode_alu_nliterals(bc, prev[i], literal, &nliteral))
706 return 0;
707 if (r600_bytecode_alu_nliterals(bc, prev[i], prev_literal, &prev_nliteral))
708 return 0;
709 if (is_alu_mova_inst(bc, prev[i])) {
710 if (have_rel)
711 return 0;
712 have_mova = 1;
713 }
714
715 if (alu_uses_rel(bc, prev[i])) {
716 if (have_mova) {
717 return 0;
718 }
719 have_rel = 1;
720 }
721
722 num_once_inst += is_alu_once_inst(bc, prev[i]);
723 }
724 if (slots[i] && r600_bytecode_alu_nliterals(bc, slots[i], literal, &nliteral))
725 return 0;
726
727 /* Let's check used slots. */
728 if (prev[i] && !slots[i]) {
729 result[i] = prev[i];
730 continue;
731 } else if (prev[i] && slots[i]) {
732 if (max_slots == 5 && result[4] == NULL && prev[4] == NULL && slots[4] == NULL) {
733 /* Trans unit is still free try to use it. */
734 if (is_alu_any_unit_inst(bc, slots[i])) {
735 result[i] = prev[i];
736 result[4] = slots[i];
737 } else if (is_alu_any_unit_inst(bc, prev[i])) {
738 if (slots[i]->dst.sel == prev[i]->dst.sel &&
739 (slots[i]->dst.write == 1 || slots[i]->is_op3) &&
740 (prev[i]->dst.write == 1 || prev[i]->is_op3))
741 return 0;
742
743 result[i] = slots[i];
744 result[4] = prev[i];
745 } else
746 return 0;
747 } else
748 return 0;
749 } else if(!slots[i]) {
750 continue;
751 } else {
752 if (max_slots == 5 && slots[i] && prev[4] &&
753 slots[i]->dst.sel == prev[4]->dst.sel &&
754 slots[i]->dst.chan == prev[4]->dst.chan &&
755 (slots[i]->dst.write == 1 || slots[i]->is_op3) &&
756 (prev[4]->dst.write == 1 || prev[4]->is_op3))
757 return 0;
758
759 result[i] = slots[i];
760 }
761
762 alu = slots[i];
763 num_once_inst += is_alu_once_inst(bc, alu);
764
765 /* don't reschedule NOPs */
766 if (is_nop_inst(bc, alu))
767 return 0;
768
769 if (is_alu_mova_inst(bc, alu)) {
770 if (have_rel) {
771 return 0;
772 }
773 have_mova = 1;
774 }
775
776 if (alu_uses_rel(bc, alu)) {
777 if (have_mova) {
778 return 0;
779 }
780 have_rel = 1;
781 }
782
783 /* Let's check source gprs */
784 num_src = r600_bytecode_get_num_operands(bc, alu);
785 for (src = 0; src < num_src; ++src) {
786
787 /* Constants don't matter. */
788 if (!is_gpr(alu->src[src].sel))
789 continue;
790
791 for (j = 0; j < max_slots; ++j) {
792 if (!prev[j] || !(prev[j]->dst.write || prev[j]->is_op3))
793 continue;
794
795 /* If it's relative then we can't determin which gpr is really used. */
796 if (prev[j]->dst.chan == alu->src[src].chan &&
797 (prev[j]->dst.sel == alu->src[src].sel ||
798 prev[j]->dst.rel || alu->src[src].rel))
799 return 0;
800 }
801 }
802 }
803
804 /* more than one PRED_ or KILL_ ? */
805 if (num_once_inst > 1)
806 return 0;
807
808 /* check if the result can still be swizzlet */
809 r = check_and_set_bank_swizzle(bc, result);
810 if (r)
811 return 0;
812
813 /* looks like everything worked out right, apply the changes */
814
815 /* undo adding previus literals */
816 bc->cf_last->ndw -= align(prev_nliteral, 2);
817
818 /* sort instructions */
819 for (i = 0; i < max_slots; ++i) {
820 slots[i] = result[i];
821 if (result[i]) {
822 LIST_DEL(&result[i]->list);
823 result[i]->last = 0;
824 LIST_ADDTAIL(&result[i]->list, &bc->cf_last->alu);
825 }
826 }
827
828 /* determine new last instruction */
829 LIST_ENTRY(struct r600_bytecode_alu, bc->cf_last->alu.prev, list)->last = 1;
830
831 /* determine new first instruction */
832 for (i = 0; i < max_slots; ++i) {
833 if (result[i]) {
834 bc->cf_last->curr_bs_head = result[i];
835 break;
836 }
837 }
838
839 bc->cf_last->prev_bs_head = bc->cf_last->prev2_bs_head;
840 bc->cf_last->prev2_bs_head = NULL;
841
842 return 0;
843 }
844
845 /* we'll keep kcache sets sorted by bank & addr */
846 static int r600_bytecode_alloc_kcache_line(struct r600_bytecode *bc,
847 struct r600_bytecode_kcache *kcache,
848 unsigned bank, unsigned line)
849 {
850 int i, kcache_banks = bc->chip_class >= EVERGREEN ? 4 : 2;
851
852 for (i = 0; i < kcache_banks; i++) {
853 if (kcache[i].mode) {
854 int d;
855
856 if (kcache[i].bank < bank)
857 continue;
858
859 if ((kcache[i].bank == bank && kcache[i].addr > line+1) ||
860 kcache[i].bank > bank) {
861 /* try to insert new line */
862 if (kcache[kcache_banks-1].mode) {
863 /* all sets are in use */
864 return -ENOMEM;
865 }
866
867 memmove(&kcache[i+1],&kcache[i], (kcache_banks-i-1)*sizeof(struct r600_bytecode_kcache));
868 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_1;
869 kcache[i].bank = bank;
870 kcache[i].addr = line;
871 return 0;
872 }
873
874 d = line - kcache[i].addr;
875
876 if (d == -1) {
877 kcache[i].addr--;
878 if (kcache[i].mode == V_SQ_CF_KCACHE_LOCK_2) {
879 /* we are prepending the line to the current set,
880 * discarding the existing second line,
881 * so we'll have to insert line+2 after it */
882 line += 2;
883 continue;
884 } else if (kcache[i].mode == V_SQ_CF_KCACHE_LOCK_1) {
885 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_2;
886 return 0;
887 } else {
888 /* V_SQ_CF_KCACHE_LOCK_LOOP_INDEX is not supported */
889 return -ENOMEM;
890 }
891 } else if (d == 1) {
892 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_2;
893 return 0;
894 } else if (d == 0)
895 return 0;
896 } else { /* free kcache set - use it */
897 kcache[i].mode = V_SQ_CF_KCACHE_LOCK_1;
898 kcache[i].bank = bank;
899 kcache[i].addr = line;
900 return 0;
901 }
902 }
903 return -ENOMEM;
904 }
905
906 static int r600_bytecode_alloc_inst_kcache_lines(struct r600_bytecode *bc,
907 struct r600_bytecode_kcache *kcache,
908 struct r600_bytecode_alu *alu)
909 {
910 int i, r;
911
912 for (i = 0; i < 3; i++) {
913 unsigned bank, line, sel = alu->src[i].sel;
914
915 if (sel < 512)
916 continue;
917
918 bank = alu->src[i].kc_bank;
919 line = (sel-512)>>4;
920
921 if ((r = r600_bytecode_alloc_kcache_line(bc, kcache, bank, line)))
922 return r;
923 }
924 return 0;
925 }
926
927 static int r600_bytecode_assign_kcache_banks(struct r600_bytecode *bc,
928 struct r600_bytecode_alu *alu,
929 struct r600_bytecode_kcache * kcache)
930 {
931 int i, j;
932
933 /* Alter the src operands to refer to the kcache. */
934 for (i = 0; i < 3; ++i) {
935 static const unsigned int base[] = {128, 160, 256, 288};
936 unsigned int line, sel = alu->src[i].sel, found = 0;
937
938 if (sel < 512)
939 continue;
940
941 sel -= 512;
942 line = sel>>4;
943
944 for (j = 0; j < 4 && !found; ++j) {
945 switch (kcache[j].mode) {
946 case V_SQ_CF_KCACHE_NOP:
947 case V_SQ_CF_KCACHE_LOCK_LOOP_INDEX:
948 R600_ERR("unexpected kcache line mode\n");
949 return -ENOMEM;
950 default:
951 if (kcache[j].bank == alu->src[i].kc_bank &&
952 kcache[j].addr <= line &&
953 line < kcache[j].addr + kcache[j].mode) {
954 alu->src[i].sel = sel - (kcache[j].addr<<4);
955 alu->src[i].sel += base[j];
956 found=1;
957 }
958 }
959 }
960 }
961 return 0;
962 }
963
964 static int r600_bytecode_alloc_kcache_lines(struct r600_bytecode *bc,
965 struct r600_bytecode_alu *alu,
966 unsigned type)
967 {
968 struct r600_bytecode_kcache kcache_sets[4];
969 struct r600_bytecode_kcache *kcache = kcache_sets;
970 int r;
971
972 memcpy(kcache, bc->cf_last->kcache, 4 * sizeof(struct r600_bytecode_kcache));
973
974 if ((r = r600_bytecode_alloc_inst_kcache_lines(bc, kcache, alu))) {
975 /* can't alloc, need to start new clause */
976 if ((r = r600_bytecode_add_cf(bc))) {
977 return r;
978 }
979 bc->cf_last->op = type;
980
981 /* retry with the new clause */
982 kcache = bc->cf_last->kcache;
983 if ((r = r600_bytecode_alloc_inst_kcache_lines(bc, kcache, alu))) {
984 /* can't alloc again- should never happen */
985 return r;
986 }
987 } else {
988 /* update kcache sets */
989 memcpy(bc->cf_last->kcache, kcache, 4 * sizeof(struct r600_bytecode_kcache));
990 }
991
992 /* if we actually used more than 2 kcache sets - use ALU_EXTENDED on eg+ */
993 if (kcache[2].mode != V_SQ_CF_KCACHE_NOP) {
994 if (bc->chip_class < EVERGREEN)
995 return -ENOMEM;
996 bc->cf_last->eg_alu_extended = 1;
997 }
998
999 return 0;
1000 }
1001
1002 static int insert_nop_r6xx(struct r600_bytecode *bc)
1003 {
1004 struct r600_bytecode_alu alu;
1005 int r, i;
1006
1007 for (i = 0; i < 4; i++) {
1008 memset(&alu, 0, sizeof(alu));
1009 alu.op = ALU_OP0_NOP;
1010 alu.src[0].chan = i;
1011 alu.dst.chan = i;
1012 alu.last = (i == 3);
1013 r = r600_bytecode_add_alu(bc, &alu);
1014 if (r)
1015 return r;
1016 }
1017 return 0;
1018 }
1019
1020 /* load AR register from gpr (bc->ar_reg) with MOVA_INT */
1021 static int load_ar_r6xx(struct r600_bytecode *bc)
1022 {
1023 struct r600_bytecode_alu alu;
1024 int r;
1025
1026 if (bc->ar_loaded)
1027 return 0;
1028
1029 /* hack to avoid making MOVA the last instruction in the clause */
1030 if ((bc->cf_last->ndw>>1) >= 110)
1031 bc->force_add_cf = 1;
1032
1033 memset(&alu, 0, sizeof(alu));
1034 alu.op = ALU_OP1_MOVA_GPR_INT;
1035 alu.src[0].sel = bc->ar_reg;
1036 alu.src[0].chan = bc->ar_chan;
1037 alu.last = 1;
1038 alu.index_mode = INDEX_MODE_LOOP;
1039 r = r600_bytecode_add_alu(bc, &alu);
1040 if (r)
1041 return r;
1042
1043 /* no requirement to set uses waterfall on MOVA_GPR_INT */
1044 bc->ar_loaded = 1;
1045 return 0;
1046 }
1047
1048 /* load AR register from gpr (bc->ar_reg) with MOVA_INT */
1049 static int load_ar(struct r600_bytecode *bc)
1050 {
1051 struct r600_bytecode_alu alu;
1052 int r;
1053
1054 if (bc->ar_handling)
1055 return load_ar_r6xx(bc);
1056
1057 if (bc->ar_loaded)
1058 return 0;
1059
1060 /* hack to avoid making MOVA the last instruction in the clause */
1061 if ((bc->cf_last->ndw>>1) >= 110)
1062 bc->force_add_cf = 1;
1063
1064 memset(&alu, 0, sizeof(alu));
1065 alu.op = ALU_OP1_MOVA_INT;
1066 alu.src[0].sel = bc->ar_reg;
1067 alu.src[0].chan = bc->ar_chan;
1068 alu.last = 1;
1069 r = r600_bytecode_add_alu(bc, &alu);
1070 if (r)
1071 return r;
1072
1073 bc->cf_last->r6xx_uses_waterfall = 1;
1074 bc->ar_loaded = 1;
1075 return 0;
1076 }
1077
1078 int r600_bytecode_add_alu_type(struct r600_bytecode *bc,
1079 const struct r600_bytecode_alu *alu, unsigned type)
1080 {
1081 struct r600_bytecode_alu *nalu = r600_bytecode_alu();
1082 struct r600_bytecode_alu *lalu;
1083 int i, r;
1084
1085 if (nalu == NULL)
1086 return -ENOMEM;
1087 memcpy(nalu, alu, sizeof(struct r600_bytecode_alu));
1088
1089 if (bc->cf_last != NULL && bc->cf_last->op != type) {
1090 /* check if we could add it anyway */
1091 if (bc->cf_last->op == CF_OP_ALU &&
1092 type == CF_OP_ALU_PUSH_BEFORE) {
1093 LIST_FOR_EACH_ENTRY(lalu, &bc->cf_last->alu, list) {
1094 if (lalu->execute_mask) {
1095 bc->force_add_cf = 1;
1096 break;
1097 }
1098 }
1099 } else
1100 bc->force_add_cf = 1;
1101 }
1102
1103 /* cf can contains only alu or only vtx or only tex */
1104 if (bc->cf_last == NULL || bc->force_add_cf) {
1105 r = r600_bytecode_add_cf(bc);
1106 if (r) {
1107 free(nalu);
1108 return r;
1109 }
1110 }
1111 bc->cf_last->op = type;
1112
1113 /* Check AR usage and load it if required */
1114 for (i = 0; i < 3; i++)
1115 if (nalu->src[i].rel && !bc->ar_loaded)
1116 load_ar(bc);
1117
1118 if (nalu->dst.rel && !bc->ar_loaded)
1119 load_ar(bc);
1120
1121 /* Setup the kcache for this ALU instruction. This will start a new
1122 * ALU clause if needed. */
1123 if ((r = r600_bytecode_alloc_kcache_lines(bc, nalu, type))) {
1124 free(nalu);
1125 return r;
1126 }
1127
1128 if (!bc->cf_last->curr_bs_head) {
1129 bc->cf_last->curr_bs_head = nalu;
1130 }
1131 /* number of gpr == the last gpr used in any alu */
1132 for (i = 0; i < 3; i++) {
1133 if (nalu->src[i].sel >= bc->ngpr && nalu->src[i].sel < 128) {
1134 bc->ngpr = nalu->src[i].sel + 1;
1135 }
1136 if (nalu->src[i].sel == V_SQ_ALU_SRC_LITERAL)
1137 r600_bytecode_special_constants(nalu->src[i].value,
1138 &nalu->src[i].sel, &nalu->src[i].neg);
1139 }
1140 if (nalu->dst.sel >= bc->ngpr) {
1141 bc->ngpr = nalu->dst.sel + 1;
1142 }
1143 LIST_ADDTAIL(&nalu->list, &bc->cf_last->alu);
1144 /* each alu use 2 dwords */
1145 bc->cf_last->ndw += 2;
1146 bc->ndw += 2;
1147
1148 /* process cur ALU instructions for bank swizzle */
1149 if (nalu->last) {
1150 uint32_t literal[4];
1151 unsigned nliteral;
1152 struct r600_bytecode_alu *slots[5];
1153 int max_slots = bc->chip_class == CAYMAN ? 4 : 5;
1154 r = assign_alu_units(bc, bc->cf_last->curr_bs_head, slots);
1155 if (r)
1156 return r;
1157
1158 if (bc->cf_last->prev_bs_head) {
1159 r = merge_inst_groups(bc, slots, bc->cf_last->prev_bs_head);
1160 if (r)
1161 return r;
1162 }
1163
1164 if (bc->cf_last->prev_bs_head) {
1165 r = replace_gpr_with_pv_ps(bc, slots, bc->cf_last->prev_bs_head);
1166 if (r)
1167 return r;
1168 }
1169
1170 r = check_and_set_bank_swizzle(bc, slots);
1171 if (r)
1172 return r;
1173
1174 for (i = 0, nliteral = 0; i < max_slots; i++) {
1175 if (slots[i]) {
1176 r = r600_bytecode_alu_nliterals(bc, slots[i], literal, &nliteral);
1177 if (r)
1178 return r;
1179 }
1180 }
1181 bc->cf_last->ndw += align(nliteral, 2);
1182
1183 /* at most 128 slots, one add alu can add 5 slots + 4 constants(2 slots)
1184 * worst case */
1185 if ((bc->cf_last->ndw >> 1) >= 120) {
1186 bc->force_add_cf = 1;
1187 }
1188
1189 bc->cf_last->prev2_bs_head = bc->cf_last->prev_bs_head;
1190 bc->cf_last->prev_bs_head = bc->cf_last->curr_bs_head;
1191 bc->cf_last->curr_bs_head = NULL;
1192 }
1193
1194 if (nalu->dst.rel && bc->r6xx_nop_after_rel_dst)
1195 insert_nop_r6xx(bc);
1196
1197 return 0;
1198 }
1199
1200 int r600_bytecode_add_alu(struct r600_bytecode *bc, const struct r600_bytecode_alu *alu)
1201 {
1202 return r600_bytecode_add_alu_type(bc, alu, CF_OP_ALU);
1203 }
1204
1205 static unsigned r600_bytecode_num_tex_and_vtx_instructions(const struct r600_bytecode *bc)
1206 {
1207 switch (bc->chip_class) {
1208 case R600:
1209 return 8;
1210
1211 case R700:
1212 case EVERGREEN:
1213 case CAYMAN:
1214 return 16;
1215
1216 default:
1217 R600_ERR("Unknown chip class %d.\n", bc->chip_class);
1218 return 8;
1219 }
1220 }
1221
1222 static inline boolean last_inst_was_not_vtx_fetch(struct r600_bytecode *bc)
1223 {
1224 return !((r600_isa_cf(bc->cf_last->op)->flags & CF_FETCH) &&
1225 (bc->chip_class == CAYMAN ||
1226 bc->cf_last->op != CF_OP_TEX));
1227 }
1228
1229 int r600_bytecode_add_vtx(struct r600_bytecode *bc, const struct r600_bytecode_vtx *vtx)
1230 {
1231 struct r600_bytecode_vtx *nvtx = r600_bytecode_vtx();
1232 int r;
1233
1234 if (nvtx == NULL)
1235 return -ENOMEM;
1236 memcpy(nvtx, vtx, sizeof(struct r600_bytecode_vtx));
1237
1238 /* cf can contains only alu or only vtx or only tex */
1239 if (bc->cf_last == NULL ||
1240 last_inst_was_not_vtx_fetch(bc) ||
1241 bc->force_add_cf) {
1242 r = r600_bytecode_add_cf(bc);
1243 if (r) {
1244 free(nvtx);
1245 return r;
1246 }
1247 switch (bc->chip_class) {
1248 case R600:
1249 case R700:
1250 case EVERGREEN:
1251 bc->cf_last->op = CF_OP_VTX;
1252 break;
1253 case CAYMAN:
1254 bc->cf_last->op = CF_OP_TEX;
1255 break;
1256 default:
1257 R600_ERR("Unknown chip class %d.\n", bc->chip_class);
1258 free(nvtx);
1259 return -EINVAL;
1260 }
1261 }
1262 LIST_ADDTAIL(&nvtx->list, &bc->cf_last->vtx);
1263 /* each fetch use 4 dwords */
1264 bc->cf_last->ndw += 4;
1265 bc->ndw += 4;
1266 if ((bc->cf_last->ndw / 4) >= r600_bytecode_num_tex_and_vtx_instructions(bc))
1267 bc->force_add_cf = 1;
1268
1269 bc->ngpr = MAX2(bc->ngpr, vtx->src_gpr + 1);
1270 bc->ngpr = MAX2(bc->ngpr, vtx->dst_gpr + 1);
1271
1272 return 0;
1273 }
1274
1275 int r600_bytecode_add_tex(struct r600_bytecode *bc, const struct r600_bytecode_tex *tex)
1276 {
1277 struct r600_bytecode_tex *ntex = r600_bytecode_tex();
1278 int r;
1279
1280 if (ntex == NULL)
1281 return -ENOMEM;
1282 memcpy(ntex, tex, sizeof(struct r600_bytecode_tex));
1283
1284 /* we can't fetch data und use it as texture lookup address in the same TEX clause */
1285 if (bc->cf_last != NULL &&
1286 bc->cf_last->op == CF_OP_TEX) {
1287 struct r600_bytecode_tex *ttex;
1288 LIST_FOR_EACH_ENTRY(ttex, &bc->cf_last->tex, list) {
1289 if (ttex->dst_gpr == ntex->src_gpr) {
1290 bc->force_add_cf = 1;
1291 break;
1292 }
1293 }
1294 /* slight hack to make gradients always go into same cf */
1295 if (ntex->op == FETCH_OP_SET_GRADIENTS_H)
1296 bc->force_add_cf = 1;
1297 }
1298
1299 /* cf can contains only alu or only vtx or only tex */
1300 if (bc->cf_last == NULL ||
1301 bc->cf_last->op != CF_OP_TEX ||
1302 bc->force_add_cf) {
1303 r = r600_bytecode_add_cf(bc);
1304 if (r) {
1305 free(ntex);
1306 return r;
1307 }
1308 bc->cf_last->op = CF_OP_TEX;
1309 }
1310 if (ntex->src_gpr >= bc->ngpr) {
1311 bc->ngpr = ntex->src_gpr + 1;
1312 }
1313 if (ntex->dst_gpr >= bc->ngpr) {
1314 bc->ngpr = ntex->dst_gpr + 1;
1315 }
1316 LIST_ADDTAIL(&ntex->list, &bc->cf_last->tex);
1317 /* each texture fetch use 4 dwords */
1318 bc->cf_last->ndw += 4;
1319 bc->ndw += 4;
1320 if ((bc->cf_last->ndw / 4) >= r600_bytecode_num_tex_and_vtx_instructions(bc))
1321 bc->force_add_cf = 1;
1322 return 0;
1323 }
1324
1325 int r600_bytecode_add_cfinst(struct r600_bytecode *bc, unsigned op)
1326 {
1327 int r;
1328 r = r600_bytecode_add_cf(bc);
1329 if (r)
1330 return r;
1331
1332 bc->cf_last->cond = V_SQ_CF_COND_ACTIVE;
1333 bc->cf_last->op = op;
1334 return 0;
1335 }
1336
1337 int cm_bytecode_add_cf_end(struct r600_bytecode *bc)
1338 {
1339 return r600_bytecode_add_cfinst(bc, CF_OP_CF_END);
1340 }
1341
1342 /* common to all 3 families */
1343 static int r600_bytecode_vtx_build(struct r600_bytecode *bc, struct r600_bytecode_vtx *vtx, unsigned id)
1344 {
1345 bc->bytecode[id] = S_SQ_VTX_WORD0_BUFFER_ID(vtx->buffer_id) |
1346 S_SQ_VTX_WORD0_FETCH_TYPE(vtx->fetch_type) |
1347 S_SQ_VTX_WORD0_SRC_GPR(vtx->src_gpr) |
1348 S_SQ_VTX_WORD0_SRC_SEL_X(vtx->src_sel_x);
1349 if (bc->chip_class < CAYMAN)
1350 bc->bytecode[id] |= S_SQ_VTX_WORD0_MEGA_FETCH_COUNT(vtx->mega_fetch_count);
1351 id++;
1352 bc->bytecode[id++] = S_SQ_VTX_WORD1_DST_SEL_X(vtx->dst_sel_x) |
1353 S_SQ_VTX_WORD1_DST_SEL_Y(vtx->dst_sel_y) |
1354 S_SQ_VTX_WORD1_DST_SEL_Z(vtx->dst_sel_z) |
1355 S_SQ_VTX_WORD1_DST_SEL_W(vtx->dst_sel_w) |
1356 S_SQ_VTX_WORD1_USE_CONST_FIELDS(vtx->use_const_fields) |
1357 S_SQ_VTX_WORD1_DATA_FORMAT(vtx->data_format) |
1358 S_SQ_VTX_WORD1_NUM_FORMAT_ALL(vtx->num_format_all) |
1359 S_SQ_VTX_WORD1_FORMAT_COMP_ALL(vtx->format_comp_all) |
1360 S_SQ_VTX_WORD1_SRF_MODE_ALL(vtx->srf_mode_all) |
1361 S_SQ_VTX_WORD1_GPR_DST_GPR(vtx->dst_gpr);
1362 bc->bytecode[id] = S_SQ_VTX_WORD2_OFFSET(vtx->offset)|
1363 S_SQ_VTX_WORD2_ENDIAN_SWAP(vtx->endian);
1364 if (bc->chip_class < CAYMAN)
1365 bc->bytecode[id] |= S_SQ_VTX_WORD2_MEGA_FETCH(1);
1366 id++;
1367 bc->bytecode[id++] = 0;
1368 return 0;
1369 }
1370
1371 /* common to all 3 families */
1372 static int r600_bytecode_tex_build(struct r600_bytecode *bc, struct r600_bytecode_tex *tex, unsigned id)
1373 {
1374 bc->bytecode[id++] = S_SQ_TEX_WORD0_TEX_INST(
1375 r600_isa_fetch_opcode(bc->isa->hw_class, tex->op)) |
1376 EG_S_SQ_TEX_WORD0_INST_MOD(tex->inst_mod) |
1377 S_SQ_TEX_WORD0_RESOURCE_ID(tex->resource_id) |
1378 S_SQ_TEX_WORD0_SRC_GPR(tex->src_gpr) |
1379 S_SQ_TEX_WORD0_SRC_REL(tex->src_rel);
1380 bc->bytecode[id++] = S_SQ_TEX_WORD1_DST_GPR(tex->dst_gpr) |
1381 S_SQ_TEX_WORD1_DST_REL(tex->dst_rel) |
1382 S_SQ_TEX_WORD1_DST_SEL_X(tex->dst_sel_x) |
1383 S_SQ_TEX_WORD1_DST_SEL_Y(tex->dst_sel_y) |
1384 S_SQ_TEX_WORD1_DST_SEL_Z(tex->dst_sel_z) |
1385 S_SQ_TEX_WORD1_DST_SEL_W(tex->dst_sel_w) |
1386 S_SQ_TEX_WORD1_LOD_BIAS(tex->lod_bias) |
1387 S_SQ_TEX_WORD1_COORD_TYPE_X(tex->coord_type_x) |
1388 S_SQ_TEX_WORD1_COORD_TYPE_Y(tex->coord_type_y) |
1389 S_SQ_TEX_WORD1_COORD_TYPE_Z(tex->coord_type_z) |
1390 S_SQ_TEX_WORD1_COORD_TYPE_W(tex->coord_type_w);
1391 bc->bytecode[id++] = S_SQ_TEX_WORD2_OFFSET_X(tex->offset_x) |
1392 S_SQ_TEX_WORD2_OFFSET_Y(tex->offset_y) |
1393 S_SQ_TEX_WORD2_OFFSET_Z(tex->offset_z) |
1394 S_SQ_TEX_WORD2_SAMPLER_ID(tex->sampler_id) |
1395 S_SQ_TEX_WORD2_SRC_SEL_X(tex->src_sel_x) |
1396 S_SQ_TEX_WORD2_SRC_SEL_Y(tex->src_sel_y) |
1397 S_SQ_TEX_WORD2_SRC_SEL_Z(tex->src_sel_z) |
1398 S_SQ_TEX_WORD2_SRC_SEL_W(tex->src_sel_w);
1399 bc->bytecode[id++] = 0;
1400 return 0;
1401 }
1402
1403 /* r600 only, r700/eg bits in r700_asm.c */
1404 static int r600_bytecode_alu_build(struct r600_bytecode *bc, struct r600_bytecode_alu *alu, unsigned id)
1405 {
1406 unsigned opcode = r600_isa_alu_opcode(bc->isa->hw_class, alu->op);
1407
1408 /* don't replace gpr by pv or ps for destination register */
1409 bc->bytecode[id++] = S_SQ_ALU_WORD0_SRC0_SEL(alu->src[0].sel) |
1410 S_SQ_ALU_WORD0_SRC0_REL(alu->src[0].rel) |
1411 S_SQ_ALU_WORD0_SRC0_CHAN(alu->src[0].chan) |
1412 S_SQ_ALU_WORD0_SRC0_NEG(alu->src[0].neg) |
1413 S_SQ_ALU_WORD0_SRC1_SEL(alu->src[1].sel) |
1414 S_SQ_ALU_WORD0_SRC1_REL(alu->src[1].rel) |
1415 S_SQ_ALU_WORD0_SRC1_CHAN(alu->src[1].chan) |
1416 S_SQ_ALU_WORD0_SRC1_NEG(alu->src[1].neg) |
1417 S_SQ_ALU_WORD0_INDEX_MODE(alu->index_mode) |
1418 S_SQ_ALU_WORD0_PRED_SEL(alu->pred_sel) |
1419 S_SQ_ALU_WORD0_LAST(alu->last);
1420
1421 if (alu->is_op3) {
1422 bc->bytecode[id++] = S_SQ_ALU_WORD1_DST_GPR(alu->dst.sel) |
1423 S_SQ_ALU_WORD1_DST_CHAN(alu->dst.chan) |
1424 S_SQ_ALU_WORD1_DST_REL(alu->dst.rel) |
1425 S_SQ_ALU_WORD1_CLAMP(alu->dst.clamp) |
1426 S_SQ_ALU_WORD1_OP3_SRC2_SEL(alu->src[2].sel) |
1427 S_SQ_ALU_WORD1_OP3_SRC2_REL(alu->src[2].rel) |
1428 S_SQ_ALU_WORD1_OP3_SRC2_CHAN(alu->src[2].chan) |
1429 S_SQ_ALU_WORD1_OP3_SRC2_NEG(alu->src[2].neg) |
1430 S_SQ_ALU_WORD1_OP3_ALU_INST(opcode) |
1431 S_SQ_ALU_WORD1_BANK_SWIZZLE(alu->bank_swizzle);
1432 } else {
1433 bc->bytecode[id++] = S_SQ_ALU_WORD1_DST_GPR(alu->dst.sel) |
1434 S_SQ_ALU_WORD1_DST_CHAN(alu->dst.chan) |
1435 S_SQ_ALU_WORD1_DST_REL(alu->dst.rel) |
1436 S_SQ_ALU_WORD1_CLAMP(alu->dst.clamp) |
1437 S_SQ_ALU_WORD1_OP2_SRC0_ABS(alu->src[0].abs) |
1438 S_SQ_ALU_WORD1_OP2_SRC1_ABS(alu->src[1].abs) |
1439 S_SQ_ALU_WORD1_OP2_WRITE_MASK(alu->dst.write) |
1440 S_SQ_ALU_WORD1_OP2_OMOD(alu->omod) |
1441 S_SQ_ALU_WORD1_OP2_ALU_INST(opcode) |
1442 S_SQ_ALU_WORD1_BANK_SWIZZLE(alu->bank_swizzle) |
1443 S_SQ_ALU_WORD1_OP2_UPDATE_EXECUTE_MASK(alu->execute_mask) |
1444 S_SQ_ALU_WORD1_OP2_UPDATE_PRED(alu->update_pred);
1445 }
1446 return 0;
1447 }
1448
1449 static void r600_bytecode_cf_vtx_build(uint32_t *bytecode, const struct r600_bytecode_cf *cf)
1450 {
1451 *bytecode++ = S_SQ_CF_WORD0_ADDR(cf->addr >> 1);
1452 *bytecode++ = S_SQ_CF_WORD1_CF_INST(r600_isa_cf_opcode(ISA_CC_R600, cf->op)) |
1453 S_SQ_CF_WORD1_BARRIER(1) |
1454 S_SQ_CF_WORD1_COUNT((cf->ndw / 4) - 1);
1455 }
1456
1457 /* common for r600/r700 - eg in eg_asm.c */
1458 static int r600_bytecode_cf_build(struct r600_bytecode *bc, struct r600_bytecode_cf *cf)
1459 {
1460 unsigned id = cf->id;
1461 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
1462 unsigned opcode = r600_isa_cf_opcode(bc->isa->hw_class, cf->op);
1463
1464 if (cfop->flags & CF_ALU) {
1465 bc->bytecode[id++] = S_SQ_CF_ALU_WORD0_ADDR(cf->addr >> 1) |
1466 S_SQ_CF_ALU_WORD0_KCACHE_MODE0(cf->kcache[0].mode) |
1467 S_SQ_CF_ALU_WORD0_KCACHE_BANK0(cf->kcache[0].bank) |
1468 S_SQ_CF_ALU_WORD0_KCACHE_BANK1(cf->kcache[1].bank);
1469
1470 bc->bytecode[id++] = S_SQ_CF_ALU_WORD1_CF_INST(opcode) |
1471 S_SQ_CF_ALU_WORD1_KCACHE_MODE1(cf->kcache[1].mode) |
1472 S_SQ_CF_ALU_WORD1_KCACHE_ADDR0(cf->kcache[0].addr) |
1473 S_SQ_CF_ALU_WORD1_KCACHE_ADDR1(cf->kcache[1].addr) |
1474 S_SQ_CF_ALU_WORD1_BARRIER(1) |
1475 S_SQ_CF_ALU_WORD1_USES_WATERFALL(bc->chip_class == R600 ? cf->r6xx_uses_waterfall : 0) |
1476 S_SQ_CF_ALU_WORD1_COUNT((cf->ndw / 2) - 1);
1477 } else if (cfop->flags & CF_FETCH) {
1478 if (bc->chip_class == R700)
1479 r700_bytecode_cf_vtx_build(&bc->bytecode[id], cf);
1480 else
1481 r600_bytecode_cf_vtx_build(&bc->bytecode[id], cf);
1482 } else if (cfop->flags & CF_EXP) {
1483 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD0_RW_GPR(cf->output.gpr) |
1484 S_SQ_CF_ALLOC_EXPORT_WORD0_ELEM_SIZE(cf->output.elem_size) |
1485 S_SQ_CF_ALLOC_EXPORT_WORD0_ARRAY_BASE(cf->output.array_base) |
1486 S_SQ_CF_ALLOC_EXPORT_WORD0_TYPE(cf->output.type);
1487 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD1_BURST_COUNT(cf->output.burst_count - 1) |
1488 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_X(cf->output.swizzle_x) |
1489 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Y(cf->output.swizzle_y) |
1490 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Z(cf->output.swizzle_z) |
1491 S_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_W(cf->output.swizzle_w) |
1492 S_SQ_CF_ALLOC_EXPORT_WORD1_BARRIER(cf->output.barrier) |
1493 S_SQ_CF_ALLOC_EXPORT_WORD1_CF_INST(opcode) |
1494 S_SQ_CF_ALLOC_EXPORT_WORD1_END_OF_PROGRAM(cf->output.end_of_program);
1495 } else if (cfop->flags & CF_STRM) {
1496 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD0_RW_GPR(cf->output.gpr) |
1497 S_SQ_CF_ALLOC_EXPORT_WORD0_ELEM_SIZE(cf->output.elem_size) |
1498 S_SQ_CF_ALLOC_EXPORT_WORD0_ARRAY_BASE(cf->output.array_base) |
1499 S_SQ_CF_ALLOC_EXPORT_WORD0_TYPE(cf->output.type);
1500 bc->bytecode[id++] = S_SQ_CF_ALLOC_EXPORT_WORD1_BURST_COUNT(cf->output.burst_count - 1) |
1501 S_SQ_CF_ALLOC_EXPORT_WORD1_BARRIER(cf->output.barrier) |
1502 S_SQ_CF_ALLOC_EXPORT_WORD1_CF_INST(opcode) |
1503 S_SQ_CF_ALLOC_EXPORT_WORD1_END_OF_PROGRAM(cf->output.end_of_program) |
1504 S_SQ_CF_ALLOC_EXPORT_WORD1_BUF_ARRAY_SIZE(cf->output.array_size) |
1505 S_SQ_CF_ALLOC_EXPORT_WORD1_BUF_COMP_MASK(cf->output.comp_mask);
1506 } else {
1507 bc->bytecode[id++] = S_SQ_CF_WORD0_ADDR(cf->cf_addr >> 1);
1508 bc->bytecode[id++] = S_SQ_CF_WORD1_CF_INST(opcode) |
1509 S_SQ_CF_WORD1_BARRIER(1) |
1510 S_SQ_CF_WORD1_COND(cf->cond) |
1511 S_SQ_CF_WORD1_POP_COUNT(cf->pop_count);
1512 }
1513 return 0;
1514 }
1515
1516 int r600_bytecode_build(struct r600_bytecode *bc)
1517 {
1518 struct r600_bytecode_cf *cf;
1519 struct r600_bytecode_alu *alu;
1520 struct r600_bytecode_vtx *vtx;
1521 struct r600_bytecode_tex *tex;
1522 uint32_t literal[4];
1523 unsigned nliteral;
1524 unsigned addr;
1525 int i, r;
1526
1527 if (bc->callstack[0].max > 0)
1528 bc->nstack = ((bc->callstack[0].max + 3) >> 2) + 2;
1529 if (bc->type == TGSI_PROCESSOR_VERTEX && !bc->nstack) {
1530 bc->nstack = 1;
1531 }
1532
1533 /* first path compute addr of each CF block */
1534 /* addr start after all the CF instructions */
1535 addr = bc->cf_last->id + 2;
1536 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
1537 if (r600_isa_cf(cf->op)->flags & CF_FETCH) {
1538 addr += 3;
1539 addr &= 0xFFFFFFFCUL;
1540 }
1541 cf->addr = addr;
1542 addr += cf->ndw;
1543 bc->ndw = cf->addr + cf->ndw;
1544 }
1545 free(bc->bytecode);
1546 bc->bytecode = calloc(1, bc->ndw * 4);
1547 if (bc->bytecode == NULL)
1548 return -ENOMEM;
1549 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
1550 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
1551 addr = cf->addr;
1552 if (bc->chip_class >= EVERGREEN)
1553 r = eg_bytecode_cf_build(bc, cf);
1554 else
1555 r = r600_bytecode_cf_build(bc, cf);
1556 if (r)
1557 return r;
1558 if (cfop->flags & CF_ALU) {
1559 nliteral = 0;
1560 memset(literal, 0, sizeof(literal));
1561 LIST_FOR_EACH_ENTRY(alu, &cf->alu, list) {
1562 r = r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
1563 if (r)
1564 return r;
1565 r600_bytecode_alu_adjust_literals(bc, alu, literal, nliteral);
1566 r600_bytecode_assign_kcache_banks(bc, alu, cf->kcache);
1567
1568 switch(bc->chip_class) {
1569 case R600:
1570 r = r600_bytecode_alu_build(bc, alu, addr);
1571 break;
1572 case R700:
1573 case EVERGREEN: /* eg alu is same encoding as r700 */
1574 case CAYMAN:
1575 r = r700_bytecode_alu_build(bc, alu, addr);
1576 break;
1577 default:
1578 R600_ERR("unknown chip class %d.\n", bc->chip_class);
1579 return -EINVAL;
1580 }
1581 if (r)
1582 return r;
1583 addr += 2;
1584 if (alu->last) {
1585 for (i = 0; i < align(nliteral, 2); ++i) {
1586 bc->bytecode[addr++] = literal[i];
1587 }
1588 nliteral = 0;
1589 memset(literal, 0, sizeof(literal));
1590 }
1591 }
1592 } else if (cf->op == CF_OP_VTX) {
1593 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
1594 r = r600_bytecode_vtx_build(bc, vtx, addr);
1595 if (r)
1596 return r;
1597 addr += 4;
1598 }
1599 } else if (cf->op == CF_OP_TEX) {
1600 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
1601 assert(bc->chip_class >= EVERGREEN);
1602 r = r600_bytecode_vtx_build(bc, vtx, addr);
1603 if (r)
1604 return r;
1605 addr += 4;
1606 }
1607 LIST_FOR_EACH_ENTRY(tex, &cf->tex, list) {
1608 r = r600_bytecode_tex_build(bc, tex, addr);
1609 if (r)
1610 return r;
1611 addr += 4;
1612 }
1613 }
1614 }
1615 return 0;
1616 }
1617
1618 void r600_bytecode_clear(struct r600_bytecode *bc)
1619 {
1620 struct r600_bytecode_cf *cf = NULL, *next_cf;
1621
1622 free(bc->bytecode);
1623 bc->bytecode = NULL;
1624
1625 LIST_FOR_EACH_ENTRY_SAFE(cf, next_cf, &bc->cf, list) {
1626 struct r600_bytecode_alu *alu = NULL, *next_alu;
1627 struct r600_bytecode_tex *tex = NULL, *next_tex;
1628 struct r600_bytecode_tex *vtx = NULL, *next_vtx;
1629
1630 LIST_FOR_EACH_ENTRY_SAFE(alu, next_alu, &cf->alu, list) {
1631 free(alu);
1632 }
1633
1634 LIST_INITHEAD(&cf->alu);
1635
1636 LIST_FOR_EACH_ENTRY_SAFE(tex, next_tex, &cf->tex, list) {
1637 free(tex);
1638 }
1639
1640 LIST_INITHEAD(&cf->tex);
1641
1642 LIST_FOR_EACH_ENTRY_SAFE(vtx, next_vtx, &cf->vtx, list) {
1643 free(vtx);
1644 }
1645
1646 LIST_INITHEAD(&cf->vtx);
1647
1648 free(cf);
1649 }
1650
1651 LIST_INITHEAD(&cf->list);
1652 }
1653
1654 static int print_swizzle(unsigned swz)
1655 {
1656 const char * swzchars = "xyzw01?_";
1657 assert(swz<8 && swz != 6);
1658 return fprintf(stderr, "%c", swzchars[swz]);
1659 }
1660
1661 static int print_sel(unsigned sel, unsigned rel, unsigned index_mode,
1662 unsigned need_brackets)
1663 {
1664 int o = 0;
1665 if (rel && index_mode >= 5 && sel < 128)
1666 o += fprintf(stderr, "G");
1667 if (rel || need_brackets) {
1668 o += fprintf(stderr, "[");
1669 }
1670 o += fprintf(stderr, "%d", sel);
1671 if (rel) {
1672 if (index_mode == 0 || index_mode == 6)
1673 o += fprintf(stderr, "+AR");
1674 else if (index_mode == 4)
1675 o += fprintf(stderr, "+AL");
1676 }
1677 if (rel || need_brackets) {
1678 o += fprintf(stderr, "]");
1679 }
1680 return o;
1681 }
1682
1683 static int print_dst(struct r600_bytecode_alu *alu)
1684 {
1685 int o = 0;
1686 unsigned sel = alu->dst.sel;
1687 char reg_char = 'R';
1688 if (sel > 128 - 4) { /* clause temporary gpr */
1689 sel -= 128 - 4;
1690 reg_char = 'T';
1691 }
1692
1693 if (alu->dst.write || alu->is_op3) {
1694 o += fprintf(stderr, "%c", reg_char);
1695 o += print_sel(alu->dst.sel, alu->dst.rel, alu->index_mode, 0);
1696 } else {
1697 o += fprintf(stderr, "__");
1698 }
1699 o += fprintf(stderr, ".");
1700 o += print_swizzle(alu->dst.chan);
1701 return o;
1702 }
1703
1704 static int print_src(struct r600_bytecode_alu *alu, unsigned idx)
1705 {
1706 int o = 0;
1707 struct r600_bytecode_alu_src *src = &alu->src[idx];
1708 unsigned sel = src->sel, need_sel = 1, need_chan = 1, need_brackets = 0;
1709
1710 if (src->neg)
1711 o += fprintf(stderr,"-");
1712 if (src->abs)
1713 o += fprintf(stderr,"|");
1714
1715 if (sel < 128 - 4) {
1716 o += fprintf(stderr, "R");
1717 } else if (sel < 128) {
1718 o += fprintf(stderr, "T");
1719 sel -= 128 - 4;
1720 } else if (sel < 160) {
1721 o += fprintf(stderr, "KC0");
1722 need_brackets = 1;
1723 sel -= 128;
1724 } else if (sel < 192) {
1725 o += fprintf(stderr, "KC1");
1726 need_brackets = 1;
1727 sel -= 160;
1728 } else if (sel >= 512) {
1729 o += fprintf(stderr, "C%d", src->kc_bank);
1730 need_brackets = 1;
1731 sel -= 512;
1732 } else if (sel >= 448) {
1733 o += fprintf(stderr, "Param");
1734 sel -= 448;
1735 need_chan = 0;
1736 } else if (sel >= 288) {
1737 o += fprintf(stderr, "KC3");
1738 need_brackets = 1;
1739 sel -= 288;
1740 } else if (sel >= 256) {
1741 o += fprintf(stderr, "KC2");
1742 need_brackets = 1;
1743 sel -= 256;
1744 } else {
1745 need_sel = 0;
1746 need_chan = 0;
1747 switch (sel) {
1748 case V_SQ_ALU_SRC_PS:
1749 o += fprintf(stderr, "PS");
1750 break;
1751 case V_SQ_ALU_SRC_PV:
1752 o += fprintf(stderr, "PV");
1753 need_chan = 1;
1754 break;
1755 case V_SQ_ALU_SRC_LITERAL:
1756 o += fprintf(stderr, "[0x%08X %f]", src->value, *(float*)&src->value);
1757 break;
1758 case V_SQ_ALU_SRC_0_5:
1759 o += fprintf(stderr, "0.5");
1760 break;
1761 case V_SQ_ALU_SRC_M_1_INT:
1762 o += fprintf(stderr, "-1");
1763 break;
1764 case V_SQ_ALU_SRC_1_INT:
1765 o += fprintf(stderr, "1");
1766 break;
1767 case V_SQ_ALU_SRC_1:
1768 o += fprintf(stderr, "1.0");
1769 break;
1770 case V_SQ_ALU_SRC_0:
1771 o += fprintf(stderr, "0");
1772 break;
1773 default:
1774 o += fprintf(stderr, "??IMM_%d", sel);
1775 break;
1776 }
1777 }
1778
1779 if (need_sel)
1780 o += print_sel(sel, src->rel, alu->index_mode, need_brackets);
1781
1782 if (need_chan) {
1783 o += fprintf(stderr, ".");
1784 o += print_swizzle(src->chan);
1785 }
1786
1787 if (src->abs)
1788 o += fprintf(stderr,"|");
1789
1790 return o;
1791 }
1792
1793 static int print_indent(int p, int c)
1794 {
1795 int o = 0;
1796 while (p++ < c)
1797 o += fprintf(stderr, " ");
1798 return o;
1799 }
1800
1801 void r600_bytecode_disasm(struct r600_bytecode *bc)
1802 {
1803 static int index = 0;
1804 struct r600_bytecode_cf *cf = NULL;
1805 struct r600_bytecode_alu *alu = NULL;
1806 struct r600_bytecode_vtx *vtx = NULL;
1807 struct r600_bytecode_tex *tex = NULL;
1808
1809 unsigned i, id, ngr = 0, last;
1810 uint32_t literal[4];
1811 unsigned nliteral;
1812 char chip = '6';
1813
1814 switch (bc->chip_class) {
1815 case R700:
1816 chip = '7';
1817 break;
1818 case EVERGREEN:
1819 chip = 'E';
1820 break;
1821 case CAYMAN:
1822 chip = 'C';
1823 break;
1824 case R600:
1825 default:
1826 chip = '6';
1827 break;
1828 }
1829 fprintf(stderr, "bytecode %d dw -- %d gprs ---------------------\n",
1830 bc->ndw, bc->ngpr);
1831 fprintf(stderr, "shader %d -- %c\n", index++, chip);
1832
1833 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
1834 id = cf->id;
1835 if (cf->op == CF_NATIVE) {
1836 fprintf(stderr, "%04d %08X %08X CF_NATIVE\n", id, bc->bytecode[id],
1837 bc->bytecode[id + 1]);
1838 } else {
1839 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
1840 if (cfop->flags & CF_ALU) {
1841 if (cf->eg_alu_extended) {
1842 fprintf(stderr, "%04d %08X %08X %s\n", id, bc->bytecode[id],
1843 bc->bytecode[id + 1], "ALU_EXT");
1844 id += 2;
1845 }
1846 fprintf(stderr, "%04d %08X %08X %s ", id, bc->bytecode[id],
1847 bc->bytecode[id + 1], cfop->name);
1848 fprintf(stderr, "%d @%d ", cf->ndw / 2, cf->addr);
1849 for (i = 0; i < 4; ++i) {
1850 if (cf->kcache[i].mode) {
1851 int c_start = (cf->kcache[i].addr << 4);
1852 int c_end = c_start + (cf->kcache[i].mode << 4);
1853 fprintf(stderr, "KC%d[CB%d:%d-%d] ",
1854 i, cf->kcache[i].bank, c_start, c_end);
1855 }
1856 }
1857 fprintf(stderr, "\n");
1858 } else if (cfop->flags & CF_FETCH) {
1859 fprintf(stderr, "%04d %08X %08X %s ", id, bc->bytecode[id],
1860 bc->bytecode[id + 1], cfop->name);
1861 fprintf(stderr, "%d @%d ", cf->ndw / 4, cf->addr);
1862 fprintf(stderr, "\n");
1863 } else if (cfop->flags & CF_EXP) {
1864 int o = 0;
1865 const char *exp_type[] = {"PIXEL", "POS ", "PARAM"};
1866 o += fprintf(stderr, "%04d %08X %08X %s ", id, bc->bytecode[id],
1867 bc->bytecode[id + 1], cfop->name);
1868 o += print_indent(o, 43);
1869 o += fprintf(stderr, "%s ", exp_type[cf->output.type]);
1870 if (cf->output.burst_count > 1) {
1871 o += fprintf(stderr, "%d-%d ", cf->output.array_base,
1872 cf->output.array_base + cf->output.burst_count - 1);
1873
1874 o += print_indent(o, 55);
1875 o += fprintf(stderr, "R%d-%d.", cf->output.gpr,
1876 cf->output.gpr + cf->output.burst_count - 1);
1877 } else {
1878 o += fprintf(stderr, "%d ", cf->output.array_base);
1879 o += print_indent(o, 55);
1880 o += fprintf(stderr, "R%d.", cf->output.gpr);
1881 }
1882
1883 o += print_swizzle(cf->output.swizzle_x);
1884 o += print_swizzle(cf->output.swizzle_y);
1885 o += print_swizzle(cf->output.swizzle_z);
1886 o += print_swizzle(cf->output.swizzle_w);
1887
1888 print_indent(o, 67);
1889
1890 fprintf(stderr, " ES:%X ", cf->output.elem_size);
1891 if (!cf->output.barrier)
1892 fprintf(stderr, "NO_BARRIER ");
1893 if (cf->output.end_of_program)
1894 fprintf(stderr, "EOP ");
1895 fprintf(stderr, "\n");
1896 } else if (r600_isa_cf(cf->op)->flags & CF_STRM) {
1897 int o = 0;
1898 const char *exp_type[] = {"WRITE", "WRITE_IND", "WRITE_ACK",
1899 "WRITE_IND_ACK"};
1900 o += fprintf(stderr, "%04d %08X %08X %s ", id,
1901 bc->bytecode[id], bc->bytecode[id + 1], cfop->name);
1902 o += print_indent(o, 43);
1903 o += fprintf(stderr, "%s ", exp_type[cf->output.type]);
1904 if (cf->output.burst_count > 1) {
1905 o += fprintf(stderr, "%d-%d ", cf->output.array_base,
1906 cf->output.array_base + cf->output.burst_count - 1);
1907 o += print_indent(o, 55);
1908 o += fprintf(stderr, "R%d-%d.", cf->output.gpr,
1909 cf->output.gpr + cf->output.burst_count - 1);
1910 } else {
1911 o += fprintf(stderr, "%d ", cf->output.array_base);
1912 o += print_indent(o, 55);
1913 o += fprintf(stderr, "R%d.", cf->output.gpr);
1914 }
1915 for (i = 0; i < 4; ++i) {
1916 if (cf->output.comp_mask & (1 << i))
1917 o += print_swizzle(i);
1918 else
1919 o += print_swizzle(7);
1920 }
1921
1922 o += print_indent(o, 67);
1923
1924 fprintf(stderr, " ES:%i ", cf->output.elem_size);
1925 if (cf->output.array_size != 0xFFF)
1926 fprintf(stderr, "AS:%i ", cf->output.array_size);
1927 if (!cf->output.barrier)
1928 fprintf(stderr, "NO_BARRIER ");
1929 if (cf->output.end_of_program)
1930 fprintf(stderr, "EOP ");
1931 fprintf(stderr, "\n");
1932 } else {
1933 fprintf(stderr, "%04d %08X %08X %s ", id, bc->bytecode[id],
1934 bc->bytecode[id + 1], cfop->name);
1935 fprintf(stderr, "@%d ", cf->cf_addr);
1936 if (cf->cond)
1937 fprintf(stderr, "CND:%X ", cf->cond);
1938 if (cf->pop_count)
1939 fprintf(stderr, "POP:%X ", cf->pop_count);
1940 fprintf(stderr, "\n");
1941 }
1942 }
1943
1944 id = cf->addr;
1945 nliteral = 0;
1946 last = 1;
1947 LIST_FOR_EACH_ENTRY(alu, &cf->alu, list) {
1948 const char *omod_str[] = {"","*2","*4","/2"};
1949 const struct alu_op_info *aop = r600_isa_alu(alu->op);
1950 int o = 0;
1951
1952 r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
1953 o += fprintf(stderr, " %04d %08X %08X ", id, bc->bytecode[id], bc->bytecode[id+1]);
1954 if (last)
1955 o += fprintf(stderr, "%4d ", ++ngr);
1956 else
1957 o += fprintf(stderr, " ");
1958 o += fprintf(stderr, "%c%c %c ", alu->execute_mask ? 'M':' ',
1959 alu->update_pred ? 'P':' ',
1960 alu->pred_sel ? alu->pred_sel==2 ? '0':'1':' ');
1961
1962 o += fprintf(stderr, "%s%s%s ", aop->name,
1963 omod_str[alu->omod], alu->dst.clamp ? "_sat":"");
1964
1965 o += print_indent(o,60);
1966 o += print_dst(alu);
1967 for (i = 0; i < aop->src_count; ++i) {
1968 o += fprintf(stderr, i == 0 ? ", ": ", ");
1969 o += print_src(alu, i);
1970 }
1971
1972 if (alu->bank_swizzle) {
1973 o += print_indent(o,75);
1974 o += fprintf(stderr, " BS:%d", alu->bank_swizzle);
1975 }
1976
1977 fprintf(stderr, "\n");
1978 id += 2;
1979
1980 if (alu->last) {
1981 for (i = 0; i < nliteral; i++, id++) {
1982 float *f = (float*)(bc->bytecode + id);
1983 o = fprintf(stderr, " %04d %08X", id, bc->bytecode[id]);
1984 print_indent(o, 60);
1985 fprintf(stderr, " %f (%d)\n", *f, *(bc->bytecode + id));
1986 }
1987 id += nliteral & 1;
1988 nliteral = 0;
1989 }
1990 last = alu->last;
1991 }
1992
1993 LIST_FOR_EACH_ENTRY(tex, &cf->tex, list) {
1994 int o = 0;
1995 o += fprintf(stderr, " %04d %08X %08X %08X ", id, bc->bytecode[id],
1996 bc->bytecode[id + 1], bc->bytecode[id + 2]);
1997
1998 o += fprintf(stderr, "%s ", r600_isa_fetch(tex->op)->name);
1999
2000 o += print_indent(o, 50);
2001
2002 o += fprintf(stderr, "R%d.", tex->dst_gpr);
2003 o += print_swizzle(tex->dst_sel_x);
2004 o += print_swizzle(tex->dst_sel_y);
2005 o += print_swizzle(tex->dst_sel_z);
2006 o += print_swizzle(tex->dst_sel_w);
2007
2008 o += fprintf(stderr, ", R%d.", tex->src_gpr);
2009 o += print_swizzle(tex->src_sel_x);
2010 o += print_swizzle(tex->src_sel_y);
2011 o += print_swizzle(tex->src_sel_z);
2012 o += print_swizzle(tex->src_sel_w);
2013
2014 o += fprintf(stderr, ", RID:%d", tex->resource_id);
2015 o += fprintf(stderr, ", SID:%d ", tex->sampler_id);
2016
2017 if (tex->lod_bias)
2018 fprintf(stderr, "LB:%d ", tex->lod_bias);
2019
2020 fprintf(stderr, "CT:%c%c%c%c ",
2021 tex->coord_type_x ? 'N' : 'U',
2022 tex->coord_type_y ? 'N' : 'U',
2023 tex->coord_type_z ? 'N' : 'U',
2024 tex->coord_type_w ? 'N' : 'U');
2025
2026 if (tex->offset_x)
2027 fprintf(stderr, "OX:%d ", tex->offset_x);
2028 if (tex->offset_y)
2029 fprintf(stderr, "OY:%d ", tex->offset_y);
2030 if (tex->offset_z)
2031 fprintf(stderr, "OZ:%d ", tex->offset_z);
2032
2033 id += 4;
2034 fprintf(stderr, "\n");
2035 }
2036
2037 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
2038 int o = 0;
2039 const char * fetch_type[] = {"VERTEX", "INSTANCE", ""};
2040 o += fprintf(stderr, " %04d %08X %08X %08X ", id, bc->bytecode[id],
2041 bc->bytecode[id + 1], bc->bytecode[id + 2]);
2042
2043 o += fprintf(stderr, "%s ", r600_isa_fetch(vtx->op)->name);
2044
2045 o += print_indent(o, 50);
2046
2047 o += fprintf(stderr, "R%d.", vtx->dst_gpr);
2048 o += print_swizzle(vtx->dst_sel_x);
2049 o += print_swizzle(vtx->dst_sel_y);
2050 o += print_swizzle(vtx->dst_sel_z);
2051 o += print_swizzle(vtx->dst_sel_w);
2052
2053 o += fprintf(stderr, ", R%d.", vtx->src_gpr);
2054 o += print_swizzle(vtx->src_sel_x);
2055
2056 if (vtx->offset)
2057 fprintf(stderr, " +%db", vtx->offset);
2058
2059 o += print_indent(o, 55);
2060
2061 fprintf(stderr, ", RID:%d ", vtx->buffer_id);
2062
2063 fprintf(stderr, "%s ", fetch_type[vtx->fetch_type]);
2064
2065 if (bc->chip_class < CAYMAN && vtx->mega_fetch_count)
2066 fprintf(stderr, "MFC:%d ", vtx->mega_fetch_count);
2067
2068 fprintf(stderr, "UCF:%d ", vtx->use_const_fields);
2069 fprintf(stderr, "FMT(DTA:%d ", vtx->data_format);
2070 fprintf(stderr, "NUM:%d ", vtx->num_format_all);
2071 fprintf(stderr, "COMP:%d ", vtx->format_comp_all);
2072 fprintf(stderr, "MODE:%d)\n", vtx->srf_mode_all);
2073
2074 id += 4;
2075 }
2076 }
2077
2078 fprintf(stderr, "--------------------------------------\n");
2079 }
2080
2081 void r600_bytecode_dump(struct r600_bytecode *bc)
2082 {
2083 struct r600_bytecode_cf *cf = NULL;
2084 struct r600_bytecode_alu *alu = NULL;
2085 struct r600_bytecode_vtx *vtx = NULL;
2086 struct r600_bytecode_tex *tex = NULL;
2087
2088 unsigned i, id;
2089 uint32_t literal[4];
2090 unsigned nliteral;
2091 char chip = '6';
2092
2093 switch (bc->chip_class) {
2094 case R700:
2095 chip = '7';
2096 break;
2097 case EVERGREEN:
2098 chip = 'E';
2099 break;
2100 case CAYMAN:
2101 chip = 'C';
2102 break;
2103 case R600:
2104 default:
2105 chip = '6';
2106 break;
2107 }
2108 fprintf(stderr, "bytecode %d dw -- %d gprs ---------------------\n", bc->ndw, bc->ngpr);
2109 fprintf(stderr, " %c\n", chip);
2110
2111 LIST_FOR_EACH_ENTRY(cf, &bc->cf, list) {
2112 id = cf->id;
2113 if (cf->op == CF_NATIVE) {
2114 fprintf(stderr, "%04d %08X CF NATIVE\n", id, bc->bytecode[id]);
2115 fprintf(stderr, "%04d %08X CF NATIVE\n", id + 1, bc->bytecode[id + 1]);
2116 } else {
2117 const struct cf_op_info *cfop = r600_isa_cf(cf->op);
2118 if (cfop->flags & CF_ALU) {
2119 if (cf->eg_alu_extended) {
2120 fprintf(stderr, "%04d %08X ALU_EXT0 ", id, bc->bytecode[id]);
2121 fprintf(stderr, "KCACHE_BANK2:%X ", cf->kcache[2].bank);
2122 fprintf(stderr, "KCACHE_BANK3:%X ", cf->kcache[3].bank);
2123 fprintf(stderr, "KCACHE_MODE2:%X\n", cf->kcache[2].mode);
2124 id++;
2125 fprintf(stderr, "%04d %08X ALU_EXT1 ", id, bc->bytecode[id]);
2126 fprintf(stderr, "KCACHE_MODE3:%X ", cf->kcache[3].mode);
2127 fprintf(stderr, "KCACHE_ADDR2:%X ", cf->kcache[2].addr);
2128 fprintf(stderr, "KCACHE_ADDR3:%X\n", cf->kcache[3].addr);
2129 id++;
2130 }
2131
2132 fprintf(stderr, "%04d %08X ALU ", id, bc->bytecode[id]);
2133 fprintf(stderr, "ADDR:%d ", cf->addr);
2134 fprintf(stderr, "KCACHE_MODE0:%X ", cf->kcache[0].mode);
2135 fprintf(stderr, "KCACHE_BANK0:%X ", cf->kcache[0].bank);
2136 fprintf(stderr, "KCACHE_BANK1:%X\n", cf->kcache[1].bank);
2137 id++;
2138 fprintf(stderr, "%04d %08X ALU ", id, bc->bytecode[id]);
2139 fprintf(stderr, "INST: %s ", cfop->name);
2140 fprintf(stderr, "KCACHE_MODE1:%X ", cf->kcache[1].mode);
2141 fprintf(stderr, "KCACHE_ADDR0:%X ", cf->kcache[0].addr);
2142 fprintf(stderr, "KCACHE_ADDR1:%X ", cf->kcache[1].addr);
2143 fprintf(stderr, "COUNT:%d\n", cf->ndw / 2);
2144 } else if (cfop->flags & CF_FETCH) {
2145 fprintf(stderr, "%04d %08X TEX/VTX ", id, bc->bytecode[id]);
2146 fprintf(stderr, "ADDR:%d\n", cf->addr);
2147 id++;
2148 fprintf(stderr, "%04d %08X TEX/VTX ", id, bc->bytecode[id]);
2149 fprintf(stderr, "INST: %s ", cfop->name);
2150 fprintf(stderr, "COUNT:%d\n", cf->ndw / 4);
2151 } else if (cfop->flags & CF_EXP) {
2152 fprintf(stderr, "%04d %08X %s ", id, bc->bytecode[id],
2153 cfop->name);
2154 fprintf(stderr, "GPR:%X ", cf->output.gpr);
2155 fprintf(stderr, "ELEM_SIZE:%X ", cf->output.elem_size);
2156 fprintf(stderr, "ARRAY_BASE:%X ", cf->output.array_base);
2157 fprintf(stderr, "TYPE:%X\n", cf->output.type);
2158 id++;
2159 fprintf(stderr, "%04d %08X %s ", id, bc->bytecode[id],
2160 cfop->name);
2161 fprintf(stderr, "SWIZ_X:%X ", cf->output.swizzle_x);
2162 fprintf(stderr, "SWIZ_Y:%X ", cf->output.swizzle_y);
2163 fprintf(stderr, "SWIZ_Z:%X ", cf->output.swizzle_z);
2164 fprintf(stderr, "SWIZ_W:%X ", cf->output.swizzle_w);
2165 fprintf(stderr, "BARRIER:%X ", cf->output.barrier);
2166 fprintf(stderr, "INST: %s ", r600_isa_cf(cf->op)->name);
2167 fprintf(stderr, "BURST_COUNT:%d ", cf->output.burst_count);
2168 fprintf(stderr, "EOP:%X\n", cf->output.end_of_program);
2169 } else if (cfop->flags & CF_STRM) {
2170 fprintf(stderr, "%04d %08X EXPORT %s ", id, bc->bytecode[id],
2171 cfop->name);
2172 fprintf(stderr, "GPR:%X ", cf->output.gpr);
2173 fprintf(stderr, "ELEM_SIZE:%i ", cf->output.elem_size);
2174 fprintf(stderr, "ARRAY_BASE:%i ", cf->output.array_base);
2175 fprintf(stderr, "TYPE:%X\n", cf->output.type);
2176 id++;
2177 fprintf(stderr, "%04d %08X EXPORT %s ", id, bc->bytecode[id],
2178 cfop->name);
2179 fprintf(stderr, "ARRAY_SIZE:%i ", cf->output.array_size);
2180 fprintf(stderr, "COMP_MASK:%X ", cf->output.comp_mask);
2181 fprintf(stderr, "BARRIER:%X ", cf->output.barrier);
2182 fprintf(stderr, "INST: %s ", cfop->name);
2183 fprintf(stderr, "BURST_COUNT:%d ", cf->output.burst_count);
2184 fprintf(stderr, "EOP:%X\n", cf->output.end_of_program);
2185
2186 } else {
2187 fprintf(stderr, "%04d %08X CF ", id, bc->bytecode[id]);
2188 fprintf(stderr, "ADDR:%d\n", cf->cf_addr);
2189 id++;
2190 fprintf(stderr, "%04d %08X CF ", id, bc->bytecode[id]);
2191 fprintf(stderr, "INST: %s ", cfop->name);
2192 fprintf(stderr, "COND:%X ", cf->cond);
2193 fprintf(stderr, "POP_COUNT:%X\n", cf->pop_count);
2194 }
2195 }
2196
2197 id = cf->addr;
2198 nliteral = 0;
2199 LIST_FOR_EACH_ENTRY(alu, &cf->alu, list) {
2200 r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
2201
2202 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2203 fprintf(stderr, " SRC0(SEL:%d ", alu->src[0].sel);
2204 fprintf(stderr, "REL:%d ", alu->src[0].rel);
2205 fprintf(stderr, "CHAN:%d ", alu->src[0].chan);
2206 fprintf(stderr, "NEG:%d) ", alu->src[0].neg);
2207 fprintf(stderr, "SRC1(SEL:%d ", alu->src[1].sel);
2208 fprintf(stderr, "REL:%d ", alu->src[1].rel);
2209 fprintf(stderr, "CHAN:%d ", alu->src[1].chan);
2210 fprintf(stderr, "NEG:%d ", alu->src[1].neg);
2211 fprintf(stderr, "IM:%d) ", alu->index_mode);
2212 fprintf(stderr, "PRED_SEL:%d ", alu->pred_sel);
2213 fprintf(stderr, "LAST:%d)\n", alu->last);
2214 id++;
2215 fprintf(stderr, "%04d %08X %c ", id, bc->bytecode[id], alu->last ? '*' : ' ');
2216 fprintf(stderr, "INST: %s ", r600_isa_alu(alu->op)->name);
2217 fprintf(stderr, "DST(SEL:%d ", alu->dst.sel);
2218 fprintf(stderr, "CHAN:%d ", alu->dst.chan);
2219 fprintf(stderr, "REL:%d ", alu->dst.rel);
2220 fprintf(stderr, "CLAMP:%d) ", alu->dst.clamp);
2221 fprintf(stderr, "BANK_SWIZZLE:%d ", alu->bank_swizzle);
2222 if (alu->is_op3) {
2223 fprintf(stderr, "SRC2(SEL:%d ", alu->src[2].sel);
2224 fprintf(stderr, "REL:%d ", alu->src[2].rel);
2225 fprintf(stderr, "CHAN:%d ", alu->src[2].chan);
2226 fprintf(stderr, "NEG:%d)\n", alu->src[2].neg);
2227 } else {
2228 fprintf(stderr, "SRC0_ABS:%d ", alu->src[0].abs);
2229 fprintf(stderr, "SRC1_ABS:%d ", alu->src[1].abs);
2230 fprintf(stderr, "WRITE_MASK:%d ", alu->dst.write);
2231 fprintf(stderr, "OMOD:%d ", alu->omod);
2232 fprintf(stderr, "EXECUTE_MASK:%d ", alu->execute_mask);
2233 fprintf(stderr, "UPDATE_PRED:%d\n", alu->update_pred);
2234 }
2235
2236 id++;
2237 if (alu->last) {
2238 for (i = 0; i < nliteral; i++, id++) {
2239 float *f = (float*)(bc->bytecode + id);
2240 fprintf(stderr, "%04d %08X\t%f (%d)\n", id, bc->bytecode[id], *f,
2241 *(bc->bytecode + id));
2242 }
2243 id += nliteral & 1;
2244 nliteral = 0;
2245 }
2246 }
2247
2248 LIST_FOR_EACH_ENTRY(tex, &cf->tex, list) {
2249 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2250 fprintf(stderr, "INST: %s ", r600_isa_fetch(tex->op)->name);
2251 fprintf(stderr, "RESOURCE_ID:%d ", tex->resource_id);
2252 fprintf(stderr, "SRC(GPR:%d ", tex->src_gpr);
2253 fprintf(stderr, "REL:%d)\n", tex->src_rel);
2254 id++;
2255 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2256 fprintf(stderr, "DST(GPR:%d ", tex->dst_gpr);
2257 fprintf(stderr, "REL:%d ", tex->dst_rel);
2258 fprintf(stderr, "SEL_X:%d ", tex->dst_sel_x);
2259 fprintf(stderr, "SEL_Y:%d ", tex->dst_sel_y);
2260 fprintf(stderr, "SEL_Z:%d ", tex->dst_sel_z);
2261 fprintf(stderr, "SEL_W:%d) ", tex->dst_sel_w);
2262 fprintf(stderr, "LOD_BIAS:%d ", tex->lod_bias);
2263 fprintf(stderr, "COORD_TYPE_X:%d ", tex->coord_type_x);
2264 fprintf(stderr, "COORD_TYPE_Y:%d ", tex->coord_type_y);
2265 fprintf(stderr, "COORD_TYPE_Z:%d ", tex->coord_type_z);
2266 fprintf(stderr, "COORD_TYPE_W:%d\n", tex->coord_type_w);
2267 id++;
2268 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2269 fprintf(stderr, "OFFSET_X:%d ", tex->offset_x);
2270 fprintf(stderr, "OFFSET_Y:%d ", tex->offset_y);
2271 fprintf(stderr, "OFFSET_Z:%d ", tex->offset_z);
2272 fprintf(stderr, "SAMPLER_ID:%d ", tex->sampler_id);
2273 fprintf(stderr, "SRC(SEL_X:%d ", tex->src_sel_x);
2274 fprintf(stderr, "SEL_Y:%d ", tex->src_sel_y);
2275 fprintf(stderr, "SEL_Z:%d ", tex->src_sel_z);
2276 fprintf(stderr, "SEL_W:%d)\n", tex->src_sel_w);
2277 id++;
2278 fprintf(stderr, "%04d %08X \n", id, bc->bytecode[id]);
2279 id++;
2280 }
2281
2282 LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
2283 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2284 fprintf(stderr, "INST: %s ", r600_isa_fetch(vtx->op)->name);
2285 fprintf(stderr, "FETCH_TYPE:%d ", vtx->fetch_type);
2286 fprintf(stderr, "BUFFER_ID:%d\n", vtx->buffer_id);
2287 id++;
2288 /* This assumes that no semantic fetches exist */
2289 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2290 fprintf(stderr, "SRC(GPR:%d ", vtx->src_gpr);
2291 fprintf(stderr, "SEL_X:%d) ", vtx->src_sel_x);
2292 if (bc->chip_class < CAYMAN)
2293 fprintf(stderr, "MEGA_FETCH_COUNT:%d ", vtx->mega_fetch_count);
2294 else
2295 fprintf(stderr, "SEL_Y:%d) ", 0);
2296 fprintf(stderr, "DST(GPR:%d ", vtx->dst_gpr);
2297 fprintf(stderr, "SEL_X:%d ", vtx->dst_sel_x);
2298 fprintf(stderr, "SEL_Y:%d ", vtx->dst_sel_y);
2299 fprintf(stderr, "SEL_Z:%d ", vtx->dst_sel_z);
2300 fprintf(stderr, "SEL_W:%d) ", vtx->dst_sel_w);
2301 fprintf(stderr, "USE_CONST_FIELDS:%d ", vtx->use_const_fields);
2302 fprintf(stderr, "FORMAT(DATA:%d ", vtx->data_format);
2303 fprintf(stderr, "NUM:%d ", vtx->num_format_all);
2304 fprintf(stderr, "COMP:%d ", vtx->format_comp_all);
2305 fprintf(stderr, "MODE:%d)\n", vtx->srf_mode_all);
2306 id++;
2307 fprintf(stderr, "%04d %08X ", id, bc->bytecode[id]);
2308 fprintf(stderr, "ENDIAN:%d ", vtx->endian);
2309 fprintf(stderr, "OFFSET:%d\n", vtx->offset);
2310 /* XXX */
2311 id++;
2312 fprintf(stderr, "%04d %08X \n", id, bc->bytecode[id]);
2313 id++;
2314 }
2315 }
2316
2317 fprintf(stderr, "--------------------------------------\n");
2318 }
2319
2320 void r600_vertex_data_type(enum pipe_format pformat,
2321 unsigned *format,
2322 unsigned *num_format, unsigned *format_comp, unsigned *endian)
2323 {
2324 const struct util_format_description *desc;
2325 unsigned i;
2326
2327 *format = 0;
2328 *num_format = 0;
2329 *format_comp = 0;
2330 *endian = ENDIAN_NONE;
2331
2332 desc = util_format_description(pformat);
2333 if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN) {
2334 goto out_unknown;
2335 }
2336
2337 /* Find the first non-VOID channel. */
2338 for (i = 0; i < 4; i++) {
2339 if (desc->channel[i].type != UTIL_FORMAT_TYPE_VOID) {
2340 break;
2341 }
2342 }
2343
2344 *endian = r600_endian_swap(desc->channel[i].size);
2345
2346 switch (desc->channel[i].type) {
2347 /* Half-floats, floats, ints */
2348 case UTIL_FORMAT_TYPE_FLOAT:
2349 switch (desc->channel[i].size) {
2350 case 16:
2351 switch (desc->nr_channels) {
2352 case 1:
2353 *format = FMT_16_FLOAT;
2354 break;
2355 case 2:
2356 *format = FMT_16_16_FLOAT;
2357 break;
2358 case 3:
2359 case 4:
2360 *format = FMT_16_16_16_16_FLOAT;
2361 break;
2362 }
2363 break;
2364 case 32:
2365 switch (desc->nr_channels) {
2366 case 1:
2367 *format = FMT_32_FLOAT;
2368 break;
2369 case 2:
2370 *format = FMT_32_32_FLOAT;
2371 break;
2372 case 3:
2373 *format = FMT_32_32_32_FLOAT;
2374 break;
2375 case 4:
2376 *format = FMT_32_32_32_32_FLOAT;
2377 break;
2378 }
2379 break;
2380 default:
2381 goto out_unknown;
2382 }
2383 break;
2384 /* Unsigned ints */
2385 case UTIL_FORMAT_TYPE_UNSIGNED:
2386 /* Signed ints */
2387 case UTIL_FORMAT_TYPE_SIGNED:
2388 switch (desc->channel[i].size) {
2389 case 8:
2390 switch (desc->nr_channels) {
2391 case 1:
2392 *format = FMT_8;
2393 break;
2394 case 2:
2395 *format = FMT_8_8;
2396 break;
2397 case 3:
2398 case 4:
2399 *format = FMT_8_8_8_8;
2400 break;
2401 }
2402 break;
2403 case 10:
2404 if (desc->nr_channels != 4)
2405 goto out_unknown;
2406
2407 *format = FMT_2_10_10_10;
2408 break;
2409 case 16:
2410 switch (desc->nr_channels) {
2411 case 1:
2412 *format = FMT_16;
2413 break;
2414 case 2:
2415 *format = FMT_16_16;
2416 break;
2417 case 3:
2418 case 4:
2419 *format = FMT_16_16_16_16;
2420 break;
2421 }
2422 break;
2423 case 32:
2424 switch (desc->nr_channels) {
2425 case 1:
2426 *format = FMT_32;
2427 break;
2428 case 2:
2429 *format = FMT_32_32;
2430 break;
2431 case 3:
2432 *format = FMT_32_32_32;
2433 break;
2434 case 4:
2435 *format = FMT_32_32_32_32;
2436 break;
2437 }
2438 break;
2439 default:
2440 goto out_unknown;
2441 }
2442 break;
2443 default:
2444 goto out_unknown;
2445 }
2446
2447 if (desc->channel[i].type == UTIL_FORMAT_TYPE_SIGNED) {
2448 *format_comp = 1;
2449 }
2450
2451 *num_format = 0;
2452 if (desc->channel[i].type == UTIL_FORMAT_TYPE_UNSIGNED ||
2453 desc->channel[i].type == UTIL_FORMAT_TYPE_SIGNED) {
2454 if (!desc->channel[i].normalized) {
2455 if (desc->channel[i].pure_integer)
2456 *num_format = 1;
2457 else
2458 *num_format = 2;
2459 }
2460 }
2461 return;
2462 out_unknown:
2463 R600_ERR("unsupported vertex format %s\n", util_format_name(pformat));
2464 }
2465
2466 void *r600_create_vertex_fetch_shader(struct pipe_context *ctx,
2467 unsigned count,
2468 const struct pipe_vertex_element *elements)
2469 {
2470 struct r600_context *rctx = (struct r600_context *)ctx;
2471 static int dump_shaders = -1;
2472 struct r600_bytecode bc;
2473 struct r600_bytecode_vtx vtx;
2474 const struct util_format_description *desc;
2475 unsigned fetch_resource_start = rctx->chip_class >= EVERGREEN ? 0 : 160;
2476 unsigned format, num_format, format_comp, endian;
2477 uint32_t *bytecode;
2478 int i, j, r, fs_size;
2479 struct r600_fetch_shader *shader;
2480
2481 assert(count < 32);
2482
2483 memset(&bc, 0, sizeof(bc));
2484 r600_bytecode_init(&bc, rctx->chip_class, rctx->family,
2485 rctx->screen->msaa_texture_support);
2486
2487 bc.isa = rctx->isa;
2488
2489 for (i = 0; i < count; i++) {
2490 if (elements[i].instance_divisor > 1) {
2491 if (rctx->chip_class == CAYMAN) {
2492 for (j = 0; j < 4; j++) {
2493 struct r600_bytecode_alu alu;
2494 memset(&alu, 0, sizeof(alu));
2495 alu.op = ALU_OP2_MULHI_UINT;
2496 alu.src[0].sel = 0;
2497 alu.src[0].chan = 3;
2498 alu.src[1].sel = V_SQ_ALU_SRC_LITERAL;
2499 alu.src[1].value = (1ll << 32) / elements[i].instance_divisor + 1;
2500 alu.dst.sel = i + 1;
2501 alu.dst.chan = j;
2502 alu.dst.write = j == 3;
2503 alu.last = j == 3;
2504 if ((r = r600_bytecode_add_alu(&bc, &alu))) {
2505 r600_bytecode_clear(&bc);
2506 return NULL;
2507 }
2508 }
2509 } else {
2510 struct r600_bytecode_alu alu;
2511 memset(&alu, 0, sizeof(alu));
2512 alu.op = ALU_OP2_MULHI_UINT;
2513 alu.src[0].sel = 0;
2514 alu.src[0].chan = 3;
2515 alu.src[1].sel = V_SQ_ALU_SRC_LITERAL;
2516 alu.src[1].value = (1ll << 32) / elements[i].instance_divisor + 1;
2517 alu.dst.sel = i + 1;
2518 alu.dst.chan = 3;
2519 alu.dst.write = 1;
2520 alu.last = 1;
2521 if ((r = r600_bytecode_add_alu(&bc, &alu))) {
2522 r600_bytecode_clear(&bc);
2523 return NULL;
2524 }
2525 }
2526 }
2527 }
2528
2529 for (i = 0; i < count; i++) {
2530 r600_vertex_data_type(elements[i].src_format,
2531 &format, &num_format, &format_comp, &endian);
2532
2533 desc = util_format_description(elements[i].src_format);
2534 if (desc == NULL) {
2535 r600_bytecode_clear(&bc);
2536 R600_ERR("unknown format %d\n", elements[i].src_format);
2537 return NULL;
2538 }
2539
2540 if (elements[i].src_offset > 65535) {
2541 r600_bytecode_clear(&bc);
2542 R600_ERR("too big src_offset: %u\n", elements[i].src_offset);
2543 return NULL;
2544 }
2545
2546 memset(&vtx, 0, sizeof(vtx));
2547 vtx.buffer_id = elements[i].vertex_buffer_index + fetch_resource_start;
2548 vtx.fetch_type = elements[i].instance_divisor ? 1 : 0;
2549 vtx.src_gpr = elements[i].instance_divisor > 1 ? i + 1 : 0;
2550 vtx.src_sel_x = elements[i].instance_divisor ? 3 : 0;
2551 vtx.mega_fetch_count = 0x1F;
2552 vtx.dst_gpr = i + 1;
2553 vtx.dst_sel_x = desc->swizzle[0];
2554 vtx.dst_sel_y = desc->swizzle[1];
2555 vtx.dst_sel_z = desc->swizzle[2];
2556 vtx.dst_sel_w = desc->swizzle[3];
2557 vtx.data_format = format;
2558 vtx.num_format_all = num_format;
2559 vtx.format_comp_all = format_comp;
2560 vtx.srf_mode_all = 1;
2561 vtx.offset = elements[i].src_offset;
2562 vtx.endian = endian;
2563
2564 if ((r = r600_bytecode_add_vtx(&bc, &vtx))) {
2565 r600_bytecode_clear(&bc);
2566 return NULL;
2567 }
2568 }
2569
2570 r600_bytecode_add_cfinst(&bc, CF_OP_RET);
2571
2572 if ((r = r600_bytecode_build(&bc))) {
2573 r600_bytecode_clear(&bc);
2574 return NULL;
2575 }
2576
2577 if (dump_shaders == -1)
2578 dump_shaders = debug_get_num_option("R600_DUMP_SHADERS", 0);
2579
2580 if (dump_shaders & 1) {
2581 fprintf(stderr, "--------------------------------------------------------------\n");
2582 r600_bytecode_dump(&bc);
2583 fprintf(stderr, "______________________________________________________________\n");
2584 }
2585 if (dump_shaders & 2) {
2586 fprintf(stderr, "--------------------------------------------------------------\n");
2587 r600_bytecode_disasm(&bc);
2588 fprintf(stderr, "______________________________________________________________\n");
2589 }
2590
2591 fs_size = bc.ndw*4;
2592
2593 /* Allocate the CSO. */
2594 shader = CALLOC_STRUCT(r600_fetch_shader);
2595 if (!shader) {
2596 r600_bytecode_clear(&bc);
2597 return NULL;
2598 }
2599
2600 u_suballocator_alloc(rctx->allocator_fetch_shader, fs_size, &shader->offset,
2601 (struct pipe_resource**)&shader->buffer);
2602 if (!shader->buffer) {
2603 r600_bytecode_clear(&bc);
2604 FREE(shader);
2605 return NULL;
2606 }
2607
2608 bytecode = r600_buffer_mmap_sync_with_rings(rctx, shader->buffer, PIPE_TRANSFER_WRITE | PIPE_TRANSFER_UNSYNCHRONIZED);
2609 bytecode += shader->offset / 4;
2610
2611 if (R600_BIG_ENDIAN) {
2612 for (i = 0; i < fs_size / 4; ++i) {
2613 bytecode[i] = bswap_32(bc.bytecode[i]);
2614 }
2615 } else {
2616 memcpy(bytecode, bc.bytecode, fs_size);
2617 }
2618 rctx->ws->buffer_unmap(shader->buffer->cs_buf);
2619
2620 r600_bytecode_clear(&bc);
2621 return shader;
2622 }
2623
2624 void r600_bytecode_alu_read(struct r600_bytecode *bc,
2625 struct r600_bytecode_alu *alu, uint32_t word0, uint32_t word1)
2626 {
2627 /* WORD0 */
2628 alu->src[0].sel = G_SQ_ALU_WORD0_SRC0_SEL(word0);
2629 alu->src[0].rel = G_SQ_ALU_WORD0_SRC0_REL(word0);
2630 alu->src[0].chan = G_SQ_ALU_WORD0_SRC0_CHAN(word0);
2631 alu->src[0].neg = G_SQ_ALU_WORD0_SRC0_NEG(word0);
2632 alu->src[1].sel = G_SQ_ALU_WORD0_SRC1_SEL(word0);
2633 alu->src[1].rel = G_SQ_ALU_WORD0_SRC1_REL(word0);
2634 alu->src[1].chan = G_SQ_ALU_WORD0_SRC1_CHAN(word0);
2635 alu->src[1].neg = G_SQ_ALU_WORD0_SRC1_NEG(word0);
2636 alu->index_mode = G_SQ_ALU_WORD0_INDEX_MODE(word0);
2637 alu->pred_sel = G_SQ_ALU_WORD0_PRED_SEL(word0);
2638 alu->last = G_SQ_ALU_WORD0_LAST(word0);
2639
2640 /* WORD1 */
2641 alu->bank_swizzle = G_SQ_ALU_WORD1_BANK_SWIZZLE(word1);
2642 if (alu->bank_swizzle)
2643 alu->bank_swizzle_force = alu->bank_swizzle;
2644 alu->dst.sel = G_SQ_ALU_WORD1_DST_GPR(word1);
2645 alu->dst.rel = G_SQ_ALU_WORD1_DST_REL(word1);
2646 alu->dst.chan = G_SQ_ALU_WORD1_DST_CHAN(word1);
2647 alu->dst.clamp = G_SQ_ALU_WORD1_CLAMP(word1);
2648 if (G_SQ_ALU_WORD1_ENCODING(word1)) /*ALU_DWORD1_OP3*/
2649 {
2650 alu->is_op3 = 1;
2651 alu->src[2].sel = G_SQ_ALU_WORD1_OP3_SRC2_SEL(word1);
2652 alu->src[2].rel = G_SQ_ALU_WORD1_OP3_SRC2_REL(word1);
2653 alu->src[2].chan = G_SQ_ALU_WORD1_OP3_SRC2_CHAN(word1);
2654 alu->src[2].neg = G_SQ_ALU_WORD1_OP3_SRC2_NEG(word1);
2655 alu->op = r600_isa_alu_by_opcode(bc->isa,
2656 G_SQ_ALU_WORD1_OP3_ALU_INST(word1), /* is_op3 = */ 1);
2657
2658 }
2659 else /*ALU_DWORD1_OP2*/
2660 {
2661 alu->src[0].abs = G_SQ_ALU_WORD1_OP2_SRC0_ABS(word1);
2662 alu->src[1].abs = G_SQ_ALU_WORD1_OP2_SRC1_ABS(word1);
2663 alu->op = r600_isa_alu_by_opcode(bc->isa,
2664 G_SQ_ALU_WORD1_OP2_ALU_INST(word1), /* is_op3 = */ 0);
2665 alu->omod = G_SQ_ALU_WORD1_OP2_OMOD(word1);
2666 alu->dst.write = G_SQ_ALU_WORD1_OP2_WRITE_MASK(word1);
2667 alu->update_pred = G_SQ_ALU_WORD1_OP2_UPDATE_PRED(word1);
2668 alu->execute_mask =
2669 G_SQ_ALU_WORD1_OP2_UPDATE_EXECUTE_MASK(word1);
2670 }
2671 }
2672
2673 void r600_bytecode_export_read(struct r600_bytecode *bc,
2674 struct r600_bytecode_output *output, uint32_t word0, uint32_t word1)
2675 {
2676 output->array_base = G_SQ_CF_ALLOC_EXPORT_WORD0_ARRAY_BASE(word0);
2677 output->type = G_SQ_CF_ALLOC_EXPORT_WORD0_TYPE(word0);
2678 output->gpr = G_SQ_CF_ALLOC_EXPORT_WORD0_RW_GPR(word0);
2679 output->elem_size = G_SQ_CF_ALLOC_EXPORT_WORD0_ELEM_SIZE(word0);
2680
2681 output->swizzle_x = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_X(word1);
2682 output->swizzle_y = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Y(word1);
2683 output->swizzle_z = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_Z(word1);
2684 output->swizzle_w = G_SQ_CF_ALLOC_EXPORT_WORD1_SWIZ_SEL_W(word1);
2685 output->burst_count = G_SQ_CF_ALLOC_EXPORT_WORD1_BURST_COUNT(word1);
2686 output->end_of_program = G_SQ_CF_ALLOC_EXPORT_WORD1_END_OF_PROGRAM(word1);
2687 output->op = r600_isa_cf_by_opcode(bc->isa,
2688 G_SQ_CF_ALLOC_EXPORT_WORD1_CF_INST(word1), 0);
2689 output->barrier = G_SQ_CF_ALLOC_EXPORT_WORD1_BARRIER(word1);
2690 output->array_size = G_SQ_CF_ALLOC_EXPORT_WORD1_BUF_ARRAY_SIZE(word1);
2691 output->comp_mask = G_SQ_CF_ALLOC_EXPORT_WORD1_BUF_COMP_MASK(word1);
2692 }