r600: Add support for B5G5R5A1.
[mesa.git] / src / gallium / drivers / r600 / r600_asm.c
index b514c58f9d8c520eaca302598b4a92dafa715844..a0f28d2776b786e616def4938ed8264a14197168 100644 (file)
@@ -27,6 +27,7 @@
 #include "r600d.h"
 
 #include <errno.h>
+#include "util/u_bitcast.h"
 #include "util/u_dump.h"
 #include "util/u_memory.h"
 #include "util/u_math.h"
 #define NUM_OF_CYCLES 3
 #define NUM_OF_COMPONENTS 4
 
-static inline unsigned int r600_bytecode_get_num_operands(
-               struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
+static inline bool alu_writes(struct r600_bytecode_alu *alu)
 {
-       return r600_isa_alu(alu->op)->src_count;
+       return alu->dst.write || alu->is_op3;
 }
 
-int r700_bytecode_alu_build(struct r600_bytecode *bc,
-               struct r600_bytecode_alu *alu, unsigned id);
+static inline unsigned int r600_bytecode_get_num_operands(const struct r600_bytecode_alu *alu)
+{
+       return r600_isa_alu(alu->op)->src_count;
+}
 
 static struct r600_bytecode_cf *r600_bytecode_cf(void)
 {
        struct r600_bytecode_cf *cf = CALLOC_STRUCT(r600_bytecode_cf);
 
-       if (cf == NULL)
+       if (!cf)
                return NULL;
        LIST_INITHEAD(&cf->list);
        LIST_INITHEAD(&cf->alu);
        LIST_INITHEAD(&cf->vtx);
        LIST_INITHEAD(&cf->tex);
+       LIST_INITHEAD(&cf->gds);
        return cf;
 }
 
@@ -63,7 +66,7 @@ static struct r600_bytecode_alu *r600_bytecode_alu(void)
 {
        struct r600_bytecode_alu *alu = CALLOC_STRUCT(r600_bytecode_alu);
 
-       if (alu == NULL)
+       if (!alu)
                return NULL;
        LIST_INITHEAD(&alu->list);
        return alu;
@@ -73,7 +76,7 @@ static struct r600_bytecode_vtx *r600_bytecode_vtx(void)
 {
        struct r600_bytecode_vtx *vtx = CALLOC_STRUCT(r600_bytecode_vtx);
 
-       if (vtx == NULL)
+       if (!vtx)
                return NULL;
        LIST_INITHEAD(&vtx->list);
        return vtx;
@@ -83,12 +86,22 @@ static struct r600_bytecode_tex *r600_bytecode_tex(void)
 {
        struct r600_bytecode_tex *tex = CALLOC_STRUCT(r600_bytecode_tex);
 
-       if (tex == NULL)
+       if (!tex)
                return NULL;
        LIST_INITHEAD(&tex->list);
        return tex;
 }
 
+static struct r600_bytecode_gds *r600_bytecode_gds(void)
+{
+       struct r600_bytecode_gds *gds = CALLOC_STRUCT(r600_bytecode_gds);
+
+       if (gds == NULL)
+               return NULL;
+       LIST_INITHEAD(&gds->list);
+       return gds;
+}
+
 static unsigned stack_entry_size(enum radeon_family chip) {
        /* Wavefront size:
         *   64: R600/RV670/RV770/Cypress/R740/Barts/Turks/Caicos/
@@ -152,7 +165,7 @@ int r600_bytecode_add_cf(struct r600_bytecode *bc)
 {
        struct r600_bytecode_cf *cf = r600_bytecode_cf();
 
-       if (cf == NULL)
+       if (!cf)
                return -ENOMEM;
        LIST_ADDTAIL(&cf->list, &bc->cf);
        if (bc->cf_last) {
@@ -219,9 +232,9 @@ int r600_bytecode_add_output(struct r600_bytecode *bc,
 }
 
 /* alu instructions that can ony exits once per group */
-static int is_alu_once_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
+static int is_alu_once_inst(struct r600_bytecode_alu *alu)
 {
-       return r600_isa_alu(alu->op)->flags & (AF_KILL | AF_PRED);
+       return r600_isa_alu(alu->op)->flags & (AF_KILL | AF_PRED) || alu->is_lds_idx_op || alu->op == ALU_OP0_GROUP_BARRIER;
 }
 
 static int is_alu_reduction_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
@@ -230,14 +243,14 @@ static int is_alu_reduction_inst(struct r600_bytecode *bc, struct r600_bytecode_
                        (r600_isa_alu_slots(bc->isa->hw_class, alu->op) == AF_4V);
 }
 
-static int is_alu_mova_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
+static int is_alu_mova_inst(struct r600_bytecode_alu *alu)
 {
        return r600_isa_alu(alu->op)->flags & AF_MOVA;
 }
 
-static int alu_uses_rel(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
+static int alu_uses_rel(struct r600_bytecode_alu *alu)
 {
-       unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
+       unsigned num_src = r600_bytecode_get_num_operands(alu);
        unsigned src;
 
        if (alu->dst.rel) {
@@ -252,6 +265,30 @@ static int alu_uses_rel(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
        return 0;
 }
 
+static int is_lds_read(int sel)
+{
+  return sel == EG_V_SQ_ALU_SRC_LDS_OQ_A_POP || sel == EG_V_SQ_ALU_SRC_LDS_OQ_B_POP;
+}
+
+static int alu_uses_lds(struct r600_bytecode_alu *alu)
+{
+       unsigned num_src = r600_bytecode_get_num_operands(alu);
+       unsigned src;
+
+       for (src = 0; src < num_src; ++src) {
+               if (is_lds_read(alu->src[src].sel)) {
+                       return 1;
+               }
+       }
+       return 0;
+}
+
+static int is_alu_64bit_inst(struct r600_bytecode_alu *alu)
+{
+       const struct alu_op_info *op = r600_isa_alu(alu->op);
+       return (op->flags & AF_64);
+}
+
 static int is_alu_vec_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
 {
        unsigned slots = r600_isa_alu_slots(bc->isa->hw_class, alu->op);
@@ -271,10 +308,10 @@ static int is_alu_any_unit_inst(struct r600_bytecode *bc, struct r600_bytecode_a
        return slots == AF_VS;
 }
 
-static int is_nop_inst(struct r600_bytecode *bc, struct r600_bytecode_alu *alu)
+static int is_nop_inst(struct r600_bytecode_alu *alu)
 {
        return alu->op == ALU_OP0_NOP;
-}              
+}
 
 static int assign_alu_units(struct r600_bytecode *bc, struct r600_bytecode_alu *alu_first,
                            struct r600_bytecode_alu *assignment[5])
@@ -365,7 +402,8 @@ static int reserve_gpr(struct alu_bank_swizzle *bs, unsigned sel, unsigned chan,
        return 0;
 }
 
-static int reserve_cfile(struct r600_bytecode *bc, struct alu_bank_swizzle *bs, unsigned sel, unsigned chan)
+static int reserve_cfile(const struct r600_bytecode *bc,
+                        struct alu_bank_swizzle *bs, unsigned sel, unsigned chan)
 {
        int res, num_res = 4;
        if (bc->chip_class >= R700) {
@@ -407,12 +445,12 @@ static int is_const(int sel)
                sel <= V_SQ_ALU_SRC_LITERAL);
 }
 
-static int check_vector(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
+static int check_vector(const struct r600_bytecode *bc, const struct r600_bytecode_alu *alu,
                        struct alu_bank_swizzle *bs, int bank_swizzle)
 {
        int r, src, num_src, sel, elem, cycle;
 
-       num_src = r600_bytecode_get_num_operands(bc, alu);
+       num_src = r600_bytecode_get_num_operands(alu);
        for (src = 0; src < num_src; src++) {
                sel = alu->src[src].sel;
                elem = alu->src[src].chan;
@@ -437,12 +475,12 @@ static int check_vector(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
        return 0;
 }
 
-static int check_scalar(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
+static int check_scalar(const struct r600_bytecode *bc, const struct r600_bytecode_alu *alu,
                        struct alu_bank_swizzle *bs, int bank_swizzle)
 {
        int r, src, num_src, const_count, sel, elem, cycle;
 
-       num_src = r600_bytecode_get_num_operands(bc, alu);
+       num_src = r600_bytecode_get_num_operands(alu);
        for (const_count = 0, src = 0; src < num_src; ++src) {
                sel = alu->src[src].sel;
                elem = alu->src[src].chan;
@@ -483,7 +521,7 @@ static int check_scalar(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
        return 0;
 }
 
-static int check_and_set_bank_swizzle(struct r600_bytecode *bc,
+static int check_and_set_bank_swizzle(const struct r600_bytecode *bc,
                                      struct r600_bytecode_alu *slots[5])
 {
        struct alu_bank_swizzle bs;
@@ -575,7 +613,13 @@ static int replace_gpr_with_pv_ps(struct r600_bytecode *bc,
                return r;
 
        for (i = 0; i < max_slots; ++i) {
-               if (prev[i] && (prev[i]->dst.write || prev[i]->is_op3) && !prev[i]->dst.rel) {
+               if (prev[i] && alu_writes(prev[i]) && !prev[i]->dst.rel) {
+
+                       if (is_alu_64bit_inst(prev[i])) {
+                               gpr[i] = -1;
+                               continue;
+                       }
+
                        gpr[i] = prev[i]->dst.sel;
                        /* cube writes more than PV.X */
                        if (is_alu_reduction_inst(bc, prev[i]))
@@ -588,10 +632,12 @@ static int replace_gpr_with_pv_ps(struct r600_bytecode *bc,
 
        for (i = 0; i < max_slots; ++i) {
                struct r600_bytecode_alu *alu = slots[i];
-               if(!alu)
+               if (!alu)
                        continue;
 
-               num_src = r600_bytecode_get_num_operands(bc, alu);
+               if (is_alu_64bit_inst(alu))
+                       continue;
+               num_src = r600_bytecode_get_num_operands(alu);
                for (src = 0; src < num_src; ++src) {
                        if (!is_gpr(alu->src[src].sel) || alu->src[src].rel)
                                continue;
@@ -621,7 +667,7 @@ static int replace_gpr_with_pv_ps(struct r600_bytecode *bc,
        return 0;
 }
 
-void r600_bytecode_special_constants(uint32_t value, unsigned *sel, unsigned *neg)
+void r600_bytecode_special_constants(uint32_t value, unsigned *sel, unsigned *neg, unsigned abs)
 {
        switch(value) {
        case 0:
@@ -641,11 +687,11 @@ void r600_bytecode_special_constants(uint32_t value, unsigned *sel, unsigned *ne
                break;
        case 0xBF800000: /* -1.0f */
                *sel = V_SQ_ALU_SRC_1;
-               *neg ^= 1;
+               *neg ^= !abs;
                break;
        case 0xBF000000: /* -0.5f */
                *sel = V_SQ_ALU_SRC_0_5;
-               *neg ^= 1;
+               *neg ^= !abs;
                break;
        default:
                *sel = V_SQ_ALU_SRC_LITERAL;
@@ -654,10 +700,10 @@ void r600_bytecode_special_constants(uint32_t value, unsigned *sel, unsigned *ne
 }
 
 /* compute how many literal are needed */
-static int r600_bytecode_alu_nliterals(struct r600_bytecode *bc, struct r600_bytecode_alu *alu,
+static int r600_bytecode_alu_nliterals(struct r600_bytecode_alu *alu,
                                 uint32_t literal[4], unsigned *nliteral)
 {
-       unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
+       unsigned num_src = r600_bytecode_get_num_operands(alu);
        unsigned i, j;
 
        for (i = 0; i < num_src; ++i) {
@@ -680,11 +726,10 @@ static int r600_bytecode_alu_nliterals(struct r600_bytecode *bc, struct r600_byt
        return 0;
 }
 
-static void r600_bytecode_alu_adjust_literals(struct r600_bytecode *bc,
-                                       struct r600_bytecode_alu *alu,
-                                       uint32_t literal[4], unsigned nliteral)
+static void r600_bytecode_alu_adjust_literals(struct r600_bytecode_alu *alu,
+                                             uint32_t literal[4], unsigned nliteral)
 {
-       unsigned num_src = r600_bytecode_get_num_operands(bc, alu);
+       unsigned num_src = r600_bytecode_get_num_operands(alu);
        unsigned i, j;
 
        for (i = 0; i < num_src; ++i) {
@@ -722,13 +767,13 @@ static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu
                if (prev[i]) {
                      if (prev[i]->pred_sel)
                              return 0;
-                     if (is_alu_once_inst(bc, prev[i]))
+                     if (is_alu_once_inst(prev[i]))
                              return 0;
                }
                if (slots[i]) {
                        if (slots[i]->pred_sel)
                                return 0;
-                       if (is_alu_once_inst(bc, slots[i]))
+                       if (is_alu_once_inst(slots[i]))
                                return 0;
                }
        }
@@ -741,26 +786,28 @@ static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu
 
                /* check number of literals */
                if (prev[i]) {
-                       if (r600_bytecode_alu_nliterals(bc, prev[i], literal, &nliteral))
+                       if (r600_bytecode_alu_nliterals(prev[i], literal, &nliteral))
                                return 0;
-                       if (r600_bytecode_alu_nliterals(bc, prev[i], prev_literal, &prev_nliteral))
+                       if (r600_bytecode_alu_nliterals(prev[i], prev_literal, &prev_nliteral))
                                return 0;
-                       if (is_alu_mova_inst(bc, prev[i])) {
+                       if (is_alu_mova_inst(prev[i])) {
                                if (have_rel)
                                        return 0;
                                have_mova = 1;
                        }
 
-                       if (alu_uses_rel(bc, prev[i])) {
+                       if (alu_uses_rel(prev[i])) {
                                if (have_mova) {
                                        return 0;
                                }
                                have_rel = 1;
                        }
+                       if (alu_uses_lds(prev[i]))
+                               return 0;
 
-                       num_once_inst += is_alu_once_inst(bc, prev[i]);
+                       num_once_inst += is_alu_once_inst(prev[i]);
                }
-               if (slots[i] && r600_bytecode_alu_nliterals(bc, slots[i], literal, &nliteral))
+               if (slots[i] && r600_bytecode_alu_nliterals(slots[i], literal, &nliteral))
                        return 0;
 
                /* Let's check used slots. */
@@ -770,13 +817,13 @@ static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu
                } else if (prev[i] && slots[i]) {
                        if (max_slots == 5 && result[4] == NULL && prev[4] == NULL && slots[4] == NULL) {
                                /* Trans unit is still free try to use it. */
-                               if (is_alu_any_unit_inst(bc, slots[i])) {
+                               if (is_alu_any_unit_inst(bc, slots[i]) && !alu_uses_lds(slots[i])) {
                                        result[i] = prev[i];
                                        result[4] = slots[i];
                                } else if (is_alu_any_unit_inst(bc, prev[i])) {
                                        if (slots[i]->dst.sel == prev[i]->dst.sel &&
-                                               (slots[i]->dst.write == 1 || slots[i]->is_op3) &&
-                                               (prev[i]->dst.write == 1 || prev[i]->is_op3))
+                                           alu_writes(slots[i]) &&
+                                           alu_writes(prev[i]))
                                                return 0;
 
                                        result[i] = slots[i];
@@ -791,28 +838,28 @@ static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu
                        if (max_slots == 5 && slots[i] && prev[4] &&
                                        slots[i]->dst.sel == prev[4]->dst.sel &&
                                        slots[i]->dst.chan == prev[4]->dst.chan &&
-                                       (slots[i]->dst.write == 1 || slots[i]->is_op3) &&
-                                       (prev[4]->dst.write == 1 || prev[4]->is_op3))
+                                       alu_writes(slots[i]) &&
+                                       alu_writes(prev[4]))
                                return 0;
 
                        result[i] = slots[i];
                }
 
                alu = slots[i];
-               num_once_inst += is_alu_once_inst(bc, alu);
+               num_once_inst += is_alu_once_inst(alu);
 
                /* don't reschedule NOPs */
-               if (is_nop_inst(bc, alu))
+               if (is_nop_inst(alu))
                        return 0;
 
-               if (is_alu_mova_inst(bc, alu)) {
+               if (is_alu_mova_inst(alu)) {
                        if (have_rel) {
                                return 0;
                        }
                        have_mova = 1;
                }
 
-               if (alu_uses_rel(bc, alu)) {
+               if (alu_uses_rel(alu)) {
                        if (have_mova) {
                                return 0;
                        }
@@ -824,7 +871,7 @@ static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu
                        return 0; /* data hazard with MOVA */
 
                /* Let's check source gprs */
-               num_src = r600_bytecode_get_num_operands(bc, alu);
+               num_src = r600_bytecode_get_num_operands(alu);
                for (src = 0; src < num_src; ++src) {
 
                        /* Constants don't matter. */
@@ -832,7 +879,7 @@ static int merge_inst_groups(struct r600_bytecode *bc, struct r600_bytecode_alu
                                continue;
 
                        for (j = 0; j < max_slots; ++j) {
-                               if (!prev[j] || !(prev[j]->dst.write || prev[j]->is_op3))
+                               if (!prev[j] || !alu_writes(prev[j]))
                                        continue;
 
                                /* If it's relative then we can't determin which gpr is really used. */
@@ -970,7 +1017,7 @@ static int r600_bytecode_alloc_inst_kcache_lines(struct r600_bytecode *bc,
        return 0;
 }
 
-static int r600_bytecode_assign_kcache_banks(struct r600_bytecode *bc,
+static int r600_bytecode_assign_kcache_banks(
                struct r600_bytecode_alu *alu,
                struct r600_bytecode_kcache * kcache)
 {
@@ -1129,7 +1176,7 @@ int r600_bytecode_add_alu_type(struct r600_bytecode *bc,
        struct r600_bytecode_alu *lalu;
        int i, r;
 
-       if (nalu == NULL)
+       if (!nalu)
                return -ENOMEM;
        memcpy(nalu, alu, sizeof(struct r600_bytecode_alu));
 
@@ -1194,7 +1241,7 @@ int r600_bytecode_add_alu_type(struct r600_bytecode *bc,
                }
                if (nalu->src[i].sel == V_SQ_ALU_SRC_LITERAL)
                        r600_bytecode_special_constants(nalu->src[i].value,
-                               &nalu->src[i].sel, &nalu->src[i].neg);
+                               &nalu->src[i].sel, &nalu->src[i].neg, nalu->src[i].abs);
        }
        if (nalu->dst.sel >= bc->ngpr) {
                bc->ngpr = nalu->dst.sel + 1;
@@ -1232,7 +1279,7 @@ int r600_bytecode_add_alu_type(struct r600_bytecode *bc,
 
                for (i = 0, nliteral = 0; i < max_slots; i++) {
                        if (slots[i]) {
-                               r = r600_bytecode_alu_nliterals(bc, slots[i], literal, &nliteral);
+                               r = r600_bytecode_alu_nliterals(slots[i], literal, &nliteral);
                                if (r)
                                        return r;
                        }
@@ -1281,23 +1328,25 @@ static unsigned r600_bytecode_num_tex_and_vtx_instructions(const struct r600_byt
 static inline boolean last_inst_was_not_vtx_fetch(struct r600_bytecode *bc)
 {
        return !((r600_isa_cf(bc->cf_last->op)->flags & CF_FETCH) &&
-                       (bc->chip_class == CAYMAN ||
-                       bc->cf_last->op != CF_OP_TEX));
+                bc->cf_last->op != CF_OP_GDS &&
+                (bc->chip_class == CAYMAN ||
+                 bc->cf_last->op != CF_OP_TEX));
 }
 
-int r600_bytecode_add_vtx(struct r600_bytecode *bc, const struct r600_bytecode_vtx *vtx)
+static int r600_bytecode_add_vtx_internal(struct r600_bytecode *bc, const struct r600_bytecode_vtx *vtx,
+                                         bool use_tc)
 {
        struct r600_bytecode_vtx *nvtx = r600_bytecode_vtx();
        int r;
 
-       if (nvtx == NULL)
+       if (!nvtx)
                return -ENOMEM;
        memcpy(nvtx, vtx, sizeof(struct r600_bytecode_vtx));
 
        /* Load index register if required */
        if (bc->chip_class >= EVERGREEN) {
                if (vtx->buffer_index_mode)
-                       egcm_load_index_reg(bc, 0, false);
+                       egcm_load_index_reg(bc, vtx->buffer_index_mode - 1, false);
        }
 
        /* cf can contains only alu or only vtx or only tex */
@@ -1312,9 +1361,14 @@ int r600_bytecode_add_vtx(struct r600_bytecode *bc, const struct r600_bytecode_v
                switch (bc->chip_class) {
                case R600:
                case R700:
-               case EVERGREEN:
                        bc->cf_last->op = CF_OP_VTX;
                        break;
+               case EVERGREEN:
+                       if (use_tc)
+                               bc->cf_last->op = CF_OP_TEX;
+                       else
+                               bc->cf_last->op = CF_OP_VTX;
+                       break;
                case CAYMAN:
                        bc->cf_last->op = CF_OP_TEX;
                        break;
@@ -1337,12 +1391,22 @@ int r600_bytecode_add_vtx(struct r600_bytecode *bc, const struct r600_bytecode_v
        return 0;
 }
 
+int r600_bytecode_add_vtx(struct r600_bytecode *bc, const struct r600_bytecode_vtx *vtx)
+{
+       return r600_bytecode_add_vtx_internal(bc, vtx, false);
+}
+
+int r600_bytecode_add_vtx_tc(struct r600_bytecode *bc, const struct r600_bytecode_vtx *vtx)
+{
+       return r600_bytecode_add_vtx_internal(bc, vtx, true);
+}
+
 int r600_bytecode_add_tex(struct r600_bytecode *bc, const struct r600_bytecode_tex *tex)
 {
        struct r600_bytecode_tex *ntex = r600_bytecode_tex();
        int r;
 
-       if (ntex == NULL)
+       if (!ntex)
                return -ENOMEM;
        memcpy(ntex, tex, sizeof(struct r600_bytecode_tex));
 
@@ -1393,6 +1457,38 @@ int r600_bytecode_add_tex(struct r600_bytecode *bc, const struct r600_bytecode_t
        return 0;
 }
 
+int r600_bytecode_add_gds(struct r600_bytecode *bc, const struct r600_bytecode_gds *gds)
+{
+       struct r600_bytecode_gds *ngds = r600_bytecode_gds();
+       int r;
+
+       if (ngds == NULL)
+               return -ENOMEM;
+       memcpy(ngds, gds, sizeof(struct r600_bytecode_gds));
+
+       if (bc->chip_class >= EVERGREEN) {
+               if (gds->uav_index_mode)
+                       egcm_load_index_reg(bc, gds->uav_index_mode - 1, false);
+       }
+
+       if (bc->cf_last == NULL ||
+           bc->cf_last->op != CF_OP_GDS ||
+           bc->force_add_cf) {
+               r = r600_bytecode_add_cf(bc);
+               if (r) {
+                       free(ngds);
+                       return r;
+               }
+               bc->cf_last->op = CF_OP_GDS;
+       }
+
+       LIST_ADDTAIL(&ngds->list, &bc->cf_last->gds);
+       bc->cf_last->ndw += 4; /* each GDS uses 4 dwords */
+       if ((bc->cf_last->ndw / 4) >= r600_bytecode_num_tex_and_vtx_instructions(bc))
+               bc->force_add_cf = 1;
+       return 0;
+}
+
 int r600_bytecode_add_cfinst(struct r600_bytecode *bc, unsigned op)
 {
        int r;
@@ -1604,16 +1700,19 @@ int r600_bytecode_build(struct r600_bytecode *bc)
        struct r600_bytecode_alu *alu;
        struct r600_bytecode_vtx *vtx;
        struct r600_bytecode_tex *tex;
+       struct r600_bytecode_gds *gds;
        uint32_t literal[4];
        unsigned nliteral;
        unsigned addr;
        int i, r;
 
-       if (!bc->nstack) // If not 0, Stack_size already provided by llvm
-               bc->nstack = bc->stack.max_entries;
-
-       if (bc->type == TGSI_PROCESSOR_VERTEX && !bc->nstack) {
-               bc->nstack = 1;
+       if (!bc->nstack) { // If not 0, Stack_size already provided by llvm
+               if (bc->stack.max_entries)
+                       bc->nstack = bc->stack.max_entries;
+               else if (bc->type == PIPE_SHADER_VERTEX ||
+                        bc->type == PIPE_SHADER_TESS_EVAL ||
+                        bc->type == PIPE_SHADER_TESS_CTRL)
+                       bc->nstack = 1;
        }
 
        /* first path compute addr of each CF block */
@@ -1645,21 +1744,23 @@ int r600_bytecode_build(struct r600_bytecode *bc)
                        nliteral = 0;
                        memset(literal, 0, sizeof(literal));
                        LIST_FOR_EACH_ENTRY(alu, &cf->alu, list) {
-                               r = r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
+                               r = r600_bytecode_alu_nliterals(alu, literal, &nliteral);
                                if (r)
                                        return r;
-                               r600_bytecode_alu_adjust_literals(bc, alu, literal, nliteral);
-                               r600_bytecode_assign_kcache_banks(bc, alu, cf->kcache);
+                               r600_bytecode_alu_adjust_literals(alu, literal, nliteral);
+                               r600_bytecode_assign_kcache_banks(alu, cf->kcache);
 
                                switch(bc->chip_class) {
                                case R600:
                                        r = r600_bytecode_alu_build(bc, alu, addr);
                                        break;
                                case R700:
-                               case EVERGREEN: /* eg alu is same encoding as r700 */
-                               case CAYMAN:
                                        r = r700_bytecode_alu_build(bc, alu, addr);
                                        break;
+                               case EVERGREEN:
+                               case CAYMAN:
+                                       r = eg_bytecode_alu_build(bc, alu, addr);
+                                       break;
                                default:
                                        R600_ERR("unknown chip class %d.\n", bc->chip_class);
                                        return -EINVAL;
@@ -1682,6 +1783,14 @@ int r600_bytecode_build(struct r600_bytecode *bc)
                                        return r;
                                addr += 4;
                        }
+               } else if (cf->op == CF_OP_GDS) {
+                       assert(bc->chip_class >= EVERGREEN);
+                       LIST_FOR_EACH_ENTRY(gds, &cf->gds, list) {
+                               r = eg_bytecode_gds_build(bc, gds, addr);
+                               if (r)
+                                       return r;
+                               addr += 4;
+                       }
                } else if (cf->op == CF_OP_TEX) {
                        LIST_FOR_EACH_ENTRY(vtx, &cf->vtx, list) {
                                assert(bc->chip_class >= EVERGREEN);
@@ -1712,6 +1821,7 @@ void r600_bytecode_clear(struct r600_bytecode *bc)
                struct r600_bytecode_alu *alu = NULL, *next_alu;
                struct r600_bytecode_tex *tex = NULL, *next_tex;
                struct r600_bytecode_tex *vtx = NULL, *next_vtx;
+               struct r600_bytecode_gds *gds = NULL, *next_gds;
 
                LIST_FOR_EACH_ENTRY_SAFE(alu, next_alu, &cf->alu, list) {
                        free(alu);
@@ -1731,6 +1841,12 @@ void r600_bytecode_clear(struct r600_bytecode *bc)
 
                LIST_INITHEAD(&cf->vtx);
 
+               LIST_FOR_EACH_ENTRY_SAFE(gds, next_gds, &cf->gds, list) {
+                       free(gds);
+               }
+
+               LIST_INITHEAD(&cf->gds);
+
                free(cf);
        }
 
@@ -1776,7 +1892,7 @@ static int print_dst(struct r600_bytecode_alu *alu)
                reg_char = 'T';
        }
 
-       if (alu->dst.write || alu->is_op3) {
+       if (alu_writes(alu)) {
                o += fprintf(stderr, "%c", reg_char);
                o += print_sel(alu->dst.sel, alu->dst.rel, alu->index_mode, 0);
        } else {
@@ -1831,6 +1947,28 @@ static int print_src(struct r600_bytecode_alu *alu, unsigned idx)
                need_sel = 0;
                need_chan = 0;
                switch (sel) {
+               case EG_V_SQ_ALU_SRC_LDS_DIRECT_A:
+                       o += fprintf(stderr, "LDS_A[0x%08X]", src->value);
+                       break;
+               case EG_V_SQ_ALU_SRC_LDS_DIRECT_B:
+                       o += fprintf(stderr, "LDS_B[0x%08X]", src->value);
+                       break;
+               case EG_V_SQ_ALU_SRC_LDS_OQ_A:
+                       o += fprintf(stderr, "LDS_OQ_A");
+                       need_chan = 1;
+                       break;
+               case EG_V_SQ_ALU_SRC_LDS_OQ_B:
+                       o += fprintf(stderr, "LDS_OQ_B");
+                       need_chan = 1;
+                       break;
+               case EG_V_SQ_ALU_SRC_LDS_OQ_A_POP:
+                       o += fprintf(stderr, "LDS_OQ_A_POP");
+                       need_chan = 1;
+                       break;
+               case EG_V_SQ_ALU_SRC_LDS_OQ_B_POP:
+                       o += fprintf(stderr, "LDS_OQ_B_POP");
+                       need_chan = 1;
+                       break;
                case V_SQ_ALU_SRC_PS:
                        o += fprintf(stderr, "PS");
                        break;
@@ -1839,7 +1977,7 @@ static int print_src(struct r600_bytecode_alu *alu, unsigned idx)
                        need_chan = 1;
                        break;
                case V_SQ_ALU_SRC_LITERAL:
-                       o += fprintf(stderr, "[0x%08X %f]", src->value, *(float*)&src->value);
+                       o += fprintf(stderr, "[0x%08X %f]", src->value, u_bitcast_u2f(src->value));
                        break;
                case V_SQ_ALU_SRC_0_5:
                        o += fprintf(stderr, "0.5");
@@ -1892,6 +2030,7 @@ void r600_bytecode_disasm(struct r600_bytecode *bc)
        struct r600_bytecode_alu *alu = NULL;
        struct r600_bytecode_vtx *vtx = NULL;
        struct r600_bytecode_tex *tex = NULL;
+       struct r600_bytecode_gds *gds = NULL;
 
        unsigned i, id, ngr = 0, last;
        uint32_t literal[4];
@@ -2008,7 +2147,8 @@ void r600_bytecode_disasm(struct r600_bytecode *bc)
                                                o += print_swizzle(7);
                                }
 
-                               if (cf->output.type == V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_WRITE_IND)
+                               if (cf->output.type == V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_WRITE_IND ||
+                                   cf->output.type == V_SQ_CF_ALLOC_EXPORT_WORD0_SQ_EXPORT_READ_IND)
                                        o += fprintf(stderr, " R%d", cf->output.index_gpr);
 
                                o += print_indent(o, 67);
@@ -2029,6 +2169,8 @@ void r600_bytecode_disasm(struct r600_bytecode *bc)
                                        fprintf(stderr, "CND:%X ", cf->cond);
                                if (cf->pop_count)
                                        fprintf(stderr, "POP:%X ", cf->pop_count);
+                               if (cf->count && (cfop->flags & CF_EMIT))
+                                       fprintf(stderr, "STREAM%d ", cf->count);
                                if (cf->end_of_program)
                                        fprintf(stderr, "EOP ");
                                fprintf(stderr, "\n");
@@ -2043,7 +2185,7 @@ void r600_bytecode_disasm(struct r600_bytecode *bc)
                        const struct alu_op_info *aop = r600_isa_alu(alu->op);
                        int o = 0;
 
-                       r600_bytecode_alu_nliterals(bc, alu, literal, &nliteral);
+                       r600_bytecode_alu_nliterals(alu, literal, &nliteral);
                        o += fprintf(stderr, " %04d %08X %08X  ", id, bc->bytecode[id], bc->bytecode[id+1]);
                        if (last)
                                o += fprintf(stderr, "%4d ", ++ngr);
@@ -2173,6 +2315,38 @@ void r600_bytecode_disasm(struct r600_bytecode *bc)
 
                        id += 4;
                }
+
+               LIST_FOR_EACH_ENTRY(gds, &cf->gds, list) {
+                       int o = 0;
+                       o += fprintf(stderr, " %04d %08X %08X %08X   ", id, bc->bytecode[id],
+                                       bc->bytecode[id + 1], bc->bytecode[id + 2]);
+
+                       o += fprintf(stderr, "%s ", r600_isa_fetch(gds->op)->name);
+
+                       if (gds->op != FETCH_OP_TF_WRITE) {
+                               o += fprintf(stderr, "R%d.", gds->dst_gpr);
+                               o += print_swizzle(gds->dst_sel_x);
+                               o += print_swizzle(gds->dst_sel_y);
+                               o += print_swizzle(gds->dst_sel_z);
+                               o += print_swizzle(gds->dst_sel_w);
+                       }
+
+                       o += fprintf(stderr, ", R%d.", gds->src_gpr);
+                       o += print_swizzle(gds->src_sel_x);
+                       o += print_swizzle(gds->src_sel_y);
+                       o += print_swizzle(gds->src_sel_z);
+
+                       if (gds->op != FETCH_OP_TF_WRITE) {
+                               o += fprintf(stderr, ", R%d.", gds->src_gpr2);
+                       }
+                       if (gds->alloc_consume) {
+                               o += fprintf(stderr, " UAV: %d", gds->uav_id);
+                               if (gds->uav_index_mode)
+                                       o += fprintf(stderr, "[%s]", index_mode[gds->uav_index_mode]);
+                       }
+                       fprintf(stderr, "\n");
+                       id += 4;
+               }
        }
 
        fprintf(stderr, "--------------------------------------\n");
@@ -2196,6 +2370,18 @@ void r600_vertex_data_type(enum pipe_format pformat,
                return;
        }
 
+       if (pformat == PIPE_FORMAT_B5G6R5_UNORM) {
+               *format = FMT_5_6_5;
+               *endian = r600_endian_swap(16);
+               return;
+       }
+
+       if (pformat == PIPE_FORMAT_B5G5R5A1_UNORM) {
+               *format = FMT_1_5_5_5;
+               *endian = r600_endian_swap(16);
+               return;
+       }
+
        desc = util_format_description(pformat);
        if (desc->layout != UTIL_FORMAT_LAYOUT_PLAIN) {
                goto out_unknown;
@@ -2399,7 +2585,7 @@ void *r600_create_vertex_fetch_shader(struct pipe_context *ctx,
                                      &format, &num_format, &format_comp, &endian);
 
                desc = util_format_description(elements[i].src_format);
-               if (desc == NULL) {
+               if (!desc) {
                        r600_bytecode_clear(&bc);
                        R600_ERR("unknown format %d\n", elements[i].src_format);
                        return NULL;
@@ -2468,7 +2654,8 @@ void *r600_create_vertex_fetch_shader(struct pipe_context *ctx,
                return NULL;
        }
 
-       u_suballocator_alloc(rctx->allocator_fetch_shader, fs_size, &shader->offset,
+       u_suballocator_alloc(rctx->allocator_fetch_shader, fs_size, 256,
+                            &shader->offset,
                             (struct pipe_resource**)&shader->buffer);
        if (!shader->buffer) {
                r600_bytecode_clear(&bc);
@@ -2486,7 +2673,7 @@ void *r600_create_vertex_fetch_shader(struct pipe_context *ctx,
        } else {
                memcpy(bytecode, bc.bytecode, fs_size);
        }
-       rctx->b.ws->buffer_unmap(shader->buffer->cs_buf);
+       rctx->b.ws->buffer_unmap(shader->buffer->buf);
 
        r600_bytecode_clear(&bc);
        return shader;