gpr(), lt(), slots(),
max_slots(sh.get_ctx().is_cayman() ? 4 : 5),
has_mova(), uses_ar(), has_predset(), has_kill(),
- updates_exec_mask(), chan_count(), interp_param(), next_id() {
+ updates_exec_mask(), consumes_lds_oqa(), produces_lds_oqa(), chan_count(), interp_param(), next_id() {
available_slots = sh.get_ctx().has_trans ? 0x1F : 0x0F;
}
memset(slots, 0, sizeof(slots));
vmap.clear();
next_id = 0;
+ produces_lds_oqa = 0;
+ consumes_lds_oqa = 0;
has_mova = false;
uses_ar = false;
has_predset = false;
has_mova |= (flags & AF_MOVA);
has_predset |= (flags & AF_ANY_PRED);
uses_ar |= n->uses_ar();
-
+ consumes_lds_oqa |= n->consumes_lds_oq();
+ produces_lds_oqa |= n->produces_lds_oq();
if (flags & AF_ANY_PRED) {
if (n->dst[2] != NULL)
updates_exec_mask = true;
}
int post_scheduler::run() {
- run_on(sh.root);
- return 0;
+ return run_on(sh.root) ? 0 : 1;
}
-void post_scheduler::run_on(container_node* n) {
-
+bool post_scheduler::run_on(container_node* n) {
+ int r = true;
for (node_riterator I = n->rbegin(), E = n->rend(); I != E; ++I) {
if (I->is_container()) {
if (I->subtype == NST_BB) {
bb_node* bb = static_cast<bb_node*>(*I);
- schedule_bb(bb);
+ r = schedule_bb(bb);
} else {
- run_on(static_cast<container_node*>(*I));
+ r = run_on(static_cast<container_node*>(*I));
}
+ if (!r)
+ break;
}
}
+ return r;
}
void post_scheduler::init_uc_val(container_node *c, value *v) {
return F == ucm.end() ? 0 : F->second;
}
-void post_scheduler::schedule_bb(bb_node* bb) {
+bool post_scheduler::schedule_bb(bb_node* bb) {
PSC_DUMP(
sblog << "scheduling BB " << bb->id << "\n";
if (!pending.empty())
if (n->is_alu_clause()) {
n->remove();
- process_alu(static_cast<container_node*>(n));
- continue;
+ bool r = process_alu(static_cast<container_node*>(n));
+ if (r)
+ continue;
+ return false;
}
n->remove();
}
this->cur_bb = NULL;
+ return true;
}
void post_scheduler::init_regmap() {
a->dst.resize(1); // Dummy needed for recolor
PSC_DUMP(
- sblog << "created IDX load: "
+ sblog << "created IDX load: ";
dump::dump_op(a);
sblog << "\n";
);
sblog << " ";
);
- if (f->bc.sampler_index_mode != V_SQ_CF_INDEX_NONE) {
+ // TODO: If same values used can avoid reloading index register
+ if (f->bc.sampler_index_mode != V_SQ_CF_INDEX_NONE ||
+ f->bc.resource_index_mode != V_SQ_CF_INDEX_NONE) {
+ unsigned index_mode = f->bc.sampler_index_mode != V_SQ_CF_INDEX_NONE ?
+ f->bc.sampler_index_mode : f->bc.resource_index_mode;
+
// Currently require prior opt passes to use one TEX per indexed op
assert(f->parent->count() == 1);
value *v = f->src.back(); // Last src is index offset
+ assert(v);
cur_bb->push_front(c);
- load_index_register(v, f->bc.sampler_index_mode);
+ load_index_register(v, index_mode);
f->src.pop_back(); // Don't need index value any more
return;
cur_bb->push_front(c);
}
-void post_scheduler::process_alu(container_node *c) {
+bool post_scheduler::process_alu(container_node *c) {
if (c->empty())
- return;
+ return true;
ucm.clear();
alu.reset();
if (uc) {
n->remove();
+
pending.push_back(n);
PSC_DUMP( sblog << "pending\n"; );
} else {
}
}
- schedule_alu(c);
+ return schedule_alu(c);
}
void post_scheduler::update_local_interferences() {
}
}
+void post_scheduler::emit_index_registers() {
+ for (unsigned i = 0; i < 2; i++) {
+ if (alu.current_idx[i]) {
+ regmap = prev_regmap;
+ alu.discard_current_group();
+
+ load_index_register(alu.current_idx[i], KC_INDEX_0 + i);
+ alu.current_idx[i] = NULL;
+ }
+ }
+}
+
void post_scheduler::emit_clause() {
if (alu.current_ar) {
emit_load_ar();
process_group();
+ if (!alu.check_clause_limits()) {
+ // Can't happen since clause only contains MOVA/CF_SET_IDX0/1
+ }
alu.emit_group();
}
- alu.emit_clause(cur_bb);
+ if (!alu.is_empty()) {
+ alu.emit_clause(cur_bb);
+ }
+
+ emit_index_registers();
}
-void post_scheduler::schedule_alu(container_node *c) {
+bool post_scheduler::schedule_alu(container_node *c) {
assert(!ready.empty() || !ready_copies.empty());
- while (1) {
-
+ bool improving = true;
+ int last_pending = pending.count();
+ while (improving) {
prev_regmap = regmap;
-
if (!prepare_alu_group()) {
+
+ int new_pending = pending.count();
+ improving = (new_pending < last_pending) || (last_pending == 0);
+ last_pending = new_pending;
+
+ if (alu.current_idx[0] || alu.current_idx[1]) {
+ regmap = prev_regmap;
+ emit_clause();
+ init_globals(live, false);
+
+ continue;
+ }
+
if (alu.current_ar) {
emit_load_ar();
continue;
regmap = prev_regmap;
emit_clause();
init_globals(live, false);
+
continue;
}
dump::dump_op_list(&pending);
assert(!"unscheduled pending instructions");
}
+ return improving;
}
void post_scheduler::add_interferences(value *v, sb_bitset &rb, val_set &vs) {
}
bool post_scheduler::map_src_vec(vvec &vv, bool src) {
+ if (src) {
+ // Handle possible UBO indexing
+ bool ubo_indexing[2] = { false, false };
+ for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I) {
+ value *v = *I;
+ if (!v)
+ continue;
+
+ if (v->is_kcache()) {
+ unsigned index_mode = v->select.kcache_index_mode();
+ if (index_mode == KC_INDEX_0 || index_mode == KC_INDEX_1) {
+ ubo_indexing[index_mode - KC_INDEX_0] = true;
+ }
+ }
+ }
+
+ // idx values stored at end of src vec, see bc_parser::prepare_alu_group
+ for (unsigned i = 2; i != 0; i--) {
+ if (ubo_indexing[i-1]) {
+ // TODO: skip adding value to kcache reservation somehow, causes
+ // unnecessary group breaks and cache line locks
+ value *v = vv.back();
+ if (alu.current_idx[i-1] && alu.current_idx[i-1] != v) {
+ PSC_DUMP(
+ sblog << "IDX" << i-1 << " already set to " <<
+ *alu.current_idx[i-1] << ", trying to set " << *v << "\n";
+ );
+ return false;
+ }
+
+ alu.current_idx[i-1] = v;
+ PSC_DUMP(sblog << "IDX" << i-1 << " set to " << *v << "\n";);
+ }
+ }
+ }
+
for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I) {
value *v = *I;
if (!v)
sblog << " current_AR: " << *alu.current_ar << "\n";
if (alu.current_pr)
sblog << " current_PR: " << *alu.current_pr << "\n";
+ if (alu.current_idx[0])
+ sblog << " current IDX0: " << *alu.current_idx[0] << "\n";
+ if (alu.current_idx[1])
+ sblog << " current IDX1: " << *alu.current_idx[1] << "\n";
}
void post_scheduler::recolor_locals() {
unsigned avail_slots = rt.avail_slots();
+ // Cannot schedule in same clause as instructions using this index value
+ if (!n->dst.empty() && n->dst[0] &&
+ (n->dst[0] == alu.current_idx[0] || n->dst[0] == alu.current_idx[1])) {
+ PSC_DUMP(sblog << " CF_IDX source: " << *n->dst[0] << "\n";);
+ return 0;
+ }
+
if (n->is_alu_packed()) {
alu_packed_node *p = static_cast<alu_packed_node*>(n);
unsigned slots = p->get_slot_mask();
value *d = a->dst.empty() ? NULL : a->dst[0];
if (d && d->is_special_reg()) {
- assert((a->bc.op_ptr->flags & AF_MOVA) || d->is_geometry_emit());
+ assert((a->bc.op_ptr->flags & AF_MOVA) || d->is_geometry_emit() || d->is_lds_oq() || d->is_lds_access());
d = NULL;
}
void alu_clause_tracker::reset() {
group = 0;
slot_count = 0;
+ outstanding_lds_oqa_reads = 0;
grp0.reset();
grp1.reset();
}
: sh(sh), kt(sh.get_ctx().hw_class), slot_count(),
grp0(sh), grp1(sh),
group(), clause(),
- push_exec_mask(),
- current_ar(), current_pr() {}
+ push_exec_mask(), outstanding_lds_oqa_reads(),
+ current_ar(), current_pr(), current_idx() {}
void alu_clause_tracker::emit_group() {
clause->push_front(g);
+ outstanding_lds_oqa_reads += grp().get_consumes_lds_oqa();
+ outstanding_lds_oqa_reads -= grp().get_produces_lds_oqa();
slot_count += grp().slot_count();
new_group();
kt.init_clause(clause->bc);
+ assert(!outstanding_lds_oqa_reads);
assert(!current_ar);
assert(!current_pr);
// reserving slots to load AR and PR values
unsigned reserve_slots = (current_ar ? 1 : 0) + (current_pr ? 1 : 0);
+ // ...and index registers
+ reserve_slots += (current_idx[0] != NULL) + (current_idx[1] != NULL);
+
+ if (gt.get_consumes_lds_oqa() && !outstanding_lds_oqa_reads)
+ reserve_slots += 60;
if (slot_count + slots > MAX_ALU_SLOTS - reserve_slots)
return false;
unsigned cnt = 0;
for (unsigned i = 0; i < sel_count; ++i) {
- unsigned line = rp[i];
+ unsigned line = rp[i] & 0x1fffffffu;
+ unsigned index_mode = rp[i] >> 29;
if (!line)
return cnt;
--line;
line = (sel_count == 2) ? line >> 5 : line >> 6;
+ line |= index_mode << 29;
if (lines.insert(line).second)
++cnt;
memcpy(old_kc, kc, sizeof(kc));
for (kc_lines::iterator I = lines.begin(), E = lines.end(); I != E; ++I) {
- unsigned line = *I;
+ unsigned index_mode = *I >> 29;
+ unsigned line = *I & 0x1fffffffu;
unsigned bank = line >> 8;
+ assert(index_mode <= KC_INDEX_INVALID);
line &= 0xFF;
- if (c && (bank == kc[c-1].bank) && (kc[c-1].addr + 1 == line))
- ++kc[c-1].mode;
- else {
+ if (c && (bank == kc[c-1].bank) && (kc[c-1].addr + 1 == line) &&
+ kc[c-1].index_mode == index_mode)
+ {
+ kc[c-1].mode = KC_LOCK_2;
+ } else {
if (c == max_kcs) {
memcpy(kc, old_kc, sizeof(kc));
return false;
kc[c].bank = bank;
kc[c].addr = line;
+ kc[c].index_mode = index_mode;
++c;
}
}
projects / mesa.git / blobdiff