projects / mesa.git / blob
? search:


f7769da05030373898de0bad6033a08ac0444915
[mesa.git] / src / gallium / drivers / nouveau / nvc0 / nvc0_vbo.c
1 /*
2 * Copyright 2010 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
24
25 #include "pipe/p_context.h"
26 #include "pipe/p_state.h"
27 #include "util/u_inlines.h"
28 #include "util/u_format.h"
29 #include "translate/translate.h"
30
31 #include "nvc0/nvc0_context.h"
32 #include "nvc0/nvc0_query_hw.h"
33 #include "nvc0/nvc0_resource.h"
34
35 #include "nvc0/nvc0_3d.xml.h"
36
37 void
38 nvc0_vertex_state_delete(struct pipe_context *pipe,
39 void *hwcso)
40 {
41 struct nvc0_vertex_stateobj *so = hwcso;
42
43 if (so->translate)
44 so->translate->release(so->translate);
45 FREE(hwcso);
46 }
47
48 void *
49 nvc0_vertex_state_create(struct pipe_context *pipe,
50 unsigned num_elements,
51 const struct pipe_vertex_element *elements)
52 {
53 struct nvc0_vertex_stateobj *so;
54 struct translate_key transkey;
55 unsigned i;
56 unsigned src_offset_max = 0;
57
58 so = MALLOC(sizeof(*so) +
59 num_elements * sizeof(struct nvc0_vertex_element));
60 if (!so)
61 return NULL;
62 so->num_elements = num_elements;
63 so->instance_elts = 0;
64 so->instance_bufs = 0;
65 so->shared_slots = false;
66 so->need_conversion = false;
67
68 memset(so->vb_access_size, 0, sizeof(so->vb_access_size));
69
70 for (i = 0; i < PIPE_MAX_ATTRIBS; ++i)
71 so->min_instance_div[i] = 0xffffffff;
72
73 transkey.nr_elements = 0;
74 transkey.output_stride = 0;
75
76 for (i = 0; i < num_elements; ++i) {
77 const struct pipe_vertex_element *ve = &elements[i];
78 const unsigned vbi = ve->vertex_buffer_index;
79 unsigned size;
80 enum pipe_format fmt = ve->src_format;
81
82 so->element[i].pipe = elements[i];
83 so->element[i].state = nvc0_format_table[fmt].vtx;
84
85 if (!so->element[i].state) {
86 switch (util_format_get_nr_components(fmt)) {
87 case 1: fmt = PIPE_FORMAT_R32_FLOAT; break;
88 case 2: fmt = PIPE_FORMAT_R32G32_FLOAT; break;
89 case 3: fmt = PIPE_FORMAT_R32G32B32_FLOAT; break;
90 case 4: fmt = PIPE_FORMAT_R32G32B32A32_FLOAT; break;
91 default:
92 assert(0);
93 FREE(so);
94 return NULL;
95 }
96 so->element[i].state = nvc0_format_table[fmt].vtx;
97 so->need_conversion = true;
98 pipe_debug_message(&nouveau_context(pipe)->debug, FALLBACK,
99 "Converting vertex element %d, no hw format %s",
100 i, util_format_name(ve->src_format));
101 }
102 size = util_format_get_blocksize(fmt);
103
104 src_offset_max = MAX2(src_offset_max, ve->src_offset);
105
106 if (so->vb_access_size[vbi] < (ve->src_offset + size))
107 so->vb_access_size[vbi] = ve->src_offset + size;
108
109 if (unlikely(ve->instance_divisor)) {
110 so->instance_elts |= 1 << i;
111 so->instance_bufs |= 1 << vbi;
112 if (ve->instance_divisor < so->min_instance_div[vbi])
113 so->min_instance_div[vbi] = ve->instance_divisor;
114 }
115
116 if (1) {
117 unsigned ca;
118 unsigned j = transkey.nr_elements++;
119
120 ca = util_format_description(fmt)->channel[0].size / 8;
121 if (ca != 1 && ca != 2)
122 ca = 4;
123
124 transkey.element[j].type = TRANSLATE_ELEMENT_NORMAL;
125 transkey.element[j].input_format = ve->src_format;
126 transkey.element[j].input_buffer = vbi;
127 transkey.element[j].input_offset = ve->src_offset;
128 transkey.element[j].instance_divisor = ve->instance_divisor;
129
130 transkey.output_stride = align(transkey.output_stride, ca);
131 transkey.element[j].output_format = fmt;
132 transkey.element[j].output_offset = transkey.output_stride;
133 transkey.output_stride += size;
134
135 so->element[i].state_alt = so->element[i].state;
136 so->element[i].state_alt |= transkey.element[j].output_offset << 7;
137 }
138
139 so->element[i].state |= i << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT;
140 }
141 transkey.output_stride = align(transkey.output_stride, 4);
142
143 so->size = transkey.output_stride;
144 so->translate = translate_create(&transkey);
145
146 if (so->instance_elts || src_offset_max >= (1 << 14))
147 return so;
148 so->shared_slots = true;
149
150 for (i = 0; i < num_elements; ++i) {
151 const unsigned b = elements[i].vertex_buffer_index;
152 const unsigned s = elements[i].src_offset;
153 so->element[i].state &= ~NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__MASK;
154 so->element[i].state |= b << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT;
155 so->element[i].state |= s << NVC0_3D_VERTEX_ATTRIB_FORMAT_OFFSET__SHIFT;
156 }
157 return so;
158 }
159
160 #define NVC0_3D_VERTEX_ATTRIB_INACTIVE \
161 NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT | \
162 NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32 | NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST
163
164 #define VTX_ATTR(a, c, t, s) \
165 ((NVC0_3D_VTX_ATTR_DEFINE_TYPE_##t) | \
166 (NVC0_3D_VTX_ATTR_DEFINE_SIZE_##s) | \
167 ((a) << NVC0_3D_VTX_ATTR_DEFINE_ATTR__SHIFT) | \
168 ((c) << NVC0_3D_VTX_ATTR_DEFINE_COMP__SHIFT))
169
170 static void
171 nvc0_set_constant_vertex_attrib(struct nvc0_context *nvc0, const unsigned a)
172 {
173 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
174 struct pipe_vertex_element *ve = &nvc0->vertex->element[a].pipe;
175 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[ve->vertex_buffer_index];
176 uint32_t mode;
177 const struct util_format_description *desc;
178 void *dst;
179 const void *src = (const uint8_t *)vb->user_buffer + ve->src_offset;
180 assert(!vb->buffer);
181
182 desc = util_format_description(ve->src_format);
183
184 PUSH_SPACE(push, 6);
185 BEGIN_NVC0(push, NVC0_3D(VTX_ATTR_DEFINE), 5);
186 dst = &push->cur[1];
187 if (desc->channel[0].pure_integer) {
188 if (desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
189 mode = VTX_ATTR(a, 4, SINT, 32);
190 desc->unpack_rgba_sint(dst, 0, src, 0, 1, 1);
191 } else {
192 mode = VTX_ATTR(a, 4, UINT, 32);
193 desc->unpack_rgba_uint(dst, 0, src, 0, 1, 1);
194 }
195 } else {
196 mode = VTX_ATTR(a, 4, FLOAT, 32);
197 desc->unpack_rgba_float(dst, 0, src, 0, 1, 1);
198 }
199 push->cur[0] = mode;
200 push->cur += 5;
201 }
202
203 static inline void
204 nvc0_user_vbuf_range(struct nvc0_context *nvc0, int vbi,
205 uint32_t *base, uint32_t *size)
206 {
207 if (unlikely(nvc0->vertex->instance_bufs & (1 << vbi))) {
208 const uint32_t div = nvc0->vertex->min_instance_div[vbi];
209 *base = nvc0->instance_off * nvc0->vtxbuf[vbi].stride;
210 *size = (nvc0->instance_max / div) * nvc0->vtxbuf[vbi].stride +
211 nvc0->vertex->vb_access_size[vbi];
212 } else {
213 /* NOTE: if there are user buffers, we *must* have index bounds */
214 assert(nvc0->vb_elt_limit != ~0);
215 *base = nvc0->vb_elt_first * nvc0->vtxbuf[vbi].stride;
216 *size = nvc0->vb_elt_limit * nvc0->vtxbuf[vbi].stride +
217 nvc0->vertex->vb_access_size[vbi];
218 }
219 }
220
221 static inline void
222 nvc0_release_user_vbufs(struct nvc0_context *nvc0)
223 {
224 if (nvc0->vbo_user) {
225 nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX_TMP);
226 nouveau_scratch_done(&nvc0->base);
227 }
228 }
229
230 static void
231 nvc0_update_user_vbufs(struct nvc0_context *nvc0)
232 {
233 uint64_t address[PIPE_MAX_ATTRIBS];
234 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
235 int i;
236 uint32_t written = 0;
237
238 PUSH_SPACE(push, nvc0->vertex->num_elements * 8);
239 for (i = 0; i < nvc0->vertex->num_elements; ++i) {
240 struct pipe_vertex_element *ve = &nvc0->vertex->element[i].pipe;
241 const unsigned b = ve->vertex_buffer_index;
242 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[b];
243 uint32_t base, size;
244
245 if (!(nvc0->vbo_user & (1 << b)))
246 continue;
247 if (nvc0->constant_vbos & (1 << b)) {
248 nvc0_set_constant_vertex_attrib(nvc0, i);
249 continue;
250 }
251 nvc0_user_vbuf_range(nvc0, b, &base, &size);
252
253 if (!(written & (1 << b))) {
254 struct nouveau_bo *bo;
255 const uint32_t bo_flags = NOUVEAU_BO_RD | NOUVEAU_BO_GART;
256 written |= 1 << b;
257 address[b] = nouveau_scratch_data(&nvc0->base, vb->user_buffer,
258 base, size, &bo);
259 if (bo)
260 BCTX_REFN_bo(nvc0->bufctx_3d, VTX_TMP, bo_flags, bo);
261
262 NOUVEAU_DRV_STAT(&nvc0->screen->base, user_buffer_upload_bytes, size);
263 }
264
265 BEGIN_1IC0(push, NVC0_3D(MACRO_VERTEX_ARRAY_SELECT), 5);
266 PUSH_DATA (push, i);
267 PUSH_DATAh(push, address[b] + base + size - 1);
268 PUSH_DATA (push, address[b] + base + size - 1);
269 PUSH_DATAh(push, address[b] + ve->src_offset);
270 PUSH_DATA (push, address[b] + ve->src_offset);
271 }
272 nvc0->base.vbo_dirty = true;
273 }
274
275 static void
276 nvc0_update_user_vbufs_shared(struct nvc0_context *nvc0)
277 {
278 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
279 uint32_t mask = nvc0->vbo_user & ~nvc0->constant_vbos;
280
281 PUSH_SPACE(push, nvc0->num_vtxbufs * 8);
282 while (mask) {
283 struct nouveau_bo *bo;
284 const uint32_t bo_flags = NOUVEAU_BO_RD | NOUVEAU_BO_GART;
285 uint64_t address;
286 uint32_t base, size;
287 const int b = ffs(mask) - 1;
288 mask &= ~(1 << b);
289
290 nvc0_user_vbuf_range(nvc0, b, &base, &size);
291
292 address = nouveau_scratch_data(&nvc0->base, nvc0->vtxbuf[b].user_buffer,
293 base, size, &bo);
294 if (bo)
295 BCTX_REFN_bo(nvc0->bufctx_3d, VTX_TMP, bo_flags, bo);
296
297 BEGIN_1IC0(push, NVC0_3D(MACRO_VERTEX_ARRAY_SELECT), 5);
298 PUSH_DATA (push, b);
299 PUSH_DATAh(push, address + base + size - 1);
300 PUSH_DATA (push, address + base + size - 1);
301 PUSH_DATAh(push, address);
302 PUSH_DATA (push, address);
303
304 NOUVEAU_DRV_STAT(&nvc0->screen->base, user_buffer_upload_bytes, size);
305 }
306
307 mask = nvc0->state.constant_elts;
308 while (mask) {
309 int i = ffs(mask) - 1;
310 mask &= ~(1 << i);
311 nvc0_set_constant_vertex_attrib(nvc0, i);
312 }
313 }
314
315 static void
316 nvc0_validate_vertex_buffers(struct nvc0_context *nvc0)
317 {
318 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
319 const struct nvc0_vertex_stateobj *vertex = nvc0->vertex;
320 uint32_t refd = 0;
321 unsigned i;
322
323 PUSH_SPACE(push, vertex->num_elements * 8);
324 for (i = 0; i < vertex->num_elements; ++i) {
325 const struct nvc0_vertex_element *ve;
326 const struct pipe_vertex_buffer *vb;
327 struct nv04_resource *res;
328 unsigned b;
329 unsigned limit, offset;
330
331 if (nvc0->state.constant_elts & (1 << i))
332 continue;
333 ve = &vertex->element[i];
334 b = ve->pipe.vertex_buffer_index;
335 vb = &nvc0->vtxbuf[b];
336
337 if (nvc0->vbo_user & (1 << b)) {
338 if (!(nvc0->constant_vbos & (1 << b))) {
339 if (ve->pipe.instance_divisor) {
340 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_DIVISOR(i)), 1);
341 PUSH_DATA (push, ve->pipe.instance_divisor);
342 }
343 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 1);
344 PUSH_DATA (push, (1 << 12) | vb->stride);
345 }
346 /* address/value set in nvc0_update_user_vbufs */
347 continue;
348 }
349 res = nv04_resource(vb->buffer);
350 offset = ve->pipe.src_offset + vb->buffer_offset;
351 limit = vb->buffer->width0 - 1;
352
353 if (unlikely(ve->pipe.instance_divisor)) {
354 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 4);
355 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | vb->stride);
356 PUSH_DATAh(push, res->address + offset);
357 PUSH_DATA (push, res->address + offset);
358 PUSH_DATA (push, ve->pipe.instance_divisor);
359 } else {
360 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 3);
361 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | vb->stride);
362 PUSH_DATAh(push, res->address + offset);
363 PUSH_DATA (push, res->address + offset);
364 }
365 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(i)), 2);
366 PUSH_DATAh(push, res->address + limit);
367 PUSH_DATA (push, res->address + limit);
368
369 if (!(refd & (1 << b))) {
370 refd |= 1 << b;
371 BCTX_REFN(nvc0->bufctx_3d, VTX, res, RD);
372 }
373 }
374 if (nvc0->vbo_user)
375 nvc0_update_user_vbufs(nvc0);
376 }
377
378 static void
379 nvc0_validate_vertex_buffers_shared(struct nvc0_context *nvc0)
380 {
381 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
382 unsigned b;
383 const uint32_t mask = nvc0->vbo_user;
384
385 PUSH_SPACE(push, nvc0->num_vtxbufs * 8);
386 for (b = 0; b < nvc0->num_vtxbufs; ++b) {
387 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[b];
388 struct nv04_resource *buf;
389 uint32_t offset, limit;
390
391 if (mask & (1 << b)) {
392 if (!(nvc0->constant_vbos & (1 << b))) {
393 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(b)), 1);
394 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | vb->stride);
395 }
396 /* address/value set in nvc0_update_user_vbufs_shared */
397 continue;
398 } else if (!vb->buffer) {
399 /* there can be holes in the vertex buffer lists */
400 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(b)), 0);
401 continue;
402 }
403 buf = nv04_resource(vb->buffer);
404 offset = vb->buffer_offset;
405 limit = buf->base.width0 - 1;
406
407 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(b)), 3);
408 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | vb->stride);
409 PUSH_DATAh(push, buf->address + offset);
410 PUSH_DATA (push, buf->address + offset);
411 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(b)), 2);
412 PUSH_DATAh(push, buf->address + limit);
413 PUSH_DATA (push, buf->address + limit);
414
415 BCTX_REFN(nvc0->bufctx_3d, VTX, buf, RD);
416 }
417 /* If there are more elements than buffers, we might not have unset
418 * fetching on the later elements.
419 */
420 PUSH_SPACE(push, nvc0->vertex->num_elements - b);
421 for (; b < nvc0->vertex->num_elements; ++b)
422 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(b)), 0);
423
424 if (nvc0->vbo_user)
425 nvc0_update_user_vbufs_shared(nvc0);
426 }
427
428 void
429 nvc0_vertex_arrays_validate(struct nvc0_context *nvc0)
430 {
431 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
432 struct nvc0_vertex_stateobj *vertex = nvc0->vertex;
433 struct nvc0_vertex_element *ve;
434 uint32_t const_vbos;
435 unsigned i;
436 uint8_t vbo_mode;
437 bool update_vertex;
438
439 nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX);
440
441 assert(vertex);
442 if (unlikely(vertex->need_conversion) ||
443 unlikely(nvc0->vertprog->vp.edgeflag < PIPE_MAX_ATTRIBS)) {
444 vbo_mode = 3;
445 } else {
446 vbo_mode = (nvc0->vbo_user && nvc0->vbo_push_hint) ? 1 : 0;
447 }
448 const_vbos = vbo_mode ? 0 : nvc0->constant_vbos;
449
450 update_vertex = (nvc0->dirty & NVC0_NEW_VERTEX) ||
451 (const_vbos != nvc0->state.constant_vbos) ||
452 (vbo_mode != nvc0->state.vbo_mode);
453
454 if (update_vertex) {
455 const unsigned n = MAX2(vertex->num_elements, nvc0->state.num_vtxelts);
456
457 nvc0->state.constant_vbos = const_vbos;
458 nvc0->state.constant_elts = 0;
459 nvc0->state.num_vtxelts = vertex->num_elements;
460 nvc0->state.vbo_mode = vbo_mode;
461
462 if (unlikely(vbo_mode)) {
463 if (unlikely(nvc0->state.instance_elts & 3)) {
464 /* translate mode uses only 2 vertex buffers */
465 nvc0->state.instance_elts &= ~3;
466 PUSH_SPACE(push, 3);
467 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_PER_INSTANCE(0)), 2);
468 PUSH_DATA (push, 0);
469 PUSH_DATA (push, 0);
470 }
471
472 PUSH_SPACE(push, n * 2 + 4);
473
474 BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(0)), n);
475 for (i = 0; i < vertex->num_elements; ++i)
476 PUSH_DATA(push, vertex->element[i].state_alt);
477 for (; i < n; ++i)
478 PUSH_DATA(push, NVC0_3D_VERTEX_ATTRIB_INACTIVE);
479
480 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(0)), 1);
481 PUSH_DATA (push, (1 << 12) | vertex->size);
482 for (i = 1; i < n; ++i)
483 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 0);
484 } else {
485 uint32_t *restrict data;
486
487 if (unlikely(vertex->instance_elts != nvc0->state.instance_elts)) {
488 nvc0->state.instance_elts = vertex->instance_elts;
489 assert(n); /* if (n == 0), both masks should be 0 */
490 PUSH_SPACE(push, 3);
491 BEGIN_NVC0(push, NVC0_3D(MACRO_VERTEX_ARRAY_PER_INSTANCE), 2);
492 PUSH_DATA (push, n);
493 PUSH_DATA (push, vertex->instance_elts);
494 }
495
496 PUSH_SPACE(push, n * 2 + 1);
497 BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(0)), n);
498 data = push->cur;
499 push->cur += n;
500 for (i = 0; i < vertex->num_elements; ++i) {
501 ve = &vertex->element[i];
502 data[i] = ve->state;
503 if (unlikely(const_vbos & (1 << ve->pipe.vertex_buffer_index))) {
504 nvc0->state.constant_elts |= 1 << i;
505 data[i] |= NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST;
506 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 0);
507 }
508 }
509 for (; i < n; ++i) {
510 data[i] = NVC0_3D_VERTEX_ATTRIB_INACTIVE;
511 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 0);
512 }
513 }
514 }
515 if (nvc0->state.vbo_mode) /* using translate, don't set up arrays here */
516 return;
517
518 if (vertex->shared_slots)
519 nvc0_validate_vertex_buffers_shared(nvc0);
520 else
521 nvc0_validate_vertex_buffers(nvc0);
522 }
523
524 void
525 nvc0_idxbuf_validate(struct nvc0_context *nvc0)
526 {
527 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
528 struct nv04_resource *buf = nv04_resource(nvc0->idxbuf.buffer);
529
530 assert(buf);
531 assert(nouveau_resource_mapped_by_gpu(&buf->base));
532
533 PUSH_SPACE(push, 6);
534 BEGIN_NVC0(push, NVC0_3D(INDEX_ARRAY_START_HIGH), 5);
535 PUSH_DATAh(push, buf->address + nvc0->idxbuf.offset);
536 PUSH_DATA (push, buf->address + nvc0->idxbuf.offset);
537 PUSH_DATAh(push, buf->address + buf->base.width0 - 1);
538 PUSH_DATA (push, buf->address + buf->base.width0 - 1);
539 PUSH_DATA (push, nvc0->idxbuf.index_size >> 1);
540
541 BCTX_REFN(nvc0->bufctx_3d, IDX, buf, RD);
542 }
543
544 #define NVC0_PRIM_GL_CASE(n) \
545 case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
546
547 static inline unsigned
548 nvc0_prim_gl(unsigned prim)
549 {
550 switch (prim) {
551 NVC0_PRIM_GL_CASE(POINTS);
552 NVC0_PRIM_GL_CASE(LINES);
553 NVC0_PRIM_GL_CASE(LINE_LOOP);
554 NVC0_PRIM_GL_CASE(LINE_STRIP);
555 NVC0_PRIM_GL_CASE(TRIANGLES);
556 NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
557 NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
558 NVC0_PRIM_GL_CASE(QUADS);
559 NVC0_PRIM_GL_CASE(QUAD_STRIP);
560 NVC0_PRIM_GL_CASE(POLYGON);
561 NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
562 NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
563 NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
564 NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
565 NVC0_PRIM_GL_CASE(PATCHES);
566 default:
567 return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
568 }
569 }
570
571 static void
572 nvc0_draw_vbo_kick_notify(struct nouveau_pushbuf *push)
573 {
574 struct nvc0_screen *screen = push->user_priv;
575
576 nouveau_fence_update(&screen->base, true);
577
578 NOUVEAU_DRV_STAT(&screen->base, pushbuf_count, 1);
579 }
580
581 static void
582 nvc0_draw_arrays(struct nvc0_context *nvc0,
583 unsigned mode, unsigned start, unsigned count,
584 unsigned instance_count)
585 {
586 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
587 unsigned prim;
588
589 if (nvc0->state.index_bias) {
590 /* index_bias is implied 0 if !info->indexed (really ?) */
591 /* TODO: can we deactivate it for the VERTEX_BUFFER_FIRST command ? */
592 PUSH_SPACE(push, 2);
593 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_BASE), 0);
594 IMMED_NVC0(push, NVC0_3D(VERTEX_ID_BASE), 0);
595 nvc0->state.index_bias = 0;
596 }
597
598 prim = nvc0_prim_gl(mode);
599
600 while (instance_count--) {
601 PUSH_SPACE(push, 6);
602 BEGIN_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), 1);
603 PUSH_DATA (push, prim);
604 BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
605 PUSH_DATA (push, start);
606 PUSH_DATA (push, count);
607 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
608
609 prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
610 }
611 NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_array, 1);
612 }
613
614 static void
615 nvc0_draw_elements_inline_u08(struct nouveau_pushbuf *push, const uint8_t *map,
616 unsigned start, unsigned count)
617 {
618 map += start;
619
620 if (count & 3) {
621 unsigned i;
622 PUSH_SPACE(push, 4);
623 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U32), count & 3);
624 for (i = 0; i < (count & 3); ++i)
625 PUSH_DATA(push, *map++);
626 count &= ~3;
627 }
628 while (count) {
629 unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 4) / 4;
630
631 PUSH_SPACE(push, nr + 1);
632 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U8), nr);
633 for (i = 0; i < nr; ++i) {
634 PUSH_DATA(push,
635 (map[3] << 24) | (map[2] << 16) | (map[1] << 8) | map[0]);
636 map += 4;
637 }
638 count -= nr * 4;
639 }
640 }
641
642 static void
643 nvc0_draw_elements_inline_u16(struct nouveau_pushbuf *push, const uint16_t *map,
644 unsigned start, unsigned count)
645 {
646 map += start;
647
648 if (count & 1) {
649 count &= ~1;
650 PUSH_SPACE(push, 2);
651 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
652 PUSH_DATA (push, *map++);
653 }
654 while (count) {
655 unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2;
656
657 PUSH_SPACE(push, nr + 1);
658 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U16), nr);
659 for (i = 0; i < nr; ++i) {
660 PUSH_DATA(push, (map[1] << 16) | map[0]);
661 map += 2;
662 }
663 count -= nr * 2;
664 }
665 }
666
667 static void
668 nvc0_draw_elements_inline_u32(struct nouveau_pushbuf *push, const uint32_t *map,
669 unsigned start, unsigned count)
670 {
671 map += start;
672
673 while (count) {
674 const unsigned nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN);
675
676 PUSH_SPACE(push, nr + 1);
677 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U32), nr);
678 PUSH_DATAp(push, map, nr);
679
680 map += nr;
681 count -= nr;
682 }
683 }
684
685 static void
686 nvc0_draw_elements_inline_u32_short(struct nouveau_pushbuf *push,
687 const uint32_t *map,
688 unsigned start, unsigned count)
689 {
690 map += start;
691
692 if (count & 1) {
693 count--;
694 PUSH_SPACE(push, 2);
695 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
696 PUSH_DATA (push, *map++);
697 }
698 while (count) {
699 unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2;
700
701 PUSH_SPACE(push, nr + 1);
702 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U16), nr);
703 for (i = 0; i < nr; ++i) {
704 PUSH_DATA(push, (map[1] << 16) | map[0]);
705 map += 2;
706 }
707 count -= nr * 2;
708 }
709 }
710
711 static void
712 nvc0_draw_elements(struct nvc0_context *nvc0, bool shorten,
713 unsigned mode, unsigned start, unsigned count,
714 unsigned instance_count, int32_t index_bias)
715 {
716 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
717 unsigned prim;
718 const unsigned index_size = nvc0->idxbuf.index_size;
719
720 prim = nvc0_prim_gl(mode);
721
722 if (index_bias != nvc0->state.index_bias) {
723 PUSH_SPACE(push, 4);
724 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_BASE), 1);
725 PUSH_DATA (push, index_bias);
726 BEGIN_NVC0(push, NVC0_3D(VERTEX_ID_BASE), 1);
727 PUSH_DATA (push, index_bias);
728 nvc0->state.index_bias = index_bias;
729 }
730
731 if (nvc0->idxbuf.buffer) {
732 PUSH_SPACE(push, 1);
733 IMMED_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), prim);
734 do {
735 PUSH_SPACE(push, 7);
736 BEGIN_NVC0(push, NVC0_3D(INDEX_BATCH_FIRST), 2);
737 PUSH_DATA (push, start);
738 PUSH_DATA (push, count);
739 if (--instance_count) {
740 BEGIN_NVC0(push, NVC0_3D(VERTEX_END_GL), 2);
741 PUSH_DATA (push, 0);
742 PUSH_DATA (push, prim | NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT);
743 }
744 } while (instance_count);
745 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
746 } else {
747 const void *data = nvc0->idxbuf.user_buffer;
748
749 while (instance_count--) {
750 PUSH_SPACE(push, 2);
751 BEGIN_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), 1);
752 PUSH_DATA (push, prim);
753 switch (index_size) {
754 case 1:
755 nvc0_draw_elements_inline_u08(push, data, start, count);
756 break;
757 case 2:
758 nvc0_draw_elements_inline_u16(push, data, start, count);
759 break;
760 case 4:
761 if (shorten)
762 nvc0_draw_elements_inline_u32_short(push, data, start, count);
763 else
764 nvc0_draw_elements_inline_u32(push, data, start, count);
765 break;
766 default:
767 assert(0);
768 return;
769 }
770 PUSH_SPACE(push, 1);
771 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
772
773 prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
774 }
775 }
776 NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_indexed, 1);
777 }
778
779 static void
780 nvc0_draw_stream_output(struct nvc0_context *nvc0,
781 const struct pipe_draw_info *info)
782 {
783 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
784 struct nvc0_so_target *so = nvc0_so_target(info->count_from_stream_output);
785 struct nv04_resource *res = nv04_resource(so->pipe.buffer);
786 unsigned mode = nvc0_prim_gl(info->mode);
787 unsigned num_instances = info->instance_count;
788
789 if (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
790 res->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
791 PUSH_SPACE(push, 2);
792 IMMED_NVC0(push, NVC0_3D(SERIALIZE), 0);
793 nvc0_hw_query_fifo_wait(nvc0, nvc0_query(so->pq));
794 if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
795 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
796
797 NOUVEAU_DRV_STAT(&nvc0->screen->base, gpu_serialize_count, 1);
798 }
799
800 while (num_instances--) {
801 nouveau_pushbuf_space(push, 9, 0, 1);
802 BEGIN_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), 1);
803 PUSH_DATA (push, mode);
804 BEGIN_NVC0(push, NVC0_3D(DRAW_TFB_BASE), 1);
805 PUSH_DATA (push, 0);
806 BEGIN_NVC0(push, NVC0_3D(DRAW_TFB_STRIDE), 1);
807 PUSH_DATA (push, so->stride);
808 BEGIN_NVC0(push, NVC0_3D(DRAW_TFB_BYTES), 1);
809 nvc0_hw_query_pushbuf_submit(push, nvc0_query(so->pq), 0x4);
810 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
811
812 mode |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
813 }
814 }
815
816 static void
817 nvc0_draw_indirect(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
818 {
819 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
820 struct nv04_resource *buf = nv04_resource(info->indirect);
821 struct nv04_resource *buf_count = nv04_resource(info->indirect_params);
822 unsigned size, macro, count = info->indirect_count, drawid = info->drawid;
823 uint32_t offset = buf->offset + info->indirect_offset;
824
825 PUSH_SPACE(push, 7);
826
827 /* must make FIFO wait for engines idle before continuing to process */
828 if ((buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr)) ||
829 (buf_count && buf_count->fence_wr &&
830 !nouveau_fence_signalled(buf_count->fence_wr))) {
831 IMMED_NVC0(push, SUBC_3D(NV10_SUBCHAN_REF_CNT), 0);
832 }
833
834 /* Queue things up to let the macros write params to the driver constbuf */
835 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
836 PUSH_DATA (push, 512);
837 PUSH_DATAh(push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
838 PUSH_DATA (push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
839
840 if (info->indexed) {
841 assert(nvc0->idxbuf.buffer);
842 assert(nouveau_resource_mapped_by_gpu(nvc0->idxbuf.buffer));
843 size = 5;
844 if (buf_count)
845 macro = NVC0_3D_MACRO_DRAW_ELEMENTS_INDIRECT_COUNT;
846 else
847 macro = NVC0_3D_MACRO_DRAW_ELEMENTS_INDIRECT;
848 } else {
849 if (nvc0->state.index_bias) {
850 /* index_bias is implied 0 if !info->indexed (really ?) */
851 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_BASE), 0);
852 IMMED_NVC0(push, NVC0_3D(VERTEX_ID_BASE), 0);
853 nvc0->state.index_bias = 0;
854 }
855 size = 4;
856 if (buf_count)
857 macro = NVC0_3D_MACRO_DRAW_ARRAYS_INDIRECT_COUNT;
858 else
859 macro = NVC0_3D_MACRO_DRAW_ARRAYS_INDIRECT;
860 }
861
862 /* If the stride is not the natural stride, we have to stick a separate
863 * push data reference for each draw. Otherwise it can all go in as one.
864 * Of course there is a maximum packet size, so we have to break things up
865 * along those borders as well.
866 */
867 while (count) {
868 unsigned draws = count, pushes, i;
869 if (info->indirect_stride == size * 4) {
870 draws = MIN2(draws, (NV04_PFIFO_MAX_PACKET_LEN - 4) / size);
871 pushes = 1;
872 } else {
873 draws = MIN2(draws, 32);
874 pushes = draws;
875 }
876
877 nouveau_pushbuf_space(push, 16, 0, pushes + !!buf_count);
878 PUSH_REFN(push, buf->bo, NOUVEAU_BO_RD | buf->domain);
879 if (buf_count)
880 PUSH_REFN(push, buf_count->bo, NOUVEAU_BO_RD | buf_count->domain);
881 PUSH_DATA(push,
882 NVC0_FIFO_PKHDR_1I(0, macro, 3 + !!buf_count + draws * size));
883 PUSH_DATA(push, nvc0_prim_gl(info->mode));
884 PUSH_DATA(push, drawid);
885 PUSH_DATA(push, draws);
886 if (buf_count) {
887 nouveau_pushbuf_data(push,
888 buf_count->bo,
889 buf_count->offset + info->indirect_params_offset,
890 NVC0_IB_ENTRY_1_NO_PREFETCH | 4);
891 }
892 if (pushes == 1) {
893 nouveau_pushbuf_data(push,
894 buf->bo, offset,
895 NVC0_IB_ENTRY_1_NO_PREFETCH | (size * 4 * draws));
896 offset += draws * info->indirect_stride;
897 } else {
898 for (i = 0; i < pushes; i++) {
899 nouveau_pushbuf_data(push,
900 buf->bo, offset,
901 NVC0_IB_ENTRY_1_NO_PREFETCH | (size * 4));
902 offset += info->indirect_stride;
903 }
904 }
905 count -= draws;
906 drawid += draws;
907 }
908 }
909
910 static inline void
911 nvc0_update_prim_restart(struct nvc0_context *nvc0, bool en, uint32_t index)
912 {
913 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
914
915 if (en != nvc0->state.prim_restart) {
916 if (en) {
917 BEGIN_NVC0(push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
918 PUSH_DATA (push, 1);
919 PUSH_DATA (push, index);
920 } else {
921 IMMED_NVC0(push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
922 }
923 nvc0->state.prim_restart = en;
924 } else
925 if (en) {
926 BEGIN_NVC0(push, NVC0_3D(PRIM_RESTART_INDEX), 1);
927 PUSH_DATA (push, index);
928 }
929 }
930
931 void
932 nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
933 {
934 struct nvc0_context *nvc0 = nvc0_context(pipe);
935 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
936 int s;
937
938 /* NOTE: caller must ensure that (min_index + index_bias) is >= 0 */
939 nvc0->vb_elt_first = info->min_index + info->index_bias;
940 nvc0->vb_elt_limit = info->max_index - info->min_index;
941 nvc0->instance_off = info->start_instance;
942 nvc0->instance_max = info->instance_count - 1;
943
944 /* For picking only a few vertices from a large user buffer, push is better,
945 * if index count is larger and we expect repeated vertices, suggest upload.
946 */
947 nvc0->vbo_push_hint =
948 info->indexed && (nvc0->vb_elt_limit >= (info->count * 2));
949
950 /* Check whether we want to switch vertex-submission mode. */
951 if (nvc0->vbo_user && !(nvc0->dirty & (NVC0_NEW_ARRAYS | NVC0_NEW_VERTEX))) {
952 if (nvc0->vbo_push_hint != !!nvc0->state.vbo_mode)
953 if (nvc0->state.vbo_mode != 3)
954 nvc0->dirty |= NVC0_NEW_ARRAYS;
955
956 if (!(nvc0->dirty & NVC0_NEW_ARRAYS) && nvc0->state.vbo_mode == 0) {
957 if (nvc0->vertex->shared_slots)
958 nvc0_update_user_vbufs_shared(nvc0);
959 else
960 nvc0_update_user_vbufs(nvc0);
961 }
962 }
963
964 if (info->mode == PIPE_PRIM_PATCHES &&
965 nvc0->state.patch_vertices != info->vertices_per_patch) {
966 nvc0->state.patch_vertices = info->vertices_per_patch;
967 PUSH_SPACE(push, 1);
968 IMMED_NVC0(push, NVC0_3D(PATCH_VERTICES), nvc0->state.patch_vertices);
969 }
970
971 /* 8 as minimum to avoid immediate double validation of new buffers */
972 nvc0_state_validate(nvc0, ~0, 8);
973
974 if (nvc0->vertprog->vp.need_draw_parameters) {
975 PUSH_SPACE(push, 9);
976 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
977 PUSH_DATA (push, 512);
978 PUSH_DATAh(push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
979 PUSH_DATA (push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
980 if (!info->indirect) {
981 BEGIN_1IC0(push, NVC0_3D(CB_POS), 1 + 3);
982 PUSH_DATA (push, 256 + 128);
983 PUSH_DATA (push, info->index_bias);
984 PUSH_DATA (push, info->start_instance);
985 PUSH_DATA (push, info->drawid);
986 }
987 }
988
989 push->kick_notify = nvc0_draw_vbo_kick_notify;
990
991 for (s = 0; s < 5 && !nvc0->cb_dirty; ++s) {
992 if (nvc0->constbuf_coherent[s])
993 nvc0->cb_dirty = true;
994 }
995
996 if (nvc0->cb_dirty) {
997 PUSH_SPACE(push, 1);
998 IMMED_NVC0(push, NVC0_3D(MEM_BARRIER), 0x1011);
999 nvc0->cb_dirty = false;
1000 }
1001
1002 for (s = 0; s < 5; ++s) {
1003 if (!nvc0->textures_coherent[s])
1004 continue;
1005
1006 PUSH_SPACE(push, nvc0->num_textures[s] * 2);
1007
1008 for (int i = 0; i < nvc0->num_textures[s]; ++i) {
1009 struct nv50_tic_entry *tic = nv50_tic_entry(nvc0->textures[s][i]);
1010 if (!(nvc0->textures_coherent[s] & (1 << i)))
1011 continue;
1012
1013 BEGIN_NVC0(push, NVC0_3D(TEX_CACHE_CTL), 1);
1014 PUSH_DATA (push, (tic->id << 4) | 1);
1015 NOUVEAU_DRV_STAT(&nvc0->screen->base, tex_cache_flush_count, 1);
1016 }
1017 }
1018
1019 if (nvc0->state.vbo_mode) {
1020 nvc0_push_vbo(nvc0, info);
1021 push->kick_notify = nvc0_default_kick_notify;
1022 nouveau_pushbuf_bufctx(push, NULL);
1023 return;
1024 }
1025
1026 /* space for base instance, flush, and prim restart */
1027 PUSH_SPACE(push, 8);
1028
1029 if (nvc0->state.instance_base != info->start_instance) {
1030 nvc0->state.instance_base = info->start_instance;
1031 /* NOTE: this does not affect the shader input, should it ? */
1032 BEGIN_NVC0(push, NVC0_3D(VB_INSTANCE_BASE), 1);
1033 PUSH_DATA (push, info->start_instance);
1034 }
1035
1036 nvc0->base.vbo_dirty |= !!nvc0->vtxbufs_coherent;
1037
1038 if (!nvc0->base.vbo_dirty && nvc0->idxbuf.buffer &&
1039 nvc0->idxbuf.buffer->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT)
1040 nvc0->base.vbo_dirty = true;
1041
1042 nvc0_update_prim_restart(nvc0, info->primitive_restart, info->restart_index);
1043
1044 if (nvc0->base.vbo_dirty) {
1045 if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
1046 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
1047 nvc0->base.vbo_dirty = false;
1048 }
1049
1050 if (unlikely(info->indirect)) {
1051 nvc0_draw_indirect(nvc0, info);
1052 } else
1053 if (unlikely(info->count_from_stream_output)) {
1054 nvc0_draw_stream_output(nvc0, info);
1055 } else
1056 if (info->indexed) {
1057 bool shorten = info->max_index <= 65535;
1058
1059 if (info->primitive_restart && info->restart_index > 65535)
1060 shorten = false;
1061
1062 nvc0_draw_elements(nvc0, shorten,
1063 info->mode, info->start, info->count,
1064 info->instance_count, info->index_bias);
1065 } else {
1066 nvc0_draw_arrays(nvc0,
1067 info->mode, info->start, info->count,
1068 info->instance_count);
1069 }
1070 push->kick_notify = nvc0_default_kick_notify;
1071
1072 nvc0_release_user_vbufs(nvc0);
1073
1074 nouveau_pushbuf_bufctx(push, NULL);
1075 }