projects / mesa.git / blob
? search:


235b1afc24b8ff3418a21f3fef54fd967d58335e
[mesa.git] / src / gallium / drivers / nouveau / nvc0 / nvc0_vbo.c
1 /*
2 * Copyright 2010 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
24
25 #include "pipe/p_context.h"
26 #include "pipe/p_state.h"
27 #include "util/u_inlines.h"
28 #include "util/u_format.h"
29 #include "translate/translate.h"
30
31 #include "nvc0/nvc0_context.h"
32 #include "nvc0/nvc0_query_hw.h"
33 #include "nvc0/nvc0_resource.h"
34
35 #include "nvc0/nvc0_3d.xml.h"
36
37 void
38 nvc0_vertex_state_delete(struct pipe_context *pipe,
39 void *hwcso)
40 {
41 struct nvc0_vertex_stateobj *so = hwcso;
42
43 if (so->translate)
44 so->translate->release(so->translate);
45 FREE(hwcso);
46 }
47
48 void *
49 nvc0_vertex_state_create(struct pipe_context *pipe,
50 unsigned num_elements,
51 const struct pipe_vertex_element *elements)
52 {
53 struct nvc0_vertex_stateobj *so;
54 struct translate_key transkey;
55 unsigned i;
56 unsigned src_offset_max = 0;
57
58 so = MALLOC(sizeof(*so) +
59 num_elements * sizeof(struct nvc0_vertex_element));
60 if (!so)
61 return NULL;
62 so->num_elements = num_elements;
63 so->instance_elts = 0;
64 so->instance_bufs = 0;
65 so->shared_slots = false;
66 so->need_conversion = false;
67
68 memset(so->vb_access_size, 0, sizeof(so->vb_access_size));
69
70 for (i = 0; i < PIPE_MAX_ATTRIBS; ++i)
71 so->min_instance_div[i] = 0xffffffff;
72
73 transkey.nr_elements = 0;
74 transkey.output_stride = 0;
75
76 for (i = 0; i < num_elements; ++i) {
77 const struct pipe_vertex_element *ve = &elements[i];
78 const unsigned vbi = ve->vertex_buffer_index;
79 unsigned size;
80 enum pipe_format fmt = ve->src_format;
81
82 so->element[i].pipe = elements[i];
83 so->element[i].state = nvc0_format_table[fmt].vtx;
84
85 if (!so->element[i].state) {
86 switch (util_format_get_nr_components(fmt)) {
87 case 1: fmt = PIPE_FORMAT_R32_FLOAT; break;
88 case 2: fmt = PIPE_FORMAT_R32G32_FLOAT; break;
89 case 3: fmt = PIPE_FORMAT_R32G32B32_FLOAT; break;
90 case 4: fmt = PIPE_FORMAT_R32G32B32A32_FLOAT; break;
91 default:
92 assert(0);
93 FREE(so);
94 return NULL;
95 }
96 so->element[i].state = nvc0_format_table[fmt].vtx;
97 so->need_conversion = true;
98 pipe_debug_message(&nouveau_context(pipe)->debug, FALLBACK,
99 "Converting vertex element %d, no hw format %s",
100 i, util_format_name(ve->src_format));
101 }
102 size = util_format_get_blocksize(fmt);
103
104 src_offset_max = MAX2(src_offset_max, ve->src_offset);
105
106 if (so->vb_access_size[vbi] < (ve->src_offset + size))
107 so->vb_access_size[vbi] = ve->src_offset + size;
108
109 if (unlikely(ve->instance_divisor)) {
110 so->instance_elts |= 1 << i;
111 so->instance_bufs |= 1 << vbi;
112 if (ve->instance_divisor < so->min_instance_div[vbi])
113 so->min_instance_div[vbi] = ve->instance_divisor;
114 }
115
116 if (1) {
117 unsigned ca;
118 unsigned j = transkey.nr_elements++;
119
120 ca = util_format_description(fmt)->channel[0].size / 8;
121 if (ca != 1 && ca != 2)
122 ca = 4;
123
124 transkey.element[j].type = TRANSLATE_ELEMENT_NORMAL;
125 transkey.element[j].input_format = ve->src_format;
126 transkey.element[j].input_buffer = vbi;
127 transkey.element[j].input_offset = ve->src_offset;
128 transkey.element[j].instance_divisor = ve->instance_divisor;
129
130 transkey.output_stride = align(transkey.output_stride, ca);
131 transkey.element[j].output_format = fmt;
132 transkey.element[j].output_offset = transkey.output_stride;
133 transkey.output_stride += size;
134
135 so->element[i].state_alt = so->element[i].state;
136 so->element[i].state_alt |= transkey.element[j].output_offset << 7;
137 }
138
139 so->element[i].state |= i << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT;
140 }
141 transkey.output_stride = align(transkey.output_stride, 4);
142
143 so->size = transkey.output_stride;
144 so->translate = translate_create(&transkey);
145
146 if (so->instance_elts || src_offset_max >= (1 << 14))
147 return so;
148 so->shared_slots = true;
149
150 for (i = 0; i < num_elements; ++i) {
151 const unsigned b = elements[i].vertex_buffer_index;
152 const unsigned s = elements[i].src_offset;
153 so->element[i].state &= ~NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__MASK;
154 so->element[i].state |= b << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT;
155 so->element[i].state |= s << NVC0_3D_VERTEX_ATTRIB_FORMAT_OFFSET__SHIFT;
156 }
157 return so;
158 }
159
160 #define NVC0_3D_VERTEX_ATTRIB_INACTIVE \
161 NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT | \
162 NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32 | NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST
163
164 #define VTX_ATTR(a, c, t, s) \
165 ((NVC0_3D_VTX_ATTR_DEFINE_TYPE_##t) | \
166 (NVC0_3D_VTX_ATTR_DEFINE_SIZE_##s) | \
167 ((a) << NVC0_3D_VTX_ATTR_DEFINE_ATTR__SHIFT) | \
168 ((c) << NVC0_3D_VTX_ATTR_DEFINE_COMP__SHIFT))
169
170 static void
171 nvc0_set_constant_vertex_attrib(struct nvc0_context *nvc0, const unsigned a)
172 {
173 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
174 struct pipe_vertex_element *ve = &nvc0->vertex->element[a].pipe;
175 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[ve->vertex_buffer_index];
176 uint32_t mode;
177 const struct util_format_description *desc;
178 void *dst;
179 const void *src = (const uint8_t *)vb->user_buffer + ve->src_offset;
180 assert(!vb->buffer);
181
182 desc = util_format_description(ve->src_format);
183
184 PUSH_SPACE(push, 6);
185 BEGIN_NVC0(push, NVC0_3D(VTX_ATTR_DEFINE), 5);
186 dst = &push->cur[1];
187 if (desc->channel[0].pure_integer) {
188 if (desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
189 mode = VTX_ATTR(a, 4, SINT, 32);
190 desc->unpack_rgba_sint(dst, 0, src, 0, 1, 1);
191 } else {
192 mode = VTX_ATTR(a, 4, UINT, 32);
193 desc->unpack_rgba_uint(dst, 0, src, 0, 1, 1);
194 }
195 } else {
196 mode = VTX_ATTR(a, 4, FLOAT, 32);
197 desc->unpack_rgba_float(dst, 0, src, 0, 1, 1);
198 }
199 push->cur[0] = mode;
200 push->cur += 5;
201 }
202
203 static inline void
204 nvc0_user_vbuf_range(struct nvc0_context *nvc0, int vbi,
205 uint32_t *base, uint32_t *size)
206 {
207 if (unlikely(nvc0->vertex->instance_bufs & (1 << vbi))) {
208 const uint32_t div = nvc0->vertex->min_instance_div[vbi];
209 *base = nvc0->instance_off * nvc0->vtxbuf[vbi].stride;
210 *size = (nvc0->instance_max / div) * nvc0->vtxbuf[vbi].stride +
211 nvc0->vertex->vb_access_size[vbi];
212 } else {
213 /* NOTE: if there are user buffers, we *must* have index bounds */
214 assert(nvc0->vb_elt_limit != ~0);
215 *base = nvc0->vb_elt_first * nvc0->vtxbuf[vbi].stride;
216 *size = nvc0->vb_elt_limit * nvc0->vtxbuf[vbi].stride +
217 nvc0->vertex->vb_access_size[vbi];
218 }
219 }
220
221 static inline void
222 nvc0_release_user_vbufs(struct nvc0_context *nvc0)
223 {
224 if (nvc0->vbo_user) {
225 nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX_TMP);
226 nouveau_scratch_done(&nvc0->base);
227 }
228 }
229
230 static void
231 nvc0_update_user_vbufs(struct nvc0_context *nvc0)
232 {
233 uint64_t address[PIPE_MAX_ATTRIBS];
234 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
235 int i;
236 uint32_t written = 0;
237
238 PUSH_SPACE(push, nvc0->vertex->num_elements * 8);
239 for (i = 0; i < nvc0->vertex->num_elements; ++i) {
240 struct pipe_vertex_element *ve = &nvc0->vertex->element[i].pipe;
241 const unsigned b = ve->vertex_buffer_index;
242 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[b];
243 uint32_t base, size;
244
245 if (!(nvc0->vbo_user & (1 << b)))
246 continue;
247 if (nvc0->constant_vbos & (1 << b)) {
248 nvc0_set_constant_vertex_attrib(nvc0, i);
249 continue;
250 }
251 nvc0_user_vbuf_range(nvc0, b, &base, &size);
252
253 if (!(written & (1 << b))) {
254 struct nouveau_bo *bo;
255 const uint32_t bo_flags = NOUVEAU_BO_RD | NOUVEAU_BO_GART;
256 written |= 1 << b;
257 address[b] = nouveau_scratch_data(&nvc0->base, vb->user_buffer,
258 base, size, &bo);
259 if (bo)
260 BCTX_REFN_bo(nvc0->bufctx_3d, VTX_TMP, bo_flags, bo);
261
262 NOUVEAU_DRV_STAT(&nvc0->screen->base, user_buffer_upload_bytes, size);
263 }
264
265 BEGIN_1IC0(push, NVC0_3D(MACRO_VERTEX_ARRAY_SELECT), 5);
266 PUSH_DATA (push, i);
267 PUSH_DATAh(push, address[b] + base + size - 1);
268 PUSH_DATA (push, address[b] + base + size - 1);
269 PUSH_DATAh(push, address[b] + ve->src_offset);
270 PUSH_DATA (push, address[b] + ve->src_offset);
271 }
272 nvc0->base.vbo_dirty = true;
273 }
274
275 static void
276 nvc0_update_user_vbufs_shared(struct nvc0_context *nvc0)
277 {
278 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
279 uint32_t mask = nvc0->vbo_user & ~nvc0->constant_vbos;
280
281 PUSH_SPACE(push, nvc0->num_vtxbufs * 8);
282 while (mask) {
283 struct nouveau_bo *bo;
284 const uint32_t bo_flags = NOUVEAU_BO_RD | NOUVEAU_BO_GART;
285 uint64_t address;
286 uint32_t base, size;
287 const int b = ffs(mask) - 1;
288 mask &= ~(1 << b);
289
290 nvc0_user_vbuf_range(nvc0, b, &base, &size);
291
292 address = nouveau_scratch_data(&nvc0->base, nvc0->vtxbuf[b].user_buffer,
293 base, size, &bo);
294 if (bo)
295 BCTX_REFN_bo(nvc0->bufctx_3d, VTX_TMP, bo_flags, bo);
296
297 BEGIN_1IC0(push, NVC0_3D(MACRO_VERTEX_ARRAY_SELECT), 5);
298 PUSH_DATA (push, b);
299 PUSH_DATAh(push, address + base + size - 1);
300 PUSH_DATA (push, address + base + size - 1);
301 PUSH_DATAh(push, address);
302 PUSH_DATA (push, address);
303
304 NOUVEAU_DRV_STAT(&nvc0->screen->base, user_buffer_upload_bytes, size);
305 }
306
307 mask = nvc0->state.constant_elts;
308 while (mask) {
309 int i = ffs(mask) - 1;
310 mask &= ~(1 << i);
311 nvc0_set_constant_vertex_attrib(nvc0, i);
312 }
313 }
314
315 static void
316 nvc0_validate_vertex_buffers(struct nvc0_context *nvc0)
317 {
318 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
319 const struct nvc0_vertex_stateobj *vertex = nvc0->vertex;
320 uint32_t refd = 0;
321 unsigned i;
322
323 PUSH_SPACE(push, vertex->num_elements * 8);
324 for (i = 0; i < vertex->num_elements; ++i) {
325 const struct nvc0_vertex_element *ve;
326 const struct pipe_vertex_buffer *vb;
327 struct nv04_resource *res;
328 unsigned b;
329 unsigned limit, offset;
330
331 if (nvc0->state.constant_elts & (1 << i))
332 continue;
333 ve = &vertex->element[i];
334 b = ve->pipe.vertex_buffer_index;
335 vb = &nvc0->vtxbuf[b];
336
337 if (!vb->buffer) {
338 if (!(nvc0->constant_vbos & (1 << b))) {
339 if (ve->pipe.instance_divisor) {
340 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_DIVISOR(i)), 1);
341 PUSH_DATA (push, ve->pipe.instance_divisor);
342 }
343 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 1);
344 PUSH_DATA (push, (1 << 12) | vb->stride);
345 }
346 /* address/value set in nvc0_update_user_vbufs */
347 continue;
348 }
349 res = nv04_resource(vb->buffer);
350 offset = ve->pipe.src_offset + vb->buffer_offset;
351 limit = vb->buffer->width0 - 1;
352
353 if (unlikely(ve->pipe.instance_divisor)) {
354 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 4);
355 PUSH_DATA (push, (1 << 12) | vb->stride);
356 PUSH_DATAh(push, res->address + offset);
357 PUSH_DATA (push, res->address + offset);
358 PUSH_DATA (push, ve->pipe.instance_divisor);
359 } else {
360 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 3);
361 PUSH_DATA (push, (1 << 12) | vb->stride);
362 PUSH_DATAh(push, res->address + offset);
363 PUSH_DATA (push, res->address + offset);
364 }
365 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(i)), 2);
366 PUSH_DATAh(push, res->address + limit);
367 PUSH_DATA (push, res->address + limit);
368
369 if (!(refd & (1 << b))) {
370 refd |= 1 << b;
371 BCTX_REFN(nvc0->bufctx_3d, VTX, res, RD);
372 }
373 }
374 if (nvc0->vbo_user)
375 nvc0_update_user_vbufs(nvc0);
376 }
377
378 static void
379 nvc0_validate_vertex_buffers_shared(struct nvc0_context *nvc0)
380 {
381 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
382 unsigned b;
383 const uint32_t mask = nvc0->vbo_user;
384
385 PUSH_SPACE(push, nvc0->num_vtxbufs * 8);
386 for (b = 0; b < nvc0->num_vtxbufs; ++b) {
387 struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[b];
388 struct nv04_resource *buf;
389 uint32_t offset, limit;
390
391 if (mask & (1 << b)) {
392 if (!(nvc0->constant_vbos & (1 << b))) {
393 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(b)), 1);
394 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | vb->stride);
395 }
396 /* address/value set in nvc0_update_user_vbufs_shared */
397 continue;
398 }
399 buf = nv04_resource(vb->buffer);
400 offset = vb->buffer_offset;
401 limit = buf->base.width0 - 1;
402
403 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(b)), 3);
404 PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | vb->stride);
405 PUSH_DATAh(push, buf->address + offset);
406 PUSH_DATA (push, buf->address + offset);
407 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(b)), 2);
408 PUSH_DATAh(push, buf->address + limit);
409 PUSH_DATA (push, buf->address + limit);
410
411 BCTX_REFN(nvc0->bufctx_3d, VTX, buf, RD);
412 }
413 if (nvc0->vbo_user)
414 nvc0_update_user_vbufs_shared(nvc0);
415 }
416
417 void
418 nvc0_vertex_arrays_validate(struct nvc0_context *nvc0)
419 {
420 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
421 struct nvc0_vertex_stateobj *vertex = nvc0->vertex;
422 struct nvc0_vertex_element *ve;
423 uint32_t const_vbos;
424 unsigned i;
425 uint8_t vbo_mode;
426 bool update_vertex;
427
428 nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_VTX);
429
430 assert(vertex);
431 if (unlikely(vertex->need_conversion) ||
432 unlikely(nvc0->vertprog->vp.edgeflag < PIPE_MAX_ATTRIBS)) {
433 vbo_mode = 3;
434 } else {
435 vbo_mode = (nvc0->vbo_user && nvc0->vbo_push_hint) ? 1 : 0;
436 }
437 const_vbos = vbo_mode ? 0 : nvc0->constant_vbos;
438
439 update_vertex = (nvc0->dirty & NVC0_NEW_VERTEX) ||
440 (const_vbos != nvc0->state.constant_vbos) ||
441 (vbo_mode != nvc0->state.vbo_mode);
442
443 if (update_vertex) {
444 const unsigned n = MAX2(vertex->num_elements, nvc0->state.num_vtxelts);
445
446 nvc0->state.constant_vbos = const_vbos;
447 nvc0->state.constant_elts = 0;
448 nvc0->state.num_vtxelts = vertex->num_elements;
449 nvc0->state.vbo_mode = vbo_mode;
450
451 if (unlikely(vbo_mode)) {
452 if (unlikely(nvc0->state.instance_elts & 3)) {
453 /* translate mode uses only 2 vertex buffers */
454 nvc0->state.instance_elts &= ~3;
455 PUSH_SPACE(push, 3);
456 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_PER_INSTANCE(0)), 2);
457 PUSH_DATA (push, 0);
458 PUSH_DATA (push, 0);
459 }
460
461 PUSH_SPACE(push, n * 2 + 4);
462
463 BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(0)), n);
464 for (i = 0; i < vertex->num_elements; ++i)
465 PUSH_DATA(push, vertex->element[i].state_alt);
466 for (; i < n; ++i)
467 PUSH_DATA(push, NVC0_3D_VERTEX_ATTRIB_INACTIVE);
468
469 BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(0)), 1);
470 PUSH_DATA (push, (1 << 12) | vertex->size);
471 for (i = 1; i < n; ++i)
472 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 0);
473 } else {
474 uint32_t *restrict data;
475
476 if (unlikely(vertex->instance_elts != nvc0->state.instance_elts)) {
477 nvc0->state.instance_elts = vertex->instance_elts;
478 assert(n); /* if (n == 0), both masks should be 0 */
479 PUSH_SPACE(push, 3);
480 BEGIN_NVC0(push, NVC0_3D(MACRO_VERTEX_ARRAY_PER_INSTANCE), 2);
481 PUSH_DATA (push, n);
482 PUSH_DATA (push, vertex->instance_elts);
483 }
484
485 PUSH_SPACE(push, n * 2 + 1);
486 BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(0)), n);
487 data = push->cur;
488 push->cur += n;
489 for (i = 0; i < vertex->num_elements; ++i) {
490 ve = &vertex->element[i];
491 data[i] = ve->state;
492 if (unlikely(const_vbos & (1 << ve->pipe.vertex_buffer_index))) {
493 nvc0->state.constant_elts |= 1 << i;
494 data[i] |= NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST;
495 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 0);
496 }
497 }
498 for (; i < n; ++i) {
499 data[i] = NVC0_3D_VERTEX_ATTRIB_INACTIVE;
500 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(i)), 0);
501 }
502 }
503 }
504 if (nvc0->state.vbo_mode) /* using translate, don't set up arrays here */
505 return;
506
507 if (vertex->shared_slots)
508 nvc0_validate_vertex_buffers_shared(nvc0);
509 else
510 nvc0_validate_vertex_buffers(nvc0);
511 }
512
513 void
514 nvc0_idxbuf_validate(struct nvc0_context *nvc0)
515 {
516 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
517 struct nv04_resource *buf = nv04_resource(nvc0->idxbuf.buffer);
518
519 assert(buf);
520 assert(nouveau_resource_mapped_by_gpu(&buf->base));
521
522 PUSH_SPACE(push, 6);
523 BEGIN_NVC0(push, NVC0_3D(INDEX_ARRAY_START_HIGH), 5);
524 PUSH_DATAh(push, buf->address + nvc0->idxbuf.offset);
525 PUSH_DATA (push, buf->address + nvc0->idxbuf.offset);
526 PUSH_DATAh(push, buf->address + buf->base.width0 - 1);
527 PUSH_DATA (push, buf->address + buf->base.width0 - 1);
528 PUSH_DATA (push, nvc0->idxbuf.index_size >> 1);
529
530 BCTX_REFN(nvc0->bufctx_3d, IDX, buf, RD);
531 }
532
533 #define NVC0_PRIM_GL_CASE(n) \
534 case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
535
536 static inline unsigned
537 nvc0_prim_gl(unsigned prim)
538 {
539 switch (prim) {
540 NVC0_PRIM_GL_CASE(POINTS);
541 NVC0_PRIM_GL_CASE(LINES);
542 NVC0_PRIM_GL_CASE(LINE_LOOP);
543 NVC0_PRIM_GL_CASE(LINE_STRIP);
544 NVC0_PRIM_GL_CASE(TRIANGLES);
545 NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
546 NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
547 NVC0_PRIM_GL_CASE(QUADS);
548 NVC0_PRIM_GL_CASE(QUAD_STRIP);
549 NVC0_PRIM_GL_CASE(POLYGON);
550 NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
551 NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
552 NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
553 NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
554 NVC0_PRIM_GL_CASE(PATCHES);
555 default:
556 return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
557 }
558 }
559
560 static void
561 nvc0_draw_vbo_kick_notify(struct nouveau_pushbuf *push)
562 {
563 struct nvc0_screen *screen = push->user_priv;
564
565 nouveau_fence_update(&screen->base, true);
566
567 NOUVEAU_DRV_STAT(&screen->base, pushbuf_count, 1);
568 }
569
570 static void
571 nvc0_draw_arrays(struct nvc0_context *nvc0,
572 unsigned mode, unsigned start, unsigned count,
573 unsigned instance_count)
574 {
575 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
576 unsigned prim;
577
578 if (nvc0->state.index_bias) {
579 /* index_bias is implied 0 if !info->indexed (really ?) */
580 /* TODO: can we deactivate it for the VERTEX_BUFFER_FIRST command ? */
581 PUSH_SPACE(push, 2);
582 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_BASE), 0);
583 IMMED_NVC0(push, NVC0_3D(VERTEX_ID_BASE), 0);
584 nvc0->state.index_bias = 0;
585 }
586
587 prim = nvc0_prim_gl(mode);
588
589 while (instance_count--) {
590 PUSH_SPACE(push, 6);
591 BEGIN_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), 1);
592 PUSH_DATA (push, prim);
593 BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
594 PUSH_DATA (push, start);
595 PUSH_DATA (push, count);
596 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
597
598 prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
599 }
600 NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_array, 1);
601 }
602
603 static void
604 nvc0_draw_elements_inline_u08(struct nouveau_pushbuf *push, const uint8_t *map,
605 unsigned start, unsigned count)
606 {
607 map += start;
608
609 if (count & 3) {
610 unsigned i;
611 PUSH_SPACE(push, 4);
612 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U32), count & 3);
613 for (i = 0; i < (count & 3); ++i)
614 PUSH_DATA(push, *map++);
615 count &= ~3;
616 }
617 while (count) {
618 unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 4) / 4;
619
620 PUSH_SPACE(push, nr + 1);
621 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U8), nr);
622 for (i = 0; i < nr; ++i) {
623 PUSH_DATA(push,
624 (map[3] << 24) | (map[2] << 16) | (map[1] << 8) | map[0]);
625 map += 4;
626 }
627 count -= nr * 4;
628 }
629 }
630
631 static void
632 nvc0_draw_elements_inline_u16(struct nouveau_pushbuf *push, const uint16_t *map,
633 unsigned start, unsigned count)
634 {
635 map += start;
636
637 if (count & 1) {
638 count &= ~1;
639 PUSH_SPACE(push, 2);
640 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
641 PUSH_DATA (push, *map++);
642 }
643 while (count) {
644 unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2;
645
646 PUSH_SPACE(push, nr + 1);
647 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U16), nr);
648 for (i = 0; i < nr; ++i) {
649 PUSH_DATA(push, (map[1] << 16) | map[0]);
650 map += 2;
651 }
652 count -= nr * 2;
653 }
654 }
655
656 static void
657 nvc0_draw_elements_inline_u32(struct nouveau_pushbuf *push, const uint32_t *map,
658 unsigned start, unsigned count)
659 {
660 map += start;
661
662 while (count) {
663 const unsigned nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN);
664
665 PUSH_SPACE(push, nr + 1);
666 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U32), nr);
667 PUSH_DATAp(push, map, nr);
668
669 map += nr;
670 count -= nr;
671 }
672 }
673
674 static void
675 nvc0_draw_elements_inline_u32_short(struct nouveau_pushbuf *push,
676 const uint32_t *map,
677 unsigned start, unsigned count)
678 {
679 map += start;
680
681 if (count & 1) {
682 count--;
683 PUSH_SPACE(push, 1);
684 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
685 PUSH_DATA (push, *map++);
686 }
687 while (count) {
688 unsigned i, nr = MIN2(count, NV04_PFIFO_MAX_PACKET_LEN * 2) / 2;
689
690 PUSH_SPACE(push, nr + 1);
691 BEGIN_NIC0(push, NVC0_3D(VB_ELEMENT_U16), nr);
692 for (i = 0; i < nr; ++i) {
693 PUSH_DATA(push, (map[1] << 16) | map[0]);
694 map += 2;
695 }
696 count -= nr * 2;
697 }
698 }
699
700 static void
701 nvc0_draw_elements(struct nvc0_context *nvc0, bool shorten,
702 unsigned mode, unsigned start, unsigned count,
703 unsigned instance_count, int32_t index_bias)
704 {
705 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
706 unsigned prim;
707 const unsigned index_size = nvc0->idxbuf.index_size;
708
709 prim = nvc0_prim_gl(mode);
710
711 if (index_bias != nvc0->state.index_bias) {
712 PUSH_SPACE(push, 4);
713 BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_BASE), 1);
714 PUSH_DATA (push, index_bias);
715 BEGIN_NVC0(push, NVC0_3D(VERTEX_ID_BASE), 1);
716 PUSH_DATA (push, index_bias);
717 nvc0->state.index_bias = index_bias;
718 }
719
720 if (nvc0->idxbuf.buffer) {
721 PUSH_SPACE(push, 1);
722 IMMED_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), prim);
723 do {
724 PUSH_SPACE(push, 7);
725 BEGIN_NVC0(push, NVC0_3D(INDEX_BATCH_FIRST), 2);
726 PUSH_DATA (push, start);
727 PUSH_DATA (push, count);
728 if (--instance_count) {
729 BEGIN_NVC0(push, NVC0_3D(VERTEX_END_GL), 2);
730 PUSH_DATA (push, 0);
731 PUSH_DATA (push, prim | NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT);
732 }
733 } while (instance_count);
734 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
735 } else {
736 const void *data = nvc0->idxbuf.user_buffer;
737
738 while (instance_count--) {
739 PUSH_SPACE(push, 2);
740 BEGIN_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), 1);
741 PUSH_DATA (push, prim);
742 switch (index_size) {
743 case 1:
744 nvc0_draw_elements_inline_u08(push, data, start, count);
745 break;
746 case 2:
747 nvc0_draw_elements_inline_u16(push, data, start, count);
748 break;
749 case 4:
750 if (shorten)
751 nvc0_draw_elements_inline_u32_short(push, data, start, count);
752 else
753 nvc0_draw_elements_inline_u32(push, data, start, count);
754 break;
755 default:
756 assert(0);
757 return;
758 }
759 PUSH_SPACE(push, 1);
760 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
761
762 prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
763 }
764 }
765 NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_indexed, 1);
766 }
767
768 static void
769 nvc0_draw_stream_output(struct nvc0_context *nvc0,
770 const struct pipe_draw_info *info)
771 {
772 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
773 struct nvc0_so_target *so = nvc0_so_target(info->count_from_stream_output);
774 struct nv04_resource *res = nv04_resource(so->pipe.buffer);
775 unsigned mode = nvc0_prim_gl(info->mode);
776 unsigned num_instances = info->instance_count;
777
778 if (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
779 res->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
780 PUSH_SPACE(push, 2);
781 IMMED_NVC0(push, NVC0_3D(SERIALIZE), 0);
782 nvc0_hw_query_fifo_wait(push, nvc0_query(so->pq));
783 if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
784 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
785
786 NOUVEAU_DRV_STAT(&nvc0->screen->base, gpu_serialize_count, 1);
787 }
788
789 while (num_instances--) {
790 nouveau_pushbuf_space(push, 9, 0, 1);
791 BEGIN_NVC0(push, NVC0_3D(VERTEX_BEGIN_GL), 1);
792 PUSH_DATA (push, mode);
793 BEGIN_NVC0(push, NVC0_3D(DRAW_TFB_BASE), 1);
794 PUSH_DATA (push, 0);
795 BEGIN_NVC0(push, NVC0_3D(DRAW_TFB_STRIDE), 1);
796 PUSH_DATA (push, so->stride);
797 BEGIN_NVC0(push, NVC0_3D(DRAW_TFB_BYTES), 1);
798 nvc0_hw_query_pushbuf_submit(push, nvc0_query(so->pq), 0x4);
799 IMMED_NVC0(push, NVC0_3D(VERTEX_END_GL), 0);
800
801 mode |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
802 }
803 }
804
805 static void
806 nvc0_draw_indirect(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
807 {
808 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
809 struct nv04_resource *buf = nv04_resource(info->indirect);
810 unsigned size;
811 const uint32_t offset = buf->offset + info->indirect_offset;
812
813 /* must make FIFO wait for engines idle before continuing to process */
814 if (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr))
815 IMMED_NVC0(push, SUBC_3D(NV10_SUBCHAN_REF_CNT), 0);
816
817 /* Queue things up to let the macros write params to the driver constbuf */
818 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
819 PUSH_DATA (push, 512);
820 PUSH_DATAh(push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
821 PUSH_DATA (push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
822 BEGIN_NVC0(push, NVC0_3D(CB_POS), 1);
823 PUSH_DATA (push, 256 + 128);
824
825 nouveau_pushbuf_space(push, 8, 0, 1);
826 PUSH_REFN(push, buf->bo, NOUVEAU_BO_RD | buf->domain);
827 if (info->indexed) {
828 assert(nvc0->idxbuf.buffer);
829 assert(nouveau_resource_mapped_by_gpu(nvc0->idxbuf.buffer));
830 size = 5 * 4;
831 BEGIN_1IC0(push, NVC0_3D(MACRO_DRAW_ELEMENTS_INDIRECT), 1 + size / 4);
832 } else {
833 if (nvc0->state.index_bias) {
834 /* index_bias is implied 0 if !info->indexed (really ?) */
835 IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_BASE), 0);
836 IMMED_NVC0(push, NVC0_3D(VERTEX_ID_BASE), 0);
837 nvc0->state.index_bias = 0;
838 }
839 size = 4 * 4;
840 BEGIN_1IC0(push, NVC0_3D(MACRO_DRAW_ARRAYS_INDIRECT), 1 + size / 4);
841 }
842 PUSH_DATA(push, nvc0_prim_gl(info->mode));
843 #define NVC0_IB_ENTRY_1_NO_PREFETCH (1 << (31 - 8))
844 nouveau_pushbuf_data(push,
845 buf->bo, offset, NVC0_IB_ENTRY_1_NO_PREFETCH | size);
846 }
847
848 static inline void
849 nvc0_update_prim_restart(struct nvc0_context *nvc0, bool en, uint32_t index)
850 {
851 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
852
853 if (en != nvc0->state.prim_restart) {
854 if (en) {
855 BEGIN_NVC0(push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
856 PUSH_DATA (push, 1);
857 PUSH_DATA (push, index);
858 } else {
859 IMMED_NVC0(push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
860 }
861 nvc0->state.prim_restart = en;
862 } else
863 if (en) {
864 BEGIN_NVC0(push, NVC0_3D(PRIM_RESTART_INDEX), 1);
865 PUSH_DATA (push, index);
866 }
867 }
868
869 void
870 nvc0_draw_vbo(struct pipe_context *pipe, const struct pipe_draw_info *info)
871 {
872 struct nvc0_context *nvc0 = nvc0_context(pipe);
873 struct nouveau_pushbuf *push = nvc0->base.pushbuf;
874 int i, s;
875
876 /* NOTE: caller must ensure that (min_index + index_bias) is >= 0 */
877 nvc0->vb_elt_first = info->min_index + info->index_bias;
878 nvc0->vb_elt_limit = info->max_index - info->min_index;
879 nvc0->instance_off = info->start_instance;
880 nvc0->instance_max = info->instance_count - 1;
881
882 /* For picking only a few vertices from a large user buffer, push is better,
883 * if index count is larger and we expect repeated vertices, suggest upload.
884 */
885 nvc0->vbo_push_hint =
886 info->indexed && (nvc0->vb_elt_limit >= (info->count * 2));
887
888 /* Check whether we want to switch vertex-submission mode. */
889 if (nvc0->vbo_user && !(nvc0->dirty & (NVC0_NEW_ARRAYS | NVC0_NEW_VERTEX))) {
890 if (nvc0->vbo_push_hint != !!nvc0->state.vbo_mode)
891 if (nvc0->state.vbo_mode != 3)
892 nvc0->dirty |= NVC0_NEW_ARRAYS;
893
894 if (!(nvc0->dirty & NVC0_NEW_ARRAYS) && nvc0->state.vbo_mode == 0) {
895 if (nvc0->vertex->shared_slots)
896 nvc0_update_user_vbufs_shared(nvc0);
897 else
898 nvc0_update_user_vbufs(nvc0);
899 }
900 }
901
902 if (info->mode == PIPE_PRIM_PATCHES &&
903 nvc0->state.patch_vertices != info->vertices_per_patch) {
904 nvc0->state.patch_vertices = info->vertices_per_patch;
905 IMMED_NVC0(push, NVC0_3D(PATCH_VERTICES), nvc0->state.patch_vertices);
906 }
907
908 /* 8 as minimum to avoid immediate double validation of new buffers */
909 nvc0_state_validate(nvc0, ~0, 8);
910
911 if (nvc0->vertprog->vp.need_draw_parameters) {
912 BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
913 PUSH_DATA (push, 512);
914 PUSH_DATAh(push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
915 PUSH_DATA (push, nvc0->screen->uniform_bo->offset + (5 << 16) + (0 << 9));
916 BEGIN_1IC0(push, NVC0_3D(CB_POS), 1 + 3);
917 PUSH_DATA (push, 256 + 128);
918 PUSH_DATA (push, info->index_bias);
919 PUSH_DATA (push, info->start_instance);
920 PUSH_DATA (push, info->drawid);
921 }
922
923 push->kick_notify = nvc0_draw_vbo_kick_notify;
924
925 /* TODO: Instead of iterating over all the buffer resources looking for
926 * coherent buffers, keep track of a context-wide count.
927 */
928 for (s = 0; s < 5 && !nvc0->cb_dirty; ++s) {
929 uint32_t valid = nvc0->constbuf_valid[s];
930
931 while (valid && !nvc0->cb_dirty) {
932 const unsigned i = ffs(valid) - 1;
933 struct pipe_resource *res;
934
935 valid &= ~(1 << i);
936 if (nvc0->constbuf[s][i].user)
937 continue;
938
939 res = nvc0->constbuf[s][i].u.buf;
940 if (!res)
941 continue;
942
943 if (res->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT)
944 nvc0->cb_dirty = true;
945 }
946 }
947
948 if (nvc0->cb_dirty) {
949 IMMED_NVC0(push, NVC0_3D(MEM_BARRIER), 0x1011);
950 nvc0->cb_dirty = false;
951 }
952
953 for (s = 0; s < 5; ++s) {
954 for (int i = 0; i < nvc0->num_textures[s]; ++i) {
955 struct nv50_tic_entry *tic = nv50_tic_entry(nvc0->textures[s][i]);
956 struct pipe_resource *res;
957 if (!tic)
958 continue;
959 res = nvc0->textures[s][i]->texture;
960 if (res->target != PIPE_BUFFER ||
961 !(res->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT))
962 continue;
963
964 BEGIN_NVC0(push, NVC0_3D(TEX_CACHE_CTL), 1);
965 PUSH_DATA (push, (tic->id << 4) | 1);
966 NOUVEAU_DRV_STAT(&nvc0->screen->base, tex_cache_flush_count, 1);
967 }
968 }
969
970 if (nvc0->state.vbo_mode) {
971 nvc0_push_vbo(nvc0, info);
972 push->kick_notify = nvc0_default_kick_notify;
973 nouveau_pushbuf_bufctx(push, NULL);
974 return;
975 }
976
977 /* space for base instance, flush, and prim restart */
978 PUSH_SPACE(push, 8);
979
980 if (nvc0->state.instance_base != info->start_instance) {
981 nvc0->state.instance_base = info->start_instance;
982 /* NOTE: this does not affect the shader input, should it ? */
983 BEGIN_NVC0(push, NVC0_3D(VB_INSTANCE_BASE), 1);
984 PUSH_DATA (push, info->start_instance);
985 }
986
987 for (i = 0; i < nvc0->num_vtxbufs && !nvc0->base.vbo_dirty; ++i) {
988 if (!nvc0->vtxbuf[i].buffer)
989 continue;
990 if (nvc0->vtxbuf[i].buffer->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT)
991 nvc0->base.vbo_dirty = true;
992 }
993
994 if (!nvc0->base.vbo_dirty && nvc0->idxbuf.buffer &&
995 nvc0->idxbuf.buffer->flags & PIPE_RESOURCE_FLAG_MAP_COHERENT)
996 nvc0->base.vbo_dirty = true;
997
998 nvc0_update_prim_restart(nvc0, info->primitive_restart, info->restart_index);
999
1000 if (nvc0->base.vbo_dirty) {
1001 if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
1002 IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
1003 nvc0->base.vbo_dirty = false;
1004 }
1005
1006 if (unlikely(info->indirect)) {
1007 nvc0_draw_indirect(nvc0, info);
1008 } else
1009 if (unlikely(info->count_from_stream_output)) {
1010 nvc0_draw_stream_output(nvc0, info);
1011 } else
1012 if (info->indexed) {
1013 bool shorten = info->max_index <= 65535;
1014
1015 if (info->primitive_restart && info->restart_index > 65535)
1016 shorten = false;
1017
1018 nvc0_draw_elements(nvc0, shorten,
1019 info->mode, info->start, info->count,
1020 info->instance_count, info->index_bias);
1021 } else {
1022 nvc0_draw_arrays(nvc0,
1023 info->mode, info->start, info->count,
1024 info->instance_count);
1025 }
1026 push->kick_notify = nvc0_default_kick_notify;
1027
1028 nvc0_release_user_vbufs(nvc0);
1029
1030 nouveau_pushbuf_bufctx(push, NULL);
1031 }