src/gallium/drivers/nouveau/nvc0/nvc0_vbo_translate.c

   1
   2 #include "pipe/p_context.h"
   3 #include "pipe/p_state.h"
   4 #include "util/u_inlines.h"
   5 #include "util/format/u_format.h"
   6 #include "translate/translate.h"
   7
   8 #include "nvc0/nvc0_context.h"
   9 #include "nvc0/nvc0_resource.h"
  10
  11 #include "nvc0/nvc0_3d.xml.h"
  12
  13 struct push_context {
  14    struct nouveau_pushbuf *push;
  15
  16    struct translate *translate;
  17    void *dest;
  18    const void *idxbuf;
  19
  20    uint32_t vertex_size;
  21    uint32_t restart_index;
  22    uint32_t start_instance;
  23    uint32_t instance_id;
  24
  25    bool prim_restart;
  26    bool need_vertex_id;
  27
  28    struct {
  29       bool enabled;
  30       bool value;
  31       uint8_t width;
  32       unsigned stride;
  33       const uint8_t *data;
  34    } edgeflag;
  35 };
  36
  37 static void nvc0_push_upload_vertex_ids(struct push_context *,
  38                                         struct nvc0_context *,
  39                                         const struct pipe_draw_info *);
  40
  41 static void
  42 nvc0_push_context_init(struct nvc0_context *nvc0, struct push_context *ctx)
  43 {
  44    ctx->push = nvc0->base.pushbuf;
  45
  46    ctx->translate = nvc0->vertex->translate;
  47    ctx->vertex_size = nvc0->vertex->size;
  48    ctx->instance_id = 0;
  49
  50    ctx->need_vertex_id =
  51       nvc0->vertprog->vp.need_vertex_id && (nvc0->vertex->num_elements < 32);
  52
  53    ctx->edgeflag.value = true;
  54    ctx->edgeflag.enabled = nvc0->vertprog->vp.edgeflag < PIPE_MAX_ATTRIBS;
  55
  56    /* silence warnings */
  57    ctx->edgeflag.data = NULL;
  58    ctx->edgeflag.stride = 0;
  59    ctx->edgeflag.width = 0;
  60 }
  61
  62 static inline void
  63 nvc0_vertex_configure_translate(struct nvc0_context *nvc0, int32_t index_bias)
  64 {
  65    struct translate *translate = nvc0->vertex->translate;
  66    unsigned i;
  67
  68    for (i = 0; i < nvc0->num_vtxbufs; ++i) {
  69       const uint8_t *map;
  70       const struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[i];
  71
  72       if (likely(vb->is_user_buffer))
  73          map = (const uint8_t *)vb->buffer.user;
  74       else {
  75          if (!vb->buffer.resource)
  76             continue;
  77
  78          map = nouveau_resource_map_offset(&nvc0->base,
  79             nv04_resource(vb->buffer.resource), vb->buffer_offset, NOUVEAU_BO_RD);
  80       }
  81
  82       if (index_bias && !unlikely(nvc0->vertex->instance_bufs & (1 << i)))
  83          map += (intptr_t)index_bias * vb->stride;
  84
  85       translate->set_buffer(translate, i, map, vb->stride, ~0);
  86    }
  87 }
  88
  89 static inline void
  90 nvc0_push_map_idxbuf(struct push_context *ctx, struct nvc0_context *nvc0,
  91                      const struct pipe_draw_info *info)
  92 {
  93    if (!info->has_user_indices) {
  94       struct nv04_resource *buf = nv04_resource(info->index.resource);
  95       ctx->idxbuf = nouveau_resource_map_offset(
  96             &nvc0->base, buf, 0, NOUVEAU_BO_RD);
  97    } else {
  98       ctx->idxbuf = info->index.user;
  99    }
 100 }
 101
 102 static inline void
 103 nvc0_push_map_edgeflag(struct push_context *ctx, struct nvc0_context *nvc0,
 104                        int32_t index_bias)
 105 {
 106    unsigned attr = nvc0->vertprog->vp.edgeflag;
 107    struct pipe_vertex_element *ve = &nvc0->vertex->element[attr].pipe;
 108    struct pipe_vertex_buffer *vb = &nvc0->vtxbuf[ve->vertex_buffer_index];
 109    struct nv04_resource *buf = nv04_resource(vb->buffer.resource);
 110
 111    ctx->edgeflag.stride = vb->stride;
 112    ctx->edgeflag.width = util_format_get_blocksize(ve->src_format);
 113    if (!vb->is_user_buffer) {
 114       unsigned offset = vb->buffer_offset + ve->src_offset;
 115       ctx->edgeflag.data = nouveau_resource_map_offset(&nvc0->base,
 116                            buf, offset, NOUVEAU_BO_RD);
 117    } else {
 118       ctx->edgeflag.data = (const uint8_t *)vb->buffer.user + ve->src_offset;
 119    }
 120
 121    if (index_bias)
 122       ctx->edgeflag.data += (intptr_t)index_bias * vb->stride;
 123 }
 124
 125 static inline unsigned
 126 prim_restart_search_i08(const uint8_t *elts, unsigned push, uint8_t index)
 127 {
 128    unsigned i;
 129    for (i = 0; i < push && elts[i] != index; ++i);
 130    return i;
 131 }
 132
 133 static inline unsigned
 134 prim_restart_search_i16(const uint16_t *elts, unsigned push, uint16_t index)
 135 {
 136    unsigned i;
 137    for (i = 0; i < push && elts[i] != index; ++i);
 138    return i;
 139 }
 140
 141 static inline unsigned
 142 prim_restart_search_i32(const uint32_t *elts, unsigned push, uint32_t index)
 143 {
 144    unsigned i;
 145    for (i = 0; i < push && elts[i] != index; ++i);
 146    return i;
 147 }
 148
 149 static inline bool
 150 ef_value_8(const struct push_context *ctx, uint32_t index)
 151 {
 152    uint8_t *pf = (uint8_t *)&ctx->edgeflag.data[index * ctx->edgeflag.stride];
 153    return !!*pf;
 154 }
 155
 156 static inline bool
 157 ef_value_32(const struct push_context *ctx, uint32_t index)
 158 {
 159    uint32_t *pf = (uint32_t *)&ctx->edgeflag.data[index * ctx->edgeflag.stride];
 160    return !!*pf;
 161 }
 162
 163 static inline bool
 164 ef_toggle(struct push_context *ctx)
 165 {
 166    ctx->edgeflag.value = !ctx->edgeflag.value;
 167    return ctx->edgeflag.value;
 168 }
 169
 170 static inline unsigned
 171 ef_toggle_search_i08(struct push_context *ctx, const uint8_t *elts, unsigned n)
 172 {
 173    unsigned i;
 174    bool ef = ctx->edgeflag.value;
 175    if (ctx->edgeflag.width == 1)
 176       for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
 177    else
 178       for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
 179    return i;
 180 }
 181
 182 static inline unsigned
 183 ef_toggle_search_i16(struct push_context *ctx, const uint16_t *elts, unsigned n)
 184 {
 185    unsigned i;
 186    bool ef = ctx->edgeflag.value;
 187    if (ctx->edgeflag.width == 1)
 188       for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
 189    else
 190       for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
 191    return i;
 192 }
 193
 194 static inline unsigned
 195 ef_toggle_search_i32(struct push_context *ctx, const uint32_t *elts, unsigned n)
 196 {
 197    unsigned i;
 198    bool ef = ctx->edgeflag.value;
 199    if (ctx->edgeflag.width == 1)
 200       for (i = 0; i < n && ef_value_8(ctx, elts[i]) == ef; ++i);
 201    else
 202       for (i = 0; i < n && ef_value_32(ctx, elts[i]) == ef; ++i);
 203    return i;
 204 }
 205
 206 static inline unsigned
 207 ef_toggle_search_seq(struct push_context *ctx, unsigned start, unsigned n)
 208 {
 209    unsigned i;
 210    bool ef = ctx->edgeflag.value;
 211    if (ctx->edgeflag.width == 1)
 212       for (i = 0; i < n && ef_value_8(ctx, start++) == ef; ++i);
 213    else
 214       for (i = 0; i < n && ef_value_32(ctx, start++) == ef; ++i);
 215    return i;
 216 }
 217
 218 static inline void *
 219 nvc0_push_setup_vertex_array(struct nvc0_context *nvc0, const unsigned count)
 220 {
 221    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 222    struct nouveau_bo *bo;
 223    uint64_t va;
 224    const unsigned size = count * nvc0->vertex->size;
 225
 226    void *const dest = nouveau_scratch_get(&nvc0->base, size, &va, &bo);
 227
 228    BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_START_HIGH(0)), 2);
 229    PUSH_DATAh(push, va);
 230    PUSH_DATA (push, va);
 231
 232    if (nvc0->screen->eng3d->oclass < TU102_3D_CLASS)
 233       BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(0)), 2);
 234    else
 235       BEGIN_NVC0(push, SUBC_3D(TU102_3D_VERTEX_ARRAY_LIMIT_HIGH(0)), 2);
 236    PUSH_DATAh(push, va + size - 1);
 237    PUSH_DATA (push, va + size - 1);
 238
 239    BCTX_REFN_bo(nvc0->bufctx_3d, 3D_VTX_TMP, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
 240                 bo);
 241    nouveau_pushbuf_validate(push);
 242
 243    return dest;
 244 }
 245
 246 static void
 247 disp_vertices_i08(struct push_context *ctx, unsigned start, unsigned count)
 248 {
 249    struct nouveau_pushbuf *push = ctx->push;
 250    struct translate *translate = ctx->translate;
 251    const uint8_t *restrict elts = (uint8_t *)ctx->idxbuf + start;
 252    unsigned pos = 0;
 253
 254    do {
 255       unsigned nR = count;
 256
 257       if (unlikely(ctx->prim_restart))
 258          nR = prim_restart_search_i08(elts, nR, ctx->restart_index);
 259
 260       translate->run_elts8(translate, elts, nR,
 261                            ctx->start_instance, ctx->instance_id, ctx->dest);
 262       count -= nR;
 263       ctx->dest += nR * ctx->vertex_size;
 264
 265       while (nR) {
 266          unsigned nE = nR;
 267
 268          if (unlikely(ctx->edgeflag.enabled))
 269             nE = ef_toggle_search_i08(ctx, elts, nR);
 270
 271          PUSH_SPACE(push, 4);
 272          if (likely(nE >= 2)) {
 273             BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
 274             PUSH_DATA (push, pos);
 275             PUSH_DATA (push, nE);
 276          } else
 277          if (nE) {
 278             if (pos <= 0xff) {
 279                IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
 280             } else {
 281                BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
 282                PUSH_DATA (push, pos);
 283             }
 284          }
 285          if (unlikely(nE != nR))
 286             IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
 287
 288          pos += nE;
 289          elts += nE;
 290          nR -= nE;
 291       }
 292       if (count) {
 293          BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
 294          PUSH_DATA (push, 0xffffffff);
 295          ++elts;
 296          ctx->dest += ctx->vertex_size;
 297          ++pos;
 298          --count;
 299       }
 300    } while (count);
 301 }
 302
 303 static void
 304 disp_vertices_i16(struct push_context *ctx, unsigned start, unsigned count)
 305 {
 306    struct nouveau_pushbuf *push = ctx->push;
 307    struct translate *translate = ctx->translate;
 308    const uint16_t *restrict elts = (uint16_t *)ctx->idxbuf + start;
 309    unsigned pos = 0;
 310
 311    do {
 312       unsigned nR = count;
 313
 314       if (unlikely(ctx->prim_restart))
 315          nR = prim_restart_search_i16(elts, nR, ctx->restart_index);
 316
 317       translate->run_elts16(translate, elts, nR,
 318                             ctx->start_instance, ctx->instance_id, ctx->dest);
 319       count -= nR;
 320       ctx->dest += nR * ctx->vertex_size;
 321
 322       while (nR) {
 323          unsigned nE = nR;
 324
 325          if (unlikely(ctx->edgeflag.enabled))
 326             nE = ef_toggle_search_i16(ctx, elts, nR);
 327
 328          PUSH_SPACE(push, 4);
 329          if (likely(nE >= 2)) {
 330             BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
 331             PUSH_DATA (push, pos);
 332             PUSH_DATA (push, nE);
 333          } else
 334          if (nE) {
 335             if (pos <= 0xff) {
 336                IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
 337             } else {
 338                BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
 339                PUSH_DATA (push, pos);
 340             }
 341          }
 342          if (unlikely(nE != nR))
 343             IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
 344
 345          pos += nE;
 346          elts += nE;
 347          nR -= nE;
 348       }
 349       if (count) {
 350          BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
 351          PUSH_DATA (push, 0xffffffff);
 352          ++elts;
 353          ctx->dest += ctx->vertex_size;
 354          ++pos;
 355          --count;
 356       }
 357    } while (count);
 358 }
 359
 360 static void
 361 disp_vertices_i32(struct push_context *ctx, unsigned start, unsigned count)
 362 {
 363    struct nouveau_pushbuf *push = ctx->push;
 364    struct translate *translate = ctx->translate;
 365    const uint32_t *restrict elts = (uint32_t *)ctx->idxbuf + start;
 366    unsigned pos = 0;
 367
 368    do {
 369       unsigned nR = count;
 370
 371       if (unlikely(ctx->prim_restart))
 372          nR = prim_restart_search_i32(elts, nR, ctx->restart_index);
 373
 374       translate->run_elts(translate, elts, nR,
 375                           ctx->start_instance, ctx->instance_id, ctx->dest);
 376       count -= nR;
 377       ctx->dest += nR * ctx->vertex_size;
 378
 379       while (nR) {
 380          unsigned nE = nR;
 381
 382          if (unlikely(ctx->edgeflag.enabled))
 383             nE = ef_toggle_search_i32(ctx, elts, nR);
 384
 385          PUSH_SPACE(push, 4);
 386          if (likely(nE >= 2)) {
 387             BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
 388             PUSH_DATA (push, pos);
 389             PUSH_DATA (push, nE);
 390          } else
 391          if (nE) {
 392             if (pos <= 0xff) {
 393                IMMED_NVC0(push, NVC0_3D(VB_ELEMENT_U32), pos);
 394             } else {
 395                BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
 396                PUSH_DATA (push, pos);
 397             }
 398          }
 399          if (unlikely(nE != nR))
 400             IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
 401
 402          pos += nE;
 403          elts += nE;
 404          nR -= nE;
 405       }
 406       if (count) {
 407          BEGIN_NVC0(push, NVC0_3D(VB_ELEMENT_U32), 1);
 408          PUSH_DATA (push, 0xffffffff);
 409          ++elts;
 410          ctx->dest += ctx->vertex_size;
 411          ++pos;
 412          --count;
 413       }
 414    } while (count);
 415 }
 416
 417 static void
 418 disp_vertices_seq(struct push_context *ctx, unsigned start, unsigned count)
 419 {
 420    struct nouveau_pushbuf *push = ctx->push;
 421    struct translate *translate = ctx->translate;
 422    unsigned pos = 0;
 423
 424    /* XXX: This will read the data corresponding to the primitive restart index,
 425     *  maybe we should avoid that ?
 426     */
 427    translate->run(translate, start, count,
 428                   ctx->start_instance, ctx->instance_id, ctx->dest);
 429    do {
 430       unsigned nr = count;
 431
 432       if (unlikely(ctx->edgeflag.enabled))
 433          nr = ef_toggle_search_seq(ctx, start + pos, nr);
 434
 435       PUSH_SPACE(push, 4);
 436       if (likely(nr)) {
 437          BEGIN_NVC0(push, NVC0_3D(VERTEX_BUFFER_FIRST), 2);
 438          PUSH_DATA (push, pos);
 439          PUSH_DATA (push, nr);
 440       }
 441       if (unlikely(nr != count))
 442          IMMED_NVC0(push, NVC0_3D(EDGEFLAG), ef_toggle(ctx));
 443
 444       pos += nr;
 445       count -= nr;
 446    } while (count);
 447 }
 448
 449
 450 #define NVC0_PRIM_GL_CASE(n) \
 451    case PIPE_PRIM_##n: return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_##n
 452
 453 static inline unsigned
 454 nvc0_prim_gl(unsigned prim)
 455 {
 456    switch (prim) {
 457    NVC0_PRIM_GL_CASE(POINTS);
 458    NVC0_PRIM_GL_CASE(LINES);
 459    NVC0_PRIM_GL_CASE(LINE_LOOP);
 460    NVC0_PRIM_GL_CASE(LINE_STRIP);
 461    NVC0_PRIM_GL_CASE(TRIANGLES);
 462    NVC0_PRIM_GL_CASE(TRIANGLE_STRIP);
 463    NVC0_PRIM_GL_CASE(TRIANGLE_FAN);
 464    NVC0_PRIM_GL_CASE(QUADS);
 465    NVC0_PRIM_GL_CASE(QUAD_STRIP);
 466    NVC0_PRIM_GL_CASE(POLYGON);
 467    NVC0_PRIM_GL_CASE(LINES_ADJACENCY);
 468    NVC0_PRIM_GL_CASE(LINE_STRIP_ADJACENCY);
 469    NVC0_PRIM_GL_CASE(TRIANGLES_ADJACENCY);
 470    NVC0_PRIM_GL_CASE(TRIANGLE_STRIP_ADJACENCY);
 471    NVC0_PRIM_GL_CASE(PATCHES);
 472    default:
 473       return NVC0_3D_VERTEX_BEGIN_GL_PRIMITIVE_POINTS;
 474    }
 475 }
 476
 477 typedef struct {
 478    uint32_t count;
 479    uint32_t primCount;
 480    uint32_t first;
 481    uint32_t baseInstance;
 482 } DrawArraysIndirectCommand;
 483
 484 typedef struct {
 485    uint32_t count;
 486    uint32_t primCount;
 487    uint32_t firstIndex;
 488    int32_t  baseVertex;
 489    uint32_t baseInstance;
 490 } DrawElementsIndirectCommand;
 491
 492 void
 493 nvc0_push_vbo_indirect(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
 494 {
 495    /* The strategy here is to just read the commands from the indirect buffer
 496     * and do the draws. This is suboptimal, but will only happen in the case
 497     * that conversion is required for FIXED or DOUBLE inputs.
 498     */
 499    struct nvc0_screen *screen = nvc0->screen;
 500    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 501    struct nv04_resource *buf = nv04_resource(info->indirect->buffer);
 502    struct nv04_resource *buf_count = nv04_resource(info->indirect->indirect_draw_count);
 503    unsigned i;
 504
 505    unsigned draw_count = info->indirect->draw_count;
 506    if (buf_count) {
 507       uint32_t *count = nouveau_resource_map_offset(
 508             &nvc0->base, buf_count, info->indirect->indirect_draw_count_offset,
 509             NOUVEAU_BO_RD);
 510       draw_count = *count;
 511    }
 512
 513    uint8_t *buf_data = nouveau_resource_map_offset(
 514             &nvc0->base, buf, info->indirect->offset, NOUVEAU_BO_RD);
 515    struct pipe_draw_info single = *info;
 516    single.indirect = NULL;
 517    for (i = 0; i < draw_count; i++, buf_data += info->indirect->stride) {
 518       if (info->index_size) {
 519          DrawElementsIndirectCommand *cmd = (void *)buf_data;
 520          single.start = info->start + cmd->firstIndex;
 521          single.count = cmd->count;
 522          single.start_instance = cmd->baseInstance;
 523          single.instance_count = cmd->primCount;
 524          single.index_bias = cmd->baseVertex;
 525       } else {
 526          DrawArraysIndirectCommand *cmd = (void *)buf_data;
 527          single.start = cmd->first;
 528          single.count = cmd->count;
 529          single.start_instance = cmd->baseInstance;
 530          single.instance_count = cmd->primCount;
 531       }
 532
 533       if (nvc0->vertprog->vp.need_draw_parameters) {
 534          PUSH_SPACE(push, 9);
 535          BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
 536          PUSH_DATA (push, NVC0_CB_AUX_SIZE);
 537          PUSH_DATAh(push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(0));
 538          PUSH_DATA (push, screen->uniform_bo->offset + NVC0_CB_AUX_INFO(0));
 539          BEGIN_1IC0(push, NVC0_3D(CB_POS), 1 + 3);
 540          PUSH_DATA (push, NVC0_CB_AUX_DRAW_INFO);
 541          PUSH_DATA (push, single.index_bias);
 542          PUSH_DATA (push, single.start_instance);
 543          PUSH_DATA (push, single.drawid + i);
 544       }
 545
 546       nvc0_push_vbo(nvc0, &single);
 547    }
 548
 549    nouveau_resource_unmap(buf);
 550    if (buf_count)
 551       nouveau_resource_unmap(buf_count);
 552 }
 553
 554 void
 555 nvc0_push_vbo(struct nvc0_context *nvc0, const struct pipe_draw_info *info)
 556 {
 557    struct push_context ctx;
 558    unsigned i, index_size;
 559    unsigned inst_count = info->instance_count;
 560    unsigned vert_count = info->count;
 561    unsigned prim;
 562
 563    nvc0_push_context_init(nvc0, &ctx);
 564
 565    nvc0_vertex_configure_translate(nvc0, info->index_bias);
 566
 567    if (nvc0->state.index_bias) {
 568       /* this is already taken care of by translate */
 569       IMMED_NVC0(ctx.push, NVC0_3D(VB_ELEMENT_BASE), 0);
 570       nvc0->state.index_bias = 0;
 571    }
 572
 573    if (unlikely(ctx.edgeflag.enabled))
 574       nvc0_push_map_edgeflag(&ctx, nvc0, info->index_bias);
 575
 576    ctx.prim_restart = info->primitive_restart;
 577    ctx.restart_index = info->restart_index;
 578
 579    if (info->primitive_restart) {
 580       /* NOTE: I hope we won't ever need that last index (~0).
 581        * If we do, we have to disable primitive restart here always and
 582        * use END,BEGIN to restart. (XXX: would that affect PrimitiveID ?)
 583        * We could also deactive PRIM_RESTART_WITH_DRAW_ARRAYS temporarily,
 584        * and add manual restart to disp_vertices_seq.
 585        */
 586       BEGIN_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 2);
 587       PUSH_DATA (ctx.push, 1);
 588       PUSH_DATA (ctx.push, info->index_size ? 0xffffffff : info->restart_index);
 589    } else
 590    if (nvc0->state.prim_restart) {
 591       IMMED_NVC0(ctx.push, NVC0_3D(PRIM_RESTART_ENABLE), 0);
 592    }
 593    nvc0->state.prim_restart = info->primitive_restart;
 594
 595    if (info->index_size) {
 596       nvc0_push_map_idxbuf(&ctx, nvc0, info);
 597       index_size = info->index_size;
 598    } else {
 599       if (unlikely(info->count_from_stream_output)) {
 600          struct pipe_context *pipe = &nvc0->base.pipe;
 601          struct nvc0_so_target *targ;
 602          targ = nvc0_so_target(info->count_from_stream_output);
 603          pipe->get_query_result(pipe, targ->pq, true, (void *)&vert_count);
 604          vert_count /= targ->stride;
 605       }
 606       ctx.idxbuf = NULL; /* shut up warnings */
 607       index_size = 0;
 608    }
 609
 610    ctx.start_instance = info->start_instance;
 611
 612    prim = nvc0_prim_gl(info->mode);
 613    do {
 614       PUSH_SPACE(ctx.push, 9);
 615
 616       ctx.dest = nvc0_push_setup_vertex_array(nvc0, vert_count);
 617       if (unlikely(!ctx.dest))
 618          break;
 619
 620       if (unlikely(ctx.need_vertex_id))
 621          nvc0_push_upload_vertex_ids(&ctx, nvc0, info);
 622
 623       if (nvc0->screen->eng3d->oclass < GM107_3D_CLASS)
 624          IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FLUSH), 0);
 625       BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_BEGIN_GL), 1);
 626       PUSH_DATA (ctx.push, prim);
 627       switch (index_size) {
 628       case 1:
 629          disp_vertices_i08(&ctx, info->start, vert_count);
 630          break;
 631       case 2:
 632          disp_vertices_i16(&ctx, info->start, vert_count);
 633          break;
 634       case 4:
 635          disp_vertices_i32(&ctx, info->start, vert_count);
 636          break;
 637       default:
 638          assert(index_size == 0);
 639          disp_vertices_seq(&ctx, info->start, vert_count);
 640          break;
 641       }
 642       PUSH_SPACE(ctx.push, 1);
 643       IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_END_GL), 0);
 644
 645       if (--inst_count) {
 646          prim |= NVC0_3D_VERTEX_BEGIN_GL_INSTANCE_NEXT;
 647          ++ctx.instance_id;
 648       }
 649       nouveau_bufctx_reset(nvc0->bufctx_3d, NVC0_BIND_3D_VTX_TMP);
 650       nouveau_scratch_done(&nvc0->base);
 651    } while (inst_count);
 652
 653
 654    /* reset state and unmap buffers (no-op) */
 655
 656    if (unlikely(!ctx.edgeflag.value)) {
 657       PUSH_SPACE(ctx.push, 1);
 658       IMMED_NVC0(ctx.push, NVC0_3D(EDGEFLAG), 1);
 659    }
 660
 661    if (unlikely(ctx.need_vertex_id)) {
 662       PUSH_SPACE(ctx.push, 4);
 663       IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ID_REPLACE), 0);
 664       BEGIN_NVC0(ctx.push, NVC0_3D(VERTEX_ATTRIB_FORMAT(1)), 1);
 665       PUSH_DATA (ctx.push,
 666                  NVC0_3D_VERTEX_ATTRIB_FORMAT_CONST |
 667                  NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_FLOAT |
 668                  NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32);
 669       IMMED_NVC0(ctx.push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 0);
 670    }
 671
 672    if (info->index_size && !info->has_user_indices)
 673       nouveau_resource_unmap(nv04_resource(info->index.resource));
 674    for (i = 0; i < nvc0->num_vtxbufs; ++i)
 675       nouveau_resource_unmap(nv04_resource(nvc0->vtxbuf[i].buffer.resource));
 676
 677    NOUVEAU_DRV_STAT(&nvc0->screen->base, draw_calls_fallback_count, 1);
 678 }
 679
 680 static inline void
 681 copy_indices_u8(uint32_t *dst, const uint8_t *elts, uint32_t bias, unsigned n)
 682 {
 683    unsigned i;
 684    for (i = 0; i < n; ++i)
 685       dst[i] = elts[i] + bias;
 686 }
 687
 688 static inline void
 689 copy_indices_u16(uint32_t *dst, const uint16_t *elts, uint32_t bias, unsigned n)
 690 {
 691    unsigned i;
 692    for (i = 0; i < n; ++i)
 693       dst[i] = elts[i] + bias;
 694 }
 695
 696 static inline void
 697 copy_indices_u32(uint32_t *dst, const uint32_t *elts, uint32_t bias, unsigned n)
 698 {
 699    unsigned i;
 700    for (i = 0; i < n; ++i)
 701       dst[i] = elts[i] + bias;
 702 }
 703
 704 static void
 705 nvc0_push_upload_vertex_ids(struct push_context *ctx,
 706                             struct nvc0_context *nvc0,
 707                             const struct pipe_draw_info *info)
 708
 709 {
 710    struct nouveau_pushbuf *push = ctx->push;
 711    struct nouveau_bo *bo;
 712    uint64_t va;
 713    uint32_t *data;
 714    uint32_t format;
 715    unsigned index_size = info->index_size;
 716    unsigned i;
 717    unsigned a = nvc0->vertex->num_elements;
 718
 719    if (!index_size || info->index_bias)
 720       index_size = 4;
 721    data = (uint32_t *)nouveau_scratch_get(&nvc0->base,
 722                                           info->count * index_size, &va, &bo);
 723
 724    BCTX_REFN_bo(nvc0->bufctx_3d, 3D_VTX_TMP, NOUVEAU_BO_GART | NOUVEAU_BO_RD,
 725                 bo);
 726    nouveau_pushbuf_validate(push);
 727
 728    if (info->index_size) {
 729       if (!info->index_bias) {
 730          memcpy(data, ctx->idxbuf, info->count * index_size);
 731       } else {
 732          switch (info->index_size) {
 733          case 1:
 734             copy_indices_u8(data, ctx->idxbuf, info->index_bias, info->count);
 735             break;
 736          case 2:
 737             copy_indices_u16(data, ctx->idxbuf, info->index_bias, info->count);
 738             break;
 739          default:
 740             copy_indices_u32(data, ctx->idxbuf, info->index_bias, info->count);
 741             break;
 742          }
 743       }
 744    } else {
 745       for (i = 0; i < info->count; ++i)
 746          data[i] = i + (info->start + info->index_bias);
 747    }
 748
 749    format = (1 << NVC0_3D_VERTEX_ATTRIB_FORMAT_BUFFER__SHIFT) |
 750       NVC0_3D_VERTEX_ATTRIB_FORMAT_TYPE_UINT;
 751
 752    switch (index_size) {
 753    case 1:
 754       format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_8;
 755       break;
 756    case 2:
 757       format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_16;
 758       break;
 759    default:
 760       format |= NVC0_3D_VERTEX_ATTRIB_FORMAT_SIZE_32;
 761       break;
 762    }
 763
 764    PUSH_SPACE(push, 12);
 765
 766    if (unlikely(nvc0->state.instance_elts & 2)) {
 767       nvc0->state.instance_elts &= ~2;
 768       IMMED_NVC0(push, NVC0_3D(VERTEX_ARRAY_PER_INSTANCE(1)), 0);
 769    }
 770
 771    BEGIN_NVC0(push, NVC0_3D(VERTEX_ATTRIB_FORMAT(a)), 1);
 772    PUSH_DATA (push, format);
 773
 774    BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_FETCH(1)), 3);
 775    PUSH_DATA (push, NVC0_3D_VERTEX_ARRAY_FETCH_ENABLE | index_size);
 776    PUSH_DATAh(push, va);
 777    PUSH_DATA (push, va);
 778
 779    if (nvc0->screen->eng3d->oclass < TU102_3D_CLASS)
 780       BEGIN_NVC0(push, NVC0_3D(VERTEX_ARRAY_LIMIT_HIGH(1)), 2);
 781    else
 782       BEGIN_NVC0(push, SUBC_3D(TU102_3D_VERTEX_ARRAY_LIMIT_HIGH(1)), 2);
 783    PUSH_DATAh(push, va + info->count * index_size - 1);
 784    PUSH_DATA (push, va + info->count * index_size - 1);
 785
 786 #define NVC0_3D_VERTEX_ID_REPLACE_SOURCE_ATTR_X(a) \
 787    (((0x80 + (a) * 0x10) / 4) << NVC0_3D_VERTEX_ID_REPLACE_SOURCE__SHIFT)
 788
 789    BEGIN_NVC0(push, NVC0_3D(VERTEX_ID_REPLACE), 1);
 790    PUSH_DATA (push, NVC0_3D_VERTEX_ID_REPLACE_SOURCE_ATTR_X(a) | 1);
 791 }