src/mesa/main/glthread_draw.c

   1 /*
   2  * Copyright © 2020 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /* Draw function marshalling for glthread.
  25  *
  26  * The purpose of these glDraw wrappers is to upload non-VBO vertex and
  27  * index data, so that glthread doesn't have to execute synchronously.
  28  */
  29
  30 #include "c99_alloca.h"
  31
  32 #include "main/glthread_marshal.h"
  33 #include "main/dispatch.h"
  34 #include "main/varray.h"
  35
  36 static inline unsigned
  37 get_index_size(GLenum type)
  38 {
  39    /* GL_UNSIGNED_BYTE  - GL_UNSIGNED_BYTE = 0
  40     * GL_UNSIGNED_SHORT - GL_UNSIGNED_BYTE = 2
  41     * GL_UNSIGNED_INT   - GL_UNSIGNED_BYTE = 4
  42     *
  43     * Divide by 2 to get n=0,1,2, then the index size is: 1 << n
  44     */
  45    return 1 << ((type - GL_UNSIGNED_BYTE) >> 1);
  46 }
  47
  48 static inline bool
  49 is_index_type_valid(GLenum type)
  50 {
  51    /* GL_UNSIGNED_BYTE  = 0x1401
  52     * GL_UNSIGNED_SHORT = 0x1403
  53     * GL_UNSIGNED_INT   = 0x1405
  54     *
  55     * The trick is that bit 1 and bit 2 mean USHORT and UINT, respectively.
  56     * After clearing those two bits (with ~6), we should get UBYTE.
  57     * Both bits can't be set, because the enum would be greater than UINT.
  58     */
  59    return type <= GL_UNSIGNED_INT && (type & ~6) == GL_UNSIGNED_BYTE;
  60 }
  61
  62 static ALWAYS_INLINE struct gl_buffer_object *
  63 upload_indices(struct gl_context *ctx, unsigned count, unsigned index_size,
  64                const GLvoid **indices)
  65 {
  66    struct gl_buffer_object *upload_buffer = NULL;
  67    unsigned upload_offset = 0;
  68
  69    assert(count);
  70
  71    _mesa_glthread_upload(ctx, *indices, index_size * count,
  72                          &upload_offset, &upload_buffer, NULL);
  73    assert(upload_buffer);
  74    *indices = (const GLvoid*)(intptr_t)upload_offset;
  75
  76    return upload_buffer;
  77 }
  78
  79 static ALWAYS_INLINE struct gl_buffer_object *
  80 upload_multi_indices(struct gl_context *ctx, unsigned total_count,
  81                      unsigned index_size, unsigned draw_count,
  82                      const GLsizei *count, const GLvoid *const *indices,
  83                      const GLvoid **out_indices)
  84 {
  85    struct gl_buffer_object *upload_buffer = NULL;
  86    unsigned upload_offset = 0;
  87    uint8_t *upload_ptr = NULL;
  88
  89    assert(total_count);
  90
  91    _mesa_glthread_upload(ctx, NULL, index_size * total_count,
  92                          &upload_offset, &upload_buffer, &upload_ptr);
  93    assert(upload_buffer);
  94
  95    for (unsigned i = 0, offset = 0; i < draw_count; i++) {
  96       if (count[i] == 0)
  97          continue;
  98
  99       unsigned size = count[i] * index_size;
 100
 101       memcpy(upload_ptr + offset, indices[i], size);
 102       out_indices[i] = (const GLvoid*)(intptr_t)(upload_offset + offset);
 103       offset += size;
 104    }
 105
 106    return upload_buffer;
 107 }
 108
 109 static ALWAYS_INLINE bool
 110 upload_vertices(struct gl_context *ctx, unsigned attrib_mask,
 111                 unsigned start_vertex, unsigned num_vertices,
 112                 unsigned start_instance, unsigned num_instances,
 113                 struct glthread_attrib_binding *attribs)
 114 {
 115    struct glthread_vao *vao = ctx->GLThread.CurrentVAO;
 116    unsigned attrib_mask_iter = attrib_mask;
 117    unsigned num_attribs = 0;
 118
 119    assert((num_vertices || !(attrib_mask & ~vao->NonZeroDivisorMask)) &&
 120           (num_instances || !(attrib_mask & vao->NonZeroDivisorMask)));
 121
 122    while (attrib_mask_iter) {
 123       unsigned i = u_bit_scan(&attrib_mask_iter);
 124       struct gl_buffer_object *upload_buffer = NULL;
 125       unsigned upload_offset = 0;
 126       unsigned stride = vao->Attrib[i].Stride;
 127       unsigned instance_div = vao->Attrib[i].Divisor;
 128       unsigned element_size = vao->Attrib[i].ElementSize;
 129       unsigned offset, size;
 130
 131       if (instance_div) {
 132          /* Per-instance attrib. */
 133
 134          /* Figure out how many instances we'll render given instance_div.  We
 135           * can't use the typical div_round_up() pattern because the CTS uses
 136           * instance_div = ~0 for a test, which overflows div_round_up()'s
 137           * addition.
 138           */
 139          unsigned count = num_instances / instance_div;
 140          if (count * instance_div != num_instances)
 141             count++;
 142
 143          offset = stride * start_instance;
 144          size = stride * (count - 1) + element_size;
 145       } else {
 146          /* Per-vertex attrib. */
 147          offset = stride * start_vertex;
 148          size = stride * (num_vertices - 1) + element_size;
 149       }
 150
 151       const void *ptr = vao->Attrib[i].Pointer;
 152       _mesa_glthread_upload(ctx, (uint8_t*)ptr + offset,
 153                             size, &upload_offset, &upload_buffer, NULL);
 154       assert(upload_buffer);
 155
 156       attribs[num_attribs].buffer = upload_buffer;
 157       attribs[num_attribs].offset = upload_offset - offset;
 158       attribs[num_attribs].original_pointer = ptr;
 159       num_attribs++;
 160    }
 161    return true;
 162 }
 163
 164 struct marshal_cmd_DrawArraysInstancedBaseInstance
 165 {
 166    struct marshal_cmd_base cmd_base;
 167    GLenum mode;
 168    GLint first;
 169    GLsizei count;
 170    GLsizei instance_count;
 171    GLuint baseinstance;
 172    GLuint non_vbo_attrib_mask;
 173 };
 174
 175 void
 176 _mesa_unmarshal_DrawArraysInstancedBaseInstance(struct gl_context *ctx,
 177                                                 const struct marshal_cmd_DrawArraysInstancedBaseInstance *cmd)
 178 {
 179    const GLenum mode = cmd->mode;
 180    const GLint first = cmd->first;
 181    const GLsizei count = cmd->count;
 182    const GLsizei instance_count = cmd->instance_count;
 183    const GLuint baseinstance = cmd->baseinstance;
 184    const GLuint non_vbo_attrib_mask = cmd->non_vbo_attrib_mask;
 185    const struct glthread_attrib_binding *attribs =
 186       (const struct glthread_attrib_binding *)(cmd + 1);
 187
 188    /* Bind uploaded buffers if needed. */
 189    if (non_vbo_attrib_mask) {
 190       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 191                                       false);
 192    }
 193
 194    CALL_DrawArraysInstancedBaseInstance(ctx->CurrentServerDispatch,
 195                                         (mode, first, count, instance_count,
 196                                          baseinstance));
 197
 198    /* Restore states. */
 199    if (non_vbo_attrib_mask) {
 200       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 201                                       true);
 202    }
 203 }
 204
 205 static ALWAYS_INLINE void
 206 draw_arrays_async(struct gl_context *ctx, GLenum mode, GLint first,
 207                   GLsizei count, GLsizei instance_count, GLuint baseinstance,
 208                   unsigned non_vbo_attrib_mask,
 209                   const struct glthread_attrib_binding *attribs)
 210 {
 211    int attribs_size = util_bitcount(non_vbo_attrib_mask) * sizeof(attribs[0]);
 212    int cmd_size = sizeof(struct marshal_cmd_DrawArraysInstancedBaseInstance) +
 213                   attribs_size;
 214    struct marshal_cmd_DrawArraysInstancedBaseInstance *cmd;
 215
 216    cmd = _mesa_glthread_allocate_command(ctx, DISPATCH_CMD_DrawArraysInstancedBaseInstance,
 217                                          cmd_size);
 218    cmd->mode = mode;
 219    cmd->first = first;
 220    cmd->count = count;
 221    cmd->instance_count = instance_count;
 222    cmd->baseinstance = baseinstance;
 223    cmd->non_vbo_attrib_mask = non_vbo_attrib_mask;
 224
 225    if (non_vbo_attrib_mask)
 226       memcpy(cmd + 1, attribs, attribs_size);
 227 }
 228
 229 void GLAPIENTRY
 230 _mesa_marshal_DrawArraysInstancedBaseInstance(GLenum mode, GLint first,
 231                                               GLsizei count, GLsizei instance_count,
 232                                               GLuint baseinstance)
 233 {
 234    GET_CURRENT_CONTEXT(ctx);
 235
 236    struct glthread_vao *vao = ctx->GLThread.CurrentVAO;
 237    unsigned non_vbo_attrib_mask = vao->UserPointerMask & vao->Enabled;
 238
 239    /* Fast path when nothing needs to be done.
 240     *
 241     * This is also an error path. Zero counts should still call the driver
 242     * for possible GL errors.
 243     */
 244    if (ctx->API == API_OPENGL_CORE || !non_vbo_attrib_mask ||
 245        count <= 0 || instance_count <= 0) {
 246       draw_arrays_async(ctx, mode, first, count, instance_count, baseinstance,
 247                         0, NULL);
 248       return;
 249    }
 250
 251    /* Upload and draw. */
 252    struct glthread_attrib_binding attribs[VERT_ATTRIB_MAX];
 253    if (!ctx->GLThread.SupportsNonVBOUploads ||
 254        !upload_vertices(ctx, non_vbo_attrib_mask, first, count, baseinstance,
 255                         instance_count, attribs)) {
 256       _mesa_glthread_finish_before(ctx, "DrawArrays");
 257       CALL_DrawArraysInstancedBaseInstance(ctx->CurrentServerDispatch,
 258                                            (mode, first, count, instance_count,
 259                                             baseinstance));
 260       return;
 261    }
 262
 263    draw_arrays_async(ctx, mode, first, count, instance_count, baseinstance,
 264                      non_vbo_attrib_mask, attribs);
 265 }
 266
 267 struct marshal_cmd_MultiDrawArrays
 268 {
 269    struct marshal_cmd_base cmd_base;
 270    GLenum mode;
 271    GLsizei draw_count;
 272    GLuint non_vbo_attrib_mask;
 273 };
 274
 275 void
 276 _mesa_unmarshal_MultiDrawArrays(struct gl_context *ctx,
 277                                 const struct marshal_cmd_MultiDrawArrays *cmd)
 278 {
 279    const GLenum mode = cmd->mode;
 280    const GLsizei draw_count = cmd->draw_count;
 281    const GLuint non_vbo_attrib_mask = cmd->non_vbo_attrib_mask;
 282
 283    const char *variable_data = (const char *)(cmd + 1);
 284    const GLint *first = (GLint *)variable_data;
 285    variable_data += sizeof(GLint) * draw_count;
 286    const GLsizei *count = (GLsizei *)variable_data;
 287    variable_data += sizeof(GLsizei) * draw_count;
 288    const struct glthread_attrib_binding *attribs =
 289       (const struct glthread_attrib_binding *)variable_data;
 290
 291    /* Bind uploaded buffers if needed. */
 292    if (non_vbo_attrib_mask) {
 293       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 294                                       false);
 295    }
 296
 297    CALL_MultiDrawArrays(ctx->CurrentServerDispatch,
 298                         (mode, first, count, draw_count));
 299
 300    /* Restore states. */
 301    if (non_vbo_attrib_mask) {
 302       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 303                                       true);
 304    }
 305 }
 306
 307 static ALWAYS_INLINE void
 308 multi_draw_arrays_async(struct gl_context *ctx, GLenum mode,
 309                         const GLint *first, const GLsizei *count,
 310                         GLsizei draw_count, unsigned non_vbo_attrib_mask,
 311                         const struct glthread_attrib_binding *attribs)
 312 {
 313    int first_size = sizeof(GLint) * draw_count;
 314    int count_size = sizeof(GLsizei) * draw_count;
 315    int attribs_size = util_bitcount(non_vbo_attrib_mask) * sizeof(attribs[0]);
 316    int cmd_size = sizeof(struct marshal_cmd_MultiDrawArrays) +
 317                   first_size + count_size + attribs_size;
 318    struct marshal_cmd_MultiDrawArrays *cmd;
 319
 320    cmd = _mesa_glthread_allocate_command(ctx, DISPATCH_CMD_MultiDrawArrays,
 321                                          cmd_size);
 322    cmd->mode = mode;
 323    cmd->draw_count = draw_count;
 324    cmd->non_vbo_attrib_mask = non_vbo_attrib_mask;
 325
 326    char *variable_data = (char*)(cmd + 1);
 327    memcpy(variable_data, first, first_size);
 328    variable_data += first_size;
 329    memcpy(variable_data, count, count_size);
 330
 331    if (non_vbo_attrib_mask) {
 332       variable_data += count_size;
 333       memcpy(variable_data, attribs, attribs_size);
 334    }
 335 }
 336
 337 void GLAPIENTRY
 338 _mesa_marshal_MultiDrawArrays(GLenum mode, const GLint *first,
 339                               const GLsizei *count, GLsizei draw_count)
 340 {
 341    GET_CURRENT_CONTEXT(ctx);
 342
 343    struct glthread_vao *vao = ctx->GLThread.CurrentVAO;
 344    unsigned non_vbo_attrib_mask = vao->UserPointerMask & vao->Enabled;
 345
 346    if (draw_count >= 0 &&
 347        (ctx->API == API_OPENGL_CORE || !non_vbo_attrib_mask)) {
 348       multi_draw_arrays_async(ctx, mode, first, count, draw_count, 0, NULL);
 349       return;
 350    }
 351
 352    /* If the draw count is too high or negative, the queue can't be used. */
 353    if (!ctx->GLThread.SupportsNonVBOUploads ||
 354        draw_count < 0 || draw_count > MARSHAL_MAX_CMD_SIZE / 16)
 355       goto sync;
 356
 357    unsigned min_index = ~0;
 358    unsigned max_index_exclusive = 0;
 359
 360    for (unsigned i = 0; i < draw_count; i++) {
 361       GLsizei vertex_count = count[i];
 362
 363       if (vertex_count < 0) {
 364          /* Just call the driver to set the error. */
 365          multi_draw_arrays_async(ctx, mode, first, count, draw_count, 0, NULL);
 366          return;
 367       }
 368       if (vertex_count == 0)
 369          continue;
 370
 371       min_index = MIN2(min_index, first[i]);
 372       max_index_exclusive = MAX2(max_index_exclusive, first[i] + vertex_count);
 373    }
 374
 375    unsigned num_vertices = max_index_exclusive - min_index;
 376    if (num_vertices == 0) {
 377       /* Nothing to do, but call the driver to set possible GL errors. */
 378       multi_draw_arrays_async(ctx, mode, first, count, draw_count, 0, NULL);
 379       return;
 380    }
 381
 382    /* Upload and draw. */
 383    struct glthread_attrib_binding attribs[VERT_ATTRIB_MAX];
 384    if (!upload_vertices(ctx, non_vbo_attrib_mask, min_index, num_vertices,
 385                         0, 1, attribs))
 386       goto sync;
 387
 388    multi_draw_arrays_async(ctx, mode, first, count, draw_count,
 389                            non_vbo_attrib_mask, attribs);
 390    return;
 391
 392 sync:
 393    _mesa_glthread_finish_before(ctx, "MultiDrawArrays");
 394    CALL_MultiDrawArrays(ctx->CurrentServerDispatch,
 395                         (mode, first, count, draw_count));
 396 }
 397
 398 struct marshal_cmd_DrawElementsInstancedBaseVertexBaseInstance
 399 {
 400    struct marshal_cmd_base cmd_base;
 401    bool index_bounds_valid;
 402    GLenum mode;
 403    GLenum type;
 404    GLsizei count;
 405    GLsizei instance_count;
 406    GLint basevertex;
 407    GLuint baseinstance;
 408    GLuint min_index;
 409    GLuint max_index;
 410    GLuint non_vbo_attrib_mask;
 411    const GLvoid *indices;
 412    struct gl_buffer_object *index_buffer;
 413 };
 414
 415 void
 416 _mesa_unmarshal_DrawElementsInstancedBaseVertexBaseInstance(struct gl_context *ctx,
 417                                                             const struct marshal_cmd_DrawElementsInstancedBaseVertexBaseInstance *cmd)
 418 {
 419    const GLenum mode = cmd->mode;
 420    const GLsizei count = cmd->count;
 421    const GLenum type = cmd->type;
 422    const GLvoid *indices = cmd->indices;
 423    const GLsizei instance_count = cmd->instance_count;
 424    const GLint basevertex = cmd->basevertex;
 425    const GLuint baseinstance = cmd->baseinstance;
 426    const GLuint min_index = cmd->min_index;
 427    const GLuint max_index = cmd->max_index;
 428    const GLuint non_vbo_attrib_mask = cmd->non_vbo_attrib_mask;
 429    struct gl_buffer_object *index_buffer = cmd->index_buffer;
 430    const struct glthread_attrib_binding *attribs =
 431       (const struct glthread_attrib_binding *)(cmd + 1);
 432
 433    /* Bind uploaded buffers if needed. */
 434    if (non_vbo_attrib_mask) {
 435       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 436                                       false);
 437    }
 438    if (index_buffer) {
 439       _mesa_InternalBindElementBuffer(ctx, index_buffer);
 440    }
 441
 442    /* Draw. */
 443    if (cmd->index_bounds_valid && instance_count == 1 && baseinstance == 0) {
 444       CALL_DrawRangeElementsBaseVertex(ctx->CurrentServerDispatch,
 445                                        (mode, min_index, max_index, count,
 446                                         type, indices, basevertex));
 447    } else {
 448       CALL_DrawElementsInstancedBaseVertexBaseInstance(ctx->CurrentServerDispatch,
 449                                                        (mode, count, type, indices,
 450                                                         instance_count, basevertex,
 451                                                         baseinstance));
 452    }
 453
 454    /* Restore states. */
 455    if (index_buffer) {
 456       _mesa_InternalBindElementBuffer(ctx, NULL);
 457    }
 458    if (non_vbo_attrib_mask) {
 459       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 460                                       true);
 461    }
 462 }
 463
 464 static ALWAYS_INLINE void
 465 draw_elements_async(struct gl_context *ctx, GLenum mode, GLsizei count,
 466                     GLenum type, const GLvoid *indices, GLsizei instance_count,
 467                     GLint basevertex, GLuint baseinstance,
 468                     bool index_bounds_valid, GLuint min_index, GLuint max_index,
 469                     struct gl_buffer_object *index_buffer,
 470                     unsigned non_vbo_attrib_mask,
 471                     const struct glthread_attrib_binding *attribs)
 472 {
 473    int attribs_size = util_bitcount(non_vbo_attrib_mask) * sizeof(attribs[0]);
 474    int cmd_size = sizeof(struct marshal_cmd_DrawElementsInstancedBaseVertexBaseInstance) +
 475                   attribs_size;
 476    struct marshal_cmd_DrawElementsInstancedBaseVertexBaseInstance *cmd;
 477
 478    cmd = _mesa_glthread_allocate_command(ctx, DISPATCH_CMD_DrawElementsInstancedBaseVertexBaseInstance, cmd_size);
 479    cmd->mode = mode;
 480    cmd->count = count;
 481    cmd->type = type;
 482    cmd->indices = indices;
 483    cmd->instance_count = instance_count;
 484    cmd->basevertex = basevertex;
 485    cmd->baseinstance = baseinstance;
 486    cmd->min_index = min_index;
 487    cmd->max_index = max_index;
 488    cmd->non_vbo_attrib_mask = non_vbo_attrib_mask;
 489    cmd->index_bounds_valid = index_bounds_valid;
 490    cmd->index_buffer = index_buffer;
 491
 492    if (non_vbo_attrib_mask)
 493       memcpy(cmd + 1, attribs, attribs_size);
 494 }
 495
 496 static void
 497 draw_elements(GLenum mode, GLsizei count, GLenum type, const GLvoid *indices,
 498               GLsizei instance_count, GLint basevertex, GLuint baseinstance,
 499               bool index_bounds_valid, GLuint min_index, GLuint max_index)
 500 {
 501    GET_CURRENT_CONTEXT(ctx);
 502
 503    struct glthread_vao *vao = ctx->GLThread.CurrentVAO;
 504    unsigned non_vbo_attrib_mask = vao->UserPointerMask & vao->Enabled;
 505    bool has_user_indices = vao->CurrentElementBufferName == 0;
 506
 507    /* Fast path when nothing needs to be done.
 508     *
 509     * This is also an error path. Zero counts should still call the driver
 510     * for possible GL errors.
 511     */
 512    if (ctx->API == API_OPENGL_CORE ||
 513        count <= 0 || instance_count <= 0 || max_index < min_index ||
 514        !is_index_type_valid(type) ||
 515        (!non_vbo_attrib_mask && !has_user_indices)) {
 516       draw_elements_async(ctx, mode, count, type, indices, instance_count,
 517                           basevertex, baseinstance, index_bounds_valid,
 518                           min_index, max_index, 0, 0, NULL);
 519       return;
 520    }
 521
 522    if (!ctx->GLThread.SupportsNonVBOUploads)
 523       goto sync;
 524
 525    bool need_index_bounds = non_vbo_attrib_mask & ~vao->NonZeroDivisorMask;
 526    unsigned index_size = get_index_size(type);
 527
 528    if (need_index_bounds && !index_bounds_valid) {
 529       /* Sync if indices come from a buffer and vertices come from memory
 530        * and index bounds are not valid.
 531        *
 532        * We would have to map the indices to compute the index bounds, and
 533        * for that we would have to sync anyway.
 534        */
 535       if (!has_user_indices)
 536          goto sync;
 537
 538       /* Compute the index bounds. */
 539       min_index = ~0;
 540       max_index = 0;
 541       vbo_get_minmax_index_mapped(count, index_size,
 542                                   ctx->GLThread._RestartIndex[index_size - 1],
 543                                   ctx->GLThread._PrimitiveRestart, indices,
 544                                   &min_index, &max_index);
 545       index_bounds_valid = true;
 546    }
 547
 548    unsigned start_vertex = min_index + basevertex;
 549    unsigned num_vertices = max_index + 1 - min_index;
 550
 551    /* If there is too much data to upload, sync and let the driver unroll
 552     * indices. */
 553    if (util_is_vbo_upload_ratio_too_large(count, num_vertices))
 554       goto sync;
 555
 556    struct glthread_attrib_binding attribs[VERT_ATTRIB_MAX];
 557    if (non_vbo_attrib_mask &&
 558        !upload_vertices(ctx, non_vbo_attrib_mask, start_vertex, num_vertices,
 559                         baseinstance, instance_count, attribs))
 560       goto sync;
 561
 562    /* Upload indices. */
 563    struct gl_buffer_object *index_buffer = NULL;
 564    if (has_user_indices)
 565       index_buffer = upload_indices(ctx, count, index_size, &indices);
 566
 567    /* Draw asynchronously. */
 568    draw_elements_async(ctx, mode, count, type, indices, instance_count,
 569                        basevertex, baseinstance, index_bounds_valid,
 570                        min_index, max_index, index_buffer,
 571                        non_vbo_attrib_mask, attribs);
 572    return;
 573
 574 sync:
 575    _mesa_glthread_finish_before(ctx, "DrawElements");
 576
 577    if (index_bounds_valid && instance_count == 1 && baseinstance == 0) {
 578       CALL_DrawRangeElementsBaseVertex(ctx->CurrentServerDispatch,
 579                                        (mode, min_index, max_index, count,
 580                                         type, indices, basevertex));
 581    } else {
 582       CALL_DrawElementsInstancedBaseVertexBaseInstance(ctx->CurrentServerDispatch,
 583                                                        (mode, count, type, indices,
 584                                                         instance_count, basevertex,
 585                                                         baseinstance));
 586    }
 587 }
 588
 589 struct marshal_cmd_MultiDrawElementsBaseVertex
 590 {
 591    struct marshal_cmd_base cmd_base;
 592    bool has_base_vertex;
 593    GLenum mode;
 594    GLenum type;
 595    GLsizei draw_count;
 596    GLuint non_vbo_attrib_mask;
 597    struct gl_buffer_object *index_buffer;
 598 };
 599
 600 void
 601 _mesa_unmarshal_MultiDrawElementsBaseVertex(struct gl_context *ctx,
 602                                             const struct marshal_cmd_MultiDrawElementsBaseVertex *cmd)
 603 {
 604    const GLenum mode = cmd->mode;
 605    const GLenum type = cmd->type;
 606    const GLsizei draw_count = cmd->draw_count;
 607    const GLuint non_vbo_attrib_mask = cmd->non_vbo_attrib_mask;
 608    struct gl_buffer_object *index_buffer = cmd->index_buffer;
 609    const bool has_base_vertex = cmd->has_base_vertex;
 610
 611    const char *variable_data = (const char *)(cmd + 1);
 612    const GLsizei *count = (GLsizei *)variable_data;
 613    variable_data += sizeof(GLsizei) * draw_count;
 614    const GLvoid *const *indices = (const GLvoid *const *)variable_data;
 615    variable_data += sizeof(const GLvoid *const *) * draw_count;
 616    const GLsizei *basevertex = NULL;
 617    if (has_base_vertex) {
 618       basevertex = (GLsizei *)variable_data;
 619       variable_data += sizeof(GLsizei) * draw_count;
 620    }
 621    const struct glthread_attrib_binding *attribs =
 622       (const struct glthread_attrib_binding *)variable_data;
 623
 624    /* Bind uploaded buffers if needed. */
 625    if (non_vbo_attrib_mask) {
 626       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 627                                       false);
 628    }
 629    if (index_buffer) {
 630       _mesa_InternalBindElementBuffer(ctx, index_buffer);
 631    }
 632
 633    /* Draw. */
 634    if (has_base_vertex) {
 635       CALL_MultiDrawElementsBaseVertex(ctx->CurrentServerDispatch,
 636                                        (mode, count, type, indices, draw_count,
 637                                         basevertex));
 638    } else {
 639       CALL_MultiDrawElementsEXT(ctx->CurrentServerDispatch,
 640                                 (mode, count, type, indices, draw_count));
 641    }
 642
 643    /* Restore states. */
 644    if (index_buffer) {
 645       _mesa_InternalBindElementBuffer(ctx, NULL);
 646    }
 647    if (non_vbo_attrib_mask) {
 648       _mesa_InternalBindVertexBuffers(ctx, attribs, non_vbo_attrib_mask,
 649                                       true);
 650    }
 651 }
 652
 653 static ALWAYS_INLINE void
 654 multi_draw_elements_async(struct gl_context *ctx, GLenum mode,
 655                           const GLsizei *count, GLenum type,
 656                           const GLvoid *const *indices, GLsizei draw_count,
 657                           const GLsizei *basevertex,
 658                           struct gl_buffer_object *index_buffer,
 659                           unsigned non_vbo_attrib_mask,
 660                           const struct glthread_attrib_binding *attribs)
 661 {
 662    int count_size = sizeof(GLsizei) * draw_count;
 663    int indices_size = sizeof(indices[0]) * draw_count;
 664    int basevertex_size = basevertex ? sizeof(GLsizei) * draw_count : 0;
 665    int attribs_size = util_bitcount(non_vbo_attrib_mask) * sizeof(attribs[0]);
 666    int cmd_size = sizeof(struct marshal_cmd_MultiDrawElementsBaseVertex) +
 667                   count_size + indices_size + basevertex_size + attribs_size;
 668    struct marshal_cmd_MultiDrawElementsBaseVertex *cmd;
 669
 670    cmd = _mesa_glthread_allocate_command(ctx, DISPATCH_CMD_MultiDrawElementsBaseVertex, cmd_size);
 671    cmd->mode = mode;
 672    cmd->type = type;
 673    cmd->draw_count = draw_count;
 674    cmd->non_vbo_attrib_mask = non_vbo_attrib_mask;
 675    cmd->index_buffer = index_buffer;
 676    cmd->has_base_vertex = basevertex != NULL;
 677
 678    char *variable_data = (char*)(cmd + 1);
 679    memcpy(variable_data, count, count_size);
 680    variable_data += count_size;
 681    memcpy(variable_data, indices, indices_size);
 682    variable_data += indices_size;
 683
 684    if (basevertex) {
 685       memcpy(variable_data, basevertex, basevertex_size);
 686       variable_data += basevertex_size;
 687    }
 688
 689    if (non_vbo_attrib_mask)
 690       memcpy(variable_data, attribs, attribs_size);
 691 }
 692
 693 void GLAPIENTRY
 694 _mesa_marshal_MultiDrawElementsBaseVertex(GLenum mode, const GLsizei *count,
 695                                           GLenum type,
 696                                           const GLvoid *const *indices,
 697                                           GLsizei draw_count,
 698                                           const GLsizei *basevertex)
 699 {
 700    GET_CURRENT_CONTEXT(ctx);
 701
 702    struct glthread_vao *vao = ctx->GLThread.CurrentVAO;
 703    unsigned non_vbo_attrib_mask = vao->UserPointerMask & vao->Enabled;
 704    bool has_user_indices = vao->CurrentElementBufferName == 0;
 705
 706    /* Fast path when nothing needs to be done. */
 707    if (draw_count >= 0 &&
 708        (ctx->API == API_OPENGL_CORE ||
 709         !is_index_type_valid(type) ||
 710         (!non_vbo_attrib_mask && !has_user_indices))) {
 711       multi_draw_elements_async(ctx, mode, count, type, indices, draw_count,
 712                                 basevertex, 0, 0, NULL);
 713       return;
 714    }
 715
 716    bool need_index_bounds = non_vbo_attrib_mask & ~vao->NonZeroDivisorMask;
 717
 718    /* If the draw count is too high or negative, the queue can't be used.
 719     *
 720     * Sync if indices come from a buffer and vertices come from memory
 721     * and index bounds are not valid. We would have to map the indices
 722     * to compute the index bounds, and for that we would have to sync anyway.
 723     */
 724    if (!ctx->GLThread.SupportsNonVBOUploads ||
 725        draw_count < 0 || draw_count > MARSHAL_MAX_CMD_SIZE / 32 ||
 726        (need_index_bounds && !has_user_indices))
 727       goto sync;
 728
 729    unsigned index_size = get_index_size(type);
 730    unsigned min_index = ~0;
 731    unsigned max_index = 0;
 732    unsigned total_count = 0;
 733    unsigned num_vertices = 0;
 734
 735    /* This is always true if there is per-vertex data that needs to be
 736     * uploaded.
 737     */
 738    if (need_index_bounds) {
 739       /* Compute the index bounds. */
 740       for (unsigned i = 0; i < draw_count; i++) {
 741          GLsizei vertex_count = count[i];
 742
 743          if (vertex_count < 0) {
 744             /* Just call the driver to set the error. */
 745             multi_draw_elements_async(ctx, mode, count, type, indices, draw_count,
 746                                       basevertex, 0, 0, NULL);
 747             return;
 748          }
 749          if (vertex_count == 0)
 750             continue;
 751
 752          unsigned min = ~0, max = 0;
 753          vbo_get_minmax_index_mapped(vertex_count, index_size,
 754                                      ctx->GLThread._RestartIndex[index_size - 1],
 755                                      ctx->GLThread._PrimitiveRestart, indices[i],
 756                                      &min, &max);
 757          if (basevertex) {
 758             min += basevertex[i];
 759             max += basevertex[i];
 760          }
 761          min_index = MIN2(min_index, min);
 762          max_index = MAX2(max_index, max);
 763          total_count += vertex_count;
 764       }
 765
 766       num_vertices = max_index + 1 - min_index;
 767
 768       if (total_count == 0 || num_vertices == 0) {
 769          /* Nothing to do, but call the driver to set possible GL errors. */
 770          multi_draw_elements_async(ctx, mode, count, type, indices, draw_count,
 771                                    basevertex, 0, 0, NULL);
 772          return;
 773       }
 774
 775       /* If there is too much data to upload, sync and let the driver unroll
 776        * indices. */
 777       if (util_is_vbo_upload_ratio_too_large(total_count, num_vertices))
 778          goto sync;
 779    } else if (has_user_indices) {
 780       /* Only compute total_count for the upload of indices. */
 781       for (unsigned i = 0; i < draw_count; i++) {
 782          GLsizei vertex_count = count[i];
 783
 784          if (vertex_count < 0) {
 785             /* Just call the driver to set the error. */
 786             multi_draw_elements_async(ctx, mode, count, type, indices, draw_count,
 787                                       basevertex, 0, 0, NULL);
 788             return;
 789          }
 790          if (vertex_count == 0)
 791             continue;
 792
 793          total_count += vertex_count;
 794       }
 795
 796       if (total_count == 0) {
 797          /* Nothing to do, but call the driver to set possible GL errors. */
 798          multi_draw_elements_async(ctx, mode, count, type, indices, draw_count,
 799                                    basevertex, 0, 0, NULL);
 800          return;
 801       }
 802    }
 803
 804    /* Upload vertices. */
 805    struct glthread_attrib_binding attribs[VERT_ATTRIB_MAX];
 806    if (non_vbo_attrib_mask &&
 807        !upload_vertices(ctx, non_vbo_attrib_mask, min_index, num_vertices,
 808                         0, 1, attribs))
 809       goto sync;
 810
 811    /* Upload indices. */
 812    struct gl_buffer_object *index_buffer = NULL;
 813    if (has_user_indices) {
 814       const GLvoid **out_indices = alloca(sizeof(indices[0]) * draw_count);
 815
 816       index_buffer = upload_multi_indices(ctx, total_count, index_size,
 817                                           draw_count, count, indices,
 818                                           out_indices);
 819       indices = out_indices;
 820    }
 821
 822    /* Draw asynchronously. */
 823    multi_draw_elements_async(ctx, mode, count, type, indices, draw_count,
 824                              basevertex, index_buffer, non_vbo_attrib_mask,
 825                              attribs);
 826    return;
 827
 828 sync:
 829    _mesa_glthread_finish_before(ctx, "DrawElements");
 830
 831    if (basevertex) {
 832       CALL_MultiDrawElementsBaseVertex(ctx->CurrentServerDispatch,
 833                                        (mode, count, type, indices, draw_count,
 834                                         basevertex));
 835    } else {
 836       CALL_MultiDrawElementsEXT(ctx->CurrentServerDispatch,
 837                                 (mode, count, type, indices, draw_count));
 838    }
 839 }
 840
 841 void GLAPIENTRY
 842 _mesa_marshal_DrawArrays(GLenum mode, GLint first, GLsizei count)
 843 {
 844    _mesa_marshal_DrawArraysInstancedBaseInstance(mode, first, count, 1, 0);
 845 }
 846
 847 void GLAPIENTRY
 848 _mesa_marshal_DrawArraysInstancedARB(GLenum mode, GLint first, GLsizei count,
 849                                      GLsizei instance_count)
 850 {
 851    _mesa_marshal_DrawArraysInstancedBaseInstance(mode, first, count,
 852                                                  instance_count, 0);
 853 }
 854
 855 void GLAPIENTRY
 856 _mesa_marshal_DrawElements(GLenum mode, GLsizei count, GLenum type,
 857                            const GLvoid *indices)
 858 {
 859    draw_elements(mode, count, type, indices, 1, 0, 0, false, 0, 0);
 860 }
 861
 862 void GLAPIENTRY
 863 _mesa_marshal_DrawRangeElements(GLenum mode, GLuint start, GLuint end,
 864                                 GLsizei count, GLenum type,
 865                                 const GLvoid *indices)
 866 {
 867    draw_elements(mode, count, type, indices, 1, 0, 0, true, start, end);
 868 }
 869
 870 void GLAPIENTRY
 871 _mesa_marshal_DrawElementsInstancedARB(GLenum mode, GLsizei count, GLenum type,
 872                                        const GLvoid *indices, GLsizei instance_count)
 873 {
 874    draw_elements(mode, count, type, indices, instance_count, 0, 0, false, 0, 0);
 875 }
 876
 877 void GLAPIENTRY
 878 _mesa_marshal_DrawElementsBaseVertex(GLenum mode, GLsizei count, GLenum type,
 879                                      const GLvoid *indices, GLint basevertex)
 880 {
 881    draw_elements(mode, count, type, indices, 1, basevertex, 0, false, 0, 0);
 882 }
 883
 884 void GLAPIENTRY
 885 _mesa_marshal_DrawRangeElementsBaseVertex(GLenum mode, GLuint start, GLuint end,
 886                                           GLsizei count, GLenum type,
 887                                           const GLvoid *indices, GLint basevertex)
 888 {
 889    draw_elements(mode, count, type, indices, 1, basevertex, 0, true, start, end);
 890 }
 891
 892 void GLAPIENTRY
 893 _mesa_marshal_DrawElementsInstancedBaseVertex(GLenum mode, GLsizei count,
 894                                               GLenum type, const GLvoid *indices,
 895                                               GLsizei instance_count, GLint basevertex)
 896 {
 897    draw_elements(mode, count, type, indices, instance_count, basevertex, 0, false, 0, 0);
 898 }
 899
 900 void GLAPIENTRY
 901 _mesa_marshal_DrawElementsInstancedBaseInstance(GLenum mode, GLsizei count,
 902                                                 GLenum type, const GLvoid *indices,
 903                                                 GLsizei instance_count, GLuint baseinstance)
 904 {
 905    draw_elements(mode, count, type, indices, instance_count, 0, baseinstance, false, 0, 0);
 906 }
 907
 908 void GLAPIENTRY
 909 _mesa_marshal_DrawElementsInstancedBaseVertexBaseInstance(GLenum mode, GLsizei count,
 910                                                           GLenum type, const GLvoid *indices,
 911                                                           GLsizei instance_count, GLint basevertex,
 912                                                           GLuint baseinstance)
 913 {
 914    draw_elements(mode, count, type, indices, instance_count, basevertex, baseinstance, false, 0, 0);
 915 }
 916
 917 void GLAPIENTRY
 918 _mesa_marshal_MultiDrawElementsEXT(GLenum mode, const GLsizei *count,
 919                                    GLenum type, const GLvoid *const *indices,
 920                                    GLsizei draw_count)
 921 {
 922    _mesa_marshal_MultiDrawElementsBaseVertex(mode, count, type, indices,
 923                                              draw_count, NULL);
 924 }
 925
 926 void
 927 _mesa_unmarshal_DrawArrays(struct gl_context *ctx, const struct marshal_cmd_DrawArrays *cmd)
 928 {
 929    unreachable("never used - DrawArraysInstancedBaseInstance is used instead");
 930 }
 931
 932 void
 933 _mesa_unmarshal_DrawArraysInstancedARB(struct gl_context *ctx, const struct marshal_cmd_DrawArraysInstancedARB *cmd)
 934 {
 935    unreachable("never used - DrawArraysInstancedBaseInstance is used instead");
 936 }
 937
 938 void
 939 _mesa_unmarshal_DrawElements(struct gl_context *ctx, const struct marshal_cmd_DrawElements *cmd)
 940 {
 941    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 942 }
 943
 944 void
 945 _mesa_unmarshal_DrawRangeElements(struct gl_context *ctx, const struct marshal_cmd_DrawRangeElements *cmd)
 946 {
 947    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 948 }
 949
 950 void
 951 _mesa_unmarshal_DrawElementsInstancedARB(struct gl_context *ctx, const struct marshal_cmd_DrawElementsInstancedARB *cmd)
 952 {
 953    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 954 }
 955
 956 void
 957 _mesa_unmarshal_DrawElementsBaseVertex(struct gl_context *ctx, const struct marshal_cmd_DrawElementsBaseVertex *cmd)
 958 {
 959    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 960 }
 961
 962 void
 963 _mesa_unmarshal_DrawRangeElementsBaseVertex(struct gl_context *ctx, const struct marshal_cmd_DrawRangeElementsBaseVertex *cmd)
 964 {
 965    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 966 }
 967
 968 void
 969 _mesa_unmarshal_DrawElementsInstancedBaseVertex(struct gl_context *ctx, const struct marshal_cmd_DrawElementsInstancedBaseVertex *cmd)
 970 {
 971    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 972 }
 973
 974 void
 975 _mesa_unmarshal_DrawElementsInstancedBaseInstance(struct gl_context *ctx, const struct marshal_cmd_DrawElementsInstancedBaseInstance *cmd)
 976 {
 977    unreachable("never used - DrawElementsInstancedBaseVertexBaseInstance is used instead");
 978 }
 979
 980 void
 981 _mesa_unmarshal_MultiDrawElementsEXT(struct gl_context *ctx, const struct marshal_cmd_MultiDrawElementsEXT *cmd)
 982 {
 983    unreachable("never used - MultiDrawElementsBaseVertex is used instead");
 984 }