src/mesa/drivers/dri/intel/intel_buffer_objects.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 #include "main/imports.h"
  30 #include "main/mfeatures.h"
  31 #include "main/mtypes.h"
  32 #include "main/macros.h"
  33 #include "main/bufferobj.h"
  34
  35 #include "intel_blit.h"
  36 #include "intel_buffer_objects.h"
  37 #include "intel_batchbuffer.h"
  38 #include "intel_context.h"
  39 #include "intel_fbo.h"
  40 #include "intel_mipmap_tree.h"
  41 #include "intel_regions.h"
  42
  43 static GLboolean
  44 intel_bufferobj_unmap(struct gl_context * ctx, struct gl_buffer_object *obj);
  45
  46 /** Allocates a new drm_intel_bo to store the data for the buffer object. */
  47 static void
  48 intel_bufferobj_alloc_buffer(struct intel_context *intel,
  49                              struct intel_buffer_object *intel_obj)
  50 {
  51    intel_obj->buffer = drm_intel_bo_alloc(intel->bufmgr, "bufferobj",
  52                                           intel_obj->Base.Size, 64);
  53 }
  54
  55 static void
  56 release_buffer(struct intel_buffer_object *intel_obj)
  57 {
  58    drm_intel_bo_unreference(intel_obj->buffer);
  59    intel_obj->buffer = NULL;
  60    intel_obj->offset = 0;
  61    intel_obj->source = 0;
  62 }
  63
  64 /**
  65  * There is some duplication between mesa's bufferobjects and our
  66  * bufmgr buffers.  Both have an integer handle and a hashtable to
  67  * lookup an opaque structure.  It would be nice if the handles and
  68  * internal structure where somehow shared.
  69  */
  70 static struct gl_buffer_object *
  71 intel_bufferobj_alloc(struct gl_context * ctx, GLuint name, GLenum target)
  72 {
  73    struct intel_buffer_object *obj = CALLOC_STRUCT(intel_buffer_object);
  74
  75    _mesa_initialize_buffer_object(&obj->Base, name, target);
  76
  77    obj->buffer = NULL;
  78
  79    return &obj->Base;
  80 }
  81
  82 /* Break the COW tie to the region.  The region gets to keep the data.
  83  */
  84 void
  85 intel_bufferobj_release_region(struct intel_buffer_object *intel_obj)
  86 {
  87    assert(intel_obj->region->buffer == intel_obj->buffer);
  88    intel_obj->region->pbo = NULL;
  89    intel_obj->region = NULL;
  90
  91    release_buffer(intel_obj);
  92 }
  93
  94 /* Break the COW tie to the region.  Both the pbo and the region end
  95  * up with a copy of the data.
  96  */
  97 void
  98 intel_bufferobj_cow(struct intel_context *intel,
  99                     struct intel_buffer_object *intel_obj)
 100 {
 101    assert(intel_obj->region);
 102    intel_region_cow(intel, intel_obj->region);
 103 }
 104
 105
 106 /**
 107  * Deallocate/free a vertex/pixel buffer object.
 108  * Called via glDeleteBuffersARB().
 109  */
 110 static void
 111 intel_bufferobj_free(struct gl_context * ctx, struct gl_buffer_object *obj)
 112 {
 113    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 114
 115    assert(intel_obj);
 116
 117    /* Buffer objects are automatically unmapped when deleting according
 118     * to the spec, but Mesa doesn't do UnmapBuffer for us at context destroy
 119     * (though it does if you call glDeleteBuffers)
 120     */
 121    if (obj->Pointer)
 122       intel_bufferobj_unmap(ctx, obj);
 123
 124    free(intel_obj->sys_buffer);
 125    if (intel_obj->region) {
 126       intel_bufferobj_release_region(intel_obj);
 127    }
 128
 129    drm_intel_bo_unreference(intel_obj->buffer);
 130    free(intel_obj);
 131 }
 132
 133
 134
 135 /**
 136  * Allocate space for and store data in a buffer object.  Any data that was
 137  * previously stored in the buffer object is lost.  If data is NULL,
 138  * memory will be allocated, but no copy will occur.
 139  * Called via ctx->Driver.BufferData().
 140  * \return GL_TRUE for success, GL_FALSE if out of memory
 141  */
 142 static GLboolean
 143 intel_bufferobj_data(struct gl_context * ctx,
 144                      GLenum target,
 145                      GLsizeiptrARB size,
 146                      const GLvoid * data,
 147                      GLenum usage, struct gl_buffer_object *obj)
 148 {
 149    struct intel_context *intel = intel_context(ctx);
 150    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 151
 152    /* Part of the ABI, but this function doesn't use it.
 153     */
 154 #ifndef I915
 155    (void) target;
 156 #endif
 157
 158    intel_obj->Base.Size = size;
 159    intel_obj->Base.Usage = usage;
 160
 161    assert(!obj->Pointer); /* Mesa should have unmapped it */
 162
 163    if (intel_obj->region)
 164       intel_bufferobj_release_region(intel_obj);
 165
 166    if (intel_obj->buffer != NULL)
 167       release_buffer(intel_obj);
 168
 169    free(intel_obj->sys_buffer);
 170    intel_obj->sys_buffer = NULL;
 171
 172    if (size != 0) {
 173       if (usage == GL_DYNAMIC_DRAW
 174 #ifdef I915
 175           /* On pre-965, stick VBOs in system memory, as we're always doing
 176            * swtnl with their contents anyway.
 177            */
 178           || target == GL_ARRAY_BUFFER || target == GL_ELEMENT_ARRAY_BUFFER
 179 #endif
 180          )
 181       {
 182          intel_obj->sys_buffer = malloc(size);
 183          if (intel_obj->sys_buffer != NULL) {
 184             if (data != NULL)
 185                memcpy(intel_obj->sys_buffer, data, size);
 186             return GL_TRUE;
 187          }
 188       }
 189       intel_bufferobj_alloc_buffer(intel, intel_obj);
 190       if (!intel_obj->buffer)
 191          return GL_FALSE;
 192
 193       if (data != NULL)
 194          drm_intel_bo_subdata(intel_obj->buffer, 0, size, data);
 195    }
 196
 197    return GL_TRUE;
 198 }
 199
 200
 201 /**
 202  * Replace data in a subrange of buffer object.  If the data range
 203  * specified by size + offset extends beyond the end of the buffer or
 204  * if data is NULL, no copy is performed.
 205  * Called via glBufferSubDataARB().
 206  */
 207 static void
 208 intel_bufferobj_subdata(struct gl_context * ctx,
 209                         GLintptrARB offset,
 210                         GLsizeiptrARB size,
 211                         const GLvoid * data, struct gl_buffer_object *obj)
 212 {
 213    struct intel_context *intel = intel_context(ctx);
 214    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 215    bool busy;
 216
 217    if (size == 0)
 218       return;
 219
 220    assert(intel_obj);
 221
 222    if (intel_obj->region)
 223       intel_bufferobj_cow(intel, intel_obj);
 224
 225    /* If we have a single copy in system memory, update that */
 226    if (intel_obj->sys_buffer) {
 227       if (intel_obj->source)
 228          release_buffer(intel_obj);
 229
 230       if (intel_obj->buffer == NULL) {
 231          memcpy((char *)intel_obj->sys_buffer + offset, data, size);
 232          return;
 233       }
 234
 235       free(intel_obj->sys_buffer);
 236       intel_obj->sys_buffer = NULL;
 237    }
 238
 239    /* Otherwise we need to update the copy in video memory. */
 240    busy =
 241       drm_intel_bo_busy(intel_obj->buffer) ||
 242       drm_intel_bo_references(intel->batch.bo, intel_obj->buffer);
 243
 244    /* replace the current busy bo with fresh data */
 245    if (busy && size == intel_obj->Base.Size) {
 246       drm_intel_bo_unreference(intel_obj->buffer);
 247       intel_bufferobj_alloc_buffer(intel, intel_obj);
 248       drm_intel_bo_subdata(intel_obj->buffer, 0, size, data);
 249    } else if (intel->gen < 6) {
 250       if (busy) {
 251          drm_intel_bo *temp_bo;
 252
 253          temp_bo = drm_intel_bo_alloc(intel->bufmgr, "subdata temp", size, 64);
 254
 255          drm_intel_bo_subdata(temp_bo, 0, size, data);
 256
 257          intel_emit_linear_blit(intel,
 258                                 intel_obj->buffer, offset,
 259                                 temp_bo, 0,
 260                                 size);
 261
 262          drm_intel_bo_unreference(temp_bo);
 263       } else {
 264          drm_intel_bo_subdata(intel_obj->buffer, offset, size, data);
 265       }
 266    } else {
 267       /* Can't use the blit to modify the buffer in the middle of batch. */
 268       if (drm_intel_bo_references(intel->batch.bo, intel_obj->buffer)) {
 269          intel_batchbuffer_flush(intel);
 270       }
 271       drm_intel_bo_subdata(intel_obj->buffer, offset, size, data);
 272    }
 273 }
 274
 275
 276 /**
 277  * Called via glGetBufferSubDataARB().
 278  */
 279 static void
 280 intel_bufferobj_get_subdata(struct gl_context * ctx,
 281                             GLintptrARB offset,
 282                             GLsizeiptrARB size,
 283                             GLvoid * data, struct gl_buffer_object *obj)
 284 {
 285    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 286    struct intel_context *intel = intel_context(ctx);
 287
 288    assert(intel_obj);
 289    if (intel_obj->sys_buffer)
 290       memcpy(data, (char *)intel_obj->sys_buffer + offset, size);
 291    else {
 292       if (drm_intel_bo_references(intel->batch.bo, intel_obj->buffer)) {
 293          intel_batchbuffer_flush(intel);
 294       }
 295       drm_intel_bo_get_subdata(intel_obj->buffer, offset, size, data);
 296    }
 297 }
 298
 299
 300
 301 /**
 302  * Called via glMapBufferRange and glMapBuffer
 303  *
 304  * The goal of this extension is to allow apps to accumulate their rendering
 305  * at the same time as they accumulate their buffer object.  Without it,
 306  * you'd end up blocking on execution of rendering every time you mapped
 307  * the buffer to put new data in.
 308  *
 309  * We support it in 3 ways: If unsynchronized, then don't bother
 310  * flushing the batchbuffer before mapping the buffer, which can save blocking
 311  * in many cases.  If we would still block, and they allow the whole buffer
 312  * to be invalidated, then just allocate a new buffer to replace the old one.
 313  * If not, and we'd block, and they allow the subrange of the buffer to be
 314  * invalidated, then we can make a new little BO, let them write into that,
 315  * and blit it into the real BO at unmap time.
 316  */
 317 static void *
 318 intel_bufferobj_map_range(struct gl_context * ctx,
 319                           GLintptr offset, GLsizeiptr length,
 320                           GLbitfield access, struct gl_buffer_object *obj)
 321 {
 322    struct intel_context *intel = intel_context(ctx);
 323    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 324
 325    assert(intel_obj);
 326
 327    /* _mesa_MapBufferRange (GL entrypoint) sets these, but the vbo module also
 328     * internally uses our functions directly.
 329     */
 330    obj->Offset = offset;
 331    obj->Length = length;
 332    obj->AccessFlags = access;
 333
 334    if (intel_obj->sys_buffer) {
 335       const bool read_only =
 336          (access & (GL_MAP_READ_BIT | GL_MAP_WRITE_BIT)) == GL_MAP_READ_BIT;
 337
 338       if (!read_only && intel_obj->source)
 339          release_buffer(intel_obj);
 340
 341       if (!intel_obj->buffer || intel_obj->source) {
 342          obj->Pointer = intel_obj->sys_buffer + offset;
 343          return obj->Pointer;
 344       }
 345
 346       free(intel_obj->sys_buffer);
 347       intel_obj->sys_buffer = NULL;
 348    }
 349
 350    if (intel_obj->region)
 351       intel_bufferobj_cow(intel, intel_obj);
 352
 353    /* If the mapping is synchronized with other GL operations, flush
 354     * the batchbuffer so that GEM knows about the buffer access for later
 355     * syncing.
 356     */
 357    if (!(access & GL_MAP_UNSYNCHRONIZED_BIT) &&
 358        drm_intel_bo_references(intel->batch.bo, intel_obj->buffer))
 359       intel_flush(ctx);
 360
 361    if (intel_obj->buffer == NULL) {
 362       obj->Pointer = NULL;
 363       return NULL;
 364    }
 365
 366    /* If the user doesn't care about existing buffer contents and mapping
 367     * would cause us to block, then throw out the old buffer.
 368     */
 369    if (!(access & GL_MAP_UNSYNCHRONIZED_BIT) &&
 370        (access & GL_MAP_INVALIDATE_BUFFER_BIT) &&
 371        drm_intel_bo_busy(intel_obj->buffer)) {
 372       drm_intel_bo_unreference(intel_obj->buffer);
 373       intel_bufferobj_alloc_buffer(intel, intel_obj);
 374    }
 375
 376    /* If the user is mapping a range of an active buffer object but
 377     * doesn't require the current contents of that range, make a new
 378     * BO, and we'll copy what they put in there out at unmap or
 379     * FlushRange time.
 380     */
 381    if ((access & GL_MAP_INVALIDATE_RANGE_BIT) &&
 382        drm_intel_bo_busy(intel_obj->buffer)) {
 383       if (access & GL_MAP_FLUSH_EXPLICIT_BIT) {
 384          intel_obj->range_map_buffer = malloc(length);
 385          obj->Pointer = intel_obj->range_map_buffer;
 386       } else {
 387          intel_obj->range_map_bo = drm_intel_bo_alloc(intel->bufmgr,
 388                                                       "range map",
 389                                                       length, 64);
 390          if (!(access & GL_MAP_READ_BIT)) {
 391             drm_intel_gem_bo_map_gtt(intel_obj->range_map_bo);
 392             intel_obj->mapped_gtt = GL_TRUE;
 393          } else {
 394             drm_intel_bo_map(intel_obj->range_map_bo,
 395                              (access & GL_MAP_WRITE_BIT) != 0);
 396             intel_obj->mapped_gtt = GL_FALSE;
 397          }
 398          obj->Pointer = intel_obj->range_map_bo->virtual;
 399       }
 400       return obj->Pointer;
 401    }
 402
 403    if (!(access & GL_MAP_READ_BIT)) {
 404       drm_intel_gem_bo_map_gtt(intel_obj->buffer);
 405       intel_obj->mapped_gtt = GL_TRUE;
 406    } else {
 407       drm_intel_bo_map(intel_obj->buffer, (access & GL_MAP_WRITE_BIT) != 0);
 408       intel_obj->mapped_gtt = GL_FALSE;
 409    }
 410
 411    obj->Pointer = intel_obj->buffer->virtual + offset;
 412    return obj->Pointer;
 413 }
 414
 415 /* Ideally we'd use a BO to avoid taking up cache space for the temporary
 416  * data, but FlushMappedBufferRange may be followed by further writes to
 417  * the pointer, so we would have to re-map after emitting our blit, which
 418  * would defeat the point.
 419  */
 420 static void
 421 intel_bufferobj_flush_mapped_range(struct gl_context *ctx,
 422                                    GLintptr offset, GLsizeiptr length,
 423                                    struct gl_buffer_object *obj)
 424 {
 425    struct intel_context *intel = intel_context(ctx);
 426    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 427    drm_intel_bo *temp_bo;
 428
 429    /* Unless we're in the range map using a temporary system buffer,
 430     * there's no work to do.
 431     */
 432    if (intel_obj->range_map_buffer == NULL)
 433       return;
 434
 435    if (length == 0)
 436       return;
 437
 438    temp_bo = drm_intel_bo_alloc(intel->bufmgr, "range map flush", length, 64);
 439
 440    drm_intel_bo_subdata(temp_bo, 0, length, intel_obj->range_map_buffer);
 441
 442    intel_emit_linear_blit(intel,
 443                           intel_obj->buffer, obj->Offset + offset,
 444                           temp_bo, 0,
 445                           length);
 446
 447    drm_intel_bo_unreference(temp_bo);
 448 }
 449
 450
 451 /**
 452  * Called via glUnmapBuffer().
 453  */
 454 static GLboolean
 455 intel_bufferobj_unmap(struct gl_context * ctx, struct gl_buffer_object *obj)
 456 {
 457    struct intel_context *intel = intel_context(ctx);
 458    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 459
 460    assert(intel_obj);
 461    assert(obj->Pointer);
 462    if (intel_obj->sys_buffer != NULL) {
 463       /* always keep the mapping around. */
 464    } else if (intel_obj->range_map_buffer != NULL) {
 465       /* Since we've emitted some blits to buffers that will (likely) be used
 466        * in rendering operations in other cache domains in this batch, emit a
 467        * flush.  Once again, we wish for a domain tracker in libdrm to cover
 468        * usage inside of a batchbuffer.
 469        */
 470       intel_batchbuffer_emit_mi_flush(intel);
 471       free(intel_obj->range_map_buffer);
 472       intel_obj->range_map_buffer = NULL;
 473    } else if (intel_obj->range_map_bo != NULL) {
 474       if (intel_obj->mapped_gtt) {
 475          drm_intel_gem_bo_unmap_gtt(intel_obj->range_map_bo);
 476       } else {
 477          drm_intel_bo_unmap(intel_obj->range_map_bo);
 478       }
 479
 480       intel_emit_linear_blit(intel,
 481                              intel_obj->buffer, obj->Offset,
 482                              intel_obj->range_map_bo, 0,
 483                              obj->Length);
 484
 485       /* Since we've emitted some blits to buffers that will (likely) be used
 486        * in rendering operations in other cache domains in this batch, emit a
 487        * flush.  Once again, we wish for a domain tracker in libdrm to cover
 488        * usage inside of a batchbuffer.
 489        */
 490       intel_batchbuffer_emit_mi_flush(intel);
 491
 492       drm_intel_bo_unreference(intel_obj->range_map_bo);
 493       intel_obj->range_map_bo = NULL;
 494    } else if (intel_obj->buffer != NULL) {
 495       if (intel_obj->mapped_gtt) {
 496          drm_intel_gem_bo_unmap_gtt(intel_obj->buffer);
 497       } else {
 498          drm_intel_bo_unmap(intel_obj->buffer);
 499       }
 500    }
 501    obj->Pointer = NULL;
 502    obj->Offset = 0;
 503    obj->Length = 0;
 504
 505    return GL_TRUE;
 506 }
 507
 508 drm_intel_bo *
 509 intel_bufferobj_buffer(struct intel_context *intel,
 510                        struct intel_buffer_object *intel_obj,
 511                        GLuint flag)
 512 {
 513    if (intel_obj->region) {
 514       if (flag == INTEL_WRITE_PART)
 515          intel_bufferobj_cow(intel, intel_obj);
 516       else if (flag == INTEL_WRITE_FULL) {
 517          intel_bufferobj_release_region(intel_obj);
 518          intel_bufferobj_alloc_buffer(intel, intel_obj);
 519       }
 520    }
 521
 522    if (intel_obj->source)
 523       release_buffer(intel_obj);
 524
 525    if (intel_obj->buffer == NULL) {
 526       intel_bufferobj_alloc_buffer(intel, intel_obj);
 527       drm_intel_bo_subdata(intel_obj->buffer,
 528                            0, intel_obj->Base.Size,
 529                            intel_obj->sys_buffer);
 530
 531       free(intel_obj->sys_buffer);
 532       intel_obj->sys_buffer = NULL;
 533       intel_obj->offset = 0;
 534    }
 535
 536    return intel_obj->buffer;
 537 }
 538
 539 #define INTEL_UPLOAD_SIZE (64*1024)
 540
 541 void
 542 intel_upload_finish(struct intel_context *intel)
 543 {
 544    if (!intel->upload.bo)
 545            return;
 546
 547    if (intel->upload.buffer_len) {
 548            drm_intel_bo_subdata(intel->upload.bo,
 549                                 intel->upload.buffer_offset,
 550                                 intel->upload.buffer_len,
 551                                 intel->upload.buffer);
 552            intel->upload.buffer_len = 0;
 553    }
 554
 555    drm_intel_bo_unreference(intel->upload.bo);
 556    intel->upload.bo = NULL;
 557 }
 558
 559 static void wrap_buffers(struct intel_context *intel, GLuint size)
 560 {
 561    intel_upload_finish(intel);
 562
 563    if (size < INTEL_UPLOAD_SIZE)
 564       size = INTEL_UPLOAD_SIZE;
 565
 566    intel->upload.bo = drm_intel_bo_alloc(intel->bufmgr, "upload", size, 0);
 567    intel->upload.offset = 0;
 568 }
 569
 570 void intel_upload_data(struct intel_context *intel,
 571                        const void *ptr, GLuint size, GLuint align,
 572                        drm_intel_bo **return_bo,
 573                        GLuint *return_offset)
 574 {
 575    GLuint base, delta;
 576
 577    base = (intel->upload.offset + align - 1) / align * align;
 578    if (intel->upload.bo == NULL || base + size > intel->upload.bo->size) {
 579       wrap_buffers(intel, size);
 580       base = 0;
 581    }
 582
 583    drm_intel_bo_reference(intel->upload.bo);
 584    *return_bo = intel->upload.bo;
 585    *return_offset = base;
 586
 587    delta = base - intel->upload.offset;
 588    if (intel->upload.buffer_len &&
 589        intel->upload.buffer_len + delta + size > sizeof(intel->upload.buffer))
 590    {
 591       drm_intel_bo_subdata(intel->upload.bo,
 592                            intel->upload.buffer_offset,
 593                            intel->upload.buffer_len,
 594                            intel->upload.buffer);
 595       intel->upload.buffer_len = 0;
 596    }
 597
 598    if (size < sizeof(intel->upload.buffer))
 599    {
 600       if (intel->upload.buffer_len == 0)
 601          intel->upload.buffer_offset = base;
 602       else
 603          intel->upload.buffer_len += delta;
 604
 605       memcpy(intel->upload.buffer + intel->upload.buffer_len, ptr, size);
 606       intel->upload.buffer_len += size;
 607    }
 608    else
 609    {
 610       drm_intel_bo_subdata(intel->upload.bo, base, size, ptr);
 611    }
 612
 613    intel->upload.offset = base + size;
 614 }
 615
 616 void *intel_upload_map(struct intel_context *intel, GLuint size, GLuint align)
 617 {
 618    GLuint base, delta;
 619    char *ptr;
 620
 621    base = (intel->upload.offset + align - 1) / align * align;
 622    if (intel->upload.bo == NULL || base + size > intel->upload.bo->size) {
 623       wrap_buffers(intel, size);
 624       base = 0;
 625    }
 626
 627    delta = base - intel->upload.offset;
 628    if (intel->upload.buffer_len &&
 629        intel->upload.buffer_len + delta + size > sizeof(intel->upload.buffer))
 630    {
 631       drm_intel_bo_subdata(intel->upload.bo,
 632                            intel->upload.buffer_offset,
 633                            intel->upload.buffer_len,
 634                            intel->upload.buffer);
 635       intel->upload.buffer_len = 0;
 636    }
 637
 638    if (size <= sizeof(intel->upload.buffer)) {
 639       if (intel->upload.buffer_len == 0)
 640          intel->upload.buffer_offset = base;
 641       else
 642          intel->upload.buffer_len += delta;
 643
 644       ptr = intel->upload.buffer + intel->upload.buffer_len;
 645       intel->upload.buffer_len += size;
 646    } else
 647       ptr = malloc(size);
 648
 649    return ptr;
 650 }
 651
 652 void intel_upload_unmap(struct intel_context *intel,
 653                         const void *ptr, GLuint size, GLuint align,
 654                         drm_intel_bo **return_bo,
 655                         GLuint *return_offset)
 656 {
 657    GLuint base;
 658
 659    base = (intel->upload.offset + align - 1) / align * align;
 660    if (size > sizeof(intel->upload.buffer)) {
 661       drm_intel_bo_subdata(intel->upload.bo, base, size, ptr);
 662       free((void*)ptr);
 663    }
 664
 665    drm_intel_bo_reference(intel->upload.bo);
 666    *return_bo = intel->upload.bo;
 667    *return_offset = base;
 668
 669    intel->upload.offset = base + size;
 670 }
 671
 672 drm_intel_bo *
 673 intel_bufferobj_source(struct intel_context *intel,
 674                        struct intel_buffer_object *intel_obj,
 675                        GLuint align, GLuint *offset)
 676 {
 677    if (intel_obj->buffer == NULL) {
 678       intel_upload_data(intel,
 679                         intel_obj->sys_buffer, intel_obj->Base.Size, align,
 680                         &intel_obj->buffer, &intel_obj->offset);
 681       intel_obj->source = 1;
 682    }
 683
 684    *offset = intel_obj->offset;
 685    return intel_obj->buffer;
 686 }
 687
 688 static void
 689 intel_bufferobj_copy_subdata(struct gl_context *ctx,
 690                              struct gl_buffer_object *src,
 691                              struct gl_buffer_object *dst,
 692                              GLintptr read_offset, GLintptr write_offset,
 693                              GLsizeiptr size)
 694 {
 695    struct intel_context *intel = intel_context(ctx);
 696    struct intel_buffer_object *intel_src = intel_buffer_object(src);
 697    struct intel_buffer_object *intel_dst = intel_buffer_object(dst);
 698    drm_intel_bo *src_bo, *dst_bo;
 699    GLuint src_offset;
 700
 701    if (size == 0)
 702       return;
 703
 704    /* If we're in system memory, just map and memcpy. */
 705    if (intel_src->sys_buffer || intel_dst->sys_buffer || intel->gen >= 6) {
 706       /* The same buffer may be used, but note that regions copied may
 707        * not overlap.
 708        */
 709       if (src == dst) {
 710          char *ptr = intel_bufferobj_map_range(ctx, 0, dst->Size,
 711                                                GL_MAP_READ_BIT, dst);
 712          memmove(ptr + write_offset, ptr + read_offset, size);
 713          intel_bufferobj_unmap(ctx, dst);
 714       } else {
 715          const char *src_ptr;
 716          char *dst_ptr;
 717
 718          src_ptr =  intel_bufferobj_map_range(ctx, 0, src->Size,
 719                                               GL_MAP_READ_BIT, src);
 720          dst_ptr =  intel_bufferobj_map_range(ctx, 0, dst->Size,
 721                                               GL_MAP_WRITE_BIT, dst);
 722
 723          memcpy(dst_ptr + write_offset, src_ptr + read_offset, size);
 724
 725          intel_bufferobj_unmap(ctx, src);
 726          intel_bufferobj_unmap(ctx, dst);
 727       }
 728       return;
 729    }
 730
 731    /* Otherwise, we have real BOs, so blit them. */
 732
 733    dst_bo = intel_bufferobj_buffer(intel, intel_dst, INTEL_WRITE_PART);
 734    src_bo = intel_bufferobj_source(intel, intel_src, 64, &src_offset);
 735
 736    intel_emit_linear_blit(intel,
 737                           dst_bo, write_offset,
 738                           src_bo, read_offset + src_offset, size);
 739
 740    /* Since we've emitted some blits to buffers that will (likely) be used
 741     * in rendering operations in other cache domains in this batch, emit a
 742     * flush.  Once again, we wish for a domain tracker in libdrm to cover
 743     * usage inside of a batchbuffer.
 744     */
 745    intel_batchbuffer_emit_mi_flush(intel);
 746 }
 747
 748 #if FEATURE_APPLE_object_purgeable
 749 static GLenum
 750 intel_buffer_purgeable(drm_intel_bo *buffer)
 751 {
 752    int retained = 0;
 753
 754    if (buffer != NULL)
 755       retained = drm_intel_bo_madvise (buffer, I915_MADV_DONTNEED);
 756
 757    return retained ? GL_VOLATILE_APPLE : GL_RELEASED_APPLE;
 758 }
 759
 760 static GLenum
 761 intel_buffer_object_purgeable(struct gl_context * ctx,
 762                               struct gl_buffer_object *obj,
 763                               GLenum option)
 764 {
 765    struct intel_buffer_object *intel_obj = intel_buffer_object (obj);
 766
 767    if (intel_obj->buffer != NULL)
 768       return intel_buffer_purgeable(intel_obj->buffer);
 769
 770    if (option == GL_RELEASED_APPLE) {
 771       if (intel_obj->sys_buffer != NULL) {
 772          free(intel_obj->sys_buffer);
 773          intel_obj->sys_buffer = NULL;
 774       }
 775
 776       return GL_RELEASED_APPLE;
 777    } else {
 778       /* XXX Create the buffer and madvise(MADV_DONTNEED)? */
 779       struct intel_context *intel = intel_context(ctx);
 780       drm_intel_bo *bo = intel_bufferobj_buffer(intel, intel_obj, INTEL_READ);
 781
 782       return intel_buffer_purgeable(bo);
 783    }
 784 }
 785
 786 static GLenum
 787 intel_texture_object_purgeable(struct gl_context * ctx,
 788                                struct gl_texture_object *obj,
 789                                GLenum option)
 790 {
 791    struct intel_texture_object *intel;
 792
 793    (void) ctx;
 794    (void) option;
 795
 796    intel = intel_texture_object(obj);
 797    if (intel->mt == NULL || intel->mt->region == NULL)
 798       return GL_RELEASED_APPLE;
 799
 800    return intel_buffer_purgeable(intel->mt->region->buffer);
 801 }
 802
 803 static GLenum
 804 intel_render_object_purgeable(struct gl_context * ctx,
 805                               struct gl_renderbuffer *obj,
 806                               GLenum option)
 807 {
 808    struct intel_renderbuffer *intel;
 809
 810    (void) ctx;
 811    (void) option;
 812
 813    intel = intel_renderbuffer(obj);
 814    if (intel->region == NULL)
 815       return GL_RELEASED_APPLE;
 816
 817    return intel_buffer_purgeable(intel->region->buffer);
 818 }
 819
 820 static GLenum
 821 intel_buffer_unpurgeable(drm_intel_bo *buffer)
 822 {
 823    int retained;
 824
 825    retained = 0;
 826    if (buffer != NULL)
 827       retained = drm_intel_bo_madvise (buffer, I915_MADV_WILLNEED);
 828
 829    return retained ? GL_RETAINED_APPLE : GL_UNDEFINED_APPLE;
 830 }
 831
 832 static GLenum
 833 intel_buffer_object_unpurgeable(struct gl_context * ctx,
 834                                 struct gl_buffer_object *obj,
 835                                 GLenum option)
 836 {
 837    (void) ctx;
 838    (void) option;
 839
 840    return intel_buffer_unpurgeable(intel_buffer_object (obj)->buffer);
 841 }
 842
 843 static GLenum
 844 intel_texture_object_unpurgeable(struct gl_context * ctx,
 845                                  struct gl_texture_object *obj,
 846                                  GLenum option)
 847 {
 848    struct intel_texture_object *intel;
 849
 850    (void) ctx;
 851    (void) option;
 852
 853    intel = intel_texture_object(obj);
 854    if (intel->mt == NULL || intel->mt->region == NULL)
 855       return GL_UNDEFINED_APPLE;
 856
 857    return intel_buffer_unpurgeable(intel->mt->region->buffer);
 858 }
 859
 860 static GLenum
 861 intel_render_object_unpurgeable(struct gl_context * ctx,
 862                                 struct gl_renderbuffer *obj,
 863                                 GLenum option)
 864 {
 865    struct intel_renderbuffer *intel;
 866
 867    (void) ctx;
 868    (void) option;
 869
 870    intel = intel_renderbuffer(obj);
 871    if (intel->region == NULL)
 872       return GL_UNDEFINED_APPLE;
 873
 874    return intel_buffer_unpurgeable(intel->region->buffer);
 875 }
 876 #endif
 877
 878 void
 879 intelInitBufferObjectFuncs(struct dd_function_table *functions)
 880 {
 881    functions->NewBufferObject = intel_bufferobj_alloc;
 882    functions->DeleteBuffer = intel_bufferobj_free;
 883    functions->BufferData = intel_bufferobj_data;
 884    functions->BufferSubData = intel_bufferobj_subdata;
 885    functions->GetBufferSubData = intel_bufferobj_get_subdata;
 886    functions->MapBufferRange = intel_bufferobj_map_range;
 887    functions->FlushMappedBufferRange = intel_bufferobj_flush_mapped_range;
 888    functions->UnmapBuffer = intel_bufferobj_unmap;
 889    functions->CopyBufferSubData = intel_bufferobj_copy_subdata;
 890
 891 #if FEATURE_APPLE_object_purgeable
 892    functions->BufferObjectPurgeable = intel_buffer_object_purgeable;
 893    functions->TextureObjectPurgeable = intel_texture_object_purgeable;
 894    functions->RenderObjectPurgeable = intel_render_object_purgeable;
 895
 896    functions->BufferObjectUnpurgeable = intel_buffer_object_unpurgeable;
 897    functions->TextureObjectUnpurgeable = intel_texture_object_unpurgeable;
 898    functions->RenderObjectUnpurgeable = intel_render_object_unpurgeable;
 899 #endif
 900 }