src/mesa/drivers/dri/intel/intel_buffer_objects.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28
  29 #include "main/imports.h"
  30 #include "main/mfeatures.h"
  31 #include "main/mtypes.h"
  32 #include "main/macros.h"
  33 #include "main/bufferobj.h"
  34
  35 #include "intel_blit.h"
  36 #include "intel_buffer_objects.h"
  37 #include "intel_batchbuffer.h"
  38 #include "intel_context.h"
  39 #include "intel_fbo.h"
  40 #include "intel_mipmap_tree.h"
  41 #include "intel_regions.h"
  42
  43 static GLboolean
  44 intel_bufferobj_unmap(struct gl_context * ctx,
  45                       GLenum target, struct gl_buffer_object *obj);
  46
  47 /** Allocates a new drm_intel_bo to store the data for the buffer object. */
  48 static void
  49 intel_bufferobj_alloc_buffer(struct intel_context *intel,
  50                              struct intel_buffer_object *intel_obj)
  51 {
  52    intel_obj->buffer = drm_intel_bo_alloc(intel->bufmgr, "bufferobj",
  53                                           intel_obj->Base.Size, 64);
  54 }
  55
  56 /**
  57  * There is some duplication between mesa's bufferobjects and our
  58  * bufmgr buffers.  Both have an integer handle and a hashtable to
  59  * lookup an opaque structure.  It would be nice if the handles and
  60  * internal structure where somehow shared.
  61  */
  62 static struct gl_buffer_object *
  63 intel_bufferobj_alloc(struct gl_context * ctx, GLuint name, GLenum target)
  64 {
  65    struct intel_buffer_object *obj = CALLOC_STRUCT(intel_buffer_object);
  66
  67    _mesa_initialize_buffer_object(&obj->Base, name, target);
  68
  69    obj->buffer = NULL;
  70
  71    return &obj->Base;
  72 }
  73
  74 /* Break the COW tie to the region.  The region gets to keep the data.
  75  */
  76 void
  77 intel_bufferobj_release_region(struct intel_context *intel,
  78                                struct intel_buffer_object *intel_obj)
  79 {
  80    assert(intel_obj->region->buffer == intel_obj->buffer);
  81    intel_obj->region->pbo = NULL;
  82    intel_obj->region = NULL;
  83
  84    drm_intel_bo_unreference(intel_obj->buffer);
  85    intel_obj->buffer = NULL;
  86 }
  87
  88 /* Break the COW tie to the region.  Both the pbo and the region end
  89  * up with a copy of the data.
  90  */
  91 void
  92 intel_bufferobj_cow(struct intel_context *intel,
  93                     struct intel_buffer_object *intel_obj)
  94 {
  95    assert(intel_obj->region);
  96    intel_region_cow(intel, intel_obj->region);
  97 }
  98
  99
 100 /**
 101  * Deallocate/free a vertex/pixel buffer object.
 102  * Called via glDeleteBuffersARB().
 103  */
 104 static void
 105 intel_bufferobj_free(struct gl_context * ctx, struct gl_buffer_object *obj)
 106 {
 107    struct intel_context *intel = intel_context(ctx);
 108    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 109
 110    assert(intel_obj);
 111
 112    /* Buffer objects are automatically unmapped when deleting according
 113     * to the spec, but Mesa doesn't do UnmapBuffer for us at context destroy
 114     * (though it does if you call glDeleteBuffers)
 115     */
 116    if (obj->Pointer)
 117       intel_bufferobj_unmap(ctx, 0, obj);
 118
 119    free(intel_obj->sys_buffer);
 120    if (intel_obj->region) {
 121       intel_bufferobj_release_region(intel, intel_obj);
 122    }
 123    else if (intel_obj->buffer) {
 124       drm_intel_bo_unreference(intel_obj->buffer);
 125    }
 126
 127    free(intel_obj);
 128 }
 129
 130
 131
 132 /**
 133  * Allocate space for and store data in a buffer object.  Any data that was
 134  * previously stored in the buffer object is lost.  If data is NULL,
 135  * memory will be allocated, but no copy will occur.
 136  * Called via ctx->Driver.BufferData().
 137  * \return GL_TRUE for success, GL_FALSE if out of memory
 138  */
 139 static GLboolean
 140 intel_bufferobj_data(struct gl_context * ctx,
 141                      GLenum target,
 142                      GLsizeiptrARB size,
 143                      const GLvoid * data,
 144                      GLenum usage, struct gl_buffer_object *obj)
 145 {
 146    struct intel_context *intel = intel_context(ctx);
 147    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 148
 149    intel_obj->Base.Size = size;
 150    intel_obj->Base.Usage = usage;
 151
 152    assert(!obj->Pointer); /* Mesa should have unmapped it */
 153
 154    if (intel_obj->region)
 155       intel_bufferobj_release_region(intel, intel_obj);
 156
 157    if (intel_obj->buffer != NULL) {
 158       drm_intel_bo_unreference(intel_obj->buffer);
 159       intel_obj->buffer = NULL;
 160       intel_obj->source = 0;
 161    }
 162    free(intel_obj->sys_buffer);
 163    intel_obj->sys_buffer = NULL;
 164
 165    if (size != 0) {
 166       if (usage == GL_DYNAMIC_DRAW
 167 #ifdef I915
 168           /* On pre-965, stick VBOs in system memory, as we're always doing
 169            * swtnl with their contents anyway.
 170            */
 171           || target == GL_ARRAY_BUFFER || target == GL_ELEMENT_ARRAY_BUFFER
 172 #endif
 173          )
 174       {
 175          intel_obj->sys_buffer = malloc(size);
 176          if (intel_obj->sys_buffer != NULL) {
 177             if (data != NULL)
 178                memcpy(intel_obj->sys_buffer, data, size);
 179             return GL_TRUE;
 180          }
 181       }
 182       intel_bufferobj_alloc_buffer(intel, intel_obj);
 183       if (!intel_obj->buffer)
 184          return GL_FALSE;
 185
 186       if (data != NULL)
 187          drm_intel_bo_subdata(intel_obj->buffer, 0, size, data);
 188    }
 189
 190    return GL_TRUE;
 191 }
 192
 193
 194 /**
 195  * Replace data in a subrange of buffer object.  If the data range
 196  * specified by size + offset extends beyond the end of the buffer or
 197  * if data is NULL, no copy is performed.
 198  * Called via glBufferSubDataARB().
 199  */
 200 static void
 201 intel_bufferobj_subdata(struct gl_context * ctx,
 202                         GLenum target,
 203                         GLintptrARB offset,
 204                         GLsizeiptrARB size,
 205                         const GLvoid * data, struct gl_buffer_object *obj)
 206 {
 207    struct intel_context *intel = intel_context(ctx);
 208    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 209
 210    if (size == 0)
 211       return;
 212
 213    assert(intel_obj);
 214
 215    if (intel_obj->region)
 216       intel_bufferobj_cow(intel, intel_obj);
 217
 218    if (intel_obj->sys_buffer) {
 219       if (intel_obj->buffer) {
 220          drm_intel_bo_unreference(intel_obj->buffer);
 221          intel_obj->buffer = NULL;
 222          intel_obj->source = 0;
 223       }
 224       memcpy((char *)intel_obj->sys_buffer + offset, data, size);
 225    } else {
 226       bool busy =
 227          drm_intel_bo_busy(intel_obj->buffer) ||
 228          drm_intel_bo_references(intel->batch.bo, intel_obj->buffer);
 229
 230       /* replace the current busy bo with fresh data */
 231       if (busy && size == intel_obj->Base.Size) {
 232          drm_intel_bo_unreference(intel_obj->buffer);
 233          intel_bufferobj_alloc_buffer(intel, intel_obj);
 234          drm_intel_bo_subdata(intel_obj->buffer, 0, size, data);
 235       } else if (intel->gen < 6) {
 236          if (busy) {
 237             drm_intel_bo *temp_bo;
 238
 239             temp_bo = drm_intel_bo_alloc(intel->bufmgr, "subdata temp", size, 64);
 240
 241             drm_intel_bo_subdata(temp_bo, 0, size, data);
 242
 243             intel_emit_linear_blit(intel,
 244                                    intel_obj->buffer, offset,
 245                                    temp_bo, 0,
 246                                    size);
 247
 248             drm_intel_bo_unreference(temp_bo);
 249          } else {
 250             drm_intel_bo_subdata(intel_obj->buffer, offset, size, data);
 251          }
 252       } else {
 253          /* Can't use the blit to modify the buffer in the middle of batch. */
 254          if (drm_intel_bo_references(intel->batch.bo, intel_obj->buffer)) {
 255             intel_batchbuffer_flush(intel);
 256          }
 257          drm_intel_bo_subdata(intel_obj->buffer, offset, size, data);
 258       }
 259    }
 260 }
 261
 262
 263 /**
 264  * Called via glGetBufferSubDataARB().
 265  */
 266 static void
 267 intel_bufferobj_get_subdata(struct gl_context * ctx,
 268                             GLenum target,
 269                             GLintptrARB offset,
 270                             GLsizeiptrARB size,
 271                             GLvoid * data, struct gl_buffer_object *obj)
 272 {
 273    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 274
 275    assert(intel_obj);
 276    if (intel_obj->sys_buffer)
 277       memcpy(data, (char *)intel_obj->sys_buffer + offset, size);
 278    else
 279       drm_intel_bo_get_subdata(intel_obj->buffer, offset, size, data);
 280 }
 281
 282
 283
 284 /**
 285  * Called via glMapBufferARB().
 286  */
 287 static void *
 288 intel_bufferobj_map(struct gl_context * ctx,
 289                     GLenum target,
 290                     GLenum access, struct gl_buffer_object *obj)
 291 {
 292    struct intel_context *intel = intel_context(ctx);
 293    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 294    GLboolean read_only = (access == GL_READ_ONLY_ARB);
 295    GLboolean write_only = (access == GL_WRITE_ONLY_ARB);
 296
 297    assert(intel_obj);
 298
 299    if (intel_obj->sys_buffer) {
 300       if (!read_only && intel_obj->buffer) {
 301          drm_intel_bo_unreference(intel_obj->buffer);
 302          intel_obj->buffer = NULL;
 303          intel_obj->source = 0;
 304       }
 305       obj->Pointer = intel_obj->sys_buffer;
 306       obj->Length = obj->Size;
 307       obj->Offset = 0;
 308       return obj->Pointer;
 309    }
 310
 311    /* Flush any existing batchbuffer that might reference this data. */
 312    if (drm_intel_bo_references(intel->batch.bo, intel_obj->buffer))
 313       intel_flush(ctx);
 314
 315    if (intel_obj->region)
 316       intel_bufferobj_cow(intel, intel_obj);
 317
 318    if (intel_obj->buffer == NULL) {
 319       obj->Pointer = NULL;
 320       return NULL;
 321    }
 322
 323    if (write_only) {
 324       drm_intel_gem_bo_map_gtt(intel_obj->buffer);
 325       intel_obj->mapped_gtt = GL_TRUE;
 326    } else {
 327       drm_intel_bo_map(intel_obj->buffer, !read_only);
 328       intel_obj->mapped_gtt = GL_FALSE;
 329    }
 330
 331    obj->Pointer = intel_obj->buffer->virtual;
 332    obj->Length = obj->Size;
 333    obj->Offset = 0;
 334
 335    return obj->Pointer;
 336 }
 337
 338 /**
 339  * Called via glMapBufferRange().
 340  *
 341  * The goal of this extension is to allow apps to accumulate their rendering
 342  * at the same time as they accumulate their buffer object.  Without it,
 343  * you'd end up blocking on execution of rendering every time you mapped
 344  * the buffer to put new data in.
 345  *
 346  * We support it in 3 ways: If unsynchronized, then don't bother
 347  * flushing the batchbuffer before mapping the buffer, which can save blocking
 348  * in many cases.  If we would still block, and they allow the whole buffer
 349  * to be invalidated, then just allocate a new buffer to replace the old one.
 350  * If not, and we'd block, and they allow the subrange of the buffer to be
 351  * invalidated, then we can make a new little BO, let them write into that,
 352  * and blit it into the real BO at unmap time.
 353  */
 354 static void *
 355 intel_bufferobj_map_range(struct gl_context * ctx,
 356                           GLenum target, GLintptr offset, GLsizeiptr length,
 357                           GLbitfield access, struct gl_buffer_object *obj)
 358 {
 359    struct intel_context *intel = intel_context(ctx);
 360    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 361
 362    assert(intel_obj);
 363
 364    /* _mesa_MapBufferRange (GL entrypoint) sets these, but the vbo module also
 365     * internally uses our functions directly.
 366     */
 367    obj->Offset = offset;
 368    obj->Length = length;
 369    obj->AccessFlags = access;
 370
 371    if (intel_obj->sys_buffer) {
 372       if (access != GL_READ_ONLY_ARB && intel_obj->buffer) {
 373          drm_intel_bo_unreference(intel_obj->buffer);
 374          intel_obj->buffer = NULL;
 375          intel_obj->source = 0;
 376       }
 377       obj->Pointer = intel_obj->sys_buffer + offset;
 378       return obj->Pointer;
 379    }
 380
 381    if (intel_obj->region)
 382       intel_bufferobj_cow(intel, intel_obj);
 383
 384    /* If the mapping is synchronized with other GL operations, flush
 385     * the batchbuffer so that GEM knows about the buffer access for later
 386     * syncing.
 387     */
 388    if (!(access & GL_MAP_UNSYNCHRONIZED_BIT) &&
 389        drm_intel_bo_references(intel->batch.bo, intel_obj->buffer))
 390       intel_flush(ctx);
 391
 392    if (intel_obj->buffer == NULL) {
 393       obj->Pointer = NULL;
 394       return NULL;
 395    }
 396
 397    /* If the user doesn't care about existing buffer contents and mapping
 398     * would cause us to block, then throw out the old buffer.
 399     */
 400    if (!(access & GL_MAP_UNSYNCHRONIZED_BIT) &&
 401        (access & GL_MAP_INVALIDATE_BUFFER_BIT) &&
 402        drm_intel_bo_busy(intel_obj->buffer)) {
 403       drm_intel_bo_unreference(intel_obj->buffer);
 404       intel_bufferobj_alloc_buffer(intel, intel_obj);
 405    }
 406
 407    /* If the user is mapping a range of an active buffer object but
 408     * doesn't require the current contents of that range, make a new
 409     * BO, and we'll copy what they put in there out at unmap or
 410     * FlushRange time.
 411     */
 412    if ((access & GL_MAP_INVALIDATE_RANGE_BIT) &&
 413        drm_intel_bo_busy(intel_obj->buffer)) {
 414       if (access & GL_MAP_FLUSH_EXPLICIT_BIT) {
 415          intel_obj->range_map_buffer = malloc(length);
 416          obj->Pointer = intel_obj->range_map_buffer;
 417       } else {
 418          intel_obj->range_map_bo = drm_intel_bo_alloc(intel->bufmgr,
 419                                                       "range map",
 420                                                       length, 64);
 421          if (!(access & GL_MAP_READ_BIT)) {
 422             drm_intel_gem_bo_map_gtt(intel_obj->range_map_bo);
 423             intel_obj->mapped_gtt = GL_TRUE;
 424          } else {
 425             drm_intel_bo_map(intel_obj->range_map_bo,
 426                              (access & GL_MAP_WRITE_BIT) != 0);
 427             intel_obj->mapped_gtt = GL_FALSE;
 428          }
 429          obj->Pointer = intel_obj->range_map_bo->virtual;
 430       }
 431       return obj->Pointer;
 432    }
 433
 434    if (!(access & GL_MAP_READ_BIT)) {
 435       drm_intel_gem_bo_map_gtt(intel_obj->buffer);
 436       intel_obj->mapped_gtt = GL_TRUE;
 437    } else {
 438       drm_intel_bo_map(intel_obj->buffer, (access & GL_MAP_WRITE_BIT) != 0);
 439       intel_obj->mapped_gtt = GL_FALSE;
 440    }
 441
 442    obj->Pointer = intel_obj->buffer->virtual + offset;
 443    return obj->Pointer;
 444 }
 445
 446 /* Ideally we'd use a BO to avoid taking up cache space for the temporary
 447  * data, but FlushMappedBufferRange may be followed by further writes to
 448  * the pointer, so we would have to re-map after emitting our blit, which
 449  * would defeat the point.
 450  */
 451 static void
 452 intel_bufferobj_flush_mapped_range(struct gl_context *ctx, GLenum target,
 453                                    GLintptr offset, GLsizeiptr length,
 454                                    struct gl_buffer_object *obj)
 455 {
 456    struct intel_context *intel = intel_context(ctx);
 457    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 458    drm_intel_bo *temp_bo;
 459
 460    /* Unless we're in the range map using a temporary system buffer,
 461     * there's no work to do.
 462     */
 463    if (intel_obj->range_map_buffer == NULL)
 464       return;
 465
 466    if (length == 0)
 467       return;
 468
 469    temp_bo = drm_intel_bo_alloc(intel->bufmgr, "range map flush", length, 64);
 470
 471    drm_intel_bo_subdata(temp_bo, 0, length, intel_obj->range_map_buffer);
 472
 473    intel_emit_linear_blit(intel,
 474                           intel_obj->buffer, obj->Offset + offset,
 475                           temp_bo, 0,
 476                           length);
 477
 478    drm_intel_bo_unreference(temp_bo);
 479 }
 480
 481
 482 /**
 483  * Called via glUnmapBuffer().
 484  */
 485 static GLboolean
 486 intel_bufferobj_unmap(struct gl_context * ctx,
 487                       GLenum target, struct gl_buffer_object *obj)
 488 {
 489    struct intel_context *intel = intel_context(ctx);
 490    struct intel_buffer_object *intel_obj = intel_buffer_object(obj);
 491
 492    assert(intel_obj);
 493    assert(obj->Pointer);
 494    if (intel_obj->sys_buffer != NULL) {
 495       /* always keep the mapping around. */
 496    } else if (intel_obj->range_map_buffer != NULL) {
 497       /* Since we've emitted some blits to buffers that will (likely) be used
 498        * in rendering operations in other cache domains in this batch, emit a
 499        * flush.  Once again, we wish for a domain tracker in libdrm to cover
 500        * usage inside of a batchbuffer.
 501        */
 502       intel_batchbuffer_emit_mi_flush(intel);
 503       free(intel_obj->range_map_buffer);
 504       intel_obj->range_map_buffer = NULL;
 505    } else if (intel_obj->range_map_bo != NULL) {
 506       if (intel_obj->mapped_gtt) {
 507          drm_intel_gem_bo_unmap_gtt(intel_obj->range_map_bo);
 508       } else {
 509          drm_intel_bo_unmap(intel_obj->range_map_bo);
 510       }
 511
 512       intel_emit_linear_blit(intel,
 513                              intel_obj->buffer, obj->Offset,
 514                              intel_obj->range_map_bo, 0,
 515                              obj->Length);
 516
 517       /* Since we've emitted some blits to buffers that will (likely) be used
 518        * in rendering operations in other cache domains in this batch, emit a
 519        * flush.  Once again, we wish for a domain tracker in libdrm to cover
 520        * usage inside of a batchbuffer.
 521        */
 522       intel_batchbuffer_emit_mi_flush(intel);
 523
 524       drm_intel_bo_unreference(intel_obj->range_map_bo);
 525       intel_obj->range_map_bo = NULL;
 526    } else if (intel_obj->buffer != NULL) {
 527       if (intel_obj->mapped_gtt) {
 528          drm_intel_gem_bo_unmap_gtt(intel_obj->buffer);
 529       } else {
 530          drm_intel_bo_unmap(intel_obj->buffer);
 531       }
 532    }
 533    obj->Pointer = NULL;
 534    obj->Offset = 0;
 535    obj->Length = 0;
 536
 537    return GL_TRUE;
 538 }
 539
 540 drm_intel_bo *
 541 intel_bufferobj_buffer(struct intel_context *intel,
 542                        struct intel_buffer_object *intel_obj,
 543                        GLuint flag)
 544 {
 545    if (intel_obj->region) {
 546       if (flag == INTEL_WRITE_PART)
 547          intel_bufferobj_cow(intel, intel_obj);
 548       else if (flag == INTEL_WRITE_FULL) {
 549          intel_bufferobj_release_region(intel, intel_obj);
 550          intel_bufferobj_alloc_buffer(intel, intel_obj);
 551       }
 552    }
 553
 554    if (intel_obj->source) {
 555       drm_intel_bo_unreference(intel_obj->buffer);
 556       intel_obj->buffer = NULL;
 557       intel_obj->source = 0;
 558    }
 559
 560    if (intel_obj->buffer == NULL) {
 561       intel_bufferobj_alloc_buffer(intel, intel_obj);
 562       drm_intel_bo_subdata(intel_obj->buffer,
 563                            0, intel_obj->Base.Size,
 564                            intel_obj->sys_buffer);
 565
 566       free(intel_obj->sys_buffer);
 567       intel_obj->sys_buffer = NULL;
 568       intel_obj->offset = 0;
 569    }
 570
 571    return intel_obj->buffer;
 572 }
 573
 574 #define INTEL_UPLOAD_SIZE (64*1024)
 575
 576 void
 577 intel_upload_finish(struct intel_context *intel)
 578 {
 579    if (!intel->upload.bo)
 580            return;
 581
 582    if (intel->upload.buffer_len) {
 583            drm_intel_bo_subdata(intel->upload.bo,
 584                                 intel->upload.buffer_offset,
 585                                 intel->upload.buffer_len,
 586                                 intel->upload.buffer);
 587            intel->upload.buffer_len = 0;
 588    }
 589
 590    drm_intel_bo_unreference(intel->upload.bo);
 591    intel->upload.bo = NULL;
 592 }
 593
 594 static void wrap_buffers(struct intel_context *intel, GLuint size)
 595 {
 596    intel_upload_finish(intel);
 597
 598    if (size < INTEL_UPLOAD_SIZE)
 599       size = INTEL_UPLOAD_SIZE;
 600
 601    intel->upload.bo = drm_intel_bo_alloc(intel->bufmgr, "upload", size, 0);
 602    intel->upload.offset = 0;
 603 }
 604
 605 void intel_upload_data(struct intel_context *intel,
 606                        const void *ptr, GLuint size, GLuint align,
 607                        drm_intel_bo **return_bo,
 608                        GLuint *return_offset)
 609 {
 610    GLuint base, delta;
 611
 612    base = (intel->upload.offset + align - 1) / align * align;
 613    if (intel->upload.bo == NULL || base + size > intel->upload.bo->size) {
 614       wrap_buffers(intel, size);
 615       base = 0;
 616    }
 617
 618    drm_intel_bo_reference(intel->upload.bo);
 619    *return_bo = intel->upload.bo;
 620    *return_offset = base;
 621
 622    delta = base - intel->upload.offset;
 623    if (intel->upload.buffer_len &&
 624        intel->upload.buffer_len + delta + size > sizeof(intel->upload.buffer))
 625    {
 626       drm_intel_bo_subdata(intel->upload.bo,
 627                            intel->upload.buffer_offset,
 628                            intel->upload.buffer_len,
 629                            intel->upload.buffer);
 630       intel->upload.buffer_len = 0;
 631    }
 632
 633    if (size < sizeof(intel->upload.buffer))
 634    {
 635       if (intel->upload.buffer_len == 0)
 636          intel->upload.buffer_offset = base;
 637       else
 638          intel->upload.buffer_len += delta;
 639
 640       memcpy(intel->upload.buffer + intel->upload.buffer_len, ptr, size);
 641       intel->upload.buffer_len += size;
 642    }
 643    else
 644    {
 645       drm_intel_bo_subdata(intel->upload.bo, base, size, ptr);
 646    }
 647
 648    intel->upload.offset = base + size;
 649 }
 650
 651 void *intel_upload_map(struct intel_context *intel, GLuint size, GLuint align)
 652 {
 653    GLuint base, delta;
 654    char *ptr;
 655
 656    base = (intel->upload.offset + align - 1) / align * align;
 657    if (intel->upload.bo == NULL || base + size > intel->upload.bo->size) {
 658       wrap_buffers(intel, size);
 659       base = 0;
 660    }
 661
 662    delta = base - intel->upload.offset;
 663    if (intel->upload.buffer_len &&
 664        intel->upload.buffer_len + delta + size > sizeof(intel->upload.buffer))
 665    {
 666       drm_intel_bo_subdata(intel->upload.bo,
 667                            intel->upload.buffer_offset,
 668                            intel->upload.buffer_len,
 669                            intel->upload.buffer);
 670       intel->upload.buffer_len = 0;
 671    }
 672
 673    if (size <= sizeof(intel->upload.buffer)) {
 674       if (intel->upload.buffer_len == 0)
 675          intel->upload.buffer_offset = base;
 676       else
 677          intel->upload.buffer_len += delta;
 678
 679       ptr = intel->upload.buffer + intel->upload.buffer_len;
 680       intel->upload.buffer_len += size;
 681    } else
 682       ptr = malloc(size);
 683
 684    return ptr;
 685 }
 686
 687 void intel_upload_unmap(struct intel_context *intel,
 688                         const void *ptr, GLuint size, GLuint align,
 689                         drm_intel_bo **return_bo,
 690                         GLuint *return_offset)
 691 {
 692    GLuint base;
 693
 694    base = (intel->upload.offset + align - 1) / align * align;
 695    if (size > sizeof(intel->upload.buffer)) {
 696       drm_intel_bo_subdata(intel->upload.bo, base, size, ptr);
 697       free((void*)ptr);
 698    }
 699
 700    drm_intel_bo_reference(intel->upload.bo);
 701    *return_bo = intel->upload.bo;
 702    *return_offset = base;
 703
 704    intel->upload.offset = base + size;
 705 }
 706
 707 drm_intel_bo *
 708 intel_bufferobj_source(struct intel_context *intel,
 709                        struct intel_buffer_object *intel_obj,
 710                        GLuint *offset)
 711 {
 712    if (intel_obj->buffer == NULL) {
 713       intel_upload_data(intel,
 714                         intel_obj->sys_buffer, intel_obj->Base.Size, 64,
 715                         &intel_obj->buffer, &intel_obj->offset);
 716       intel_obj->source = 1;
 717    }
 718
 719    *offset = intel_obj->offset;
 720    return intel_obj->buffer;
 721 }
 722
 723 static void
 724 intel_bufferobj_copy_subdata(struct gl_context *ctx,
 725                              struct gl_buffer_object *src,
 726                              struct gl_buffer_object *dst,
 727                              GLintptr read_offset, GLintptr write_offset,
 728                              GLsizeiptr size)
 729 {
 730    struct intel_context *intel = intel_context(ctx);
 731    struct intel_buffer_object *intel_src = intel_buffer_object(src);
 732    struct intel_buffer_object *intel_dst = intel_buffer_object(dst);
 733    drm_intel_bo *src_bo, *dst_bo;
 734    GLuint src_offset;
 735
 736    if (size == 0)
 737       return;
 738
 739    /* If we're in system memory, just map and memcpy. */
 740    if (intel_src->sys_buffer || intel_dst->sys_buffer || intel->gen >= 6) {
 741       /* The same buffer may be used, but note that regions copied may
 742        * not overlap.
 743        */
 744       if (src == dst) {
 745          char *ptr = intel_bufferobj_map(ctx, GL_COPY_WRITE_BUFFER,
 746                                          GL_READ_WRITE, dst);
 747          memcpy(ptr + write_offset, ptr + read_offset, size);
 748          intel_bufferobj_unmap(ctx, GL_COPY_WRITE_BUFFER, dst);
 749       } else {
 750          const char *src_ptr;
 751          char *dst_ptr;
 752
 753          src_ptr =  intel_bufferobj_map(ctx, GL_COPY_READ_BUFFER,
 754                                         GL_READ_ONLY, src);
 755          dst_ptr =  intel_bufferobj_map(ctx, GL_COPY_WRITE_BUFFER,
 756                                         GL_WRITE_ONLY, dst);
 757
 758          memcpy(dst_ptr + write_offset, src_ptr + read_offset, size);
 759
 760          intel_bufferobj_unmap(ctx, GL_COPY_READ_BUFFER, src);
 761          intel_bufferobj_unmap(ctx, GL_COPY_WRITE_BUFFER, dst);
 762       }
 763       return;
 764    }
 765
 766    /* Otherwise, we have real BOs, so blit them. */
 767
 768    dst_bo = intel_bufferobj_buffer(intel, intel_dst, INTEL_WRITE_PART);
 769    src_bo = intel_bufferobj_source(intel, intel_src, &src_offset);
 770
 771    intel_emit_linear_blit(intel,
 772                           dst_bo, write_offset,
 773                           src_bo, read_offset + src_offset, size);
 774
 775    /* Since we've emitted some blits to buffers that will (likely) be used
 776     * in rendering operations in other cache domains in this batch, emit a
 777     * flush.  Once again, we wish for a domain tracker in libdrm to cover
 778     * usage inside of a batchbuffer.
 779     */
 780    intel_batchbuffer_emit_mi_flush(intel);
 781 }
 782
 783 #if FEATURE_APPLE_object_purgeable
 784 static GLenum
 785 intel_buffer_purgeable(struct gl_context * ctx,
 786                        drm_intel_bo *buffer,
 787                        GLenum option)
 788 {
 789    int retained = 0;
 790
 791    if (buffer != NULL)
 792       retained = drm_intel_bo_madvise (buffer, I915_MADV_DONTNEED);
 793
 794    return retained ? GL_VOLATILE_APPLE : GL_RELEASED_APPLE;
 795 }
 796
 797 static GLenum
 798 intel_buffer_object_purgeable(struct gl_context * ctx,
 799                               struct gl_buffer_object *obj,
 800                               GLenum option)
 801 {
 802    struct intel_buffer_object *intel;
 803
 804    intel = intel_buffer_object (obj);
 805    if (intel->buffer != NULL)
 806       return intel_buffer_purgeable (ctx, intel->buffer, option);
 807
 808    if (option == GL_RELEASED_APPLE) {
 809       if (intel->sys_buffer != NULL) {
 810          free(intel->sys_buffer);
 811          intel->sys_buffer = NULL;
 812       }
 813
 814       return GL_RELEASED_APPLE;
 815    } else {
 816       /* XXX Create the buffer and madvise(MADV_DONTNEED)? */
 817       return intel_buffer_purgeable (ctx,
 818                                      intel_bufferobj_buffer(intel_context(ctx),
 819                                                             intel, INTEL_READ),
 820                                      option);
 821    }
 822 }
 823
 824 static GLenum
 825 intel_texture_object_purgeable(struct gl_context * ctx,
 826                                struct gl_texture_object *obj,
 827                                GLenum option)
 828 {
 829    struct intel_texture_object *intel;
 830
 831    intel = intel_texture_object(obj);
 832    if (intel->mt == NULL || intel->mt->region == NULL)
 833       return GL_RELEASED_APPLE;
 834
 835    return intel_buffer_purgeable (ctx, intel->mt->region->buffer, option);
 836 }
 837
 838 static GLenum
 839 intel_render_object_purgeable(struct gl_context * ctx,
 840                               struct gl_renderbuffer *obj,
 841                               GLenum option)
 842 {
 843    struct intel_renderbuffer *intel;
 844
 845    intel = intel_renderbuffer(obj);
 846    if (intel->region == NULL)
 847       return GL_RELEASED_APPLE;
 848
 849    return intel_buffer_purgeable (ctx, intel->region->buffer, option);
 850 }
 851
 852 static GLenum
 853 intel_buffer_unpurgeable(struct gl_context * ctx,
 854                          drm_intel_bo *buffer,
 855                          GLenum option)
 856 {
 857    int retained;
 858
 859    retained = 0;
 860    if (buffer != NULL)
 861       retained = drm_intel_bo_madvise (buffer, I915_MADV_WILLNEED);
 862
 863    return retained ? GL_RETAINED_APPLE : GL_UNDEFINED_APPLE;
 864 }
 865
 866 static GLenum
 867 intel_buffer_object_unpurgeable(struct gl_context * ctx,
 868                                 struct gl_buffer_object *obj,
 869                                 GLenum option)
 870 {
 871    return intel_buffer_unpurgeable (ctx, intel_buffer_object (obj)->buffer, option);
 872 }
 873
 874 static GLenum
 875 intel_texture_object_unpurgeable(struct gl_context * ctx,
 876                                  struct gl_texture_object *obj,
 877                                  GLenum option)
 878 {
 879    struct intel_texture_object *intel;
 880
 881    intel = intel_texture_object(obj);
 882    if (intel->mt == NULL || intel->mt->region == NULL)
 883       return GL_UNDEFINED_APPLE;
 884
 885    return intel_buffer_unpurgeable (ctx, intel->mt->region->buffer, option);
 886 }
 887
 888 static GLenum
 889 intel_render_object_unpurgeable(struct gl_context * ctx,
 890                                 struct gl_renderbuffer *obj,
 891                                 GLenum option)
 892 {
 893    struct intel_renderbuffer *intel;
 894
 895    intel = intel_renderbuffer(obj);
 896    if (intel->region == NULL)
 897       return GL_UNDEFINED_APPLE;
 898
 899    return intel_buffer_unpurgeable (ctx, intel->region->buffer, option);
 900 }
 901 #endif
 902
 903 void
 904 intelInitBufferObjectFuncs(struct dd_function_table *functions)
 905 {
 906    functions->NewBufferObject = intel_bufferobj_alloc;
 907    functions->DeleteBuffer = intel_bufferobj_free;
 908    functions->BufferData = intel_bufferobj_data;
 909    functions->BufferSubData = intel_bufferobj_subdata;
 910    functions->GetBufferSubData = intel_bufferobj_get_subdata;
 911    functions->MapBuffer = intel_bufferobj_map;
 912    functions->MapBufferRange = intel_bufferobj_map_range;
 913    functions->FlushMappedBufferRange = intel_bufferobj_flush_mapped_range;
 914    functions->UnmapBuffer = intel_bufferobj_unmap;
 915    functions->CopyBufferSubData = intel_bufferobj_copy_subdata;
 916
 917 #if FEATURE_APPLE_object_purgeable
 918    functions->BufferObjectPurgeable = intel_buffer_object_purgeable;
 919    functions->TextureObjectPurgeable = intel_texture_object_purgeable;
 920    functions->RenderObjectPurgeable = intel_render_object_purgeable;
 921
 922    functions->BufferObjectUnpurgeable = intel_buffer_object_unpurgeable;
 923    functions->TextureObjectUnpurgeable = intel_texture_object_unpurgeable;
 924    functions->RenderObjectUnpurgeable = intel_render_object_unpurgeable;
 925 #endif
 926 }