* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
*
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
*/
/**
static enum iris_memory_zone
memzone_for_address(uint64_t address)
{
- STATIC_ASSERT(IRIS_MEMZONE_OTHER_START > IRIS_MEMZONE_DYNAMIC_START);
+ STATIC_ASSERT(IRIS_MEMZONE_OTHER_START > IRIS_MEMZONE_DYNAMIC_START);
STATIC_ASSERT(IRIS_MEMZONE_DYNAMIC_START > IRIS_MEMZONE_SURFACE_START);
- STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START > IRIS_MEMZONE_SHADER_START);
- STATIC_ASSERT(IRIS_BINDER_ADDRESS == IRIS_MEMZONE_SURFACE_START);
+ STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START > IRIS_MEMZONE_BINDER_START);
+ STATIC_ASSERT(IRIS_MEMZONE_BINDER_START > IRIS_MEMZONE_SHADER_START);
STATIC_ASSERT(IRIS_BORDER_COLOR_POOL_ADDRESS == IRIS_MEMZONE_DYNAMIC_START);
if (address >= IRIS_MEMZONE_OTHER_START)
if (address > IRIS_MEMZONE_DYNAMIC_START)
return IRIS_MEMZONE_DYNAMIC;
- if (address == IRIS_BINDER_ADDRESS)
- return IRIS_MEMZONE_BINDER;
-
- if (address > IRIS_MEMZONE_SURFACE_START)
+ if (address >= IRIS_MEMZONE_SURFACE_START)
return IRIS_MEMZONE_SURFACE;
+ if (address >= IRIS_MEMZONE_BINDER_START)
+ return IRIS_MEMZONE_BINDER;
+
return IRIS_MEMZONE_SHADER;
}
}
static struct bo_cache_bucket *
-get_bucket_allocator(struct iris_bufmgr *bufmgr, uint64_t size)
+get_bucket_allocator(struct iris_bufmgr *bufmgr,
+ enum iris_memory_zone memzone,
+ uint64_t size)
{
/* Skip using the bucket allocator for very large sizes, as it allocates
* 64 of them and this can balloon rather quickly.
uint64_t size,
uint64_t alignment)
{
- if (memzone == IRIS_MEMZONE_BINDER)
- return IRIS_BINDER_ADDRESS;
- else if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL)
+ if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL)
return IRIS_BORDER_COLOR_POOL_ADDRESS;
- struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size);
+ /* The binder handles its own allocations. Return non-zero here. */
+ if (memzone == IRIS_MEMZONE_BINDER)
+ return IRIS_MEMZONE_BINDER_START;
+
+ struct bo_cache_bucket *bucket =
+ get_bucket_allocator(bufmgr, memzone, size);
uint64_t addr;
if (bucket) {
uint64_t address,
uint64_t size)
{
- if (address == IRIS_BINDER_ADDRESS ||
- address == IRIS_BORDER_COLOR_POOL_ADDRESS)
+ if (address == IRIS_BORDER_COLOR_POOL_ADDRESS)
return;
/* Un-canonicalize the address. */
if (address == 0ull)
return;
- struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size);
+ enum iris_memory_zone memzone = memzone_for_address(address);
+
+ /* The binder handles its own allocations. */
+ if (memzone == IRIS_MEMZONE_BINDER)
+ return;
+
+ struct bo_cache_bucket *bucket =
+ get_bucket_allocator(bufmgr, memzone, size);
if (bucket) {
bucket_vma_free(bucket, address);
} else {
- enum iris_memory_zone memzone = memzone_for_address(address);
util_vma_heap_free(&bufmgr->vma_allocator[memzone], address, size);
}
}
}
}
+static struct iris_bo *
+bo_calloc(void)
+{
+ struct iris_bo *bo = calloc(1, sizeof(*bo));
+ if (bo) {
+ bo->hash = _mesa_hash_pointer(bo);
+ }
+ return bo;
+}
+
static struct iris_bo *
bo_alloc_internal(struct iris_bufmgr *bufmgr,
const char *name,
* allocation up.
*/
if (bucket == NULL) {
- bo_size = size;
- if (bo_size < page_size)
- bo_size = page_size;
+ bo_size = MAX2(ALIGN(size, page_size), page_size);
} else {
bo_size = bucket->size;
}
* memory and assign it a new address.
*/
if (memzone != memzone_for_address(bo->gtt_offset)) {
- vma_free(bufmgr, bo->gtt_offset, bo_size);
+ vma_free(bufmgr, bo->gtt_offset, bo->size);
bo->gtt_offset = 0ull;
}
} else {
- bo = calloc(1, sizeof(*bo));
+ bo = bo_calloc();
if (!bo)
goto err;
bo->name = name;
p_atomic_set(&bo->refcount, 1);
- bo->reusable = true;
+ bo->reusable = bucket && bufmgr->bo_reuse;
bo->cache_coherent = bufmgr->has_llc;
bo->index = -1;
bo->kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
+ /* By default, capture all driver-internal buffers like shader kernels,
+ * surface states, dynamic states, border colors, and so on.
+ */
+ if (memzone < IRIS_MEMZONE_OTHER)
+ bo->kflags |= EXEC_OBJECT_CAPTURE;
+
if (bo->gtt_offset == 0ull) {
bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, 1);
{
struct iris_bo *bo;
- bo = calloc(1, sizeof(*bo));
+ bo = bo_calloc();
if (!bo)
return NULL;
goto out;
}
- bo = calloc(1, sizeof(*bo));
+ bo = bo_calloc();
if (!bo)
goto out;
DBG("bo_unreference final: %d (%s)\n", bo->gem_handle, bo->name);
- bucket = bucket_for_size(bufmgr, bo->size);
+ bucket = NULL;
+ if (bo->reusable)
+ bucket = bucket_for_size(bufmgr, bo->size);
/* Put the buffer into our internal cache for reuse if we can. */
- if (bufmgr->bo_reuse && bo->reusable && bucket != NULL &&
- iris_bo_madvise(bo, I915_MADV_DONTNEED)) {
+ if (bucket && iris_bo_madvise(bo, I915_MADV_DONTNEED)) {
bo->free_time = time;
bo->name = NULL;
_mesa_hash_table_destroy(bufmgr->handle_table, NULL);
for (int z = 0; z < IRIS_MEMZONE_COUNT; z++) {
- util_vma_heap_finish(&bufmgr->vma_allocator[z]);
+ if (z != IRIS_MEMZONE_BINDER)
+ util_vma_heap_finish(&bufmgr->vma_allocator[z]);
}
free(bufmgr);
goto out;
}
- bo = calloc(1, sizeof(*bo));
+ bo = bo_calloc();
if (!bo)
goto out;
return NULL;
}
+static void
+iris_bo_make_external_locked(struct iris_bo *bo)
+{
+ if (!bo->external) {
+ _mesa_hash_table_insert(bo->bufmgr->handle_table, &bo->gem_handle, bo);
+ bo->external = true;
+ }
+}
+
static void
iris_bo_make_external(struct iris_bo *bo)
{
struct iris_bufmgr *bufmgr = bo->bufmgr;
- if (!bo->external) {
- mtx_lock(&bufmgr->lock);
- if (!bo->external) {
- _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
- bo->external = true;
- }
- mtx_unlock(&bufmgr->lock);
- }
+ if (bo->external)
+ return;
+
+ mtx_lock(&bufmgr->lock);
+ iris_bo_make_external_locked(bo);
+ mtx_unlock(&bufmgr->lock);
}
int
if (drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink))
return -errno;
- iris_bo_make_external(bo);
mtx_lock(&bufmgr->lock);
if (!bo->global_name) {
+ iris_bo_make_external_locked(bo);
bo->global_name = flink.name;
_mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo);
}
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SHADER],
PAGE_SIZE, _4GB);
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SURFACE],
- IRIS_MEMZONE_SURFACE_START + IRIS_BINDER_SIZE,
- _4GB - IRIS_BINDER_SIZE);
+ IRIS_MEMZONE_SURFACE_START,
+ _4GB - IRIS_MAX_BINDERS * IRIS_BINDER_SIZE);
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_DYNAMIC],
IRIS_MEMZONE_DYNAMIC_START + IRIS_BORDER_COLOR_POOL_SIZE,
_4GB - IRIS_BORDER_COLOR_POOL_SIZE);