projects
/
mesa.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
iris: create aux surface if needed
[mesa.git]
/
src
/
gallium
/
drivers
/
iris
/
iris_bufmgr.c
diff --git
a/src/gallium/drivers/iris/iris_bufmgr.c
b/src/gallium/drivers/iris/iris_bufmgr.c
index ebec10bf093317b3d9ff5c48b69a2e4487a78cc1..92ede93405622051bcbb4e90d5a5e4e0489383dc 100644
(file)
--- a/
src/gallium/drivers/iris/iris_bufmgr.c
+++ b/
src/gallium/drivers/iris/iris_bufmgr.c
@@
-8,17
+8,16
@@
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
*
*
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+ *
OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS
- * IN THE SOFTWARE.
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ *
DEALINGS
IN THE SOFTWARE.
*/
/**
*/
/**
@@
-245,10
+244,10
@@
bucket_for_size(struct iris_bufmgr *bufmgr, uint64_t size)
static enum iris_memory_zone
memzone_for_address(uint64_t address)
{
static enum iris_memory_zone
memzone_for_address(uint64_t address)
{
- STATIC_ASSERT(IRIS_MEMZONE_OTHER_START > IRIS_MEMZONE_DYNAMIC_START);
+ STATIC_ASSERT(IRIS_MEMZONE_OTHER_START
> IRIS_MEMZONE_DYNAMIC_START);
STATIC_ASSERT(IRIS_MEMZONE_DYNAMIC_START > IRIS_MEMZONE_SURFACE_START);
STATIC_ASSERT(IRIS_MEMZONE_DYNAMIC_START > IRIS_MEMZONE_SURFACE_START);
- STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START > IRIS_MEMZONE_
SHA
DER_START);
- STATIC_ASSERT(IRIS_
BINDER_ADDRESS == IRIS_MEMZONE_SURFACE
_START);
+ STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START > IRIS_MEMZONE_
BIN
DER_START);
+ STATIC_ASSERT(IRIS_
MEMZONE_BINDER_START > IRIS_MEMZONE_SHADER
_START);
STATIC_ASSERT(IRIS_BORDER_COLOR_POOL_ADDRESS == IRIS_MEMZONE_DYNAMIC_START);
if (address >= IRIS_MEMZONE_OTHER_START)
STATIC_ASSERT(IRIS_BORDER_COLOR_POOL_ADDRESS == IRIS_MEMZONE_DYNAMIC_START);
if (address >= IRIS_MEMZONE_OTHER_START)
@@
-260,12
+259,12
@@
memzone_for_address(uint64_t address)
if (address > IRIS_MEMZONE_DYNAMIC_START)
return IRIS_MEMZONE_DYNAMIC;
if (address > IRIS_MEMZONE_DYNAMIC_START)
return IRIS_MEMZONE_DYNAMIC;
- if (address == IRIS_BINDER_ADDRESS)
- return IRIS_MEMZONE_BINDER;
-
- if (address > IRIS_MEMZONE_SURFACE_START)
+ if (address >= IRIS_MEMZONE_SURFACE_START)
return IRIS_MEMZONE_SURFACE;
return IRIS_MEMZONE_SURFACE;
+ if (address >= IRIS_MEMZONE_BINDER_START)
+ return IRIS_MEMZONE_BINDER;
+
return IRIS_MEMZONE_SHADER;
}
return IRIS_MEMZONE_SHADER;
}
@@
-366,7
+365,9
@@
bucket_vma_free(struct bo_cache_bucket *bucket, uint64_t address)
}
static struct bo_cache_bucket *
}
static struct bo_cache_bucket *
-get_bucket_allocator(struct iris_bufmgr *bufmgr, uint64_t size)
+get_bucket_allocator(struct iris_bufmgr *bufmgr,
+ enum iris_memory_zone memzone,
+ uint64_t size)
{
/* Skip using the bucket allocator for very large sizes, as it allocates
* 64 of them and this can balloon rather quickly.
{
/* Skip using the bucket allocator for very large sizes, as it allocates
* 64 of them and this can balloon rather quickly.
@@
-394,12
+395,15
@@
vma_alloc(struct iris_bufmgr *bufmgr,
uint64_t size,
uint64_t alignment)
{
uint64_t size,
uint64_t alignment)
{
- if (memzone == IRIS_MEMZONE_BINDER)
- return IRIS_BINDER_ADDRESS;
- else if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL)
+ if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL)
return IRIS_BORDER_COLOR_POOL_ADDRESS;
return IRIS_BORDER_COLOR_POOL_ADDRESS;
- struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size);
+ /* The binder handles its own allocations. Return non-zero here. */
+ if (memzone == IRIS_MEMZONE_BINDER)
+ return IRIS_MEMZONE_BINDER_START;
+
+ struct bo_cache_bucket *bucket =
+ get_bucket_allocator(bufmgr, memzone, size);
uint64_t addr;
if (bucket) {
uint64_t addr;
if (bucket) {
@@
-420,8
+424,7
@@
vma_free(struct iris_bufmgr *bufmgr,
uint64_t address,
uint64_t size)
{
uint64_t address,
uint64_t size)
{
- if (address == IRIS_BINDER_ADDRESS ||
- address == IRIS_BORDER_COLOR_POOL_ADDRESS)
+ if (address == IRIS_BORDER_COLOR_POOL_ADDRESS)
return;
/* Un-canonicalize the address. */
return;
/* Un-canonicalize the address. */
@@
-430,12
+433,18
@@
vma_free(struct iris_bufmgr *bufmgr,
if (address == 0ull)
return;
if (address == 0ull)
return;
- struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size);
+ enum iris_memory_zone memzone = memzone_for_address(address);
+
+ /* The binder handles its own allocations. */
+ if (memzone == IRIS_MEMZONE_BINDER)
+ return;
+
+ struct bo_cache_bucket *bucket =
+ get_bucket_allocator(bufmgr, memzone, size);
if (bucket) {
bucket_vma_free(bucket, address);
} else {
if (bucket) {
bucket_vma_free(bucket, address);
} else {
- enum iris_memory_zone memzone = memzone_for_address(address);
util_vma_heap_free(&bufmgr->vma_allocator[memzone], address, size);
}
}
util_vma_heap_free(&bufmgr->vma_allocator[memzone], address, size);
}
}
@@
-482,6
+491,16
@@
iris_bo_cache_purge_bucket(struct iris_bufmgr *bufmgr,
}
}
}
}
+static struct iris_bo *
+bo_calloc(void)
+{
+ struct iris_bo *bo = calloc(1, sizeof(*bo));
+ if (bo) {
+ bo->hash = _mesa_hash_pointer(bo);
+ }
+ return bo;
+}
+
static struct iris_bo *
bo_alloc_internal(struct iris_bufmgr *bufmgr,
const char *name,
static struct iris_bo *
bo_alloc_internal(struct iris_bufmgr *bufmgr,
const char *name,
@@
-509,9
+528,7
@@
bo_alloc_internal(struct iris_bufmgr *bufmgr,
* allocation up.
*/
if (bucket == NULL) {
* allocation up.
*/
if (bucket == NULL) {
- bo_size = size;
- if (bo_size < page_size)
- bo_size = page_size;
+ bo_size = MAX2(ALIGN(size, page_size), page_size);
} else {
bo_size = bucket->size;
}
} else {
bo_size = bucket->size;
}
@@
-558,11
+575,11
@@
retry:
* memory and assign it a new address.
*/
if (memzone != memzone_for_address(bo->gtt_offset)) {
* memory and assign it a new address.
*/
if (memzone != memzone_for_address(bo->gtt_offset)) {
- vma_free(bufmgr, bo->gtt_offset, bo
_
size);
+ vma_free(bufmgr, bo->gtt_offset, bo
->
size);
bo->gtt_offset = 0ull;
}
} else {
bo->gtt_offset = 0ull;
}
} else {
- bo =
calloc(1, sizeof(*bo)
);
+ bo =
bo_calloc(
);
if (!bo)
goto err;
if (!bo)
goto err;
@@
-607,11
+624,17
@@
retry:
bo->name = name;
p_atomic_set(&bo->refcount, 1);
bo->name = name;
p_atomic_set(&bo->refcount, 1);
- bo->reusable =
tru
e;
+ bo->reusable =
bucket && bufmgr->bo_reus
e;
bo->cache_coherent = bufmgr->has_llc;
bo->index = -1;
bo->kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
bo->cache_coherent = bufmgr->has_llc;
bo->index = -1;
bo->kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
+ /* By default, capture all driver-internal buffers like shader kernels,
+ * surface states, dynamic states, border colors, and so on.
+ */
+ if (memzone < IRIS_MEMZONE_OTHER)
+ bo->kflags |= EXEC_OBJECT_CAPTURE;
+
if (bo->gtt_offset == 0ull) {
bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, 1);
if (bo->gtt_offset == 0ull) {
bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, 1);
@@
-659,7
+682,7
@@
iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
{
struct iris_bo *bo;
{
struct iris_bo *bo;
- bo =
calloc(1, sizeof(*bo)
);
+ bo =
bo_calloc(
);
if (!bo)
return NULL;
if (!bo)
return NULL;
@@
-747,7
+770,7
@@
iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
goto out;
}
goto out;
}
- bo =
calloc(1, sizeof(*bo)
);
+ bo =
bo_calloc(
);
if (!bo)
goto out;
if (!bo)
goto out;
@@
-863,10
+886,11
@@
bo_unreference_final(struct iris_bo *bo, time_t time)
DBG("bo_unreference final: %d (%s)\n", bo->gem_handle, bo->name);
DBG("bo_unreference final: %d (%s)\n", bo->gem_handle, bo->name);
- bucket = bucket_for_size(bufmgr, bo->size);
+ bucket = NULL;
+ if (bo->reusable)
+ bucket = bucket_for_size(bufmgr, bo->size);
/* Put the buffer into our internal cache for reuse if we can. */
/* Put the buffer into our internal cache for reuse if we can. */
- if (bufmgr->bo_reuse && bo->reusable && bucket != NULL &&
- iris_bo_madvise(bo, I915_MADV_DONTNEED)) {
+ if (bucket && iris_bo_madvise(bo, I915_MADV_DONTNEED)) {
bo->free_time = time;
bo->name = NULL;
bo->free_time = time;
bo->name = NULL;
@@
-1267,7
+1291,8
@@
iris_bufmgr_destroy(struct iris_bufmgr *bufmgr)
_mesa_hash_table_destroy(bufmgr->handle_table, NULL);
for (int z = 0; z < IRIS_MEMZONE_COUNT; z++) {
_mesa_hash_table_destroy(bufmgr->handle_table, NULL);
for (int z = 0; z < IRIS_MEMZONE_COUNT; z++) {
- util_vma_heap_finish(&bufmgr->vma_allocator[z]);
+ if (z != IRIS_MEMZONE_BINDER)
+ util_vma_heap_finish(&bufmgr->vma_allocator[z]);
}
free(bufmgr);
}
free(bufmgr);
@@
-1341,7
+1366,7
@@
iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd)
goto out;
}
goto out;
}
- bo =
calloc(1, sizeof(*bo)
);
+ bo =
bo_calloc(
);
if (!bo)
goto out;
if (!bo)
goto out;
@@
-1385,19
+1410,26
@@
err:
return NULL;
}
return NULL;
}
+static void
+iris_bo_make_external_locked(struct iris_bo *bo)
+{
+ if (!bo->external) {
+ _mesa_hash_table_insert(bo->bufmgr->handle_table, &bo->gem_handle, bo);
+ bo->external = true;
+ }
+}
+
static void
iris_bo_make_external(struct iris_bo *bo)
{
struct iris_bufmgr *bufmgr = bo->bufmgr;
static void
iris_bo_make_external(struct iris_bo *bo)
{
struct iris_bufmgr *bufmgr = bo->bufmgr;
- if (!bo->external) {
- mtx_lock(&bufmgr->lock);
- if (!bo->external) {
- _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
- bo->external = true;
- }
- mtx_unlock(&bufmgr->lock);
- }
+ if (bo->external)
+ return;
+
+ mtx_lock(&bufmgr->lock);
+ iris_bo_make_external_locked(bo);
+ mtx_unlock(&bufmgr->lock);
}
int
}
int
@@
-1435,9
+1467,9
@@
iris_bo_flink(struct iris_bo *bo, uint32_t *name)
if (drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink))
return -errno;
if (drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink))
return -errno;
- iris_bo_make_external(bo);
mtx_lock(&bufmgr->lock);
if (!bo->global_name) {
mtx_lock(&bufmgr->lock);
if (!bo->global_name) {
+ iris_bo_make_external_locked(bo);
bo->global_name = flink.name;
_mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo);
}
bo->global_name = flink.name;
_mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo);
}
@@
-1586,8
+1618,8
@@
iris_bufmgr_init(struct gen_device_info *devinfo, int fd)
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SHADER],
PAGE_SIZE, _4GB);
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SURFACE],
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SHADER],
PAGE_SIZE, _4GB);
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SURFACE],
- IRIS_MEMZONE_SURFACE_START
+ IRIS_BINDER_SIZE
,
- _4GB - IRIS_BINDER_SIZE);
+ IRIS_MEMZONE_SURFACE_START,
+ _4GB - IRIS_
MAX_BINDERS * IRIS_
BINDER_SIZE);
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_DYNAMIC],
IRIS_MEMZONE_DYNAMIC_START + IRIS_BORDER_COLOR_POOL_SIZE,
_4GB - IRIS_BORDER_COLOR_POOL_SIZE);
util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_DYNAMIC],
IRIS_MEMZONE_DYNAMIC_START + IRIS_BORDER_COLOR_POOL_SIZE,
_4GB - IRIS_BORDER_COLOR_POOL_SIZE);