X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Firis%2Firis_bufmgr.c;h=92ede93405622051bcbb4e90d5a5e4e0489383dc;hb=f879349398cbb46fbe150049db18018c3635842c;hp=3a43371b00d602d0ec1b23ed4afad1d3e8524509;hpb=5911fb8801cb0c7962c83d69c0b477a634523f27;p=mesa.git diff --git a/src/gallium/drivers/iris/iris_bufmgr.c b/src/gallium/drivers/iris/iris_bufmgr.c index 3a43371b00d..92ede934056 100644 --- a/src/gallium/drivers/iris/iris_bufmgr.c +++ b/src/gallium/drivers/iris/iris_bufmgr.c @@ -8,17 +8,16 @@ * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. */ /** @@ -245,10 +244,10 @@ bucket_for_size(struct iris_bufmgr *bufmgr, uint64_t size) static enum iris_memory_zone memzone_for_address(uint64_t address) { - STATIC_ASSERT(IRIS_MEMZONE_OTHER_START > IRIS_MEMZONE_DYNAMIC_START); + STATIC_ASSERT(IRIS_MEMZONE_OTHER_START > IRIS_MEMZONE_DYNAMIC_START); STATIC_ASSERT(IRIS_MEMZONE_DYNAMIC_START > IRIS_MEMZONE_SURFACE_START); - STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START > IRIS_MEMZONE_SHADER_START); - STATIC_ASSERT(IRIS_BINDER_ADDRESS == IRIS_MEMZONE_SURFACE_START); + STATIC_ASSERT(IRIS_MEMZONE_SURFACE_START > IRIS_MEMZONE_BINDER_START); + STATIC_ASSERT(IRIS_MEMZONE_BINDER_START > IRIS_MEMZONE_SHADER_START); STATIC_ASSERT(IRIS_BORDER_COLOR_POOL_ADDRESS == IRIS_MEMZONE_DYNAMIC_START); if (address >= IRIS_MEMZONE_OTHER_START) @@ -260,12 +259,12 @@ memzone_for_address(uint64_t address) if (address > IRIS_MEMZONE_DYNAMIC_START) return IRIS_MEMZONE_DYNAMIC; - if (address == IRIS_BINDER_ADDRESS) - return IRIS_MEMZONE_BINDER; - - if (address > IRIS_MEMZONE_SURFACE_START) + if (address >= IRIS_MEMZONE_SURFACE_START) return IRIS_MEMZONE_SURFACE; + if (address >= IRIS_MEMZONE_BINDER_START) + return IRIS_MEMZONE_BINDER; + return IRIS_MEMZONE_SHADER; } @@ -366,7 +365,9 @@ bucket_vma_free(struct bo_cache_bucket *bucket, uint64_t address) } static struct bo_cache_bucket * -get_bucket_allocator(struct iris_bufmgr *bufmgr, uint64_t size) +get_bucket_allocator(struct iris_bufmgr *bufmgr, + enum iris_memory_zone memzone, + uint64_t size) { /* Skip using the bucket allocator for very large sizes, as it allocates * 64 of them and this can balloon rather quickly. @@ -394,12 +395,15 @@ vma_alloc(struct iris_bufmgr *bufmgr, uint64_t size, uint64_t alignment) { - if (memzone == IRIS_MEMZONE_BINDER) - return IRIS_BINDER_ADDRESS; - else if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL) + if (memzone == IRIS_MEMZONE_BORDER_COLOR_POOL) return IRIS_BORDER_COLOR_POOL_ADDRESS; - struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size); + /* The binder handles its own allocations. Return non-zero here. */ + if (memzone == IRIS_MEMZONE_BINDER) + return IRIS_MEMZONE_BINDER_START; + + struct bo_cache_bucket *bucket = + get_bucket_allocator(bufmgr, memzone, size); uint64_t addr; if (bucket) { @@ -420,8 +424,7 @@ vma_free(struct iris_bufmgr *bufmgr, uint64_t address, uint64_t size) { - if (address == IRIS_BINDER_ADDRESS || - address == IRIS_BORDER_COLOR_POOL_ADDRESS) + if (address == IRIS_BORDER_COLOR_POOL_ADDRESS) return; /* Un-canonicalize the address. */ @@ -430,12 +433,18 @@ vma_free(struct iris_bufmgr *bufmgr, if (address == 0ull) return; - struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size); + enum iris_memory_zone memzone = memzone_for_address(address); + + /* The binder handles its own allocations. */ + if (memzone == IRIS_MEMZONE_BINDER) + return; + + struct bo_cache_bucket *bucket = + get_bucket_allocator(bufmgr, memzone, size); if (bucket) { bucket_vma_free(bucket, address); } else { - enum iris_memory_zone memzone = memzone_for_address(address); util_vma_heap_free(&bufmgr->vma_allocator[memzone], address, size); } } @@ -482,6 +491,16 @@ iris_bo_cache_purge_bucket(struct iris_bufmgr *bufmgr, } } +static struct iris_bo * +bo_calloc(void) +{ + struct iris_bo *bo = calloc(1, sizeof(*bo)); + if (bo) { + bo->hash = _mesa_hash_pointer(bo); + } + return bo; +} + static struct iris_bo * bo_alloc_internal(struct iris_bufmgr *bufmgr, const char *name, @@ -509,9 +528,7 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr, * allocation up. */ if (bucket == NULL) { - bo_size = size; - if (bo_size < page_size) - bo_size = page_size; + bo_size = MAX2(ALIGN(size, page_size), page_size); } else { bo_size = bucket->size; } @@ -558,11 +575,11 @@ retry: * memory and assign it a new address. */ if (memzone != memzone_for_address(bo->gtt_offset)) { - vma_free(bufmgr, bo->gtt_offset, bo_size); + vma_free(bufmgr, bo->gtt_offset, bo->size); bo->gtt_offset = 0ull; } } else { - bo = calloc(1, sizeof(*bo)); + bo = bo_calloc(); if (!bo) goto err; @@ -607,11 +624,17 @@ retry: bo->name = name; p_atomic_set(&bo->refcount, 1); - bo->reusable = true; + bo->reusable = bucket && bufmgr->bo_reuse; bo->cache_coherent = bufmgr->has_llc; bo->index = -1; bo->kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED; + /* By default, capture all driver-internal buffers like shader kernels, + * surface states, dynamic states, border colors, and so on. + */ + if (memzone < IRIS_MEMZONE_OTHER) + bo->kflags |= EXEC_OBJECT_CAPTURE; + if (bo->gtt_offset == 0ull) { bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, 1); @@ -652,6 +675,58 @@ iris_bo_alloc_tiled(struct iris_bufmgr *bufmgr, const char *name, flags, tiling_mode, pitch); } +struct iris_bo * +iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name, + void *ptr, size_t size, + enum iris_memory_zone memzone) +{ + struct iris_bo *bo; + + bo = bo_calloc(); + if (!bo) + return NULL; + + struct drm_i915_gem_userptr arg = { + .user_ptr = (uintptr_t)ptr, + .user_size = size, + }; + if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_USERPTR, &arg)) + goto err_free; + bo->gem_handle = arg.handle; + + /* Check the buffer for validity before we try and use it in a batch */ + struct drm_i915_gem_set_domain sd = { + .handle = bo->gem_handle, + .read_domains = I915_GEM_DOMAIN_CPU, + }; + if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd)) + goto err_close; + + bo->name = name; + bo->size = size; + bo->map_cpu = ptr; + + bo->bufmgr = bufmgr; + bo->kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED; + bo->gtt_offset = vma_alloc(bufmgr, memzone, size, 1); + if (bo->gtt_offset == 0ull) + goto err_close; + + p_atomic_set(&bo->refcount, 1); + bo->userptr = true; + bo->cache_coherent = true; + bo->index = -1; + bo->idle = true; + + return bo; + +err_close: + drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &bo->gem_handle); +err_free: + free(bo); + return NULL; +} + /** * Returns a iris_bo wrapping the given buffer object handle. * @@ -695,7 +770,7 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr, goto out; } - bo = calloc(1, sizeof(*bo)); + bo = bo_calloc(); if (!bo) goto out; @@ -740,7 +815,7 @@ bo_free(struct iris_bo *bo) { struct iris_bufmgr *bufmgr = bo->bufmgr; - if (bo->map_cpu) { + if (bo->map_cpu && !bo->userptr) { VG_NOACCESS(bo->map_cpu, bo->size); munmap(bo->map_cpu, bo->size); } @@ -811,10 +886,11 @@ bo_unreference_final(struct iris_bo *bo, time_t time) DBG("bo_unreference final: %d (%s)\n", bo->gem_handle, bo->name); - bucket = bucket_for_size(bufmgr, bo->size); + bucket = NULL; + if (bo->reusable) + bucket = bucket_for_size(bufmgr, bo->size); /* Put the buffer into our internal cache for reuse if we can. */ - if (bufmgr->bo_reuse && bo->reusable && bucket != NULL && - iris_bo_madvise(bo, I915_MADV_DONTNEED)) { + if (bucket && iris_bo_madvise(bo, I915_MADV_DONTNEED)) { bo->free_time = time; bo->name = NULL; @@ -907,7 +983,6 @@ iris_bo_map_cpu(struct pipe_debug_callback *dbg, }; int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg); if (ret != 0) { - ret = -errno; DBG("%s:%d: Error mapping buffer %d (%s): %s .\n", __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno)); return NULL; @@ -969,7 +1044,6 @@ iris_bo_map_wc(struct pipe_debug_callback *dbg, }; int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg); if (ret != 0) { - ret = -errno; DBG("%s:%d: Error mapping buffer %d (%s): %s .\n", __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno)); return NULL; @@ -1135,30 +1209,6 @@ iris_bo_map(struct pipe_debug_callback *dbg, return map; } -int -iris_bo_subdata(struct iris_bo *bo, uint64_t offset, - uint64_t size, const void *data) -{ - struct iris_bufmgr *bufmgr = bo->bufmgr; - - struct drm_i915_gem_pwrite pwrite = { - .handle = bo->gem_handle, - .offset = offset, - .size = size, - .data_ptr = (uint64_t) (uintptr_t) data, - }; - - int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_PWRITE, &pwrite); - if (ret != 0) { - ret = -errno; - DBG("%s:%d: Error writing data to buffer %d: " - "(%"PRIu64" %"PRIu64") %s .\n", - __FILE__, __LINE__, bo->gem_handle, offset, size, strerror(errno)); - } - - return ret; -} - /** Waits for all GPU rendering with the object to have completed. */ void iris_bo_wait_rendering(struct iris_bo *bo) @@ -1241,7 +1291,8 @@ iris_bufmgr_destroy(struct iris_bufmgr *bufmgr) _mesa_hash_table_destroy(bufmgr->handle_table, NULL); for (int z = 0; z < IRIS_MEMZONE_COUNT; z++) { - util_vma_heap_finish(&bufmgr->vma_allocator[z]); + if (z != IRIS_MEMZONE_BINDER) + util_vma_heap_finish(&bufmgr->vma_allocator[z]); } free(bufmgr); @@ -1315,7 +1366,7 @@ iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd) goto out; } - bo = calloc(1, sizeof(*bo)); + bo = bo_calloc(); if (!bo) goto out; @@ -1359,19 +1410,26 @@ err: return NULL; } +static void +iris_bo_make_external_locked(struct iris_bo *bo) +{ + if (!bo->external) { + _mesa_hash_table_insert(bo->bufmgr->handle_table, &bo->gem_handle, bo); + bo->external = true; + } +} + static void iris_bo_make_external(struct iris_bo *bo) { struct iris_bufmgr *bufmgr = bo->bufmgr; - if (!bo->external) { - mtx_lock(&bufmgr->lock); - if (!bo->external) { - _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo); - bo->external = true; - } - mtx_unlock(&bufmgr->lock); - } + if (bo->external) + return; + + mtx_lock(&bufmgr->lock); + iris_bo_make_external_locked(bo); + mtx_unlock(&bufmgr->lock); } int @@ -1409,9 +1467,9 @@ iris_bo_flink(struct iris_bo *bo, uint32_t *name) if (drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink)) return -errno; - iris_bo_make_external(bo); mtx_lock(&bufmgr->lock); if (!bo->global_name) { + iris_bo_make_external_locked(bo); bo->global_name = flink.name; _mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo); } @@ -1560,8 +1618,8 @@ iris_bufmgr_init(struct gen_device_info *devinfo, int fd) util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SHADER], PAGE_SIZE, _4GB); util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SURFACE], - IRIS_MEMZONE_SURFACE_START + IRIS_BINDER_SIZE, - _4GB - IRIS_BINDER_SIZE); + IRIS_MEMZONE_SURFACE_START, + _4GB - IRIS_MAX_BINDERS * IRIS_BINDER_SIZE); util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_DYNAMIC], IRIS_MEMZONE_DYNAMIC_START + IRIS_BORDER_COLOR_POOL_SIZE, _4GB - IRIS_BORDER_COLOR_POOL_SIZE);