src/gallium/drivers/iris/iris_bufmgr.h

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #ifndef IRIS_BUFMGR_H
  25 #define IRIS_BUFMGR_H
  26
  27 #include <stdbool.h>
  28 #include <stdint.h>
  29 #include <stdio.h>
  30 #include <sys/types.h>
  31 #include "c11/threads.h"
  32 #include "util/macros.h"
  33 #include "util/u_atomic.h"
  34 #include "util/list.h"
  35 #include "pipe/p_defines.h"
  36
  37 struct iris_batch;
  38 struct gen_device_info;
  39 struct pipe_debug_callback;
  40
  41 /**
  42  * Memory zones.  When allocating a buffer, you can request that it is
  43  * placed into a specific region of the virtual address space (PPGTT).
  44  *
  45  * Most buffers can go anywhere (IRIS_MEMZONE_OTHER).  Some buffers are
  46  * accessed via an offset from a base address.  STATE_BASE_ADDRESS has
  47  * a maximum 4GB size for each region, so we need to restrict those
  48  * buffers to be within 4GB of the base.  Each memory zone corresponds
  49  * to a particular base address.
  50  *
  51  * We lay out the virtual address space as follows:
  52  *
  53  * - [0,   4K): Nothing            (empty page for null address)
  54  * - [4K,  4G): Shaders            (Instruction Base Address)
  55  * - [4G,  8G): Surfaces & Binders (Surface State Base Address, Bindless ...)
  56  * - [8G, 12G): Dynamic            (Dynamic State Base Address)
  57  * - [12G, *):  Other              (everything else in the full 48-bit VMA)
  58  *
  59  * A special buffer for border color lives at the start of the dynamic state
  60  * memory zone.  This unfortunately has to be handled specially because the
  61  * SAMPLER_STATE "Indirect State Pointer" field is only a 24-bit pointer.
  62  *
  63  * Each GL context uses a separate GEM context, which technically gives them
  64  * each a separate VMA.  However, we assign address globally, so buffers will
  65  * have the same address in all GEM contexts.  This lets us have a single BO
  66  * field for the address, which is easy and cheap.
  67  */
  68 enum iris_memory_zone {
  69    IRIS_MEMZONE_SHADER,
  70    IRIS_MEMZONE_BINDER,
  71    IRIS_MEMZONE_SURFACE,
  72    IRIS_MEMZONE_DYNAMIC,
  73    IRIS_MEMZONE_OTHER,
  74
  75    IRIS_MEMZONE_BORDER_COLOR_POOL,
  76 };
  77
  78 /* Intentionally exclude single buffer "zones" */
  79 #define IRIS_MEMZONE_COUNT (IRIS_MEMZONE_OTHER + 1)
  80
  81 #define IRIS_BINDER_SIZE (64 * 1024)
  82 #define IRIS_MAX_BINDERS 100
  83
  84 #define IRIS_MEMZONE_SHADER_START     (0ull * (1ull << 32))
  85 #define IRIS_MEMZONE_BINDER_START     (1ull * (1ull << 32))
  86 #define IRIS_MEMZONE_SURFACE_START    (IRIS_MEMZONE_BINDER_START + IRIS_MAX_BINDERS * IRIS_BINDER_SIZE)
  87 #define IRIS_MEMZONE_DYNAMIC_START    (2ull * (1ull << 32))
  88 #define IRIS_MEMZONE_OTHER_START      (3ull * (1ull << 32))
  89
  90 #define IRIS_BORDER_COLOR_POOL_ADDRESS IRIS_MEMZONE_DYNAMIC_START
  91 #define IRIS_BORDER_COLOR_POOL_SIZE (64 * 1024)
  92
  93 /**
  94  * Classification of the various incoherent caches of the GPU into a number of
  95  * caching domains.
  96  */
  97 enum iris_domain {
  98    /** Render color cache. */
  99    IRIS_DOMAIN_RENDER_WRITE = 0,
 100    /** (Hi)Z/stencil cache. */
 101    IRIS_DOMAIN_DEPTH_WRITE,
 102    /** Any other read-write cache. */
 103    IRIS_DOMAIN_OTHER_WRITE,
 104    /** Any other read-only cache. */
 105    IRIS_DOMAIN_OTHER_READ,
 106    /** Number of caching domains. */
 107    NUM_IRIS_DOMAINS,
 108    /** Not a real cache, use to opt out of the cache tracking mechanism. */
 109    IRIS_DOMAIN_NONE = NUM_IRIS_DOMAINS
 110 };
 111
 112 /**
 113  * Whether a caching domain is guaranteed not to write any data to memory.
 114  */
 115 static inline bool
 116 iris_domain_is_read_only(enum iris_domain access)
 117 {
 118    return access == IRIS_DOMAIN_OTHER_READ;
 119 }
 120
 121 struct iris_bo {
 122    /**
 123     * Size in bytes of the buffer object.
 124     *
 125     * The size may be larger than the size originally requested for the
 126     * allocation, such as being aligned to page size.
 127     */
 128    uint64_t size;
 129
 130    /** Buffer manager context associated with this buffer object */
 131    struct iris_bufmgr *bufmgr;
 132
 133    /** Pre-computed hash using _mesa_hash_pointer for cache tracking sets */
 134    uint32_t hash;
 135
 136    /** The GEM handle for this buffer object. */
 137    uint32_t gem_handle;
 138
 139    /**
 140     * Virtual address of the buffer inside the PPGTT (Per-Process Graphics
 141     * Translation Table).
 142     *
 143     * Although each hardware context has its own VMA, we assign BO's to the
 144     * same address in all contexts, for simplicity.
 145     */
 146    uint64_t gtt_offset;
 147
 148    /**
 149     * If non-zero, then this bo has an aux-map translation to this address.
 150     */
 151    uint64_t aux_map_address;
 152
 153    /**
 154     * The validation list index for this buffer, or -1 when not in a batch.
 155     * Note that a single buffer may be in multiple batches (contexts), and
 156     * this is a global field, which refers to the last batch using the BO.
 157     * It should not be considered authoritative, but can be used to avoid a
 158     * linear walk of the validation list in the common case by guessing that
 159     * exec_bos[bo->index] == bo and confirming whether that's the case.
 160     *
 161     * XXX: this is not ideal now that we have more than one batch per context,
 162     * XXX: as the index will flop back and forth between the render index and
 163     * XXX: compute index...
 164     */
 165    unsigned index;
 166
 167    int refcount;
 168    const char *name;
 169
 170    uint64_t kflags;
 171
 172    /**
 173     * Kenel-assigned global name for this object
 174     *
 175     * List contains both flink named and prime fd'd objects
 176     */
 177    unsigned global_name;
 178
 179    /**
 180     * Current tiling mode
 181     */
 182    uint32_t tiling_mode;
 183    uint32_t stride;
 184
 185    time_t free_time;
 186
 187    /** Mapped address for the buffer, saved across map/unmap cycles */
 188    void *map_cpu;
 189    /** GTT virtual address for the buffer, saved across map/unmap cycles */
 190    void *map_gtt;
 191    /** WC CPU address for the buffer, saved across map/unmap cycles */
 192    void *map_wc;
 193
 194    /** BO cache list */
 195    struct list_head head;
 196
 197    /** List of GEM handle exports of this buffer (bo_export) */
 198    struct list_head exports;
 199
 200    /**
 201     * Synchronization sequence number of most recent access of this BO from
 202     * each caching domain.
 203     *
 204     * Although this is a global field, use in multiple contexts should be
 205     * safe, see iris_emit_buffer_barrier_for() for details.
 206     *
 207     * Also align it to 64 bits. This will make atomic operations faster on 32
 208     * bit platforms.
 209     */
 210    uint64_t last_seqnos[NUM_IRIS_DOMAINS] __attribute__ ((aligned (8)));
 211
 212    /**
 213     * Boolean of whether the GPU is definitely not accessing the buffer.
 214     *
 215     * This is only valid when reusable, since non-reusable
 216     * buffers are those that have been shared with other
 217     * processes, so we don't know their state.
 218     */
 219    bool idle;
 220
 221    /**
 222     * Boolean of whether this buffer can be re-used
 223     */
 224    bool reusable;
 225
 226    /**
 227     * Boolean of whether this buffer has been shared with an external client.
 228     */
 229    bool external;
 230
 231    /**
 232     * Boolean of whether this buffer is cache coherent
 233     */
 234    bool cache_coherent;
 235
 236    /**
 237     * Boolean of whether this buffer points into user memory
 238     */
 239    bool userptr;
 240 };
 241
 242 #define BO_ALLOC_ZEROED     (1<<0)
 243 #define BO_ALLOC_COHERENT   (1<<1)
 244
 245 /**
 246  * Allocate a buffer object.
 247  *
 248  * Buffer objects are not necessarily initially mapped into CPU virtual
 249  * address space or graphics device aperture.  They must be mapped
 250  * using iris_bo_map() to be used by the CPU.
 251  */
 252 struct iris_bo *iris_bo_alloc(struct iris_bufmgr *bufmgr,
 253                               const char *name,
 254                               uint64_t size,
 255                               enum iris_memory_zone memzone);
 256
 257 /**
 258  * Allocate a tiled buffer object.
 259  *
 260  * Alignment for tiled objects is set automatically; the 'flags'
 261  * argument provides a hint about how the object will be used initially.
 262  *
 263  * Valid tiling formats are:
 264  *  I915_TILING_NONE
 265  *  I915_TILING_X
 266  *  I915_TILING_Y
 267  */
 268 struct iris_bo *iris_bo_alloc_tiled(struct iris_bufmgr *bufmgr,
 269                                     const char *name,
 270                                     uint64_t size,
 271                                     uint32_t alignment,
 272                                     enum iris_memory_zone memzone,
 273                                     uint32_t tiling_mode,
 274                                     uint32_t pitch,
 275                                     unsigned flags);
 276
 277 struct iris_bo *
 278 iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
 279                        void *ptr, size_t size,
 280                        enum iris_memory_zone memzone);
 281
 282 /** Takes a reference on a buffer object */
 283 static inline void
 284 iris_bo_reference(struct iris_bo *bo)
 285 {
 286    p_atomic_inc(&bo->refcount);
 287 }
 288
 289 /**
 290  * Releases a reference on a buffer object, freeing the data if
 291  * no references remain.
 292  */
 293 void iris_bo_unreference(struct iris_bo *bo);
 294
 295 #define MAP_READ          PIPE_TRANSFER_READ
 296 #define MAP_WRITE         PIPE_TRANSFER_WRITE
 297 #define MAP_ASYNC         PIPE_TRANSFER_UNSYNCHRONIZED
 298 #define MAP_PERSISTENT    PIPE_TRANSFER_PERSISTENT
 299 #define MAP_COHERENT      PIPE_TRANSFER_COHERENT
 300 /* internal */
 301 #define MAP_INTERNAL_MASK (0xffu << 24)
 302 #define MAP_RAW           (0x01 << 24)
 303
 304 #define MAP_FLAGS         (MAP_READ | MAP_WRITE | MAP_ASYNC | \
 305                            MAP_PERSISTENT | MAP_COHERENT | MAP_INTERNAL_MASK)
 306
 307 /**
 308  * Maps the buffer into userspace.
 309  *
 310  * This function will block waiting for any existing execution on the
 311  * buffer to complete, first.  The resulting mapping is returned.
 312  */
 313 MUST_CHECK void *iris_bo_map(struct pipe_debug_callback *dbg,
 314                              struct iris_bo *bo, unsigned flags);
 315
 316 /**
 317  * Reduces the refcount on the userspace mapping of the buffer
 318  * object.
 319  */
 320 static inline int iris_bo_unmap(struct iris_bo *bo) { return 0; }
 321
 322 /**
 323  * Waits for rendering to an object by the GPU to have completed.
 324  *
 325  * This is not required for any access to the BO by bo_map,
 326  * bo_subdata, etc.  It is merely a way for the driver to implement
 327  * glFinish.
 328  */
 329 void iris_bo_wait_rendering(struct iris_bo *bo);
 330
 331
 332 /**
 333  * Unref a buffer manager instance.
 334  */
 335 void iris_bufmgr_unref(struct iris_bufmgr *bufmgr);
 336
 337 /**
 338  * Create a visible name for a buffer which can be used by other apps
 339  *
 340  * \param buf Buffer to create a name for
 341  * \param name Returned name
 342  */
 343 int iris_bo_flink(struct iris_bo *bo, uint32_t *name);
 344
 345 /**
 346  * Make a BO externally accessible.
 347  *
 348  * \param bo Buffer to make external
 349  */
 350 void iris_bo_make_external(struct iris_bo *bo);
 351
 352 /**
 353  * Returns 1 if mapping the buffer for write could cause the process
 354  * to block, due to the object being active in the GPU.
 355  */
 356 int iris_bo_busy(struct iris_bo *bo);
 357
 358 /**
 359  * Specify the volatility of the buffer.
 360  * \param bo Buffer to create a name for
 361  * \param madv The purgeable status
 362  *
 363  * Use I915_MADV_DONTNEED to mark the buffer as purgeable, and it will be
 364  * reclaimed under memory pressure. If you subsequently require the buffer,
 365  * then you must pass I915_MADV_WILLNEED to mark the buffer as required.
 366  *
 367  * Returns 1 if the buffer was retained, or 0 if it was discarded whilst
 368  * marked as I915_MADV_DONTNEED.
 369  */
 370 int iris_bo_madvise(struct iris_bo *bo, int madv);
 371
 372 /* drm_bacon_bufmgr_gem.c */
 373 struct iris_bufmgr *iris_bufmgr_get_for_fd(struct gen_device_info *devinfo, int fd,
 374                                            bool bo_reuse);
 375 int iris_bufmgr_get_fd(struct iris_bufmgr *bufmgr);
 376
 377 struct iris_bo *iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
 378                                              const char *name,
 379                                              unsigned handle);
 380
 381 void* iris_bufmgr_get_aux_map_context(struct iris_bufmgr *bufmgr);
 382
 383 int iris_bo_wait(struct iris_bo *bo, int64_t timeout_ns);
 384
 385 uint32_t iris_create_hw_context(struct iris_bufmgr *bufmgr);
 386 uint32_t iris_clone_hw_context(struct iris_bufmgr *bufmgr, uint32_t ctx_id);
 387
 388 #define IRIS_CONTEXT_LOW_PRIORITY    ((I915_CONTEXT_MIN_USER_PRIORITY-1)/2)
 389 #define IRIS_CONTEXT_MEDIUM_PRIORITY (I915_CONTEXT_DEFAULT_PRIORITY)
 390 #define IRIS_CONTEXT_HIGH_PRIORITY   ((I915_CONTEXT_MAX_USER_PRIORITY+1)/2)
 391
 392 int iris_hw_context_set_priority(struct iris_bufmgr *bufmgr,
 393                                  uint32_t ctx_id, int priority);
 394
 395 void iris_destroy_hw_context(struct iris_bufmgr *bufmgr, uint32_t ctx_id);
 396
 397 int iris_bo_export_dmabuf(struct iris_bo *bo, int *prime_fd);
 398 struct iris_bo *iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd,
 399                                       int tiling);
 400
 401 /**
 402  * Exports a bo as a GEM handle into a given DRM file descriptor
 403  * \param bo Buffer to export
 404  * \param drm_fd File descriptor where the new handle is created
 405  * \param out_handle Pointer to store the new handle
 406  *
 407  * Returns 0 if the buffer was successfully exported, a non zero error code
 408  * otherwise.
 409  */
 410 int iris_bo_export_gem_handle_for_device(struct iris_bo *bo, int drm_fd,
 411                                          uint32_t *out_handle);
 412
 413 uint32_t iris_bo_export_gem_handle(struct iris_bo *bo);
 414
 415 int iris_reg_read(struct iris_bufmgr *bufmgr, uint32_t offset, uint64_t *out);
 416
 417 int drm_ioctl(int fd, unsigned long request, void *arg);
 418
 419 /**
 420  * Returns the BO's address relative to the appropriate base address.
 421  *
 422  * All of our base addresses are programmed to the start of a 4GB region,
 423  * so simply returning the bottom 32 bits of the BO address will give us
 424  * the offset from whatever base address corresponds to that memory region.
 425  */
 426 static inline uint32_t
 427 iris_bo_offset_from_base_address(struct iris_bo *bo)
 428 {
 429    /* This only works for buffers in the memory zones corresponding to a
 430     * base address - the top, unbounded memory zone doesn't have a base.
 431     */
 432    assert(bo->gtt_offset < IRIS_MEMZONE_OTHER_START);
 433    return bo->gtt_offset;
 434 }
 435
 436 /**
 437  * Track access of a BO from the specified caching domain and sequence number.
 438  *
 439  * Can be used without locking.  Only the most recent access (i.e. highest
 440  * seqno) is tracked.
 441  */
 442 static inline void
 443 iris_bo_bump_seqno(struct iris_bo *bo, uint64_t seqno,
 444                    enum iris_domain type)
 445 {
 446    uint64_t *const last_seqno = &bo->last_seqnos[type];
 447    uint64_t tmp, prev_seqno = p_atomic_read(last_seqno);
 448
 449    while (prev_seqno < seqno &&
 450           prev_seqno != (tmp = p_atomic_cmpxchg(last_seqno, prev_seqno, seqno)))
 451       prev_seqno = tmp;
 452 }
 453
 454 enum iris_memory_zone iris_memzone_for_address(uint64_t address);
 455
 456 #endif /* IRIS_BUFMGR_H */