src/gallium/winsys/amdgpu/drm/amdgpu_cs.h

   1 /*
   2  * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
   3  * Copyright © 2015 Advanced Micro Devices, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  15  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  16  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  17  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  18  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * The above copyright notice and this permission notice (including the
  24  * next paragraph) shall be included in all copies or substantial portions
  25  * of the Software.
  26  */
  27
  28 #ifndef AMDGPU_CS_H
  29 #define AMDGPU_CS_H
  30
  31 #include "amdgpu_bo.h"
  32 #include "util/u_memory.h"
  33 #include "drm-uapi/amdgpu_drm.h"
  34
  35 struct amdgpu_ctx {
  36    struct amdgpu_winsys *ws;
  37    amdgpu_context_handle ctx;
  38    amdgpu_bo_handle user_fence_bo;
  39    uint64_t *user_fence_cpu_address_base;
  40    int refcount;
  41    unsigned initial_num_total_rejected_cs;
  42    unsigned num_rejected_cs;
  43 };
  44
  45 struct amdgpu_cs_buffer {
  46    struct amdgpu_winsys_bo *bo;
  47    union {
  48       struct {
  49          uint32_t priority_usage;
  50       } real;
  51       struct {
  52          uint32_t real_idx; /* index of underlying real BO */
  53       } slab;
  54    } u;
  55    enum radeon_bo_usage usage;
  56 };
  57
  58 enum ib_type {
  59    IB_MAIN,
  60    IB_PARALLEL_COMPUTE,
  61    IB_NUM,
  62 };
  63
  64 struct amdgpu_ib {
  65    struct radeon_cmdbuf base;
  66
  67    /* A buffer out of which new IBs are allocated. */
  68    struct pb_buffer        *big_ib_buffer;
  69    uint8_t                 *ib_mapped;
  70    unsigned                used_ib_space;
  71
  72    /* The maximum seen size from cs_check_space. If the driver does
  73     * cs_check_space and flush, the newly allocated IB should have at least
  74     * this size.
  75     */
  76    unsigned                max_check_space_size;
  77
  78    unsigned                max_ib_size;
  79    uint32_t                *ptr_ib_size;
  80    bool                    ptr_ib_size_inside_ib;
  81    enum ib_type            ib_type;
  82 };
  83
  84 struct amdgpu_fence_list {
  85    struct pipe_fence_handle    **list;
  86    unsigned                    num;
  87    unsigned                    max;
  88 };
  89
  90 struct amdgpu_cs_context {
  91    struct drm_amdgpu_cs_chunk_ib ib[IB_NUM];
  92
  93    /* Buffers. */
  94    unsigned                    max_real_buffers;
  95    unsigned                    num_real_buffers;
  96    struct amdgpu_cs_buffer     *real_buffers;
  97
  98    unsigned                    num_slab_buffers;
  99    unsigned                    max_slab_buffers;
 100    struct amdgpu_cs_buffer     *slab_buffers;
 101
 102    unsigned                    num_sparse_buffers;
 103    unsigned                    max_sparse_buffers;
 104    struct amdgpu_cs_buffer     *sparse_buffers;
 105
 106    int                         buffer_indices_hashlist[4096];
 107
 108    struct amdgpu_winsys_bo     *last_added_bo;
 109    unsigned                    last_added_bo_index;
 110    unsigned                    last_added_bo_usage;
 111    uint32_t                    last_added_bo_priority_usage;
 112
 113    struct amdgpu_fence_list    fence_dependencies;
 114    struct amdgpu_fence_list    syncobj_dependencies;
 115    struct amdgpu_fence_list    syncobj_to_signal;
 116
 117    /* The compute IB uses the dependencies above + these: */
 118    struct amdgpu_fence_list    compute_fence_dependencies;
 119    struct amdgpu_fence_list    compute_start_fence_dependencies;
 120
 121    struct pipe_fence_handle    *fence;
 122
 123    /* the error returned from cs_flush for non-async submissions */
 124    int                         error_code;
 125
 126    /* TMZ: will this command be submitted using the TMZ flag */
 127    bool secure;
 128 };
 129
 130 struct amdgpu_cs {
 131    struct amdgpu_ib main; /* must be first because this is inherited */
 132    struct amdgpu_ib compute_ib; /* optional parallel compute IB */
 133    struct amdgpu_ctx *ctx;
 134    enum ring_type ring_type;
 135    struct drm_amdgpu_cs_chunk_fence fence_chunk;
 136
 137    /* We flip between these two CS. While one is being consumed
 138     * by the kernel in another thread, the other one is being filled
 139     * by the pipe driver. */
 140    struct amdgpu_cs_context csc1;
 141    struct amdgpu_cs_context csc2;
 142    /* The currently-used CS. */
 143    struct amdgpu_cs_context *csc;
 144    /* The CS being currently-owned by the other thread. */
 145    struct amdgpu_cs_context *cst;
 146
 147    /* Flush CS. */
 148    void (*flush_cs)(void *ctx, unsigned flags, struct pipe_fence_handle **fence);
 149    void *flush_data;
 150    bool stop_exec_on_failure;
 151
 152    struct util_queue_fence flush_completed;
 153    struct pipe_fence_handle *next_fence;
 154 };
 155
 156 struct amdgpu_fence {
 157    struct pipe_reference reference;
 158    /* If ctx == NULL, this fence is syncobj-based. */
 159    uint32_t syncobj;
 160
 161    struct amdgpu_winsys *ws;
 162    struct amdgpu_ctx *ctx;  /* submission context */
 163    struct amdgpu_cs_fence fence;
 164    uint64_t *user_fence_cpu_address;
 165
 166    /* If the fence has been submitted. This is unsignalled for deferred fences
 167     * (cs->next_fence) and while an IB is still being submitted in the submit
 168     * thread. */
 169    struct util_queue_fence submitted;
 170
 171    volatile int signalled;              /* bool (int for atomicity) */
 172 };
 173
 174 static inline bool amdgpu_fence_is_syncobj(struct amdgpu_fence *fence)
 175 {
 176    return fence->ctx == NULL;
 177 }
 178
 179 static inline void amdgpu_ctx_unref(struct amdgpu_ctx *ctx)
 180 {
 181    if (p_atomic_dec_zero(&ctx->refcount)) {
 182       amdgpu_cs_ctx_free(ctx->ctx);
 183       amdgpu_bo_free(ctx->user_fence_bo);
 184       FREE(ctx);
 185    }
 186 }
 187
 188 static inline void amdgpu_fence_reference(struct pipe_fence_handle **dst,
 189                                           struct pipe_fence_handle *src)
 190 {
 191    struct amdgpu_fence **adst = (struct amdgpu_fence **)dst;
 192    struct amdgpu_fence *asrc = (struct amdgpu_fence *)src;
 193
 194    if (pipe_reference(&(*adst)->reference, &asrc->reference)) {
 195       struct amdgpu_fence *fence = *adst;
 196
 197       if (amdgpu_fence_is_syncobj(fence))
 198          amdgpu_cs_destroy_syncobj(fence->ws->dev, fence->syncobj);
 199       else
 200          amdgpu_ctx_unref(fence->ctx);
 201
 202       util_queue_fence_destroy(&fence->submitted);
 203       FREE(fence);
 204    }
 205    *adst = asrc;
 206 }
 207
 208 int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo);
 209
 210 static inline struct amdgpu_ib *
 211 amdgpu_ib(struct radeon_cmdbuf *base)
 212 {
 213    return (struct amdgpu_ib *)base;
 214 }
 215
 216 static inline struct amdgpu_cs *
 217 amdgpu_cs(struct radeon_cmdbuf *base)
 218 {
 219    assert(amdgpu_ib(base)->ib_type == IB_MAIN);
 220    return (struct amdgpu_cs*)base;
 221 }
 222
 223 #define get_container(member_ptr, container_type, container_member) \
 224    (container_type *)((char *)(member_ptr) - offsetof(container_type, container_member))
 225
 226 static inline struct amdgpu_cs *
 227 amdgpu_cs_from_ib(struct amdgpu_ib *ib)
 228 {
 229    switch (ib->ib_type) {
 230    case IB_MAIN:
 231       return get_container(ib, struct amdgpu_cs, main);
 232    case IB_PARALLEL_COMPUTE:
 233       return get_container(ib, struct amdgpu_cs, compute_ib);
 234    default:
 235       unreachable("bad ib_type");
 236    }
 237 }
 238
 239 static inline bool
 240 amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs *cs,
 241                               struct amdgpu_winsys_bo *bo)
 242 {
 243    int num_refs = bo->num_cs_references;
 244    return num_refs == bo->ws->num_cs ||
 245          (num_refs && amdgpu_lookup_buffer(cs->csc, bo) != -1);
 246 }
 247
 248 static inline bool
 249 amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs *cs,
 250                                          struct amdgpu_winsys_bo *bo,
 251                                          enum radeon_bo_usage usage)
 252 {
 253    int index;
 254    struct amdgpu_cs_buffer *buffer;
 255
 256    if (!bo->num_cs_references)
 257       return false;
 258
 259    index = amdgpu_lookup_buffer(cs->csc, bo);
 260    if (index == -1)
 261       return false;
 262
 263    buffer = bo->bo ? &cs->csc->real_buffers[index] :
 264             bo->sparse ? &cs->csc->sparse_buffers[index] :
 265             &cs->csc->slab_buffers[index];
 266
 267    return (buffer->usage & usage) != 0;
 268 }
 269
 270 static inline bool
 271 amdgpu_bo_is_referenced_by_any_cs(struct amdgpu_winsys_bo *bo)
 272 {
 273    return bo->num_cs_references != 0;
 274 }
 275
 276 bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
 277                        bool absolute);
 278 void amdgpu_add_fences(struct amdgpu_winsys_bo *bo,
 279                        unsigned num_fences,
 280                        struct pipe_fence_handle **fences);
 281 void amdgpu_cs_sync_flush(struct radeon_cmdbuf *rcs);
 282 void amdgpu_cs_init_functions(struct amdgpu_screen_winsys *ws);
 283 void amdgpu_cs_submit_ib(void *job, int thread_index);
 284
 285 #endif