freedreno/drm: remove idx_lock

Since it ends up contended, it is a bit of a bottleneck for workloads
with high driver overhead.  Worth nearly +10% at gfxbench driver2.

Signed-off-by: Rob Clark <robdclark@chromium.org>

diff --git a/src/freedreno/drm/msm_priv.h b/src/freedreno/drm/msm_priv.h
index acf22bcb6f8bd83069a8518a79edf9fb144f35b4..fd4267fa37106911f276e781b316248f2de29827 100644 (file)
--- a/src/freedreno/drm/msm_priv.h
+++ b/src/freedreno/drm/msm_priv.h
@@ -66,11 +66,6 @@ struct fd_submit * msm_submit_sp_new(struct fd_pipe *pipe);
 struct msm_bo {
        struct fd_bo base;
        uint64_t offset;
-       /* to avoid excess hashtable lookups, cache the ring this bo was
-        * last emitted on (since that will probably also be the next ring
-        * it is emitted on)
-        */
-       unsigned current_submit_seqno;
        uint32_t idx;
 };
 FD_DEFINE_CAST(fd_bo, msm_bo);
@@ -138,4 +133,6 @@ grow(void *ptr, uint16_t nr, uint16_t *max, uint16_t sz)
        (x)->nr_ ## name ++; \
 })
 
+#define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
+
 #endif /* MSM_PRIV_H_ */

diff --git a/src/freedreno/drm/msm_ringbuffer.c b/src/freedreno/drm/msm_ringbuffer.c
index 369f26f9837609cb45868b68e723cacb9dc23bce..c4b352ee283316814ab7456496bb0b5168d7e46a 100644 (file)
--- a/src/freedreno/drm/msm_ringbuffer.c
+++ b/src/freedreno/drm/msm_ringbuffer.c
@@ -41,8 +41,6 @@
 
 #define INIT_SIZE 0x1000
 
-static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
-
 
 struct msm_submit {
        struct fd_submit base;
@@ -50,8 +48,6 @@ struct msm_submit {
        DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
        DECLARE_ARRAY(struct fd_bo *, bos);
 
-       unsigned seqno;
-
        /* maps fd_bo to idx in bos table: */
        struct hash_table *bo_table;
 
@@ -146,10 +142,15 @@ append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
 {
        struct msm_bo *msm_bo = to_msm_bo(bo);
        uint32_t idx;
-       pthread_mutex_lock(&idx_lock);
-       if (likely(msm_bo->current_submit_seqno == submit->seqno)) {
-               idx = msm_bo->idx;
-       } else {
+
+       /* NOTE: it is legal to use the same bo on different threads for
+        * different submits.  But it is not legal to use the same submit
+        * from given threads.
+        */
+       idx = READ_ONCE(msm_bo->idx);
+
+       if (unlikely((idx >= submit->nr_submit_bos) ||
+                       (submit->submit_bos[idx].handle != bo->handle))) {
                uint32_t hash = _mesa_hash_pointer(bo);
                struct hash_entry *entry;
 
@@ -170,14 +171,14 @@ append_bo(struct msm_submit *submit, struct fd_bo *bo, uint32_t flags)
                        _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
                                        (void *)(uintptr_t)idx);
                }
-               msm_bo->current_submit_seqno = submit->seqno;
                msm_bo->idx = idx;
        }
-       pthread_mutex_unlock(&idx_lock);
+
        if (flags & FD_RELOC_READ)
                submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
        if (flags & FD_RELOC_WRITE)
                submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
+
        return idx;
 }
 
@@ -455,9 +456,7 @@ msm_submit_new(struct fd_pipe *pipe)
 {
        struct msm_submit *msm_submit = calloc(1, sizeof(*msm_submit));
        struct fd_submit *submit;
-       static unsigned submit_cnt = 0;
 
-       msm_submit->seqno = ++submit_cnt;
        msm_submit->bo_table = _mesa_hash_table_create(NULL,
                        _mesa_hash_pointer, _mesa_key_pointer_equal);
        msm_submit->ring_set = _mesa_set_create(NULL,

diff --git a/src/freedreno/drm/msm_ringbuffer_sp.c b/src/freedreno/drm/msm_ringbuffer_sp.c
index 2b8f531721e69801ede501123306e47ffa188a74..c87ad5128100865731652e4d1b6c28ce46f519f3 100644 (file)
--- a/src/freedreno/drm/msm_ringbuffer_sp.c
+++ b/src/freedreno/drm/msm_ringbuffer_sp.c
@@ -41,8 +41,6 @@
 
 #define INIT_SIZE 0x1000
 
-static pthread_mutex_t idx_lock = PTHREAD_MUTEX_INITIALIZER;
-
 
 struct msm_submit_sp {
        struct fd_submit base;
@@ -50,8 +48,6 @@ struct msm_submit_sp {
        DECLARE_ARRAY(struct drm_msm_gem_submit_bo, submit_bos);
        DECLARE_ARRAY(struct fd_bo *, bos);
 
-       unsigned seqno;
-
        /* maps fd_bo to idx in bos table: */
        struct hash_table *bo_table;
 
@@ -124,10 +120,15 @@ append_bo(struct msm_submit_sp *submit, struct fd_bo *bo, uint32_t flags)
 {
        struct msm_bo *msm_bo = to_msm_bo(bo);
        uint32_t idx;
-       pthread_mutex_lock(&idx_lock);
-       if (likely(msm_bo->current_submit_seqno == submit->seqno)) {
-               idx = msm_bo->idx;
-       } else {
+
+       /* NOTE: it is legal to use the same bo on different threads for
+        * different submits.  But it is not legal to use the same submit
+        * from given threads.
+        */
+       idx = READ_ONCE(msm_bo->idx);
+
+       if (unlikely((idx >= submit->nr_submit_bos) ||
+                       (submit->submit_bos[idx].handle != bo->handle))) {
                uint32_t hash = _mesa_hash_pointer(bo);
                struct hash_entry *entry;
 
@@ -148,16 +149,16 @@ append_bo(struct msm_submit_sp *submit, struct fd_bo *bo, uint32_t flags)
                        _mesa_hash_table_insert_pre_hashed(submit->bo_table, hash, bo,
                                        (void *)(uintptr_t)idx);
                }
-               msm_bo->current_submit_seqno = submit->seqno;
                msm_bo->idx = idx;
        }
-       pthread_mutex_unlock(&idx_lock);
+
        if (flags & FD_RELOC_READ)
                submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_READ;
        if (flags & FD_RELOC_WRITE)
                submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_WRITE;
        if (flags & FD_RELOC_DUMP)
                submit->submit_bos[idx].flags |= MSM_SUBMIT_BO_DUMP;
+
        return idx;
 }
 
@@ -337,9 +338,7 @@ msm_submit_sp_new(struct fd_pipe *pipe)
 {
        struct msm_submit_sp *msm_submit = calloc(1, sizeof(*msm_submit));
        struct fd_submit *submit;
-       static unsigned submit_cnt = 0;
 
-       msm_submit->seqno = ++submit_cnt;
        msm_submit->bo_table = _mesa_hash_table_create(NULL,
                        _mesa_hash_pointer, _mesa_key_pointer_equal);
        // TODO tune size: