src/gallium/drivers/freedreno/freedreno_batch_cache.c

   1 /*
   2  * Copyright (C) 2016 Rob Clark <robclark@freedesktop.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors:
  24  *    Rob Clark <robclark@freedesktop.org>
  25  */
  26
  27 #include "util/hash_table.h"
  28 #include "util/set.h"
  29 #include "util/list.h"
  30 #include "util/u_string.h"
  31
  32 #include "freedreno_batch.h"
  33 #include "freedreno_batch_cache.h"
  34 #include "freedreno_context.h"
  35 #include "freedreno_resource.h"
  36
  37 /* Overview:
  38  *
  39  *   The batch cache provides lookup for mapping pipe_framebuffer_state
  40  *   to a batch.
  41  *
  42  *   It does this via hashtable, with key that roughly matches the
  43  *   pipe_framebuffer_state, as described below.
  44  *
  45  * Batch Cache hashtable key:
  46  *
  47  *   To serialize the key, and to avoid dealing with holding a reference to
  48  *   pipe_surface's (which hold a reference to pipe_resource and complicate
  49  *   the whole refcnting thing), the key is variable length and inline's the
  50  *   pertinent details of the pipe_surface.
  51  *
  52  * Batch:
  53  *
  54  *   Each batch needs to hold a reference to each resource it depends on (ie.
  55  *   anything that needs a mem2gmem).  And a weak reference to resources it
  56  *   renders to.  (If both src[n] and dst[n] are not NULL then they are the
  57  *   same.)
  58  *
  59  *   When a resource is destroyed, we need to remove entries in the batch
  60  *   cache that reference the resource, to avoid dangling pointer issues.
  61  *   So each resource holds a hashset of batches which have reference them
  62  *   in their hashtable key.
  63  *
  64  *   When a batch has weak reference to no more resources (ie. all the
  65  *   surfaces it rendered to are destroyed) the batch can be destroyed.
  66  *   Could happen in an app that renders and never uses the result.  More
  67  *   common scenario, I think, will be that some, but not all, of the
  68  *   surfaces are destroyed before the batch is submitted.
  69  *
  70  *   If (for example), batch writes to zsbuf but that surface is destroyed
  71  *   before batch is submitted, we can skip gmem2mem (but still need to
  72  *   alloc gmem space as before.  If the batch depended on previous contents
  73  *   of that surface, it would be holding a reference so the surface would
  74  *   not have been destroyed.
  75  */
  76
  77 struct key {
  78         uint32_t width, height, layers;
  79         uint16_t samples, num_surfs;
  80         struct fd_context *ctx;
  81         struct {
  82                 struct pipe_resource *texture;
  83                 union pipe_surface_desc u;
  84                 uint16_t pos, format;
  85         } surf[0];
  86 };
  87
  88 static struct key *
  89 key_alloc(unsigned num_surfs)
  90 {
  91         struct key *key =
  92                 CALLOC_VARIANT_LENGTH_STRUCT(key, sizeof(key->surf[0]) * num_surfs);
  93         return key;
  94 }
  95
  96 static uint32_t
  97 key_hash(const void *_key)
  98 {
  99         const struct key *key = _key;
 100         uint32_t hash = _mesa_fnv32_1a_offset_bias;
 101         hash = _mesa_fnv32_1a_accumulate_block(hash, key, offsetof(struct key, surf[0]));
 102         hash = _mesa_fnv32_1a_accumulate_block(hash, key->surf, sizeof(key->surf[0]) * key->num_surfs);
 103         return hash;
 104 }
 105
 106 static bool
 107 key_equals(const void *_a, const void *_b)
 108 {
 109         const struct key *a = _a;
 110         const struct key *b = _b;
 111         return (memcmp(a, b, offsetof(struct key, surf[0])) == 0) &&
 112                 (memcmp(a->surf, b->surf, sizeof(a->surf[0]) * a->num_surfs) == 0);
 113 }
 114
 115 void
 116 fd_bc_init(struct fd_batch_cache *cache)
 117 {
 118         cache->ht = _mesa_hash_table_create(NULL, key_hash, key_equals);
 119 }
 120
 121 void
 122 fd_bc_fini(struct fd_batch_cache *cache)
 123 {
 124         _mesa_hash_table_destroy(cache->ht, NULL);
 125 }
 126
 127 void
 128 fd_bc_flush(struct fd_batch_cache *cache, struct fd_context *ctx)
 129 {
 130         struct hash_entry *entry;
 131         struct fd_batch *last_batch = NULL;
 132
 133         mtx_lock(&ctx->screen->lock);
 134
 135         hash_table_foreach(cache->ht, entry) {
 136                 struct fd_batch *batch = NULL;
 137                 /* hold a reference since we can drop screen->lock: */
 138                 fd_batch_reference_locked(&batch, (struct fd_batch *)entry->data);
 139                 if (batch->ctx == ctx) {
 140                         mtx_unlock(&ctx->screen->lock);
 141                         fd_batch_reference(&last_batch, batch);
 142                         fd_batch_flush(batch, false, false);
 143                         mtx_lock(&ctx->screen->lock);
 144                 }
 145                 fd_batch_reference_locked(&batch, NULL);
 146         }
 147
 148         mtx_unlock(&ctx->screen->lock);
 149
 150         if (last_batch) {
 151                 fd_batch_sync(last_batch);
 152                 fd_batch_reference(&last_batch, NULL);
 153         }
 154 }
 155
 156 /* deferred flush doesn't actually flush, but it marks every other
 157  * batch associated with the context as dependent on the current
 158  * batch.  So when the current batch gets flushed, all other batches
 159  * that came before also get flushed.
 160  */
 161 void
 162 fd_bc_flush_deferred(struct fd_batch_cache *cache, struct fd_context *ctx)
 163 {
 164         struct fd_batch *current_batch = ctx->batch;
 165         struct hash_entry *entry;
 166
 167         mtx_lock(&ctx->screen->lock);
 168
 169         hash_table_foreach(cache->ht, entry) {
 170                 struct fd_batch *batch = entry->data;
 171                 if (batch == current_batch)
 172                         continue;
 173                 if (batch->ctx == ctx)
 174                         fd_batch_add_dep(current_batch, batch);
 175         }
 176
 177         mtx_unlock(&ctx->screen->lock);
 178 }
 179
 180 void
 181 fd_bc_invalidate_context(struct fd_context *ctx)
 182 {
 183         struct fd_batch_cache *cache = &ctx->screen->batch_cache;
 184         struct fd_batch *batch;
 185
 186         mtx_lock(&ctx->screen->lock);
 187
 188         foreach_batch(batch, cache, cache->batch_mask) {
 189                 if (batch->ctx == ctx)
 190                         fd_batch_reference_locked(&batch, NULL);
 191         }
 192
 193         mtx_unlock(&ctx->screen->lock);
 194 }
 195
 196 void
 197 fd_bc_invalidate_batch(struct fd_batch *batch, bool destroy)
 198 {
 199         if (!batch)
 200                 return;
 201
 202         struct fd_batch_cache *cache = &batch->ctx->screen->batch_cache;
 203         struct key *key = (struct key *)batch->key;
 204
 205         pipe_mutex_assert_locked(batch->ctx->screen->lock);
 206
 207         if (destroy) {
 208                 cache->batches[batch->idx] = NULL;
 209                 cache->batch_mask &= ~(1 << batch->idx);
 210         }
 211
 212         if (!key)
 213                 return;
 214
 215         DBG("%p: key=%p", batch, batch->key);
 216         for (unsigned idx = 0; idx < key->num_surfs; idx++) {
 217                 struct fd_resource *rsc = fd_resource(key->surf[idx].texture);
 218                 rsc->bc_batch_mask &= ~(1 << batch->idx);
 219         }
 220
 221         struct hash_entry *entry =
 222                 _mesa_hash_table_search_pre_hashed(cache->ht, batch->hash, key);
 223         _mesa_hash_table_remove(cache->ht, entry);
 224
 225         batch->key = NULL;
 226         free(key);
 227 }
 228
 229 void
 230 fd_bc_invalidate_resource(struct fd_resource *rsc, bool destroy)
 231 {
 232         struct fd_screen *screen = fd_screen(rsc->base.screen);
 233         struct fd_batch *batch;
 234
 235         mtx_lock(&screen->lock);
 236
 237         if (destroy) {
 238                 foreach_batch(batch, &screen->batch_cache, rsc->batch_mask) {
 239                         struct set_entry *entry = _mesa_set_search(batch->resources, rsc);
 240                         _mesa_set_remove(batch->resources, entry);
 241                 }
 242                 rsc->batch_mask = 0;
 243
 244                 fd_batch_reference_locked(&rsc->write_batch, NULL);
 245         }
 246
 247         foreach_batch(batch, &screen->batch_cache, rsc->bc_batch_mask)
 248                 fd_bc_invalidate_batch(batch, false);
 249
 250         rsc->bc_batch_mask = 0;
 251
 252         mtx_unlock(&screen->lock);
 253 }
 254
 255 struct fd_batch *
 256 fd_bc_alloc_batch(struct fd_batch_cache *cache, struct fd_context *ctx)
 257 {
 258         struct fd_batch *batch;
 259         uint32_t idx;
 260
 261         mtx_lock(&ctx->screen->lock);
 262
 263         while ((idx = ffs(~cache->batch_mask)) == 0) {
 264 #if 0
 265                 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
 266                         batch = cache->batches[i];
 267                         debug_printf("%d: needs_flush=%d, depends:", batch->idx, batch->needs_flush);
 268                         struct set_entry *entry;
 269                         set_foreach(batch->dependencies, entry) {
 270                                 struct fd_batch *dep = (struct fd_batch *)entry->key;
 271                                 debug_printf(" %d", dep->idx);
 272                         }
 273                         debug_printf("\n");
 274                 }
 275 #endif
 276                 /* TODO: is LRU the better policy?  Or perhaps the batch that
 277                  * depends on the fewest other batches?
 278                  */
 279                 struct fd_batch *flush_batch = NULL;
 280                 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
 281                         if ((cache->batches[i] == ctx->batch) ||
 282                                         !cache->batches[i]->needs_flush)
 283                                 continue;
 284                         if (!flush_batch || (cache->batches[i]->seqno < flush_batch->seqno))
 285                                 fd_batch_reference_locked(&flush_batch, cache->batches[i]);
 286                 }
 287
 288                 /* we can drop lock temporarily here, since we hold a ref,
 289                  * flush_batch won't disappear under us.
 290                  */
 291                 mtx_unlock(&ctx->screen->lock);
 292                 DBG("%p: too many batches!  flush forced!", flush_batch);
 293                 fd_batch_flush(flush_batch, true, false);
 294                 mtx_lock(&ctx->screen->lock);
 295
 296                 /* While the resources get cleaned up automatically, the flush_batch
 297                  * doesn't get removed from the dependencies of other batches, so
 298                  * it won't be unref'd and will remain in the table.
 299                  *
 300                  * TODO maybe keep a bitmask of batches that depend on me, to make
 301                  * this easier:
 302                  */
 303                 for (unsigned i = 0; i < ARRAY_SIZE(cache->batches); i++) {
 304                         struct fd_batch *other = cache->batches[i];
 305                         if (!other)
 306                                 continue;
 307                         if (other->dependents_mask & (1 << flush_batch->idx)) {
 308                                 other->dependents_mask &= ~(1 << flush_batch->idx);
 309                                 struct fd_batch *ref = flush_batch;
 310                                 fd_batch_reference_locked(&ref, NULL);
 311                         }
 312                 }
 313
 314                 fd_batch_reference_locked(&flush_batch, NULL);
 315         }
 316
 317         idx--;              /* bit zero returns 1 for ffs() */
 318
 319         batch = fd_batch_create(ctx, false);
 320         if (!batch)
 321                 goto out;
 322
 323         batch->seqno = cache->cnt++;
 324         batch->idx = idx;
 325         cache->batch_mask |= (1 << idx);
 326
 327         debug_assert(cache->batches[idx] == NULL);
 328         cache->batches[idx] = batch;
 329
 330 out:
 331         mtx_unlock(&ctx->screen->lock);
 332
 333         return batch;
 334 }
 335
 336 static struct fd_batch *
 337 batch_from_key(struct fd_batch_cache *cache, struct key *key,
 338                 struct fd_context *ctx)
 339 {
 340         struct fd_batch *batch = NULL;
 341         uint32_t hash = key_hash(key);
 342         struct hash_entry *entry =
 343                 _mesa_hash_table_search_pre_hashed(cache->ht, hash, key);
 344
 345         if (entry) {
 346                 free(key);
 347                 fd_batch_reference(&batch, (struct fd_batch *)entry->data);
 348                 return batch;
 349         }
 350
 351         batch = fd_bc_alloc_batch(cache, ctx);
 352 #ifdef DEBUG
 353         DBG("%p: hash=0x%08x, %ux%u, %u layers, %u samples", batch, hash,
 354                         key->width, key->height, key->layers, key->samples);
 355         for (unsigned idx = 0; idx < key->num_surfs; idx++) {
 356                 DBG("%p:  surf[%u]: %p (%s) (%u,%u / %u,%u,%u)", batch, key->surf[idx].pos,
 357                         key->surf[idx].texture, util_format_name(key->surf[idx].format),
 358                         key->surf[idx].u.buf.first_element, key->surf[idx].u.buf.last_element,
 359                         key->surf[idx].u.tex.first_layer, key->surf[idx].u.tex.last_layer,
 360                         key->surf[idx].u.tex.level);
 361         }
 362 #endif
 363         if (!batch)
 364                 return NULL;
 365
 366         mtx_lock(&ctx->screen->lock);
 367
 368         _mesa_hash_table_insert_pre_hashed(cache->ht, hash, key, batch);
 369         batch->key = key;
 370         batch->hash = hash;
 371
 372         for (unsigned idx = 0; idx < key->num_surfs; idx++) {
 373                 struct fd_resource *rsc = fd_resource(key->surf[idx].texture);
 374                 rsc->bc_batch_mask = (1 << batch->idx);
 375         }
 376
 377         mtx_unlock(&ctx->screen->lock);
 378
 379         return batch;
 380 }
 381
 382 static void
 383 key_surf(struct key *key, unsigned idx, unsigned pos, struct pipe_surface *psurf)
 384 {
 385         key->surf[idx].texture = psurf->texture;
 386         key->surf[idx].u = psurf->u;
 387         key->surf[idx].pos = pos;
 388         key->surf[idx].format = psurf->format;
 389 }
 390
 391 struct fd_batch *
 392 fd_batch_from_fb(struct fd_batch_cache *cache, struct fd_context *ctx,
 393                 const struct pipe_framebuffer_state *pfb)
 394 {
 395         unsigned idx = 0, n = pfb->nr_cbufs + (pfb->zsbuf ? 1 : 0);
 396         struct key *key = key_alloc(n);
 397
 398         key->width = pfb->width;
 399         key->height = pfb->height;
 400         key->layers = pfb->layers;
 401         key->samples = pfb->samples;
 402         key->ctx = ctx;
 403
 404         if (pfb->zsbuf)
 405                 key_surf(key, idx++, 0, pfb->zsbuf);
 406
 407         for (unsigned i = 0; i < pfb->nr_cbufs; i++)
 408                 if (pfb->cbufs[i])
 409                         key_surf(key, idx++, i + 1, pfb->cbufs[i]);
 410
 411         key->num_surfs = idx;
 412
 413         return batch_from_key(cache, key, ctx);
 414 }