src/util/u_queue.c

   1 /*
   2  * Copyright © 2016 Advanced Micro Devices, Inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sub license, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  14  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
  15  * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  16  * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
  17  * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  20  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * The above copyright notice and this permission notice (including the
  23  * next paragraph) shall be included in all copies or substantial portions
  24  * of the Software.
  25  */
  26
  27 #include "u_queue.h"
  28 #include "util/u_string.h"
  29
  30 static void util_queue_killall_and_wait(struct util_queue *queue);
  31
  32 /****************************************************************************
  33  * Wait for all queues to assert idle when exit() is called.
  34  *
  35  * Otherwise, C++ static variable destructors can be called while threads
  36  * are using the static variables.
  37  */
  38
  39 static once_flag atexit_once_flag = ONCE_FLAG_INIT;
  40 static struct list_head queue_list;
  41 static mtx_t exit_mutex = _MTX_INITIALIZER_NP;
  42
  43 static void
  44 atexit_handler(void)
  45 {
  46    struct util_queue *iter;
  47
  48    mtx_lock(&exit_mutex);
  49    /* Wait for all queues to assert idle. */
  50    LIST_FOR_EACH_ENTRY(iter, &queue_list, head) {
  51       util_queue_killall_and_wait(iter);
  52    }
  53    mtx_unlock(&exit_mutex);
  54 }
  55
  56 static void
  57 global_init(void)
  58 {
  59    LIST_INITHEAD(&queue_list);
  60    atexit(atexit_handler);
  61 }
  62
  63 static void
  64 add_to_atexit_list(struct util_queue *queue)
  65 {
  66    call_once(&atexit_once_flag, global_init);
  67
  68    mtx_lock(&exit_mutex);
  69    LIST_ADD(&queue->head, &queue_list);
  70    mtx_unlock(&exit_mutex);
  71 }
  72
  73 static void
  74 remove_from_atexit_list(struct util_queue *queue)
  75 {
  76    struct util_queue *iter, *tmp;
  77
  78    mtx_lock(&exit_mutex);
  79    LIST_FOR_EACH_ENTRY_SAFE(iter, tmp, &queue_list, head) {
  80       if (iter == queue) {
  81          LIST_DEL(&iter->head);
  82          break;
  83       }
  84    }
  85    mtx_unlock(&exit_mutex);
  86 }
  87
  88 /****************************************************************************
  89  * util_queue_fence
  90  */
  91
  92 static void
  93 util_queue_fence_signal(struct util_queue_fence *fence)
  94 {
  95    mtx_lock(&fence->mutex);
  96    fence->signalled = true;
  97    cnd_broadcast(&fence->cond);
  98    mtx_unlock(&fence->mutex);
  99 }
 100
 101 void
 102 util_queue_fence_wait(struct util_queue_fence *fence)
 103 {
 104    mtx_lock(&fence->mutex);
 105    while (!fence->signalled)
 106       cnd_wait(&fence->cond, &fence->mutex);
 107    mtx_unlock(&fence->mutex);
 108 }
 109
 110 void
 111 util_queue_fence_init(struct util_queue_fence *fence)
 112 {
 113    memset(fence, 0, sizeof(*fence));
 114    (void) mtx_init(&fence->mutex, mtx_plain);
 115    cnd_init(&fence->cond);
 116    fence->signalled = true;
 117 }
 118
 119 void
 120 util_queue_fence_destroy(struct util_queue_fence *fence)
 121 {
 122    assert(fence->signalled);
 123    cnd_destroy(&fence->cond);
 124    mtx_destroy(&fence->mutex);
 125 }
 126
 127 /****************************************************************************
 128  * util_queue implementation
 129  */
 130
 131 struct thread_input {
 132    struct util_queue *queue;
 133    int thread_index;
 134 };
 135
 136 static int
 137 util_queue_thread_func(void *input)
 138 {
 139    struct util_queue *queue = ((struct thread_input*)input)->queue;
 140    int thread_index = ((struct thread_input*)input)->thread_index;
 141
 142    free(input);
 143
 144    if (queue->name) {
 145       char name[16];
 146       util_snprintf(name, sizeof(name), "%s:%i", queue->name, thread_index);
 147       u_thread_setname(name);
 148    }
 149
 150    while (1) {
 151       struct util_queue_job job;
 152
 153       mtx_lock(&queue->lock);
 154       assert(queue->num_queued >= 0 && queue->num_queued <= queue->max_jobs);
 155
 156       /* wait if the queue is empty */
 157       while (!queue->kill_threads && queue->num_queued == 0)
 158          cnd_wait(&queue->has_queued_cond, &queue->lock);
 159
 160       if (queue->kill_threads) {
 161          mtx_unlock(&queue->lock);
 162          break;
 163       }
 164
 165       job = queue->jobs[queue->read_idx];
 166       memset(&queue->jobs[queue->read_idx], 0, sizeof(struct util_queue_job));
 167       queue->read_idx = (queue->read_idx + 1) % queue->max_jobs;
 168
 169       queue->num_queued--;
 170       cnd_signal(&queue->has_space_cond);
 171       mtx_unlock(&queue->lock);
 172
 173       if (job.job) {
 174          job.execute(job.job, thread_index);
 175          util_queue_fence_signal(job.fence);
 176          if (job.cleanup)
 177             job.cleanup(job.job, thread_index);
 178       }
 179    }
 180
 181    /* signal remaining jobs before terminating */
 182    mtx_lock(&queue->lock);
 183    for (unsigned i = queue->read_idx; i != queue->write_idx;
 184         i = (i + 1) % queue->max_jobs) {
 185       if (queue->jobs[i].job) {
 186          util_queue_fence_signal(queue->jobs[i].fence);
 187          queue->jobs[i].job = NULL;
 188       }
 189    }
 190    queue->read_idx = queue->write_idx;
 191    queue->num_queued = 0;
 192    mtx_unlock(&queue->lock);
 193    return 0;
 194 }
 195
 196 bool
 197 util_queue_init(struct util_queue *queue,
 198                 const char *name,
 199                 unsigned max_jobs,
 200                 unsigned num_threads)
 201 {
 202    unsigned i;
 203
 204    memset(queue, 0, sizeof(*queue));
 205    queue->name = name;
 206    queue->num_threads = num_threads;
 207    queue->max_jobs = max_jobs;
 208
 209    queue->jobs = (struct util_queue_job*)
 210                  calloc(max_jobs, sizeof(struct util_queue_job));
 211    if (!queue->jobs)
 212       goto fail;
 213
 214    (void) mtx_init(&queue->lock, mtx_plain);
 215
 216    queue->num_queued = 0;
 217    cnd_init(&queue->has_queued_cond);
 218    cnd_init(&queue->has_space_cond);
 219
 220    queue->threads = (thrd_t*) calloc(num_threads, sizeof(thrd_t));
 221    if (!queue->threads)
 222       goto fail;
 223
 224    /* start threads */
 225    for (i = 0; i < num_threads; i++) {
 226       struct thread_input *input =
 227          (struct thread_input *) malloc(sizeof(struct thread_input));
 228       input->queue = queue;
 229       input->thread_index = i;
 230
 231       queue->threads[i] = u_thread_create(util_queue_thread_func, input);
 232
 233       if (!queue->threads[i]) {
 234          free(input);
 235
 236          if (i == 0) {
 237             /* no threads created, fail */
 238             goto fail;
 239          } else {
 240             /* at least one thread created, so use it */
 241             queue->num_threads = i;
 242             break;
 243          }
 244       }
 245    }
 246
 247    add_to_atexit_list(queue);
 248    return true;
 249
 250 fail:
 251    free(queue->threads);
 252
 253    if (queue->jobs) {
 254       cnd_destroy(&queue->has_space_cond);
 255       cnd_destroy(&queue->has_queued_cond);
 256       mtx_destroy(&queue->lock);
 257       free(queue->jobs);
 258    }
 259    /* also util_queue_is_initialized can be used to check for success */
 260    memset(queue, 0, sizeof(*queue));
 261    return false;
 262 }
 263
 264 static void
 265 util_queue_killall_and_wait(struct util_queue *queue)
 266 {
 267    unsigned i;
 268
 269    /* Signal all threads to terminate. */
 270    mtx_lock(&queue->lock);
 271    queue->kill_threads = 1;
 272    cnd_broadcast(&queue->has_queued_cond);
 273    mtx_unlock(&queue->lock);
 274
 275    for (i = 0; i < queue->num_threads; i++)
 276       thrd_join(queue->threads[i], NULL);
 277    queue->num_threads = 0;
 278 }
 279
 280 void
 281 util_queue_destroy(struct util_queue *queue)
 282 {
 283    util_queue_killall_and_wait(queue);
 284    remove_from_atexit_list(queue);
 285
 286    cnd_destroy(&queue->has_space_cond);
 287    cnd_destroy(&queue->has_queued_cond);
 288    mtx_destroy(&queue->lock);
 289    free(queue->jobs);
 290    free(queue->threads);
 291 }
 292
 293 void
 294 util_queue_add_job(struct util_queue *queue,
 295                    void *job,
 296                    struct util_queue_fence *fence,
 297                    util_queue_execute_func execute,
 298                    util_queue_execute_func cleanup)
 299 {
 300    struct util_queue_job *ptr;
 301
 302    assert(fence->signalled);
 303
 304    mtx_lock(&queue->lock);
 305    if (queue->kill_threads) {
 306       mtx_unlock(&queue->lock);
 307       /* well no good option here, but any leaks will be
 308        * short-lived as things are shutting down..
 309        */
 310       return;
 311    }
 312
 313    fence->signalled = false;
 314
 315    assert(queue->num_queued >= 0 && queue->num_queued <= queue->max_jobs);
 316
 317    /* if the queue is full, wait until there is space */
 318    while (queue->num_queued == queue->max_jobs)
 319       cnd_wait(&queue->has_space_cond, &queue->lock);
 320
 321    ptr = &queue->jobs[queue->write_idx];
 322    assert(ptr->job == NULL);
 323    ptr->job = job;
 324    ptr->fence = fence;
 325    ptr->execute = execute;
 326    ptr->cleanup = cleanup;
 327    queue->write_idx = (queue->write_idx + 1) % queue->max_jobs;
 328
 329    queue->num_queued++;
 330    cnd_signal(&queue->has_queued_cond);
 331    mtx_unlock(&queue->lock);
 332 }
 333
 334 /**
 335  * Remove a queued job. If the job hasn't started execution, it's removed from
 336  * the queue. If the job has started execution, the function waits for it to
 337  * complete.
 338  *
 339  * In all cases, the fence is signalled when the function returns.
 340  *
 341  * The function can be used when destroying an object associated with the job
 342  * when you don't care about the job completion state.
 343  */
 344 void
 345 util_queue_drop_job(struct util_queue *queue, struct util_queue_fence *fence)
 346 {
 347    bool removed = false;
 348
 349    if (util_queue_fence_is_signalled(fence))
 350       return;
 351
 352    mtx_lock(&queue->lock);
 353    for (unsigned i = queue->read_idx; i != queue->write_idx;
 354         i = (i + 1) % queue->max_jobs) {
 355       if (queue->jobs[i].fence == fence) {
 356          if (queue->jobs[i].cleanup)
 357             queue->jobs[i].cleanup(queue->jobs[i].job, -1);
 358
 359          /* Just clear it. The threads will treat as a no-op job. */
 360          memset(&queue->jobs[i], 0, sizeof(queue->jobs[i]));
 361          removed = true;
 362          break;
 363       }
 364    }
 365    mtx_unlock(&queue->lock);
 366
 367    if (removed)
 368       util_queue_fence_signal(fence);
 369    else
 370       util_queue_fence_wait(fence);
 371 }
 372
 373 int64_t
 374 util_queue_get_thread_time_nano(struct util_queue *queue, unsigned thread_index)
 375 {
 376    /* Allow some flexibility by not raising an error. */
 377    if (thread_index >= queue->num_threads)
 378       return 0;
 379
 380    return u_thread_get_time_nano(queue->threads[thread_index]);
 381 }