src/gallium/state_trackers/clover/core/kernel.cpp

   1 //
   2 // Copyright 2012 Francisco Jerez
   3 //
   4 // Permission is hereby granted, free of charge, to any person obtaining a
   5 // copy of this software and associated documentation files (the "Software"),
   6 // to deal in the Software without restriction, including without limitation
   7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 // and/or sell copies of the Software, and to permit persons to whom the
   9 // Software is furnished to do so, subject to the following conditions:
  10 //
  11 // The above copyright notice and this permission notice shall be included in
  12 // all copies or substantial portions of the Software.
  13 //
  14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 // OTHER DEALINGS IN THE SOFTWARE.
  21 //
  22
  23 #include "core/kernel.hpp"
  24 #include "core/resource.hpp"
  25 #include "util/factor.hpp"
  26 #include "util/u_math.h"
  27 #include "pipe/p_context.h"
  28
  29 using namespace clover;
  30
  31 kernel::kernel(clover::program &prog, const std::string &name,
  32                const std::vector<module::argument> &margs) :
  33    program(prog), _name(name), exec(*this),
  34    program_ref(prog._kernel_ref_counter) {
  35    for (auto &marg : margs) {
  36       if (marg.semantic == module::argument::general)
  37          _args.emplace_back(argument::create(marg));
  38    }
  39 }
  40
  41 template<typename V>
  42 static inline std::vector<uint>
  43 pad_vector(command_queue &q, const V &v, uint x) {
  44    std::vector<uint> w { v.begin(), v.end() };
  45    w.resize(q.device().max_block_size().size(), x);
  46    return w;
  47 }
  48
  49 void
  50 kernel::launch(command_queue &q,
  51                const std::vector<size_t> &grid_offset,
  52                const std::vector<size_t> &grid_size,
  53                const std::vector<size_t> &block_size) {
  54    const auto m = program().binary(q.device());
  55    const auto reduced_grid_size =
  56       map(divides(), grid_size, block_size);
  57    void *st = exec.bind(&q, grid_offset);
  58    struct pipe_grid_info info = {};
  59
  60    // The handles are created during exec_context::bind(), so we need make
  61    // sure to call exec_context::bind() before retrieving them.
  62    std::vector<uint32_t *> g_handles = map([&](size_t h) {
  63          return (uint32_t *)&exec.input[h];
  64       }, exec.g_handles);
  65
  66    q.pipe->bind_compute_state(q.pipe, st);
  67    q.pipe->bind_sampler_states(q.pipe, PIPE_SHADER_COMPUTE,
  68                                0, exec.samplers.size(),
  69                                exec.samplers.data());
  70
  71    q.pipe->set_sampler_views(q.pipe, PIPE_SHADER_COMPUTE, 0,
  72                              exec.sviews.size(), exec.sviews.data());
  73    q.pipe->set_compute_resources(q.pipe, 0, exec.resources.size(),
  74                                  exec.resources.data());
  75    q.pipe->set_global_binding(q.pipe, 0, exec.g_buffers.size(),
  76                               exec.g_buffers.data(), g_handles.data());
  77
  78    // Fill information for the launch_grid() call.
  79    info.work_dim = grid_size.size();
  80    copy(pad_vector(q, block_size, 1), info.block);
  81    copy(pad_vector(q, reduced_grid_size, 1), info.grid);
  82    info.pc = find(name_equals(_name), m.syms).offset;
  83    info.input = exec.input.data();
  84
  85    q.pipe->launch_grid(q.pipe, &info);
  86
  87    q.pipe->set_global_binding(q.pipe, 0, exec.g_buffers.size(), NULL, NULL);
  88    q.pipe->set_compute_resources(q.pipe, 0, exec.resources.size(), NULL);
  89    q.pipe->set_sampler_views(q.pipe, PIPE_SHADER_COMPUTE, 0,
  90                              exec.sviews.size(), NULL);
  91    q.pipe->bind_sampler_states(q.pipe, PIPE_SHADER_COMPUTE, 0,
  92                                exec.samplers.size(), NULL);
  93
  94    q.pipe->memory_barrier(q.pipe, PIPE_BARRIER_GLOBAL_BUFFER);
  95    exec.unbind();
  96 }
  97
  98 size_t
  99 kernel::mem_local() const {
 100    size_t sz = 0;
 101
 102    for (auto &arg : args()) {
 103       if (dynamic_cast<local_argument *>(&arg))
 104          sz += arg.storage();
 105    }
 106
 107    return sz;
 108 }
 109
 110 size_t
 111 kernel::mem_private() const {
 112    return 0;
 113 }
 114
 115 const std::string &
 116 kernel::name() const {
 117    return _name;
 118 }
 119
 120 std::vector<size_t>
 121 kernel::optimal_block_size(const command_queue &q,
 122                            const std::vector<size_t> &grid_size) const {
 123    return factor::find_grid_optimal_factor<size_t>(
 124       q.device().max_threads_per_block(), q.device().max_block_size(),
 125       grid_size);
 126 }
 127
 128 std::vector<size_t>
 129 kernel::required_block_size() const {
 130    return { 0, 0, 0 };
 131 }
 132
 133 kernel::argument_range
 134 kernel::args() {
 135    return map(derefs(), _args);
 136 }
 137
 138 kernel::const_argument_range
 139 kernel::args() const {
 140    return map(derefs(), _args);
 141 }
 142
 143 const module &
 144 kernel::module(const command_queue &q) const {
 145    return program().binary(q.device());
 146 }
 147
 148 kernel::exec_context::exec_context(kernel &kern) :
 149    kern(kern), q(NULL), mem_local(0), st(NULL), cs() {
 150 }
 151
 152 kernel::exec_context::~exec_context() {
 153    if (st)
 154       q->pipe->delete_compute_state(q->pipe, st);
 155 }
 156
 157 void *
 158 kernel::exec_context::bind(intrusive_ptr<command_queue> _q,
 159                            const std::vector<size_t> &grid_offset) {
 160    std::swap(q, _q);
 161
 162    // Bind kernel arguments.
 163    auto &m = kern.program().binary(q->device());
 164    auto margs = find(name_equals(kern.name()), m.syms).args;
 165    auto msec = find(type_equals(module::section::text), m.secs);
 166    auto explicit_arg = kern._args.begin();
 167
 168    for (auto &marg : margs) {
 169       switch (marg.semantic) {
 170       case module::argument::general:
 171          (*(explicit_arg++))->bind(*this, marg);
 172          break;
 173
 174       case module::argument::grid_dimension: {
 175          const cl_uint dimension = grid_offset.size();
 176          auto arg = argument::create(marg);
 177
 178          arg->set(sizeof(dimension), &dimension);
 179          arg->bind(*this, marg);
 180          break;
 181       }
 182       case module::argument::grid_offset: {
 183          for (cl_uint x : pad_vector(*q, grid_offset, 0)) {
 184             auto arg = argument::create(marg);
 185
 186             arg->set(sizeof(x), &x);
 187             arg->bind(*this, marg);
 188          }
 189          break;
 190       }
 191       case module::argument::image_size: {
 192          auto img = dynamic_cast<image_argument &>(**(explicit_arg - 1)).get();
 193          std::vector<cl_uint> image_size{
 194                static_cast<cl_uint>(img->width()),
 195                static_cast<cl_uint>(img->height()),
 196                static_cast<cl_uint>(img->depth())};
 197          for (auto x : image_size) {
 198             auto arg = argument::create(marg);
 199
 200             arg->set(sizeof(x), &x);
 201             arg->bind(*this, marg);
 202          }
 203          break;
 204       }
 205       case module::argument::image_format: {
 206          auto img = dynamic_cast<image_argument &>(**(explicit_arg - 1)).get();
 207          cl_image_format fmt = img->format();
 208          std::vector<cl_uint> image_format{
 209                static_cast<cl_uint>(fmt.image_channel_data_type),
 210                static_cast<cl_uint>(fmt.image_channel_order)};
 211          for (auto x : image_format) {
 212             auto arg = argument::create(marg);
 213
 214             arg->set(sizeof(x), &x);
 215             arg->bind(*this, marg);
 216          }
 217          break;
 218       }
 219       }
 220    }
 221
 222    // Create a new compute state if anything changed.
 223    if (!st || q != _q ||
 224        cs.req_local_mem != mem_local ||
 225        cs.req_input_mem != input.size()) {
 226       if (st)
 227          _q->pipe->delete_compute_state(_q->pipe, st);
 228
 229       cs.ir_type = q->device().ir_format();
 230       cs.prog = &(msec.data[0]);
 231       cs.req_local_mem = mem_local;
 232       cs.req_input_mem = input.size();
 233       st = q->pipe->create_compute_state(q->pipe, &cs);
 234    }
 235
 236    return st;
 237 }
 238
 239 void
 240 kernel::exec_context::unbind() {
 241    for (auto &arg : kern.args())
 242       arg.unbind(*this);
 243
 244    input.clear();
 245    samplers.clear();
 246    sviews.clear();
 247    resources.clear();
 248    g_buffers.clear();
 249    g_handles.clear();
 250    mem_local = 0;
 251 }
 252
 253 namespace {
 254    template<typename T>
 255    std::vector<uint8_t>
 256    bytes(const T& x) {
 257       return { (uint8_t *)&x, (uint8_t *)&x + sizeof(x) };
 258    }
 259
 260    ///
 261    /// Transform buffer \a v from the native byte order into the byte
 262    /// order specified by \a e.
 263    ///
 264    template<typename T>
 265    void
 266    byteswap(T &v, pipe_endian e) {
 267       if (PIPE_ENDIAN_NATIVE != e)
 268          std::reverse(v.begin(), v.end());
 269    }
 270
 271    ///
 272    /// Pad buffer \a v to the next multiple of \a n.
 273    ///
 274    template<typename T>
 275    void
 276    align(T &v, size_t n) {
 277       v.resize(util_align_npot(v.size(), n));
 278    }
 279
 280    bool
 281    msb(const std::vector<uint8_t> &s) {
 282       if (PIPE_ENDIAN_NATIVE == PIPE_ENDIAN_LITTLE)
 283          return s.back() & 0x80;
 284       else
 285          return s.front() & 0x80;
 286    }
 287
 288    ///
 289    /// Resize buffer \a v to size \a n using sign or zero extension
 290    /// according to \a ext.
 291    ///
 292    template<typename T>
 293    void
 294    extend(T &v, enum module::argument::ext_type ext, size_t n) {
 295       const size_t m = std::min(v.size(), n);
 296       const bool sign_ext = (ext == module::argument::sign_ext);
 297       const uint8_t fill = (sign_ext && msb(v) ? ~0 : 0);
 298       T w(n, fill);
 299
 300       if (PIPE_ENDIAN_NATIVE == PIPE_ENDIAN_LITTLE)
 301          std::copy_n(v.begin(), m, w.begin());
 302       else
 303          std::copy_n(v.end() - m, m, w.end() - m);
 304
 305       std::swap(v, w);
 306    }
 307
 308    ///
 309    /// Append buffer \a w to \a v.
 310    ///
 311    template<typename T>
 312    void
 313    insert(T &v, const T &w) {
 314       v.insert(v.end(), w.begin(), w.end());
 315    }
 316
 317    ///
 318    /// Append \a n elements to the end of buffer \a v.
 319    ///
 320    template<typename T>
 321    size_t
 322    allocate(T &v, size_t n) {
 323       size_t pos = v.size();
 324       v.resize(pos + n);
 325       return pos;
 326    }
 327 }
 328
 329 std::unique_ptr<kernel::argument>
 330 kernel::argument::create(const module::argument &marg) {
 331    switch (marg.type) {
 332    case module::argument::scalar:
 333       return std::unique_ptr<kernel::argument>(new scalar_argument(marg.size));
 334
 335    case module::argument::global:
 336       return std::unique_ptr<kernel::argument>(new global_argument);
 337
 338    case module::argument::local:
 339       return std::unique_ptr<kernel::argument>(new local_argument);
 340
 341    case module::argument::constant:
 342       return std::unique_ptr<kernel::argument>(new constant_argument);
 343
 344    case module::argument::image2d_rd:
 345    case module::argument::image3d_rd:
 346       return std::unique_ptr<kernel::argument>(new image_rd_argument);
 347
 348    case module::argument::image2d_wr:
 349    case module::argument::image3d_wr:
 350       return std::unique_ptr<kernel::argument>(new image_wr_argument);
 351
 352    case module::argument::sampler:
 353       return std::unique_ptr<kernel::argument>(new sampler_argument);
 354
 355    }
 356    throw error(CL_INVALID_KERNEL_DEFINITION);
 357 }
 358
 359 kernel::argument::argument() : _set(false) {
 360 }
 361
 362 bool
 363 kernel::argument::set() const {
 364    return _set;
 365 }
 366
 367 size_t
 368 kernel::argument::storage() const {
 369    return 0;
 370 }
 371
 372 kernel::scalar_argument::scalar_argument(size_t size) : size(size) {
 373 }
 374
 375 void
 376 kernel::scalar_argument::set(size_t size, const void *value) {
 377    if (!value)
 378       throw error(CL_INVALID_ARG_VALUE);
 379
 380    if (size != this->size)
 381       throw error(CL_INVALID_ARG_SIZE);
 382
 383    v = { (uint8_t *)value, (uint8_t *)value + size };
 384    _set = true;
 385 }
 386
 387 void
 388 kernel::scalar_argument::bind(exec_context &ctx,
 389                               const module::argument &marg) {
 390    auto w = v;
 391
 392    extend(w, marg.ext_type, marg.target_size);
 393    byteswap(w, ctx.q->device().endianness());
 394    align(ctx.input, marg.target_align);
 395    insert(ctx.input, w);
 396 }
 397
 398 void
 399 kernel::scalar_argument::unbind(exec_context &ctx) {
 400 }
 401
 402 void
 403 kernel::global_argument::set(size_t size, const void *value) {
 404    if (size != sizeof(cl_mem))
 405       throw error(CL_INVALID_ARG_SIZE);
 406
 407    buf = pobj<buffer>(value ? *(cl_mem *)value : NULL);
 408    _set = true;
 409 }
 410
 411 void
 412 kernel::global_argument::bind(exec_context &ctx,
 413                               const module::argument &marg) {
 414    align(ctx.input, marg.target_align);
 415
 416    if (buf) {
 417       const resource &r = buf->resource(*ctx.q);
 418       ctx.g_handles.push_back(ctx.input.size());
 419       ctx.g_buffers.push_back(r.pipe);
 420
 421       // How to handle multi-demensional offsets?
 422       // We don't need to.  Buffer offsets are always
 423       // one-dimensional.
 424       auto v = bytes(r.offset[0]);
 425       extend(v, marg.ext_type, marg.target_size);
 426       byteswap(v, ctx.q->device().endianness());
 427       insert(ctx.input, v);
 428    } else {
 429       // Null pointer.
 430       allocate(ctx.input, marg.target_size);
 431    }
 432 }
 433
 434 void
 435 kernel::global_argument::unbind(exec_context &ctx) {
 436 }
 437
 438 size_t
 439 kernel::local_argument::storage() const {
 440    return _storage;
 441 }
 442
 443 void
 444 kernel::local_argument::set(size_t size, const void *value) {
 445    if (value)
 446       throw error(CL_INVALID_ARG_VALUE);
 447
 448    if (!size)
 449       throw error(CL_INVALID_ARG_SIZE);
 450
 451    _storage = size;
 452    _set = true;
 453 }
 454
 455 void
 456 kernel::local_argument::bind(exec_context &ctx,
 457                              const module::argument &marg) {
 458    auto v = bytes(ctx.mem_local);
 459
 460    extend(v, module::argument::zero_ext, marg.target_size);
 461    byteswap(v, ctx.q->device().endianness());
 462    align(ctx.input, marg.target_align);
 463    insert(ctx.input, v);
 464
 465    ctx.mem_local += _storage;
 466 }
 467
 468 void
 469 kernel::local_argument::unbind(exec_context &ctx) {
 470 }
 471
 472 void
 473 kernel::constant_argument::set(size_t size, const void *value) {
 474    if (size != sizeof(cl_mem))
 475       throw error(CL_INVALID_ARG_SIZE);
 476
 477    buf = pobj<buffer>(value ? *(cl_mem *)value : NULL);
 478    _set = true;
 479 }
 480
 481 void
 482 kernel::constant_argument::bind(exec_context &ctx,
 483                                 const module::argument &marg) {
 484    align(ctx.input, marg.target_align);
 485
 486    if (buf) {
 487       resource &r = buf->resource(*ctx.q);
 488       auto v = bytes(ctx.resources.size() << 24 | r.offset[0]);
 489
 490       extend(v, module::argument::zero_ext, marg.target_size);
 491       byteswap(v, ctx.q->device().endianness());
 492       insert(ctx.input, v);
 493
 494       st = r.bind_surface(*ctx.q, false);
 495       ctx.resources.push_back(st);
 496    } else {
 497       // Null pointer.
 498       allocate(ctx.input, marg.target_size);
 499    }
 500 }
 501
 502 void
 503 kernel::constant_argument::unbind(exec_context &ctx) {
 504    if (buf)
 505       buf->resource(*ctx.q).unbind_surface(*ctx.q, st);
 506 }
 507
 508 void
 509 kernel::image_rd_argument::set(size_t size, const void *value) {
 510    if (!value)
 511       throw error(CL_INVALID_ARG_VALUE);
 512
 513    if (size != sizeof(cl_mem))
 514       throw error(CL_INVALID_ARG_SIZE);
 515
 516    img = &obj<image>(*(cl_mem *)value);
 517    _set = true;
 518 }
 519
 520 void
 521 kernel::image_rd_argument::bind(exec_context &ctx,
 522                                 const module::argument &marg) {
 523    auto v = bytes(ctx.sviews.size());
 524
 525    extend(v, module::argument::zero_ext, marg.target_size);
 526    byteswap(v, ctx.q->device().endianness());
 527    align(ctx.input, marg.target_align);
 528    insert(ctx.input, v);
 529
 530    st = img->resource(*ctx.q).bind_sampler_view(*ctx.q);
 531    ctx.sviews.push_back(st);
 532 }
 533
 534 void
 535 kernel::image_rd_argument::unbind(exec_context &ctx) {
 536    img->resource(*ctx.q).unbind_sampler_view(*ctx.q, st);
 537 }
 538
 539 void
 540 kernel::image_wr_argument::set(size_t size, const void *value) {
 541    if (!value)
 542       throw error(CL_INVALID_ARG_VALUE);
 543
 544    if (size != sizeof(cl_mem))
 545       throw error(CL_INVALID_ARG_SIZE);
 546
 547    img = &obj<image>(*(cl_mem *)value);
 548    _set = true;
 549 }
 550
 551 void
 552 kernel::image_wr_argument::bind(exec_context &ctx,
 553                                 const module::argument &marg) {
 554    auto v = bytes(ctx.resources.size());
 555
 556    extend(v, module::argument::zero_ext, marg.target_size);
 557    byteswap(v, ctx.q->device().endianness());
 558    align(ctx.input, marg.target_align);
 559    insert(ctx.input, v);
 560
 561    st = img->resource(*ctx.q).bind_surface(*ctx.q, true);
 562    ctx.resources.push_back(st);
 563 }
 564
 565 void
 566 kernel::image_wr_argument::unbind(exec_context &ctx) {
 567    img->resource(*ctx.q).unbind_surface(*ctx.q, st);
 568 }
 569
 570 void
 571 kernel::sampler_argument::set(size_t size, const void *value) {
 572    if (!value)
 573       throw error(CL_INVALID_SAMPLER);
 574
 575    if (size != sizeof(cl_sampler))
 576       throw error(CL_INVALID_ARG_SIZE);
 577
 578    s = &obj(*(cl_sampler *)value);
 579    _set = true;
 580 }
 581
 582 void
 583 kernel::sampler_argument::bind(exec_context &ctx,
 584                                const module::argument &marg) {
 585    st = s->bind(*ctx.q);
 586    ctx.samplers.push_back(st);
 587 }
 588
 589 void
 590 kernel::sampler_argument::unbind(exec_context &ctx) {
 591    s->unbind(*ctx.q, st);
 592 }