src/gallium/frontends/clover/api/memory.cpp

   1 //
   2 // Copyright 2012 Francisco Jerez
   3 //
   4 // Permission is hereby granted, free of charge, to any person obtaining a
   5 // copy of this software and associated documentation files (the "Software"),
   6 // to deal in the Software without restriction, including without limitation
   7 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 // and/or sell copies of the Software, and to permit persons to whom the
   9 // Software is furnished to do so, subject to the following conditions:
  10 //
  11 // The above copyright notice and this permission notice shall be included in
  12 // all copies or substantial portions of the Software.
  13 //
  14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 // THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 // OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 // ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 // OTHER DEALINGS IN THE SOFTWARE.
  21 //
  22
  23 #include "util/u_math.h"
  24 #include "api/util.hpp"
  25 #include "core/memory.hpp"
  26 #include "core/format.hpp"
  27
  28 using namespace clover;
  29
  30 namespace {
  31    cl_mem_flags
  32    validate_flags(cl_mem d_parent, cl_mem_flags d_flags, bool svm) {
  33       const cl_mem_flags dev_access_flags =
  34          CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY;
  35       const cl_mem_flags host_ptr_flags =
  36          CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR;
  37       const cl_mem_flags host_access_flags =
  38          CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS;
  39       const cl_mem_flags svm_flags =
  40          CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_SVM_ATOMICS;
  41
  42       const cl_mem_flags valid_flags =
  43          dev_access_flags
  44             | (svm || d_parent ? 0 : host_ptr_flags)
  45             | (svm ? svm_flags : host_access_flags);
  46
  47       if ((d_flags & ~valid_flags) ||
  48           util_bitcount(d_flags & dev_access_flags) > 1 ||
  49           util_bitcount(d_flags & host_access_flags) > 1)
  50          throw error(CL_INVALID_VALUE);
  51
  52       if ((d_flags & CL_MEM_USE_HOST_PTR) &&
  53           (d_flags & (CL_MEM_COPY_HOST_PTR | CL_MEM_ALLOC_HOST_PTR)))
  54          throw error(CL_INVALID_VALUE);
  55
  56       if ((d_flags & CL_MEM_SVM_ATOMICS) &&
  57           !(d_flags & CL_MEM_SVM_FINE_GRAIN_BUFFER))
  58          throw error(CL_INVALID_VALUE);
  59
  60       if (d_parent) {
  61          const auto &parent = obj(d_parent);
  62          const cl_mem_flags flags = (d_flags |
  63                                      (d_flags & dev_access_flags ? 0 :
  64                                       parent.flags() & dev_access_flags) |
  65                                      (d_flags & host_access_flags ? 0 :
  66                                       parent.flags() & host_access_flags) |
  67                                      (parent.flags() & host_ptr_flags));
  68
  69          if (~flags & parent.flags() & (dev_access_flags & ~CL_MEM_READ_WRITE))
  70             throw error(CL_INVALID_VALUE);
  71
  72          // Check if new host access flags cause a mismatch between
  73          // host-read/write-only.
  74          if (!(flags & CL_MEM_HOST_NO_ACCESS) &&
  75              (~flags & parent.flags() & host_access_flags))
  76             throw error(CL_INVALID_VALUE);
  77
  78          return flags;
  79
  80       } else {
  81          return d_flags | (d_flags & dev_access_flags ? 0 : CL_MEM_READ_WRITE);
  82       }
  83    }
  84 }
  85
  86 CLOVER_API cl_mem
  87 clCreateBuffer(cl_context d_ctx, cl_mem_flags d_flags, size_t size,
  88                void *host_ptr, cl_int *r_errcode) try {
  89    const cl_mem_flags flags = validate_flags(NULL, d_flags, false);
  90    auto &ctx = obj(d_ctx);
  91
  92    if (bool(host_ptr) != bool(flags & (CL_MEM_USE_HOST_PTR |
  93                                        CL_MEM_COPY_HOST_PTR)))
  94       throw error(CL_INVALID_HOST_PTR);
  95
  96    if (!size ||
  97        size > fold(maximum(), cl_ulong(0),
  98                    map(std::mem_fn(&device::max_mem_alloc_size), ctx.devices())
  99           ))
 100       throw error(CL_INVALID_BUFFER_SIZE);
 101
 102    ret_error(r_errcode, CL_SUCCESS);
 103    return new root_buffer(ctx, flags, size, host_ptr);
 104
 105 } catch (error &e) {
 106    ret_error(r_errcode, e);
 107    return NULL;
 108 }
 109
 110 CLOVER_API cl_mem
 111 clCreateSubBuffer(cl_mem d_mem, cl_mem_flags d_flags,
 112                   cl_buffer_create_type op,
 113                   const void *op_info, cl_int *r_errcode) try {
 114    auto &parent = obj<root_buffer>(d_mem);
 115    const cl_mem_flags flags = validate_flags(d_mem, d_flags, false);
 116
 117    if (op == CL_BUFFER_CREATE_TYPE_REGION) {
 118       auto reg = reinterpret_cast<const cl_buffer_region *>(op_info);
 119
 120       if (!reg ||
 121           reg->origin > parent.size() ||
 122           reg->origin + reg->size > parent.size())
 123          throw error(CL_INVALID_VALUE);
 124
 125       if (!reg->size)
 126          throw error(CL_INVALID_BUFFER_SIZE);
 127
 128       ret_error(r_errcode, CL_SUCCESS);
 129       return new sub_buffer(parent, flags, reg->origin, reg->size);
 130
 131    } else {
 132       throw error(CL_INVALID_VALUE);
 133    }
 134
 135 } catch (error &e) {
 136    ret_error(r_errcode, e);
 137    return NULL;
 138 }
 139
 140 CLOVER_API cl_mem
 141 clCreateImage(cl_context d_ctx, cl_mem_flags d_flags,
 142               const cl_image_format *format,
 143               const cl_image_desc *desc,
 144               void *host_ptr, cl_int *r_errcode) try {
 145    auto &ctx = obj(d_ctx);
 146
 147    if (!any_of(std::mem_fn(&device::image_support), ctx.devices()))
 148       throw error(CL_INVALID_OPERATION);
 149
 150    if (!format)
 151       throw error(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR);
 152
 153    if (!desc)
 154       throw error(CL_INVALID_IMAGE_DESCRIPTOR);
 155
 156    if (desc->image_array_size == 0 &&
 157        (desc->image_type == CL_MEM_OBJECT_IMAGE1D_ARRAY ||
 158         desc->image_type == CL_MEM_OBJECT_IMAGE2D_ARRAY))
 159       throw error(CL_INVALID_IMAGE_DESCRIPTOR);
 160
 161    if (!host_ptr &&
 162        (desc->image_row_pitch || desc->image_slice_pitch))
 163       throw error(CL_INVALID_IMAGE_DESCRIPTOR);
 164
 165    if (desc->num_mip_levels || desc->num_samples)
 166       throw error(CL_INVALID_IMAGE_DESCRIPTOR);
 167
 168    if (bool(desc->buffer) != (desc->image_type == CL_MEM_OBJECT_IMAGE1D_BUFFER))
 169       throw error(CL_INVALID_IMAGE_DESCRIPTOR);
 170
 171    if (bool(host_ptr) != bool(d_flags & (CL_MEM_USE_HOST_PTR |
 172                                          CL_MEM_COPY_HOST_PTR)))
 173       throw error(CL_INVALID_HOST_PTR);
 174
 175    const cl_mem_flags flags = validate_flags(desc->buffer, d_flags, false);
 176
 177    if (!supported_formats(ctx, desc->image_type).count(*format))
 178       throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED);
 179
 180    ret_error(r_errcode, CL_SUCCESS);
 181
 182    switch (desc->image_type) {
 183    case CL_MEM_OBJECT_IMAGE2D:
 184       if (!desc->image_width || !desc->image_height)
 185          throw error(CL_INVALID_IMAGE_SIZE);
 186
 187       if (all_of([=](const device &dev) {
 188                const size_t max = 1 << dev.max_image_levels_2d();
 189                return (desc->image_width > max ||
 190                        desc->image_height > max);
 191             }, ctx.devices()))
 192          throw error(CL_INVALID_IMAGE_SIZE);
 193
 194       return new image2d(ctx, flags, format,
 195                          desc->image_width, desc->image_height,
 196                          desc->image_row_pitch, host_ptr);
 197
 198    case CL_MEM_OBJECT_IMAGE3D:
 199       if (!desc->image_width || !desc->image_height || !desc->image_depth)
 200          throw error(CL_INVALID_IMAGE_SIZE);
 201
 202       if (all_of([=](const device &dev) {
 203                const size_t max = 1 << dev.max_image_levels_3d();
 204                return (desc->image_width > max ||
 205                        desc->image_height > max ||
 206                        desc->image_depth > max);
 207             }, ctx.devices()))
 208          throw error(CL_INVALID_IMAGE_SIZE);
 209
 210       return new image3d(ctx, flags, format,
 211                          desc->image_width, desc->image_height,
 212                          desc->image_depth, desc->image_row_pitch,
 213                          desc->image_slice_pitch, host_ptr);
 214
 215    case CL_MEM_OBJECT_IMAGE1D:
 216    case CL_MEM_OBJECT_IMAGE1D_ARRAY:
 217    case CL_MEM_OBJECT_IMAGE1D_BUFFER:
 218    case CL_MEM_OBJECT_IMAGE2D_ARRAY:
 219       // XXX - Not implemented.
 220       throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED);
 221
 222    default:
 223       throw error(CL_INVALID_IMAGE_DESCRIPTOR);
 224    }
 225
 226 } catch (error &e) {
 227    ret_error(r_errcode, e);
 228    return NULL;
 229 }
 230
 231 CLOVER_API cl_mem
 232 clCreateImage2D(cl_context d_ctx, cl_mem_flags d_flags,
 233                 const cl_image_format *format,
 234                 size_t width, size_t height, size_t row_pitch,
 235                 void *host_ptr, cl_int *r_errcode) {
 236    const cl_image_desc desc = { CL_MEM_OBJECT_IMAGE2D, width, height, 0, 0,
 237                                 row_pitch, 0, 0, 0, NULL };
 238
 239    return clCreateImage(d_ctx, d_flags, format, &desc, host_ptr, r_errcode);
 240 }
 241
 242 CLOVER_API cl_mem
 243 clCreateImage3D(cl_context d_ctx, cl_mem_flags d_flags,
 244                 const cl_image_format *format,
 245                 size_t width, size_t height, size_t depth,
 246                 size_t row_pitch, size_t slice_pitch,
 247                 void *host_ptr, cl_int *r_errcode) {
 248    const cl_image_desc desc = { CL_MEM_OBJECT_IMAGE3D, width, height, depth, 0,
 249                                 row_pitch, slice_pitch, 0, 0, NULL };
 250
 251    return clCreateImage(d_ctx, d_flags, format, &desc, host_ptr, r_errcode);
 252 }
 253
 254 CLOVER_API cl_int
 255 clGetSupportedImageFormats(cl_context d_ctx, cl_mem_flags flags,
 256                            cl_mem_object_type type, cl_uint count,
 257                            cl_image_format *r_buf, cl_uint *r_count) try {
 258    auto &ctx = obj(d_ctx);
 259    auto formats = supported_formats(ctx, type);
 260
 261    validate_flags(NULL, flags, false);
 262
 263    if (r_buf && !r_count)
 264       throw error(CL_INVALID_VALUE);
 265
 266    if (r_buf)
 267       std::copy_n(formats.begin(),
 268                   std::min((cl_uint)formats.size(), count),
 269                   r_buf);
 270
 271    if (r_count)
 272       *r_count = formats.size();
 273
 274    return CL_SUCCESS;
 275
 276 } catch (error &e) {
 277    return e.get();
 278 }
 279
 280 CLOVER_API cl_int
 281 clGetMemObjectInfo(cl_mem d_mem, cl_mem_info param,
 282                    size_t size, void *r_buf, size_t *r_size) try {
 283    property_buffer buf { r_buf, size, r_size };
 284    auto &mem = obj(d_mem);
 285
 286    switch (param) {
 287    case CL_MEM_TYPE:
 288       buf.as_scalar<cl_mem_object_type>() = mem.type();
 289       break;
 290
 291    case CL_MEM_FLAGS:
 292       buf.as_scalar<cl_mem_flags>() = mem.flags();
 293       break;
 294
 295    case CL_MEM_SIZE:
 296       buf.as_scalar<size_t>() = mem.size();
 297       break;
 298
 299    case CL_MEM_HOST_PTR:
 300       buf.as_scalar<void *>() = mem.host_ptr();
 301       break;
 302
 303    case CL_MEM_MAP_COUNT:
 304       buf.as_scalar<cl_uint>() = 0;
 305       break;
 306
 307    case CL_MEM_REFERENCE_COUNT:
 308       buf.as_scalar<cl_uint>() = mem.ref_count();
 309       break;
 310
 311    case CL_MEM_CONTEXT:
 312       buf.as_scalar<cl_context>() = desc(mem.context());
 313       break;
 314
 315    case CL_MEM_ASSOCIATED_MEMOBJECT: {
 316       sub_buffer *sub = dynamic_cast<sub_buffer *>(&mem);
 317       buf.as_scalar<cl_mem>() = (sub ? desc(sub->parent()) : NULL);
 318       break;
 319    }
 320    case CL_MEM_OFFSET: {
 321       sub_buffer *sub = dynamic_cast<sub_buffer *>(&mem);
 322       buf.as_scalar<size_t>() = (sub ? sub->offset() : 0);
 323       break;
 324    }
 325    case CL_MEM_USES_SVM_POINTER:
 326    case CL_MEM_USES_SVM_POINTER_ARM: {
 327       // with system SVM all host ptrs are SVM pointers
 328       // TODO: once we support devices with lower levels of SVM, we have to
 329       // check the ptr in more detail
 330       const bool system_svm = all_of(std::mem_fn(&device::has_system_svm),
 331                                      mem.context().devices());
 332       buf.as_scalar<cl_bool>() = mem.host_ptr() && system_svm;
 333       break;
 334    }
 335    default:
 336       throw error(CL_INVALID_VALUE);
 337    }
 338
 339    return CL_SUCCESS;
 340
 341 } catch (error &e) {
 342    return e.get();
 343 }
 344
 345 CLOVER_API cl_int
 346 clGetImageInfo(cl_mem d_mem, cl_image_info param,
 347                size_t size, void *r_buf, size_t *r_size) try {
 348    property_buffer buf { r_buf, size, r_size };
 349    auto &img = obj<image>(d_mem);
 350
 351    switch (param) {
 352    case CL_IMAGE_FORMAT:
 353       buf.as_scalar<cl_image_format>() = img.format();
 354       break;
 355
 356    case CL_IMAGE_ELEMENT_SIZE:
 357       buf.as_scalar<size_t>() = 0;
 358       break;
 359
 360    case CL_IMAGE_ROW_PITCH:
 361       buf.as_scalar<size_t>() = img.row_pitch();
 362       break;
 363
 364    case CL_IMAGE_SLICE_PITCH:
 365       buf.as_scalar<size_t>() = img.slice_pitch();
 366       break;
 367
 368    case CL_IMAGE_WIDTH:
 369       buf.as_scalar<size_t>() = img.width();
 370       break;
 371
 372    case CL_IMAGE_HEIGHT:
 373       buf.as_scalar<size_t>() = img.height();
 374       break;
 375
 376    case CL_IMAGE_DEPTH:
 377       buf.as_scalar<size_t>() = img.depth();
 378       break;
 379
 380    default:
 381       throw error(CL_INVALID_VALUE);
 382    }
 383
 384    return CL_SUCCESS;
 385
 386 } catch (error &e) {
 387    return e.get();
 388 }
 389
 390 CLOVER_API cl_int
 391 clRetainMemObject(cl_mem d_mem) try {
 392    obj(d_mem).retain();
 393    return CL_SUCCESS;
 394
 395 } catch (error &e) {
 396    return e.get();
 397 }
 398
 399 CLOVER_API cl_int
 400 clReleaseMemObject(cl_mem d_mem) try {
 401    if (obj(d_mem).release())
 402       delete pobj(d_mem);
 403
 404    return CL_SUCCESS;
 405
 406 } catch (error &e) {
 407    return e.get();
 408 }
 409
 410 CLOVER_API cl_int
 411 clSetMemObjectDestructorCallback(cl_mem d_mem,
 412                                  void (CL_CALLBACK *pfn_notify)(cl_mem, void *),
 413                                  void *user_data) try {
 414    auto &mem = obj(d_mem);
 415
 416    if (!pfn_notify)
 417       return CL_INVALID_VALUE;
 418
 419    mem.destroy_notify([=]{ pfn_notify(d_mem, user_data); });
 420
 421    return CL_SUCCESS;
 422
 423 } catch (error &e) {
 424    return e.get();
 425 }
 426
 427 CLOVER_API cl_int
 428 clEnqueueFillBuffer(cl_command_queue command_queue, cl_mem buffer,
 429                     const void *pattern, size_t pattern_size,
 430                     size_t offset, size_t size,
 431                     cl_uint num_events_in_wait_list,
 432                     const cl_event *event_wait_list,
 433                     cl_event *event) {
 434    CLOVER_NOT_SUPPORTED_UNTIL("1.2");
 435    return CL_INVALID_VALUE;
 436 }
 437
 438 CLOVER_API cl_int
 439 clEnqueueFillImage(cl_command_queue command_queue, cl_mem image,
 440                    const void *fill_color,
 441                    const size_t *origin, const size_t *region,
 442                    cl_uint num_events_in_wait_list,
 443                    const cl_event *event_wait_list,
 444                    cl_event *event) {
 445    CLOVER_NOT_SUPPORTED_UNTIL("1.2");
 446    return CL_INVALID_VALUE;
 447 }
 448
 449 CLOVER_API void *
 450 clSVMAlloc(cl_context d_ctx,
 451            cl_svm_mem_flags flags,
 452            size_t size,
 453            unsigned int alignment) try {
 454    auto &ctx = obj(d_ctx);
 455    validate_flags(NULL, flags, true);
 456
 457    if (!size ||
 458        size > fold(minimum(), cl_ulong(ULONG_MAX),
 459                    map(std::mem_fn(&device::max_mem_alloc_size), ctx.devices())))
 460       return nullptr;
 461
 462    if (!util_is_power_of_two_or_zero(alignment))
 463       return nullptr;
 464
 465    if (!alignment)
 466       alignment = 0x80; // sizeof(long16)
 467
 468    bool can_emulate = all_of(std::mem_fn(&device::has_system_svm), ctx.devices());
 469    if (can_emulate) {
 470       // we can ignore all the flags as it's not required to honor them.
 471       void *ptr = nullptr;
 472       if (alignment < sizeof(void*))
 473          alignment = sizeof(void*);
 474       posix_memalign(&ptr, alignment, size);
 475       return ptr;
 476    }
 477
 478    CLOVER_NOT_SUPPORTED_UNTIL("2.0");
 479    return nullptr;
 480
 481 } catch (error &e) {
 482    return nullptr;
 483 }
 484
 485 CLOVER_API void
 486 clSVMFree(cl_context d_ctx,
 487           void *svm_pointer) try {
 488    auto &ctx = obj(d_ctx);
 489    bool can_emulate = all_of(std::mem_fn(&device::has_system_svm), ctx.devices());
 490
 491    if (can_emulate)
 492       return free(svm_pointer);
 493
 494    CLOVER_NOT_SUPPORTED_UNTIL("2.0");
 495
 496 } catch (error &e) {
 497 }