src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_struct.h"
   8 #include "gallivm/lp_bld_type.h"
   9 #include "gallivm/lp_bld_flow.h"
  10 #include "gallivm/lp_bld_debug.h"
  11 #include "gallivm/lp_bld_tgsi.h"
  12 #include "gallivm/lp_bld_printf.h"
  13
  14 #include "tgsi/tgsi_exec.h"
  15
  16 #include "util/u_cpu_detect.h"
  17 #include "util/u_string.h"
  18
  19 #include <llvm-c/Transforms/Scalar.h>
  20
  21 #define DEBUG_STORE 0
  22
  23
  24 /* generates the draw jit function */
  25 static void
  26 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  27 static void
  28 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  29
  30 static void
  31 init_globals(struct draw_llvm *llvm)
  32 {
  33    LLVMTypeRef texture_type;
  34
  35    /* struct draw_jit_texture */
  36    {
  37       LLVMTypeRef elem_types[4];
  38
  39       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  41       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  42       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  43
  44       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  45
  46       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  47                              llvm->target, texture_type,
  48                              DRAW_JIT_TEXTURE_WIDTH);
  49       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  50                              llvm->target, texture_type,
  51                              DRAW_JIT_TEXTURE_HEIGHT);
  52       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  53                              llvm->target, texture_type,
  54                              DRAW_JIT_TEXTURE_STRIDE);
  55       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  56                              llvm->target, texture_type,
  57                              DRAW_JIT_TEXTURE_DATA);
  58       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  59                            llvm->target, texture_type);
  60
  61       LLVMAddTypeName(llvm->module, "texture", texture_type);
  62    }
  63
  64
  65    /* struct draw_jit_context */
  66    {
  67       LLVMTypeRef elem_types[3];
  68       LLVMTypeRef context_type;
  69
  70       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  71       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  72       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  73
  74       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  75
  76       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  77                              llvm->target, context_type, 0);
  78       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  79                              llvm->target, context_type, 1);
  80       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  81                              llvm->target, context_type,
  82                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  83       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  84                            llvm->target, context_type);
  85
  86       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  87
  88       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  89    }
  90    {
  91       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  92       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  93    }
  94    /* struct pipe_vertex_buffer */
  95    {
  96       LLVMTypeRef elem_types[4];
  97       LLVMTypeRef vb_type;
  98
  99       elem_types[0] = LLVMInt32Type();
 100       elem_types[1] = LLVMInt32Type();
 101       elem_types[2] = LLVMInt32Type();
 102       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 103
 104       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 105
 106       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 107                              llvm->target, vb_type, 0);
 108       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 109                              llvm->target, vb_type, 2);
 110       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 111                            llvm->target, vb_type);
 112
 113       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 114
 115       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 116    }
 117 }
 118
 119 static LLVMTypeRef
 120 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 121 {
 122    /* struct vertex_header */
 123    LLVMTypeRef elem_types[3];
 124    LLVMTypeRef vertex_header;
 125    char struct_name[24];
 126
 127    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 128
 129    elem_types[0]  = LLVMIntType(32);
 130    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 131    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 132
 133    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 134
 135    /* these are bit-fields and we can't take address of them
 136       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 137       llvm->target, vertex_header,
 138       DRAW_JIT_VERTEX_CLIPMASK);
 139       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 140       llvm->target, vertex_header,
 141       DRAW_JIT_VERTEX_EDGEFLAG);
 142       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 143       llvm->target, vertex_header,
 144       DRAW_JIT_VERTEX_PAD);
 145       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 146       llvm->target, vertex_header,
 147       DRAW_JIT_VERTEX_VERTEX_ID);
 148    */
 149    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 150                           llvm->target, vertex_header,
 151                           DRAW_JIT_VERTEX_CLIP);
 152    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 153                           llvm->target, vertex_header,
 154                           DRAW_JIT_VERTEX_DATA);
 155
 156    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 157
 158    return LLVMPointerType(vertex_header, 0);
 159 }
 160
 161 struct draw_llvm *
 162 draw_llvm_create(struct draw_context *draw)
 163 {
 164    struct draw_llvm *llvm;
 165
 166 #ifdef PIPE_ARCH_X86
 167    util_cpu_detect();
 168    /* require SSE2 due to LLVM PR6960. */
 169    if (!util_cpu_caps.has_sse2)
 170        return NULL;
 171 #endif
 172
 173    llvm = CALLOC_STRUCT( draw_llvm );
 174
 175    llvm->draw = draw;
 176    llvm->engine = draw->engine;
 177
 178    debug_assert(llvm->engine);
 179
 180    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 181    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 182
 183    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 184
 185    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 186
 187    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 188    LLVMAddTargetData(llvm->target, llvm->pass);
 189    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 190     * but there are more on SVN. */
 191    /* TODO: Add more passes */
 192    LLVMAddCFGSimplificationPass(llvm->pass);
 193    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 194    LLVMAddConstantPropagationPass(llvm->pass);
 195    if(util_cpu_caps.has_sse4_1) {
 196       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 197        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 198        * somehow becomes invalid code.
 199        */
 200       LLVMAddInstructionCombiningPass(llvm->pass);
 201    }
 202    LLVMAddGVNPass(llvm->pass);
 203
 204    init_globals(llvm);
 205
 206
 207 #if 0
 208    LLVMDumpModule(lp_build_module);
 209 #endif
 210
 211    return llvm;
 212 }
 213
 214 void
 215 draw_llvm_destroy(struct draw_llvm *llvm)
 216 {
 217    LLVMDisposePassManager(llvm->pass);
 218
 219    FREE(llvm);
 220 }
 221
 222 struct draw_llvm_variant *
 223 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 224 {
 225    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 226
 227    draw_llvm_make_variant_key(llvm, &variant->key);
 228
 229    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 230
 231    draw_llvm_generate(llvm, variant);
 232    draw_llvm_generate_elts(llvm, variant);
 233
 234    return variant;
 235 }
 236
 237 static void
 238 generate_vs(struct draw_llvm *llvm,
 239             LLVMBuilderRef builder,
 240             LLVMValueRef (*outputs)[NUM_CHANNELS],
 241             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 242             LLVMValueRef context_ptr)
 243 {
 244    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 245    struct lp_type vs_type;
 246    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 247
 248    memset(&vs_type, 0, sizeof vs_type);
 249    vs_type.floating = TRUE; /* floating point values */
 250    vs_type.sign = TRUE;     /* values are signed */
 251    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 252    vs_type.width = 32;      /* 32-bit float */
 253    vs_type.length = 4;      /* 4 elements per vector */
 254 #if 0
 255    num_vs = 4;              /* number of vertices per block */
 256 #endif
 257
 258    /*tgsi_dump(tokens, 0);*/
 259    lp_build_tgsi_soa(builder,
 260                      tokens,
 261                      vs_type,
 262                      NULL /*struct lp_build_mask_context *mask*/,
 263                      consts_ptr,
 264                      NULL /*pos*/,
 265                      inputs,
 266                      outputs,
 267                      NULL/*sampler*/,
 268                      &llvm->draw->vs.vertex_shader->info);
 269 }
 270
 271 #if DEBUG_STORE
 272 static void print_vectorf(LLVMBuilderRef builder,
 273                          LLVMValueRef vec)
 274 {
 275    LLVMValueRef val[4];
 276    val[0] = LLVMBuildExtractElement(builder, vec,
 277                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 278    val[1] = LLVMBuildExtractElement(builder, vec,
 279                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 280    val[2] = LLVMBuildExtractElement(builder, vec,
 281                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 282    val[3] = LLVMBuildExtractElement(builder, vec,
 283                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 284    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 285                    val[0], val[1], val[2], val[3]);
 286 }
 287 #endif
 288
 289 static void
 290 generate_fetch(LLVMBuilderRef builder,
 291                LLVMValueRef vbuffers_ptr,
 292                LLVMValueRef *res,
 293                struct pipe_vertex_element *velem,
 294                LLVMValueRef vbuf,
 295                LLVMValueRef index)
 296 {
 297    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 298    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 299                                            &indices, 1, "");
 300    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 301    LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf);
 302    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 303    LLVMValueRef cond;
 304    LLVMValueRef stride;
 305
 306    cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, "");
 307
 308    index = LLVMBuildSelect(builder, cond, index, vb_max_index, "");
 309
 310    stride = LLVMBuildMul(builder, vb_stride, index, "");
 311
 312    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 313
 314    stride = LLVMBuildAdd(builder, stride,
 315                          vb_buffer_offset,
 316                          "");
 317    stride = LLVMBuildAdd(builder, stride,
 318                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 319                          "");
 320
 321    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 322    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 323
 324    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 325 }
 326
 327 static LLVMValueRef
 328 aos_to_soa(LLVMBuilderRef builder,
 329            LLVMValueRef val0,
 330            LLVMValueRef val1,
 331            LLVMValueRef val2,
 332            LLVMValueRef val3,
 333            LLVMValueRef channel)
 334 {
 335    LLVMValueRef ex, res;
 336
 337    ex = LLVMBuildExtractElement(builder, val0,
 338                                 channel, "");
 339    res = LLVMBuildInsertElement(builder,
 340                                 LLVMConstNull(LLVMTypeOf(val0)),
 341                                 ex,
 342                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 343                                 "");
 344
 345    ex = LLVMBuildExtractElement(builder, val1,
 346                                 channel, "");
 347    res = LLVMBuildInsertElement(builder,
 348                                 res, ex,
 349                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 350                                 "");
 351
 352    ex = LLVMBuildExtractElement(builder, val2,
 353                                 channel, "");
 354    res = LLVMBuildInsertElement(builder,
 355                                 res, ex,
 356                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 357                                 "");
 358
 359    ex = LLVMBuildExtractElement(builder, val3,
 360                                 channel, "");
 361    res = LLVMBuildInsertElement(builder,
 362                                 res, ex,
 363                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 364                                 "");
 365
 366    return res;
 367 }
 368
 369 static void
 370 soa_to_aos(LLVMBuilderRef builder,
 371            LLVMValueRef soa[NUM_CHANNELS],
 372            LLVMValueRef aos[NUM_CHANNELS])
 373 {
 374    LLVMValueRef comp;
 375    int i = 0;
 376
 377    debug_assert(NUM_CHANNELS == 4);
 378
 379    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 380    aos[1] = aos[2] = aos[3] = aos[0];
 381
 382    for (i = 0; i < NUM_CHANNELS; ++i) {
 383       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 384
 385       comp = LLVMBuildExtractElement(builder, soa[i],
 386                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 387       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 388
 389       comp = LLVMBuildExtractElement(builder, soa[i],
 390                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 391       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 392
 393       comp = LLVMBuildExtractElement(builder, soa[i],
 394                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 395       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 396
 397       comp = LLVMBuildExtractElement(builder, soa[i],
 398                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 399       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 400
 401    }
 402 }
 403
 404 static void
 405 convert_to_soa(LLVMBuilderRef builder,
 406                LLVMValueRef (*aos)[NUM_CHANNELS],
 407                LLVMValueRef (*soa)[NUM_CHANNELS],
 408                int num_attribs)
 409 {
 410    int i;
 411
 412    debug_assert(NUM_CHANNELS == 4);
 413
 414    for (i = 0; i < num_attribs; ++i) {
 415       LLVMValueRef val0 = aos[i][0];
 416       LLVMValueRef val1 = aos[i][1];
 417       LLVMValueRef val2 = aos[i][2];
 418       LLVMValueRef val3 = aos[i][3];
 419
 420       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 421                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 422       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 423                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 424       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 425                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 426       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 427                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 428    }
 429 }
 430
 431 static void
 432 store_aos(LLVMBuilderRef builder,
 433           LLVMValueRef io_ptr,
 434           LLVMValueRef index,
 435           LLVMValueRef value)
 436 {
 437    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 438    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 439    LLVMValueRef indices[3];
 440
 441    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 442    indices[1] = index;
 443    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 444
 445    /* undefined vertex */
 446    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 447                                         0xffff, 0), id_ptr);
 448
 449 #if DEBUG_STORE
 450    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 451 #endif
 452 #if 0
 453    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 454      print_vectorf(builder, value);*/
 455    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 456                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 457                                "datavec");
 458    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 459
 460    LLVMBuildStore(builder, value, data_ptr);
 461 #else
 462    {
 463       LLVMValueRef x, y, z, w;
 464       LLVMValueRef idx0, idx1, idx2, idx3;
 465       LLVMValueRef gep0, gep1, gep2, gep3;
 466       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 467
 468       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 469       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 470       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 471       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 472
 473       x = LLVMBuildExtractElement(builder, value,
 474                                   idx0, "");
 475       y = LLVMBuildExtractElement(builder, value,
 476                                   idx1, "");
 477       z = LLVMBuildExtractElement(builder, value,
 478                                   idx2, "");
 479       w = LLVMBuildExtractElement(builder, value,
 480                                   idx3, "");
 481
 482       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 483       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 484       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 485       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 486
 487       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 488         x, gep0, y, gep1, z, gep2, w, gep3);*/
 489       LLVMBuildStore(builder, x, gep0);
 490       LLVMBuildStore(builder, y, gep1);
 491       LLVMBuildStore(builder, z, gep2);
 492       LLVMBuildStore(builder, w, gep3);
 493    }
 494 #endif
 495 }
 496
 497 static void
 498 store_aos_array(LLVMBuilderRef builder,
 499                 LLVMValueRef io_ptr,
 500                 LLVMValueRef aos[NUM_CHANNELS],
 501                 int attrib,
 502                 int num_outputs)
 503 {
 504    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 505    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 506    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 507    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 508    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 509    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 510
 511    debug_assert(NUM_CHANNELS == 4);
 512
 513    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 514                           &ind0, 1, "");
 515    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 516                           &ind1, 1, "");
 517    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 518                           &ind2, 1, "");
 519    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 520                           &ind3, 1, "");
 521
 522 #if DEBUG_STORE
 523    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 524                    io_ptr, ind0, ind1, ind2, ind3);
 525 #endif
 526
 527    store_aos(builder, io0_ptr, attr_index, aos[0]);
 528    store_aos(builder, io1_ptr, attr_index, aos[1]);
 529    store_aos(builder, io2_ptr, attr_index, aos[2]);
 530    store_aos(builder, io3_ptr, attr_index, aos[3]);
 531 }
 532
 533 static void
 534 convert_to_aos(LLVMBuilderRef builder,
 535                LLVMValueRef io,
 536                LLVMValueRef (*outputs)[NUM_CHANNELS],
 537                int num_outputs,
 538                int max_vertices)
 539 {
 540    unsigned chan, attrib;
 541
 542 #if DEBUG_STORE
 543    lp_build_printf(builder, "   # storing begin\n");
 544 #endif
 545    for (attrib = 0; attrib < num_outputs; ++attrib) {
 546       LLVMValueRef soa[4];
 547       LLVMValueRef aos[4];
 548       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 549          if(outputs[attrib][chan]) {
 550             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 551             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 552             /*lp_build_printf(builder, "output %d : %d ",
 553                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 554                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 555               print_vectorf(builder, out);*/
 556             soa[chan] = out;
 557          } else
 558             soa[chan] = 0;
 559       }
 560       soa_to_aos(builder, soa, aos);
 561       store_aos_array(builder,
 562                       io,
 563                       aos,
 564                       attrib,
 565                       num_outputs);
 566    }
 567 #if DEBUG_STORE
 568    lp_build_printf(builder, "   # storing end\n");
 569 #endif
 570 }
 571
 572 static void
 573 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 574 {
 575    LLVMTypeRef arg_types[7];
 576    LLVMTypeRef func_type;
 577    LLVMValueRef context_ptr;
 578    LLVMBasicBlockRef block;
 579    LLVMBuilderRef builder;
 580    LLVMValueRef start, end, count, stride, step, io_itr;
 581    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 582    struct draw_context *draw = llvm->draw;
 583    unsigned i, j;
 584    struct lp_build_context bld;
 585    struct lp_build_loop_state lp_loop;
 586    struct lp_type vs_type = lp_type_float_vec(32);
 587    const int max_vertices = 4;
 588    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 589
 590    arg_types[0] = llvm->context_ptr_type;           /* context */
 591    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 592    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 593    arg_types[3] = LLVMInt32Type();                  /* start */
 594    arg_types[4] = LLVMInt32Type();                  /* count */
 595    arg_types[5] = LLVMInt32Type();                  /* stride */
 596    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 597
 598    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 599
 600    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 601    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 602    for(i = 0; i < Elements(arg_types); ++i)
 603       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 604          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 605
 606    context_ptr  = LLVMGetParam(variant->function, 0);
 607    io_ptr       = LLVMGetParam(variant->function, 1);
 608    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 609    start        = LLVMGetParam(variant->function, 3);
 610    count        = LLVMGetParam(variant->function, 4);
 611    stride       = LLVMGetParam(variant->function, 5);
 612    vb_ptr       = LLVMGetParam(variant->function, 6);
 613
 614    lp_build_name(context_ptr, "context");
 615    lp_build_name(io_ptr, "io");
 616    lp_build_name(vbuffers_ptr, "vbuffers");
 617    lp_build_name(start, "start");
 618    lp_build_name(count, "count");
 619    lp_build_name(stride, "stride");
 620    lp_build_name(vb_ptr, "vb");
 621
 622    /*
 623     * Function body
 624     */
 625
 626    block = LLVMAppendBasicBlock(variant->function, "entry");
 627    builder = LLVMCreateBuilder();
 628    LLVMPositionBuilderAtEnd(builder, block);
 629
 630    lp_build_context_init(&bld, builder, vs_type);
 631
 632    end = lp_build_add(&bld, start, count);
 633
 634    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 635
 636 #if DEBUG_STORE
 637    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 638                    start, end, step);
 639 #endif
 640    lp_build_loop_begin(builder, start, &lp_loop);
 641    {
 642       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 643       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 644       LLVMValueRef io;
 645       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 646
 647       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 648       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 649 #if DEBUG_STORE
 650       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 651                       io_itr, io, lp_loop.counter);
 652 #endif
 653       for (i = 0; i < NUM_CHANNELS; ++i) {
 654          LLVMValueRef true_index = LLVMBuildAdd(
 655             builder,
 656             lp_loop.counter,
 657             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 658          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 659             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 660             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 661                                                  velem->vertex_buffer_index,
 662                                                  0);
 663             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 664                                            &vb_index, 1, "");
 665             generate_fetch(builder, vbuffers_ptr,
 666                            &aos_attribs[j][i], velem, vb, true_index);
 667          }
 668       }
 669       convert_to_soa(builder, aos_attribs, inputs,
 670                      draw->pt.nr_vertex_elements);
 671
 672       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 673       generate_vs(llvm,
 674                   builder,
 675                   outputs,
 676                   ptr_aos,
 677                   context_ptr);
 678
 679       convert_to_aos(builder, io, outputs,
 680                      draw->vs.vertex_shader->info.num_outputs,
 681                      max_vertices);
 682    }
 683    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 684
 685    LLVMBuildRetVoid(builder);
 686
 687    LLVMDisposeBuilder(builder);
 688
 689    /*
 690     * Translate the LLVM IR into machine code.
 691     */
 692 #ifdef DEBUG
 693    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 694       lp_debug_dump_value(variant->function);
 695       assert(0);
 696    }
 697 #endif
 698
 699    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 700
 701    if (0) {
 702       lp_debug_dump_value(variant->function);
 703       debug_printf("\n");
 704    }
 705    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 706
 707    if (0)
 708       lp_disassemble(variant->jit_func);
 709 }
 710
 711
 712 static void
 713 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 714 {
 715    LLVMTypeRef arg_types[7];
 716    LLVMTypeRef func_type;
 717    LLVMValueRef context_ptr;
 718    LLVMBasicBlockRef block;
 719    LLVMBuilderRef builder;
 720    LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
 721    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 722    struct draw_context *draw = llvm->draw;
 723    unsigned i, j;
 724    struct lp_build_context bld;
 725    struct lp_build_context bld_int;
 726    struct lp_build_loop_state lp_loop;
 727    struct lp_type vs_type = lp_type_float_vec(32);
 728    const int max_vertices = 4;
 729    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 730    LLVMValueRef fetch_max;
 731
 732    arg_types[0] = llvm->context_ptr_type;               /* context */
 733    arg_types[1] = llvm->vertex_header_ptr_type;         /* vertex_header */
 734    arg_types[2] = llvm->buffer_ptr_type;                /* vbuffers */
 735    arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0);  /* fetch_elts * */
 736    arg_types[4] = LLVMInt32Type();                      /* fetch_count */
 737    arg_types[5] = LLVMInt32Type();                      /* stride */
 738    arg_types[6] = llvm->vb_ptr_type;                    /* pipe_vertex_buffer's */
 739
 740    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 741
 742    variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type);
 743    LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
 744    for(i = 0; i < Elements(arg_types); ++i)
 745       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 746          LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute);
 747
 748    context_ptr  = LLVMGetParam(variant->function_elts, 0);
 749    io_ptr       = LLVMGetParam(variant->function_elts, 1);
 750    vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
 751    fetch_elts   = LLVMGetParam(variant->function_elts, 3);
 752    fetch_count  = LLVMGetParam(variant->function_elts, 4);
 753    stride       = LLVMGetParam(variant->function_elts, 5);
 754    vb_ptr       = LLVMGetParam(variant->function_elts, 6);
 755
 756    lp_build_name(context_ptr, "context");
 757    lp_build_name(io_ptr, "io");
 758    lp_build_name(vbuffers_ptr, "vbuffers");
 759    lp_build_name(fetch_elts, "fetch_elts");
 760    lp_build_name(fetch_count, "fetch_count");
 761    lp_build_name(stride, "stride");
 762    lp_build_name(vb_ptr, "vb");
 763
 764    /*
 765     * Function body
 766     */
 767
 768    block = LLVMAppendBasicBlock(variant->function_elts, "entry");
 769    builder = LLVMCreateBuilder();
 770    LLVMPositionBuilderAtEnd(builder, block);
 771
 772    lp_build_context_init(&bld, builder, vs_type);
 773    lp_build_context_init(&bld_int, builder, lp_type_int(32));
 774
 775    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 776
 777    fetch_max = LLVMBuildSub(builder, fetch_count,
 778                             LLVMConstInt(LLVMInt32Type(), 1, 0),
 779                             "fetch_max");
 780
 781    lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop);
 782    {
 783       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 784       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 785       LLVMValueRef io;
 786       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 787
 788       io_itr = lp_loop.counter;
 789       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 790 #if DEBUG_STORE
 791       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 792                       io_itr, io, lp_loop.counter);
 793 #endif
 794       for (i = 0; i < NUM_CHANNELS; ++i) {
 795          LLVMValueRef true_index = LLVMBuildAdd(
 796             builder,
 797             lp_loop.counter,
 798             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 799          LLVMValueRef fetch_ptr;
 800
 801          /* make sure we're not out of bounds which can happen
 802           * if fetch_count % 4 != 0, because on the last iteration
 803           * a few of the 4 vertex fetches will be out of bounds */
 804          true_index = lp_build_min(&bld_int, true_index, fetch_max);
 805
 806          fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
 807                                   &true_index, 1, "");
 808          true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
 809          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 810             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 811             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 812                                                  velem->vertex_buffer_index,
 813                                                  0);
 814             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 815                                            &vb_index, 1, "");
 816             generate_fetch(builder, vbuffers_ptr,
 817                            &aos_attribs[j][i], velem, vb, true_index);
 818          }
 819       }
 820       convert_to_soa(builder, aos_attribs, inputs,
 821                      draw->pt.nr_vertex_elements);
 822
 823       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 824       generate_vs(llvm,
 825                   builder,
 826                   outputs,
 827                   ptr_aos,
 828                   context_ptr);
 829
 830       convert_to_aos(builder, io, outputs,
 831                      draw->vs.vertex_shader->info.num_outputs,
 832                      max_vertices);
 833    }
 834    lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop);
 835
 836    LLVMBuildRetVoid(builder);
 837
 838    LLVMDisposeBuilder(builder);
 839
 840    /*
 841     * Translate the LLVM IR into machine code.
 842     */
 843 #ifdef DEBUG
 844    if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
 845       lp_debug_dump_value(variant->function_elts);
 846       assert(0);
 847    }
 848 #endif
 849
 850    LLVMRunFunctionPassManager(llvm->pass, variant->function_elts);
 851
 852    if (0) {
 853       lp_debug_dump_value(variant->function_elts);
 854       debug_printf("\n");
 855    }
 856    variant->jit_func_elts = (draw_jit_vert_func_elts)LLVMGetPointerToGlobal(
 857       llvm->draw->engine, variant->function_elts);
 858
 859    if (0)
 860       lp_disassemble(variant->jit_func_elts);
 861 }
 862
 863 void
 864 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 865                            struct draw_llvm_variant_key *key)
 866 {
 867    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 868
 869    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 870
 871    memcpy(key->vertex_element,
 872           llvm->draw->pt.vertex_element,
 873           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 874
 875    memcpy(&key->vs,
 876           &llvm->draw->vs.vertex_shader->state,
 877           sizeof(struct pipe_shader_state));
 878 }