src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_struct.h"
   8 #include "gallivm/lp_bld_type.h"
   9 #include "gallivm/lp_bld_flow.h"
  10 #include "gallivm/lp_bld_debug.h"
  11 #include "gallivm/lp_bld_tgsi.h"
  12 #include "gallivm/lp_bld_printf.h"
  13
  14 #include "tgsi/tgsi_exec.h"
  15
  16 #include "util/u_cpu_detect.h"
  17 #include "util/u_string.h"
  18
  19 #include <llvm-c/Transforms/Scalar.h>
  20
  21 #define DEBUG_STORE 0
  22
  23
  24 /** cast wrapper */
  25 static INLINE draw_jit_vert_func_elts
  26 voidptr_to_draw_vert_func_elts(void *v)
  27 {
  28    union {
  29       void *v;
  30       draw_jit_vert_func_elts f;
  31    } u;
  32    assert(sizeof(u.v) == sizeof(u.f));
  33    u.v = v;
  34    return u.f;
  35 }
  36
  37
  38 /** cast wrapper */
  39 static INLINE draw_jit_vert_func
  40 voidptr_to_draw_jit_vert_func(void *v)
  41 {
  42    union {
  43       void *v;
  44       draw_jit_vert_func f;
  45    } u;
  46    assert(sizeof(u.v) == sizeof(u.f));
  47    u.v = v;
  48    return u.f;
  49 }
  50
  51
  52 /* generates the draw jit function */
  53 static void
  54 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  55 static void
  56 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  57
  58 static void
  59 init_globals(struct draw_llvm *llvm)
  60 {
  61    LLVMTypeRef texture_type;
  62
  63    /* struct draw_jit_texture */
  64    {
  65       LLVMTypeRef elem_types[4];
  66
  67       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  68       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  69       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  70       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  71
  72       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  73
  74       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  75                              llvm->target, texture_type,
  76                              DRAW_JIT_TEXTURE_WIDTH);
  77       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  78                              llvm->target, texture_type,
  79                              DRAW_JIT_TEXTURE_HEIGHT);
  80       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  81                              llvm->target, texture_type,
  82                              DRAW_JIT_TEXTURE_STRIDE);
  83       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  84                              llvm->target, texture_type,
  85                              DRAW_JIT_TEXTURE_DATA);
  86       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  87                            llvm->target, texture_type);
  88
  89       LLVMAddTypeName(llvm->module, "texture", texture_type);
  90    }
  91
  92
  93    /* struct draw_jit_context */
  94    {
  95       LLVMTypeRef elem_types[3];
  96       LLVMTypeRef context_type;
  97
  98       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  99       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
 100       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
 101
 102       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 103
 104       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
 105                              llvm->target, context_type, 0);
 106       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
 107                              llvm->target, context_type, 1);
 108       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
 109                              llvm->target, context_type,
 110                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
 111       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
 112                            llvm->target, context_type);
 113
 114       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
 115
 116       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
 117    }
 118    {
 119       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
 120       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
 121    }
 122    /* struct pipe_vertex_buffer */
 123    {
 124       LLVMTypeRef elem_types[4];
 125       LLVMTypeRef vb_type;
 126
 127       elem_types[0] = LLVMInt32Type();
 128       elem_types[1] = LLVMInt32Type();
 129       elem_types[2] = LLVMInt32Type();
 130       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 131
 132       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 133
 134       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 135                              llvm->target, vb_type, 0);
 136       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 137                              llvm->target, vb_type, 2);
 138       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 139                            llvm->target, vb_type);
 140
 141       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 142
 143       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 144    }
 145 }
 146
 147 static LLVMTypeRef
 148 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 149 {
 150    /* struct vertex_header */
 151    LLVMTypeRef elem_types[3];
 152    LLVMTypeRef vertex_header;
 153    char struct_name[24];
 154
 155    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 156
 157    elem_types[0]  = LLVMIntType(32);
 158    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 159    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 160
 161    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 162
 163    /* these are bit-fields and we can't take address of them
 164       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 165       llvm->target, vertex_header,
 166       DRAW_JIT_VERTEX_CLIPMASK);
 167       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 168       llvm->target, vertex_header,
 169       DRAW_JIT_VERTEX_EDGEFLAG);
 170       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 171       llvm->target, vertex_header,
 172       DRAW_JIT_VERTEX_PAD);
 173       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 174       llvm->target, vertex_header,
 175       DRAW_JIT_VERTEX_VERTEX_ID);
 176    */
 177    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 178                           llvm->target, vertex_header,
 179                           DRAW_JIT_VERTEX_CLIP);
 180    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 181                           llvm->target, vertex_header,
 182                           DRAW_JIT_VERTEX_DATA);
 183
 184    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 185
 186    return LLVMPointerType(vertex_header, 0);
 187 }
 188
 189 struct draw_llvm *
 190 draw_llvm_create(struct draw_context *draw)
 191 {
 192    struct draw_llvm *llvm;
 193
 194 #ifdef PIPE_ARCH_X86
 195    util_cpu_detect();
 196    /* require SSE2 due to LLVM PR6960. */
 197    if (!util_cpu_caps.has_sse2)
 198        return NULL;
 199 #endif
 200
 201    llvm = CALLOC_STRUCT( draw_llvm );
 202
 203    llvm->draw = draw;
 204    llvm->engine = draw->engine;
 205
 206    debug_assert(llvm->engine);
 207
 208    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 209    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 210
 211    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 212
 213    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 214
 215    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 216    LLVMAddTargetData(llvm->target, llvm->pass);
 217    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 218     * but there are more on SVN. */
 219    /* TODO: Add more passes */
 220    LLVMAddCFGSimplificationPass(llvm->pass);
 221    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 222    LLVMAddConstantPropagationPass(llvm->pass);
 223    if(util_cpu_caps.has_sse4_1) {
 224       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 225        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 226        * somehow becomes invalid code.
 227        */
 228       LLVMAddInstructionCombiningPass(llvm->pass);
 229    }
 230    LLVMAddGVNPass(llvm->pass);
 231
 232    init_globals(llvm);
 233
 234
 235 #if 0
 236    LLVMDumpModule(lp_build_module);
 237 #endif
 238
 239    return llvm;
 240 }
 241
 242 void
 243 draw_llvm_destroy(struct draw_llvm *llvm)
 244 {
 245    LLVMDisposePassManager(llvm->pass);
 246
 247    FREE(llvm);
 248 }
 249
 250 struct draw_llvm_variant *
 251 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 252 {
 253    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 254
 255    draw_llvm_make_variant_key(llvm, &variant->key);
 256
 257    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 258
 259    draw_llvm_generate(llvm, variant);
 260    draw_llvm_generate_elts(llvm, variant);
 261
 262    return variant;
 263 }
 264
 265 static void
 266 generate_vs(struct draw_llvm *llvm,
 267             LLVMBuilderRef builder,
 268             LLVMValueRef (*outputs)[NUM_CHANNELS],
 269             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 270             LLVMValueRef context_ptr)
 271 {
 272    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 273    struct lp_type vs_type;
 274    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 275
 276    memset(&vs_type, 0, sizeof vs_type);
 277    vs_type.floating = TRUE; /* floating point values */
 278    vs_type.sign = TRUE;     /* values are signed */
 279    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 280    vs_type.width = 32;      /* 32-bit float */
 281    vs_type.length = 4;      /* 4 elements per vector */
 282 #if 0
 283    num_vs = 4;              /* number of vertices per block */
 284 #endif
 285
 286    /*tgsi_dump(tokens, 0);*/
 287    lp_build_tgsi_soa(builder,
 288                      tokens,
 289                      vs_type,
 290                      NULL /*struct lp_build_mask_context *mask*/,
 291                      consts_ptr,
 292                      NULL /*pos*/,
 293                      inputs,
 294                      outputs,
 295                      NULL/*sampler*/,
 296                      &llvm->draw->vs.vertex_shader->info);
 297 }
 298
 299 #if DEBUG_STORE
 300 static void print_vectorf(LLVMBuilderRef builder,
 301                          LLVMValueRef vec)
 302 {
 303    LLVMValueRef val[4];
 304    val[0] = LLVMBuildExtractElement(builder, vec,
 305                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 306    val[1] = LLVMBuildExtractElement(builder, vec,
 307                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 308    val[2] = LLVMBuildExtractElement(builder, vec,
 309                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 310    val[3] = LLVMBuildExtractElement(builder, vec,
 311                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 312    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 313                    val[0], val[1], val[2], val[3]);
 314 }
 315 #endif
 316
 317 static void
 318 generate_fetch(LLVMBuilderRef builder,
 319                LLVMValueRef vbuffers_ptr,
 320                LLVMValueRef *res,
 321                struct pipe_vertex_element *velem,
 322                LLVMValueRef vbuf,
 323                LLVMValueRef index)
 324 {
 325    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 326    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 327                                            &indices, 1, "");
 328    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 329    LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf);
 330    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 331    LLVMValueRef cond;
 332    LLVMValueRef stride;
 333
 334    cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, "");
 335
 336    index = LLVMBuildSelect(builder, cond, index, vb_max_index, "");
 337
 338    stride = LLVMBuildMul(builder, vb_stride, index, "");
 339
 340    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 341
 342    stride = LLVMBuildAdd(builder, stride,
 343                          vb_buffer_offset,
 344                          "");
 345    stride = LLVMBuildAdd(builder, stride,
 346                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 347                          "");
 348
 349    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 350    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 351
 352    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 353 }
 354
 355 static LLVMValueRef
 356 aos_to_soa(LLVMBuilderRef builder,
 357            LLVMValueRef val0,
 358            LLVMValueRef val1,
 359            LLVMValueRef val2,
 360            LLVMValueRef val3,
 361            LLVMValueRef channel)
 362 {
 363    LLVMValueRef ex, res;
 364
 365    ex = LLVMBuildExtractElement(builder, val0,
 366                                 channel, "");
 367    res = LLVMBuildInsertElement(builder,
 368                                 LLVMConstNull(LLVMTypeOf(val0)),
 369                                 ex,
 370                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 371                                 "");
 372
 373    ex = LLVMBuildExtractElement(builder, val1,
 374                                 channel, "");
 375    res = LLVMBuildInsertElement(builder,
 376                                 res, ex,
 377                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 378                                 "");
 379
 380    ex = LLVMBuildExtractElement(builder, val2,
 381                                 channel, "");
 382    res = LLVMBuildInsertElement(builder,
 383                                 res, ex,
 384                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 385                                 "");
 386
 387    ex = LLVMBuildExtractElement(builder, val3,
 388                                 channel, "");
 389    res = LLVMBuildInsertElement(builder,
 390                                 res, ex,
 391                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 392                                 "");
 393
 394    return res;
 395 }
 396
 397 static void
 398 soa_to_aos(LLVMBuilderRef builder,
 399            LLVMValueRef soa[NUM_CHANNELS],
 400            LLVMValueRef aos[NUM_CHANNELS])
 401 {
 402    LLVMValueRef comp;
 403    int i = 0;
 404
 405    debug_assert(NUM_CHANNELS == 4);
 406
 407    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 408    aos[1] = aos[2] = aos[3] = aos[0];
 409
 410    for (i = 0; i < NUM_CHANNELS; ++i) {
 411       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 412
 413       comp = LLVMBuildExtractElement(builder, soa[i],
 414                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 415       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 416
 417       comp = LLVMBuildExtractElement(builder, soa[i],
 418                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 419       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 420
 421       comp = LLVMBuildExtractElement(builder, soa[i],
 422                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 423       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 424
 425       comp = LLVMBuildExtractElement(builder, soa[i],
 426                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 427       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 428
 429    }
 430 }
 431
 432 static void
 433 convert_to_soa(LLVMBuilderRef builder,
 434                LLVMValueRef (*aos)[NUM_CHANNELS],
 435                LLVMValueRef (*soa)[NUM_CHANNELS],
 436                int num_attribs)
 437 {
 438    int i;
 439
 440    debug_assert(NUM_CHANNELS == 4);
 441
 442    for (i = 0; i < num_attribs; ++i) {
 443       LLVMValueRef val0 = aos[i][0];
 444       LLVMValueRef val1 = aos[i][1];
 445       LLVMValueRef val2 = aos[i][2];
 446       LLVMValueRef val3 = aos[i][3];
 447
 448       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 449                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 450       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 451                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 452       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 453                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 454       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 455                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 456    }
 457 }
 458
 459 static void
 460 store_aos(LLVMBuilderRef builder,
 461           LLVMValueRef io_ptr,
 462           LLVMValueRef index,
 463           LLVMValueRef value)
 464 {
 465    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 466    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 467    LLVMValueRef indices[3];
 468
 469    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 470    indices[1] = index;
 471    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 472
 473    /* undefined vertex */
 474    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 475                                         0xffff, 0), id_ptr);
 476
 477 #if DEBUG_STORE
 478    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 479 #endif
 480 #if 0
 481    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 482      print_vectorf(builder, value);*/
 483    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 484                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 485                                "datavec");
 486    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 487
 488    LLVMBuildStore(builder, value, data_ptr);
 489 #else
 490    {
 491       LLVMValueRef x, y, z, w;
 492       LLVMValueRef idx0, idx1, idx2, idx3;
 493       LLVMValueRef gep0, gep1, gep2, gep3;
 494       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 495
 496       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 497       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 498       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 499       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 500
 501       x = LLVMBuildExtractElement(builder, value,
 502                                   idx0, "");
 503       y = LLVMBuildExtractElement(builder, value,
 504                                   idx1, "");
 505       z = LLVMBuildExtractElement(builder, value,
 506                                   idx2, "");
 507       w = LLVMBuildExtractElement(builder, value,
 508                                   idx3, "");
 509
 510       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 511       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 512       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 513       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 514
 515       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 516         x, gep0, y, gep1, z, gep2, w, gep3);*/
 517       LLVMBuildStore(builder, x, gep0);
 518       LLVMBuildStore(builder, y, gep1);
 519       LLVMBuildStore(builder, z, gep2);
 520       LLVMBuildStore(builder, w, gep3);
 521    }
 522 #endif
 523 }
 524
 525 static void
 526 store_aos_array(LLVMBuilderRef builder,
 527                 LLVMValueRef io_ptr,
 528                 LLVMValueRef aos[NUM_CHANNELS],
 529                 int attrib,
 530                 int num_outputs)
 531 {
 532    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 533    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 534    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 535    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 536    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 537    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 538
 539    debug_assert(NUM_CHANNELS == 4);
 540
 541    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 542                           &ind0, 1, "");
 543    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 544                           &ind1, 1, "");
 545    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 546                           &ind2, 1, "");
 547    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 548                           &ind3, 1, "");
 549
 550 #if DEBUG_STORE
 551    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 552                    io_ptr, ind0, ind1, ind2, ind3);
 553 #endif
 554
 555    store_aos(builder, io0_ptr, attr_index, aos[0]);
 556    store_aos(builder, io1_ptr, attr_index, aos[1]);
 557    store_aos(builder, io2_ptr, attr_index, aos[2]);
 558    store_aos(builder, io3_ptr, attr_index, aos[3]);
 559 }
 560
 561 static void
 562 convert_to_aos(LLVMBuilderRef builder,
 563                LLVMValueRef io,
 564                LLVMValueRef (*outputs)[NUM_CHANNELS],
 565                int num_outputs,
 566                int max_vertices)
 567 {
 568    unsigned chan, attrib;
 569
 570 #if DEBUG_STORE
 571    lp_build_printf(builder, "   # storing begin\n");
 572 #endif
 573    for (attrib = 0; attrib < num_outputs; ++attrib) {
 574       LLVMValueRef soa[4];
 575       LLVMValueRef aos[4];
 576       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 577          if(outputs[attrib][chan]) {
 578             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 579             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 580             /*lp_build_printf(builder, "output %d : %d ",
 581                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 582                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 583               print_vectorf(builder, out);*/
 584             soa[chan] = out;
 585          } else
 586             soa[chan] = 0;
 587       }
 588       soa_to_aos(builder, soa, aos);
 589       store_aos_array(builder,
 590                       io,
 591                       aos,
 592                       attrib,
 593                       num_outputs);
 594    }
 595 #if DEBUG_STORE
 596    lp_build_printf(builder, "   # storing end\n");
 597 #endif
 598 }
 599
 600 static void
 601 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 602 {
 603    LLVMTypeRef arg_types[7];
 604    LLVMTypeRef func_type;
 605    LLVMValueRef context_ptr;
 606    LLVMBasicBlockRef block;
 607    LLVMBuilderRef builder;
 608    LLVMValueRef start, end, count, stride, step, io_itr;
 609    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 610    struct draw_context *draw = llvm->draw;
 611    unsigned i, j;
 612    struct lp_build_context bld;
 613    struct lp_build_loop_state lp_loop;
 614    struct lp_type vs_type = lp_type_float_vec(32);
 615    const int max_vertices = 4;
 616    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 617    void *code;
 618
 619    arg_types[0] = llvm->context_ptr_type;           /* context */
 620    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 621    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 622    arg_types[3] = LLVMInt32Type();                  /* start */
 623    arg_types[4] = LLVMInt32Type();                  /* count */
 624    arg_types[5] = LLVMInt32Type();                  /* stride */
 625    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 626
 627    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 628
 629    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 630    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 631    for(i = 0; i < Elements(arg_types); ++i)
 632       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 633          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 634
 635    context_ptr  = LLVMGetParam(variant->function, 0);
 636    io_ptr       = LLVMGetParam(variant->function, 1);
 637    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 638    start        = LLVMGetParam(variant->function, 3);
 639    count        = LLVMGetParam(variant->function, 4);
 640    stride       = LLVMGetParam(variant->function, 5);
 641    vb_ptr       = LLVMGetParam(variant->function, 6);
 642
 643    lp_build_name(context_ptr, "context");
 644    lp_build_name(io_ptr, "io");
 645    lp_build_name(vbuffers_ptr, "vbuffers");
 646    lp_build_name(start, "start");
 647    lp_build_name(count, "count");
 648    lp_build_name(stride, "stride");
 649    lp_build_name(vb_ptr, "vb");
 650
 651    /*
 652     * Function body
 653     */
 654
 655    block = LLVMAppendBasicBlock(variant->function, "entry");
 656    builder = LLVMCreateBuilder();
 657    LLVMPositionBuilderAtEnd(builder, block);
 658
 659    lp_build_context_init(&bld, builder, vs_type);
 660
 661    end = lp_build_add(&bld, start, count);
 662
 663    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 664
 665 #if DEBUG_STORE
 666    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 667                    start, end, step);
 668 #endif
 669    lp_build_loop_begin(builder, start, &lp_loop);
 670    {
 671       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 672       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 673       LLVMValueRef io;
 674       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 675
 676       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 677       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 678 #if DEBUG_STORE
 679       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 680                       io_itr, io, lp_loop.counter);
 681 #endif
 682       for (i = 0; i < NUM_CHANNELS; ++i) {
 683          LLVMValueRef true_index = LLVMBuildAdd(
 684             builder,
 685             lp_loop.counter,
 686             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 687          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 688             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 689             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 690                                                  velem->vertex_buffer_index,
 691                                                  0);
 692             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 693                                            &vb_index, 1, "");
 694             generate_fetch(builder, vbuffers_ptr,
 695                            &aos_attribs[j][i], velem, vb, true_index);
 696          }
 697       }
 698       convert_to_soa(builder, aos_attribs, inputs,
 699                      draw->pt.nr_vertex_elements);
 700
 701       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 702       generate_vs(llvm,
 703                   builder,
 704                   outputs,
 705                   ptr_aos,
 706                   context_ptr);
 707
 708       convert_to_aos(builder, io, outputs,
 709                      draw->vs.vertex_shader->info.num_outputs,
 710                      max_vertices);
 711    }
 712    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 713
 714    LLVMBuildRetVoid(builder);
 715
 716    LLVMDisposeBuilder(builder);
 717
 718    /*
 719     * Translate the LLVM IR into machine code.
 720     */
 721 #ifdef DEBUG
 722    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 723       lp_debug_dump_value(variant->function);
 724       assert(0);
 725    }
 726 #endif
 727
 728    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 729
 730    if (0) {
 731       lp_debug_dump_value(variant->function);
 732       debug_printf("\n");
 733    }
 734
 735    code = LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 736    variant->jit_func = voidptr_to_draw_jit_vert_func(code);
 737
 738    if (0)
 739       lp_disassemble(code);
 740 }
 741
 742
 743 static void
 744 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 745 {
 746    LLVMTypeRef arg_types[7];
 747    LLVMTypeRef func_type;
 748    LLVMValueRef context_ptr;
 749    LLVMBasicBlockRef block;
 750    LLVMBuilderRef builder;
 751    LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
 752    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 753    struct draw_context *draw = llvm->draw;
 754    unsigned i, j;
 755    struct lp_build_context bld;
 756    struct lp_build_context bld_int;
 757    struct lp_build_loop_state lp_loop;
 758    struct lp_type vs_type = lp_type_float_vec(32);
 759    const int max_vertices = 4;
 760    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 761    LLVMValueRef fetch_max;
 762    void *code;
 763
 764    arg_types[0] = llvm->context_ptr_type;               /* context */
 765    arg_types[1] = llvm->vertex_header_ptr_type;         /* vertex_header */
 766    arg_types[2] = llvm->buffer_ptr_type;                /* vbuffers */
 767    arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0);  /* fetch_elts * */
 768    arg_types[4] = LLVMInt32Type();                      /* fetch_count */
 769    arg_types[5] = LLVMInt32Type();                      /* stride */
 770    arg_types[6] = llvm->vb_ptr_type;                    /* pipe_vertex_buffer's */
 771
 772    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 773
 774    variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type);
 775    LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
 776    for(i = 0; i < Elements(arg_types); ++i)
 777       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 778          LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute);
 779
 780    context_ptr  = LLVMGetParam(variant->function_elts, 0);
 781    io_ptr       = LLVMGetParam(variant->function_elts, 1);
 782    vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
 783    fetch_elts   = LLVMGetParam(variant->function_elts, 3);
 784    fetch_count  = LLVMGetParam(variant->function_elts, 4);
 785    stride       = LLVMGetParam(variant->function_elts, 5);
 786    vb_ptr       = LLVMGetParam(variant->function_elts, 6);
 787
 788    lp_build_name(context_ptr, "context");
 789    lp_build_name(io_ptr, "io");
 790    lp_build_name(vbuffers_ptr, "vbuffers");
 791    lp_build_name(fetch_elts, "fetch_elts");
 792    lp_build_name(fetch_count, "fetch_count");
 793    lp_build_name(stride, "stride");
 794    lp_build_name(vb_ptr, "vb");
 795
 796    /*
 797     * Function body
 798     */
 799
 800    block = LLVMAppendBasicBlock(variant->function_elts, "entry");
 801    builder = LLVMCreateBuilder();
 802    LLVMPositionBuilderAtEnd(builder, block);
 803
 804    lp_build_context_init(&bld, builder, vs_type);
 805    lp_build_context_init(&bld_int, builder, lp_type_int(32));
 806
 807    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 808
 809    fetch_max = LLVMBuildSub(builder, fetch_count,
 810                             LLVMConstInt(LLVMInt32Type(), 1, 0),
 811                             "fetch_max");
 812
 813    lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop);
 814    {
 815       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 816       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 817       LLVMValueRef io;
 818       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 819
 820       io_itr = lp_loop.counter;
 821       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 822 #if DEBUG_STORE
 823       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 824                       io_itr, io, lp_loop.counter);
 825 #endif
 826       for (i = 0; i < NUM_CHANNELS; ++i) {
 827          LLVMValueRef true_index = LLVMBuildAdd(
 828             builder,
 829             lp_loop.counter,
 830             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 831          LLVMValueRef fetch_ptr;
 832
 833          /* make sure we're not out of bounds which can happen
 834           * if fetch_count % 4 != 0, because on the last iteration
 835           * a few of the 4 vertex fetches will be out of bounds */
 836          true_index = lp_build_min(&bld_int, true_index, fetch_max);
 837
 838          fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
 839                                   &true_index, 1, "");
 840          true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
 841          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 842             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 843             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 844                                                  velem->vertex_buffer_index,
 845                                                  0);
 846             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 847                                            &vb_index, 1, "");
 848             generate_fetch(builder, vbuffers_ptr,
 849                            &aos_attribs[j][i], velem, vb, true_index);
 850          }
 851       }
 852       convert_to_soa(builder, aos_attribs, inputs,
 853                      draw->pt.nr_vertex_elements);
 854
 855       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 856       generate_vs(llvm,
 857                   builder,
 858                   outputs,
 859                   ptr_aos,
 860                   context_ptr);
 861
 862       convert_to_aos(builder, io, outputs,
 863                      draw->vs.vertex_shader->info.num_outputs,
 864                      max_vertices);
 865    }
 866    lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop);
 867
 868    LLVMBuildRetVoid(builder);
 869
 870    LLVMDisposeBuilder(builder);
 871
 872    /*
 873     * Translate the LLVM IR into machine code.
 874     */
 875 #ifdef DEBUG
 876    if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
 877       lp_debug_dump_value(variant->function_elts);
 878       assert(0);
 879    }
 880 #endif
 881
 882    LLVMRunFunctionPassManager(llvm->pass, variant->function_elts);
 883
 884    if (0) {
 885       lp_debug_dump_value(variant->function_elts);
 886       debug_printf("\n");
 887    }
 888
 889    code = LLVMGetPointerToGlobal(llvm->draw->engine, variant->function_elts);
 890    variant->jit_func_elts = voidptr_to_draw_vert_func_elts(code);
 891
 892    if (0)
 893       lp_disassemble(code);
 894 }
 895
 896 void
 897 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 898                            struct draw_llvm_variant_key *key)
 899 {
 900    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 901
 902    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 903
 904    memcpy(key->vertex_element,
 905           llvm->draw->pt.vertex_element,
 906           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 907
 908    memcpy(&key->vs,
 909           &llvm->draw->vs.vertex_shader->state,
 910           sizeof(struct pipe_shader_state));
 911 }