src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_interp.h"
   8 #include "gallivm/lp_bld_struct.h"
   9 #include "gallivm/lp_bld_type.h"
  10 #include "gallivm/lp_bld_flow.h"
  11 #include "gallivm/lp_bld_debug.h"
  12 #include "gallivm/lp_bld_tgsi.h"
  13 #include "gallivm/lp_bld_printf.h"
  14 #include "gallivm/lp_bld_init.h"
  15
  16 #include "util/u_cpu_detect.h"
  17 #include "util/u_string.h"
  18
  19 #include <llvm-c/Transforms/Scalar.h>
  20
  21 #define DEBUG_STORE 0
  22
  23
  24 /* generates the draw jit function */
  25 static void
  26 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  27
  28 static void
  29 init_globals(struct draw_llvm *llvm)
  30 {
  31    LLVMTypeRef texture_type;
  32
  33    /* struct draw_jit_texture */
  34    {
  35       LLVMTypeRef elem_types[4];
  36
  37       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  38       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  39       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  41
  42       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  43
  44       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  45                              llvm->target, texture_type,
  46                              DRAW_JIT_TEXTURE_WIDTH);
  47       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  48                              llvm->target, texture_type,
  49                              DRAW_JIT_TEXTURE_HEIGHT);
  50       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  51                              llvm->target, texture_type,
  52                              DRAW_JIT_TEXTURE_STRIDE);
  53       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  54                              llvm->target, texture_type,
  55                              DRAW_JIT_TEXTURE_DATA);
  56       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  57                            llvm->target, texture_type);
  58
  59       LLVMAddTypeName(llvm->module, "texture", texture_type);
  60    }
  61
  62
  63    /* struct draw_jit_context */
  64    {
  65       LLVMTypeRef elem_types[3];
  66       LLVMTypeRef context_type;
  67
  68       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  69       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  70       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  71
  72       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  73
  74       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  75                              llvm->target, context_type, 0);
  76       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  77                              llvm->target, context_type, 1);
  78       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  79                              llvm->target, context_type,
  80                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  81       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  82                            llvm->target, context_type);
  83
  84       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  85
  86       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  87    }
  88    {
  89       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  90       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  91    }
  92    /* struct pipe_vertex_buffer */
  93    {
  94       LLVMTypeRef elem_types[4];
  95       LLVMTypeRef vb_type;
  96
  97       elem_types[0] = LLVMInt32Type();
  98       elem_types[1] = LLVMInt32Type();
  99       elem_types[2] = LLVMInt32Type();
 100       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 101
 102       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 103
 104       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 105                              llvm->target, vb_type, 0);
 106       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 107                              llvm->target, vb_type, 2);
 108       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 109                            llvm->target, vb_type);
 110
 111       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 112
 113       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 114    }
 115 }
 116
 117 static LLVMTypeRef
 118 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 119 {
 120    /* struct vertex_header */
 121    LLVMTypeRef elem_types[3];
 122    LLVMTypeRef vertex_header;
 123    char struct_name[24];
 124
 125    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 126
 127    elem_types[0]  = LLVMIntType(32);
 128    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 129    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 130
 131    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 132
 133    /* these are bit-fields and we can't take address of them
 134       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 135       llvm->target, vertex_header,
 136       DRAW_JIT_VERTEX_CLIPMASK);
 137       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 138       llvm->target, vertex_header,
 139       DRAW_JIT_VERTEX_EDGEFLAG);
 140       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 141       llvm->target, vertex_header,
 142       DRAW_JIT_VERTEX_PAD);
 143       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 144       llvm->target, vertex_header,
 145       DRAW_JIT_VERTEX_VERTEX_ID);
 146    */
 147    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 148                           llvm->target, vertex_header,
 149                           DRAW_JIT_VERTEX_CLIP);
 150    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 151                           llvm->target, vertex_header,
 152                           DRAW_JIT_VERTEX_DATA);
 153
 154    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 155
 156    return LLVMPointerType(vertex_header, 0);
 157 }
 158
 159 struct draw_llvm *
 160 draw_llvm_create(struct draw_context *draw)
 161 {
 162    struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
 163
 164    util_cpu_detect();
 165
 166    llvm->draw = draw;
 167    llvm->engine = draw->engine;
 168
 169    debug_assert(llvm->engine);
 170
 171    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 172    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 173
 174    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 175
 176    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 177
 178    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 179    LLVMAddTargetData(llvm->target, llvm->pass);
 180    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 181     * but there are more on SVN. */
 182    /* TODO: Add more passes */
 183    LLVMAddConstantPropagationPass(llvm->pass);
 184    if(util_cpu_caps.has_sse4_1) {
 185       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 186        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 187        * somehow becomes invalid code.
 188        */
 189       LLVMAddInstructionCombiningPass(llvm->pass);
 190    }
 191    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 192    LLVMAddGVNPass(llvm->pass);
 193    LLVMAddCFGSimplificationPass(llvm->pass);
 194
 195    init_globals(llvm);
 196
 197
 198 #if 1
 199    LLVMDumpModule(llvm->module);
 200 #endif
 201
 202    return llvm;
 203 }
 204
 205 void
 206 draw_llvm_destroy(struct draw_llvm *llvm)
 207 {
 208    free(llvm);
 209 }
 210
 211 struct draw_llvm_variant *
 212 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 213 {
 214    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 215
 216    draw_llvm_make_variant_key(llvm, &variant->key);
 217
 218    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 219
 220    draw_llvm_generate(llvm, variant);
 221
 222    return variant;
 223 }
 224
 225
 226 struct draw_context *draw_create_with_llvm(void)
 227 {
 228    struct draw_context *draw = CALLOC_STRUCT( draw_context );
 229    if (draw == NULL)
 230       goto fail;
 231
 232    assert(lp_build_engine);
 233    draw->engine = lp_build_engine;
 234
 235    if (!draw_init(draw))
 236       goto fail;
 237
 238    return draw;
 239
 240 fail:
 241    draw_destroy( draw );
 242    return NULL;
 243 }
 244
 245 static void
 246 generate_vs(struct draw_llvm *llvm,
 247             LLVMBuilderRef builder,
 248             LLVMValueRef (*outputs)[NUM_CHANNELS],
 249             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 250             LLVMValueRef context_ptr)
 251 {
 252    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 253    struct lp_type vs_type;
 254    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 255
 256    memset(&vs_type, 0, sizeof vs_type);
 257    vs_type.floating = TRUE; /* floating point values */
 258    vs_type.sign = TRUE;     /* values are signed */
 259    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 260    vs_type.width = 32;      /* 32-bit float */
 261    vs_type.length = 4;      /* 4 elements per vector */
 262 #if 0
 263    num_vs = 4;              /* number of vertices per block */
 264 #endif
 265
 266    /*tgsi_dump(tokens, 0);*/
 267    lp_build_tgsi_soa(builder,
 268                      tokens,
 269                      vs_type,
 270                      NULL /*struct lp_build_mask_context *mask*/,
 271                      consts_ptr,
 272                      NULL /*pos*/,
 273                      inputs,
 274                      outputs,
 275                      NULL/*sampler*/);
 276 }
 277
 278 #if DEBUG_STORE
 279 static void print_vectorf(LLVMBuilderRef builder,
 280                          LLVMValueRef vec)
 281 {
 282    LLVMValueRef val[4];
 283    val[0] = LLVMBuildExtractElement(builder, vec,
 284                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 285    val[1] = LLVMBuildExtractElement(builder, vec,
 286                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 287    val[2] = LLVMBuildExtractElement(builder, vec,
 288                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 289    val[3] = LLVMBuildExtractElement(builder, vec,
 290                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 291    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 292                    val[0], val[1], val[2], val[3]);
 293 }
 294 #endif
 295
 296 static void
 297 generate_fetch(LLVMBuilderRef builder,
 298                LLVMValueRef vbuffers_ptr,
 299                LLVMValueRef *res,
 300                struct pipe_vertex_element *velem,
 301                LLVMValueRef vbuf,
 302                LLVMValueRef index)
 303 {
 304    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 305    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 306                                            &indices, 1, "");
 307    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 308    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 309    LLVMValueRef stride = LLVMBuildMul(builder,
 310                                       vb_stride,
 311                                       index, "");
 312
 313    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 314
 315    stride = LLVMBuildAdd(builder, stride,
 316                          vb_buffer_offset,
 317                          "");
 318    stride = LLVMBuildAdd(builder, stride,
 319                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 320                          "");
 321
 322    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 323    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 324
 325    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 326 }
 327
 328 static LLVMValueRef
 329 aos_to_soa(LLVMBuilderRef builder,
 330            LLVMValueRef val0,
 331            LLVMValueRef val1,
 332            LLVMValueRef val2,
 333            LLVMValueRef val3,
 334            LLVMValueRef channel)
 335 {
 336    LLVMValueRef ex, res;
 337
 338    ex = LLVMBuildExtractElement(builder, val0,
 339                                 channel, "");
 340    res = LLVMBuildInsertElement(builder,
 341                                 LLVMConstNull(LLVMTypeOf(val0)),
 342                                 ex,
 343                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 344                                 "");
 345
 346    ex = LLVMBuildExtractElement(builder, val1,
 347                                 channel, "");
 348    res = LLVMBuildInsertElement(builder,
 349                                 res, ex,
 350                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 351                                 "");
 352
 353    ex = LLVMBuildExtractElement(builder, val2,
 354                                 channel, "");
 355    res = LLVMBuildInsertElement(builder,
 356                                 res, ex,
 357                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 358                                 "");
 359
 360    ex = LLVMBuildExtractElement(builder, val3,
 361                                 channel, "");
 362    res = LLVMBuildInsertElement(builder,
 363                                 res, ex,
 364                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 365                                 "");
 366
 367    return res;
 368 }
 369
 370 static void
 371 soa_to_aos(LLVMBuilderRef builder,
 372            LLVMValueRef soa[NUM_CHANNELS],
 373            LLVMValueRef aos[NUM_CHANNELS])
 374 {
 375    LLVMValueRef comp;
 376    int i = 0;
 377
 378    debug_assert(NUM_CHANNELS == 4);
 379
 380    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 381    aos[1] = aos[2] = aos[3] = aos[0];
 382
 383    for (i = 0; i < NUM_CHANNELS; ++i) {
 384       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 385
 386       comp = LLVMBuildExtractElement(builder, soa[i],
 387                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 388       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 389
 390       comp = LLVMBuildExtractElement(builder, soa[i],
 391                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 392       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 393
 394       comp = LLVMBuildExtractElement(builder, soa[i],
 395                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 396       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 397
 398       comp = LLVMBuildExtractElement(builder, soa[i],
 399                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 400       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 401
 402    }
 403 }
 404
 405 static void
 406 convert_to_soa(LLVMBuilderRef builder,
 407                LLVMValueRef (*aos)[NUM_CHANNELS],
 408                LLVMValueRef (*soa)[NUM_CHANNELS],
 409                int num_attribs)
 410 {
 411    int i;
 412
 413    debug_assert(NUM_CHANNELS == 4);
 414
 415    for (i = 0; i < num_attribs; ++i) {
 416       LLVMValueRef val0 = aos[i][0];
 417       LLVMValueRef val1 = aos[i][1];
 418       LLVMValueRef val2 = aos[i][2];
 419       LLVMValueRef val3 = aos[i][3];
 420
 421       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 422                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 423       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 424                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 425       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 426                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 427       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 428                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 429    }
 430 }
 431
 432 static void
 433 store_aos(LLVMBuilderRef builder,
 434           LLVMValueRef io_ptr,
 435           LLVMValueRef index,
 436           LLVMValueRef value)
 437 {
 438    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 439    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 440    LLVMValueRef indices[3];
 441
 442    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 443    indices[1] = index;
 444    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 445
 446    /* undefined vertex */
 447    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 448                                         0xffff, 0), id_ptr);
 449
 450 #if DEBUG_STORE
 451    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 452 #endif
 453 #if 0
 454    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 455      print_vectorf(builder, value);*/
 456    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 457                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 458                                "datavec");
 459    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 460
 461    LLVMBuildStore(builder, value, data_ptr);
 462 #else
 463    {
 464       LLVMValueRef x, y, z, w;
 465       LLVMValueRef idx0, idx1, idx2, idx3;
 466       LLVMValueRef gep0, gep1, gep2, gep3;
 467       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 468
 469       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 470       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 471       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 472       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 473
 474       x = LLVMBuildExtractElement(builder, value,
 475                                   idx0, "");
 476       y = LLVMBuildExtractElement(builder, value,
 477                                   idx1, "");
 478       z = LLVMBuildExtractElement(builder, value,
 479                                   idx2, "");
 480       w = LLVMBuildExtractElement(builder, value,
 481                                   idx3, "");
 482
 483       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 484       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 485       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 486       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 487
 488       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 489         x, gep0, y, gep1, z, gep2, w, gep3);*/
 490       LLVMBuildStore(builder, x, gep0);
 491       LLVMBuildStore(builder, y, gep1);
 492       LLVMBuildStore(builder, z, gep2);
 493       LLVMBuildStore(builder, w, gep3);
 494    }
 495 #endif
 496 }
 497
 498 static void
 499 store_aos_array(LLVMBuilderRef builder,
 500                 LLVMValueRef io_ptr,
 501                 LLVMValueRef aos[NUM_CHANNELS],
 502                 int attrib,
 503                 int num_outputs)
 504 {
 505    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 506    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 507    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 508    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 509    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 510    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 511
 512    debug_assert(NUM_CHANNELS == 4);
 513
 514    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 515                           &ind0, 1, "");
 516    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 517                           &ind1, 1, "");
 518    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 519                           &ind2, 1, "");
 520    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 521                           &ind3, 1, "");
 522
 523 #if DEBUG_STORE
 524    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 525                    io_ptr, ind0, ind1, ind2, ind3);
 526 #endif
 527
 528    store_aos(builder, io0_ptr, attr_index, aos[0]);
 529    store_aos(builder, io1_ptr, attr_index, aos[1]);
 530    store_aos(builder, io2_ptr, attr_index, aos[2]);
 531    store_aos(builder, io3_ptr, attr_index, aos[3]);
 532 }
 533
 534 static void
 535 convert_to_aos(LLVMBuilderRef builder,
 536                LLVMValueRef io,
 537                LLVMValueRef (*outputs)[NUM_CHANNELS],
 538                int num_outputs,
 539                int max_vertices)
 540 {
 541    unsigned chan, attrib;
 542
 543 #if DEBUG_STORE
 544    lp_build_printf(builder, "   # storing begin\n");
 545 #endif
 546    for (attrib = 0; attrib < num_outputs; ++attrib) {
 547       LLVMValueRef soa[4];
 548       LLVMValueRef aos[4];
 549       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 550          if(outputs[attrib][chan]) {
 551             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 552             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 553             /*lp_build_printf(builder, "output %d : %d ",
 554                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 555                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 556               print_vectorf(builder, out);*/
 557             soa[chan] = out;
 558          } else
 559             soa[chan] = 0;
 560       }
 561       soa_to_aos(builder, soa, aos);
 562       store_aos_array(builder,
 563                       io,
 564                       aos,
 565                       attrib,
 566                       num_outputs);
 567    }
 568 #if DEBUG_STORE
 569    lp_build_printf(builder, "   # storing end\n");
 570 #endif
 571 }
 572
 573 static void
 574 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 575 {
 576    LLVMTypeRef arg_types[7];
 577    LLVMTypeRef func_type;
 578    LLVMValueRef context_ptr;
 579    LLVMBasicBlockRef block;
 580    LLVMBuilderRef builder;
 581    LLVMValueRef start, end, count, stride, step, io_itr;
 582    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 583    struct draw_context *draw = llvm->draw;
 584    unsigned i, j;
 585    struct lp_build_context bld;
 586    struct lp_build_loop_state lp_loop;
 587    struct lp_type vs_type = lp_type_float_vec(32);
 588    const int max_vertices = 4;
 589    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 590
 591    arg_types[0] = llvm->context_ptr_type;           /* context */
 592    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 593    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 594    arg_types[3] = LLVMInt32Type();                  /* start */
 595    arg_types[4] = LLVMInt32Type();                  /* count */
 596    arg_types[5] = LLVMInt32Type();                  /* stride */
 597    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 598
 599    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 600
 601    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 602    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 603    for(i = 0; i < Elements(arg_types); ++i)
 604       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 605          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 606
 607    context_ptr  = LLVMGetParam(variant->function, 0);
 608    io_ptr       = LLVMGetParam(variant->function, 1);
 609    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 610    start        = LLVMGetParam(variant->function, 3);
 611    count        = LLVMGetParam(variant->function, 4);
 612    stride       = LLVMGetParam(variant->function, 5);
 613    vb_ptr       = LLVMGetParam(variant->function, 6);
 614
 615    lp_build_name(context_ptr, "context");
 616    lp_build_name(io_ptr, "io");
 617    lp_build_name(vbuffers_ptr, "vbuffers");
 618    lp_build_name(start, "start");
 619    lp_build_name(count, "count");
 620    lp_build_name(stride, "stride");
 621    lp_build_name(vb_ptr, "vb");
 622
 623    /*
 624     * Function body
 625     */
 626
 627    block = LLVMAppendBasicBlock(variant->function, "entry");
 628    builder = LLVMCreateBuilder();
 629    LLVMPositionBuilderAtEnd(builder, block);
 630
 631    lp_build_context_init(&bld, builder, vs_type);
 632
 633    end = lp_build_add(&bld, start, count);
 634
 635    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 636
 637 #if DEBUG_STORE
 638    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 639                    start, end, step);
 640 #endif
 641    lp_build_loop_begin(builder, start, &lp_loop);
 642    {
 643       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 644       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 645       LLVMValueRef io;
 646       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 647
 648       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 649       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 650 #if DEBUG_STORE
 651       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 652                       io_itr, io, lp_loop.counter);
 653 #endif
 654       for (i = 0; i < NUM_CHANNELS; ++i) {
 655          LLVMValueRef true_index = LLVMBuildAdd(
 656             builder,
 657             lp_loop.counter,
 658             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 659          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 660             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 661             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 662                                                  velem->vertex_buffer_index,
 663                                                  0);
 664             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 665                                            &vb_index, 1, "");
 666             generate_fetch(builder, vbuffers_ptr,
 667                            &aos_attribs[j][i], velem, vb, true_index);
 668          }
 669       }
 670       convert_to_soa(builder, aos_attribs, inputs,
 671                      draw->pt.nr_vertex_elements);
 672
 673       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 674       generate_vs(llvm,
 675                   builder,
 676                   outputs,
 677                   ptr_aos,
 678                   context_ptr);
 679
 680       convert_to_aos(builder, io, outputs,
 681                      draw->vs.vertex_shader->info.num_outputs,
 682                      max_vertices);
 683    }
 684    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 685
 686    LLVMBuildRetVoid(builder);
 687
 688    LLVMDisposeBuilder(builder);
 689
 690    /*
 691     * Translate the LLVM IR into machine code.
 692     */
 693 #ifdef DEBUG
 694    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 695       LLVMDumpValue(variant->function);
 696       assert(0);
 697    }
 698 #endif
 699
 700    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 701
 702    if (0) {
 703       LLVMDumpValue(variant->function);
 704       debug_printf("\n");
 705    }
 706    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 707
 708    if (0)
 709       lp_disassemble(variant->jit_func);
 710 }
 711
 712 void
 713 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 714                            struct draw_llvm_variant_key *key)
 715 {
 716    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 717
 718    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 719
 720    memcpy(key->vertex_element,
 721           llvm->draw->pt.vertex_element,
 722           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 723
 724    memcpy(&key->vs,
 725           &llvm->draw->vs.vertex_shader->state,
 726           sizeof(struct pipe_shader_state));
 727 }