src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_struct.h"
   8 #include "gallivm/lp_bld_type.h"
   9 #include "gallivm/lp_bld_flow.h"
  10 #include "gallivm/lp_bld_debug.h"
  11 #include "gallivm/lp_bld_tgsi.h"
  12 #include "gallivm/lp_bld_printf.h"
  13 #include "gallivm/lp_bld_init.h"
  14
  15 #include "tgsi/tgsi_exec.h"
  16
  17 #include "util/u_cpu_detect.h"
  18 #include "util/u_string.h"
  19
  20 #include <llvm-c/Transforms/Scalar.h>
  21
  22 #define DEBUG_STORE 0
  23
  24
  25 /* generates the draw jit function */
  26 static void
  27 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  28
  29 static void
  30 init_globals(struct draw_llvm *llvm)
  31 {
  32    LLVMTypeRef texture_type;
  33
  34    /* struct draw_jit_texture */
  35    {
  36       LLVMTypeRef elem_types[4];
  37
  38       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  39       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  41       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  42
  43       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  44
  45       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  46                              llvm->target, texture_type,
  47                              DRAW_JIT_TEXTURE_WIDTH);
  48       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  49                              llvm->target, texture_type,
  50                              DRAW_JIT_TEXTURE_HEIGHT);
  51       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  52                              llvm->target, texture_type,
  53                              DRAW_JIT_TEXTURE_STRIDE);
  54       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  55                              llvm->target, texture_type,
  56                              DRAW_JIT_TEXTURE_DATA);
  57       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  58                            llvm->target, texture_type);
  59
  60       LLVMAddTypeName(llvm->module, "texture", texture_type);
  61    }
  62
  63
  64    /* struct draw_jit_context */
  65    {
  66       LLVMTypeRef elem_types[3];
  67       LLVMTypeRef context_type;
  68
  69       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  70       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  71       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  72
  73       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  74
  75       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  76                              llvm->target, context_type, 0);
  77       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  78                              llvm->target, context_type, 1);
  79       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  80                              llvm->target, context_type,
  81                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  82       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  83                            llvm->target, context_type);
  84
  85       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  86
  87       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  88    }
  89    {
  90       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  91       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  92    }
  93    /* struct pipe_vertex_buffer */
  94    {
  95       LLVMTypeRef elem_types[4];
  96       LLVMTypeRef vb_type;
  97
  98       elem_types[0] = LLVMInt32Type();
  99       elem_types[1] = LLVMInt32Type();
 100       elem_types[2] = LLVMInt32Type();
 101       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 102
 103       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 104
 105       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 106                              llvm->target, vb_type, 0);
 107       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 108                              llvm->target, vb_type, 2);
 109       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 110                            llvm->target, vb_type);
 111
 112       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 113
 114       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 115    }
 116 }
 117
 118 static LLVMTypeRef
 119 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 120 {
 121    /* struct vertex_header */
 122    LLVMTypeRef elem_types[3];
 123    LLVMTypeRef vertex_header;
 124    char struct_name[24];
 125
 126    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 127
 128    elem_types[0]  = LLVMIntType(32);
 129    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 130    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 131
 132    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 133
 134    /* these are bit-fields and we can't take address of them
 135       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 136       llvm->target, vertex_header,
 137       DRAW_JIT_VERTEX_CLIPMASK);
 138       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 139       llvm->target, vertex_header,
 140       DRAW_JIT_VERTEX_EDGEFLAG);
 141       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 142       llvm->target, vertex_header,
 143       DRAW_JIT_VERTEX_PAD);
 144       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 145       llvm->target, vertex_header,
 146       DRAW_JIT_VERTEX_VERTEX_ID);
 147    */
 148    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 149                           llvm->target, vertex_header,
 150                           DRAW_JIT_VERTEX_CLIP);
 151    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 152                           llvm->target, vertex_header,
 153                           DRAW_JIT_VERTEX_DATA);
 154
 155    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 156
 157    return LLVMPointerType(vertex_header, 0);
 158 }
 159
 160 struct draw_llvm *
 161 draw_llvm_create(struct draw_context *draw)
 162 {
 163    struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
 164
 165    util_cpu_detect();
 166
 167    llvm->draw = draw;
 168    llvm->engine = draw->engine;
 169
 170    debug_assert(llvm->engine);
 171
 172    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 173    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 174
 175    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 176
 177    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 178
 179    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 180    LLVMAddTargetData(llvm->target, llvm->pass);
 181    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 182     * but there are more on SVN. */
 183    /* TODO: Add more passes */
 184    LLVMAddConstantPropagationPass(llvm->pass);
 185    if(util_cpu_caps.has_sse4_1) {
 186       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 187        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 188        * somehow becomes invalid code.
 189        */
 190       LLVMAddInstructionCombiningPass(llvm->pass);
 191    }
 192    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 193    LLVMAddGVNPass(llvm->pass);
 194    LLVMAddCFGSimplificationPass(llvm->pass);
 195
 196    init_globals(llvm);
 197
 198
 199 #if 0
 200    LLVMDumpModule(llvm->module);
 201 #endif
 202
 203    return llvm;
 204 }
 205
 206 void
 207 draw_llvm_destroy(struct draw_llvm *llvm)
 208 {
 209    free(llvm);
 210 }
 211
 212 struct draw_llvm_variant *
 213 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 214 {
 215    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 216
 217    draw_llvm_make_variant_key(llvm, &variant->key);
 218
 219    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 220
 221    draw_llvm_generate(llvm, variant);
 222
 223    return variant;
 224 }
 225
 226 static void
 227 generate_vs(struct draw_llvm *llvm,
 228             LLVMBuilderRef builder,
 229             LLVMValueRef (*outputs)[NUM_CHANNELS],
 230             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 231             LLVMValueRef context_ptr)
 232 {
 233    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 234    struct lp_type vs_type;
 235    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 236
 237    memset(&vs_type, 0, sizeof vs_type);
 238    vs_type.floating = TRUE; /* floating point values */
 239    vs_type.sign = TRUE;     /* values are signed */
 240    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 241    vs_type.width = 32;      /* 32-bit float */
 242    vs_type.length = 4;      /* 4 elements per vector */
 243 #if 0
 244    num_vs = 4;              /* number of vertices per block */
 245 #endif
 246
 247    /*tgsi_dump(tokens, 0);*/
 248    lp_build_tgsi_soa(builder,
 249                      tokens,
 250                      vs_type,
 251                      NULL /*struct lp_build_mask_context *mask*/,
 252                      consts_ptr,
 253                      NULL /*pos*/,
 254                      inputs,
 255                      outputs,
 256                      NULL/*sampler*/);
 257 }
 258
 259 #if DEBUG_STORE
 260 static void print_vectorf(LLVMBuilderRef builder,
 261                          LLVMValueRef vec)
 262 {
 263    LLVMValueRef val[4];
 264    val[0] = LLVMBuildExtractElement(builder, vec,
 265                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 266    val[1] = LLVMBuildExtractElement(builder, vec,
 267                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 268    val[2] = LLVMBuildExtractElement(builder, vec,
 269                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 270    val[3] = LLVMBuildExtractElement(builder, vec,
 271                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 272    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 273                    val[0], val[1], val[2], val[3]);
 274 }
 275 #endif
 276
 277 static void
 278 generate_fetch(LLVMBuilderRef builder,
 279                LLVMValueRef vbuffers_ptr,
 280                LLVMValueRef *res,
 281                struct pipe_vertex_element *velem,
 282                LLVMValueRef vbuf,
 283                LLVMValueRef index)
 284 {
 285    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 286    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 287                                            &indices, 1, "");
 288    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 289    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 290    LLVMValueRef stride = LLVMBuildMul(builder,
 291                                       vb_stride,
 292                                       index, "");
 293
 294    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 295
 296    stride = LLVMBuildAdd(builder, stride,
 297                          vb_buffer_offset,
 298                          "");
 299    stride = LLVMBuildAdd(builder, stride,
 300                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 301                          "");
 302
 303    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 304    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 305
 306    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 307 }
 308
 309 static LLVMValueRef
 310 aos_to_soa(LLVMBuilderRef builder,
 311            LLVMValueRef val0,
 312            LLVMValueRef val1,
 313            LLVMValueRef val2,
 314            LLVMValueRef val3,
 315            LLVMValueRef channel)
 316 {
 317    LLVMValueRef ex, res;
 318
 319    ex = LLVMBuildExtractElement(builder, val0,
 320                                 channel, "");
 321    res = LLVMBuildInsertElement(builder,
 322                                 LLVMConstNull(LLVMTypeOf(val0)),
 323                                 ex,
 324                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 325                                 "");
 326
 327    ex = LLVMBuildExtractElement(builder, val1,
 328                                 channel, "");
 329    res = LLVMBuildInsertElement(builder,
 330                                 res, ex,
 331                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 332                                 "");
 333
 334    ex = LLVMBuildExtractElement(builder, val2,
 335                                 channel, "");
 336    res = LLVMBuildInsertElement(builder,
 337                                 res, ex,
 338                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 339                                 "");
 340
 341    ex = LLVMBuildExtractElement(builder, val3,
 342                                 channel, "");
 343    res = LLVMBuildInsertElement(builder,
 344                                 res, ex,
 345                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 346                                 "");
 347
 348    return res;
 349 }
 350
 351 static void
 352 soa_to_aos(LLVMBuilderRef builder,
 353            LLVMValueRef soa[NUM_CHANNELS],
 354            LLVMValueRef aos[NUM_CHANNELS])
 355 {
 356    LLVMValueRef comp;
 357    int i = 0;
 358
 359    debug_assert(NUM_CHANNELS == 4);
 360
 361    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 362    aos[1] = aos[2] = aos[3] = aos[0];
 363
 364    for (i = 0; i < NUM_CHANNELS; ++i) {
 365       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 366
 367       comp = LLVMBuildExtractElement(builder, soa[i],
 368                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 369       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 370
 371       comp = LLVMBuildExtractElement(builder, soa[i],
 372                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 373       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 374
 375       comp = LLVMBuildExtractElement(builder, soa[i],
 376                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 377       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 378
 379       comp = LLVMBuildExtractElement(builder, soa[i],
 380                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 381       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 382
 383    }
 384 }
 385
 386 static void
 387 convert_to_soa(LLVMBuilderRef builder,
 388                LLVMValueRef (*aos)[NUM_CHANNELS],
 389                LLVMValueRef (*soa)[NUM_CHANNELS],
 390                int num_attribs)
 391 {
 392    int i;
 393
 394    debug_assert(NUM_CHANNELS == 4);
 395
 396    for (i = 0; i < num_attribs; ++i) {
 397       LLVMValueRef val0 = aos[i][0];
 398       LLVMValueRef val1 = aos[i][1];
 399       LLVMValueRef val2 = aos[i][2];
 400       LLVMValueRef val3 = aos[i][3];
 401
 402       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 403                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 404       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 405                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 406       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 407                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 408       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 409                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 410    }
 411 }
 412
 413 static void
 414 store_aos(LLVMBuilderRef builder,
 415           LLVMValueRef io_ptr,
 416           LLVMValueRef index,
 417           LLVMValueRef value)
 418 {
 419    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 420    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 421    LLVMValueRef indices[3];
 422
 423    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 424    indices[1] = index;
 425    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 426
 427    /* undefined vertex */
 428    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 429                                         0xffff, 0), id_ptr);
 430
 431 #if DEBUG_STORE
 432    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 433 #endif
 434 #if 0
 435    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 436      print_vectorf(builder, value);*/
 437    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 438                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 439                                "datavec");
 440    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 441
 442    LLVMBuildStore(builder, value, data_ptr);
 443 #else
 444    {
 445       LLVMValueRef x, y, z, w;
 446       LLVMValueRef idx0, idx1, idx2, idx3;
 447       LLVMValueRef gep0, gep1, gep2, gep3;
 448       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 449
 450       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 451       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 452       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 453       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 454
 455       x = LLVMBuildExtractElement(builder, value,
 456                                   idx0, "");
 457       y = LLVMBuildExtractElement(builder, value,
 458                                   idx1, "");
 459       z = LLVMBuildExtractElement(builder, value,
 460                                   idx2, "");
 461       w = LLVMBuildExtractElement(builder, value,
 462                                   idx3, "");
 463
 464       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 465       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 466       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 467       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 468
 469       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 470         x, gep0, y, gep1, z, gep2, w, gep3);*/
 471       LLVMBuildStore(builder, x, gep0);
 472       LLVMBuildStore(builder, y, gep1);
 473       LLVMBuildStore(builder, z, gep2);
 474       LLVMBuildStore(builder, w, gep3);
 475    }
 476 #endif
 477 }
 478
 479 static void
 480 store_aos_array(LLVMBuilderRef builder,
 481                 LLVMValueRef io_ptr,
 482                 LLVMValueRef aos[NUM_CHANNELS],
 483                 int attrib,
 484                 int num_outputs)
 485 {
 486    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 487    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 488    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 489    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 490    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 491    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 492
 493    debug_assert(NUM_CHANNELS == 4);
 494
 495    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 496                           &ind0, 1, "");
 497    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 498                           &ind1, 1, "");
 499    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 500                           &ind2, 1, "");
 501    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 502                           &ind3, 1, "");
 503
 504 #if DEBUG_STORE
 505    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 506                    io_ptr, ind0, ind1, ind2, ind3);
 507 #endif
 508
 509    store_aos(builder, io0_ptr, attr_index, aos[0]);
 510    store_aos(builder, io1_ptr, attr_index, aos[1]);
 511    store_aos(builder, io2_ptr, attr_index, aos[2]);
 512    store_aos(builder, io3_ptr, attr_index, aos[3]);
 513 }
 514
 515 static void
 516 convert_to_aos(LLVMBuilderRef builder,
 517                LLVMValueRef io,
 518                LLVMValueRef (*outputs)[NUM_CHANNELS],
 519                int num_outputs,
 520                int max_vertices)
 521 {
 522    unsigned chan, attrib;
 523
 524 #if DEBUG_STORE
 525    lp_build_printf(builder, "   # storing begin\n");
 526 #endif
 527    for (attrib = 0; attrib < num_outputs; ++attrib) {
 528       LLVMValueRef soa[4];
 529       LLVMValueRef aos[4];
 530       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 531          if(outputs[attrib][chan]) {
 532             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 533             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 534             /*lp_build_printf(builder, "output %d : %d ",
 535                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 536                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 537               print_vectorf(builder, out);*/
 538             soa[chan] = out;
 539          } else
 540             soa[chan] = 0;
 541       }
 542       soa_to_aos(builder, soa, aos);
 543       store_aos_array(builder,
 544                       io,
 545                       aos,
 546                       attrib,
 547                       num_outputs);
 548    }
 549 #if DEBUG_STORE
 550    lp_build_printf(builder, "   # storing end\n");
 551 #endif
 552 }
 553
 554 static void
 555 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 556 {
 557    LLVMTypeRef arg_types[7];
 558    LLVMTypeRef func_type;
 559    LLVMValueRef context_ptr;
 560    LLVMBasicBlockRef block;
 561    LLVMBuilderRef builder;
 562    LLVMValueRef start, end, count, stride, step, io_itr;
 563    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 564    struct draw_context *draw = llvm->draw;
 565    unsigned i, j;
 566    struct lp_build_context bld;
 567    struct lp_build_loop_state lp_loop;
 568    struct lp_type vs_type = lp_type_float_vec(32);
 569    const int max_vertices = 4;
 570    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 571
 572    arg_types[0] = llvm->context_ptr_type;           /* context */
 573    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 574    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 575    arg_types[3] = LLVMInt32Type();                  /* start */
 576    arg_types[4] = LLVMInt32Type();                  /* count */
 577    arg_types[5] = LLVMInt32Type();                  /* stride */
 578    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 579
 580    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 581
 582    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 583    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 584    for(i = 0; i < Elements(arg_types); ++i)
 585       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 586          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 587
 588    context_ptr  = LLVMGetParam(variant->function, 0);
 589    io_ptr       = LLVMGetParam(variant->function, 1);
 590    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 591    start        = LLVMGetParam(variant->function, 3);
 592    count        = LLVMGetParam(variant->function, 4);
 593    stride       = LLVMGetParam(variant->function, 5);
 594    vb_ptr       = LLVMGetParam(variant->function, 6);
 595
 596    lp_build_name(context_ptr, "context");
 597    lp_build_name(io_ptr, "io");
 598    lp_build_name(vbuffers_ptr, "vbuffers");
 599    lp_build_name(start, "start");
 600    lp_build_name(count, "count");
 601    lp_build_name(stride, "stride");
 602    lp_build_name(vb_ptr, "vb");
 603
 604    /*
 605     * Function body
 606     */
 607
 608    block = LLVMAppendBasicBlock(variant->function, "entry");
 609    builder = LLVMCreateBuilder();
 610    LLVMPositionBuilderAtEnd(builder, block);
 611
 612    lp_build_context_init(&bld, builder, vs_type);
 613
 614    end = lp_build_add(&bld, start, count);
 615
 616    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 617
 618 #if DEBUG_STORE
 619    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 620                    start, end, step);
 621 #endif
 622    lp_build_loop_begin(builder, start, &lp_loop);
 623    {
 624       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 625       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 626       LLVMValueRef io;
 627       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 628
 629       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 630       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 631 #if DEBUG_STORE
 632       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 633                       io_itr, io, lp_loop.counter);
 634 #endif
 635       for (i = 0; i < NUM_CHANNELS; ++i) {
 636          LLVMValueRef true_index = LLVMBuildAdd(
 637             builder,
 638             lp_loop.counter,
 639             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 640          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 641             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 642             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 643                                                  velem->vertex_buffer_index,
 644                                                  0);
 645             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 646                                            &vb_index, 1, "");
 647             generate_fetch(builder, vbuffers_ptr,
 648                            &aos_attribs[j][i], velem, vb, true_index);
 649          }
 650       }
 651       convert_to_soa(builder, aos_attribs, inputs,
 652                      draw->pt.nr_vertex_elements);
 653
 654       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 655       generate_vs(llvm,
 656                   builder,
 657                   outputs,
 658                   ptr_aos,
 659                   context_ptr);
 660
 661       convert_to_aos(builder, io, outputs,
 662                      draw->vs.vertex_shader->info.num_outputs,
 663                      max_vertices);
 664    }
 665    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 666
 667    LLVMBuildRetVoid(builder);
 668
 669    LLVMDisposeBuilder(builder);
 670
 671    /*
 672     * Translate the LLVM IR into machine code.
 673     */
 674 #ifdef DEBUG
 675    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 676       LLVMDumpValue(variant->function);
 677       assert(0);
 678    }
 679 #endif
 680
 681    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 682
 683    if (0) {
 684       LLVMDumpValue(variant->function);
 685       debug_printf("\n");
 686    }
 687    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 688
 689    if (0)
 690       lp_disassemble(variant->jit_func);
 691 }
 692
 693 void
 694 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 695                            struct draw_llvm_variant_key *key)
 696 {
 697    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 698
 699    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 700
 701    memcpy(key->vertex_element,
 702           llvm->draw->pt.vertex_element,
 703           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 704
 705    memcpy(&key->vs,
 706           &llvm->draw->vs.vertex_shader->state,
 707           sizeof(struct pipe_shader_state));
 708 }