src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_struct.h"
   8 #include "gallivm/lp_bld_type.h"
   9 #include "gallivm/lp_bld_flow.h"
  10 #include "gallivm/lp_bld_debug.h"
  11 #include "gallivm/lp_bld_tgsi.h"
  12 #include "gallivm/lp_bld_printf.h"
  13
  14 #include "tgsi/tgsi_exec.h"
  15
  16 #include "util/u_cpu_detect.h"
  17 #include "util/u_string.h"
  18
  19 #include <llvm-c/Transforms/Scalar.h>
  20
  21 #define DEBUG_STORE 0
  22
  23
  24 /* generates the draw jit function */
  25 static void
  26 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  27
  28 static void
  29 init_globals(struct draw_llvm *llvm)
  30 {
  31    LLVMTypeRef texture_type;
  32
  33    /* struct draw_jit_texture */
  34    {
  35       LLVMTypeRef elem_types[4];
  36
  37       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  38       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  39       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  41
  42       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  43
  44       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  45                              llvm->target, texture_type,
  46                              DRAW_JIT_TEXTURE_WIDTH);
  47       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  48                              llvm->target, texture_type,
  49                              DRAW_JIT_TEXTURE_HEIGHT);
  50       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  51                              llvm->target, texture_type,
  52                              DRAW_JIT_TEXTURE_STRIDE);
  53       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  54                              llvm->target, texture_type,
  55                              DRAW_JIT_TEXTURE_DATA);
  56       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  57                            llvm->target, texture_type);
  58
  59       LLVMAddTypeName(llvm->module, "texture", texture_type);
  60    }
  61
  62
  63    /* struct draw_jit_context */
  64    {
  65       LLVMTypeRef elem_types[3];
  66       LLVMTypeRef context_type;
  67
  68       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  69       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  70       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  71
  72       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  73
  74       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  75                              llvm->target, context_type, 0);
  76       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  77                              llvm->target, context_type, 1);
  78       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  79                              llvm->target, context_type,
  80                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  81       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  82                            llvm->target, context_type);
  83
  84       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  85
  86       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  87    }
  88    {
  89       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  90       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  91    }
  92    /* struct pipe_vertex_buffer */
  93    {
  94       LLVMTypeRef elem_types[4];
  95       LLVMTypeRef vb_type;
  96
  97       elem_types[0] = LLVMInt32Type();
  98       elem_types[1] = LLVMInt32Type();
  99       elem_types[2] = LLVMInt32Type();
 100       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 101
 102       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 103
 104       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 105                              llvm->target, vb_type, 0);
 106       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 107                              llvm->target, vb_type, 2);
 108       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 109                            llvm->target, vb_type);
 110
 111       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 112
 113       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 114    }
 115 }
 116
 117 static LLVMTypeRef
 118 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 119 {
 120    /* struct vertex_header */
 121    LLVMTypeRef elem_types[3];
 122    LLVMTypeRef vertex_header;
 123    char struct_name[24];
 124
 125    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 126
 127    elem_types[0]  = LLVMIntType(32);
 128    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 129    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 130
 131    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 132
 133    /* these are bit-fields and we can't take address of them
 134       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 135       llvm->target, vertex_header,
 136       DRAW_JIT_VERTEX_CLIPMASK);
 137       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 138       llvm->target, vertex_header,
 139       DRAW_JIT_VERTEX_EDGEFLAG);
 140       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 141       llvm->target, vertex_header,
 142       DRAW_JIT_VERTEX_PAD);
 143       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 144       llvm->target, vertex_header,
 145       DRAW_JIT_VERTEX_VERTEX_ID);
 146    */
 147    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 148                           llvm->target, vertex_header,
 149                           DRAW_JIT_VERTEX_CLIP);
 150    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 151                           llvm->target, vertex_header,
 152                           DRAW_JIT_VERTEX_DATA);
 153
 154    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 155
 156    return LLVMPointerType(vertex_header, 0);
 157 }
 158
 159 struct draw_llvm *
 160 draw_llvm_create(struct draw_context *draw)
 161 {
 162    struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
 163
 164    util_cpu_detect();
 165
 166    llvm->draw = draw;
 167    llvm->engine = draw->engine;
 168
 169    debug_assert(llvm->engine);
 170
 171    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 172    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 173
 174    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 175
 176    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 177
 178    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 179    LLVMAddTargetData(llvm->target, llvm->pass);
 180    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 181     * but there are more on SVN. */
 182    /* TODO: Add more passes */
 183    LLVMAddConstantPropagationPass(llvm->pass);
 184    if(util_cpu_caps.has_sse4_1) {
 185       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 186        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 187        * somehow becomes invalid code.
 188        */
 189       LLVMAddInstructionCombiningPass(llvm->pass);
 190    }
 191    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 192    LLVMAddGVNPass(llvm->pass);
 193    LLVMAddCFGSimplificationPass(llvm->pass);
 194
 195    init_globals(llvm);
 196
 197
 198 #if 0
 199    LLVMDumpModule(llvm->module);
 200 #endif
 201
 202    return llvm;
 203 }
 204
 205 void
 206 draw_llvm_destroy(struct draw_llvm *llvm)
 207 {
 208    free(llvm);
 209 }
 210
 211 struct draw_llvm_variant *
 212 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 213 {
 214    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 215
 216    draw_llvm_make_variant_key(llvm, &variant->key);
 217
 218    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 219
 220    draw_llvm_generate(llvm, variant);
 221
 222    return variant;
 223 }
 224
 225 static void
 226 generate_vs(struct draw_llvm *llvm,
 227             LLVMBuilderRef builder,
 228             LLVMValueRef (*outputs)[NUM_CHANNELS],
 229             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 230             LLVMValueRef context_ptr)
 231 {
 232    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 233    struct lp_type vs_type;
 234    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 235
 236    memset(&vs_type, 0, sizeof vs_type);
 237    vs_type.floating = TRUE; /* floating point values */
 238    vs_type.sign = TRUE;     /* values are signed */
 239    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 240    vs_type.width = 32;      /* 32-bit float */
 241    vs_type.length = 4;      /* 4 elements per vector */
 242 #if 0
 243    num_vs = 4;              /* number of vertices per block */
 244 #endif
 245
 246    /*tgsi_dump(tokens, 0);*/
 247    lp_build_tgsi_soa(builder,
 248                      tokens,
 249                      vs_type,
 250                      NULL /*struct lp_build_mask_context *mask*/,
 251                      consts_ptr,
 252                      NULL /*pos*/,
 253                      inputs,
 254                      outputs,
 255                      NULL/*sampler*/);
 256 }
 257
 258 #if DEBUG_STORE
 259 static void print_vectorf(LLVMBuilderRef builder,
 260                          LLVMValueRef vec)
 261 {
 262    LLVMValueRef val[4];
 263    val[0] = LLVMBuildExtractElement(builder, vec,
 264                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 265    val[1] = LLVMBuildExtractElement(builder, vec,
 266                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 267    val[2] = LLVMBuildExtractElement(builder, vec,
 268                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 269    val[3] = LLVMBuildExtractElement(builder, vec,
 270                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 271    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 272                    val[0], val[1], val[2], val[3]);
 273 }
 274 #endif
 275
 276 static void
 277 generate_fetch(LLVMBuilderRef builder,
 278                LLVMValueRef vbuffers_ptr,
 279                LLVMValueRef *res,
 280                struct pipe_vertex_element *velem,
 281                LLVMValueRef vbuf,
 282                LLVMValueRef index)
 283 {
 284    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 285    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 286                                            &indices, 1, "");
 287    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 288    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 289    LLVMValueRef stride = LLVMBuildMul(builder,
 290                                       vb_stride,
 291                                       index, "");
 292
 293    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 294
 295    stride = LLVMBuildAdd(builder, stride,
 296                          vb_buffer_offset,
 297                          "");
 298    stride = LLVMBuildAdd(builder, stride,
 299                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 300                          "");
 301
 302    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 303    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 304
 305    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 306 }
 307
 308 static LLVMValueRef
 309 aos_to_soa(LLVMBuilderRef builder,
 310            LLVMValueRef val0,
 311            LLVMValueRef val1,
 312            LLVMValueRef val2,
 313            LLVMValueRef val3,
 314            LLVMValueRef channel)
 315 {
 316    LLVMValueRef ex, res;
 317
 318    ex = LLVMBuildExtractElement(builder, val0,
 319                                 channel, "");
 320    res = LLVMBuildInsertElement(builder,
 321                                 LLVMConstNull(LLVMTypeOf(val0)),
 322                                 ex,
 323                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 324                                 "");
 325
 326    ex = LLVMBuildExtractElement(builder, val1,
 327                                 channel, "");
 328    res = LLVMBuildInsertElement(builder,
 329                                 res, ex,
 330                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 331                                 "");
 332
 333    ex = LLVMBuildExtractElement(builder, val2,
 334                                 channel, "");
 335    res = LLVMBuildInsertElement(builder,
 336                                 res, ex,
 337                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 338                                 "");
 339
 340    ex = LLVMBuildExtractElement(builder, val3,
 341                                 channel, "");
 342    res = LLVMBuildInsertElement(builder,
 343                                 res, ex,
 344                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 345                                 "");
 346
 347    return res;
 348 }
 349
 350 static void
 351 soa_to_aos(LLVMBuilderRef builder,
 352            LLVMValueRef soa[NUM_CHANNELS],
 353            LLVMValueRef aos[NUM_CHANNELS])
 354 {
 355    LLVMValueRef comp;
 356    int i = 0;
 357
 358    debug_assert(NUM_CHANNELS == 4);
 359
 360    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 361    aos[1] = aos[2] = aos[3] = aos[0];
 362
 363    for (i = 0; i < NUM_CHANNELS; ++i) {
 364       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 365
 366       comp = LLVMBuildExtractElement(builder, soa[i],
 367                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 368       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 369
 370       comp = LLVMBuildExtractElement(builder, soa[i],
 371                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 372       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 373
 374       comp = LLVMBuildExtractElement(builder, soa[i],
 375                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 376       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 377
 378       comp = LLVMBuildExtractElement(builder, soa[i],
 379                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 380       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 381
 382    }
 383 }
 384
 385 static void
 386 convert_to_soa(LLVMBuilderRef builder,
 387                LLVMValueRef (*aos)[NUM_CHANNELS],
 388                LLVMValueRef (*soa)[NUM_CHANNELS],
 389                int num_attribs)
 390 {
 391    int i;
 392
 393    debug_assert(NUM_CHANNELS == 4);
 394
 395    for (i = 0; i < num_attribs; ++i) {
 396       LLVMValueRef val0 = aos[i][0];
 397       LLVMValueRef val1 = aos[i][1];
 398       LLVMValueRef val2 = aos[i][2];
 399       LLVMValueRef val3 = aos[i][3];
 400
 401       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 402                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 403       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 404                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 405       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 406                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 407       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 408                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 409    }
 410 }
 411
 412 static void
 413 store_aos(LLVMBuilderRef builder,
 414           LLVMValueRef io_ptr,
 415           LLVMValueRef index,
 416           LLVMValueRef value)
 417 {
 418    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 419    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 420    LLVMValueRef indices[3];
 421
 422    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 423    indices[1] = index;
 424    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 425
 426    /* undefined vertex */
 427    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 428                                         0xffff, 0), id_ptr);
 429
 430 #if DEBUG_STORE
 431    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 432 #endif
 433 #if 0
 434    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 435      print_vectorf(builder, value);*/
 436    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 437                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 438                                "datavec");
 439    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 440
 441    LLVMBuildStore(builder, value, data_ptr);
 442 #else
 443    {
 444       LLVMValueRef x, y, z, w;
 445       LLVMValueRef idx0, idx1, idx2, idx3;
 446       LLVMValueRef gep0, gep1, gep2, gep3;
 447       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 448
 449       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 450       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 451       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 452       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 453
 454       x = LLVMBuildExtractElement(builder, value,
 455                                   idx0, "");
 456       y = LLVMBuildExtractElement(builder, value,
 457                                   idx1, "");
 458       z = LLVMBuildExtractElement(builder, value,
 459                                   idx2, "");
 460       w = LLVMBuildExtractElement(builder, value,
 461                                   idx3, "");
 462
 463       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 464       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 465       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 466       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 467
 468       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 469         x, gep0, y, gep1, z, gep2, w, gep3);*/
 470       LLVMBuildStore(builder, x, gep0);
 471       LLVMBuildStore(builder, y, gep1);
 472       LLVMBuildStore(builder, z, gep2);
 473       LLVMBuildStore(builder, w, gep3);
 474    }
 475 #endif
 476 }
 477
 478 static void
 479 store_aos_array(LLVMBuilderRef builder,
 480                 LLVMValueRef io_ptr,
 481                 LLVMValueRef aos[NUM_CHANNELS],
 482                 int attrib,
 483                 int num_outputs)
 484 {
 485    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 486    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 487    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 488    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 489    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 490    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 491
 492    debug_assert(NUM_CHANNELS == 4);
 493
 494    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 495                           &ind0, 1, "");
 496    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 497                           &ind1, 1, "");
 498    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 499                           &ind2, 1, "");
 500    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 501                           &ind3, 1, "");
 502
 503 #if DEBUG_STORE
 504    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 505                    io_ptr, ind0, ind1, ind2, ind3);
 506 #endif
 507
 508    store_aos(builder, io0_ptr, attr_index, aos[0]);
 509    store_aos(builder, io1_ptr, attr_index, aos[1]);
 510    store_aos(builder, io2_ptr, attr_index, aos[2]);
 511    store_aos(builder, io3_ptr, attr_index, aos[3]);
 512 }
 513
 514 static void
 515 convert_to_aos(LLVMBuilderRef builder,
 516                LLVMValueRef io,
 517                LLVMValueRef (*outputs)[NUM_CHANNELS],
 518                int num_outputs,
 519                int max_vertices)
 520 {
 521    unsigned chan, attrib;
 522
 523 #if DEBUG_STORE
 524    lp_build_printf(builder, "   # storing begin\n");
 525 #endif
 526    for (attrib = 0; attrib < num_outputs; ++attrib) {
 527       LLVMValueRef soa[4];
 528       LLVMValueRef aos[4];
 529       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 530          if(outputs[attrib][chan]) {
 531             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 532             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 533             /*lp_build_printf(builder, "output %d : %d ",
 534                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 535                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 536               print_vectorf(builder, out);*/
 537             soa[chan] = out;
 538          } else
 539             soa[chan] = 0;
 540       }
 541       soa_to_aos(builder, soa, aos);
 542       store_aos_array(builder,
 543                       io,
 544                       aos,
 545                       attrib,
 546                       num_outputs);
 547    }
 548 #if DEBUG_STORE
 549    lp_build_printf(builder, "   # storing end\n");
 550 #endif
 551 }
 552
 553 static void
 554 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 555 {
 556    LLVMTypeRef arg_types[7];
 557    LLVMTypeRef func_type;
 558    LLVMValueRef context_ptr;
 559    LLVMBasicBlockRef block;
 560    LLVMBuilderRef builder;
 561    LLVMValueRef start, end, count, stride, step, io_itr;
 562    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 563    struct draw_context *draw = llvm->draw;
 564    unsigned i, j;
 565    struct lp_build_context bld;
 566    struct lp_build_loop_state lp_loop;
 567    struct lp_type vs_type = lp_type_float_vec(32);
 568    const int max_vertices = 4;
 569    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 570
 571    arg_types[0] = llvm->context_ptr_type;           /* context */
 572    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 573    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 574    arg_types[3] = LLVMInt32Type();                  /* start */
 575    arg_types[4] = LLVMInt32Type();                  /* count */
 576    arg_types[5] = LLVMInt32Type();                  /* stride */
 577    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 578
 579    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 580
 581    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 582    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 583    for(i = 0; i < Elements(arg_types); ++i)
 584       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 585          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 586
 587    context_ptr  = LLVMGetParam(variant->function, 0);
 588    io_ptr       = LLVMGetParam(variant->function, 1);
 589    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 590    start        = LLVMGetParam(variant->function, 3);
 591    count        = LLVMGetParam(variant->function, 4);
 592    stride       = LLVMGetParam(variant->function, 5);
 593    vb_ptr       = LLVMGetParam(variant->function, 6);
 594
 595    lp_build_name(context_ptr, "context");
 596    lp_build_name(io_ptr, "io");
 597    lp_build_name(vbuffers_ptr, "vbuffers");
 598    lp_build_name(start, "start");
 599    lp_build_name(count, "count");
 600    lp_build_name(stride, "stride");
 601    lp_build_name(vb_ptr, "vb");
 602
 603    /*
 604     * Function body
 605     */
 606
 607    block = LLVMAppendBasicBlock(variant->function, "entry");
 608    builder = LLVMCreateBuilder();
 609    LLVMPositionBuilderAtEnd(builder, block);
 610
 611    lp_build_context_init(&bld, builder, vs_type);
 612
 613    end = lp_build_add(&bld, start, count);
 614
 615    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 616
 617 #if DEBUG_STORE
 618    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 619                    start, end, step);
 620 #endif
 621    lp_build_loop_begin(builder, start, &lp_loop);
 622    {
 623       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 624       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 625       LLVMValueRef io;
 626       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 627
 628       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 629       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 630 #if DEBUG_STORE
 631       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 632                       io_itr, io, lp_loop.counter);
 633 #endif
 634       for (i = 0; i < NUM_CHANNELS; ++i) {
 635          LLVMValueRef true_index = LLVMBuildAdd(
 636             builder,
 637             lp_loop.counter,
 638             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 639          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 640             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 641             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 642                                                  velem->vertex_buffer_index,
 643                                                  0);
 644             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 645                                            &vb_index, 1, "");
 646             generate_fetch(builder, vbuffers_ptr,
 647                            &aos_attribs[j][i], velem, vb, true_index);
 648          }
 649       }
 650       convert_to_soa(builder, aos_attribs, inputs,
 651                      draw->pt.nr_vertex_elements);
 652
 653       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 654       generate_vs(llvm,
 655                   builder,
 656                   outputs,
 657                   ptr_aos,
 658                   context_ptr);
 659
 660       convert_to_aos(builder, io, outputs,
 661                      draw->vs.vertex_shader->info.num_outputs,
 662                      max_vertices);
 663    }
 664    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 665
 666    LLVMBuildRetVoid(builder);
 667
 668    LLVMDisposeBuilder(builder);
 669
 670    /*
 671     * Translate the LLVM IR into machine code.
 672     */
 673 #ifdef DEBUG
 674    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 675       LLVMDumpValue(variant->function);
 676       assert(0);
 677    }
 678 #endif
 679
 680    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 681
 682    if (0) {
 683       LLVMDumpValue(variant->function);
 684       debug_printf("\n");
 685    }
 686    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 687
 688    if (0)
 689       lp_disassemble(variant->jit_func);
 690 }
 691
 692 void
 693 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 694                            struct draw_llvm_variant_key *key)
 695 {
 696    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 697
 698    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 699
 700    memcpy(key->vertex_element,
 701           llvm->draw->pt.vertex_element,
 702           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 703
 704    memcpy(&key->vs,
 705           &llvm->draw->vs.vertex_shader->state,
 706           sizeof(struct pipe_shader_state));
 707 }