src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_interp.h"
   8 #include "gallivm/lp_bld_struct.h"
   9 #include "gallivm/lp_bld_type.h"
  10 #include "gallivm/lp_bld_flow.h"
  11 #include "gallivm/lp_bld_debug.h"
  12 #include "gallivm/lp_bld_tgsi.h"
  13 #include "gallivm/lp_bld_printf.h"
  14 #include "gallivm/lp_bld_init.h"
  15
  16 #include "util/u_cpu_detect.h"
  17 #include "tgsi/tgsi_dump.h"
  18
  19 #include <llvm-c/Transforms/Scalar.h>
  20
  21 #define DEBUG_STORE 0
  22
  23
  24 /* generates the draw jit function */
  25 static void
  26 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  27
  28 static void
  29 init_globals(struct draw_llvm *llvm)
  30 {
  31    LLVMTypeRef texture_type;
  32
  33    /* struct draw_jit_texture */
  34    {
  35       LLVMTypeRef elem_types[4];
  36
  37       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  38       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  39       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  41
  42       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  43
  44       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  45                              llvm->target, texture_type,
  46                              DRAW_JIT_TEXTURE_WIDTH);
  47       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  48                              llvm->target, texture_type,
  49                              DRAW_JIT_TEXTURE_HEIGHT);
  50       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  51                              llvm->target, texture_type,
  52                              DRAW_JIT_TEXTURE_STRIDE);
  53       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  54                              llvm->target, texture_type,
  55                              DRAW_JIT_TEXTURE_DATA);
  56       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  57                            llvm->target, texture_type);
  58
  59       LLVMAddTypeName(llvm->module, "texture", texture_type);
  60    }
  61
  62
  63    /* struct draw_jit_context */
  64    {
  65       LLVMTypeRef elem_types[3];
  66       LLVMTypeRef context_type;
  67
  68       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  69       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  70       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  71
  72       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  73
  74       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  75                              llvm->target, context_type, 0);
  76       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  77                              llvm->target, context_type, 1);
  78       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  79                              llvm->target, context_type,
  80                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  81       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  82                            llvm->target, context_type);
  83
  84       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  85
  86       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  87    }
  88    {
  89       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  90       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  91    }
  92 }
  93
  94 static LLVMTypeRef
  95 create_vertex_header(struct draw_llvm *llvm, int data_elems)
  96 {
  97    /* struct vertex_header */
  98    LLVMTypeRef elem_types[3];
  99    LLVMTypeRef vertex_header;
 100    char struct_name[24];
 101
 102    snprintf(struct_name, 23, "vertex_header%d", data_elems);
 103
 104    elem_types[0]  = LLVMIntType(32);
 105    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 106    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 107
 108    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 109
 110    /* these are bit-fields and we can't take address of them
 111       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 112       llvm->target, vertex_header,
 113       DRAW_JIT_VERTEX_CLIPMASK);
 114       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 115       llvm->target, vertex_header,
 116       DRAW_JIT_VERTEX_EDGEFLAG);
 117       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 118       llvm->target, vertex_header,
 119       DRAW_JIT_VERTEX_PAD);
 120       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 121       llvm->target, vertex_header,
 122       DRAW_JIT_VERTEX_VERTEX_ID);
 123    */
 124    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 125                           llvm->target, vertex_header,
 126                           DRAW_JIT_VERTEX_CLIP);
 127    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 128                           llvm->target, vertex_header,
 129                           DRAW_JIT_VERTEX_DATA);
 130
 131    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 132
 133    return LLVMPointerType(vertex_header, 0);
 134 }
 135
 136 struct draw_llvm *
 137 draw_llvm_create(struct draw_context *draw)
 138 {
 139    struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
 140
 141    util_cpu_detect();
 142
 143    llvm->draw = draw;
 144    llvm->engine = draw->engine;
 145
 146    debug_assert(llvm->engine);
 147
 148    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 149    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 150
 151    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 152
 153    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 154
 155    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 156    LLVMAddTargetData(llvm->target, llvm->pass);
 157    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 158     * but there are more on SVN. */
 159    /* TODO: Add more passes */
 160    LLVMAddConstantPropagationPass(llvm->pass);
 161    if(util_cpu_caps.has_sse4_1) {
 162       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 163        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 164        * somehow becomes invalid code.
 165        */
 166       LLVMAddInstructionCombiningPass(llvm->pass);
 167    }
 168    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 169    LLVMAddGVNPass(llvm->pass);
 170    LLVMAddCFGSimplificationPass(llvm->pass);
 171
 172    init_globals(llvm);
 173
 174
 175 #if 1
 176    LLVMDumpModule(llvm->module);
 177 #endif
 178
 179    return llvm;
 180 }
 181
 182 void
 183 draw_llvm_destroy(struct draw_llvm *llvm)
 184 {
 185    free(llvm);
 186 }
 187
 188 struct draw_llvm_variant *
 189 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 190 {
 191    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 192
 193    draw_llvm_make_variant_key(llvm, &variant->key);
 194
 195    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 196
 197    draw_llvm_generate(llvm, variant);
 198
 199    return variant;
 200 }
 201
 202
 203 struct draw_context *draw_create_with_llvm(void)
 204 {
 205    struct draw_context *draw = CALLOC_STRUCT( draw_context );
 206    if (draw == NULL)
 207       goto fail;
 208
 209    assert(lp_build_engine);
 210    draw->engine = lp_build_engine;
 211
 212    if (!draw_init(draw))
 213       goto fail;
 214
 215    return draw;
 216
 217 fail:
 218    draw_destroy( draw );
 219    return NULL;
 220 }
 221
 222 static void
 223 generate_vs(struct draw_llvm *llvm,
 224             LLVMBuilderRef builder,
 225             LLVMValueRef (*outputs)[NUM_CHANNELS],
 226             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 227             LLVMValueRef context_ptr)
 228 {
 229    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 230    struct lp_type vs_type;
 231    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 232
 233    memset(&vs_type, 0, sizeof vs_type);
 234    vs_type.floating = TRUE; /* floating point values */
 235    vs_type.sign = TRUE;     /* values are signed */
 236    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 237    vs_type.width = 32;      /* 32-bit float */
 238    vs_type.length = 4;      /* 4 elements per vector */
 239 #if 0
 240    num_vs = 4;              /* number of vertices per block */
 241 #endif
 242
 243    tgsi_dump(tokens, 0);
 244    lp_build_tgsi_soa(builder,
 245                      tokens,
 246                      vs_type,
 247                      NULL /*struct lp_build_mask_context *mask*/,
 248                      consts_ptr,
 249                      NULL /*pos*/,
 250                      inputs,
 251                      outputs,
 252                      NULL/*sampler*/);
 253 }
 254
 255 #if DEBUG_STORE
 256 static void print_vectorf(LLVMBuilderRef builder,
 257                          LLVMValueRef vec)
 258 {
 259    LLVMValueRef val[4];
 260    val[0] = LLVMBuildExtractElement(builder, vec,
 261                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 262    val[1] = LLVMBuildExtractElement(builder, vec,
 263                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 264    val[2] = LLVMBuildExtractElement(builder, vec,
 265                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 266    val[3] = LLVMBuildExtractElement(builder, vec,
 267                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 268    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 269                    val[0], val[1], val[2], val[3]);
 270 }
 271 #endif
 272
 273 static void
 274 generate_fetch(LLVMBuilderRef builder,
 275                LLVMValueRef vbuffers_ptr,
 276                LLVMValueRef *res,
 277                struct pipe_vertex_element *velem,
 278                struct pipe_vertex_buffer *vbuf,
 279                LLVMValueRef index)
 280 {
 281    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 282    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 283                                            &indices, 1, "");
 284    LLVMValueRef stride = LLVMBuildMul(builder,
 285                                       LLVMConstInt(LLVMInt32Type(), vbuf->stride, 0),
 286                                       index, "");
 287
 288    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 289
 290    stride = LLVMBuildAdd(builder, stride,
 291                          LLVMConstInt(LLVMInt32Type(), vbuf->buffer_offset, 0),
 292                          "");
 293    stride = LLVMBuildAdd(builder, stride,
 294                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 295                          "");
 296
 297    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 298    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 299
 300    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 301 }
 302
 303 static LLVMValueRef
 304 aos_to_soa(LLVMBuilderRef builder,
 305            LLVMValueRef val0,
 306            LLVMValueRef val1,
 307            LLVMValueRef val2,
 308            LLVMValueRef val3,
 309            LLVMValueRef channel)
 310 {
 311    LLVMValueRef ex, res;
 312
 313    ex = LLVMBuildExtractElement(builder, val0,
 314                                 channel, "");
 315    res = LLVMBuildInsertElement(builder,
 316                                 LLVMConstNull(LLVMTypeOf(val0)),
 317                                 ex,
 318                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 319                                 "");
 320
 321    ex = LLVMBuildExtractElement(builder, val1,
 322                                 channel, "");
 323    res = LLVMBuildInsertElement(builder,
 324                                 res, ex,
 325                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 326                                 "");
 327
 328    ex = LLVMBuildExtractElement(builder, val2,
 329                                 channel, "");
 330    res = LLVMBuildInsertElement(builder,
 331                                 res, ex,
 332                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 333                                 "");
 334
 335    ex = LLVMBuildExtractElement(builder, val3,
 336                                 channel, "");
 337    res = LLVMBuildInsertElement(builder,
 338                                 res, ex,
 339                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 340                                 "");
 341
 342    return res;
 343 }
 344
 345 static void
 346 soa_to_aos(LLVMBuilderRef builder,
 347            LLVMValueRef soa[NUM_CHANNELS],
 348            LLVMValueRef aos[NUM_CHANNELS])
 349 {
 350    LLVMValueRef comp;
 351    int i = 0;
 352
 353    debug_assert(NUM_CHANNELS == 4);
 354
 355    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 356    aos[1] = aos[2] = aos[3] = aos[0];
 357
 358    for (i = 0; i < NUM_CHANNELS; ++i) {
 359       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 360
 361       comp = LLVMBuildExtractElement(builder, soa[i],
 362                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 363       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 364
 365       comp = LLVMBuildExtractElement(builder, soa[i],
 366                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 367       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 368
 369       comp = LLVMBuildExtractElement(builder, soa[i],
 370                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 371       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 372
 373       comp = LLVMBuildExtractElement(builder, soa[i],
 374                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 375       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 376
 377    }
 378 }
 379
 380 static void
 381 convert_to_soa(LLVMBuilderRef builder,
 382                LLVMValueRef (*aos)[NUM_CHANNELS],
 383                LLVMValueRef (*soa)[NUM_CHANNELS],
 384                int num_attribs)
 385 {
 386    int i;
 387
 388    debug_assert(NUM_CHANNELS == 4);
 389
 390    for (i = 0; i < num_attribs; ++i) {
 391       LLVMValueRef val0 = aos[i][0];
 392       LLVMValueRef val1 = aos[i][1];
 393       LLVMValueRef val2 = aos[i][2];
 394       LLVMValueRef val3 = aos[i][3];
 395
 396       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 397                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 398       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 399                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 400       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 401                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 402       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 403                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 404    }
 405 }
 406
 407 static void
 408 store_aos(LLVMBuilderRef builder,
 409           LLVMValueRef io_ptr,
 410           LLVMValueRef index,
 411           LLVMValueRef value)
 412 {
 413    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 414    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 415    LLVMValueRef indices[3];
 416
 417    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 418    indices[1] = index;
 419    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 420
 421    /* undefined vertex */
 422    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 423                                         0xffff, 0), id_ptr);
 424
 425 #if DEBUG_STORE
 426    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 427 #endif
 428 #if 0
 429    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 430      print_vectorf(builder, value);*/
 431    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 432                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 433                                "datavec");
 434    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 435
 436    LLVMBuildStore(builder, value, data_ptr);
 437 #else
 438    {
 439       LLVMValueRef x, y, z, w;
 440       LLVMValueRef idx0, idx1, idx2, idx3;
 441       LLVMValueRef gep0, gep1, gep2, gep3;
 442       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 443
 444       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 445       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 446       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 447       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 448
 449       x = LLVMBuildExtractElement(builder, value,
 450                                   idx0, "");
 451       y = LLVMBuildExtractElement(builder, value,
 452                                   idx1, "");
 453       z = LLVMBuildExtractElement(builder, value,
 454                                   idx2, "");
 455       w = LLVMBuildExtractElement(builder, value,
 456                                   idx3, "");
 457
 458       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 459       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 460       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 461       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 462
 463       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 464         x, gep0, y, gep1, z, gep2, w, gep3);*/
 465       LLVMBuildStore(builder, x, gep0);
 466       LLVMBuildStore(builder, y, gep1);
 467       LLVMBuildStore(builder, z, gep2);
 468       LLVMBuildStore(builder, w, gep3);
 469    }
 470 #endif
 471 }
 472
 473 static void
 474 store_aos_array(LLVMBuilderRef builder,
 475                 LLVMValueRef io_ptr,
 476                 LLVMValueRef aos[NUM_CHANNELS],
 477                 int attrib,
 478                 int num_outputs)
 479 {
 480    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 481    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 482    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 483    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 484    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 485    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 486
 487    debug_assert(NUM_CHANNELS == 4);
 488
 489    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 490                           &ind0, 1, "");
 491    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 492                           &ind1, 1, "");
 493    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 494                           &ind2, 1, "");
 495    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 496                           &ind3, 1, "");
 497
 498 #if DEBUG_STORE
 499    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 500                    io_ptr, ind0, ind1, ind2, ind3);
 501 #endif
 502
 503    store_aos(builder, io0_ptr, attr_index, aos[0]);
 504    store_aos(builder, io1_ptr, attr_index, aos[1]);
 505    store_aos(builder, io2_ptr, attr_index, aos[2]);
 506    store_aos(builder, io3_ptr, attr_index, aos[3]);
 507 }
 508
 509 static void
 510 convert_to_aos(LLVMBuilderRef builder,
 511                LLVMValueRef io,
 512                LLVMValueRef (*outputs)[NUM_CHANNELS],
 513                int num_outputs,
 514                int max_vertices)
 515 {
 516    unsigned chan, attrib;
 517
 518 #if DEBUG_STORE
 519    lp_build_printf(builder, "   # storing begin\n");
 520 #endif
 521    for (attrib = 0; attrib < num_outputs; ++attrib) {
 522       LLVMValueRef soa[4];
 523       LLVMValueRef aos[4];
 524       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 525          if(outputs[attrib][chan]) {
 526             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 527             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 528             /*lp_build_printf(builder, "output %d : %d ",
 529                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 530                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 531               print_vectorf(builder, out);*/
 532             soa[chan] = out;
 533          } else
 534             soa[chan] = 0;
 535       }
 536       soa_to_aos(builder, soa, aos);
 537       store_aos_array(builder,
 538                       io,
 539                       aos,
 540                       attrib,
 541                       num_outputs);
 542    }
 543 #if DEBUG_STORE
 544    lp_build_printf(builder, "   # storing end\n");
 545 #endif
 546 }
 547
 548 static void
 549 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 550 {
 551    LLVMTypeRef arg_types[6];
 552    LLVMTypeRef func_type;
 553    LLVMValueRef context_ptr;
 554    LLVMBasicBlockRef block;
 555    LLVMBuilderRef builder;
 556    LLVMValueRef start, end, count, stride, step, io_itr;
 557    LLVMValueRef io_ptr, vbuffers_ptr;
 558    struct draw_context *draw = llvm->draw;
 559    unsigned i, j;
 560    struct lp_build_context bld;
 561    struct lp_build_loop_state lp_loop;
 562    struct lp_type vs_type = lp_type_float_vec(32);
 563    const int max_vertices = 4;
 564    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 565
 566    arg_types[0] = llvm->context_ptr_type;           /* context */
 567    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 568    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 569    arg_types[3] = LLVMInt32Type();                  /* start */
 570    arg_types[4] = LLVMInt32Type();                  /* count */
 571    arg_types[5] = LLVMInt32Type();                  /* stride */
 572
 573    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 574
 575    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 576    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 577    for(i = 0; i < Elements(arg_types); ++i)
 578       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 579          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 580
 581    context_ptr  = LLVMGetParam(variant->function, 0);
 582    io_ptr       = LLVMGetParam(variant->function, 1);
 583    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 584    start        = LLVMGetParam(variant->function, 3);
 585    count        = LLVMGetParam(variant->function, 4);
 586    stride       = LLVMGetParam(variant->function, 5);
 587
 588    lp_build_name(context_ptr, "context");
 589    lp_build_name(io_ptr, "io");
 590    lp_build_name(vbuffers_ptr, "vbuffers");
 591    lp_build_name(start, "start");
 592    lp_build_name(count, "count");
 593    lp_build_name(stride, "stride");
 594
 595    /*
 596     * Function body
 597     */
 598
 599    block = LLVMAppendBasicBlock(variant->function, "entry");
 600    builder = LLVMCreateBuilder();
 601    LLVMPositionBuilderAtEnd(builder, block);
 602
 603    lp_build_context_init(&bld, builder, vs_type);
 604
 605    end = lp_build_add(&bld, start, count);
 606
 607    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 608
 609 #if DEBUG_STORE
 610    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 611                    start, end, step);
 612 #endif
 613    lp_build_loop_begin(builder, start, &lp_loop);
 614    {
 615       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 616       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 617       LLVMValueRef io;
 618       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 619
 620       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 621       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 622 #if DEBUG_STORE
 623       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 624                       io_itr, io, lp_loop.counter);
 625 #endif
 626       for (i = 0; i < NUM_CHANNELS; ++i) {
 627          LLVMValueRef true_index = LLVMBuildAdd(
 628             builder,
 629             lp_loop.counter,
 630             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 631          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 632             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 633             struct pipe_vertex_buffer *vbuf = &draw->pt.vertex_buffer[
 634                velem->vertex_buffer_index];
 635             generate_fetch(builder, vbuffers_ptr,
 636                            &aos_attribs[j][i], velem, vbuf, true_index);
 637          }
 638       }
 639       convert_to_soa(builder, aos_attribs, inputs,
 640                      draw->pt.nr_vertex_elements);
 641
 642       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 643       generate_vs(llvm,
 644                   builder,
 645                   outputs,
 646                   ptr_aos,
 647                   context_ptr);
 648
 649       convert_to_aos(builder, io, outputs,
 650                      draw->vs.vertex_shader->info.num_outputs,
 651                      max_vertices);
 652    }
 653    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 654
 655    LLVMBuildRetVoid(builder);
 656
 657    LLVMDisposeBuilder(builder);
 658
 659    /*
 660     * Translate the LLVM IR into machine code.
 661     */
 662
 663 #ifdef DEBUG
 664    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 665       LLVMDumpValue(variant->function);
 666       assert(0);
 667    }
 668 #endif
 669
 670    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 671
 672    if (0) {
 673       LLVMDumpValue(variant->function);
 674       debug_printf("\n");
 675    }
 676    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 677
 678    if (0)
 679       lp_disassemble(variant->jit_func);
 680 }
 681
 682 void
 683 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 684                            struct draw_llvm_variant_key *key)
 685 {
 686    key->nr_vertex_buffers = llvm->draw->pt.nr_vertex_buffers;
 687    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 688
 689    memcpy(key->vertex_buffer,
 690           llvm->draw->pt.vertex_buffer,
 691           sizeof(struct pipe_vertex_buffer) * PIPE_MAX_ATTRIBS);
 692
 693    memcpy(key->vertex_element,
 694           llvm->draw->pt.vertex_element,
 695           sizeof(struct pipe_vertex_element) * PIPE_MAX_ATTRIBS);
 696
 697    memcpy(&key->vs,
 698           &llvm->draw->vs.vertex_shader->state,
 699           sizeof(struct pipe_shader_state));
 700 }