src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_struct.h"
   8 #include "gallivm/lp_bld_type.h"
   9 #include "gallivm/lp_bld_flow.h"
  10 #include "gallivm/lp_bld_debug.h"
  11 #include "gallivm/lp_bld_tgsi.h"
  12 #include "gallivm/lp_bld_printf.h"
  13 #include "gallivm/lp_bld_init.h"
  14
  15 #include "tgsi/tgsi_exec.h"
  16
  17 #include "util/u_cpu_detect.h"
  18 #include "util/u_string.h"
  19
  20 #include <llvm-c/Transforms/Scalar.h>
  21
  22 #define DEBUG_STORE 0
  23
  24
  25 /* generates the draw jit function */
  26 static void
  27 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  28
  29 static void
  30 init_globals(struct draw_llvm *llvm)
  31 {
  32    LLVMTypeRef texture_type;
  33
  34    /* struct draw_jit_texture */
  35    {
  36       LLVMTypeRef elem_types[4];
  37
  38       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  39       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  41       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  42
  43       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  44
  45       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  46                              llvm->target, texture_type,
  47                              DRAW_JIT_TEXTURE_WIDTH);
  48       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  49                              llvm->target, texture_type,
  50                              DRAW_JIT_TEXTURE_HEIGHT);
  51       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  52                              llvm->target, texture_type,
  53                              DRAW_JIT_TEXTURE_STRIDE);
  54       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  55                              llvm->target, texture_type,
  56                              DRAW_JIT_TEXTURE_DATA);
  57       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  58                            llvm->target, texture_type);
  59
  60       LLVMAddTypeName(llvm->module, "texture", texture_type);
  61    }
  62
  63
  64    /* struct draw_jit_context */
  65    {
  66       LLVMTypeRef elem_types[3];
  67       LLVMTypeRef context_type;
  68
  69       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  70       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  71       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  72
  73       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  74
  75       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  76                              llvm->target, context_type, 0);
  77       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  78                              llvm->target, context_type, 1);
  79       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  80                              llvm->target, context_type,
  81                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  82       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  83                            llvm->target, context_type);
  84
  85       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  86
  87       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  88    }
  89    {
  90       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  91       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  92    }
  93    /* struct pipe_vertex_buffer */
  94    {
  95       LLVMTypeRef elem_types[4];
  96       LLVMTypeRef vb_type;
  97
  98       elem_types[0] = LLVMInt32Type();
  99       elem_types[1] = LLVMInt32Type();
 100       elem_types[2] = LLVMInt32Type();
 101       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 102
 103       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 104
 105       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 106                              llvm->target, vb_type, 0);
 107       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 108                              llvm->target, vb_type, 2);
 109       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 110                            llvm->target, vb_type);
 111
 112       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 113
 114       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 115    }
 116 }
 117
 118 static LLVMTypeRef
 119 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 120 {
 121    /* struct vertex_header */
 122    LLVMTypeRef elem_types[3];
 123    LLVMTypeRef vertex_header;
 124    char struct_name[24];
 125
 126    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 127
 128    elem_types[0]  = LLVMIntType(32);
 129    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 130    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 131
 132    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 133
 134    /* these are bit-fields and we can't take address of them
 135       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 136       llvm->target, vertex_header,
 137       DRAW_JIT_VERTEX_CLIPMASK);
 138       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 139       llvm->target, vertex_header,
 140       DRAW_JIT_VERTEX_EDGEFLAG);
 141       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 142       llvm->target, vertex_header,
 143       DRAW_JIT_VERTEX_PAD);
 144       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 145       llvm->target, vertex_header,
 146       DRAW_JIT_VERTEX_VERTEX_ID);
 147    */
 148    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 149                           llvm->target, vertex_header,
 150                           DRAW_JIT_VERTEX_CLIP);
 151    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 152                           llvm->target, vertex_header,
 153                           DRAW_JIT_VERTEX_DATA);
 154
 155    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 156
 157    return LLVMPointerType(vertex_header, 0);
 158 }
 159
 160 struct draw_llvm *
 161 draw_llvm_create(struct draw_context *draw)
 162 {
 163    struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
 164
 165    util_cpu_detect();
 166
 167    llvm->draw = draw;
 168    llvm->engine = draw->engine;
 169
 170    debug_assert(llvm->engine);
 171
 172    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 173    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 174
 175    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 176
 177    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 178
 179    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 180    LLVMAddTargetData(llvm->target, llvm->pass);
 181    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 182     * but there are more on SVN. */
 183    /* TODO: Add more passes */
 184    LLVMAddConstantPropagationPass(llvm->pass);
 185    if(util_cpu_caps.has_sse4_1) {
 186       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 187        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 188        * somehow becomes invalid code.
 189        */
 190       LLVMAddInstructionCombiningPass(llvm->pass);
 191    }
 192    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 193    LLVMAddGVNPass(llvm->pass);
 194    LLVMAddCFGSimplificationPass(llvm->pass);
 195
 196    init_globals(llvm);
 197
 198
 199 #if 1
 200    LLVMDumpModule(llvm->module);
 201 #endif
 202
 203    return llvm;
 204 }
 205
 206 void
 207 draw_llvm_destroy(struct draw_llvm *llvm)
 208 {
 209    free(llvm);
 210 }
 211
 212 struct draw_llvm_variant *
 213 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 214 {
 215    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 216
 217    draw_llvm_make_variant_key(llvm, &variant->key);
 218
 219    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 220
 221    draw_llvm_generate(llvm, variant);
 222
 223    return variant;
 224 }
 225
 226
 227 struct draw_context *draw_create_with_llvm(void)
 228 {
 229    struct draw_context *draw = CALLOC_STRUCT( draw_context );
 230    if (draw == NULL)
 231       goto fail;
 232
 233    assert(lp_build_engine);
 234    draw->engine = lp_build_engine;
 235
 236    if (!draw_init(draw))
 237       goto fail;
 238
 239    return draw;
 240
 241 fail:
 242    draw_destroy( draw );
 243    return NULL;
 244 }
 245
 246 static void
 247 generate_vs(struct draw_llvm *llvm,
 248             LLVMBuilderRef builder,
 249             LLVMValueRef (*outputs)[NUM_CHANNELS],
 250             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 251             LLVMValueRef context_ptr)
 252 {
 253    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 254    struct lp_type vs_type;
 255    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 256
 257    memset(&vs_type, 0, sizeof vs_type);
 258    vs_type.floating = TRUE; /* floating point values */
 259    vs_type.sign = TRUE;     /* values are signed */
 260    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 261    vs_type.width = 32;      /* 32-bit float */
 262    vs_type.length = 4;      /* 4 elements per vector */
 263 #if 0
 264    num_vs = 4;              /* number of vertices per block */
 265 #endif
 266
 267    /*tgsi_dump(tokens, 0);*/
 268    lp_build_tgsi_soa(builder,
 269                      tokens,
 270                      vs_type,
 271                      NULL /*struct lp_build_mask_context *mask*/,
 272                      consts_ptr,
 273                      NULL /*pos*/,
 274                      inputs,
 275                      outputs,
 276                      NULL/*sampler*/);
 277 }
 278
 279 #if DEBUG_STORE
 280 static void print_vectorf(LLVMBuilderRef builder,
 281                          LLVMValueRef vec)
 282 {
 283    LLVMValueRef val[4];
 284    val[0] = LLVMBuildExtractElement(builder, vec,
 285                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 286    val[1] = LLVMBuildExtractElement(builder, vec,
 287                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 288    val[2] = LLVMBuildExtractElement(builder, vec,
 289                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 290    val[3] = LLVMBuildExtractElement(builder, vec,
 291                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 292    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 293                    val[0], val[1], val[2], val[3]);
 294 }
 295 #endif
 296
 297 static void
 298 generate_fetch(LLVMBuilderRef builder,
 299                LLVMValueRef vbuffers_ptr,
 300                LLVMValueRef *res,
 301                struct pipe_vertex_element *velem,
 302                LLVMValueRef vbuf,
 303                LLVMValueRef index)
 304 {
 305    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 306    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 307                                            &indices, 1, "");
 308    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 309    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 310    LLVMValueRef stride = LLVMBuildMul(builder,
 311                                       vb_stride,
 312                                       index, "");
 313
 314    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 315
 316    stride = LLVMBuildAdd(builder, stride,
 317                          vb_buffer_offset,
 318                          "");
 319    stride = LLVMBuildAdd(builder, stride,
 320                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 321                          "");
 322
 323    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 324    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 325
 326    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 327 }
 328
 329 static LLVMValueRef
 330 aos_to_soa(LLVMBuilderRef builder,
 331            LLVMValueRef val0,
 332            LLVMValueRef val1,
 333            LLVMValueRef val2,
 334            LLVMValueRef val3,
 335            LLVMValueRef channel)
 336 {
 337    LLVMValueRef ex, res;
 338
 339    ex = LLVMBuildExtractElement(builder, val0,
 340                                 channel, "");
 341    res = LLVMBuildInsertElement(builder,
 342                                 LLVMConstNull(LLVMTypeOf(val0)),
 343                                 ex,
 344                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 345                                 "");
 346
 347    ex = LLVMBuildExtractElement(builder, val1,
 348                                 channel, "");
 349    res = LLVMBuildInsertElement(builder,
 350                                 res, ex,
 351                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 352                                 "");
 353
 354    ex = LLVMBuildExtractElement(builder, val2,
 355                                 channel, "");
 356    res = LLVMBuildInsertElement(builder,
 357                                 res, ex,
 358                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 359                                 "");
 360
 361    ex = LLVMBuildExtractElement(builder, val3,
 362                                 channel, "");
 363    res = LLVMBuildInsertElement(builder,
 364                                 res, ex,
 365                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 366                                 "");
 367
 368    return res;
 369 }
 370
 371 static void
 372 soa_to_aos(LLVMBuilderRef builder,
 373            LLVMValueRef soa[NUM_CHANNELS],
 374            LLVMValueRef aos[NUM_CHANNELS])
 375 {
 376    LLVMValueRef comp;
 377    int i = 0;
 378
 379    debug_assert(NUM_CHANNELS == 4);
 380
 381    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 382    aos[1] = aos[2] = aos[3] = aos[0];
 383
 384    for (i = 0; i < NUM_CHANNELS; ++i) {
 385       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 386
 387       comp = LLVMBuildExtractElement(builder, soa[i],
 388                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 389       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 390
 391       comp = LLVMBuildExtractElement(builder, soa[i],
 392                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 393       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 394
 395       comp = LLVMBuildExtractElement(builder, soa[i],
 396                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 397       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 398
 399       comp = LLVMBuildExtractElement(builder, soa[i],
 400                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 401       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 402
 403    }
 404 }
 405
 406 static void
 407 convert_to_soa(LLVMBuilderRef builder,
 408                LLVMValueRef (*aos)[NUM_CHANNELS],
 409                LLVMValueRef (*soa)[NUM_CHANNELS],
 410                int num_attribs)
 411 {
 412    int i;
 413
 414    debug_assert(NUM_CHANNELS == 4);
 415
 416    for (i = 0; i < num_attribs; ++i) {
 417       LLVMValueRef val0 = aos[i][0];
 418       LLVMValueRef val1 = aos[i][1];
 419       LLVMValueRef val2 = aos[i][2];
 420       LLVMValueRef val3 = aos[i][3];
 421
 422       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 423                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 424       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 425                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 426       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 427                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 428       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 429                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 430    }
 431 }
 432
 433 static void
 434 store_aos(LLVMBuilderRef builder,
 435           LLVMValueRef io_ptr,
 436           LLVMValueRef index,
 437           LLVMValueRef value)
 438 {
 439    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 440    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 441    LLVMValueRef indices[3];
 442
 443    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 444    indices[1] = index;
 445    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 446
 447    /* undefined vertex */
 448    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 449                                         0xffff, 0), id_ptr);
 450
 451 #if DEBUG_STORE
 452    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 453 #endif
 454 #if 0
 455    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 456      print_vectorf(builder, value);*/
 457    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 458                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 459                                "datavec");
 460    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 461
 462    LLVMBuildStore(builder, value, data_ptr);
 463 #else
 464    {
 465       LLVMValueRef x, y, z, w;
 466       LLVMValueRef idx0, idx1, idx2, idx3;
 467       LLVMValueRef gep0, gep1, gep2, gep3;
 468       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 469
 470       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 471       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 472       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 473       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 474
 475       x = LLVMBuildExtractElement(builder, value,
 476                                   idx0, "");
 477       y = LLVMBuildExtractElement(builder, value,
 478                                   idx1, "");
 479       z = LLVMBuildExtractElement(builder, value,
 480                                   idx2, "");
 481       w = LLVMBuildExtractElement(builder, value,
 482                                   idx3, "");
 483
 484       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 485       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 486       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 487       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 488
 489       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 490         x, gep0, y, gep1, z, gep2, w, gep3);*/
 491       LLVMBuildStore(builder, x, gep0);
 492       LLVMBuildStore(builder, y, gep1);
 493       LLVMBuildStore(builder, z, gep2);
 494       LLVMBuildStore(builder, w, gep3);
 495    }
 496 #endif
 497 }
 498
 499 static void
 500 store_aos_array(LLVMBuilderRef builder,
 501                 LLVMValueRef io_ptr,
 502                 LLVMValueRef aos[NUM_CHANNELS],
 503                 int attrib,
 504                 int num_outputs)
 505 {
 506    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 507    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 508    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 509    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 510    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 511    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 512
 513    debug_assert(NUM_CHANNELS == 4);
 514
 515    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 516                           &ind0, 1, "");
 517    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 518                           &ind1, 1, "");
 519    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 520                           &ind2, 1, "");
 521    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 522                           &ind3, 1, "");
 523
 524 #if DEBUG_STORE
 525    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 526                    io_ptr, ind0, ind1, ind2, ind3);
 527 #endif
 528
 529    store_aos(builder, io0_ptr, attr_index, aos[0]);
 530    store_aos(builder, io1_ptr, attr_index, aos[1]);
 531    store_aos(builder, io2_ptr, attr_index, aos[2]);
 532    store_aos(builder, io3_ptr, attr_index, aos[3]);
 533 }
 534
 535 static void
 536 convert_to_aos(LLVMBuilderRef builder,
 537                LLVMValueRef io,
 538                LLVMValueRef (*outputs)[NUM_CHANNELS],
 539                int num_outputs,
 540                int max_vertices)
 541 {
 542    unsigned chan, attrib;
 543
 544 #if DEBUG_STORE
 545    lp_build_printf(builder, "   # storing begin\n");
 546 #endif
 547    for (attrib = 0; attrib < num_outputs; ++attrib) {
 548       LLVMValueRef soa[4];
 549       LLVMValueRef aos[4];
 550       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 551          if(outputs[attrib][chan]) {
 552             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 553             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 554             /*lp_build_printf(builder, "output %d : %d ",
 555                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 556                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 557               print_vectorf(builder, out);*/
 558             soa[chan] = out;
 559          } else
 560             soa[chan] = 0;
 561       }
 562       soa_to_aos(builder, soa, aos);
 563       store_aos_array(builder,
 564                       io,
 565                       aos,
 566                       attrib,
 567                       num_outputs);
 568    }
 569 #if DEBUG_STORE
 570    lp_build_printf(builder, "   # storing end\n");
 571 #endif
 572 }
 573
 574 static void
 575 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 576 {
 577    LLVMTypeRef arg_types[7];
 578    LLVMTypeRef func_type;
 579    LLVMValueRef context_ptr;
 580    LLVMBasicBlockRef block;
 581    LLVMBuilderRef builder;
 582    LLVMValueRef start, end, count, stride, step, io_itr;
 583    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 584    struct draw_context *draw = llvm->draw;
 585    unsigned i, j;
 586    struct lp_build_context bld;
 587    struct lp_build_loop_state lp_loop;
 588    struct lp_type vs_type = lp_type_float_vec(32);
 589    const int max_vertices = 4;
 590    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 591
 592    arg_types[0] = llvm->context_ptr_type;           /* context */
 593    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 594    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 595    arg_types[3] = LLVMInt32Type();                  /* start */
 596    arg_types[4] = LLVMInt32Type();                  /* count */
 597    arg_types[5] = LLVMInt32Type();                  /* stride */
 598    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 599
 600    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 601
 602    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 603    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 604    for(i = 0; i < Elements(arg_types); ++i)
 605       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 606          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 607
 608    context_ptr  = LLVMGetParam(variant->function, 0);
 609    io_ptr       = LLVMGetParam(variant->function, 1);
 610    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 611    start        = LLVMGetParam(variant->function, 3);
 612    count        = LLVMGetParam(variant->function, 4);
 613    stride       = LLVMGetParam(variant->function, 5);
 614    vb_ptr       = LLVMGetParam(variant->function, 6);
 615
 616    lp_build_name(context_ptr, "context");
 617    lp_build_name(io_ptr, "io");
 618    lp_build_name(vbuffers_ptr, "vbuffers");
 619    lp_build_name(start, "start");
 620    lp_build_name(count, "count");
 621    lp_build_name(stride, "stride");
 622    lp_build_name(vb_ptr, "vb");
 623
 624    /*
 625     * Function body
 626     */
 627
 628    block = LLVMAppendBasicBlock(variant->function, "entry");
 629    builder = LLVMCreateBuilder();
 630    LLVMPositionBuilderAtEnd(builder, block);
 631
 632    lp_build_context_init(&bld, builder, vs_type);
 633
 634    end = lp_build_add(&bld, start, count);
 635
 636    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 637
 638 #if DEBUG_STORE
 639    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 640                    start, end, step);
 641 #endif
 642    lp_build_loop_begin(builder, start, &lp_loop);
 643    {
 644       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 645       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 646       LLVMValueRef io;
 647       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 648
 649       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 650       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 651 #if DEBUG_STORE
 652       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 653                       io_itr, io, lp_loop.counter);
 654 #endif
 655       for (i = 0; i < NUM_CHANNELS; ++i) {
 656          LLVMValueRef true_index = LLVMBuildAdd(
 657             builder,
 658             lp_loop.counter,
 659             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 660          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 661             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 662             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 663                                                  velem->vertex_buffer_index,
 664                                                  0);
 665             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 666                                            &vb_index, 1, "");
 667             generate_fetch(builder, vbuffers_ptr,
 668                            &aos_attribs[j][i], velem, vb, true_index);
 669          }
 670       }
 671       convert_to_soa(builder, aos_attribs, inputs,
 672                      draw->pt.nr_vertex_elements);
 673
 674       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 675       generate_vs(llvm,
 676                   builder,
 677                   outputs,
 678                   ptr_aos,
 679                   context_ptr);
 680
 681       convert_to_aos(builder, io, outputs,
 682                      draw->vs.vertex_shader->info.num_outputs,
 683                      max_vertices);
 684    }
 685    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 686
 687    LLVMBuildRetVoid(builder);
 688
 689    LLVMDisposeBuilder(builder);
 690
 691    /*
 692     * Translate the LLVM IR into machine code.
 693     */
 694 #ifdef DEBUG
 695    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 696       LLVMDumpValue(variant->function);
 697       assert(0);
 698    }
 699 #endif
 700
 701    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 702
 703    if (0) {
 704       LLVMDumpValue(variant->function);
 705       debug_printf("\n");
 706    }
 707    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 708
 709    if (0)
 710       lp_disassemble(variant->jit_func);
 711 }
 712
 713 void
 714 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 715                            struct draw_llvm_variant_key *key)
 716 {
 717    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 718
 719    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 720
 721    memcpy(key->vertex_element,
 722           llvm->draw->pt.vertex_element,
 723           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 724
 725    memcpy(&key->vs,
 726           &llvm->draw->vs.vertex_shader->state,
 727           sizeof(struct pipe_shader_state));
 728 }