src/gallium/auxiliary/draw/draw_llvm.c

   1 #include "draw_llvm.h"
   2
   3 #include "draw_context.h"
   4 #include "draw_vs.h"
   5
   6 #include "gallivm/lp_bld_arit.h"
   7 #include "gallivm/lp_bld_struct.h"
   8 #include "gallivm/lp_bld_type.h"
   9 #include "gallivm/lp_bld_flow.h"
  10 #include "gallivm/lp_bld_debug.h"
  11 #include "gallivm/lp_bld_tgsi.h"
  12 #include "gallivm/lp_bld_printf.h"
  13
  14 #include "tgsi/tgsi_exec.h"
  15
  16 #include "util/u_cpu_detect.h"
  17 #include "util/u_string.h"
  18
  19 #include <llvm-c/Transforms/Scalar.h>
  20
  21 #define DEBUG_STORE 0
  22
  23
  24 /* generates the draw jit function */
  25 static void
  26 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  27 static void
  28 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
  29
  30 static void
  31 init_globals(struct draw_llvm *llvm)
  32 {
  33    LLVMTypeRef texture_type;
  34
  35    /* struct draw_jit_texture */
  36    {
  37       LLVMTypeRef elem_types[4];
  38
  39       elem_types[DRAW_JIT_TEXTURE_WIDTH]  = LLVMInt32Type();
  40       elem_types[DRAW_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
  41       elem_types[DRAW_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
  42       elem_types[DRAW_JIT_TEXTURE_DATA]   = LLVMPointerType(LLVMInt8Type(), 0);
  43
  44       texture_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  45
  46       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, width,
  47                              llvm->target, texture_type,
  48                              DRAW_JIT_TEXTURE_WIDTH);
  49       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, height,
  50                              llvm->target, texture_type,
  51                              DRAW_JIT_TEXTURE_HEIGHT);
  52       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, stride,
  53                              llvm->target, texture_type,
  54                              DRAW_JIT_TEXTURE_STRIDE);
  55       LP_CHECK_MEMBER_OFFSET(struct draw_jit_texture, data,
  56                              llvm->target, texture_type,
  57                              DRAW_JIT_TEXTURE_DATA);
  58       LP_CHECK_STRUCT_SIZE(struct draw_jit_texture,
  59                            llvm->target, texture_type);
  60
  61       LLVMAddTypeName(llvm->module, "texture", texture_type);
  62    }
  63
  64
  65    /* struct draw_jit_context */
  66    {
  67       LLVMTypeRef elem_types[3];
  68       LLVMTypeRef context_type;
  69
  70       elem_types[0] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  71       elem_types[1] = LLVMPointerType(LLVMFloatType(), 0); /* vs_constants */
  72       elem_types[2] = LLVMArrayType(texture_type, PIPE_MAX_SAMPLERS); /* textures */
  73
  74       context_type = LLVMStructType(elem_types, Elements(elem_types), 0);
  75
  76       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_constants,
  77                              llvm->target, context_type, 0);
  78       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, gs_constants,
  79                              llvm->target, context_type, 1);
  80       LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, textures,
  81                              llvm->target, context_type,
  82                              DRAW_JIT_CONTEXT_TEXTURES_INDEX);
  83       LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
  84                            llvm->target, context_type);
  85
  86       LLVMAddTypeName(llvm->module, "draw_jit_context", context_type);
  87
  88       llvm->context_ptr_type = LLVMPointerType(context_type, 0);
  89    }
  90    {
  91       LLVMTypeRef buffer_ptr = LLVMPointerType(LLVMIntType(8), 0);
  92       llvm->buffer_ptr_type = LLVMPointerType(buffer_ptr, 0);
  93    }
  94    /* struct pipe_vertex_buffer */
  95    {
  96       LLVMTypeRef elem_types[4];
  97       LLVMTypeRef vb_type;
  98
  99       elem_types[0] = LLVMInt32Type();
 100       elem_types[1] = LLVMInt32Type();
 101       elem_types[2] = LLVMInt32Type();
 102       elem_types[3] = LLVMPointerType(LLVMOpaqueType(), 0); /* vs_constants */
 103
 104       vb_type = LLVMStructType(elem_types, Elements(elem_types), 0);
 105
 106       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
 107                              llvm->target, vb_type, 0);
 108       LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
 109                              llvm->target, vb_type, 2);
 110       LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer,
 111                            llvm->target, vb_type);
 112
 113       LLVMAddTypeName(llvm->module, "pipe_vertex_buffer", vb_type);
 114
 115       llvm->vb_ptr_type = LLVMPointerType(vb_type, 0);
 116    }
 117 }
 118
 119 static LLVMTypeRef
 120 create_vertex_header(struct draw_llvm *llvm, int data_elems)
 121 {
 122    /* struct vertex_header */
 123    LLVMTypeRef elem_types[3];
 124    LLVMTypeRef vertex_header;
 125    char struct_name[24];
 126
 127    util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
 128
 129    elem_types[0]  = LLVMIntType(32);
 130    elem_types[1]  = LLVMArrayType(LLVMFloatType(), 4);
 131    elem_types[2]  = LLVMArrayType(elem_types[1], data_elems);
 132
 133    vertex_header = LLVMStructType(elem_types, Elements(elem_types), 0);
 134
 135    /* these are bit-fields and we can't take address of them
 136       LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
 137       llvm->target, vertex_header,
 138       DRAW_JIT_VERTEX_CLIPMASK);
 139       LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
 140       llvm->target, vertex_header,
 141       DRAW_JIT_VERTEX_EDGEFLAG);
 142       LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
 143       llvm->target, vertex_header,
 144       DRAW_JIT_VERTEX_PAD);
 145       LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
 146       llvm->target, vertex_header,
 147       DRAW_JIT_VERTEX_VERTEX_ID);
 148    */
 149    LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip,
 150                           llvm->target, vertex_header,
 151                           DRAW_JIT_VERTEX_CLIP);
 152    LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
 153                           llvm->target, vertex_header,
 154                           DRAW_JIT_VERTEX_DATA);
 155
 156    LLVMAddTypeName(llvm->module, struct_name, vertex_header);
 157
 158    return LLVMPointerType(vertex_header, 0);
 159 }
 160
 161 struct draw_llvm *
 162 draw_llvm_create(struct draw_context *draw)
 163 {
 164    struct draw_llvm *llvm = CALLOC_STRUCT( draw_llvm );
 165
 166    util_cpu_detect();
 167
 168    llvm->draw = draw;
 169    llvm->engine = draw->engine;
 170
 171    debug_assert(llvm->engine);
 172
 173    llvm->module = LLVMModuleCreateWithName("draw_llvm");
 174    llvm->provider = LLVMCreateModuleProviderForExistingModule(llvm->module);
 175
 176    LLVMAddModuleProvider(llvm->engine, llvm->provider);
 177
 178    llvm->target = LLVMGetExecutionEngineTargetData(llvm->engine);
 179
 180    llvm->pass = LLVMCreateFunctionPassManager(llvm->provider);
 181    LLVMAddTargetData(llvm->target, llvm->pass);
 182    /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
 183     * but there are more on SVN. */
 184    /* TODO: Add more passes */
 185    LLVMAddConstantPropagationPass(llvm->pass);
 186    if(util_cpu_caps.has_sse4_1) {
 187       /* FIXME: There is a bug in this pass, whereby the combination of fptosi
 188        * and sitofp (necessary for trunc/floor/ceil/round implementation)
 189        * somehow becomes invalid code.
 190        */
 191       LLVMAddInstructionCombiningPass(llvm->pass);
 192    }
 193    LLVMAddPromoteMemoryToRegisterPass(llvm->pass);
 194    LLVMAddGVNPass(llvm->pass);
 195    LLVMAddCFGSimplificationPass(llvm->pass);
 196
 197    init_globals(llvm);
 198
 199
 200 #if 0
 201    LLVMDumpModule(llvm->module);
 202 #endif
 203
 204    return llvm;
 205 }
 206
 207 void
 208 draw_llvm_destroy(struct draw_llvm *llvm)
 209 {
 210    FREE(llvm);
 211 }
 212
 213 struct draw_llvm_variant *
 214 draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
 215 {
 216    struct draw_llvm_variant *variant = MALLOC(sizeof(struct draw_llvm_variant));
 217
 218    draw_llvm_make_variant_key(llvm, &variant->key);
 219
 220    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 221
 222    draw_llvm_generate(llvm, variant);
 223    draw_llvm_generate_elts(llvm, variant);
 224
 225    return variant;
 226 }
 227
 228 static void
 229 generate_vs(struct draw_llvm *llvm,
 230             LLVMBuilderRef builder,
 231             LLVMValueRef (*outputs)[NUM_CHANNELS],
 232             const LLVMValueRef (*inputs)[NUM_CHANNELS],
 233             LLVMValueRef context_ptr)
 234 {
 235    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
 236    struct lp_type vs_type;
 237    LLVMValueRef consts_ptr = draw_jit_context_vs_constants(builder, context_ptr);
 238
 239    memset(&vs_type, 0, sizeof vs_type);
 240    vs_type.floating = TRUE; /* floating point values */
 241    vs_type.sign = TRUE;     /* values are signed */
 242    vs_type.norm = FALSE;    /* values are not limited to [0,1] or [-1,1] */
 243    vs_type.width = 32;      /* 32-bit float */
 244    vs_type.length = 4;      /* 4 elements per vector */
 245 #if 0
 246    num_vs = 4;              /* number of vertices per block */
 247 #endif
 248
 249    /*tgsi_dump(tokens, 0);*/
 250    lp_build_tgsi_soa(builder,
 251                      tokens,
 252                      vs_type,
 253                      NULL /*struct lp_build_mask_context *mask*/,
 254                      consts_ptr,
 255                      NULL /*pos*/,
 256                      inputs,
 257                      outputs,
 258                      NULL/*sampler*/,
 259                      &llvm->draw->vs.vertex_shader->info);
 260 }
 261
 262 #if DEBUG_STORE
 263 static void print_vectorf(LLVMBuilderRef builder,
 264                          LLVMValueRef vec)
 265 {
 266    LLVMValueRef val[4];
 267    val[0] = LLVMBuildExtractElement(builder, vec,
 268                                     LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 269    val[1] = LLVMBuildExtractElement(builder, vec,
 270                                     LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 271    val[2] = LLVMBuildExtractElement(builder, vec,
 272                                     LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 273    val[3] = LLVMBuildExtractElement(builder, vec,
 274                                     LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 275    lp_build_printf(builder, "vector = [%f, %f, %f, %f]\n",
 276                    val[0], val[1], val[2], val[3]);
 277 }
 278 #endif
 279
 280 static void
 281 generate_fetch(LLVMBuilderRef builder,
 282                LLVMValueRef vbuffers_ptr,
 283                LLVMValueRef *res,
 284                struct pipe_vertex_element *velem,
 285                LLVMValueRef vbuf,
 286                LLVMValueRef index)
 287 {
 288    LLVMValueRef indices = LLVMConstInt(LLVMInt64Type(), velem->vertex_buffer_index, 0);
 289    LLVMValueRef vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr,
 290                                            &indices, 1, "");
 291    LLVMValueRef vb_stride = draw_jit_vbuffer_stride(builder, vbuf);
 292    LLVMValueRef vb_max_index = draw_jit_vbuffer_max_index(builder, vbuf);
 293    LLVMValueRef vb_buffer_offset = draw_jit_vbuffer_offset(builder, vbuf);
 294    LLVMValueRef cond;
 295    LLVMValueRef stride;
 296
 297    cond = LLVMBuildICmp(builder, LLVMIntULE, index, vb_max_index, "");
 298
 299    index = LLVMBuildSelect(builder, cond, index, vb_max_index, "");
 300
 301    stride = LLVMBuildMul(builder, vb_stride, index, "");
 302
 303    vbuffer_ptr = LLVMBuildLoad(builder, vbuffer_ptr, "vbuffer");
 304
 305    stride = LLVMBuildAdd(builder, stride,
 306                          vb_buffer_offset,
 307                          "");
 308    stride = LLVMBuildAdd(builder, stride,
 309                          LLVMConstInt(LLVMInt32Type(), velem->src_offset, 0),
 310                          "");
 311
 312    /*lp_build_printf(builder, "vbuf index = %d, stride is %d\n", indices, stride);*/
 313    vbuffer_ptr = LLVMBuildGEP(builder, vbuffer_ptr, &stride, 1, "");
 314
 315    *res = draw_llvm_translate_from(builder, vbuffer_ptr, velem->src_format);
 316 }
 317
 318 static LLVMValueRef
 319 aos_to_soa(LLVMBuilderRef builder,
 320            LLVMValueRef val0,
 321            LLVMValueRef val1,
 322            LLVMValueRef val2,
 323            LLVMValueRef val3,
 324            LLVMValueRef channel)
 325 {
 326    LLVMValueRef ex, res;
 327
 328    ex = LLVMBuildExtractElement(builder, val0,
 329                                 channel, "");
 330    res = LLVMBuildInsertElement(builder,
 331                                 LLVMConstNull(LLVMTypeOf(val0)),
 332                                 ex,
 333                                 LLVMConstInt(LLVMInt32Type(), 0, 0),
 334                                 "");
 335
 336    ex = LLVMBuildExtractElement(builder, val1,
 337                                 channel, "");
 338    res = LLVMBuildInsertElement(builder,
 339                                 res, ex,
 340                                 LLVMConstInt(LLVMInt32Type(), 1, 0),
 341                                 "");
 342
 343    ex = LLVMBuildExtractElement(builder, val2,
 344                                 channel, "");
 345    res = LLVMBuildInsertElement(builder,
 346                                 res, ex,
 347                                 LLVMConstInt(LLVMInt32Type(), 2, 0),
 348                                 "");
 349
 350    ex = LLVMBuildExtractElement(builder, val3,
 351                                 channel, "");
 352    res = LLVMBuildInsertElement(builder,
 353                                 res, ex,
 354                                 LLVMConstInt(LLVMInt32Type(), 3, 0),
 355                                 "");
 356
 357    return res;
 358 }
 359
 360 static void
 361 soa_to_aos(LLVMBuilderRef builder,
 362            LLVMValueRef soa[NUM_CHANNELS],
 363            LLVMValueRef aos[NUM_CHANNELS])
 364 {
 365    LLVMValueRef comp;
 366    int i = 0;
 367
 368    debug_assert(NUM_CHANNELS == 4);
 369
 370    aos[0] = LLVMConstNull(LLVMTypeOf(soa[0]));
 371    aos[1] = aos[2] = aos[3] = aos[0];
 372
 373    for (i = 0; i < NUM_CHANNELS; ++i) {
 374       LLVMValueRef channel = LLVMConstInt(LLVMInt32Type(), i, 0);
 375
 376       comp = LLVMBuildExtractElement(builder, soa[i],
 377                                      LLVMConstInt(LLVMInt32Type(), 0, 0), "");
 378       aos[0] = LLVMBuildInsertElement(builder, aos[0], comp, channel, "");
 379
 380       comp = LLVMBuildExtractElement(builder, soa[i],
 381                                      LLVMConstInt(LLVMInt32Type(), 1, 0), "");
 382       aos[1] = LLVMBuildInsertElement(builder, aos[1], comp, channel, "");
 383
 384       comp = LLVMBuildExtractElement(builder, soa[i],
 385                                      LLVMConstInt(LLVMInt32Type(), 2, 0), "");
 386       aos[2] = LLVMBuildInsertElement(builder, aos[2], comp, channel, "");
 387
 388       comp = LLVMBuildExtractElement(builder, soa[i],
 389                                      LLVMConstInt(LLVMInt32Type(), 3, 0), "");
 390       aos[3] = LLVMBuildInsertElement(builder, aos[3], comp, channel, "");
 391
 392    }
 393 }
 394
 395 static void
 396 convert_to_soa(LLVMBuilderRef builder,
 397                LLVMValueRef (*aos)[NUM_CHANNELS],
 398                LLVMValueRef (*soa)[NUM_CHANNELS],
 399                int num_attribs)
 400 {
 401    int i;
 402
 403    debug_assert(NUM_CHANNELS == 4);
 404
 405    for (i = 0; i < num_attribs; ++i) {
 406       LLVMValueRef val0 = aos[i][0];
 407       LLVMValueRef val1 = aos[i][1];
 408       LLVMValueRef val2 = aos[i][2];
 409       LLVMValueRef val3 = aos[i][3];
 410
 411       soa[i][0] = aos_to_soa(builder, val0, val1, val2, val3,
 412                              LLVMConstInt(LLVMInt32Type(), 0, 0));
 413       soa[i][1] = aos_to_soa(builder, val0, val1, val2, val3,
 414                              LLVMConstInt(LLVMInt32Type(), 1, 0));
 415       soa[i][2] = aos_to_soa(builder, val0, val1, val2, val3,
 416                              LLVMConstInt(LLVMInt32Type(), 2, 0));
 417       soa[i][3] = aos_to_soa(builder, val0, val1, val2, val3,
 418                              LLVMConstInt(LLVMInt32Type(), 3, 0));
 419    }
 420 }
 421
 422 static void
 423 store_aos(LLVMBuilderRef builder,
 424           LLVMValueRef io_ptr,
 425           LLVMValueRef index,
 426           LLVMValueRef value)
 427 {
 428    LLVMValueRef id_ptr = draw_jit_header_id(builder, io_ptr);
 429    LLVMValueRef data_ptr = draw_jit_header_data(builder, io_ptr);
 430    LLVMValueRef indices[3];
 431
 432    indices[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 433    indices[1] = index;
 434    indices[2] = LLVMConstInt(LLVMInt32Type(), 0, 0);
 435
 436    /* undefined vertex */
 437    LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(),
 438                                         0xffff, 0), id_ptr);
 439
 440 #if DEBUG_STORE
 441    lp_build_printf(builder, "    ---- %p storing attribute %d (io = %p)\n", data_ptr, index, io_ptr);
 442 #endif
 443 #if 0
 444    /*lp_build_printf(builder, " ---- %p storing at %d (%p)  ", io_ptr, index, data_ptr);
 445      print_vectorf(builder, value);*/
 446    data_ptr = LLVMBuildBitCast(builder, data_ptr,
 447                                LLVMPointerType(LLVMArrayType(LLVMVectorType(LLVMFloatType(), 4), 0), 0),
 448                                "datavec");
 449    data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 2, "");
 450
 451    LLVMBuildStore(builder, value, data_ptr);
 452 #else
 453    {
 454       LLVMValueRef x, y, z, w;
 455       LLVMValueRef idx0, idx1, idx2, idx3;
 456       LLVMValueRef gep0, gep1, gep2, gep3;
 457       data_ptr = LLVMBuildGEP(builder, data_ptr, indices, 3, "");
 458
 459       idx0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 460       idx1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 461       idx2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 462       idx3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 463
 464       x = LLVMBuildExtractElement(builder, value,
 465                                   idx0, "");
 466       y = LLVMBuildExtractElement(builder, value,
 467                                   idx1, "");
 468       z = LLVMBuildExtractElement(builder, value,
 469                                   idx2, "");
 470       w = LLVMBuildExtractElement(builder, value,
 471                                   idx3, "");
 472
 473       gep0 = LLVMBuildGEP(builder, data_ptr, &idx0, 1, "");
 474       gep1 = LLVMBuildGEP(builder, data_ptr, &idx1, 1, "");
 475       gep2 = LLVMBuildGEP(builder, data_ptr, &idx2, 1, "");
 476       gep3 = LLVMBuildGEP(builder, data_ptr, &idx3, 1, "");
 477
 478       /*lp_build_printf(builder, "##### x = %f (%p), y = %f (%p), z = %f (%p), w = %f (%p)\n",
 479         x, gep0, y, gep1, z, gep2, w, gep3);*/
 480       LLVMBuildStore(builder, x, gep0);
 481       LLVMBuildStore(builder, y, gep1);
 482       LLVMBuildStore(builder, z, gep2);
 483       LLVMBuildStore(builder, w, gep3);
 484    }
 485 #endif
 486 }
 487
 488 static void
 489 store_aos_array(LLVMBuilderRef builder,
 490                 LLVMValueRef io_ptr,
 491                 LLVMValueRef aos[NUM_CHANNELS],
 492                 int attrib,
 493                 int num_outputs)
 494 {
 495    LLVMValueRef attr_index = LLVMConstInt(LLVMInt32Type(), attrib, 0);
 496    LLVMValueRef ind0 = LLVMConstInt(LLVMInt32Type(), 0, 0);
 497    LLVMValueRef ind1 = LLVMConstInt(LLVMInt32Type(), 1, 0);
 498    LLVMValueRef ind2 = LLVMConstInt(LLVMInt32Type(), 2, 0);
 499    LLVMValueRef ind3 = LLVMConstInt(LLVMInt32Type(), 3, 0);
 500    LLVMValueRef io0_ptr, io1_ptr, io2_ptr, io3_ptr;
 501
 502    debug_assert(NUM_CHANNELS == 4);
 503
 504    io0_ptr = LLVMBuildGEP(builder, io_ptr,
 505                           &ind0, 1, "");
 506    io1_ptr = LLVMBuildGEP(builder, io_ptr,
 507                           &ind1, 1, "");
 508    io2_ptr = LLVMBuildGEP(builder, io_ptr,
 509                           &ind2, 1, "");
 510    io3_ptr = LLVMBuildGEP(builder, io_ptr,
 511                           &ind3, 1, "");
 512
 513 #if DEBUG_STORE
 514    lp_build_printf(builder, "   io = %p, indexes[%d, %d, %d, %d]\n",
 515                    io_ptr, ind0, ind1, ind2, ind3);
 516 #endif
 517
 518    store_aos(builder, io0_ptr, attr_index, aos[0]);
 519    store_aos(builder, io1_ptr, attr_index, aos[1]);
 520    store_aos(builder, io2_ptr, attr_index, aos[2]);
 521    store_aos(builder, io3_ptr, attr_index, aos[3]);
 522 }
 523
 524 static void
 525 convert_to_aos(LLVMBuilderRef builder,
 526                LLVMValueRef io,
 527                LLVMValueRef (*outputs)[NUM_CHANNELS],
 528                int num_outputs,
 529                int max_vertices)
 530 {
 531    unsigned chan, attrib;
 532
 533 #if DEBUG_STORE
 534    lp_build_printf(builder, "   # storing begin\n");
 535 #endif
 536    for (attrib = 0; attrib < num_outputs; ++attrib) {
 537       LLVMValueRef soa[4];
 538       LLVMValueRef aos[4];
 539       for(chan = 0; chan < NUM_CHANNELS; ++chan) {
 540          if(outputs[attrib][chan]) {
 541             LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
 542             lp_build_name(out, "output%u.%c", attrib, "xyzw"[chan]);
 543             /*lp_build_printf(builder, "output %d : %d ",
 544                             LLVMConstInt(LLVMInt32Type(), attrib, 0),
 545                             LLVMConstInt(LLVMInt32Type(), chan, 0));
 546               print_vectorf(builder, out);*/
 547             soa[chan] = out;
 548          } else
 549             soa[chan] = 0;
 550       }
 551       soa_to_aos(builder, soa, aos);
 552       store_aos_array(builder,
 553                       io,
 554                       aos,
 555                       attrib,
 556                       num_outputs);
 557    }
 558 #if DEBUG_STORE
 559    lp_build_printf(builder, "   # storing end\n");
 560 #endif
 561 }
 562
 563 static void
 564 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 565 {
 566    LLVMTypeRef arg_types[7];
 567    LLVMTypeRef func_type;
 568    LLVMValueRef context_ptr;
 569    LLVMBasicBlockRef block;
 570    LLVMBuilderRef builder;
 571    LLVMValueRef start, end, count, stride, step, io_itr;
 572    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 573    struct draw_context *draw = llvm->draw;
 574    unsigned i, j;
 575    struct lp_build_context bld;
 576    struct lp_build_loop_state lp_loop;
 577    struct lp_type vs_type = lp_type_float_vec(32);
 578    const int max_vertices = 4;
 579    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 580
 581    arg_types[0] = llvm->context_ptr_type;           /* context */
 582    arg_types[1] = llvm->vertex_header_ptr_type;     /* vertex_header */
 583    arg_types[2] = llvm->buffer_ptr_type;            /* vbuffers */
 584    arg_types[3] = LLVMInt32Type();                  /* start */
 585    arg_types[4] = LLVMInt32Type();                  /* count */
 586    arg_types[5] = LLVMInt32Type();                  /* stride */
 587    arg_types[6] = llvm->vb_ptr_type;                /* pipe_vertex_buffer's */
 588
 589    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 590
 591    variant->function = LLVMAddFunction(llvm->module, "draw_llvm_shader", func_type);
 592    LLVMSetFunctionCallConv(variant->function, LLVMCCallConv);
 593    for(i = 0; i < Elements(arg_types); ++i)
 594       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 595          LLVMAddAttribute(LLVMGetParam(variant->function, i), LLVMNoAliasAttribute);
 596
 597    context_ptr  = LLVMGetParam(variant->function, 0);
 598    io_ptr       = LLVMGetParam(variant->function, 1);
 599    vbuffers_ptr = LLVMGetParam(variant->function, 2);
 600    start        = LLVMGetParam(variant->function, 3);
 601    count        = LLVMGetParam(variant->function, 4);
 602    stride       = LLVMGetParam(variant->function, 5);
 603    vb_ptr       = LLVMGetParam(variant->function, 6);
 604
 605    lp_build_name(context_ptr, "context");
 606    lp_build_name(io_ptr, "io");
 607    lp_build_name(vbuffers_ptr, "vbuffers");
 608    lp_build_name(start, "start");
 609    lp_build_name(count, "count");
 610    lp_build_name(stride, "stride");
 611    lp_build_name(vb_ptr, "vb");
 612
 613    /*
 614     * Function body
 615     */
 616
 617    block = LLVMAppendBasicBlock(variant->function, "entry");
 618    builder = LLVMCreateBuilder();
 619    LLVMPositionBuilderAtEnd(builder, block);
 620
 621    lp_build_context_init(&bld, builder, vs_type);
 622
 623    end = lp_build_add(&bld, start, count);
 624
 625    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 626
 627 #if DEBUG_STORE
 628    lp_build_printf(builder, "start = %d, end = %d, step = %d\n",
 629                    start, end, step);
 630 #endif
 631    lp_build_loop_begin(builder, start, &lp_loop);
 632    {
 633       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 634       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 635       LLVMValueRef io;
 636       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 637
 638       io_itr = LLVMBuildSub(builder, lp_loop.counter, start, "");
 639       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 640 #if DEBUG_STORE
 641       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 642                       io_itr, io, lp_loop.counter);
 643 #endif
 644       for (i = 0; i < NUM_CHANNELS; ++i) {
 645          LLVMValueRef true_index = LLVMBuildAdd(
 646             builder,
 647             lp_loop.counter,
 648             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 649          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 650             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 651             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 652                                                  velem->vertex_buffer_index,
 653                                                  0);
 654             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 655                                            &vb_index, 1, "");
 656             generate_fetch(builder, vbuffers_ptr,
 657                            &aos_attribs[j][i], velem, vb, true_index);
 658          }
 659       }
 660       convert_to_soa(builder, aos_attribs, inputs,
 661                      draw->pt.nr_vertex_elements);
 662
 663       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 664       generate_vs(llvm,
 665                   builder,
 666                   outputs,
 667                   ptr_aos,
 668                   context_ptr);
 669
 670       convert_to_aos(builder, io, outputs,
 671                      draw->vs.vertex_shader->info.num_outputs,
 672                      max_vertices);
 673    }
 674    lp_build_loop_end_cond(builder, end, step, LLVMIntUGE, &lp_loop);
 675
 676    LLVMBuildRetVoid(builder);
 677
 678    LLVMDisposeBuilder(builder);
 679
 680    /*
 681     * Translate the LLVM IR into machine code.
 682     */
 683 #ifdef DEBUG
 684    if(LLVMVerifyFunction(variant->function, LLVMPrintMessageAction)) {
 685       LLVMDumpValue(variant->function);
 686       assert(0);
 687    }
 688 #endif
 689
 690    LLVMRunFunctionPassManager(llvm->pass, variant->function);
 691
 692    if (0) {
 693       LLVMDumpValue(variant->function);
 694       debug_printf("\n");
 695    }
 696    variant->jit_func = (draw_jit_vert_func)LLVMGetPointerToGlobal(llvm->draw->engine, variant->function);
 697
 698    if (0)
 699       lp_disassemble(variant->jit_func);
 700 }
 701
 702
 703 static void
 704 draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 705 {
 706    LLVMTypeRef arg_types[7];
 707    LLVMTypeRef func_type;
 708    LLVMValueRef context_ptr;
 709    LLVMBasicBlockRef block;
 710    LLVMBuilderRef builder;
 711    LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
 712    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
 713    struct draw_context *draw = llvm->draw;
 714    unsigned i, j;
 715    struct lp_build_context bld;
 716    struct lp_build_loop_state lp_loop;
 717    struct lp_type vs_type = lp_type_float_vec(32);
 718    const int max_vertices = 4;
 719    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
 720
 721    arg_types[0] = llvm->context_ptr_type;               /* context */
 722    arg_types[1] = llvm->vertex_header_ptr_type;         /* vertex_header */
 723    arg_types[2] = llvm->buffer_ptr_type;                /* vbuffers */
 724    arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0);  /* fetch_elts * */
 725    arg_types[4] = LLVMInt32Type();                      /* fetch_count */
 726    arg_types[5] = LLVMInt32Type();                      /* stride */
 727    arg_types[6] = llvm->vb_ptr_type;                    /* pipe_vertex_buffer's */
 728
 729    func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
 730
 731    variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type);
 732    LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
 733    for(i = 0; i < Elements(arg_types); ++i)
 734       if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
 735          LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute);
 736
 737    context_ptr  = LLVMGetParam(variant->function_elts, 0);
 738    io_ptr       = LLVMGetParam(variant->function_elts, 1);
 739    vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
 740    fetch_elts   = LLVMGetParam(variant->function_elts, 3);
 741    fetch_count  = LLVMGetParam(variant->function_elts, 4);
 742    stride       = LLVMGetParam(variant->function_elts, 5);
 743    vb_ptr       = LLVMGetParam(variant->function_elts, 6);
 744
 745    lp_build_name(context_ptr, "context");
 746    lp_build_name(io_ptr, "io");
 747    lp_build_name(vbuffers_ptr, "vbuffers");
 748    lp_build_name(fetch_elts, "fetch_elts");
 749    lp_build_name(fetch_count, "fetch_count");
 750    lp_build_name(stride, "stride");
 751    lp_build_name(vb_ptr, "vb");
 752
 753    /*
 754     * Function body
 755     */
 756
 757    block = LLVMAppendBasicBlock(variant->function_elts, "entry");
 758    builder = LLVMCreateBuilder();
 759    LLVMPositionBuilderAtEnd(builder, block);
 760
 761    lp_build_context_init(&bld, builder, vs_type);
 762
 763    step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
 764
 765    lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop);
 766    {
 767       LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
 768       LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
 769       LLVMValueRef io;
 770       const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
 771
 772       io_itr = lp_loop.counter;
 773       io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
 774 #if DEBUG_STORE
 775       lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
 776                       io_itr, io, lp_loop.counter);
 777 #endif
 778       for (i = 0; i < NUM_CHANNELS; ++i) {
 779          LLVMValueRef true_index = LLVMBuildAdd(
 780             builder,
 781             lp_loop.counter,
 782             LLVMConstInt(LLVMInt32Type(), i, 0), "");
 783          LLVMValueRef fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
 784                                                &true_index, 1, "");
 785          true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
 786          for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
 787             struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
 788             LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
 789                                                  velem->vertex_buffer_index,
 790                                                  0);
 791             LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
 792                                            &vb_index, 1, "");
 793             generate_fetch(builder, vbuffers_ptr,
 794                            &aos_attribs[j][i], velem, vb, true_index);
 795          }
 796       }
 797       convert_to_soa(builder, aos_attribs, inputs,
 798                      draw->pt.nr_vertex_elements);
 799
 800       ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
 801       generate_vs(llvm,
 802                   builder,
 803                   outputs,
 804                   ptr_aos,
 805                   context_ptr);
 806
 807       convert_to_aos(builder, io, outputs,
 808                      draw->vs.vertex_shader->info.num_outputs,
 809                      max_vertices);
 810    }
 811    lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop);
 812
 813    LLVMBuildRetVoid(builder);
 814
 815    LLVMDisposeBuilder(builder);
 816
 817    /*
 818     * Translate the LLVM IR into machine code.
 819     */
 820 #ifdef DEBUG
 821    if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
 822       LLVMDumpValue(variant->function_elts);
 823       assert(0);
 824    }
 825 #endif
 826
 827    LLVMRunFunctionPassManager(llvm->pass, variant->function_elts);
 828
 829    if (0) {
 830       LLVMDumpValue(variant->function_elts);
 831       debug_printf("\n");
 832    }
 833    variant->jit_func_elts = (draw_jit_vert_func_elts)LLVMGetPointerToGlobal(
 834       llvm->draw->engine, variant->function_elts);
 835
 836    if (0)
 837       lp_disassemble(variant->jit_func_elts);
 838 }
 839
 840 void
 841 draw_llvm_make_variant_key(struct draw_llvm *llvm,
 842                            struct draw_llvm_variant_key *key)
 843 {
 844    memset(key, 0, sizeof(struct draw_llvm_variant_key));
 845
 846    key->nr_vertex_elements = llvm->draw->pt.nr_vertex_elements;
 847
 848    memcpy(key->vertex_element,
 849           llvm->draw->pt.vertex_element,
 850           sizeof(struct pipe_vertex_element) * key->nr_vertex_elements);
 851
 852    memcpy(&key->vs,
 853           &llvm->draw->vs.vertex_shader->state,
 854           sizeof(struct pipe_shader_state));
 855 }