src/gallium/drivers/swr/swr_shader.cpp

   1 /****************************************************************************
   2  * Copyright (C) 2015 Intel Corporation.   All Rights Reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  ***************************************************************************/
  23
  24 // llvm redefines DEBUG
  25 #pragma push_macro("DEBUG")
  26 #undef DEBUG
  27 #include "JitManager.h"
  28 #include <llvm/Config/llvm-config.h>
  29 #include "llvm-c/Core.h"
  30 #include "llvm/Support/CBindingWrapping.h"
  31 #include "llvm/IR/LegacyPassManager.h"
  32 #pragma pop_macro("DEBUG")
  33
  34 #include "state.h"
  35 #include "gen_state_llvm.h"
  36 #include "builder.h"
  37 #include "functionpasses/passes.h"
  38
  39 #include "tgsi/tgsi_strings.h"
  40 #include "util/u_format.h"
  41 #include "util/u_prim.h"
  42 #include "gallivm/lp_bld_init.h"
  43 #include "gallivm/lp_bld_flow.h"
  44 #include "gallivm/lp_bld_struct.h"
  45 #include "gallivm/lp_bld_tgsi.h"
  46
  47 #include "swr_context.h"
  48 #include "gen_surf_state_llvm.h"
  49 #include "gen_swr_context_llvm.h"
  50 #include "swr_resource.h"
  51 #include "swr_state.h"
  52 #include "swr_screen.h"
  53
  54 using namespace SwrJit;
  55 using namespace llvm;
  56
  57 static unsigned
  58 locate_linkage(ubyte name, ubyte index, struct tgsi_shader_info *info);
  59
  60 bool operator==(const swr_jit_fs_key &lhs, const swr_jit_fs_key &rhs)
  61 {
  62    return !memcmp(&lhs, &rhs, sizeof(lhs));
  63 }
  64
  65 bool operator==(const swr_jit_vs_key &lhs, const swr_jit_vs_key &rhs)
  66 {
  67    return !memcmp(&lhs, &rhs, sizeof(lhs));
  68 }
  69
  70 bool operator==(const swr_jit_fetch_key &lhs, const swr_jit_fetch_key &rhs)
  71 {
  72    return !memcmp(&lhs, &rhs, sizeof(lhs));
  73 }
  74
  75 bool operator==(const swr_jit_gs_key &lhs, const swr_jit_gs_key &rhs)
  76 {
  77    return !memcmp(&lhs, &rhs, sizeof(lhs));
  78 }
  79
  80 static void
  81 swr_generate_sampler_key(const struct lp_tgsi_info &info,
  82                          struct swr_context *ctx,
  83                          enum pipe_shader_type shader_type,
  84                          struct swr_jit_sampler_key &key)
  85 {
  86    key.nr_samplers = info.base.file_max[TGSI_FILE_SAMPLER] + 1;
  87
  88    for (unsigned i = 0; i < key.nr_samplers; i++) {
  89       if (info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
  90          lp_sampler_static_sampler_state(
  91             &key.sampler[i].sampler_state,
  92             ctx->samplers[shader_type][i]);
  93       }
  94    }
  95
  96    /*
  97     * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
  98     * are dx10-style? Can't really have mixed opcodes, at least not
  99     * if we want to skip the holes here (without rescanning tgsi).
 100     */
 101    if (info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
 102       key.nr_sampler_views =
 103          info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
 104       for (unsigned i = 0; i < key.nr_sampler_views; i++) {
 105          if (info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
 106             const struct pipe_sampler_view *view =
 107                ctx->sampler_views[shader_type][i];
 108             lp_sampler_static_texture_state(
 109                &key.sampler[i].texture_state, view);
 110             if (view) {
 111                struct swr_resource *swr_res = swr_resource(view->texture);
 112                const struct util_format_description *desc =
 113                   util_format_description(view->format);
 114                if (swr_res->has_depth && swr_res->has_stencil &&
 115                    !util_format_has_depth(desc))
 116                   key.sampler[i].texture_state.format = PIPE_FORMAT_S8_UINT;
 117             }
 118          }
 119       }
 120    } else {
 121       key.nr_sampler_views = key.nr_samplers;
 122       for (unsigned i = 0; i < key.nr_sampler_views; i++) {
 123          if (info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
 124             const struct pipe_sampler_view *view =
 125                ctx->sampler_views[shader_type][i];
 126             lp_sampler_static_texture_state(
 127                &key.sampler[i].texture_state, view);
 128             if (view) {
 129                struct swr_resource *swr_res = swr_resource(view->texture);
 130                const struct util_format_description *desc =
 131                   util_format_description(view->format);
 132                if (swr_res->has_depth && swr_res->has_stencil &&
 133                    !util_format_has_depth(desc))
 134                   key.sampler[i].texture_state.format = PIPE_FORMAT_S8_UINT;
 135             }
 136          }
 137       }
 138    }
 139 }
 140
 141 void
 142 swr_generate_fs_key(struct swr_jit_fs_key &key,
 143                     struct swr_context *ctx,
 144                     swr_fragment_shader *swr_fs)
 145 {
 146    memset(&key, 0, sizeof(key));
 147
 148    key.nr_cbufs = ctx->framebuffer.nr_cbufs;
 149    key.light_twoside = ctx->rasterizer->light_twoside;
 150    key.sprite_coord_enable = ctx->rasterizer->sprite_coord_enable;
 151
 152    struct tgsi_shader_info *pPrevShader;
 153    if (ctx->gs)
 154       pPrevShader = &ctx->gs->info.base;
 155    else
 156       pPrevShader = &ctx->vs->info.base;
 157
 158    memcpy(&key.vs_output_semantic_name,
 159           &pPrevShader->output_semantic_name,
 160           sizeof(key.vs_output_semantic_name));
 161    memcpy(&key.vs_output_semantic_idx,
 162           &pPrevShader->output_semantic_index,
 163           sizeof(key.vs_output_semantic_idx));
 164
 165    swr_generate_sampler_key(swr_fs->info, ctx, PIPE_SHADER_FRAGMENT, key);
 166
 167    key.poly_stipple_enable = ctx->rasterizer->poly_stipple_enable &&
 168       ctx->poly_stipple.prim_is_poly;
 169 }
 170
 171 void
 172 swr_generate_vs_key(struct swr_jit_vs_key &key,
 173                     struct swr_context *ctx,
 174                     swr_vertex_shader *swr_vs)
 175 {
 176    memset(&key, 0, sizeof(key));
 177
 178    key.clip_plane_mask =
 179       swr_vs->info.base.clipdist_writemask ?
 180       swr_vs->info.base.clipdist_writemask & ctx->rasterizer->clip_plane_enable :
 181       ctx->rasterizer->clip_plane_enable;
 182
 183    swr_generate_sampler_key(swr_vs->info, ctx, PIPE_SHADER_VERTEX, key);
 184 }
 185
 186 void
 187 swr_generate_fetch_key(struct swr_jit_fetch_key &key,
 188                        struct swr_vertex_element_state *velems)
 189 {
 190    memset(&key, 0, sizeof(key));
 191
 192    key.fsState = velems->fsState;
 193 }
 194
 195 void
 196 swr_generate_gs_key(struct swr_jit_gs_key &key,
 197                     struct swr_context *ctx,
 198                     swr_geometry_shader *swr_gs)
 199 {
 200    memset(&key, 0, sizeof(key));
 201
 202    struct tgsi_shader_info *pPrevShader = &ctx->vs->info.base;
 203
 204    memcpy(&key.vs_output_semantic_name,
 205           &pPrevShader->output_semantic_name,
 206           sizeof(key.vs_output_semantic_name));
 207    memcpy(&key.vs_output_semantic_idx,
 208           &pPrevShader->output_semantic_index,
 209           sizeof(key.vs_output_semantic_idx));
 210
 211    swr_generate_sampler_key(swr_gs->info, ctx, PIPE_SHADER_GEOMETRY, key);
 212 }
 213
 214 struct BuilderSWR : public Builder {
 215    BuilderSWR(JitManager *pJitMgr, const char *pName)
 216       : Builder(pJitMgr)
 217    {
 218       pJitMgr->SetupNewModule();
 219       gallivm = gallivm_create(pName, wrap(&JM()->mContext));
 220       pJitMgr->mpCurrentModule = unwrap(gallivm->module);
 221    }
 222
 223    ~BuilderSWR() {
 224       gallivm_free_ir(gallivm);
 225    }
 226
 227    void WriteVS(Value *pVal, Value *pVsContext, Value *pVtxOutput,
 228                 unsigned slot, unsigned channel);
 229
 230    struct gallivm_state *gallivm;
 231    PFN_VERTEX_FUNC CompileVS(struct swr_context *ctx, swr_jit_vs_key &key);
 232    PFN_PIXEL_KERNEL CompileFS(struct swr_context *ctx, swr_jit_fs_key &key);
 233    PFN_GS_FUNC CompileGS(struct swr_context *ctx, swr_jit_gs_key &key);
 234
 235    LLVMValueRef
 236    swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
 237                            struct lp_build_tgsi_context * bld_base,
 238                            boolean is_vindex_indirect,
 239                            LLVMValueRef vertex_index,
 240                            boolean is_aindex_indirect,
 241                            LLVMValueRef attrib_index,
 242                            LLVMValueRef swizzle_index);
 243    void
 244    swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
 245                            struct lp_build_tgsi_context * bld_base,
 246                            LLVMValueRef (*outputs)[4],
 247                            LLVMValueRef emitted_vertices_vec);
 248
 249    void
 250    swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
 251                              struct lp_build_tgsi_context * bld_base,
 252                              LLVMValueRef verts_per_prim_vec,
 253                              LLVMValueRef emitted_prims_vec);
 254
 255    void
 256    swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
 257                         struct lp_build_tgsi_context * bld_base,
 258                         LLVMValueRef total_emitted_vertices_vec,
 259                         LLVMValueRef emitted_prims_vec);
 260
 261 };
 262
 263 struct swr_gs_llvm_iface {
 264    struct lp_build_tgsi_gs_iface base;
 265    struct tgsi_shader_info *info;
 266
 267    BuilderSWR *pBuilder;
 268
 269    Value *pGsCtx;
 270    SWR_GS_STATE *pGsState;
 271    uint32_t num_outputs;
 272    uint32_t num_verts_per_prim;
 273
 274    Value *pVtxAttribMap;
 275 };
 276
 277 // trampoline functions so we can use the builder llvm construction methods
 278 static LLVMValueRef
 279 swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
 280                            struct lp_build_tgsi_context * bld_base,
 281                            boolean is_vindex_indirect,
 282                            LLVMValueRef vertex_index,
 283                            boolean is_aindex_indirect,
 284                            LLVMValueRef attrib_index,
 285                            LLVMValueRef swizzle_index)
 286 {
 287     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_iface;
 288
 289     return iface->pBuilder->swr_gs_llvm_fetch_input(gs_iface, bld_base,
 290                                                    is_vindex_indirect,
 291                                                    vertex_index,
 292                                                    is_aindex_indirect,
 293                                                    attrib_index,
 294                                                    swizzle_index);
 295 }
 296
 297 static void
 298 swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
 299                            struct lp_build_tgsi_context * bld_base,
 300                            LLVMValueRef (*outputs)[4],
 301                            LLVMValueRef emitted_vertices_vec)
 302 {
 303     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
 304
 305     iface->pBuilder->swr_gs_llvm_emit_vertex(gs_base, bld_base,
 306                                             outputs,
 307                                             emitted_vertices_vec);
 308 }
 309
 310 static void
 311 swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
 312                              struct lp_build_tgsi_context * bld_base,
 313                              LLVMValueRef verts_per_prim_vec,
 314                              LLVMValueRef emitted_prims_vec)
 315 {
 316     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
 317
 318     iface->pBuilder->swr_gs_llvm_end_primitive(gs_base, bld_base,
 319                                               verts_per_prim_vec,
 320                                               emitted_prims_vec);
 321 }
 322
 323 static void
 324 swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
 325                         struct lp_build_tgsi_context * bld_base,
 326                         LLVMValueRef total_emitted_vertices_vec,
 327                         LLVMValueRef emitted_prims_vec)
 328 {
 329     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
 330
 331     iface->pBuilder->swr_gs_llvm_epilogue(gs_base, bld_base,
 332                                          total_emitted_vertices_vec,
 333                                          emitted_prims_vec);
 334 }
 335
 336 LLVMValueRef
 337 BuilderSWR::swr_gs_llvm_fetch_input(const struct lp_build_tgsi_gs_iface *gs_iface,
 338                            struct lp_build_tgsi_context * bld_base,
 339                            boolean is_vindex_indirect,
 340                            LLVMValueRef vertex_index,
 341                            boolean is_aindex_indirect,
 342                            LLVMValueRef attrib_index,
 343                            LLVMValueRef swizzle_index)
 344 {
 345     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_iface;
 346     Value *vert_index = unwrap(vertex_index);
 347     Value *attr_index = unwrap(attrib_index);
 348
 349     IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
 350
 351     if (is_vindex_indirect || is_aindex_indirect) {
 352        int i;
 353        Value *res = unwrap(bld_base->base.zero);
 354        struct lp_type type = bld_base->base.type;
 355
 356        for (i = 0; i < type.length; i++) {
 357           Value *vert_chan_index = vert_index;
 358           Value *attr_chan_index = attr_index;
 359
 360           if (is_vindex_indirect) {
 361              vert_chan_index = VEXTRACT(vert_index, C(i));
 362           }
 363           if (is_aindex_indirect) {
 364              attr_chan_index = VEXTRACT(attr_index, C(i));
 365           }
 366
 367           Value *attrib =
 368              LOAD(GEP(iface->pVtxAttribMap, {C(0), attr_chan_index}));
 369
 370           Value *pVertex = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pVerts});
 371           Value *pInputVertStride = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_inputVertStride});
 372
 373           Value *pVector = ADD(MUL(vert_chan_index, pInputVertStride), attrib);
 374           Value *pInput = LOAD(GEP(pVertex, {pVector, unwrap(swizzle_index)}));
 375
 376           Value *value = VEXTRACT(pInput, C(i));
 377           res = VINSERT(res, value, C(i));
 378        }
 379
 380        return wrap(res);
 381     } else {
 382        Value *attrib = LOAD(GEP(iface->pVtxAttribMap, {C(0), attr_index}));
 383
 384        Value *pVertex = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pVerts});
 385        Value *pInputVertStride = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_inputVertStride});
 386
 387        Value *pVector = ADD(MUL(vert_index, pInputVertStride), attrib);
 388
 389        Value *pInput = LOAD(GEP(pVertex, {pVector, unwrap(swizzle_index)}));
 390
 391        return wrap(pInput);
 392     }
 393 }
 394
 395 // GS output stream layout
 396 #define VERTEX_COUNT_SIZE 32
 397 #define CONTROL_HEADER_SIZE (8*32)
 398
 399 void
 400 BuilderSWR::swr_gs_llvm_emit_vertex(const struct lp_build_tgsi_gs_iface *gs_base,
 401                            struct lp_build_tgsi_context * bld_base,
 402                            LLVMValueRef (*outputs)[4],
 403                            LLVMValueRef emitted_vertices_vec)
 404 {
 405     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
 406
 407     IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
 408
 409     const uint32_t headerSize = VERTEX_COUNT_SIZE + CONTROL_HEADER_SIZE;
 410     const uint32_t attribSize = 4 * sizeof(float);
 411     const uint32_t vertSize = attribSize * SWR_VTX_NUM_SLOTS;
 412     Value *pVertexOffset = MUL(unwrap(emitted_vertices_vec), VIMMED1(vertSize));
 413
 414     Value *vMask = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_mask});
 415     Value *vMask1 = TRUNC(vMask, VectorType::get(mInt1Ty, mVWidth));
 416
 417     Value *pStack = STACKSAVE();
 418     Value *pTmpPtr = ALLOCA(mFP32Ty, C(4)); // used for dummy write for lane masking
 419
 420     for (uint32_t attrib = 0; attrib < iface->num_outputs; ++attrib) {
 421        uint32_t attribSlot = attrib;
 422        uint32_t sgvChannel = 0;
 423        if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_PSIZE) {
 424           attribSlot = VERTEX_SGV_SLOT;
 425           sgvChannel = VERTEX_SGV_POINT_SIZE_COMP;
 426        } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_LAYER) {
 427           attribSlot = VERTEX_SGV_SLOT;
 428           sgvChannel = VERTEX_SGV_RTAI_COMP;
 429        } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_VIEWPORT_INDEX) {
 430           attribSlot = VERTEX_SGV_SLOT;
 431           sgvChannel = VERTEX_SGV_VAI_COMP;
 432        } else if (iface->info->output_semantic_name[attrib] == TGSI_SEMANTIC_POSITION) {
 433           attribSlot = VERTEX_POSITION_SLOT;
 434        } else {
 435           attribSlot = VERTEX_ATTRIB_START_SLOT + attrib;
 436           if (iface->info->writes_position) {
 437              attribSlot--;
 438           }
 439        }
 440
 441        Value *pOutputOffset = ADD(pVertexOffset, VIMMED1(headerSize + attribSize * attribSlot)); // + sgvChannel ?
 442
 443        for (uint32_t lane = 0; lane < mVWidth; ++lane) {
 444           Value *pLaneOffset = VEXTRACT(pOutputOffset, C(lane));
 445           Value *pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
 446           Value *pStreamOffset = GEP(pStream, pLaneOffset);
 447           pStreamOffset = BITCAST(pStreamOffset, mFP32PtrTy);
 448
 449           Value *pLaneMask = VEXTRACT(vMask1, C(lane));
 450           pStreamOffset = SELECT(pLaneMask, pStreamOffset, pTmpPtr);
 451
 452           for (uint32_t channel = 0; channel < 4; ++channel) {
 453              Value *vData;
 454
 455              if (attribSlot == VERTEX_SGV_SLOT)
 456                 vData = LOAD(unwrap(outputs[attrib][0]));
 457              else
 458                 vData = LOAD(unwrap(outputs[attrib][channel]));
 459
 460              if (attribSlot != VERTEX_SGV_SLOT ||
 461                  sgvChannel == channel) {
 462                 vData = VEXTRACT(vData, C(lane));
 463                 STORE(vData, pStreamOffset);
 464              }
 465              pStreamOffset = GEP(pStreamOffset, C(1));
 466           }
 467        }
 468     }
 469
 470     STACKRESTORE(pStack);
 471 }
 472
 473 void
 474 BuilderSWR::swr_gs_llvm_end_primitive(const struct lp_build_tgsi_gs_iface *gs_base,
 475                              struct lp_build_tgsi_context * bld_base,
 476                              LLVMValueRef verts_per_prim_vec,
 477                              LLVMValueRef emitted_prims_vec)
 478 {
 479     swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
 480
 481     IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
 482
 483     Value *vMask = LOAD(iface->pGsCtx, { 0, SWR_GS_CONTEXT_mask });
 484     Value *vMask1 = TRUNC(vMask, VectorType::get(mInt1Ty, 8));
 485
 486     uint32_t vertsPerPrim = iface->num_verts_per_prim;
 487
 488     Value *vCount =
 489        ADD(MUL(unwrap(emitted_prims_vec), VIMMED1(vertsPerPrim)),
 490            unwrap(verts_per_prim_vec));
 491
 492     struct lp_build_tgsi_soa_context *bld = lp_soa_context(bld_base);
 493     vCount = LOAD(unwrap(bld->total_emitted_vertices_vec_ptr));
 494
 495     struct lp_exec_mask *exec_mask = &bld->exec_mask;
 496     Value *mask = unwrap(lp_build_mask_value(bld->mask));
 497     if (exec_mask->has_mask)
 498        mask = AND(mask, unwrap(exec_mask->exec_mask));
 499
 500     Value *cmpMask = VMASK(ICMP_NE(unwrap(verts_per_prim_vec), VIMMED1(0)));
 501     mask = AND(mask, cmpMask);
 502     vMask1 = TRUNC(mask, VectorType::get(mInt1Ty, 8));
 503
 504     vCount = SUB(vCount, VIMMED1(1));
 505     Value *vOffset = ADD(UDIV(vCount, VIMMED1(8)), VIMMED1(VERTEX_COUNT_SIZE));
 506     Value *vValue = SHL(VIMMED1(1), UREM(vCount, VIMMED1(8)));
 507
 508     vValue = TRUNC(vValue, VectorType::get(mInt8Ty, 8));
 509
 510     Value *pStack = STACKSAVE();
 511     Value *pTmpPtr = ALLOCA(mInt8Ty, C(4)); // used for dummy read/write for lane masking
 512
 513     for (uint32_t lane = 0; lane < mVWidth; ++lane) {
 514        Value *vLaneOffset = VEXTRACT(vOffset, C(lane));
 515        Value *pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
 516        Value *pStreamOffset = GEP(pStream, vLaneOffset);
 517
 518        Value *pLaneMask = VEXTRACT(vMask1, C(lane));
 519        pStreamOffset = SELECT(pLaneMask, pStreamOffset, pTmpPtr);
 520
 521        Value *vVal = LOAD(pStreamOffset);
 522        vVal = OR(vVal, VEXTRACT(vValue, C(lane)));
 523        STORE(vVal, pStreamOffset);
 524     }
 525
 526     STACKRESTORE(pStack);
 527 }
 528
 529 void
 530 BuilderSWR::swr_gs_llvm_epilogue(const struct lp_build_tgsi_gs_iface *gs_base,
 531                         struct lp_build_tgsi_context * bld_base,
 532                         LLVMValueRef total_emitted_vertices_vec,
 533                         LLVMValueRef emitted_prims_vec)
 534 {
 535    swr_gs_llvm_iface *iface = (swr_gs_llvm_iface*)gs_base;
 536
 537    IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
 538
 539    // Store emit count to each output stream in the first DWORD
 540    for (uint32_t lane = 0; lane < mVWidth; ++lane)
 541    {
 542       Value* pStream = LOAD(iface->pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
 543       pStream = BITCAST(pStream, mInt32PtrTy);
 544       Value* pLaneCount = VEXTRACT(unwrap(total_emitted_vertices_vec), C(lane));
 545       STORE(pLaneCount, pStream);
 546    }
 547 }
 548
 549 PFN_GS_FUNC
 550 BuilderSWR::CompileGS(struct swr_context *ctx, swr_jit_gs_key &key)
 551 {
 552    SWR_GS_STATE *pGS = &ctx->gs->gsState;
 553    struct tgsi_shader_info *info = &ctx->gs->info.base;
 554
 555    memset(pGS, 0, sizeof(*pGS));
 556
 557    pGS->gsEnable = true;
 558
 559    pGS->numInputAttribs = (VERTEX_ATTRIB_START_SLOT - VERTEX_POSITION_SLOT) + info->num_inputs;
 560    pGS->outputTopology =
 561       swr_convert_prim_topology(info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM]);
 562    pGS->maxNumVerts = info->properties[TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES];
 563    pGS->instanceCount = info->properties[TGSI_PROPERTY_GS_INVOCATIONS];
 564
 565    // XXX: single stream for now...
 566    pGS->isSingleStream = true;
 567    pGS->singleStreamID = 0;
 568
 569    pGS->vertexAttribOffset = VERTEX_POSITION_SLOT;
 570    pGS->inputVertStride = pGS->numInputAttribs + pGS->vertexAttribOffset;
 571    pGS->outputVertexSize = SWR_VTX_NUM_SLOTS;
 572    pGS->controlDataSize = 8; // GS ouputs max of 8 32B units
 573    pGS->controlDataOffset = VERTEX_COUNT_SIZE;
 574    pGS->outputVertexOffset = pGS->controlDataOffset + CONTROL_HEADER_SIZE;
 575
 576    pGS->allocationSize =
 577       VERTEX_COUNT_SIZE + // vertex count
 578       CONTROL_HEADER_SIZE + // control header
 579       (SWR_VTX_NUM_SLOTS * 16) * // sizeof vertex
 580       pGS->maxNumVerts; // num verts
 581
 582    struct swr_geometry_shader *gs = ctx->gs;
 583
 584    LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
 585    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
 586
 587    memset(outputs, 0, sizeof(outputs));
 588
 589    AttrBuilder attrBuilder;
 590    attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
 591
 592    std::vector<Type *> gsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
 593                               PointerType::get(mInt8Ty, 0),
 594                               PointerType::get(Gen_SWR_GS_CONTEXT(JM()), 0)};
 595    FunctionType *vsFuncType =
 596       FunctionType::get(Type::getVoidTy(JM()->mContext), gsArgs, false);
 597
 598    // create new vertex shader function
 599    auto pFunction = Function::Create(vsFuncType,
 600                                      GlobalValue::ExternalLinkage,
 601                                      "GS",
 602                                      JM()->mpCurrentModule);
 603 #if LLVM_VERSION_MAJOR < 5
 604    AttributeSet attrSet = AttributeSet::get(
 605       JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
 606    pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
 607 #else
 608    pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
 609 #endif
 610
 611    BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
 612    IRB()->SetInsertPoint(block);
 613    LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
 614
 615    auto argitr = pFunction->arg_begin();
 616    Value *hPrivateData = &*argitr++;
 617    hPrivateData->setName("hPrivateData");
 618    Value *pWorkerData = &*argitr++;
 619    pWorkerData->setName("pWorkerData");
 620    Value *pGsCtx = &*argitr++;
 621    pGsCtx->setName("gsCtx");
 622
 623    Value *consts_ptr =
 624       GEP(hPrivateData, {C(0), C(swr_draw_context_constantGS)});
 625    consts_ptr->setName("gs_constants");
 626    Value *const_sizes_ptr =
 627       GEP(hPrivateData, {0, swr_draw_context_num_constantsGS});
 628    const_sizes_ptr->setName("num_gs_constants");
 629
 630    struct lp_build_sampler_soa *sampler =
 631       swr_sampler_soa_create(key.sampler, PIPE_SHADER_GEOMETRY);
 632
 633    struct lp_bld_tgsi_system_values system_values;
 634    memset(&system_values, 0, sizeof(system_values));
 635    system_values.prim_id = wrap(LOAD(pGsCtx, {0, SWR_GS_CONTEXT_PrimitiveID}));
 636    system_values.instance_id = wrap(LOAD(pGsCtx, {0, SWR_GS_CONTEXT_InstanceID}));
 637
 638    std::vector<Constant*> mapConstants;
 639    Value *vtxAttribMap = ALLOCA(ArrayType::get(mInt32Ty, PIPE_MAX_SHADER_INPUTS));
 640    for (unsigned slot = 0; slot < info->num_inputs; slot++) {
 641       ubyte semantic_name = info->input_semantic_name[slot];
 642       ubyte semantic_idx = info->input_semantic_index[slot];
 643
 644       unsigned vs_slot = locate_linkage(semantic_name, semantic_idx, &ctx->vs->info.base);
 645
 646       vs_slot += VERTEX_ATTRIB_START_SLOT;
 647
 648       if (ctx->vs->info.base.output_semantic_name[0] == TGSI_SEMANTIC_POSITION)
 649          vs_slot--;
 650
 651       if (semantic_name == TGSI_SEMANTIC_POSITION)
 652          vs_slot = VERTEX_POSITION_SLOT;
 653
 654       STORE(C(vs_slot), vtxAttribMap, {0, slot});
 655       mapConstants.push_back(C(vs_slot));
 656    }
 657
 658    struct lp_build_mask_context mask;
 659    Value *mask_val = LOAD(pGsCtx, {0, SWR_GS_CONTEXT_mask}, "gsMask");
 660    lp_build_mask_begin(&mask, gallivm,
 661                        lp_type_float_vec(32, 32 * 8), wrap(mask_val));
 662
 663    // zero out cut buffer so we can load/modify/store bits
 664    for (uint32_t lane = 0; lane < mVWidth; ++lane)
 665    {
 666       Value* pStream = LOAD(pGsCtx, {0, SWR_GS_CONTEXT_pStreams, lane});
 667       MEMSET(pStream, C((char)0), VERTEX_COUNT_SIZE + CONTROL_HEADER_SIZE, sizeof(float) * KNOB_SIMD_WIDTH);
 668    }
 669
 670    struct swr_gs_llvm_iface gs_iface;
 671    gs_iface.base.fetch_input = ::swr_gs_llvm_fetch_input;
 672    gs_iface.base.emit_vertex = ::swr_gs_llvm_emit_vertex;
 673    gs_iface.base.end_primitive = ::swr_gs_llvm_end_primitive;
 674    gs_iface.base.gs_epilogue = ::swr_gs_llvm_epilogue;
 675    gs_iface.pBuilder = this;
 676    gs_iface.pGsCtx = pGsCtx;
 677    gs_iface.pGsState = pGS;
 678    gs_iface.num_outputs = gs->info.base.num_outputs;
 679    gs_iface.num_verts_per_prim =
 680       u_vertices_per_prim((pipe_prim_type)info->properties[TGSI_PROPERTY_GS_OUTPUT_PRIM]);
 681    gs_iface.info = info;
 682    gs_iface.pVtxAttribMap = vtxAttribMap;
 683
 684    struct lp_build_tgsi_params params;
 685    memset(&params, 0, sizeof(params));
 686    params.type = lp_type_float_vec(32, 32 * 8);
 687    params.mask = & mask;
 688    params.consts_ptr = wrap(consts_ptr);
 689    params.const_sizes_ptr = wrap(const_sizes_ptr);
 690    params.system_values = &system_values;
 691    params.inputs = inputs;
 692    params.context_ptr = wrap(hPrivateData);
 693    params.sampler = sampler;
 694    params.info = &gs->info.base;
 695    params.gs_iface = &gs_iface.base;
 696
 697    lp_build_tgsi_soa(gallivm,
 698                      gs->pipe.tokens,
 699                      &params,
 700                      outputs);
 701
 702    lp_build_mask_end(&mask);
 703
 704    sampler->destroy(sampler);
 705
 706    IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
 707
 708    RET_VOID();
 709
 710    gallivm_verify_function(gallivm, wrap(pFunction));
 711    gallivm_compile_module(gallivm);
 712
 713    PFN_GS_FUNC pFunc =
 714       (PFN_GS_FUNC)gallivm_jit_function(gallivm, wrap(pFunction));
 715
 716    debug_printf("geom shader  %p\n", pFunc);
 717    assert(pFunc && "Error: GeomShader = NULL");
 718
 719    JM()->mIsModuleFinalized = true;
 720
 721    return pFunc;
 722 }
 723
 724 PFN_GS_FUNC
 725 swr_compile_gs(struct swr_context *ctx, swr_jit_gs_key &key)
 726 {
 727    BuilderSWR builder(
 728       reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
 729       "GS");
 730    PFN_GS_FUNC func = builder.CompileGS(ctx, key);
 731
 732    ctx->gs->map.insert(std::make_pair(key, std::make_unique<VariantGS>(builder.gallivm, func)));
 733    return func;
 734 }
 735
 736 void
 737 BuilderSWR::WriteVS(Value *pVal, Value *pVsContext, Value *pVtxOutput, unsigned slot, unsigned channel)
 738 {
 739 #if USE_SIMD16_FRONTEND && !USE_SIMD16_VS
 740    // interleave the simdvertex components into the dest simd16vertex
 741    //   slot16offset = slot8offset * 2
 742    //   comp16offset = comp8offset * 2 + alternateOffset
 743
 744    Value *offset = LOAD(pVsContext, { 0, SWR_VS_CONTEXT_AlternateOffset });
 745    Value *pOut = GEP(pVtxOutput, { C(0), C(0), C(slot * 2), offset } );
 746    STORE(pVal, pOut, {channel * 2});
 747 #else
 748    Value *pOut = GEP(pVtxOutput, {0, 0, slot});
 749    STORE(pVal, pOut, {0, channel});
 750 #endif
 751 }
 752
 753 PFN_VERTEX_FUNC
 754 BuilderSWR::CompileVS(struct swr_context *ctx, swr_jit_vs_key &key)
 755 {
 756    struct swr_vertex_shader *swr_vs = ctx->vs;
 757
 758    LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
 759    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
 760
 761    memset(outputs, 0, sizeof(outputs));
 762
 763    AttrBuilder attrBuilder;
 764    attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
 765
 766    std::vector<Type *> vsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
 767                               PointerType::get(mInt8Ty, 0),
 768                               PointerType::get(Gen_SWR_VS_CONTEXT(JM()), 0)};
 769    FunctionType *vsFuncType =
 770       FunctionType::get(Type::getVoidTy(JM()->mContext), vsArgs, false);
 771
 772    // create new vertex shader function
 773    auto pFunction = Function::Create(vsFuncType,
 774                                      GlobalValue::ExternalLinkage,
 775                                      "VS",
 776                                      JM()->mpCurrentModule);
 777 #if LLVM_VERSION_MAJOR < 5
 778    AttributeSet attrSet = AttributeSet::get(
 779       JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
 780    pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
 781 #else
 782    pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
 783 #endif
 784
 785    BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
 786    IRB()->SetInsertPoint(block);
 787    LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
 788
 789    auto argitr = pFunction->arg_begin();
 790    Value *hPrivateData = &*argitr++;
 791    hPrivateData->setName("hPrivateData");
 792    Value *pWorkerData = &*argitr++;
 793    pWorkerData->setName("pWorkerData");
 794    Value *pVsCtx = &*argitr++;
 795    pVsCtx->setName("vsCtx");
 796
 797    Value *consts_ptr = GEP(hPrivateData, {C(0), C(swr_draw_context_constantVS)});
 798
 799    consts_ptr->setName("vs_constants");
 800    Value *const_sizes_ptr =
 801       GEP(hPrivateData, {0, swr_draw_context_num_constantsVS});
 802    const_sizes_ptr->setName("num_vs_constants");
 803
 804    Value *vtxInput = LOAD(pVsCtx, {0, SWR_VS_CONTEXT_pVin});
 805 #if USE_SIMD16_VS
 806    vtxInput = BITCAST(vtxInput, PointerType::get(Gen_simd16vertex(JM()), 0));
 807 #endif
 808
 809    for (uint32_t attrib = 0; attrib < PIPE_MAX_SHADER_INPUTS; attrib++) {
 810       const unsigned mask = swr_vs->info.base.input_usage_mask[attrib];
 811       for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
 812          if (mask & (1 << channel)) {
 813             inputs[attrib][channel] =
 814                wrap(LOAD(vtxInput, {0, 0, attrib, channel}));
 815          }
 816       }
 817    }
 818
 819    struct lp_build_sampler_soa *sampler =
 820       swr_sampler_soa_create(key.sampler, PIPE_SHADER_VERTEX);
 821
 822    struct lp_bld_tgsi_system_values system_values;
 823    memset(&system_values, 0, sizeof(system_values));
 824    system_values.instance_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_InstanceID}));
 825
 826 #if USE_SIMD16_VS
 827    system_values.vertex_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_VertexID16}));
 828 #else
 829    system_values.vertex_id = wrap(LOAD(pVsCtx, {0, SWR_VS_CONTEXT_VertexID}));
 830 #endif
 831
 832 #if USE_SIMD16_VS
 833    uint32_t vectorWidth = mVWidth16;
 834 #else
 835    uint32_t vectorWidth = mVWidth;
 836 #endif
 837
 838    struct lp_build_tgsi_params params;
 839    memset(&params, 0, sizeof(params));
 840    params.type = lp_type_float_vec(32, 32 * vectorWidth);
 841    params.consts_ptr = wrap(consts_ptr);
 842    params.const_sizes_ptr = wrap(const_sizes_ptr);
 843    params.system_values = &system_values;
 844    params.inputs = inputs;
 845    params.context_ptr = wrap(hPrivateData);
 846    params.sampler = sampler;
 847    params.info = &swr_vs->info.base;
 848
 849    lp_build_tgsi_soa(gallivm,
 850                      swr_vs->pipe.tokens,
 851                      &params,
 852                      outputs);
 853
 854    sampler->destroy(sampler);
 855
 856    IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
 857
 858    Value *vtxOutput = LOAD(pVsCtx, {0, SWR_VS_CONTEXT_pVout});
 859 #if USE_SIMD16_VS
 860    vtxOutput = BITCAST(vtxOutput, PointerType::get(Gen_simd16vertex(JM()), 0));
 861 #endif
 862
 863    for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
 864       for (uint32_t attrib = 0; attrib < PIPE_MAX_SHADER_OUTPUTS; attrib++) {
 865          if (!outputs[attrib][channel])
 866             continue;
 867
 868          Value *val;
 869          uint32_t outSlot;
 870
 871          if (swr_vs->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_PSIZE) {
 872             if (channel != VERTEX_SGV_POINT_SIZE_COMP)
 873                continue;
 874             val = LOAD(unwrap(outputs[attrib][0]));
 875             outSlot = VERTEX_SGV_SLOT;
 876          } else if (swr_vs->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_POSITION) {
 877             val = LOAD(unwrap(outputs[attrib][channel]));
 878             outSlot = VERTEX_POSITION_SLOT;
 879          } else {
 880             val = LOAD(unwrap(outputs[attrib][channel]));
 881             outSlot = VERTEX_ATTRIB_START_SLOT + attrib;
 882             if (swr_vs->info.base.output_semantic_name[0] == TGSI_SEMANTIC_POSITION)
 883                outSlot--;
 884          }
 885
 886          WriteVS(val, pVsCtx, vtxOutput, outSlot, channel);
 887       }
 888    }
 889
 890    if (ctx->rasterizer->clip_plane_enable ||
 891        swr_vs->info.base.culldist_writemask) {
 892       unsigned clip_mask = ctx->rasterizer->clip_plane_enable;
 893
 894       unsigned cv = 0;
 895       if (swr_vs->info.base.writes_clipvertex) {
 896          cv = locate_linkage(TGSI_SEMANTIC_CLIPVERTEX, 0,
 897                              &swr_vs->info.base);
 898       } else {
 899          for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
 900             if (swr_vs->info.base.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
 901                 swr_vs->info.base.output_semantic_index[i] == 0) {
 902                cv = i;
 903                break;
 904             }
 905          }
 906       }
 907       LLVMValueRef cx = LLVMBuildLoad(gallivm->builder, outputs[cv][0], "");
 908       LLVMValueRef cy = LLVMBuildLoad(gallivm->builder, outputs[cv][1], "");
 909       LLVMValueRef cz = LLVMBuildLoad(gallivm->builder, outputs[cv][2], "");
 910       LLVMValueRef cw = LLVMBuildLoad(gallivm->builder, outputs[cv][3], "");
 911
 912       for (unsigned val = 0; val < PIPE_MAX_CLIP_PLANES; val++) {
 913          // clip distance overrides user clip planes
 914          if ((swr_vs->info.base.clipdist_writemask & clip_mask & (1 << val)) ||
 915              ((swr_vs->info.base.culldist_writemask << swr_vs->info.base.num_written_clipdistance) & (1 << val))) {
 916             unsigned cv = locate_linkage(TGSI_SEMANTIC_CLIPDIST, val < 4 ? 0 : 1,
 917                                          &swr_vs->info.base);
 918             if (val < 4) {
 919                LLVMValueRef dist = LLVMBuildLoad(gallivm->builder, outputs[cv][val], "");
 920                WriteVS(unwrap(dist), pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_LO_SLOT, val);
 921             } else {
 922                LLVMValueRef dist = LLVMBuildLoad(gallivm->builder, outputs[cv][val - 4], "");
 923                WriteVS(unwrap(dist), pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_HI_SLOT, val - 4);
 924             }
 925             continue;
 926          }
 927
 928          if (!(clip_mask & (1 << val)))
 929             continue;
 930
 931          Value *px = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 0}));
 932          Value *py = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 1}));
 933          Value *pz = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 2}));
 934          Value *pw = LOAD(GEP(hPrivateData, {0, swr_draw_context_userClipPlanes, val, 3}));
 935 #if USE_SIMD16_VS
 936          Value *bpx = VBROADCAST_16(px);
 937          Value *bpy = VBROADCAST_16(py);
 938          Value *bpz = VBROADCAST_16(pz);
 939          Value *bpw = VBROADCAST_16(pw);
 940 #else
 941          Value *bpx = VBROADCAST(px);
 942          Value *bpy = VBROADCAST(py);
 943          Value *bpz = VBROADCAST(pz);
 944          Value *bpw = VBROADCAST(pw);
 945 #endif
 946          Value *dist = FADD(FMUL(unwrap(cx), bpx),
 947                             FADD(FMUL(unwrap(cy), bpy),
 948                                  FADD(FMUL(unwrap(cz), bpz),
 949                                       FMUL(unwrap(cw), bpw))));
 950
 951          if (val < 4)
 952             WriteVS(dist, pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_LO_SLOT, val);
 953          else
 954             WriteVS(dist, pVsCtx, vtxOutput, VERTEX_CLIPCULL_DIST_HI_SLOT, val - 4);
 955       }
 956    }
 957
 958    RET_VOID();
 959
 960    gallivm_verify_function(gallivm, wrap(pFunction));
 961    gallivm_compile_module(gallivm);
 962
 963    //   lp_debug_dump_value(func);
 964
 965    PFN_VERTEX_FUNC pFunc =
 966       (PFN_VERTEX_FUNC)gallivm_jit_function(gallivm, wrap(pFunction));
 967
 968    debug_printf("vert shader  %p\n", pFunc);
 969    assert(pFunc && "Error: VertShader = NULL");
 970
 971    JM()->mIsModuleFinalized = true;
 972
 973    return pFunc;
 974 }
 975
 976 PFN_VERTEX_FUNC
 977 swr_compile_vs(struct swr_context *ctx, swr_jit_vs_key &key)
 978 {
 979    if (!ctx->vs->pipe.tokens)
 980       return NULL;
 981
 982    BuilderSWR builder(
 983       reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
 984       "VS");
 985    PFN_VERTEX_FUNC func = builder.CompileVS(ctx, key);
 986
 987    ctx->vs->map.insert(std::make_pair(key, std::make_unique<VariantVS>(builder.gallivm, func)));
 988    return func;
 989 }
 990
 991 unsigned
 992 swr_so_adjust_attrib(unsigned in_attrib,
 993                      swr_vertex_shader *swr_vs)
 994 {
 995    ubyte semantic_name;
 996    unsigned attrib;
 997
 998    attrib = in_attrib + VERTEX_ATTRIB_START_SLOT;
 999
1000    if (swr_vs) {
1001       semantic_name = swr_vs->info.base.output_semantic_name[in_attrib];
1002       if (semantic_name == TGSI_SEMANTIC_POSITION) {
1003          attrib = VERTEX_POSITION_SLOT;
1004       } else if (semantic_name == TGSI_SEMANTIC_PSIZE) {
1005          attrib = VERTEX_SGV_SLOT;
1006       } else if (semantic_name == TGSI_SEMANTIC_LAYER) {
1007          attrib = VERTEX_SGV_SLOT;
1008       } else {
1009          if (swr_vs->info.base.writes_position) {
1010                attrib--;
1011          }
1012       }
1013    }
1014
1015    return attrib;
1016 }
1017
1018 static unsigned
1019 locate_linkage(ubyte name, ubyte index, struct tgsi_shader_info *info)
1020 {
1021    for (int i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
1022       if ((info->output_semantic_name[i] == name)
1023           && (info->output_semantic_index[i] == index)) {
1024          return i;
1025       }
1026    }
1027
1028    return 0xFFFFFFFF;
1029 }
1030
1031 PFN_PIXEL_KERNEL
1032 BuilderSWR::CompileFS(struct swr_context *ctx, swr_jit_fs_key &key)
1033 {
1034    struct swr_fragment_shader *swr_fs = ctx->fs;
1035
1036    struct tgsi_shader_info *pPrevShader;
1037    if (ctx->gs)
1038       pPrevShader = &ctx->gs->info.base;
1039    else
1040       pPrevShader = &ctx->vs->info.base;
1041
1042    LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS];
1043    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
1044
1045    memset(inputs, 0, sizeof(inputs));
1046    memset(outputs, 0, sizeof(outputs));
1047
1048    struct lp_build_sampler_soa *sampler = NULL;
1049
1050    AttrBuilder attrBuilder;
1051    attrBuilder.addStackAlignmentAttr(JM()->mVWidth * sizeof(float));
1052
1053    std::vector<Type *> fsArgs{PointerType::get(Gen_swr_draw_context(JM()), 0),
1054                               PointerType::get(mInt8Ty, 0),
1055                               PointerType::get(Gen_SWR_PS_CONTEXT(JM()), 0)};
1056    FunctionType *funcType =
1057       FunctionType::get(Type::getVoidTy(JM()->mContext), fsArgs, false);
1058
1059    auto pFunction = Function::Create(funcType,
1060                                      GlobalValue::ExternalLinkage,
1061                                      "FS",
1062                                      JM()->mpCurrentModule);
1063 #if LLVM_VERSION_MAJOR < 5
1064    AttributeSet attrSet = AttributeSet::get(
1065       JM()->mContext, AttributeSet::FunctionIndex, attrBuilder);
1066    pFunction->addAttributes(AttributeSet::FunctionIndex, attrSet);
1067 #else
1068    pFunction->addAttributes(AttributeList::FunctionIndex, attrBuilder);
1069 #endif
1070
1071    BasicBlock *block = BasicBlock::Create(JM()->mContext, "entry", pFunction);
1072    IRB()->SetInsertPoint(block);
1073    LLVMPositionBuilderAtEnd(gallivm->builder, wrap(block));
1074
1075    auto args = pFunction->arg_begin();
1076    Value *hPrivateData = &*args++;
1077    hPrivateData->setName("hPrivateData");
1078    Value *pWorkerData = &*args++;
1079    pWorkerData->setName("pWorkerData");
1080    Value *pPS = &*args++;
1081    pPS->setName("psCtx");
1082
1083    Value *consts_ptr = GEP(hPrivateData, {0, swr_draw_context_constantFS});
1084    consts_ptr->setName("fs_constants");
1085    Value *const_sizes_ptr =
1086       GEP(hPrivateData, {0, swr_draw_context_num_constantsFS});
1087    const_sizes_ptr->setName("num_fs_constants");
1088
1089    // load *pAttribs, *pPerspAttribs
1090    Value *pRawAttribs = LOAD(pPS, {0, SWR_PS_CONTEXT_pAttribs}, "pRawAttribs");
1091    Value *pPerspAttribs =
1092       LOAD(pPS, {0, SWR_PS_CONTEXT_pPerspAttribs}, "pPerspAttribs");
1093
1094    swr_fs->constantMask = 0;
1095    swr_fs->flatConstantMask = 0;
1096    swr_fs->pointSpriteMask = 0;
1097
1098    for (int attrib = 0; attrib < PIPE_MAX_SHADER_INPUTS; attrib++) {
1099       const unsigned mask = swr_fs->info.base.input_usage_mask[attrib];
1100       const unsigned interpMode = swr_fs->info.base.input_interpolate[attrib];
1101       const unsigned interpLoc = swr_fs->info.base.input_interpolate_loc[attrib];
1102
1103       if (!mask)
1104          continue;
1105
1106       // load i,j
1107       Value *vi = nullptr, *vj = nullptr;
1108       switch (interpLoc) {
1109       case TGSI_INTERPOLATE_LOC_CENTER:
1110          vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_center}, "i");
1111          vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_center}, "j");
1112          break;
1113       case TGSI_INTERPOLATE_LOC_CENTROID:
1114          vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_centroid}, "i");
1115          vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_centroid}, "j");
1116          break;
1117       case TGSI_INTERPOLATE_LOC_SAMPLE:
1118          vi = LOAD(pPS, {0, SWR_PS_CONTEXT_vI, PixelPositions_sample}, "i");
1119          vj = LOAD(pPS, {0, SWR_PS_CONTEXT_vJ, PixelPositions_sample}, "j");
1120          break;
1121       }
1122
1123       // load/compute w
1124       Value *vw = nullptr, *pAttribs;
1125       if (interpMode == TGSI_INTERPOLATE_PERSPECTIVE ||
1126           interpMode == TGSI_INTERPOLATE_COLOR) {
1127          pAttribs = pPerspAttribs;
1128          switch (interpLoc) {
1129          case TGSI_INTERPOLATE_LOC_CENTER:
1130             vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_center}));
1131             break;
1132          case TGSI_INTERPOLATE_LOC_CENTROID:
1133             vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_centroid}));
1134             break;
1135          case TGSI_INTERPOLATE_LOC_SAMPLE:
1136             vw = VRCP(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_sample}));
1137             break;
1138          }
1139       } else {
1140          pAttribs = pRawAttribs;
1141          vw = VIMMED1(1.f);
1142       }
1143
1144       vw->setName("w");
1145
1146       ubyte semantic_name = swr_fs->info.base.input_semantic_name[attrib];
1147       ubyte semantic_idx = swr_fs->info.base.input_semantic_index[attrib];
1148
1149       if (semantic_name == TGSI_SEMANTIC_FACE) {
1150          Value *ff =
1151             UI_TO_FP(LOAD(pPS, {0, SWR_PS_CONTEXT_frontFace}), mFP32Ty);
1152          ff = FSUB(FMUL(ff, C(2.0f)), C(1.0f));
1153          ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vFrontFace");
1154
1155          inputs[attrib][0] = wrap(ff);
1156          inputs[attrib][1] = wrap(VIMMED1(0.0f));
1157          inputs[attrib][2] = wrap(VIMMED1(0.0f));
1158          inputs[attrib][3] = wrap(VIMMED1(1.0f));
1159          continue;
1160       } else if (semantic_name == TGSI_SEMANTIC_POSITION) { // gl_FragCoord
1161          if (swr_fs->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER] ==
1162              TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER) {
1163             inputs[attrib][0] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_center}, "vX"));
1164             inputs[attrib][1] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_center}, "vY"));
1165          } else {
1166             inputs[attrib][0] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_UL}, "vX"));
1167             inputs[attrib][1] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_UL}, "vY"));
1168          }
1169          inputs[attrib][2] = wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vZ}, "vZ"));
1170          inputs[attrib][3] =
1171             wrap(LOAD(pPS, {0, SWR_PS_CONTEXT_vOneOverW, PixelPositions_center}, "vOneOverW"));
1172          continue;
1173       } else if (semantic_name == TGSI_SEMANTIC_LAYER) { // gl_Layer
1174          Value *ff = LOAD(pPS, {0, SWR_PS_CONTEXT_renderTargetArrayIndex});
1175          ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vRenderTargetArrayIndex");
1176          inputs[attrib][0] = wrap(ff);
1177          inputs[attrib][1] = wrap(VIMMED1(0.0f));
1178          inputs[attrib][2] = wrap(VIMMED1(0.0f));
1179          inputs[attrib][3] = wrap(VIMMED1(0.0f));
1180          continue;
1181       } else if (semantic_name == TGSI_SEMANTIC_VIEWPORT_INDEX) { // gl_ViewportIndex
1182          Value *ff = LOAD(pPS, {0, SWR_PS_CONTEXT_viewportIndex});
1183          ff = VECTOR_SPLAT(JM()->mVWidth, ff, "vViewportIndex");
1184          inputs[attrib][0] = wrap(ff);
1185          inputs[attrib][1] = wrap(VIMMED1(0.0f));
1186          inputs[attrib][2] = wrap(VIMMED1(0.0f));
1187          inputs[attrib][3] = wrap(VIMMED1(0.0f));
1188          continue;
1189       }
1190       unsigned linkedAttrib =
1191          locate_linkage(semantic_name, semantic_idx, pPrevShader) - 1;
1192
1193       uint32_t extraAttribs = 0;
1194       if (semantic_name == TGSI_SEMANTIC_PRIMID && !ctx->gs) {
1195          /* non-gs generated primID - need to grab from swizzleMap override */
1196          linkedAttrib = pPrevShader->num_outputs - 1;
1197          swr_fs->constantMask |= 1 << linkedAttrib;
1198          extraAttribs++;
1199       } else if (semantic_name == TGSI_SEMANTIC_GENERIC &&
1200           key.sprite_coord_enable & (1 << semantic_idx)) {
1201          /* we add an extra attrib to the backendState in swr_update_derived. */
1202          linkedAttrib = pPrevShader->num_outputs + extraAttribs - 1;
1203          swr_fs->pointSpriteMask |= (1 << linkedAttrib);
1204          extraAttribs++;
1205       } else if (linkedAttrib == 0xFFFFFFFF) {
1206          inputs[attrib][0] = wrap(VIMMED1(0.0f));
1207          inputs[attrib][1] = wrap(VIMMED1(0.0f));
1208          inputs[attrib][2] = wrap(VIMMED1(0.0f));
1209          inputs[attrib][3] = wrap(VIMMED1(1.0f));
1210          /* If we're reading in color and 2-sided lighting is enabled, we have
1211           * to keep going.
1212           */
1213          if (semantic_name != TGSI_SEMANTIC_COLOR || !key.light_twoside)
1214             continue;
1215       } else {
1216          if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1217             swr_fs->constantMask |= 1 << linkedAttrib;
1218          } else if (interpMode == TGSI_INTERPOLATE_COLOR) {
1219             swr_fs->flatConstantMask |= 1 << linkedAttrib;
1220          }
1221       }
1222
1223       unsigned bcolorAttrib = 0xFFFFFFFF;
1224       Value *offset = NULL;
1225       if (semantic_name == TGSI_SEMANTIC_COLOR && key.light_twoside) {
1226          bcolorAttrib = locate_linkage(
1227                TGSI_SEMANTIC_BCOLOR, semantic_idx, pPrevShader) - 1;
1228          /* Neither front nor back colors were available. Nothing to load. */
1229          if (bcolorAttrib == 0xFFFFFFFF && linkedAttrib == 0xFFFFFFFF)
1230             continue;
1231          /* If there is no front color, just always use the back color. */
1232          if (linkedAttrib == 0xFFFFFFFF)
1233             linkedAttrib = bcolorAttrib;
1234
1235          if (bcolorAttrib != 0xFFFFFFFF) {
1236             if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1237                swr_fs->constantMask |= 1 << bcolorAttrib;
1238             } else if (interpMode == TGSI_INTERPOLATE_COLOR) {
1239                swr_fs->flatConstantMask |= 1 << bcolorAttrib;
1240             }
1241
1242             unsigned diff = 12 * (bcolorAttrib - linkedAttrib);
1243
1244             if (diff) {
1245                Value *back =
1246                   XOR(C(1), LOAD(pPS, {0, SWR_PS_CONTEXT_frontFace}), "backFace");
1247
1248                offset = MUL(back, C(diff));
1249                offset->setName("offset");
1250             }
1251          }
1252       }
1253
1254       for (int channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
1255          if (mask & (1 << channel)) {
1256             Value *indexA = C(linkedAttrib * 12 + channel);
1257             Value *indexB = C(linkedAttrib * 12 + channel + 4);
1258             Value *indexC = C(linkedAttrib * 12 + channel + 8);
1259
1260             if (offset) {
1261                indexA = ADD(indexA, offset);
1262                indexB = ADD(indexB, offset);
1263                indexC = ADD(indexC, offset);
1264             }
1265
1266             Value *va = VBROADCAST(LOAD(GEP(pAttribs, indexA)));
1267             Value *vb = VBROADCAST(LOAD(GEP(pAttribs, indexB)));
1268             Value *vc = VBROADCAST(LOAD(GEP(pAttribs, indexC)));
1269
1270             if (interpMode == TGSI_INTERPOLATE_CONSTANT) {
1271                inputs[attrib][channel] = wrap(va);
1272             } else {
1273                Value *vk = FSUB(FSUB(VIMMED1(1.0f), vi), vj);
1274
1275                vc = FMUL(vk, vc);
1276
1277                Value *interp = FMUL(va, vi);
1278                Value *interp1 = FMUL(vb, vj);
1279                interp = FADD(interp, interp1);
1280                interp = FADD(interp, vc);
1281                if (interpMode == TGSI_INTERPOLATE_PERSPECTIVE ||
1282                    interpMode == TGSI_INTERPOLATE_COLOR)
1283                   interp = FMUL(interp, vw);
1284                inputs[attrib][channel] = wrap(interp);
1285             }
1286          }
1287       }
1288    }
1289
1290    sampler = swr_sampler_soa_create(key.sampler, PIPE_SHADER_FRAGMENT);
1291
1292    struct lp_bld_tgsi_system_values system_values;
1293    memset(&system_values, 0, sizeof(system_values));
1294
1295    struct lp_build_mask_context mask;
1296    bool uses_mask = false;
1297
1298    if (swr_fs->info.base.uses_kill ||
1299        key.poly_stipple_enable) {
1300       Value *vActiveMask = NULL;
1301       if (swr_fs->info.base.uses_kill) {
1302          vActiveMask = LOAD(pPS, {0, SWR_PS_CONTEXT_activeMask}, "activeMask");
1303       }
1304       if (key.poly_stipple_enable) {
1305          // first get fragment xy coords and clip to stipple bounds
1306          Value *vXf = LOAD(pPS, {0, SWR_PS_CONTEXT_vX, PixelPositions_UL});
1307          Value *vYf = LOAD(pPS, {0, SWR_PS_CONTEXT_vY, PixelPositions_UL});
1308          Value *vXu = FP_TO_UI(vXf, mSimdInt32Ty);
1309          Value *vYu = FP_TO_UI(vYf, mSimdInt32Ty);
1310
1311          // stipple pattern is 32x32, which means that one line of stipple
1312          // is stored in one word:
1313          // vXstipple is bit offset inside 32-bit stipple word
1314          // vYstipple is word index is stipple array
1315          Value *vXstipple = AND(vXu, VIMMED1(0x1f)); // & (32-1)
1316          Value *vYstipple = AND(vYu, VIMMED1(0x1f)); // & (32-1)
1317
1318          // grab stipple pattern base address
1319          Value *stipplePtr = GEP(hPrivateData, {0, swr_draw_context_polyStipple, 0});
1320          stipplePtr = BITCAST(stipplePtr, mInt8PtrTy);
1321
1322          // peform a gather to grab stipple words for each lane
1323          Value *vStipple = GATHERDD(VUNDEF_I(), stipplePtr, vYstipple,
1324                                     VIMMED1(0xffffffff), 4);
1325
1326          // create a mask with one bit corresponding to the x stipple
1327          // and AND it with the pattern, to see if we have a bit
1328          Value *vBitMask = LSHR(VIMMED1(0x80000000), vXstipple);
1329          Value *vStippleMask = AND(vStipple, vBitMask);
1330          vStippleMask = ICMP_NE(vStippleMask, VIMMED1(0));
1331          vStippleMask = VMASK(vStippleMask);
1332
1333          if (swr_fs->info.base.uses_kill) {
1334             vActiveMask = AND(vActiveMask, vStippleMask);
1335          } else {
1336             vActiveMask = vStippleMask;
1337          }
1338       }
1339       lp_build_mask_begin(
1340          &mask, gallivm, lp_type_float_vec(32, 32 * 8), wrap(vActiveMask));
1341       uses_mask = true;
1342    }
1343
1344    struct lp_build_tgsi_params params;
1345    memset(&params, 0, sizeof(params));
1346    params.type = lp_type_float_vec(32, 32 * 8);
1347    params.mask = uses_mask ? &mask : NULL;
1348    params.consts_ptr = wrap(consts_ptr);
1349    params.const_sizes_ptr = wrap(const_sizes_ptr);
1350    params.system_values = &system_values;
1351    params.inputs = inputs;
1352    params.context_ptr = wrap(hPrivateData);
1353    params.sampler = sampler;
1354    params.info = &swr_fs->info.base;
1355
1356    lp_build_tgsi_soa(gallivm,
1357                      swr_fs->pipe.tokens,
1358                      &params,
1359                      outputs);
1360
1361    sampler->destroy(sampler);
1362
1363    IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
1364
1365    for (uint32_t attrib = 0; attrib < swr_fs->info.base.num_outputs;
1366         attrib++) {
1367       switch (swr_fs->info.base.output_semantic_name[attrib]) {
1368       case TGSI_SEMANTIC_POSITION: {
1369          // write z
1370          LLVMValueRef outZ =
1371             LLVMBuildLoad(gallivm->builder, outputs[attrib][2], "");
1372          STORE(unwrap(outZ), pPS, {0, SWR_PS_CONTEXT_vZ});
1373          break;
1374       }
1375       case TGSI_SEMANTIC_COLOR: {
1376          for (uint32_t channel = 0; channel < TGSI_NUM_CHANNELS; channel++) {
1377             if (!outputs[attrib][channel])
1378                continue;
1379
1380             LLVMValueRef out =
1381                LLVMBuildLoad(gallivm->builder, outputs[attrib][channel], "");
1382             if (swr_fs->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS] &&
1383                 swr_fs->info.base.output_semantic_index[attrib] == 0) {
1384                for (uint32_t rt = 0; rt < key.nr_cbufs; rt++) {
1385                   STORE(unwrap(out),
1386                         pPS,
1387                         {0, SWR_PS_CONTEXT_shaded, rt, channel});
1388                }
1389             } else {
1390                STORE(unwrap(out),
1391                      pPS,
1392                      {0,
1393                            SWR_PS_CONTEXT_shaded,
1394                            swr_fs->info.base.output_semantic_index[attrib],
1395                            channel});
1396             }
1397          }
1398          break;
1399       }
1400       default: {
1401          fprintf(stderr,
1402                  "unknown output from FS %s[%d]\n",
1403                  tgsi_semantic_names[swr_fs->info.base
1404                                         .output_semantic_name[attrib]],
1405                  swr_fs->info.base.output_semantic_index[attrib]);
1406          break;
1407       }
1408       }
1409    }
1410
1411    LLVMValueRef mask_result = 0;
1412    if (uses_mask) {
1413       mask_result = lp_build_mask_end(&mask);
1414    }
1415
1416    IRB()->SetInsertPoint(unwrap(LLVMGetInsertBlock(gallivm->builder)));
1417
1418    if (uses_mask) {
1419       STORE(unwrap(mask_result), pPS, {0, SWR_PS_CONTEXT_activeMask});
1420    }
1421
1422    RET_VOID();
1423
1424    gallivm_verify_function(gallivm, wrap(pFunction));
1425
1426    gallivm_compile_module(gallivm);
1427
1428    // after the gallivm passes, we have to lower the core's intrinsics
1429    llvm::legacy::FunctionPassManager lowerPass(JM()->mpCurrentModule);
1430    lowerPass.add(createLowerX86Pass(this));
1431    lowerPass.run(*pFunction);
1432
1433    PFN_PIXEL_KERNEL kernel =
1434       (PFN_PIXEL_KERNEL)gallivm_jit_function(gallivm, wrap(pFunction));
1435    debug_printf("frag shader  %p\n", kernel);
1436    assert(kernel && "Error: FragShader = NULL");
1437
1438    JM()->mIsModuleFinalized = true;
1439
1440    return kernel;
1441 }
1442
1443 PFN_PIXEL_KERNEL
1444 swr_compile_fs(struct swr_context *ctx, swr_jit_fs_key &key)
1445 {
1446    if (!ctx->fs->pipe.tokens)
1447       return NULL;
1448
1449    BuilderSWR builder(
1450       reinterpret_cast<JitManager *>(swr_screen(ctx->pipe.screen)->hJitMgr),
1451       "FS");
1452    PFN_PIXEL_KERNEL func = builder.CompileFS(ctx, key);
1453
1454    ctx->fs->map.insert(std::make_pair(key, std::make_unique<VariantFS>(builder.gallivm, func)));
1455    return func;
1456 }