src/gallium/auxiliary/gallivm/lp_bld_ir_common.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * Copyright 2007-2008 VMware, Inc.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  **************************************************************************/
  28
  29 #include "util/u_memory.h"
  30 #include "lp_bld_type.h"
  31 #include "lp_bld_init.h"
  32 #include "lp_bld_flow.h"
  33 #include "lp_bld_ir_common.h"
  34 #include "lp_bld_logic.h"
  35
  36 /*
  37  * Return the context for the current function.
  38  * (always 'main', if shader doesn't do any function calls)
  39  */
  40 static inline struct function_ctx *
  41 func_ctx(struct lp_exec_mask *mask)
  42 {
  43    assert(mask->function_stack_size > 0);
  44    assert(mask->function_stack_size <= LP_MAX_NUM_FUNCS);
  45    return &mask->function_stack[mask->function_stack_size - 1];
  46 }
  47
  48 /*
  49  * Returns true if we're in a loop.
  50  * It's global, meaning that it returns true even if there's
  51  * no loop inside the current function, but we were inside
  52  * a loop inside another function, from which this one was called.
  53  */
  54 static inline boolean
  55 mask_has_loop(struct lp_exec_mask *mask)
  56 {
  57    int i;
  58    for (i = mask->function_stack_size - 1; i >= 0; --i) {
  59       const struct function_ctx *ctx = &mask->function_stack[i];
  60       if (ctx->loop_stack_size > 0)
  61          return TRUE;
  62    }
  63    return FALSE;
  64 }
  65
  66 /*
  67  * Returns true if we're inside a switch statement.
  68  * It's global, meaning that it returns true even if there's
  69  * no switch in the current function, but we were inside
  70  * a switch inside another function, from which this one was called.
  71  */
  72 static inline boolean
  73 mask_has_switch(struct lp_exec_mask *mask)
  74 {
  75    int i;
  76    for (i = mask->function_stack_size - 1; i >= 0; --i) {
  77       const struct function_ctx *ctx = &mask->function_stack[i];
  78       if (ctx->switch_stack_size > 0)
  79          return TRUE;
  80    }
  81    return FALSE;
  82 }
  83
  84 /*
  85  * Returns true if we're inside a conditional.
  86  * It's global, meaning that it returns true even if there's
  87  * no conditional in the current function, but we were inside
  88  * a conditional inside another function, from which this one was called.
  89  */
  90 static inline boolean
  91 mask_has_cond(struct lp_exec_mask *mask)
  92 {
  93    int i;
  94    for (i = mask->function_stack_size - 1; i >= 0; --i) {
  95       const struct function_ctx *ctx = &mask->function_stack[i];
  96       if (ctx->cond_stack_size > 0)
  97          return TRUE;
  98    }
  99    return FALSE;
 100 }
 101
 102 void lp_exec_mask_update(struct lp_exec_mask *mask)
 103 {
 104    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 105    boolean has_loop_mask = mask_has_loop(mask);
 106    boolean has_cond_mask = mask_has_cond(mask);
 107    boolean has_switch_mask = mask_has_switch(mask);
 108    boolean has_ret_mask = mask->function_stack_size > 1 ||
 109          mask->ret_in_main;
 110
 111    if (has_loop_mask) {
 112       /*for loops we need to update the entire mask at runtime */
 113       LLVMValueRef tmp;
 114       assert(mask->break_mask);
 115       tmp = LLVMBuildAnd(builder,
 116                          mask->cont_mask,
 117                          mask->break_mask,
 118                          "maskcb");
 119       mask->exec_mask = LLVMBuildAnd(builder,
 120                                      mask->cond_mask,
 121                                      tmp,
 122                                      "maskfull");
 123    } else
 124       mask->exec_mask = mask->cond_mask;
 125
 126    if (has_switch_mask) {
 127       mask->exec_mask = LLVMBuildAnd(builder,
 128                                      mask->exec_mask,
 129                                      mask->switch_mask,
 130                                      "switchmask");
 131    }
 132
 133    if (has_ret_mask) {
 134       mask->exec_mask = LLVMBuildAnd(builder,
 135                                      mask->exec_mask,
 136                                      mask->ret_mask,
 137                                      "callmask");
 138    }
 139
 140    mask->has_mask = (has_cond_mask ||
 141                      has_loop_mask ||
 142                      has_switch_mask ||
 143                      has_ret_mask);
 144 }
 145
 146 /*
 147  * Initialize a function context at the specified index.
 148  */
 149 void
 150 lp_exec_mask_function_init(struct lp_exec_mask *mask, int function_idx)
 151 {
 152    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 153    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 154    struct function_ctx *ctx =  &mask->function_stack[function_idx];
 155
 156    ctx->cond_stack_size = 0;
 157    ctx->loop_stack_size = 0;
 158    ctx->bgnloop_stack_size = 0;
 159    ctx->switch_stack_size = 0;
 160
 161    if (function_idx == 0) {
 162       ctx->ret_mask = mask->ret_mask;
 163    }
 164
 165    ctx->loop_limiter = lp_build_alloca(mask->bld->gallivm,
 166                                        int_type, "looplimiter");
 167    LLVMBuildStore(
 168       builder,
 169       LLVMConstInt(int_type, LP_MAX_TGSI_LOOP_ITERATIONS, false),
 170       ctx->loop_limiter);
 171 }
 172
 173 void lp_exec_mask_init(struct lp_exec_mask *mask, struct lp_build_context *bld)
 174 {
 175    mask->bld = bld;
 176    mask->has_mask = FALSE;
 177    mask->ret_in_main = FALSE;
 178    /* For the main function */
 179    mask->function_stack_size = 1;
 180
 181    mask->int_vec_type = lp_build_int_vec_type(bld->gallivm, mask->bld->type);
 182    mask->exec_mask = mask->ret_mask = mask->break_mask = mask->cont_mask =
 183          mask->cond_mask = mask->switch_mask =
 184          LLVMConstAllOnes(mask->int_vec_type);
 185
 186    mask->function_stack = CALLOC(LP_MAX_NUM_FUNCS,
 187                                  sizeof(mask->function_stack[0]));
 188    lp_exec_mask_function_init(mask, 0);
 189 }
 190
 191 void
 192 lp_exec_mask_fini(struct lp_exec_mask *mask)
 193 {
 194    FREE(mask->function_stack);
 195 }
 196
 197 /* stores val into an address pointed to by dst_ptr.
 198  * mask->exec_mask is used to figure out which bits of val
 199  * should be stored into the address
 200  * (0 means don't store this bit, 1 means do store).
 201  */
 202 void lp_exec_mask_store(struct lp_exec_mask *mask,
 203                         struct lp_build_context *bld_store,
 204                         LLVMValueRef val,
 205                         LLVMValueRef dst_ptr)
 206 {
 207    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 208    LLVMValueRef exec_mask = mask->has_mask ? mask->exec_mask : NULL;
 209
 210    assert(lp_check_value(bld_store->type, val));
 211    assert(LLVMGetTypeKind(LLVMTypeOf(dst_ptr)) == LLVMPointerTypeKind);
 212    assert(LLVMGetElementType(LLVMTypeOf(dst_ptr)) == LLVMTypeOf(val) ||
 213           LLVMGetTypeKind(LLVMGetElementType(LLVMTypeOf(dst_ptr))) == LLVMArrayTypeKind);
 214
 215    if (exec_mask) {
 216       LLVMValueRef res, dst;
 217
 218       dst = LLVMBuildLoad(builder, dst_ptr, "");
 219       res = lp_build_select(bld_store, exec_mask, val, dst);
 220       LLVMBuildStore(builder, res, dst_ptr);
 221    } else
 222       LLVMBuildStore(builder, val, dst_ptr);
 223 }
 224
 225 void lp_exec_bgnloop_post_phi(struct lp_exec_mask *mask)
 226 {
 227    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 228    struct function_ctx *ctx = func_ctx(mask);
 229
 230    if (ctx->loop_stack_size != ctx->bgnloop_stack_size) {
 231       mask->break_mask = LLVMBuildLoad(builder, ctx->break_var, "");
 232       lp_exec_mask_update(mask);
 233       ctx->bgnloop_stack_size = ctx->loop_stack_size;
 234    }
 235 }
 236
 237 void lp_exec_bgnloop(struct lp_exec_mask *mask, bool load)
 238 {
 239    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 240    struct function_ctx *ctx = func_ctx(mask);
 241
 242    if (ctx->loop_stack_size >= LP_MAX_TGSI_NESTING) {
 243       ++ctx->loop_stack_size;
 244       return;
 245    }
 246
 247    ctx->break_type_stack[ctx->loop_stack_size + ctx->switch_stack_size] =
 248       ctx->break_type;
 249    ctx->break_type = LP_EXEC_MASK_BREAK_TYPE_LOOP;
 250
 251    ctx->loop_stack[ctx->loop_stack_size].loop_block = ctx->loop_block;
 252    ctx->loop_stack[ctx->loop_stack_size].cont_mask = mask->cont_mask;
 253    ctx->loop_stack[ctx->loop_stack_size].break_mask = mask->break_mask;
 254    ctx->loop_stack[ctx->loop_stack_size].break_var = ctx->break_var;
 255    ++ctx->loop_stack_size;
 256
 257    ctx->break_var = lp_build_alloca(mask->bld->gallivm, mask->int_vec_type, "");
 258    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 259
 260    ctx->loop_block = lp_build_insert_new_block(mask->bld->gallivm, "bgnloop");
 261
 262    LLVMBuildBr(builder, ctx->loop_block);
 263    LLVMPositionBuilderAtEnd(builder, ctx->loop_block);
 264
 265    if (load) {
 266       lp_exec_bgnloop_post_phi(mask);
 267    }
 268 }
 269
 270 void lp_exec_endloop(struct gallivm_state *gallivm,
 271                      struct lp_exec_mask *mask)
 272 {
 273    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 274    struct function_ctx *ctx = func_ctx(mask);
 275    LLVMBasicBlockRef endloop;
 276    LLVMTypeRef int_type = LLVMInt32TypeInContext(mask->bld->gallivm->context);
 277    LLVMTypeRef reg_type = LLVMIntTypeInContext(gallivm->context,
 278                                                mask->bld->type.width *
 279                                                mask->bld->type.length);
 280    LLVMValueRef i1cond, i2cond, icond, limiter;
 281
 282    assert(mask->break_mask);
 283
 284    assert(ctx->loop_stack_size);
 285    if (ctx->loop_stack_size > LP_MAX_TGSI_NESTING) {
 286       --ctx->loop_stack_size;
 287       --ctx->bgnloop_stack_size;
 288       return;
 289    }
 290
 291    /*
 292     * Restore the cont_mask, but don't pop
 293     */
 294    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size - 1].cont_mask;
 295    lp_exec_mask_update(mask);
 296
 297    /*
 298     * Unlike the continue mask, the break_mask must be preserved across loop
 299     * iterations
 300     */
 301    LLVMBuildStore(builder, mask->break_mask, ctx->break_var);
 302
 303    /* Decrement the loop limiter */
 304    limiter = LLVMBuildLoad(builder, ctx->loop_limiter, "");
 305
 306    limiter = LLVMBuildSub(
 307       builder,
 308       limiter,
 309       LLVMConstInt(int_type, 1, false),
 310       "");
 311
 312    LLVMBuildStore(builder, limiter, ctx->loop_limiter);
 313
 314    /* i1cond = (mask != 0) */
 315    i1cond = LLVMBuildICmp(
 316       builder,
 317       LLVMIntNE,
 318       LLVMBuildBitCast(builder, mask->exec_mask, reg_type, ""),
 319       LLVMConstNull(reg_type), "i1cond");
 320
 321    /* i2cond = (looplimiter > 0) */
 322    i2cond = LLVMBuildICmp(
 323       builder,
 324       LLVMIntSGT,
 325       limiter,
 326       LLVMConstNull(int_type), "i2cond");
 327
 328    /* if( i1cond && i2cond ) */
 329    icond = LLVMBuildAnd(builder, i1cond, i2cond, "");
 330
 331    endloop = lp_build_insert_new_block(mask->bld->gallivm, "endloop");
 332
 333    LLVMBuildCondBr(builder,
 334                    icond, ctx->loop_block, endloop);
 335
 336    LLVMPositionBuilderAtEnd(builder, endloop);
 337
 338    assert(ctx->loop_stack_size);
 339    --ctx->loop_stack_size;
 340    --ctx->bgnloop_stack_size;
 341    mask->cont_mask = ctx->loop_stack[ctx->loop_stack_size].cont_mask;
 342    mask->break_mask = ctx->loop_stack[ctx->loop_stack_size].break_mask;
 343    ctx->loop_block = ctx->loop_stack[ctx->loop_stack_size].loop_block;
 344    ctx->break_var = ctx->loop_stack[ctx->loop_stack_size].break_var;
 345    ctx->break_type = ctx->break_type_stack[ctx->loop_stack_size +
 346          ctx->switch_stack_size];
 347
 348    lp_exec_mask_update(mask);
 349 }
 350
 351 void lp_exec_mask_cond_push(struct lp_exec_mask *mask,
 352                             LLVMValueRef val)
 353 {
 354    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 355    struct function_ctx *ctx = func_ctx(mask);
 356
 357    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING) {
 358       ctx->cond_stack_size++;
 359       return;
 360    }
 361    if (ctx->cond_stack_size == 0 && mask->function_stack_size == 1) {
 362       assert(mask->cond_mask == LLVMConstAllOnes(mask->int_vec_type));
 363    }
 364    ctx->cond_stack[ctx->cond_stack_size++] = mask->cond_mask;
 365    assert(LLVMTypeOf(val) == mask->int_vec_type);
 366    mask->cond_mask = LLVMBuildAnd(builder,
 367                                   mask->cond_mask,
 368                                   val,
 369                                   "");
 370    lp_exec_mask_update(mask);
 371 }
 372
 373 void lp_exec_mask_cond_invert(struct lp_exec_mask *mask)
 374 {
 375    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 376    struct function_ctx *ctx = func_ctx(mask);
 377    LLVMValueRef prev_mask;
 378    LLVMValueRef inv_mask;
 379
 380    assert(ctx->cond_stack_size);
 381    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 382       return;
 383    prev_mask = ctx->cond_stack[ctx->cond_stack_size - 1];
 384    if (ctx->cond_stack_size == 1 && mask->function_stack_size == 1) {
 385       assert(prev_mask == LLVMConstAllOnes(mask->int_vec_type));
 386    }
 387
 388    inv_mask = LLVMBuildNot(builder, mask->cond_mask, "");
 389
 390    mask->cond_mask = LLVMBuildAnd(builder,
 391                                   inv_mask,
 392                                   prev_mask, "");
 393    lp_exec_mask_update(mask);
 394 }
 395
 396 void lp_exec_mask_cond_pop(struct lp_exec_mask *mask)
 397 {
 398    struct function_ctx *ctx = func_ctx(mask);
 399    assert(ctx->cond_stack_size);
 400    --ctx->cond_stack_size;
 401    if (ctx->cond_stack_size >= LP_MAX_TGSI_NESTING)
 402       return;
 403    mask->cond_mask = ctx->cond_stack[ctx->cond_stack_size];
 404    lp_exec_mask_update(mask);
 405 }
 406
 407
 408 void lp_exec_continue(struct lp_exec_mask *mask)
 409 {
 410    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 411    LLVMValueRef exec_mask = LLVMBuildNot(builder,
 412                                          mask->exec_mask,
 413                                          "");
 414
 415    mask->cont_mask = LLVMBuildAnd(builder,
 416                                   mask->cont_mask,
 417                                   exec_mask, "");
 418
 419    lp_exec_mask_update(mask);
 420 }
 421
 422 void lp_exec_break(struct lp_exec_mask *mask, int *pc,
 423                    bool break_always)
 424 {
 425    LLVMBuilderRef builder = mask->bld->gallivm->builder;
 426    struct function_ctx *ctx = func_ctx(mask);
 427
 428    if (ctx->break_type == LP_EXEC_MASK_BREAK_TYPE_LOOP) {
 429       LLVMValueRef exec_mask = LLVMBuildNot(builder,
 430                                             mask->exec_mask,
 431                                             "break");
 432
 433       mask->break_mask = LLVMBuildAnd(builder,
 434                                       mask->break_mask,
 435                                       exec_mask, "break_full");
 436    }
 437    else {
 438       if (ctx->switch_in_default) {
 439          /*
 440           * stop default execution but only if this is an unconditional switch.
 441           * (The condition here is not perfect since dead code after break is
 442           * allowed but should be sufficient since false negatives are just
 443           * unoptimized - so we don't have to pre-evaluate that).
 444           */
 445          if(break_always && ctx->switch_pc) {
 446             if (pc)
 447                *pc = ctx->switch_pc;
 448             return;
 449          }
 450       }
 451
 452       if (break_always) {
 453          mask->switch_mask = LLVMConstNull(mask->bld->int_vec_type);
 454       }
 455       else {
 456          LLVMValueRef exec_mask = LLVMBuildNot(builder,
 457                                                mask->exec_mask,
 458                                                "break");
 459          mask->switch_mask = LLVMBuildAnd(builder,
 460                                           mask->switch_mask,
 461                                           exec_mask, "break_switch");
 462       }
 463    }
 464
 465    lp_exec_mask_update(mask);
 466 }