src/gallium/auxiliary/draw/draw_vf_sse.c

   1 /*
   2  * Copyright 2003 Tungsten Graphics, inc.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * on the rights to use, copy, modify, merge, publish, distribute, sub
   9  * license, and/or sell copies of the Software, and to permit persons to whom
  10  * the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.  IN NO EVENT SHALL
  19  * TUNGSTEN GRAPHICS AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  20  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  21  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  22  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Keith Whitwell <keithw@tungstengraphics.com>
  26  */
  27
  28
  29 #include "pipe/p_compiler.h"
  30 #include "util/u_simple_list.h"
  31
  32 #include "draw_vf.h"
  33
  34
  35 #if defined(USE_SSE_ASM)
  36
  37 #include "rtasm/rtasm_cpu.h"
  38 #include "rtasm/rtasm_x86sse.h"
  39
  40
  41 #define X    0
  42 #define Y    1
  43 #define Z    2
  44 #define W    3
  45
  46
  47 struct x86_program {
  48    struct x86_function func;
  49
  50    struct draw_vertex_fetch *vf;
  51    boolean inputs_safe;
  52    boolean outputs_safe;
  53    boolean have_sse2;
  54
  55    struct x86_reg identity;
  56    struct x86_reg chan0;
  57 };
  58
  59
  60 static struct x86_reg get_identity( struct x86_program *p )
  61 {
  62    return p->identity;
  63 }
  64
  65 static void emit_load4f_4( struct x86_program *p,
  66                            struct x86_reg dest,
  67                            struct x86_reg arg0 )
  68 {
  69    sse_movups(&p->func, dest, arg0);
  70 }
  71
  72 static void emit_load4f_3( struct x86_program *p,
  73                            struct x86_reg dest,
  74                            struct x86_reg arg0 )
  75 {
  76    /* Have to jump through some hoops:
  77     *
  78     * c 0 0 0
  79     * c 0 0 1
  80     * 0 0 c 1
  81     * a b c 1
  82     */
  83    sse_movss(&p->func, dest, x86_make_disp(arg0, 8));
  84    sse_shufps(&p->func, dest, get_identity(p), SHUF(X,Y,Z,W) );
  85    sse_shufps(&p->func, dest, dest, SHUF(Y,Z,X,W) );
  86    sse_movlps(&p->func, dest, arg0);
  87 }
  88
  89 static void emit_load4f_2( struct x86_program *p,
  90                            struct x86_reg dest,
  91                            struct x86_reg arg0 )
  92 {
  93    /* Initialize from identity, then pull in low two words:
  94     */
  95    sse_movups(&p->func, dest, get_identity(p));
  96    sse_movlps(&p->func, dest, arg0);
  97 }
  98
  99 static void emit_load4f_1( struct x86_program *p,
 100                            struct x86_reg dest,
 101                            struct x86_reg arg0 )
 102 {
 103    /* Pull in low word, then swizzle in identity */
 104    sse_movss(&p->func, dest, arg0);
 105    sse_shufps(&p->func, dest, get_identity(p), SHUF(X,Y,Z,W) );
 106 }
 107
 108
 109
 110 static void emit_load3f_3( struct x86_program *p,
 111                            struct x86_reg dest,
 112                            struct x86_reg arg0 )
 113 {
 114    /* Over-reads by 1 dword - potential SEGV if input is a vertex
 115     * array.
 116     */
 117    if (p->inputs_safe) {
 118       sse_movups(&p->func, dest, arg0);
 119    }
 120    else {
 121       /* c 0 0 0
 122        * c c c c
 123        * a b c c
 124        */
 125       sse_movss(&p->func, dest, x86_make_disp(arg0, 8));
 126       sse_shufps(&p->func, dest, dest, SHUF(X,X,X,X));
 127       sse_movlps(&p->func, dest, arg0);
 128    }
 129 }
 130
 131 static void emit_load3f_2( struct x86_program *p,
 132                            struct x86_reg dest,
 133                            struct x86_reg arg0 )
 134 {
 135    emit_load4f_2(p, dest, arg0);
 136 }
 137
 138 static void emit_load3f_1( struct x86_program *p,
 139                            struct x86_reg dest,
 140                            struct x86_reg arg0 )
 141 {
 142    emit_load4f_1(p, dest, arg0);
 143 }
 144
 145 static void emit_load2f_2( struct x86_program *p,
 146                            struct x86_reg dest,
 147                            struct x86_reg arg0 )
 148 {
 149    sse_movlps(&p->func, dest, arg0);
 150 }
 151
 152 static void emit_load2f_1( struct x86_program *p,
 153                            struct x86_reg dest,
 154                            struct x86_reg arg0 )
 155 {
 156    emit_load4f_1(p, dest, arg0);
 157 }
 158
 159 static void emit_load1f_1( struct x86_program *p,
 160                            struct x86_reg dest,
 161                            struct x86_reg arg0 )
 162 {
 163    sse_movss(&p->func, dest, arg0);
 164 }
 165
 166 static void (*load[4][4])( struct x86_program *p,
 167                            struct x86_reg dest,
 168                            struct x86_reg arg0 ) = {
 169    { emit_load1f_1,
 170      emit_load1f_1,
 171      emit_load1f_1,
 172      emit_load1f_1 },
 173
 174    { emit_load2f_1,
 175      emit_load2f_2,
 176      emit_load2f_2,
 177      emit_load2f_2 },
 178
 179    { emit_load3f_1,
 180      emit_load3f_2,
 181      emit_load3f_3,
 182      emit_load3f_3 },
 183
 184    { emit_load4f_1,
 185      emit_load4f_2,
 186      emit_load4f_3,
 187      emit_load4f_4 }
 188 };
 189
 190 static void emit_load( struct x86_program *p,
 191                        struct x86_reg dest,
 192                        unsigned sz,
 193                        struct x86_reg src,
 194                        unsigned src_sz)
 195 {
 196    load[sz-1][src_sz-1](p, dest, src);
 197 }
 198
 199 static void emit_store4f( struct x86_program *p,
 200                           struct x86_reg dest,
 201                           struct x86_reg arg0 )
 202 {
 203    sse_movups(&p->func, dest, arg0);
 204 }
 205
 206 static void emit_store3f( struct x86_program *p,
 207                           struct x86_reg dest,
 208                           struct x86_reg arg0 )
 209 {
 210    if (p->outputs_safe) {
 211       /* Emit the extra dword anyway.  This may hurt writecombining,
 212        * may cause other problems.
 213        */
 214       sse_movups(&p->func, dest, arg0);
 215    }
 216    else {
 217       /* Alternate strategy - emit two, shuffle, emit one.
 218        */
 219       sse_movlps(&p->func, dest, arg0);
 220       sse_shufps(&p->func, arg0, arg0, SHUF(Z,Z,Z,Z) ); /* NOTE! destructive */
 221       sse_movss(&p->func, x86_make_disp(dest,8), arg0);
 222    }
 223 }
 224
 225 static void emit_store2f( struct x86_program *p,
 226                            struct x86_reg dest,
 227                            struct x86_reg arg0 )
 228 {
 229    sse_movlps(&p->func, dest, arg0);
 230 }
 231
 232 static void emit_store1f( struct x86_program *p,
 233                           struct x86_reg dest,
 234                           struct x86_reg arg0 )
 235 {
 236    sse_movss(&p->func, dest, arg0);
 237 }
 238
 239
 240 static void (*store[4])( struct x86_program *p,
 241                          struct x86_reg dest,
 242                          struct x86_reg arg0 ) =
 243 {
 244    emit_store1f,
 245    emit_store2f,
 246    emit_store3f,
 247    emit_store4f
 248 };
 249
 250 static void emit_store( struct x86_program *p,
 251                         struct x86_reg dest,
 252                         unsigned sz,
 253                         struct x86_reg temp )
 254
 255 {
 256    store[sz-1](p, dest, temp);
 257 }
 258
 259 static void emit_pack_store_4ub( struct x86_program *p,
 260                                  struct x86_reg dest,
 261                                  struct x86_reg temp )
 262 {
 263    /* Scale by 255.0
 264     */
 265    sse_mulps(&p->func, temp, p->chan0);
 266
 267    if (p->have_sse2) {
 268       sse2_cvtps2dq(&p->func, temp, temp);
 269       sse2_packssdw(&p->func, temp, temp);
 270       sse2_packuswb(&p->func, temp, temp);
 271       sse_movss(&p->func, dest, temp);
 272    }
 273    else {
 274       struct x86_reg mmx0 = x86_make_reg(file_MMX, 0);
 275       struct x86_reg mmx1 = x86_make_reg(file_MMX, 1);
 276       sse_cvtps2pi(&p->func, mmx0, temp);
 277       sse_movhlps(&p->func, temp, temp);
 278       sse_cvtps2pi(&p->func, mmx1, temp);
 279       mmx_packssdw(&p->func, mmx0, mmx1);
 280       mmx_packuswb(&p->func, mmx0, mmx0);
 281       mmx_movd(&p->func, dest, mmx0);
 282    }
 283 }
 284
 285 static int get_offset( const void *a, const void *b )
 286 {
 287    return (const char *)b - (const char *)a;
 288 }
 289
 290 /* Not much happens here.  Eventually use this function to try and
 291  * avoid saving/reloading the source pointers each vertex (if some of
 292  * them can fit in registers).
 293  */
 294 static void get_src_ptr( struct x86_program *p,
 295                          struct x86_reg srcREG,
 296                          struct x86_reg vfREG,
 297                          struct draw_vf_attr *a )
 298 {
 299    struct draw_vertex_fetch *vf = p->vf;
 300    struct x86_reg ptr_to_src = x86_make_disp(vfREG, get_offset(vf, &a->inputptr));
 301
 302    /* Load current a[j].inputptr
 303     */
 304    x86_mov(&p->func, srcREG, ptr_to_src);
 305 }
 306
 307 static void update_src_ptr( struct x86_program *p,
 308                          struct x86_reg srcREG,
 309                          struct x86_reg vfREG,
 310                          struct draw_vf_attr *a )
 311 {
 312    if (a->inputstride) {
 313       struct draw_vertex_fetch *vf = p->vf;
 314       struct x86_reg ptr_to_src = x86_make_disp(vfREG, get_offset(vf, &a->inputptr));
 315
 316       /* add a[j].inputstride (hardcoded value - could just as easily
 317        * pull the stride value from memory each time).
 318        */
 319       x86_lea(&p->func, srcREG, x86_make_disp(srcREG, a->inputstride));
 320
 321       /* save new value of a[j].inputptr
 322        */
 323       x86_mov(&p->func, ptr_to_src, srcREG);
 324    }
 325 }
 326
 327
 328 /* Lots of hardcoding
 329  *
 330  * EAX -- pointer to current output vertex
 331  * ECX -- pointer to current attribute
 332  *
 333  */
 334 static boolean build_vertex_emit( struct x86_program *p )
 335 {
 336    struct draw_vertex_fetch *vf = p->vf;
 337    unsigned j = 0;
 338
 339    struct x86_reg vertexEAX = x86_make_reg(file_REG32, reg_AX);
 340    struct x86_reg srcECX = x86_make_reg(file_REG32, reg_CX);
 341    struct x86_reg countEBP = x86_make_reg(file_REG32, reg_BP);
 342    struct x86_reg vfESI = x86_make_reg(file_REG32, reg_SI);
 343    struct x86_reg temp = x86_make_reg(file_XMM, 0);
 344    uint8_t *fixup, *label;
 345
 346    /* Push a few regs?
 347     */
 348    x86_push(&p->func, countEBP);
 349    x86_push(&p->func, vfESI);
 350
 351
 352    /* Get vertex count, compare to zero
 353     */
 354    x86_xor(&p->func, srcECX, srcECX);
 355    x86_mov(&p->func, countEBP, x86_fn_arg(&p->func, 2));
 356    x86_cmp(&p->func, countEBP, srcECX);
 357    fixup = x86_jcc_forward(&p->func, cc_E);
 358
 359    /* Initialize destination register.
 360     */
 361    x86_mov(&p->func, vertexEAX, x86_fn_arg(&p->func, 3));
 362
 363    /* Move argument 1 (vf) into a reg:
 364     */
 365    x86_mov(&p->func, vfESI, x86_fn_arg(&p->func, 1));
 366
 367
 368    /* always load, needed or not:
 369     */
 370    sse_movups(&p->func, p->identity, x86_make_disp(vfESI, get_offset(vf, &vf->identity[0])));
 371
 372    /* Note address for loop jump */
 373    label = x86_get_label(&p->func);
 374
 375    /* Emit code for each of the attributes.  Currently routes
 376     * everything through SSE registers, even when it might be more
 377     * efficient to stick with regular old x86.  No optimization or
 378     * other tricks - enough new ground to cover here just getting
 379     * things working.
 380     */
 381    while (j < vf->attr_count) {
 382       struct draw_vf_attr *a = &vf->attr[j];
 383       struct x86_reg dest = x86_make_disp(vertexEAX, a->vertoffset);
 384
 385       /* Now, load an XMM reg from src, perhaps transform, then save.
 386        * Could be shortcircuited in specific cases:
 387        */
 388       switch (a->format) {
 389       case DRAW_EMIT_1F:
 390       case DRAW_EMIT_1F_CONST:
 391          get_src_ptr(p, srcECX, vfESI, a);
 392          emit_load(p, temp, 1, x86_deref(srcECX), a->inputsize);
 393          emit_store(p, dest, 1, temp);
 394          update_src_ptr(p, srcECX, vfESI, a);
 395          break;
 396       case DRAW_EMIT_2F:
 397       case DRAW_EMIT_2F_CONST:
 398          get_src_ptr(p, srcECX, vfESI, a);
 399          emit_load(p, temp, 2, x86_deref(srcECX), a->inputsize);
 400          emit_store(p, dest, 2, temp);
 401          update_src_ptr(p, srcECX, vfESI, a);
 402          break;
 403       case DRAW_EMIT_3F:
 404       case DRAW_EMIT_3F_CONST:
 405          /* Potentially the worst case - hardcode 2+1 copying:
 406           */
 407          if (0) {
 408             get_src_ptr(p, srcECX, vfESI, a);
 409             emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
 410             emit_store(p, dest, 3, temp);
 411             update_src_ptr(p, srcECX, vfESI, a);
 412          }
 413          else {
 414             get_src_ptr(p, srcECX, vfESI, a);
 415             emit_load(p, temp, 2, x86_deref(srcECX), a->inputsize);
 416             emit_store(p, dest, 2, temp);
 417             if (a->inputsize > 2) {
 418                emit_load(p, temp, 1, x86_make_disp(srcECX, 8), 1);
 419                emit_store(p, x86_make_disp(dest,8), 1, temp);
 420             }
 421             else {
 422                sse_movss(&p->func, x86_make_disp(dest,8), get_identity(p));
 423             }
 424             update_src_ptr(p, srcECX, vfESI, a);
 425          }
 426          break;
 427       case DRAW_EMIT_4F:
 428       case DRAW_EMIT_4F_CONST:
 429          get_src_ptr(p, srcECX, vfESI, a);
 430          emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 431          emit_store(p, dest, 4, temp);
 432          update_src_ptr(p, srcECX, vfESI, a);
 433          break;
 434       case DRAW_EMIT_3F_XYW:
 435          get_src_ptr(p, srcECX, vfESI, a);
 436          emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 437          sse_shufps(&p->func, temp, temp, SHUF(X,Y,W,Z));
 438          emit_store(p, dest, 3, temp);
 439          update_src_ptr(p, srcECX, vfESI, a);
 440          break;
 441
 442       case DRAW_EMIT_1UB_1F:
 443          /* Test for PAD3 + 1UB:
 444           */
 445          if (j > 0 &&
 446              a[-1].vertoffset + a[-1].vertattrsize <= a->vertoffset - 3)
 447          {
 448             get_src_ptr(p, srcECX, vfESI, a);
 449             emit_load(p, temp, 1, x86_deref(srcECX), a->inputsize);
 450             sse_shufps(&p->func, temp, temp, SHUF(X,X,X,X));
 451             emit_pack_store_4ub(p, x86_make_disp(dest, -3), temp); /* overkill! */
 452             update_src_ptr(p, srcECX, vfESI, a);
 453          }
 454          else {
 455             debug_printf("Can't emit 1ub %x %x %d\n",
 456                     a->vertoffset, a[-1].vertoffset, a[-1].vertattrsize );
 457             return FALSE;
 458          }
 459          break;
 460       case DRAW_EMIT_3UB_3F_RGB:
 461       case DRAW_EMIT_3UB_3F_BGR:
 462          /* Test for 3UB + PAD1:
 463           */
 464          if (j == vf->attr_count - 1 ||
 465              a[1].vertoffset >= a->vertoffset + 4) {
 466             get_src_ptr(p, srcECX, vfESI, a);
 467             emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
 468             if (a->format == DRAW_EMIT_3UB_3F_BGR)
 469                sse_shufps(&p->func, temp, temp, SHUF(Z,Y,X,W));
 470             emit_pack_store_4ub(p, dest, temp);
 471             update_src_ptr(p, srcECX, vfESI, a);
 472          }
 473          /* Test for 3UB + 1UB:
 474           */
 475          else if (j < vf->attr_count - 1 &&
 476                   a[1].format == DRAW_EMIT_1UB_1F &&
 477                   a[1].vertoffset == a->vertoffset + 3) {
 478             get_src_ptr(p, srcECX, vfESI, a);
 479             emit_load(p, temp, 3, x86_deref(srcECX), a->inputsize);
 480             update_src_ptr(p, srcECX, vfESI, a);
 481
 482             /* Make room for incoming value:
 483              */
 484             sse_shufps(&p->func, temp, temp, SHUF(W,X,Y,Z));
 485
 486             get_src_ptr(p, srcECX, vfESI, &a[1]);
 487             emit_load(p, temp, 1, x86_deref(srcECX), a[1].inputsize);
 488             update_src_ptr(p, srcECX, vfESI, &a[1]);
 489
 490             /* Rearrange and possibly do BGR conversion:
 491              */
 492             if (a->format == DRAW_EMIT_3UB_3F_BGR)
 493                sse_shufps(&p->func, temp, temp, SHUF(W,Z,Y,X));
 494             else
 495                sse_shufps(&p->func, temp, temp, SHUF(Y,Z,W,X));
 496
 497             emit_pack_store_4ub(p, dest, temp);
 498             j++;                /* NOTE: two attrs consumed */
 499          }
 500          else {
 501             debug_printf("Can't emit 3ub\n");
 502          }
 503          return FALSE;  /* add this later */
 504          break;
 505
 506       case DRAW_EMIT_4UB_4F_RGBA:
 507          get_src_ptr(p, srcECX, vfESI, a);
 508          emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 509          emit_pack_store_4ub(p, dest, temp);
 510          update_src_ptr(p, srcECX, vfESI, a);
 511          break;
 512       case DRAW_EMIT_4UB_4F_BGRA:
 513          get_src_ptr(p, srcECX, vfESI, a);
 514          emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 515          sse_shufps(&p->func, temp, temp, SHUF(Z,Y,X,W));
 516          emit_pack_store_4ub(p, dest, temp);
 517          update_src_ptr(p, srcECX, vfESI, a);
 518          break;
 519       case DRAW_EMIT_4UB_4F_ARGB:
 520          get_src_ptr(p, srcECX, vfESI, a);
 521          emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 522          sse_shufps(&p->func, temp, temp, SHUF(W,X,Y,Z));
 523          emit_pack_store_4ub(p, dest, temp);
 524          update_src_ptr(p, srcECX, vfESI, a);
 525          break;
 526       case DRAW_EMIT_4UB_4F_ABGR:
 527          get_src_ptr(p, srcECX, vfESI, a);
 528          emit_load(p, temp, 4, x86_deref(srcECX), a->inputsize);
 529          sse_shufps(&p->func, temp, temp, SHUF(W,Z,Y,X));
 530          emit_pack_store_4ub(p, dest, temp);
 531          update_src_ptr(p, srcECX, vfESI, a);
 532          break;
 533       default:
 534          debug_printf("unknown a[%d].format %d\n", j, a->format);
 535          return FALSE;  /* catch any new opcodes */
 536       }
 537
 538       /* Increment j by at least 1 - may have been incremented above also:
 539        */
 540       j++;
 541    }
 542
 543    /* Next vertex:
 544     */
 545    x86_lea(&p->func, vertexEAX, x86_make_disp(vertexEAX, vf->vertex_stride));
 546
 547    /* decr count, loop if not zero
 548     */
 549    x86_dec(&p->func, countEBP);
 550    x86_test(&p->func, countEBP, countEBP);
 551    x86_jcc(&p->func, cc_NZ, label);
 552
 553    /* Exit mmx state?
 554     */
 555    if (p->func.need_emms)
 556       mmx_emms(&p->func);
 557
 558    /* Land forward jump here:
 559     */
 560    x86_fixup_fwd_jump(&p->func, fixup);
 561
 562    /* Pop regs and return
 563     */
 564    x86_pop(&p->func, x86_get_base_reg(vfESI));
 565    x86_pop(&p->func, countEBP);
 566    x86_ret(&p->func);
 567
 568    vf->emit = (draw_vf_emit_func)x86_get_func(&p->func);
 569    return TRUE;
 570 }
 571
 572
 573
 574 void draw_vf_generate_sse_emit( struct draw_vertex_fetch *vf )
 575 {
 576    struct x86_program p;
 577
 578    if (!rtasm_cpu_has_sse()) {
 579       vf->codegen_emit = NULL;
 580       return;
 581    }
 582
 583    memset(&p, 0, sizeof(p));
 584
 585    p.vf = vf;
 586    p.inputs_safe = 0;           /* for now */
 587    p.outputs_safe = 1;          /* for now */
 588    p.have_sse2 = rtasm_cpu_has_sse2();
 589    p.identity = x86_make_reg(file_XMM, 6);
 590    p.chan0 = x86_make_reg(file_XMM, 7);
 591
 592    x86_init_func(&p.func);
 593
 594    if (build_vertex_emit(&p)) {
 595       draw_vf_register_fastpath( vf, TRUE );
 596    }
 597    else {
 598       /* Note the failure so that we don't keep trying to codegen an
 599        * impossible state:
 600        */
 601       draw_vf_register_fastpath( vf, FALSE );
 602       x86_release_func(&p.func);
 603    }
 604 }
 605
 606 #else
 607
 608 void draw_vf_generate_sse_emit( struct draw_vertex_fetch *vf )
 609 {
 610    /* Dummy version for when USE_SSE_ASM not defined */
 611 }
 612
 613 #endif