src/gallium/auxiliary/rtasm/rtasm_x86sse.c

   1 /**************************************************************************
   2  *
   3  * Copyright (C) 1999-2005  Brian Paul   All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included
  13  * in all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  16  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * BRIAN PAUL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  19  * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  20  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  **************************************************************************/
  23
  24 #if defined(__i386__) || defined(__386__) || defined(i386)
  25
  26 #include "pipe/p_compiler.h"
  27 #include "pipe/p_debug.h"
  28 #include "pipe/p_pointer.h"
  29
  30 #include "rtasm_execmem.h"
  31 #include "rtasm_x86sse.h"
  32
  33 #define DISASSEM 0
  34 #define X86_TWOB 0x0f
  35
  36 static unsigned char *cptr( void (*label)() )
  37 {
  38    return (unsigned char *) label;
  39 }
  40
  41
  42 static void do_realloc( struct x86_function *p )
  43 {
  44    if (p->size == 0) {
  45       p->size = 1024;
  46       p->store = rtasm_exec_malloc(p->size);
  47       p->csr = p->store;
  48    }
  49    else {
  50       uintptr_t used = pointer_to_uintptr( p->csr ) - pointer_to_uintptr( p->store );
  51       unsigned char *tmp = p->store;
  52       p->size *= 2;
  53       p->store = rtasm_exec_malloc(p->size);
  54       memcpy(p->store, tmp, used);
  55       p->csr = p->store + used;
  56       rtasm_exec_free(tmp);
  57    }
  58 }
  59
  60 /* Emit bytes to the instruction stream:
  61  */
  62 static unsigned char *reserve( struct x86_function *p, int bytes )
  63 {
  64    if (p->csr + bytes - p->store > (int) p->size)
  65       do_realloc(p);
  66
  67    {
  68       unsigned char *csr = p->csr;
  69       p->csr += bytes;
  70       return csr;
  71    }
  72 }
  73
  74
  75
  76 static void emit_1b( struct x86_function *p, char b0 )
  77 {
  78    char *csr = (char *)reserve(p, 1);
  79    *csr = b0;
  80 }
  81
  82 static void emit_1i( struct x86_function *p, int i0 )
  83 {
  84    int *icsr = (int *)reserve(p, sizeof(i0));
  85    *icsr = i0;
  86 }
  87
  88 static void emit_1ub( struct x86_function *p, unsigned char b0 )
  89 {
  90    unsigned char *csr = reserve(p, 1);
  91    *csr++ = b0;
  92 }
  93
  94 static void emit_2ub( struct x86_function *p, unsigned char b0, unsigned char b1 )
  95 {
  96    unsigned char *csr = reserve(p, 2);
  97    *csr++ = b0;
  98    *csr++ = b1;
  99 }
 100
 101 static void emit_3ub( struct x86_function *p, unsigned char b0, unsigned char b1, unsigned char b2 )
 102 {
 103    unsigned char *csr = reserve(p, 3);
 104    *csr++ = b0;
 105    *csr++ = b1;
 106    *csr++ = b2;
 107 }
 108
 109
 110 /* Build a modRM byte + possible displacement.  No treatment of SIB
 111  * indexing.  BZZT - no way to encode an absolute address.
 112  */
 113 static void emit_modrm( struct x86_function *p,
 114                         struct x86_reg reg,
 115                         struct x86_reg regmem )
 116 {
 117    unsigned char val = 0;
 118
 119    assert(reg.mod == mod_REG);
 120
 121    val |= regmem.mod << 6;      /* mod field */
 122    val |= reg.idx << 3;         /* reg field */
 123    val |= regmem.idx;           /* r/m field */
 124
 125    emit_1ub(p, val);
 126
 127    /* Oh-oh we've stumbled into the SIB thing.
 128     */
 129    if (regmem.file == file_REG32 &&
 130        regmem.idx == reg_SP) {
 131       emit_1ub(p, 0x24);                /* simplistic! */
 132    }
 133
 134    switch (regmem.mod) {
 135    case mod_REG:
 136    case mod_INDIRECT:
 137       break;
 138    case mod_DISP8:
 139       emit_1b(p, (char) regmem.disp);
 140       break;
 141    case mod_DISP32:
 142       emit_1i(p, regmem.disp);
 143       break;
 144    default:
 145       assert(0);
 146       break;
 147    }
 148 }
 149
 150
 151 static void emit_modrm_noreg( struct x86_function *p,
 152                               unsigned op,
 153                               struct x86_reg regmem )
 154 {
 155    struct x86_reg dummy = x86_make_reg(file_REG32, op);
 156    emit_modrm(p, dummy, regmem);
 157 }
 158
 159 /* Many x86 instructions have two opcodes to cope with the situations
 160  * where the destination is a register or memory reference
 161  * respectively.  This function selects the correct opcode based on
 162  * the arguments presented.
 163  */
 164 static void emit_op_modrm( struct x86_function *p,
 165                            unsigned char op_dst_is_reg,
 166                            unsigned char op_dst_is_mem,
 167                            struct x86_reg dst,
 168                            struct x86_reg src )
 169 {
 170    switch (dst.mod) {
 171    case mod_REG:
 172       emit_1ub(p, op_dst_is_reg);
 173       emit_modrm(p, dst, src);
 174       break;
 175    case mod_INDIRECT:
 176    case mod_DISP32:
 177    case mod_DISP8:
 178       assert(src.mod == mod_REG);
 179       emit_1ub(p, op_dst_is_mem);
 180       emit_modrm(p, src, dst);
 181       break;
 182    default:
 183       assert(0);
 184       break;
 185    }
 186 }
 187
 188
 189
 190
 191
 192
 193
 194 /* Create and manipulate registers and regmem values:
 195  */
 196 struct x86_reg x86_make_reg( enum x86_reg_file file,
 197                              enum x86_reg_name idx )
 198 {
 199    struct x86_reg reg;
 200
 201    reg.file = file;
 202    reg.idx = idx;
 203    reg.mod = mod_REG;
 204    reg.disp = 0;
 205
 206    return reg;
 207 }
 208
 209 struct x86_reg x86_make_disp( struct x86_reg reg,
 210                               int disp )
 211 {
 212    assert(reg.file == file_REG32);
 213
 214    if (reg.mod == mod_REG)
 215       reg.disp = disp;
 216    else
 217       reg.disp += disp;
 218
 219    if (reg.disp == 0)
 220       reg.mod = mod_INDIRECT;
 221    else if (reg.disp <= 127 && reg.disp >= -128)
 222       reg.mod = mod_DISP8;
 223    else
 224       reg.mod = mod_DISP32;
 225
 226    return reg;
 227 }
 228
 229 struct x86_reg x86_deref( struct x86_reg reg )
 230 {
 231    return x86_make_disp(reg, 0);
 232 }
 233
 234 struct x86_reg x86_get_base_reg( struct x86_reg reg )
 235 {
 236    return x86_make_reg( reg.file, reg.idx );
 237 }
 238
 239 unsigned char *x86_get_label( struct x86_function *p )
 240 {
 241    return p->csr;
 242 }
 243
 244
 245
 246 /***********************************************************************
 247  * x86 instructions
 248  */
 249
 250
 251 void x86_jcc( struct x86_function *p,
 252               enum x86_cc cc,
 253               unsigned char *label )
 254 {
 255    intptr_t offset = pointer_to_intptr( label ) - (pointer_to_intptr( x86_get_label(p) ) + 2);
 256
 257    if (offset <= 127 && offset >= -128) {
 258       emit_1ub(p, 0x70 + cc);
 259       emit_1b(p, (char) offset);
 260    }
 261    else {
 262       offset = pointer_to_intptr( label ) - (pointer_to_intptr( x86_get_label(p) ) + 6);
 263       emit_2ub(p, 0x0f, 0x80 + cc);
 264       emit_1i(p, offset);
 265    }
 266 }
 267
 268 /* Always use a 32bit offset for forward jumps:
 269  */
 270 unsigned char *x86_jcc_forward( struct x86_function *p,
 271                           enum x86_cc cc )
 272 {
 273    emit_2ub(p, 0x0f, 0x80 + cc);
 274    emit_1i(p, 0);
 275    return x86_get_label(p);
 276 }
 277
 278 unsigned char *x86_jmp_forward( struct x86_function *p)
 279 {
 280    emit_1ub(p, 0xe9);
 281    emit_1i(p, 0);
 282    return x86_get_label(p);
 283 }
 284
 285 unsigned char *x86_call_forward( struct x86_function *p)
 286 {
 287    emit_1ub(p, 0xe8);
 288    emit_1i(p, 0);
 289    return x86_get_label(p);
 290 }
 291
 292 /* Fixup offset from forward jump:
 293  */
 294 void x86_fixup_fwd_jump( struct x86_function *p,
 295                          unsigned char *fixup )
 296 {
 297    *(int *)(fixup - 4) = pointer_to_intptr( x86_get_label(p) ) - pointer_to_intptr( fixup );
 298 }
 299
 300 void x86_jmp( struct x86_function *p, unsigned char *label)
 301 {
 302    emit_1ub(p, 0xe9);
 303    emit_1i(p, pointer_to_intptr( label ) - pointer_to_intptr( x86_get_label(p) ) - 4);
 304 }
 305
 306 #if 0
 307 /* This doesn't work once we start reallocating & copying the
 308  * generated code on buffer fills, because the call is relative to the
 309  * current pc.
 310  */
 311 void x86_call( struct x86_function *p, void (*label)())
 312 {
 313    emit_1ub(p, 0xe8);
 314    emit_1i(p, cptr(label) - x86_get_label(p) - 4);
 315 }
 316 #else
 317 void x86_call( struct x86_function *p, struct x86_reg reg)
 318 {
 319    emit_1ub(p, 0xff);
 320    emit_modrm(p, reg, reg);
 321 }
 322 #endif
 323
 324
 325 /* michal:
 326  * Temporary. As I need immediate operands, and dont want to mess with the codegen,
 327  * I load the immediate into general purpose register and use it.
 328  */
 329 void x86_mov_reg_imm( struct x86_function *p, struct x86_reg dst, int imm )
 330 {
 331    assert(dst.mod == mod_REG);
 332    emit_1ub(p, 0xb8 + dst.idx);
 333    emit_1i(p, imm);
 334 }
 335
 336 void x86_push( struct x86_function *p,
 337                struct x86_reg reg )
 338 {
 339    assert(reg.mod == mod_REG);
 340    emit_1ub(p, 0x50 + reg.idx);
 341    p->stack_offset += 4;
 342 }
 343
 344 void x86_pop( struct x86_function *p,
 345               struct x86_reg reg )
 346 {
 347    assert(reg.mod == mod_REG);
 348    emit_1ub(p, 0x58 + reg.idx);
 349    p->stack_offset -= 4;
 350 }
 351
 352 void x86_inc( struct x86_function *p,
 353               struct x86_reg reg )
 354 {
 355    assert(reg.mod == mod_REG);
 356    emit_1ub(p, 0x40 + reg.idx);
 357 }
 358
 359 void x86_dec( struct x86_function *p,
 360               struct x86_reg reg )
 361 {
 362    assert(reg.mod == mod_REG);
 363    emit_1ub(p, 0x48 + reg.idx);
 364 }
 365
 366 void x86_ret( struct x86_function *p )
 367 {
 368    emit_1ub(p, 0xc3);
 369 }
 370
 371 void x86_sahf( struct x86_function *p )
 372 {
 373    emit_1ub(p, 0x9e);
 374 }
 375
 376 void x86_mov( struct x86_function *p,
 377               struct x86_reg dst,
 378               struct x86_reg src )
 379 {
 380    emit_op_modrm( p, 0x8b, 0x89, dst, src );
 381 }
 382
 383 void x86_xor( struct x86_function *p,
 384               struct x86_reg dst,
 385               struct x86_reg src )
 386 {
 387    emit_op_modrm( p, 0x33, 0x31, dst, src );
 388 }
 389
 390 void x86_cmp( struct x86_function *p,
 391               struct x86_reg dst,
 392               struct x86_reg src )
 393 {
 394    emit_op_modrm( p, 0x3b, 0x39, dst, src );
 395 }
 396
 397 void x86_lea( struct x86_function *p,
 398               struct x86_reg dst,
 399               struct x86_reg src )
 400 {
 401    emit_1ub(p, 0x8d);
 402    emit_modrm( p, dst, src );
 403 }
 404
 405 void x86_test( struct x86_function *p,
 406                struct x86_reg dst,
 407                struct x86_reg src )
 408 {
 409    emit_1ub(p, 0x85);
 410    emit_modrm( p, dst, src );
 411 }
 412
 413 void x86_add( struct x86_function *p,
 414                struct x86_reg dst,
 415                struct x86_reg src )
 416 {
 417    emit_op_modrm(p, 0x03, 0x01, dst, src );
 418 }
 419
 420 void x86_mul( struct x86_function *p,
 421                struct x86_reg src )
 422 {
 423    assert (src.file == file_REG32 && src.mod == mod_REG);
 424    emit_op_modrm(p, 0xf7, 0, x86_make_reg (file_REG32, reg_SP), src );
 425 }
 426
 427 void x86_sub( struct x86_function *p,
 428                struct x86_reg dst,
 429                struct x86_reg src )
 430 {
 431    emit_op_modrm(p, 0x2b, 0x29, dst, src );
 432 }
 433
 434 void x86_or( struct x86_function *p,
 435              struct x86_reg dst,
 436              struct x86_reg src )
 437 {
 438    emit_op_modrm( p, 0x0b, 0x09, dst, src );
 439 }
 440
 441 void x86_and( struct x86_function *p,
 442               struct x86_reg dst,
 443               struct x86_reg src )
 444 {
 445    emit_op_modrm( p, 0x23, 0x21, dst, src );
 446 }
 447
 448
 449
 450 /***********************************************************************
 451  * SSE instructions
 452  */
 453
 454
 455 void sse_movss( struct x86_function *p,
 456                 struct x86_reg dst,
 457                 struct x86_reg src )
 458 {
 459    emit_2ub(p, 0xF3, X86_TWOB);
 460    emit_op_modrm( p, 0x10, 0x11, dst, src );
 461 }
 462
 463 void sse_movaps( struct x86_function *p,
 464                  struct x86_reg dst,
 465                  struct x86_reg src )
 466 {
 467    emit_1ub(p, X86_TWOB);
 468    emit_op_modrm( p, 0x28, 0x29, dst, src );
 469 }
 470
 471 void sse_movups( struct x86_function *p,
 472                  struct x86_reg dst,
 473                  struct x86_reg src )
 474 {
 475    emit_1ub(p, X86_TWOB);
 476    emit_op_modrm( p, 0x10, 0x11, dst, src );
 477 }
 478
 479 void sse_movhps( struct x86_function *p,
 480                  struct x86_reg dst,
 481                  struct x86_reg src )
 482 {
 483    assert(dst.mod != mod_REG || src.mod != mod_REG);
 484    emit_1ub(p, X86_TWOB);
 485    emit_op_modrm( p, 0x16, 0x17, dst, src ); /* cf movlhps */
 486 }
 487
 488 void sse_movlps( struct x86_function *p,
 489                  struct x86_reg dst,
 490                  struct x86_reg src )
 491 {
 492    assert(dst.mod != mod_REG || src.mod != mod_REG);
 493    emit_1ub(p, X86_TWOB);
 494    emit_op_modrm( p, 0x12, 0x13, dst, src ); /* cf movhlps */
 495 }
 496
 497 void sse_maxps( struct x86_function *p,
 498                 struct x86_reg dst,
 499                 struct x86_reg src )
 500 {
 501    emit_2ub(p, X86_TWOB, 0x5F);
 502    emit_modrm( p, dst, src );
 503 }
 504
 505 void sse_maxss( struct x86_function *p,
 506                 struct x86_reg dst,
 507                 struct x86_reg src )
 508 {
 509    emit_3ub(p, 0xF3, X86_TWOB, 0x5F);
 510    emit_modrm( p, dst, src );
 511 }
 512
 513 void sse_divss( struct x86_function *p,
 514                 struct x86_reg dst,
 515                 struct x86_reg src )
 516 {
 517    emit_3ub(p, 0xF3, X86_TWOB, 0x5E);
 518    emit_modrm( p, dst, src );
 519 }
 520
 521 void sse_minps( struct x86_function *p,
 522                 struct x86_reg dst,
 523                 struct x86_reg src )
 524 {
 525    emit_2ub(p, X86_TWOB, 0x5D);
 526    emit_modrm( p, dst, src );
 527 }
 528
 529 void sse_subps( struct x86_function *p,
 530                 struct x86_reg dst,
 531                 struct x86_reg src )
 532 {
 533    emit_2ub(p, X86_TWOB, 0x5C);
 534    emit_modrm( p, dst, src );
 535 }
 536
 537 void sse_mulps( struct x86_function *p,
 538                 struct x86_reg dst,
 539                 struct x86_reg src )
 540 {
 541    emit_2ub(p, X86_TWOB, 0x59);
 542    emit_modrm( p, dst, src );
 543 }
 544
 545 void sse_mulss( struct x86_function *p,
 546                 struct x86_reg dst,
 547                 struct x86_reg src )
 548 {
 549    emit_3ub(p, 0xF3, X86_TWOB, 0x59);
 550    emit_modrm( p, dst, src );
 551 }
 552
 553 void sse_addps( struct x86_function *p,
 554                 struct x86_reg dst,
 555                 struct x86_reg src )
 556 {
 557    emit_2ub(p, X86_TWOB, 0x58);
 558    emit_modrm( p, dst, src );
 559 }
 560
 561 void sse_addss( struct x86_function *p,
 562                 struct x86_reg dst,
 563                 struct x86_reg src )
 564 {
 565    emit_3ub(p, 0xF3, X86_TWOB, 0x58);
 566    emit_modrm( p, dst, src );
 567 }
 568
 569 void sse_andnps( struct x86_function *p,
 570                  struct x86_reg dst,
 571                  struct x86_reg src )
 572 {
 573    emit_2ub(p, X86_TWOB, 0x55);
 574    emit_modrm( p, dst, src );
 575 }
 576
 577 void sse_andps( struct x86_function *p,
 578                 struct x86_reg dst,
 579                 struct x86_reg src )
 580 {
 581    emit_2ub(p, X86_TWOB, 0x54);
 582    emit_modrm( p, dst, src );
 583 }
 584
 585 void sse_rsqrtps( struct x86_function *p,
 586                   struct x86_reg dst,
 587                   struct x86_reg src )
 588 {
 589    emit_2ub(p, X86_TWOB, 0x52);
 590    emit_modrm( p, dst, src );
 591 }
 592
 593 void sse_rsqrtss( struct x86_function *p,
 594                   struct x86_reg dst,
 595                   struct x86_reg src )
 596 {
 597    emit_3ub(p, 0xF3, X86_TWOB, 0x52);
 598    emit_modrm( p, dst, src );
 599
 600 }
 601
 602 void sse_movhlps( struct x86_function *p,
 603                   struct x86_reg dst,
 604                   struct x86_reg src )
 605 {
 606    assert(dst.mod == mod_REG && src.mod == mod_REG);
 607    emit_2ub(p, X86_TWOB, 0x12);
 608    emit_modrm( p, dst, src );
 609 }
 610
 611 void sse_movlhps( struct x86_function *p,
 612                   struct x86_reg dst,
 613                   struct x86_reg src )
 614 {
 615    assert(dst.mod == mod_REG && src.mod == mod_REG);
 616    emit_2ub(p, X86_TWOB, 0x16);
 617    emit_modrm( p, dst, src );
 618 }
 619
 620 void sse_orps( struct x86_function *p,
 621                struct x86_reg dst,
 622                struct x86_reg src )
 623 {
 624    emit_2ub(p, X86_TWOB, 0x56);
 625    emit_modrm( p, dst, src );
 626 }
 627
 628 void sse_xorps( struct x86_function *p,
 629                 struct x86_reg dst,
 630                 struct x86_reg src )
 631 {
 632    emit_2ub(p, X86_TWOB, 0x57);
 633    emit_modrm( p, dst, src );
 634 }
 635
 636 void sse_cvtps2pi( struct x86_function *p,
 637                    struct x86_reg dst,
 638                    struct x86_reg src )
 639 {
 640    assert(dst.file == file_MMX &&
 641           (src.file == file_XMM || src.mod != mod_REG));
 642
 643    p->need_emms = 1;
 644
 645    emit_2ub(p, X86_TWOB, 0x2d);
 646    emit_modrm( p, dst, src );
 647 }
 648
 649
 650 /* Shufps can also be used to implement a reduced swizzle when dest ==
 651  * arg0.
 652  */
 653 void sse_shufps( struct x86_function *p,
 654                  struct x86_reg dest,
 655                  struct x86_reg arg0,
 656                  unsigned char shuf)
 657 {
 658    emit_2ub(p, X86_TWOB, 0xC6);
 659    emit_modrm(p, dest, arg0);
 660    emit_1ub(p, shuf);
 661 }
 662
 663 void sse_cmpps( struct x86_function *p,
 664                 struct x86_reg dest,
 665                 struct x86_reg arg0,
 666                 unsigned char cc)
 667 {
 668    emit_2ub(p, X86_TWOB, 0xC2);
 669    emit_modrm(p, dest, arg0);
 670    emit_1ub(p, cc);
 671 }
 672
 673 void sse_pmovmskb( struct x86_function *p,
 674                    struct x86_reg dest,
 675                    struct x86_reg src)
 676 {
 677     emit_3ub(p, 0x66, X86_TWOB, 0xD7);
 678     emit_modrm(p, dest, src);
 679 }
 680
 681 /***********************************************************************
 682  * SSE2 instructions
 683  */
 684
 685 /**
 686  * Perform a reduced swizzle:
 687  */
 688 void sse2_pshufd( struct x86_function *p,
 689                   struct x86_reg dest,
 690                   struct x86_reg arg0,
 691                   unsigned char shuf)
 692 {
 693    emit_3ub(p, 0x66, X86_TWOB, 0x70);
 694    emit_modrm(p, dest, arg0);
 695    emit_1ub(p, shuf);
 696 }
 697
 698 void sse2_cvttps2dq( struct x86_function *p,
 699                      struct x86_reg dst,
 700                      struct x86_reg src )
 701 {
 702    emit_3ub( p, 0xF3, X86_TWOB, 0x5B );
 703    emit_modrm( p, dst, src );
 704 }
 705
 706 void sse2_cvtps2dq( struct x86_function *p,
 707                     struct x86_reg dst,
 708                     struct x86_reg src )
 709 {
 710    emit_3ub(p, 0x66, X86_TWOB, 0x5B);
 711    emit_modrm( p, dst, src );
 712 }
 713
 714 void sse2_packssdw( struct x86_function *p,
 715                     struct x86_reg dst,
 716                     struct x86_reg src )
 717 {
 718    emit_3ub(p, 0x66, X86_TWOB, 0x6B);
 719    emit_modrm( p, dst, src );
 720 }
 721
 722 void sse2_packsswb( struct x86_function *p,
 723                     struct x86_reg dst,
 724                     struct x86_reg src )
 725 {
 726    emit_3ub(p, 0x66, X86_TWOB, 0x63);
 727    emit_modrm( p, dst, src );
 728 }
 729
 730 void sse2_packuswb( struct x86_function *p,
 731                     struct x86_reg dst,
 732                     struct x86_reg src )
 733 {
 734    emit_3ub(p, 0x66, X86_TWOB, 0x67);
 735    emit_modrm( p, dst, src );
 736 }
 737
 738 void sse2_rcpps( struct x86_function *p,
 739                  struct x86_reg dst,
 740                  struct x86_reg src )
 741 {
 742    emit_2ub(p, X86_TWOB, 0x53);
 743    emit_modrm( p, dst, src );
 744 }
 745
 746 void sse2_rcpss( struct x86_function *p,
 747                 struct x86_reg dst,
 748                 struct x86_reg src )
 749 {
 750    emit_3ub(p, 0xF3, X86_TWOB, 0x53);
 751    emit_modrm( p, dst, src );
 752 }
 753
 754 void sse2_movd( struct x86_function *p,
 755                 struct x86_reg dst,
 756                 struct x86_reg src )
 757 {
 758    emit_2ub(p, 0x66, X86_TWOB);
 759    emit_op_modrm( p, 0x6e, 0x7e, dst, src );
 760 }
 761
 762
 763
 764
 765 /***********************************************************************
 766  * x87 instructions
 767  */
 768 void x87_fist( struct x86_function *p, struct x86_reg dst )
 769 {
 770    emit_1ub(p, 0xdb);
 771    emit_modrm_noreg(p, 2, dst);
 772 }
 773
 774 void x87_fistp( struct x86_function *p, struct x86_reg dst )
 775 {
 776    emit_1ub(p, 0xdb);
 777    emit_modrm_noreg(p, 3, dst);
 778 }
 779
 780 void x87_fild( struct x86_function *p, struct x86_reg arg )
 781 {
 782    emit_1ub(p, 0xdf);
 783    emit_modrm_noreg(p, 0, arg);
 784 }
 785
 786 void x87_fldz( struct x86_function *p )
 787 {
 788    emit_2ub(p, 0xd9, 0xee);
 789 }
 790
 791
 792 void x87_fldcw( struct x86_function *p, struct x86_reg arg )
 793 {
 794    assert(arg.file == file_REG32);
 795    assert(arg.mod != mod_REG);
 796    emit_1ub(p, 0xd9);
 797    emit_modrm_noreg(p, 5, arg);
 798 }
 799
 800 void x87_fld1( struct x86_function *p )
 801 {
 802    emit_2ub(p, 0xd9, 0xe8);
 803 }
 804
 805 void x87_fldl2e( struct x86_function *p )
 806 {
 807    emit_2ub(p, 0xd9, 0xea);
 808 }
 809
 810 void x87_fldln2( struct x86_function *p )
 811 {
 812    emit_2ub(p, 0xd9, 0xed);
 813 }
 814
 815 void x87_fwait( struct x86_function *p )
 816 {
 817    emit_1ub(p, 0x9b);
 818 }
 819
 820 void x87_fnclex( struct x86_function *p )
 821 {
 822    emit_2ub(p, 0xdb, 0xe2);
 823 }
 824
 825 void x87_fclex( struct x86_function *p )
 826 {
 827    x87_fwait(p);
 828    x87_fnclex(p);
 829 }
 830
 831
 832 static void x87_arith_op( struct x86_function *p, struct x86_reg dst, struct x86_reg arg,
 833                           unsigned char dst0ub0,
 834                           unsigned char dst0ub1,
 835                           unsigned char arg0ub0,
 836                           unsigned char arg0ub1,
 837                           unsigned char argmem_noreg)
 838 {
 839    assert(dst.file == file_x87);
 840
 841    if (arg.file == file_x87) {
 842       if (dst.idx == 0)
 843          emit_2ub(p, dst0ub0, dst0ub1+arg.idx);
 844       else if (arg.idx == 0)
 845          emit_2ub(p, arg0ub0, arg0ub1+arg.idx);
 846       else
 847          assert(0);
 848    }
 849    else if (dst.idx == 0) {
 850       assert(arg.file == file_REG32);
 851       emit_1ub(p, 0xd8);
 852       emit_modrm_noreg(p, argmem_noreg, arg);
 853    }
 854    else
 855       assert(0);
 856 }
 857
 858 void x87_fmul( struct x86_function *p, struct x86_reg dst, struct x86_reg arg )
 859 {
 860    x87_arith_op(p, dst, arg,
 861                 0xd8, 0xc8,
 862                 0xdc, 0xc8,
 863                 4);
 864 }
 865
 866 void x87_fsub( struct x86_function *p, struct x86_reg dst, struct x86_reg arg )
 867 {
 868    x87_arith_op(p, dst, arg,
 869                 0xd8, 0xe0,
 870                 0xdc, 0xe8,
 871                 4);
 872 }
 873
 874 void x87_fsubr( struct x86_function *p, struct x86_reg dst, struct x86_reg arg )
 875 {
 876    x87_arith_op(p, dst, arg,
 877                 0xd8, 0xe8,
 878                 0xdc, 0xe0,
 879                 5);
 880 }
 881
 882 void x87_fadd( struct x86_function *p, struct x86_reg dst, struct x86_reg arg )
 883 {
 884    x87_arith_op(p, dst, arg,
 885                 0xd8, 0xc0,
 886                 0xdc, 0xc0,
 887                 0);
 888 }
 889
 890 void x87_fdiv( struct x86_function *p, struct x86_reg dst, struct x86_reg arg )
 891 {
 892    x87_arith_op(p, dst, arg,
 893                 0xd8, 0xf0,
 894                 0xdc, 0xf8,
 895                 6);
 896 }
 897
 898 void x87_fdivr( struct x86_function *p, struct x86_reg dst, struct x86_reg arg )
 899 {
 900    x87_arith_op(p, dst, arg,
 901                 0xd8, 0xf8,
 902                 0xdc, 0xf0,
 903                 7);
 904 }
 905
 906 void x87_fmulp( struct x86_function *p, struct x86_reg dst )
 907 {
 908    assert(dst.file == file_x87);
 909    assert(dst.idx >= 1);
 910    emit_2ub(p, 0xde, 0xc8+dst.idx);
 911 }
 912
 913 void x87_fsubp( struct x86_function *p, struct x86_reg dst )
 914 {
 915    assert(dst.file == file_x87);
 916    assert(dst.idx >= 1);
 917    emit_2ub(p, 0xde, 0xe8+dst.idx);
 918 }
 919
 920 void x87_fsubrp( struct x86_function *p, struct x86_reg dst )
 921 {
 922    assert(dst.file == file_x87);
 923    assert(dst.idx >= 1);
 924    emit_2ub(p, 0xde, 0xe0+dst.idx);
 925 }
 926
 927 void x87_faddp( struct x86_function *p, struct x86_reg dst )
 928 {
 929    assert(dst.file == file_x87);
 930    assert(dst.idx >= 1);
 931    emit_2ub(p, 0xde, 0xc0+dst.idx);
 932 }
 933
 934 void x87_fdivp( struct x86_function *p, struct x86_reg dst )
 935 {
 936    assert(dst.file == file_x87);
 937    assert(dst.idx >= 1);
 938    emit_2ub(p, 0xde, 0xf8+dst.idx);
 939 }
 940
 941 void x87_fdivrp( struct x86_function *p, struct x86_reg dst )
 942 {
 943    assert(dst.file == file_x87);
 944    assert(dst.idx >= 1);
 945    emit_2ub(p, 0xde, 0xf0+dst.idx);
 946 }
 947
 948 void x87_fucom( struct x86_function *p, struct x86_reg arg )
 949 {
 950    assert(arg.file == file_x87);
 951    emit_2ub(p, 0xdd, 0xe0+arg.idx);
 952 }
 953
 954 void x87_fucomp( struct x86_function *p, struct x86_reg arg )
 955 {
 956    assert(arg.file == file_x87);
 957    emit_2ub(p, 0xdd, 0xe8+arg.idx);
 958 }
 959
 960 void x87_fucompp( struct x86_function *p )
 961 {
 962    emit_2ub(p, 0xda, 0xe9);
 963 }
 964
 965 void x87_fxch( struct x86_function *p, struct x86_reg arg )
 966 {
 967    assert(arg.file == file_x87);
 968    emit_2ub(p, 0xd9, 0xc8+arg.idx);
 969 }
 970
 971 void x87_fabs( struct x86_function *p )
 972 {
 973    emit_2ub(p, 0xd9, 0xe1);
 974 }
 975
 976 void x87_fchs( struct x86_function *p )
 977 {
 978    emit_2ub(p, 0xd9, 0xe0);
 979 }
 980
 981 void x87_fcos( struct x86_function *p )
 982 {
 983    emit_2ub(p, 0xd9, 0xff);
 984 }
 985
 986
 987 void x87_fprndint( struct x86_function *p )
 988 {
 989    emit_2ub(p, 0xd9, 0xfc);
 990 }
 991
 992 void x87_fscale( struct x86_function *p )
 993 {
 994    emit_2ub(p, 0xd9, 0xfd);
 995 }
 996
 997 void x87_fsin( struct x86_function *p )
 998 {
 999    emit_2ub(p, 0xd9, 0xfe);
1000 }
1001
1002 void x87_fsincos( struct x86_function *p )
1003 {
1004    emit_2ub(p, 0xd9, 0xfb);
1005 }
1006
1007 void x87_fsqrt( struct x86_function *p )
1008 {
1009    emit_2ub(p, 0xd9, 0xfa);
1010 }
1011
1012 void x87_fxtract( struct x86_function *p )
1013 {
1014    emit_2ub(p, 0xd9, 0xf4);
1015 }
1016
1017 /* st0 = (2^st0)-1
1018  *
1019  * Restrictions: -1.0 <= st0 <= 1.0
1020  */
1021 void x87_f2xm1( struct x86_function *p )
1022 {
1023    emit_2ub(p, 0xd9, 0xf0);
1024 }
1025
1026 /* st1 = st1 * log2(st0);
1027  * pop_stack;
1028  */
1029 void x87_fyl2x( struct x86_function *p )
1030 {
1031    emit_2ub(p, 0xd9, 0xf1);
1032 }
1033
1034 /* st1 = st1 * log2(st0 + 1.0);
1035  * pop_stack;
1036  *
1037  * A fast operation, with restrictions: -.29 < st0 < .29
1038  */
1039 void x87_fyl2xp1( struct x86_function *p )
1040 {
1041    emit_2ub(p, 0xd9, 0xf9);
1042 }
1043
1044
1045 void x87_fld( struct x86_function *p, struct x86_reg arg )
1046 {
1047    if (arg.file == file_x87)
1048       emit_2ub(p, 0xd9, 0xc0 + arg.idx);
1049    else {
1050       emit_1ub(p, 0xd9);
1051       emit_modrm_noreg(p, 0, arg);
1052    }
1053 }
1054
1055 void x87_fst( struct x86_function *p, struct x86_reg dst )
1056 {
1057    if (dst.file == file_x87)
1058       emit_2ub(p, 0xdd, 0xd0 + dst.idx);
1059    else {
1060       emit_1ub(p, 0xd9);
1061       emit_modrm_noreg(p, 2, dst);
1062    }
1063 }
1064
1065 void x87_fstp( struct x86_function *p, struct x86_reg dst )
1066 {
1067    if (dst.file == file_x87)
1068       emit_2ub(p, 0xdd, 0xd8 + dst.idx);
1069    else {
1070       emit_1ub(p, 0xd9);
1071       emit_modrm_noreg(p, 3, dst);
1072    }
1073 }
1074
1075 void x87_fcom( struct x86_function *p, struct x86_reg dst )
1076 {
1077    if (dst.file == file_x87)
1078       emit_2ub(p, 0xd8, 0xd0 + dst.idx);
1079    else {
1080       emit_1ub(p, 0xd8);
1081       emit_modrm_noreg(p, 2, dst);
1082    }
1083 }
1084
1085 void x87_fcomp( struct x86_function *p, struct x86_reg dst )
1086 {
1087    if (dst.file == file_x87)
1088       emit_2ub(p, 0xd8, 0xd8 + dst.idx);
1089    else {
1090       emit_1ub(p, 0xd8);
1091       emit_modrm_noreg(p, 3, dst);
1092    }
1093 }
1094
1095
1096 void x87_fnstsw( struct x86_function *p, struct x86_reg dst )
1097 {
1098    assert(dst.file == file_REG32);
1099
1100    if (dst.idx == reg_AX &&
1101        dst.mod == mod_REG)
1102       emit_2ub(p, 0xdf, 0xe0);
1103    else {
1104       emit_1ub(p, 0xdd);
1105       emit_modrm_noreg(p, 7, dst);
1106    }
1107 }
1108
1109
1110
1111
1112 /***********************************************************************
1113  * MMX instructions
1114  */
1115
1116 void mmx_emms( struct x86_function *p )
1117 {
1118    assert(p->need_emms);
1119    emit_2ub(p, 0x0f, 0x77);
1120    p->need_emms = 0;
1121 }
1122
1123 void mmx_packssdw( struct x86_function *p,
1124                    struct x86_reg dst,
1125                    struct x86_reg src )
1126 {
1127    assert(dst.file == file_MMX &&
1128           (src.file == file_MMX || src.mod != mod_REG));
1129
1130    p->need_emms = 1;
1131
1132    emit_2ub(p, X86_TWOB, 0x6b);
1133    emit_modrm( p, dst, src );
1134 }
1135
1136 void mmx_packuswb( struct x86_function *p,
1137                    struct x86_reg dst,
1138                    struct x86_reg src )
1139 {
1140    assert(dst.file == file_MMX &&
1141           (src.file == file_MMX || src.mod != mod_REG));
1142
1143    p->need_emms = 1;
1144
1145    emit_2ub(p, X86_TWOB, 0x67);
1146    emit_modrm( p, dst, src );
1147 }
1148
1149 void mmx_movd( struct x86_function *p,
1150                struct x86_reg dst,
1151                struct x86_reg src )
1152 {
1153    p->need_emms = 1;
1154    emit_1ub(p, X86_TWOB);
1155    emit_op_modrm( p, 0x6e, 0x7e, dst, src );
1156 }
1157
1158 void mmx_movq( struct x86_function *p,
1159                struct x86_reg dst,
1160                struct x86_reg src )
1161 {
1162    p->need_emms = 1;
1163    emit_1ub(p, X86_TWOB);
1164    emit_op_modrm( p, 0x6f, 0x7f, dst, src );
1165 }
1166
1167
1168 /***********************************************************************
1169  * Helper functions
1170  */
1171
1172
1173 /* Retreive a reference to one of the function arguments, taking into
1174  * account any push/pop activity:
1175  */
1176 struct x86_reg x86_fn_arg( struct x86_function *p,
1177                            unsigned arg )
1178 {
1179    return x86_make_disp(x86_make_reg(file_REG32, reg_SP),
1180                         p->stack_offset + arg * 4);     /* ??? */
1181 }
1182
1183
1184 void x86_init_func( struct x86_function *p )
1185 {
1186    p->size = 0;
1187    p->store = NULL;
1188    p->csr = p->store;
1189 }
1190
1191 void x86_init_func_size( struct x86_function *p, unsigned code_size )
1192 {
1193    p->size = code_size;
1194    p->store = rtasm_exec_malloc(code_size);
1195    p->csr = p->store;
1196 }
1197
1198 void x86_release_func( struct x86_function *p )
1199 {
1200    rtasm_exec_free(p->store);
1201    p->store = NULL;
1202    p->csr = NULL;
1203    p->size = 0;
1204 }
1205
1206
1207 void (*x86_get_func( struct x86_function *p ))(void)
1208 {
1209    if (DISASSEM && p->store)
1210       debug_printf("disassemble %p %p\n", p->store, p->csr);
1211    return (void (*)(void)) p->store;
1212 }
1213
1214 #else
1215
1216 void x86sse_dummy( void )
1217 {
1218 }
1219
1220 #endif