src/arch/hsail/insts/mem.hh

   1 /*
   2  * Copyright (c) 2012-2015 Advanced Micro Devices, Inc.
   3  * All rights reserved.
   4  *
   5  * For use for simulation and test purposes only
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  * 1. Redistributions of source code must retain the above copyright notice,
  11  * this list of conditions and the following disclaimer.
  12  *
  13  * 2. Redistributions in binary form must reproduce the above copyright notice,
  14  * this list of conditions and the following disclaimer in the documentation
  15  * and/or other materials provided with the distribution.
  16  *
  17  * 3. Neither the name of the copyright holder nor the names of its contributors
  18  * may be used to endorse or promote products derived from this software
  19  * without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  25  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31  * POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * Author: Steve Reinhardt
  34  */
  35
  36 #ifndef __ARCH_HSAIL_INSTS_MEM_HH__
  37 #define __ARCH_HSAIL_INSTS_MEM_HH__
  38
  39 #include "arch/hsail/insts/decl.hh"
  40 #include "arch/hsail/insts/gpu_static_inst.hh"
  41 #include "arch/hsail/operand.hh"
  42
  43 namespace HsailISA
  44 {
  45     class MemInst
  46     {
  47       public:
  48         MemInst() : size(0), addr_operand(nullptr) { }
  49
  50         MemInst(Enums::MemType m_type)
  51         {
  52             if (m_type == Enums::M_U64 ||
  53                 m_type == Enums::M_S64 ||
  54                 m_type == Enums::M_F64) {
  55                 size = 8;
  56             } else if (m_type == Enums::M_U32 ||
  57                        m_type == Enums::M_S32 ||
  58                        m_type == Enums::M_F32) {
  59                 size = 4;
  60             } else if (m_type == Enums::M_U16 ||
  61                        m_type == Enums::M_S16 ||
  62                        m_type == Enums::M_F16) {
  63                 size = 2;
  64             } else {
  65                 size = 1;
  66             }
  67
  68             addr_operand = nullptr;
  69         }
  70
  71         void
  72         init_addr(AddrOperandBase *_addr_operand)
  73         {
  74             addr_operand = _addr_operand;
  75         }
  76
  77       private:
  78         int size;
  79         AddrOperandBase *addr_operand;
  80
  81       public:
  82         int getMemOperandSize() { return size; }
  83         AddrOperandBase *getAddressOperand() { return addr_operand; }
  84     };
  85
  86     template<typename DestOperandType, typename AddrOperandType>
  87     class LdaInstBase : public HsailGPUStaticInst
  88     {
  89       public:
  90         typename DestOperandType::DestOperand dest;
  91         AddrOperandType addr;
  92
  93         LdaInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
  94                     const char *_opcode)
  95            : HsailGPUStaticInst(obj, _opcode)
  96         {
  97             using namespace Brig;
  98
  99             unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
 100             dest.init(op_offs, obj);
 101             op_offs = obj->getOperandPtr(ib->operands, 1);
 102             addr.init(op_offs, obj);
 103         }
 104
 105         int numSrcRegOperands() { return(this->addr.isVectorRegister()); }
 106         int numDstRegOperands() { return dest.isVectorRegister(); }
 107         bool isVectorRegister(int operandIndex)
 108         {
 109             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 110             return((operandIndex == 0) ? dest.isVectorRegister() :
 111                    this->addr.isVectorRegister());
 112         }
 113         bool isCondRegister(int operandIndex)
 114         {
 115             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 116             return((operandIndex == 0) ? dest.isCondRegister() :
 117                    this->addr.isCondRegister());
 118         }
 119         bool isScalarRegister(int operandIndex)
 120         {
 121             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 122             return((operandIndex == 0) ? dest.isScalarRegister() :
 123                    this->addr.isScalarRegister());
 124         }
 125         bool isSrcOperand(int operandIndex)
 126         {
 127             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 128             if (operandIndex > 0)
 129                 return(this->addr.isVectorRegister());
 130             return false;
 131         }
 132         bool isDstOperand(int operandIndex) {
 133             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 134             return(operandIndex == 0);
 135         }
 136         int getOperandSize(int operandIndex)
 137         {
 138             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 139             return((operandIndex == 0) ? dest.opSize() :
 140                    this->addr.opSize());
 141         }
 142         int getRegisterIndex(int operandIndex)
 143         {
 144             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 145             return((operandIndex == 0) ? dest.regIndex() :
 146                    this->addr.regIndex());
 147         }
 148         int getNumOperands()
 149         {
 150             if (this->addr.isVectorRegister())
 151                 return 2;
 152             return 1;
 153         }
 154     };
 155
 156     template<typename DestDataType, typename AddrOperandType>
 157     class LdaInst :
 158         public LdaInstBase<typename DestDataType::OperandType, AddrOperandType>,
 159         public MemInst
 160     {
 161       public:
 162         void generateDisassembly();
 163
 164         LdaInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
 165                         const char *_opcode)
 166             : LdaInstBase<typename DestDataType::OperandType,
 167                           AddrOperandType>(ib, obj, _opcode)
 168         {
 169             init_addr(&this->addr);
 170         }
 171
 172         void execute(GPUDynInstPtr gpuDynInst);
 173     };
 174
 175     template<typename DataType>
 176     GPUStaticInst*
 177     decodeLda(const Brig::BrigInstBase *ib, const BrigObject *obj)
 178     {
 179         unsigned op_offs = obj->getOperandPtr(ib->operands, 1);
 180         BrigRegOperandInfo regDataType = findRegDataType(op_offs, obj);
 181
 182         if (regDataType.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
 183             return new LdaInst<DataType, NoRegAddrOperand>(ib, obj, "ldas");
 184         } else if (regDataType.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
 185             // V2/V4 not allowed
 186             switch (regDataType.regKind) {
 187               case Brig::BRIG_REGISTER_KIND_SINGLE:
 188                 return new LdaInst<DataType, SRegAddrOperand>(ib, obj, "ldas");
 189               case Brig::BRIG_REGISTER_KIND_DOUBLE:
 190                 return new LdaInst<DataType, DRegAddrOperand>(ib, obj, "ldas");
 191               default:
 192                 fatal("Bad ldas register operand type %d\n", regDataType.type);
 193             }
 194         } else {
 195             fatal("Bad ldas register operand kind %d\n", regDataType.kind);
 196         }
 197     }
 198
 199     template<typename MemOperandType, typename DestOperandType,
 200              typename AddrOperandType>
 201     class LdInstBase : public HsailGPUStaticInst
 202     {
 203       public:
 204         Brig::BrigWidth8_t width;
 205         typename DestOperandType::DestOperand dest;
 206         AddrOperandType addr;
 207
 208         Brig::BrigSegment segment;
 209         Brig::BrigMemoryOrder memoryOrder;
 210         Brig::BrigMemoryScope memoryScope;
 211         unsigned int equivClass;
 212         bool isArgLoad()
 213         {
 214             return segment == Brig::BRIG_SEGMENT_KERNARG ||
 215                    segment == Brig::BRIG_SEGMENT_ARG;
 216         }
 217         void
 218         initLd(const Brig::BrigInstBase *ib, const BrigObject *obj,
 219                const char *_opcode)
 220         {
 221             using namespace Brig;
 222
 223             const BrigInstMem *ldst = (const BrigInstMem*)ib;
 224
 225             segment = (BrigSegment)ldst->segment;
 226             memoryOrder = BRIG_MEMORY_ORDER_NONE;
 227             memoryScope = BRIG_MEMORY_SCOPE_NONE;
 228             equivClass = ldst->equivClass;
 229
 230             switch (segment) {
 231               case BRIG_SEGMENT_GLOBAL:
 232                 o_type = Enums::OT_GLOBAL_READ;
 233                 break;
 234
 235               case BRIG_SEGMENT_GROUP:
 236                 o_type = Enums::OT_SHARED_READ;
 237                 break;
 238
 239               case BRIG_SEGMENT_PRIVATE:
 240                 o_type = Enums::OT_PRIVATE_READ;
 241                 break;
 242
 243               case BRIG_SEGMENT_READONLY:
 244                 o_type = Enums::OT_READONLY_READ;
 245                 break;
 246
 247               case BRIG_SEGMENT_SPILL:
 248                 o_type = Enums::OT_SPILL_READ;
 249                 break;
 250
 251               case BRIG_SEGMENT_FLAT:
 252                 o_type = Enums::OT_FLAT_READ;
 253                 break;
 254
 255               case BRIG_SEGMENT_KERNARG:
 256                 o_type = Enums::OT_KERN_READ;
 257                 break;
 258
 259               case BRIG_SEGMENT_ARG:
 260                 o_type = Enums::OT_ARG;
 261                 break;
 262
 263               default:
 264                 panic("Ld: segment %d not supported\n", segment);
 265             }
 266
 267             width = ldst->width;
 268             unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
 269             const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
 270             if (brigOp->kind == BRIG_KIND_OPERAND_REGISTER)
 271                 dest.init(op_offs, obj);
 272
 273             op_offs = obj->getOperandPtr(ib->operands, 1);
 274             addr.init(op_offs, obj);
 275         }
 276
 277         void
 278         initAtomicLd(const Brig::BrigInstBase *ib, const BrigObject *obj,
 279                      const char *_opcode)
 280         {
 281             using namespace Brig;
 282
 283             const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
 284
 285             segment = (BrigSegment)at->segment;
 286             memoryOrder = (BrigMemoryOrder)at->memoryOrder;
 287             memoryScope = (BrigMemoryScope)at->memoryScope;
 288             equivClass = 0;
 289
 290             switch (segment) {
 291               case BRIG_SEGMENT_GLOBAL:
 292                 o_type = Enums::OT_GLOBAL_READ;
 293                 break;
 294
 295               case BRIG_SEGMENT_GROUP:
 296                 o_type = Enums::OT_SHARED_READ;
 297                 break;
 298
 299               case BRIG_SEGMENT_PRIVATE:
 300                 o_type = Enums::OT_PRIVATE_READ;
 301                 break;
 302
 303               case BRIG_SEGMENT_READONLY:
 304                 o_type = Enums::OT_READONLY_READ;
 305                 break;
 306
 307               case BRIG_SEGMENT_SPILL:
 308                 o_type = Enums::OT_SPILL_READ;
 309                 break;
 310
 311               case BRIG_SEGMENT_FLAT:
 312                 o_type = Enums::OT_FLAT_READ;
 313                 break;
 314
 315               case BRIG_SEGMENT_KERNARG:
 316                 o_type = Enums::OT_KERN_READ;
 317                 break;
 318
 319               case BRIG_SEGMENT_ARG:
 320                 o_type = Enums::OT_ARG;
 321                 break;
 322
 323               default:
 324                 panic("Ld: segment %d not supported\n", segment);
 325             }
 326
 327             width = BRIG_WIDTH_1;
 328             unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
 329             const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
 330
 331             if (brigOp->kind == BRIG_KIND_OPERAND_REGISTER)
 332                 dest.init(op_offs, obj);
 333
 334             op_offs = obj->getOperandPtr(ib->operands,1);
 335             addr.init(op_offs, obj);
 336         }
 337
 338         LdInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
 339                    const char *_opcode)
 340            : HsailGPUStaticInst(obj, _opcode)
 341         {
 342             using namespace Brig;
 343
 344             if (ib->opcode == BRIG_OPCODE_LD) {
 345                 initLd(ib, obj, _opcode);
 346             } else {
 347                 initAtomicLd(ib, obj, _opcode);
 348             }
 349         }
 350
 351         int numSrcRegOperands() { return(this->addr.isVectorRegister()); }
 352         int numDstRegOperands() { return dest.isVectorRegister(); }
 353         int getNumOperands()
 354         {
 355             if (this->addr.isVectorRegister())
 356                 return 2;
 357             else
 358                 return 1;
 359         }
 360         bool isVectorRegister(int operandIndex)
 361         {
 362             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 363             return((operandIndex == 0) ? dest.isVectorRegister() :
 364                    this->addr.isVectorRegister());
 365         }
 366         bool isCondRegister(int operandIndex)
 367         {
 368             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 369             return((operandIndex == 0) ? dest.isCondRegister() :
 370                    this->addr.isCondRegister());
 371         }
 372         bool isScalarRegister(int operandIndex)
 373         {
 374             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 375             return((operandIndex == 0) ? dest.isScalarRegister() :
 376                    this->addr.isScalarRegister());
 377         }
 378         bool isSrcOperand(int operandIndex)
 379         {
 380             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 381             if (operandIndex > 0)
 382                 return(this->addr.isVectorRegister());
 383             return false;
 384         }
 385         bool isDstOperand(int operandIndex)
 386         {
 387             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 388             return(operandIndex == 0);
 389         }
 390         int getOperandSize(int operandIndex)
 391         {
 392             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 393             return((operandIndex == 0) ? dest.opSize() :
 394                    this->addr.opSize());
 395         }
 396         int getRegisterIndex(int operandIndex)
 397         {
 398             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 399             return((operandIndex == 0) ? dest.regIndex() :
 400                    this->addr.regIndex());
 401         }
 402     };
 403
 404     template<typename MemDataType, typename DestDataType,
 405              typename AddrOperandType>
 406     class LdInst :
 407         public LdInstBase<typename MemDataType::CType,
 408                           typename DestDataType::OperandType, AddrOperandType>,
 409         public MemInst
 410     {
 411         typename DestDataType::OperandType::DestOperand dest_vect[4];
 412         uint16_t num_dest_operands;
 413         void generateDisassembly();
 414
 415       public:
 416         LdInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
 417                const char *_opcode)
 418             : LdInstBase<typename MemDataType::CType,
 419                          typename DestDataType::OperandType,
 420                          AddrOperandType>(ib, obj, _opcode),
 421               MemInst(MemDataType::memType)
 422         {
 423             init_addr(&this->addr);
 424
 425             unsigned op_offs = obj->getOperandPtr(ib->operands,0);
 426             const Brig::BrigOperand *brigOp = obj->getOperand(op_offs);
 427
 428             if (brigOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
 429                 const Brig::BrigOperandOperandList *brigRegVecOp =
 430                     (const Brig::BrigOperandOperandList*)brigOp;
 431
 432                 num_dest_operands =
 433                     *((unsigned*)obj->getData(brigRegVecOp->elements)) / 4;
 434
 435                 assert(num_dest_operands <= 4);
 436             } else {
 437                 num_dest_operands = 1;
 438             }
 439
 440             if (num_dest_operands > 1) {
 441                 assert(brigOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
 442
 443                 for (int i = 0; i < num_dest_operands; ++i) {
 444                     dest_vect[i].init_from_vect(op_offs, obj, i);
 445                 }
 446             }
 447         }
 448
 449         void
 450         initiateAcc(GPUDynInstPtr gpuDynInst) override
 451         {
 452             typedef typename MemDataType::CType c0;
 453
 454             gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
 455
 456             if (num_dest_operands > 1) {
 457                 for (int i = 0; i < VSZ; ++i)
 458                     if (gpuDynInst->exec_mask[i])
 459                         gpuDynInst->statusVector.push_back(num_dest_operands);
 460                     else
 461                         gpuDynInst->statusVector.push_back(0);
 462             }
 463
 464             for (int k = 0; k < num_dest_operands; ++k) {
 465
 466                 c0 *d = &((c0*)gpuDynInst->d_data)[k * VSZ];
 467
 468                 for (int i = 0; i < VSZ; ++i) {
 469                     if (gpuDynInst->exec_mask[i]) {
 470                         Addr vaddr = gpuDynInst->addr[i] + k * sizeof(c0);
 471
 472                         if (isLocalMem()) {
 473                             // load from shared memory
 474                             *d = gpuDynInst->wavefront()->ldsChunk->
 475                                 read<c0>(vaddr);
 476                         } else {
 477                             Request *req = new Request(0, vaddr, sizeof(c0), 0,
 478                                           gpuDynInst->computeUnit()->masterId(),
 479                                           0, gpuDynInst->wfDynId, i);
 480
 481                             gpuDynInst->setRequestFlags(req);
 482                             PacketPtr pkt = new Packet(req, MemCmd::ReadReq);
 483                             pkt->dataStatic(d);
 484
 485                             if (gpuDynInst->computeUnit()->shader->
 486                                 separate_acquire_release &&
 487                                 gpuDynInst->memoryOrder ==
 488                                 Enums::MEMORY_ORDER_SC_ACQUIRE) {
 489                                 // if this load has acquire semantics,
 490                                 // set the response continuation function
 491                                 // to perform an Acquire request
 492                                 gpuDynInst->execContinuation =
 493                                     &GPUStaticInst::execLdAcq;
 494
 495                                 gpuDynInst->useContinuation = true;
 496                             } else {
 497                                 // the request will be finished when
 498                                 // the load completes
 499                                 gpuDynInst->useContinuation = false;
 500                             }
 501                             // translation is performed in sendRequest()
 502                             gpuDynInst->computeUnit()->sendRequest(gpuDynInst,
 503                                                                    i, pkt);
 504                         }
 505                     }
 506                     ++d;
 507                 }
 508             }
 509
 510             gpuDynInst->updateStats();
 511         }
 512
 513       private:
 514         void
 515         execLdAcq(GPUDynInstPtr gpuDynInst) override
 516         {
 517             // after the load has complete and if the load has acquire
 518             // semantics, issue an acquire request.
 519             if (!isLocalMem()) {
 520                 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
 521                     && gpuDynInst->memoryOrder ==
 522                     Enums::MEMORY_ORDER_SC_ACQUIRE) {
 523                     gpuDynInst->statusBitVector = VectorMask(1);
 524                     gpuDynInst->useContinuation = false;
 525                     // create request
 526                     Request *req = new Request(0, 0, 0, 0,
 527                                   gpuDynInst->computeUnit()->masterId(),
 528                                   0, gpuDynInst->wfDynId, -1);
 529                     req->setFlags(Request::ACQUIRE);
 530                     gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
 531                 }
 532             }
 533         }
 534
 535       public:
 536         bool
 537         isLocalMem() const override
 538         {
 539             return this->segment == Brig::BRIG_SEGMENT_GROUP;
 540         }
 541
 542         bool isVectorRegister(int operandIndex)
 543         {
 544             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 545             if ((num_dest_operands != getNumOperands()) &&
 546                 (operandIndex == (getNumOperands()-1)))
 547                 return(this->addr.isVectorRegister());
 548             if (num_dest_operands > 1) {
 549                 return dest_vect[operandIndex].isVectorRegister();
 550             }
 551             else if (num_dest_operands == 1) {
 552                 return LdInstBase<typename MemDataType::CType,
 553                        typename DestDataType::OperandType,
 554                        AddrOperandType>::dest.isVectorRegister();
 555             }
 556             return false;
 557         }
 558         bool isCondRegister(int operandIndex)
 559         {
 560             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 561             if ((num_dest_operands != getNumOperands()) &&
 562                 (operandIndex == (getNumOperands()-1)))
 563                 return(this->addr.isCondRegister());
 564             if (num_dest_operands > 1)
 565                 return dest_vect[operandIndex].isCondRegister();
 566             else if (num_dest_operands == 1)
 567                 return LdInstBase<typename MemDataType::CType,
 568                        typename DestDataType::OperandType,
 569                        AddrOperandType>::dest.isCondRegister();
 570             return false;
 571         }
 572         bool isScalarRegister(int operandIndex)
 573         {
 574             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 575             if ((num_dest_operands != getNumOperands()) &&
 576                 (operandIndex == (getNumOperands()-1)))
 577                 return(this->addr.isScalarRegister());
 578             if (num_dest_operands > 1)
 579                 return dest_vect[operandIndex].isScalarRegister();
 580             else if (num_dest_operands == 1)
 581                 return LdInstBase<typename MemDataType::CType,
 582                        typename DestDataType::OperandType,
 583                        AddrOperandType>::dest.isScalarRegister();
 584             return false;
 585         }
 586         bool isSrcOperand(int operandIndex)
 587         {
 588             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 589             if ((num_dest_operands != getNumOperands()) &&
 590                 (operandIndex == (getNumOperands()-1)))
 591                 return(this->addr.isVectorRegister());
 592             return false;
 593         }
 594         bool isDstOperand(int operandIndex)
 595         {
 596             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 597             if ((num_dest_operands != getNumOperands()) &&
 598                 (operandIndex == (getNumOperands()-1)))
 599                 return false;
 600             return true;
 601         }
 602         int getOperandSize(int operandIndex)
 603         {
 604             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 605             if ((num_dest_operands != getNumOperands()) &&
 606                 (operandIndex == (getNumOperands()-1)))
 607                 return(this->addr.opSize());
 608             if (num_dest_operands > 1)
 609                 return(dest_vect[operandIndex].opSize());
 610             else if (num_dest_operands == 1)
 611                 return(LdInstBase<typename MemDataType::CType,
 612                        typename DestDataType::OperandType,
 613                        AddrOperandType>::dest.opSize());
 614             return 0;
 615         }
 616         int getRegisterIndex(int operandIndex)
 617         {
 618             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 619             if ((num_dest_operands != getNumOperands()) &&
 620                 (operandIndex == (getNumOperands()-1)))
 621                 return(this->addr.regIndex());
 622             if (num_dest_operands > 1)
 623                 return(dest_vect[operandIndex].regIndex());
 624             else if (num_dest_operands == 1)
 625                 return(LdInstBase<typename MemDataType::CType,
 626                        typename DestDataType::OperandType,
 627                        AddrOperandType>::dest.regIndex());
 628             return -1;
 629         }
 630         int getNumOperands()
 631         {
 632             if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
 633                 return(num_dest_operands+1);
 634             else
 635                 return(num_dest_operands);
 636         }
 637         void execute(GPUDynInstPtr gpuDynInst);
 638     };
 639
 640     template<typename MemDT, typename DestDT>
 641     GPUStaticInst*
 642     decodeLd2(const Brig::BrigInstBase *ib, const BrigObject *obj)
 643     {
 644         unsigned op_offs = obj->getOperandPtr(ib->operands,1);
 645         BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
 646
 647         if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
 648             return new LdInst<MemDT, DestDT, NoRegAddrOperand>(ib, obj, "ld");
 649         } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER ||
 650                    tmp.kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
 651             switch (tmp.regKind) {
 652               case Brig::BRIG_REGISTER_KIND_SINGLE:
 653                 return new LdInst<MemDT, DestDT,
 654                                   SRegAddrOperand>(ib, obj, "ld");
 655               case Brig::BRIG_REGISTER_KIND_DOUBLE:
 656                 return new LdInst<MemDT, DestDT,
 657                                   DRegAddrOperand>(ib, obj, "ld");
 658               default:
 659                 fatal("Bad ld register operand type %d\n", tmp.regKind);
 660             }
 661         } else {
 662             fatal("Bad ld register operand kind %d\n", tmp.kind);
 663         }
 664     }
 665
 666     template<typename MemDT>
 667     GPUStaticInst*
 668     decodeLd(const Brig::BrigInstBase *ib, const BrigObject *obj)
 669     {
 670         unsigned op_offs = obj->getOperandPtr(ib->operands,0);
 671         BrigRegOperandInfo dest = findRegDataType(op_offs, obj);
 672
 673         assert(dest.kind == Brig::BRIG_KIND_OPERAND_REGISTER ||
 674                dest.kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
 675         switch(dest.regKind) {
 676           case Brig::BRIG_REGISTER_KIND_SINGLE:
 677             switch (ib->type) {
 678               case Brig::BRIG_TYPE_B8:
 679               case Brig::BRIG_TYPE_B16:
 680               case Brig::BRIG_TYPE_B32:
 681                 return decodeLd2<MemDT, B32>(ib, obj);
 682               case Brig::BRIG_TYPE_U8:
 683               case Brig::BRIG_TYPE_U16:
 684               case Brig::BRIG_TYPE_U32:
 685                 return decodeLd2<MemDT, U32>(ib, obj);
 686               case Brig::BRIG_TYPE_S8:
 687               case Brig::BRIG_TYPE_S16:
 688               case Brig::BRIG_TYPE_S32:
 689                 return decodeLd2<MemDT, S32>(ib, obj);
 690               case Brig::BRIG_TYPE_F16:
 691               case Brig::BRIG_TYPE_F32:
 692                 return decodeLd2<MemDT, U32>(ib, obj);
 693               default:
 694                 fatal("Bad ld register operand type %d, %d\n",
 695                       dest.regKind, ib->type);
 696             };
 697           case Brig::BRIG_REGISTER_KIND_DOUBLE:
 698             switch (ib->type) {
 699               case Brig::BRIG_TYPE_B64:
 700                 return decodeLd2<MemDT, B64>(ib, obj);
 701               case Brig::BRIG_TYPE_U64:
 702                 return decodeLd2<MemDT, U64>(ib, obj);
 703               case Brig::BRIG_TYPE_S64:
 704                 return decodeLd2<MemDT, S64>(ib, obj);
 705               case Brig::BRIG_TYPE_F64:
 706                 return decodeLd2<MemDT, U64>(ib, obj);
 707               default:
 708                 fatal("Bad ld register operand type %d, %d\n",
 709                       dest.regKind, ib->type);
 710             };
 711           default:
 712             fatal("Bad ld register operand type %d, %d\n", dest.regKind,
 713                   ib->type);
 714         }
 715     }
 716
 717     template<typename MemDataType, typename SrcOperandType,
 718              typename AddrOperandType>
 719     class StInstBase : public HsailGPUStaticInst
 720     {
 721       public:
 722         typename SrcOperandType::SrcOperand src;
 723         AddrOperandType addr;
 724
 725         Brig::BrigSegment segment;
 726         Brig::BrigMemoryScope memoryScope;
 727         Brig::BrigMemoryOrder memoryOrder;
 728         unsigned int equivClass;
 729
 730         void
 731         initSt(const Brig::BrigInstBase *ib, const BrigObject *obj,
 732                const char *_opcode)
 733         {
 734             using namespace Brig;
 735
 736             const BrigInstMem *ldst = (const BrigInstMem*)ib;
 737
 738             segment = (BrigSegment)ldst->segment;
 739             memoryOrder = BRIG_MEMORY_ORDER_NONE;
 740             memoryScope = BRIG_MEMORY_SCOPE_NONE;
 741             equivClass = ldst->equivClass;
 742
 743             switch (segment) {
 744               case BRIG_SEGMENT_GLOBAL:
 745                 o_type = Enums::OT_GLOBAL_WRITE;
 746                 break;
 747
 748               case BRIG_SEGMENT_GROUP:
 749                 o_type = Enums::OT_SHARED_WRITE;
 750                 break;
 751
 752               case BRIG_SEGMENT_PRIVATE:
 753                 o_type = Enums::OT_PRIVATE_WRITE;
 754                 break;
 755
 756               case BRIG_SEGMENT_READONLY:
 757                 o_type = Enums::OT_READONLY_WRITE;
 758                 break;
 759
 760               case BRIG_SEGMENT_SPILL:
 761                 o_type = Enums::OT_SPILL_WRITE;
 762                 break;
 763
 764               case BRIG_SEGMENT_FLAT:
 765                 o_type = Enums::OT_FLAT_WRITE;
 766                 break;
 767
 768               case BRIG_SEGMENT_ARG:
 769                 o_type = Enums::OT_ARG;
 770                 break;
 771
 772               default:
 773                 panic("St: segment %d not supported\n", segment);
 774             }
 775
 776             unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
 777             const BrigOperand *baseOp = obj->getOperand(op_offs);
 778
 779             if ((baseOp->kind == BRIG_KIND_OPERAND_CONSTANT_BYTES) ||
 780                 (baseOp->kind == BRIG_KIND_OPERAND_REGISTER)) {
 781                 src.init(op_offs, obj);
 782             }
 783
 784             op_offs = obj->getOperandPtr(ib->operands, 1);
 785             addr.init(op_offs, obj);
 786         }
 787
 788         void
 789         initAtomicSt(const Brig::BrigInstBase *ib, const BrigObject *obj,
 790                      const char *_opcode)
 791         {
 792             using namespace Brig;
 793
 794             const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
 795
 796             segment = (BrigSegment)at->segment;
 797             memoryScope = (BrigMemoryScope)at->memoryScope;
 798             memoryOrder = (BrigMemoryOrder)at->memoryOrder;
 799             equivClass = 0;
 800
 801             switch (segment) {
 802               case BRIG_SEGMENT_GLOBAL:
 803                 o_type = Enums::OT_GLOBAL_WRITE;
 804                 break;
 805
 806               case BRIG_SEGMENT_GROUP:
 807                 o_type = Enums::OT_SHARED_WRITE;
 808                 break;
 809
 810               case BRIG_SEGMENT_PRIVATE:
 811                 o_type = Enums::OT_PRIVATE_WRITE;
 812                 break;
 813
 814               case BRIG_SEGMENT_READONLY:
 815                 o_type = Enums::OT_READONLY_WRITE;
 816                 break;
 817
 818               case BRIG_SEGMENT_SPILL:
 819                 o_type = Enums::OT_SPILL_WRITE;
 820                 break;
 821
 822               case BRIG_SEGMENT_FLAT:
 823                 o_type = Enums::OT_FLAT_WRITE;
 824                 break;
 825
 826               case BRIG_SEGMENT_ARG:
 827                 o_type = Enums::OT_ARG;
 828                 break;
 829
 830               default:
 831                 panic("St: segment %d not supported\n", segment);
 832             }
 833
 834             unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
 835             addr.init(op_offs, obj);
 836
 837             op_offs = obj->getOperandPtr(ib->operands, 1);
 838             src.init(op_offs, obj);
 839         }
 840
 841         StInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
 842                    const char *_opcode)
 843            : HsailGPUStaticInst(obj, _opcode)
 844         {
 845             using namespace Brig;
 846
 847             if (ib->opcode == BRIG_OPCODE_ST) {
 848                 initSt(ib, obj, _opcode);
 849             } else {
 850                 initAtomicSt(ib, obj, _opcode);
 851             }
 852         }
 853
 854         int numDstRegOperands() { return 0; }
 855         int numSrcRegOperands()
 856         {
 857             return src.isVectorRegister() + this->addr.isVectorRegister();
 858         }
 859         int getNumOperands()
 860         {
 861             if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
 862                 return 2;
 863             else
 864                 return 1;
 865         }
 866         bool isVectorRegister(int operandIndex)
 867         {
 868             assert(operandIndex >= 0 && operandIndex < getNumOperands());
 869             return !operandIndex ? src.isVectorRegister() :
 870                    this->addr.isVectorRegister();
 871         }
 872         bool isCondRegister(int operandIndex)
 873         {
 874             assert(operandIndex >= 0 && operandIndex < getNumOperands());
 875             return !operandIndex ? src.isCondRegister() :
 876                    this->addr.isCondRegister();
 877         }
 878         bool isScalarRegister(int operandIndex)
 879         {
 880             assert(operandIndex >= 0 && operandIndex < getNumOperands());
 881             return !operandIndex ? src.isScalarRegister() :
 882                    this->addr.isScalarRegister();
 883         }
 884         bool isSrcOperand(int operandIndex)
 885         {
 886             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
 887             return true;
 888         }
 889         bool isDstOperand(int operandIndex) { return false; }
 890         int getOperandSize(int operandIndex)
 891         {
 892             assert(operandIndex >= 0 && operandIndex < getNumOperands());
 893             return !operandIndex ? src.opSize() : this->addr.opSize();
 894         }
 895         int getRegisterIndex(int operandIndex)
 896         {
 897             assert(operandIndex >= 0 && operandIndex < getNumOperands());
 898             return !operandIndex ? src.regIndex() : this->addr.regIndex();
 899         }
 900     };
 901
 902
 903     template<typename MemDataType, typename SrcDataType,
 904              typename AddrOperandType>
 905     class StInst :
 906         public StInstBase<MemDataType, typename SrcDataType::OperandType,
 907                           AddrOperandType>,
 908         public MemInst
 909     {
 910       public:
 911         typename SrcDataType::OperandType::SrcOperand src_vect[4];
 912         uint16_t num_src_operands;
 913         void generateDisassembly();
 914
 915         StInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
 916                         const char *_opcode, int srcIdx)
 917             : StInstBase<MemDataType, typename SrcDataType::OperandType,
 918                          AddrOperandType>(ib, obj, _opcode),
 919               MemInst(SrcDataType::memType)
 920         {
 921             init_addr(&this->addr);
 922
 923             BrigRegOperandInfo rinfo;
 924             unsigned op_offs = obj->getOperandPtr(ib->operands,srcIdx);
 925             const Brig::BrigOperand *baseOp = obj->getOperand(op_offs);
 926
 927             if (baseOp->kind == Brig::BRIG_KIND_OPERAND_CONSTANT_BYTES) {
 928                 const Brig::BrigOperandConstantBytes *op =
 929                     (Brig::BrigOperandConstantBytes*)baseOp;
 930
 931                 rinfo = BrigRegOperandInfo((Brig::BrigKind16_t)op->base.kind,
 932                                            Brig::BRIG_TYPE_NONE);
 933             } else {
 934                 rinfo = findRegDataType(op_offs, obj);
 935             }
 936
 937             if (baseOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST) {
 938                 const Brig::BrigOperandOperandList *brigRegVecOp =
 939                     (const Brig::BrigOperandOperandList*)baseOp;
 940
 941                 num_src_operands =
 942                     *((unsigned*)obj->getData(brigRegVecOp->elements)) / 4;
 943
 944                 assert(num_src_operands <= 4);
 945             } else {
 946                 num_src_operands = 1;
 947             }
 948
 949             if (num_src_operands > 1) {
 950                 assert(baseOp->kind == Brig::BRIG_KIND_OPERAND_OPERAND_LIST);
 951
 952                 for (int i = 0; i < num_src_operands; ++i) {
 953                     src_vect[i].init_from_vect(op_offs, obj, i);
 954                 }
 955             }
 956         }
 957
 958         void
 959         initiateAcc(GPUDynInstPtr gpuDynInst) override
 960         {
 961             // before performing a store, check if this store has
 962             // release semantics, and if so issue a release first
 963             if (!isLocalMem()) {
 964                 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
 965                     && gpuDynInst->memoryOrder ==
 966                     Enums::MEMORY_ORDER_SC_RELEASE) {
 967
 968                     gpuDynInst->statusBitVector = VectorMask(1);
 969                     gpuDynInst->execContinuation = &GPUStaticInst::execSt;
 970                     gpuDynInst->useContinuation = true;
 971                     // create request
 972                     Request *req = new Request(0, 0, 0, 0,
 973                                   gpuDynInst->computeUnit()->masterId(),
 974                                   0, gpuDynInst->wfDynId, -1);
 975                     req->setFlags(Request::RELEASE);
 976                     gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
 977
 978                     return;
 979                 }
 980             }
 981
 982             // if there is no release semantic, perform stores immediately
 983             execSt(gpuDynInst);
 984         }
 985
 986         bool
 987         isLocalMem() const override
 988         {
 989             return this->segment == Brig::BRIG_SEGMENT_GROUP;
 990         }
 991
 992       private:
 993         // execSt may be called through a continuation
 994         // if the store had release semantics. see comment for
 995         // execSt in gpu_static_inst.hh
 996         void
 997         execSt(GPUDynInstPtr gpuDynInst) override
 998         {
 999             typedef typename MemDataType::CType c0;
1000
1001             gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
1002
1003             if (num_src_operands > 1) {
1004                 for (int i = 0; i < VSZ; ++i)
1005                     if (gpuDynInst->exec_mask[i])
1006                         gpuDynInst->statusVector.push_back(num_src_operands);
1007                     else
1008                         gpuDynInst->statusVector.push_back(0);
1009             }
1010
1011             for (int k = 0; k < num_src_operands; ++k) {
1012                 c0 *d = &((c0*)gpuDynInst->d_data)[k * VSZ];
1013
1014                 for (int i = 0; i < VSZ; ++i) {
1015                     if (gpuDynInst->exec_mask[i]) {
1016                         Addr vaddr = gpuDynInst->addr[i] + k * sizeof(c0);
1017
1018                         if (isLocalMem()) {
1019                             //store to shared memory
1020                             gpuDynInst->wavefront()->ldsChunk->write<c0>(vaddr,
1021                                                                          *d);
1022                         } else {
1023                             Request *req =
1024                               new Request(0, vaddr, sizeof(c0), 0,
1025                                           gpuDynInst->computeUnit()->masterId(),
1026                                           0, gpuDynInst->wfDynId, i);
1027
1028                             gpuDynInst->setRequestFlags(req);
1029                             PacketPtr pkt = new Packet(req, MemCmd::WriteReq);
1030                             pkt->dataStatic<c0>(d);
1031
1032                             // translation is performed in sendRequest()
1033                             // the request will be finished when the store completes
1034                             gpuDynInst->useContinuation = false;
1035                             gpuDynInst->computeUnit()->sendRequest(gpuDynInst,
1036                                                                    i, pkt);
1037
1038                         }
1039                     }
1040                     ++d;
1041                 }
1042             }
1043
1044             gpuDynInst->updateStats();
1045         }
1046
1047       public:
1048         bool isVectorRegister(int operandIndex)
1049         {
1050             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1051             if (operandIndex == num_src_operands)
1052                 return this->addr.isVectorRegister();
1053             if (num_src_operands > 1)
1054                 return src_vect[operandIndex].isVectorRegister();
1055             else if (num_src_operands == 1)
1056                 return StInstBase<MemDataType,
1057                        typename SrcDataType::OperandType,
1058                        AddrOperandType>::src.isVectorRegister();
1059             return false;
1060         }
1061         bool isCondRegister(int operandIndex)
1062         {
1063             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1064             if (operandIndex == num_src_operands)
1065                 return this->addr.isCondRegister();
1066             if (num_src_operands > 1)
1067                 return src_vect[operandIndex].isCondRegister();
1068             else if (num_src_operands == 1)
1069                 return StInstBase<MemDataType,
1070                        typename SrcDataType::OperandType,
1071                        AddrOperandType>::src.isCondRegister();
1072             return false;
1073         }
1074         bool isScalarRegister(int operandIndex)
1075         {
1076             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1077             if (operandIndex == num_src_operands)
1078                 return this->addr.isScalarRegister();
1079             if (num_src_operands > 1)
1080                 return src_vect[operandIndex].isScalarRegister();
1081             else if (num_src_operands == 1)
1082                 return StInstBase<MemDataType,
1083                        typename SrcDataType::OperandType,
1084                        AddrOperandType>::src.isScalarRegister();
1085             return false;
1086         }
1087         bool isSrcOperand(int operandIndex)
1088         {
1089             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1090             return true;
1091         }
1092         bool isDstOperand(int operandIndex) { return false; }
1093         int getOperandSize(int operandIndex)
1094         {
1095             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1096             if (operandIndex == num_src_operands)
1097                 return this->addr.opSize();
1098             if (num_src_operands > 1)
1099                 return src_vect[operandIndex].opSize();
1100             else if (num_src_operands == 1)
1101                 return StInstBase<MemDataType,
1102                        typename SrcDataType::OperandType,
1103                        AddrOperandType>::src.opSize();
1104             return 0;
1105         }
1106         int getRegisterIndex(int operandIndex)
1107         {
1108             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1109             if (operandIndex == num_src_operands)
1110                 return this->addr.regIndex();
1111             if (num_src_operands > 1)
1112                 return src_vect[operandIndex].regIndex();
1113             else if (num_src_operands == 1)
1114                 return StInstBase<MemDataType,
1115                        typename SrcDataType::OperandType,
1116                        AddrOperandType>::src.regIndex();
1117             return -1;
1118         }
1119         int getNumOperands()
1120         {
1121             if (this->addr.isVectorRegister() || this->addr.isScalarRegister())
1122                 return num_src_operands + 1;
1123             else
1124                 return num_src_operands;
1125         }
1126         void execute(GPUDynInstPtr gpuDynInst);
1127     };
1128
1129     template<typename DataType, typename SrcDataType>
1130     GPUStaticInst*
1131     decodeSt(const Brig::BrigInstBase *ib, const BrigObject *obj)
1132     {
1133         int srcIdx = 0;
1134         int destIdx = 1;
1135         if (ib->opcode == Brig::BRIG_OPCODE_ATOMIC ||
1136             ib->opcode == Brig::BRIG_OPCODE_ATOMICNORET) {
1137             srcIdx = 1;
1138             destIdx = 0;
1139         }
1140         unsigned op_offs = obj->getOperandPtr(ib->operands,destIdx);
1141
1142         BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
1143
1144         if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
1145             return new StInst<DataType, SrcDataType,
1146                               NoRegAddrOperand>(ib, obj, "st", srcIdx);
1147         } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
1148             // V2/V4 not allowed
1149             switch (tmp.regKind) {
1150               case Brig::BRIG_REGISTER_KIND_SINGLE:
1151                 return new StInst<DataType, SrcDataType,
1152                                   SRegAddrOperand>(ib, obj, "st", srcIdx);
1153               case Brig::BRIG_REGISTER_KIND_DOUBLE:
1154                 return new StInst<DataType, SrcDataType,
1155                                   DRegAddrOperand>(ib, obj, "st", srcIdx);
1156               default:
1157                 fatal("Bad st register operand type %d\n", tmp.type);
1158             }
1159         } else {
1160             fatal("Bad st register operand kind %d\n", tmp.kind);
1161         }
1162     }
1163
1164     Enums::MemOpType brigAtomicToMemOpType(Brig::BrigOpcode brigOpCode,
1165                                            Brig::BrigAtomicOperation brigOp);
1166
1167     template<typename OperandType, typename AddrOperandType, int NumSrcOperands,
1168              bool HasDst>
1169     class AtomicInstBase : public HsailGPUStaticInst
1170     {
1171       public:
1172         typename OperandType::DestOperand dest;
1173         typename OperandType::SrcOperand src[NumSrcOperands];
1174         AddrOperandType addr;
1175
1176         Brig::BrigSegment segment;
1177         Brig::BrigMemoryOrder memoryOrder;
1178         Brig::BrigAtomicOperation atomicOperation;
1179         Brig::BrigMemoryScope memoryScope;
1180         Brig::BrigOpcode opcode;
1181         Enums::MemOpType opType;
1182
1183         AtomicInstBase(const Brig::BrigInstBase *ib, const BrigObject *obj,
1184                        const char *_opcode)
1185            : HsailGPUStaticInst(obj, _opcode)
1186         {
1187             using namespace Brig;
1188
1189             const BrigInstAtomic *at = (const BrigInstAtomic*)ib;
1190
1191             segment = (BrigSegment)at->segment;
1192             memoryScope = (BrigMemoryScope)at->memoryScope;
1193             memoryOrder = (BrigMemoryOrder)at->memoryOrder;
1194             atomicOperation = (BrigAtomicOperation)at->atomicOperation;
1195             opcode = (BrigOpcode)ib->opcode;
1196             opType = brigAtomicToMemOpType(opcode, atomicOperation);
1197
1198             switch (segment) {
1199               case BRIG_SEGMENT_GLOBAL:
1200                 o_type = Enums::OT_GLOBAL_ATOMIC;
1201                 break;
1202
1203               case BRIG_SEGMENT_GROUP:
1204                 o_type = Enums::OT_SHARED_ATOMIC;
1205                 break;
1206
1207               case BRIG_SEGMENT_FLAT:
1208                 o_type = Enums::OT_FLAT_ATOMIC;
1209                 break;
1210
1211               default:
1212                 panic("Atomic: segment %d not supported\n", segment);
1213             }
1214
1215             if (HasDst) {
1216                 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
1217                 dest.init(op_offs, obj);
1218
1219                 op_offs = obj->getOperandPtr(ib->operands, 1);
1220                 addr.init(op_offs, obj);
1221
1222                 for (int i = 0; i < NumSrcOperands; ++i) {
1223                     op_offs = obj->getOperandPtr(ib->operands, i + 2);
1224                     src[i].init(op_offs, obj);
1225                 }
1226             } else {
1227
1228                 unsigned op_offs = obj->getOperandPtr(ib->operands, 0);
1229                 addr.init(op_offs, obj);
1230
1231                 for (int i = 0; i < NumSrcOperands; ++i) {
1232                     op_offs = obj->getOperandPtr(ib->operands, i + 1);
1233                     src[i].init(op_offs, obj);
1234                 }
1235             }
1236         }
1237
1238         int numSrcRegOperands()
1239         {
1240             int operands = 0;
1241             for (int i = 0; i < NumSrcOperands; i++) {
1242                 if (src[i].isVectorRegister() == true) {
1243                     operands++;
1244                 }
1245             }
1246             if (addr.isVectorRegister())
1247                 operands++;
1248             return operands;
1249         }
1250         int numDstRegOperands() { return dest.isVectorRegister(); }
1251         int getNumOperands()
1252         {
1253             if (addr.isVectorRegister())
1254                 return(NumSrcOperands + 2);
1255             return(NumSrcOperands + 1);
1256         }
1257         bool isVectorRegister(int operandIndex)
1258         {
1259             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1260             if (operandIndex < NumSrcOperands)
1261                 return src[operandIndex].isVectorRegister();
1262             else if (operandIndex == NumSrcOperands)
1263                 return(addr.isVectorRegister());
1264             else
1265                 return dest.isVectorRegister();
1266         }
1267         bool isCondRegister(int operandIndex)
1268         {
1269             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1270             if (operandIndex < NumSrcOperands)
1271                 return src[operandIndex].isCondRegister();
1272             else if (operandIndex == NumSrcOperands)
1273                 return(addr.isCondRegister());
1274             else
1275                 return dest.isCondRegister();
1276         }
1277         bool isScalarRegister(int operandIndex)
1278         {
1279             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1280             if (operandIndex < NumSrcOperands)
1281                 return src[operandIndex].isScalarRegister();
1282             else if (operandIndex == NumSrcOperands)
1283                 return(addr.isScalarRegister());
1284             else
1285                 return dest.isScalarRegister();
1286         }
1287         bool isSrcOperand(int operandIndex)
1288         {
1289             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1290             if (operandIndex < NumSrcOperands)
1291                 return true;
1292             else if (operandIndex == NumSrcOperands)
1293                 return(addr.isVectorRegister());
1294             else
1295                 return false;
1296         }
1297         bool isDstOperand(int operandIndex)
1298         {
1299             if (operandIndex <= NumSrcOperands)
1300                 return false;
1301             else
1302                 return true;
1303         }
1304         int getOperandSize(int operandIndex)
1305         {
1306             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1307             if (operandIndex < NumSrcOperands)
1308                 return(src[operandIndex].opSize());
1309             else if (operandIndex == NumSrcOperands)
1310                 return(addr.opSize());
1311             else
1312                 return(dest.opSize());
1313         }
1314         int getRegisterIndex(int operandIndex)
1315         {
1316             assert((operandIndex >= 0) && (operandIndex < getNumOperands()));
1317             if (operandIndex < NumSrcOperands)
1318                 return(src[operandIndex].regIndex());
1319             else if (operandIndex == NumSrcOperands)
1320                 return(addr.regIndex());
1321             else
1322                 return(dest.regIndex());
1323             return -1;
1324         }
1325     };
1326
1327     template<typename MemDataType, typename AddrOperandType, int NumSrcOperands,
1328              bool HasDst>
1329     class AtomicInst :
1330         public AtomicInstBase<typename MemDataType::OperandType,
1331                               AddrOperandType, NumSrcOperands, HasDst>,
1332         public MemInst
1333     {
1334       public:
1335         void generateDisassembly();
1336
1337         AtomicInst(const Brig::BrigInstBase *ib, const BrigObject *obj,
1338                    const char *_opcode)
1339             : AtomicInstBase<typename MemDataType::OperandType, AddrOperandType,
1340                              NumSrcOperands, HasDst>
1341                 (ib, obj, _opcode),
1342               MemInst(MemDataType::memType)
1343         {
1344             init_addr(&this->addr);
1345         }
1346
1347         void
1348         initiateAcc(GPUDynInstPtr gpuDynInst) override
1349         {
1350             // before doing the RMW, check if this atomic has
1351             // release semantics, and if so issue a release first
1352             if (!isLocalMem()) {
1353                 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1354                     && (gpuDynInst->memoryOrder ==
1355                     Enums::MEMORY_ORDER_SC_RELEASE || gpuDynInst->memoryOrder ==
1356                     Enums::MEMORY_ORDER_SC_ACQUIRE_RELEASE)) {
1357
1358                     gpuDynInst->statusBitVector = VectorMask(1);
1359
1360                     gpuDynInst->execContinuation = &GPUStaticInst::execAtomic;
1361                     gpuDynInst->useContinuation = true;
1362
1363                     // create request
1364                     Request *req = new Request(0, 0, 0, 0,
1365                                   gpuDynInst->computeUnit()->masterId(),
1366                                   0, gpuDynInst->wfDynId, -1);
1367                     req->setFlags(Request::RELEASE);
1368                     gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1369
1370                     return;
1371                 }
1372             }
1373
1374             // if there is no release semantic, execute the RMW immediately
1375             execAtomic(gpuDynInst);
1376
1377         }
1378
1379         void execute(GPUDynInstPtr gpuDynInst);
1380
1381         bool
1382         isLocalMem() const override
1383         {
1384             return this->segment == Brig::BRIG_SEGMENT_GROUP;
1385         }
1386
1387       private:
1388         // execAtomic may be called through a continuation
1389         // if the RMW had release semantics. see comment for
1390         // execContinuation in gpu_dyn_inst.hh
1391         void
1392         execAtomic(GPUDynInstPtr gpuDynInst) override
1393         {
1394             gpuDynInst->statusBitVector = gpuDynInst->exec_mask;
1395
1396             typedef typename MemDataType::CType c0;
1397
1398             c0 *d = &((c0*) gpuDynInst->d_data)[0];
1399             c0 *e = &((c0*) gpuDynInst->a_data)[0];
1400             c0 *f = &((c0*) gpuDynInst->x_data)[0];
1401
1402             for (int i = 0; i < VSZ; ++i) {
1403                 if (gpuDynInst->exec_mask[i]) {
1404                     Addr vaddr = gpuDynInst->addr[i];
1405
1406                     if (isLocalMem()) {
1407                         Wavefront *wavefront = gpuDynInst->wavefront();
1408                         *d = wavefront->ldsChunk->read<c0>(vaddr);
1409
1410                         switch (this->opType) {
1411                           case Enums::MO_AADD:
1412                           case Enums::MO_ANRADD:
1413                             wavefront->ldsChunk->write<c0>(vaddr,
1414                             wavefront->ldsChunk->read<c0>(vaddr) + (*e));
1415                             break;
1416                           case Enums::MO_ASUB:
1417                           case Enums::MO_ANRSUB:
1418                             wavefront->ldsChunk->write<c0>(vaddr,
1419                             wavefront->ldsChunk->read<c0>(vaddr) - (*e));
1420                             break;
1421                           case Enums::MO_AMAX:
1422                           case Enums::MO_ANRMAX:
1423                             wavefront->ldsChunk->write<c0>(vaddr,
1424                             std::max(wavefront->ldsChunk->read<c0>(vaddr),
1425                             (*e)));
1426                             break;
1427                           case Enums::MO_AMIN:
1428                           case Enums::MO_ANRMIN:
1429                             wavefront->ldsChunk->write<c0>(vaddr,
1430                             std::min(wavefront->ldsChunk->read<c0>(vaddr),
1431                             (*e)));
1432                             break;
1433                           case Enums::MO_AAND:
1434                           case Enums::MO_ANRAND:
1435                             wavefront->ldsChunk->write<c0>(vaddr,
1436                             wavefront->ldsChunk->read<c0>(vaddr) & (*e));
1437                             break;
1438                           case Enums::MO_AOR:
1439                           case Enums::MO_ANROR:
1440                             wavefront->ldsChunk->write<c0>(vaddr,
1441                             wavefront->ldsChunk->read<c0>(vaddr) | (*e));
1442                             break;
1443                           case Enums::MO_AXOR:
1444                           case Enums::MO_ANRXOR:
1445                             wavefront->ldsChunk->write<c0>(vaddr,
1446                             wavefront->ldsChunk->read<c0>(vaddr) ^ (*e));
1447                             break;
1448                           case Enums::MO_AINC:
1449                           case Enums::MO_ANRINC:
1450                             wavefront->ldsChunk->write<c0>(vaddr,
1451                             wavefront->ldsChunk->read<c0>(vaddr) + 1);
1452                             break;
1453                           case Enums::MO_ADEC:
1454                           case Enums::MO_ANRDEC:
1455                             wavefront->ldsChunk->write<c0>(vaddr,
1456                             wavefront->ldsChunk->read<c0>(vaddr) - 1);
1457                             break;
1458                           case Enums::MO_AEXCH:
1459                           case Enums::MO_ANREXCH:
1460                             wavefront->ldsChunk->write<c0>(vaddr, (*e));
1461                             break;
1462                           case Enums::MO_ACAS:
1463                           case Enums::MO_ANRCAS:
1464                             wavefront->ldsChunk->write<c0>(vaddr,
1465                             (wavefront->ldsChunk->read<c0>(vaddr) == (*e)) ?
1466                             (*f) : wavefront->ldsChunk->read<c0>(vaddr));
1467                             break;
1468                           default:
1469                             fatal("Unrecognized or invalid HSAIL atomic op "
1470                                   "type.\n");
1471                             break;
1472                         }
1473                     } else {
1474                         Request *req =
1475                             new Request(0, vaddr, sizeof(c0), 0,
1476                                         gpuDynInst->computeUnit()->masterId(),
1477                                         0, gpuDynInst->wfDynId, i,
1478                                         gpuDynInst->makeAtomicOpFunctor<c0>(e,
1479                                         f, this->opType));
1480
1481                         gpuDynInst->setRequestFlags(req);
1482                         PacketPtr pkt = new Packet(req, MemCmd::SwapReq);
1483                         pkt->dataStatic(d);
1484
1485                         if (gpuDynInst->computeUnit()->shader->
1486                             separate_acquire_release &&
1487                             (gpuDynInst->memoryOrder ==
1488                              Enums::MEMORY_ORDER_SC_ACQUIRE)) {
1489                             // if this atomic has acquire semantics,
1490                             // schedule the continuation to perform an
1491                             // acquire after the RMW completes
1492                             gpuDynInst->execContinuation =
1493                                 &GPUStaticInst::execAtomicAcq;
1494
1495                             gpuDynInst->useContinuation = true;
1496                         } else {
1497                             // the request will be finished when the RMW completes
1498                             gpuDynInst->useContinuation = false;
1499                         }
1500                         // translation is performed in sendRequest()
1501                         gpuDynInst->computeUnit()->sendRequest(gpuDynInst, i,
1502                                                                pkt);
1503                     }
1504                 }
1505
1506                 ++d;
1507                 ++e;
1508                 ++f;
1509             }
1510
1511             gpuDynInst->updateStats();
1512         }
1513
1514         // execAtomicACq will always be called through a continuation.
1515         // see comment for execContinuation in gpu_dyn_inst.hh
1516         void
1517         execAtomicAcq(GPUDynInstPtr gpuDynInst) override
1518         {
1519             // after performing the RMW, check to see if this instruction
1520             // has acquire semantics, and if so, issue an acquire
1521             if (!isLocalMem()) {
1522                 if (gpuDynInst->computeUnit()->shader->separate_acquire_release
1523                      && gpuDynInst->memoryOrder ==
1524                      Enums::MEMORY_ORDER_SC_ACQUIRE) {
1525                     gpuDynInst->statusBitVector = VectorMask(1);
1526
1527                     // the request will be finished when
1528                     // the acquire completes
1529                     gpuDynInst->useContinuation = false;
1530                     // create request
1531                     Request *req = new Request(0, 0, 0, 0,
1532                                   gpuDynInst->computeUnit()->masterId(),
1533                                   0, gpuDynInst->wfDynId, -1);
1534                     req->setFlags(Request::ACQUIRE);
1535                     gpuDynInst->computeUnit()->injectGlobalMemFence(gpuDynInst, false, req);
1536                 }
1537             }
1538         }
1539     };
1540
1541     template<typename DataType, typename AddrOperandType, int NumSrcOperands>
1542     GPUStaticInst*
1543     constructAtomic(const Brig::BrigInstBase *ib, const BrigObject *obj)
1544     {
1545         const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1546
1547         if (at->atomicOperation == Brig::BRIG_ATOMIC_LD) {
1548             return decodeLd<DataType>(ib, obj);
1549         } else if (at->atomicOperation == Brig::BRIG_ATOMIC_ST) {
1550             switch (ib->type) {
1551               case Brig::BRIG_TYPE_B8:
1552                 return decodeSt<S8,S8>(ib, obj);
1553               case Brig::BRIG_TYPE_B16:
1554                 return decodeSt<S8,S16>(ib, obj);
1555               case Brig::BRIG_TYPE_B32:
1556                 return decodeSt<S8,S32>(ib, obj);
1557               case Brig::BRIG_TYPE_B64:
1558                 return decodeSt<S8,S64>(ib, obj);
1559               default: fatal("AtomicSt: Operand type mismatch %d\n", ib->type);
1560             }
1561         } else {
1562             if ((Brig::BrigOpcode)ib->opcode == Brig::BRIG_OPCODE_ATOMICNORET)
1563                 return new AtomicInst<DataType, AddrOperandType,
1564                     NumSrcOperands, false>(ib, obj, "atomicnoret");
1565             else
1566                 return new AtomicInst<DataType, AddrOperandType,
1567                     NumSrcOperands, true>(ib, obj, "atomic");
1568         }
1569     }
1570
1571     template<typename DataType, int NumSrcOperands>
1572     GPUStaticInst*
1573     decodeAtomicHelper(const Brig::BrigInstBase *ib, const BrigObject *obj)
1574     {
1575         unsigned addrIndex = (Brig::BrigOpcode)ib->opcode ==
1576             Brig::BRIG_OPCODE_ATOMICNORET ? 0 : 1;
1577
1578         unsigned op_offs = obj->getOperandPtr(ib->operands,addrIndex);
1579
1580         BrigRegOperandInfo tmp = findRegDataType(op_offs, obj);
1581
1582         if (tmp.kind == Brig::BRIG_KIND_OPERAND_ADDRESS) {
1583             return constructAtomic<DataType, NoRegAddrOperand,
1584                                    NumSrcOperands>(ib, obj);
1585         } else if (tmp.kind == Brig::BRIG_KIND_OPERAND_REGISTER) {
1586             // V2/V4 not allowed
1587             switch (tmp.regKind) {
1588               case Brig::BRIG_REGISTER_KIND_SINGLE:
1589                   return constructAtomic<DataType, SRegAddrOperand,
1590                                          NumSrcOperands>(ib, obj);
1591               case Brig::BRIG_REGISTER_KIND_DOUBLE:
1592                 return constructAtomic<DataType, DRegAddrOperand,
1593                                        NumSrcOperands>(ib, obj);
1594               default:
1595                 fatal("Bad atomic register operand type %d\n", tmp.type);
1596             }
1597         } else {
1598             fatal("Bad atomic register operand kind %d\n", tmp.kind);
1599         }
1600     }
1601
1602
1603     template<typename DataType>
1604     GPUStaticInst*
1605     decodeAtomic(const Brig::BrigInstBase *ib, const BrigObject *obj)
1606     {
1607         const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1608
1609         if (at->atomicOperation == Brig::BRIG_ATOMIC_CAS) {
1610             return decodeAtomicHelper<DataType, 2>(ib, obj);
1611         } else {
1612             return decodeAtomicHelper<DataType, 1>(ib, obj);
1613         }
1614     }
1615
1616     template<typename DataType>
1617     GPUStaticInst*
1618     decodeAtomicNoRet(const Brig::BrigInstBase *ib, const BrigObject *obj)
1619     {
1620         const Brig::BrigInstAtomic *at = (const Brig::BrigInstAtomic*)ib;
1621         if (at->atomicOperation == Brig::BRIG_ATOMIC_CAS) {
1622             return decodeAtomicHelper<DataType, 2>(ib, obj);
1623         } else {
1624             return decodeAtomicHelper<DataType, 1>(ib, obj);
1625         }
1626     }
1627 } // namespace HsailISA
1628
1629 #endif // __ARCH_HSAIL_INSTS_MEM_HH__