src/gallium/drivers/radeon/SIISelLowering.cpp

   1 //===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // Most of the DAG lowering is handled in AMDGPUISelLowering.cpp.  This file is
  11 // mostly EmitInstrWithCustomInserter().
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "SIISelLowering.h"
  16 #include "AMDIL.h"
  17 #include "AMDILIntrinsicInfo.h"
  18 #include "SIInstrInfo.h"
  19 #include "SIRegisterInfo.h"
  20 #include "llvm/CodeGen/MachineInstrBuilder.h"
  21 #include "llvm/CodeGen/MachineRegisterInfo.h"
  22 #include "llvm/CodeGen/SelectionDAG.h"
  23
  24 using namespace llvm;
  25
  26 SITargetLowering::SITargetLowering(TargetMachine &TM) :
  27     AMDGPUTargetLowering(TM),
  28     TII(static_cast<const SIInstrInfo*>(TM.getInstrInfo()))
  29 {
  30   addRegisterClass(MVT::v4f32, &AMDGPU::VReg_128RegClass);
  31   addRegisterClass(MVT::f32, &AMDGPU::VReg_32RegClass);
  32   addRegisterClass(MVT::i32, &AMDGPU::VReg_32RegClass);
  33   addRegisterClass(MVT::i64, &AMDGPU::VReg_64RegClass);
  34   addRegisterClass(MVT::i1, &AMDGPU::SCCRegRegClass);
  35   addRegisterClass(MVT::i1, &AMDGPU::VCCRegRegClass);
  36
  37   addRegisterClass(MVT::v4i32, &AMDGPU::SReg_128RegClass);
  38   addRegisterClass(MVT::v8i32, &AMDGPU::SReg_256RegClass);
  39
  40   computeRegisterProperties();
  41
  42   setOperationAction(ISD::AND, MVT::i1, Custom);
  43
  44   setOperationAction(ISD::ADD, MVT::i64, Legal);
  45   setOperationAction(ISD::ADD, MVT::i32, Legal);
  46
  47   setOperationAction(ISD::BR_CC, MVT::i32, Custom);
  48
  49   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
  50
  51   // We need to custom lower loads from the USER_SGPR address space, so we can
  52   // add the SGPRs as livein registers.
  53   setOperationAction(ISD::LOAD, MVT::i32, Custom);
  54   setOperationAction(ISD::LOAD, MVT::i64, Custom);
  55
  56   setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
  57   setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
  58
  59   setOperationAction(ISD::SELECT_CC, MVT::Other, Expand);
  60   setTargetDAGCombine(ISD::SELECT_CC);
  61
  62   setTargetDAGCombine(ISD::SETCC);
  63 }
  64
  65 MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter(
  66     MachineInstr * MI, MachineBasicBlock * BB) const
  67 {
  68   const TargetInstrInfo * TII = getTargetMachine().getInstrInfo();
  69   MachineRegisterInfo & MRI = BB->getParent()->getRegInfo();
  70   MachineBasicBlock::iterator I = MI;
  71
  72   if (TII->get(MI->getOpcode()).TSFlags & SIInstrFlags::NEED_WAIT) {
  73     AppendS_WAITCNT(MI, *BB, llvm::next(I));
  74     return BB;
  75   }
  76
  77   switch (MI->getOpcode()) {
  78   default:
  79     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
  80
  81   case AMDGPU::CLAMP_SI:
  82     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::V_MOV_B32_e64))
  83            .addOperand(MI->getOperand(0))
  84            .addOperand(MI->getOperand(1))
  85            // VSRC1-2 are unused, but we still need to fill all the
  86            // operand slots, so we just reuse the VSRC0 operand
  87            .addOperand(MI->getOperand(1))
  88            .addOperand(MI->getOperand(1))
  89            .addImm(0) // ABS
  90            .addImm(1) // CLAMP
  91            .addImm(0) // OMOD
  92            .addImm(0); // NEG
  93     MI->eraseFromParent();
  94     break;
  95
  96   case AMDGPU::FABS_SI:
  97     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::V_MOV_B32_e64))
  98                  .addOperand(MI->getOperand(0))
  99                  .addOperand(MI->getOperand(1))
 100                  // VSRC1-2 are unused, but we still need to fill all the
 101                  // operand slots, so we just reuse the VSRC0 operand
 102                  .addOperand(MI->getOperand(1))
 103                  .addOperand(MI->getOperand(1))
 104                  .addImm(1) // ABS
 105                  .addImm(0) // CLAMP
 106                  .addImm(0) // OMOD
 107                  .addImm(0); // NEG
 108     MI->eraseFromParent();
 109     break;
 110
 111   case AMDGPU::FNEG_SI:
 112     BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::V_MOV_B32_e64))
 113                  .addOperand(MI->getOperand(0))
 114                  .addOperand(MI->getOperand(1))
 115                  // VSRC1-2 are unused, but we still need to fill all the
 116                  // operand slots, so we just reuse the VSRC0 operand
 117                  .addOperand(MI->getOperand(1))
 118                  .addOperand(MI->getOperand(1))
 119                  .addImm(0) // ABS
 120                  .addImm(0) // CLAMP
 121                  .addImm(0) // OMOD
 122                  .addImm(1); // NEG
 123     MI->eraseFromParent();
 124     break;
 125
 126   case AMDGPU::SI_INTERP:
 127     LowerSI_INTERP(MI, *BB, I, MRI);
 128     break;
 129   case AMDGPU::SI_INTERP_CONST:
 130     LowerSI_INTERP_CONST(MI, *BB, I);
 131     break;
 132   case AMDGPU::SI_KIL:
 133     LowerSI_KIL(MI, *BB, I, MRI);
 134     break;
 135   case AMDGPU::SI_V_CNDLT:
 136     LowerSI_V_CNDLT(MI, *BB, I, MRI);
 137     break;
 138   }
 139   return BB;
 140 }
 141
 142 void SITargetLowering::AppendS_WAITCNT(MachineInstr *MI, MachineBasicBlock &BB,
 143     MachineBasicBlock::iterator I) const
 144 {
 145   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_WAITCNT))
 146           .addImm(0);
 147 }
 148
 149 void SITargetLowering::LowerSI_INTERP(MachineInstr *MI, MachineBasicBlock &BB,
 150     MachineBasicBlock::iterator I, MachineRegisterInfo & MRI) const
 151 {
 152   unsigned tmp = MRI.createVirtualRegister(&AMDGPU::VReg_32RegClass);
 153   MachineOperand dst = MI->getOperand(0);
 154   MachineOperand iReg = MI->getOperand(1);
 155   MachineOperand jReg = MI->getOperand(2);
 156   MachineOperand attr_chan = MI->getOperand(3);
 157   MachineOperand attr = MI->getOperand(4);
 158   MachineOperand params = MI->getOperand(5);
 159
 160   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_MOV_B32))
 161           .addReg(AMDGPU::M0)
 162           .addOperand(params);
 163
 164   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_INTERP_P1_F32), tmp)
 165           .addOperand(iReg)
 166           .addOperand(attr_chan)
 167           .addOperand(attr);
 168
 169   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_INTERP_P2_F32))
 170           .addOperand(dst)
 171           .addReg(tmp)
 172           .addOperand(jReg)
 173           .addOperand(attr_chan)
 174           .addOperand(attr);
 175
 176   MI->eraseFromParent();
 177 }
 178
 179 void SITargetLowering::LowerSI_INTERP_CONST(MachineInstr *MI,
 180     MachineBasicBlock &BB, MachineBasicBlock::iterator I) const
 181 {
 182   MachineOperand dst = MI->getOperand(0);
 183   MachineOperand attr_chan = MI->getOperand(1);
 184   MachineOperand attr = MI->getOperand(2);
 185   MachineOperand params = MI->getOperand(3);
 186
 187   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_MOV_B32))
 188           .addReg(AMDGPU::M0)
 189           .addOperand(params);
 190
 191   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_INTERP_MOV_F32))
 192           .addOperand(dst)
 193           .addOperand(attr_chan)
 194           .addOperand(attr);
 195
 196   MI->eraseFromParent();
 197 }
 198
 199 void SITargetLowering::LowerSI_KIL(MachineInstr *MI, MachineBasicBlock &BB,
 200     MachineBasicBlock::iterator I, MachineRegisterInfo & MRI) const
 201 {
 202   // Clear this pixel from the exec mask if the operand is negative
 203   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_CMPX_LE_F32_e32),
 204           AMDGPU::VCC)
 205           .addReg(AMDGPU::SREG_LIT_0)
 206           .addOperand(MI->getOperand(0));
 207
 208   // If the exec mask is non-zero, skip the next two instructions
 209   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_CBRANCH_EXECNZ))
 210           .addImm(3)
 211           .addReg(AMDGPU::EXEC);
 212
 213   // Exec mask is zero: Export to NULL target...
 214   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::EXP))
 215           .addImm(0)
 216           .addImm(0x09) // V_008DFC_SQ_EXP_NULL
 217           .addImm(0)
 218           .addImm(1)
 219           .addImm(1)
 220           .addReg(AMDGPU::SREG_LIT_0)
 221           .addReg(AMDGPU::SREG_LIT_0)
 222           .addReg(AMDGPU::SREG_LIT_0)
 223           .addReg(AMDGPU::SREG_LIT_0);
 224
 225   // ... and terminate wavefront
 226   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_ENDPGM));
 227
 228   MI->eraseFromParent();
 229 }
 230
 231 void SITargetLowering::LowerSI_V_CNDLT(MachineInstr *MI, MachineBasicBlock &BB,
 232     MachineBasicBlock::iterator I, MachineRegisterInfo & MRI) const
 233 {
 234   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_CMP_LT_F32_e32),
 235           AMDGPU::VCC)
 236           .addOperand(MI->getOperand(1))
 237           .addReg(AMDGPU::SREG_LIT_0);
 238
 239   BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_CNDMASK_B32))
 240           .addOperand(MI->getOperand(0))
 241           .addReg(AMDGPU::VCC)
 242           .addOperand(MI->getOperand(2))
 243           .addOperand(MI->getOperand(3));
 244
 245   MI->eraseFromParent();
 246 }
 247
 248 EVT SITargetLowering::getSetCCResultType(EVT VT) const
 249 {
 250   return MVT::i1;
 251 }
 252
 253 //===----------------------------------------------------------------------===//
 254 // Custom DAG Lowering Operations
 255 //===----------------------------------------------------------------------===//
 256
 257 SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const
 258 {
 259   switch (Op.getOpcode()) {
 260   default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
 261   case ISD::BR_CC: return LowerBR_CC(Op, DAG);
 262   case ISD::LOAD: return LowerLOAD(Op, DAG);
 263   case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
 264   case ISD::AND: return Loweri1ContextSwitch(Op, DAG, ISD::AND);
 265   case ISD::INTRINSIC_WO_CHAIN: {
 266     unsigned IntrinsicID =
 267                          cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
 268     EVT VT = Op.getValueType();
 269     switch (IntrinsicID) {
 270     case AMDGPUIntrinsic::SI_vs_load_buffer_index:
 271       return CreateLiveInRegister(DAG, &AMDGPU::VReg_32RegClass,
 272                                   AMDGPU::VGPR0, VT);
 273     default: return AMDGPUTargetLowering::LowerOperation(Op, DAG);
 274     }
 275     break;
 276   }
 277   }
 278   return SDValue();
 279 }
 280
 281 /// Loweri1ContextSwitch - The function is for lowering i1 operations on the
 282 /// VCC register.  In the VALU context, VCC is a one bit register, but in the
 283 /// SALU context the VCC is a 64-bit register (1-bit per thread).  Since only
 284 /// the SALU can perform operations on the VCC register, we need to promote
 285 /// the operand types from i1 to i64 in order for tablegen to be able to match
 286 /// this operation to the correct SALU instruction.  We do this promotion by
 287 /// wrapping the operands in a CopyToReg node.
 288 ///
 289 SDValue SITargetLowering::Loweri1ContextSwitch(SDValue Op,
 290                                                SelectionDAG &DAG,
 291                                                unsigned VCCNode) const
 292 {
 293   DebugLoc DL = Op.getDebugLoc();
 294
 295   SDValue OpNode = DAG.getNode(VCCNode, DL, MVT::i64,
 296                                DAG.getNode(SIISD::VCC_BITCAST, DL, MVT::i64,
 297                                            Op.getOperand(0)),
 298                                DAG.getNode(SIISD::VCC_BITCAST, DL, MVT::i64,
 299                                            Op.getOperand(1)));
 300
 301   return DAG.getNode(SIISD::VCC_BITCAST, DL, MVT::i1, OpNode);
 302 }
 303
 304 SDValue SITargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const
 305 {
 306   SDValue Chain = Op.getOperand(0);
 307   SDValue CC = Op.getOperand(1);
 308   SDValue LHS   = Op.getOperand(2);
 309   SDValue RHS   = Op.getOperand(3);
 310   SDValue JumpT  = Op.getOperand(4);
 311   SDValue CmpValue;
 312   SDValue Result;
 313   CmpValue = DAG.getNode(
 314       ISD::SETCC,
 315       Op.getDebugLoc(),
 316       MVT::i1,
 317       LHS, RHS,
 318       CC);
 319
 320   Result = DAG.getNode(
 321       AMDGPUISD::BRANCH_COND,
 322       CmpValue.getDebugLoc(),
 323       MVT::Other, Chain,
 324       JumpT, CmpValue);
 325   return Result;
 326 }
 327
 328 SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const
 329 {
 330   EVT VT = Op.getValueType();
 331   LoadSDNode *Ptr = dyn_cast<LoadSDNode>(Op);
 332
 333   assert(Ptr);
 334
 335   unsigned AddrSpace = Ptr->getPointerInfo().getAddrSpace();
 336
 337   // We only need to lower USER_SGPR address space loads
 338   if (AddrSpace != AMDGPUAS::USER_SGPR_ADDRESS) {
 339     return SDValue();
 340   }
 341
 342   // Loads from the USER_SGPR address space can only have constant value
 343   // pointers.
 344   ConstantSDNode *BasePtr = dyn_cast<ConstantSDNode>(Ptr->getBasePtr());
 345   assert(BasePtr);
 346
 347   unsigned TypeDwordWidth = VT.getSizeInBits() / 32;
 348   const TargetRegisterClass * dstClass;
 349   switch (TypeDwordWidth) {
 350     default:
 351       assert(!"USER_SGPR value size not implemented");
 352       return SDValue();
 353     case 1:
 354       dstClass = &AMDGPU::SReg_32RegClass;
 355       break;
 356     case 2:
 357       dstClass = &AMDGPU::SReg_64RegClass;
 358       break;
 359   }
 360   uint64_t Index = BasePtr->getZExtValue();
 361   assert(Index % TypeDwordWidth == 0 && "USER_SGPR not properly aligned");
 362   unsigned SGPRIndex = Index / TypeDwordWidth;
 363   unsigned Reg = dstClass->getRegister(SGPRIndex);
 364
 365   DAG.ReplaceAllUsesOfValueWith(Op, CreateLiveInRegister(DAG, dstClass, Reg,
 366                                                          VT));
 367   return SDValue();
 368 }
 369
 370 SDValue SITargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const
 371 {
 372   SDValue LHS = Op.getOperand(0);
 373   SDValue RHS = Op.getOperand(1);
 374   SDValue True = Op.getOperand(2);
 375   SDValue False = Op.getOperand(3);
 376   SDValue CC = Op.getOperand(4);
 377   EVT VT = Op.getValueType();
 378   DebugLoc DL = Op.getDebugLoc();
 379
 380   SDValue Cond = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, CC);
 381   return DAG.getNode(ISD::SELECT, DL, VT, Cond, True, False);
 382 }
 383
 384 //===----------------------------------------------------------------------===//
 385 // Custom DAG optimizations
 386 //===----------------------------------------------------------------------===//
 387
 388 SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
 389                                             DAGCombinerInfo &DCI) const {
 390   SelectionDAG &DAG = DCI.DAG;
 391   DebugLoc DL = N->getDebugLoc();
 392   EVT VT = N->getValueType(0);
 393
 394   switch (N->getOpcode()) {
 395     default: break;
 396     case ISD::SELECT_CC: {
 397       N->dump();
 398       ConstantSDNode *True, *False;
 399       // i1 selectcc(l, r, -1, 0, cc) -> i1 setcc(l, r, cc)
 400       if ((True = dyn_cast<ConstantSDNode>(N->getOperand(2)))
 401           && (False = dyn_cast<ConstantSDNode>(N->getOperand(3)))
 402           && True->isAllOnesValue()
 403           && False->isNullValue()
 404           && VT == MVT::i1) {
 405         return DAG.getNode(ISD::SETCC, DL, VT, N->getOperand(0),
 406                            N->getOperand(1), N->getOperand(4));
 407
 408       }
 409       break;
 410     }
 411     case ISD::SETCC: {
 412       SDValue Arg0 = N->getOperand(0);
 413       SDValue Arg1 = N->getOperand(1);
 414       SDValue CC = N->getOperand(2);
 415       ConstantSDNode * C = NULL;
 416       ISD::CondCode CCOp = dyn_cast<CondCodeSDNode>(CC)->get();
 417
 418       // i1 setcc (sext(i1), 0, setne) -> i1 setcc(i1, 0, setne)
 419       if (VT == MVT::i1
 420           && Arg0.getOpcode() == ISD::SIGN_EXTEND
 421           && Arg0.getOperand(0).getValueType() == MVT::i1
 422           && (C = dyn_cast<ConstantSDNode>(Arg1))
 423           && C->isNullValue()
 424           && CCOp == ISD::SETNE) {
 425         return SimplifySetCC(VT, Arg0.getOperand(0),
 426                              DAG.getConstant(0, MVT::i1), CCOp, true, DCI, DL);
 427       }
 428       break;
 429     }
 430   }
 431   return SDValue();
 432 }
 433
 434 #define NODE_NAME_CASE(node) case SIISD::node: return #node;
 435
 436 const char* SITargetLowering::getTargetNodeName(unsigned Opcode) const
 437 {
 438   switch (Opcode) {
 439   default: return AMDGPUTargetLowering::getTargetNodeName(Opcode);
 440   NODE_NAME_CASE(VCC_AND)
 441   NODE_NAME_CASE(VCC_BITCAST)
 442   }
 443 }