llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp - rust-lang/llvm-project - Git at Google

 //===-- RISCVExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. ---===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains a pass that expands atomic pseudo instructions into
 // target instructions. This pass should be run at the last possible moment,
 // avoiding the possibility for other passes to break the requirements for
 // forward progress in the LR/SC block.
 //
 //===----------------------------------------------------------------------===//

 #include "RISCV.h"
 #include "RISCVInstrInfo.h"
 #include "RISCVTargetMachine.h"

 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"

 using namespace llvm;

 #define RISCV_EXPAND_ATOMIC_PSEUDO_NAME                                        \
   "RISCV atomic pseudo instruction expansion pass"

 namespace {

 class RISCVExpandAtomicPseudo : public MachineFunctionPass {
 public:
   const RISCVInstrInfo *TII;
   static char ID;

   RISCVExpandAtomicPseudo() : MachineFunctionPass(ID) {
     initializeRISCVExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
   }

   bool runOnMachineFunction(MachineFunction &MF) override;

   StringRef getPassName() const override {
     return RISCV_EXPAND_ATOMIC_PSEUDO_NAME;
   }

 private:
   bool expandMBB(MachineBasicBlock &MBB);
   bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                 MachineBasicBlock::iterator &NextMBBI);
   bool expandAtomicBinOp(MachineBasicBlock &MBB,
                          MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
                          bool IsMasked, int Width,
                          MachineBasicBlock::iterator &NextMBBI);
   bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator MBBI,
                             AtomicRMWInst::BinOp, bool IsMasked, int Width,
                             MachineBasicBlock::iterator &NextMBBI);
   bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator MBBI, bool IsMasked,
                            int Width, MachineBasicBlock::iterator &NextMBBI);
 };

 char RISCVExpandAtomicPseudo::ID = 0;

 bool RISCVExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
   TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
   bool Modified = false;
   for (auto &MBB : MF)
     Modified |= expandMBB(MBB);
   return Modified;
 }

 bool RISCVExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
   bool Modified = false;

   MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
   while (MBBI != E) {
     MachineBasicBlock::iterator NMBBI = std::next(MBBI);
     Modified |= expandMI(MBB, MBBI, NMBBI);
     MBBI = NMBBI;
   }

   return Modified;
 }

 bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MBBI,
                                        MachineBasicBlock::iterator &NextMBBI) {
   // RISCVInstrInfo::getInstSizeInBytes expects that the total size of the
   // expanded instructions for each pseudo is correct in the Size field of the
   // tablegen definition for the pseudo.
   switch (MBBI->getOpcode()) {
   case RISCV::PseudoAtomicLoadNand32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
                              NextMBBI);
   case RISCV::PseudoAtomicLoadNand64:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
                              NextMBBI);
   case RISCV::PseudoMaskedAtomicSwap32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
                              NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadAdd32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadSub32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadNand32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
                              NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadMax32:
     return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
                                 NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadMin32:
     return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
                                 NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadUMax32:
     return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
                                 NextMBBI);
   case RISCV::PseudoMaskedAtomicLoadUMin32:
     return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
                                 NextMBBI);
   case RISCV::PseudoCmpXchg32:
     return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
   case RISCV::PseudoCmpXchg64:
     return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
   case RISCV::PseudoMaskedCmpXchg32:
     return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
   }

   return false;
 }

 static unsigned getLRForRMW32(AtomicOrdering Ordering) {
   switch (Ordering) {
   default:
     llvm_unreachable("Unexpected AtomicOrdering");
   case AtomicOrdering::Monotonic:
     return RISCV::LR_W;
   case AtomicOrdering::Acquire:
     return RISCV::LR_W_AQ;
   case AtomicOrdering::Release:
     return RISCV::LR_W;
   case AtomicOrdering::AcquireRelease:
     return RISCV::LR_W_AQ;
   case AtomicOrdering::SequentiallyConsistent:
     return RISCV::LR_W_AQ_RL;
   }
 }

 static unsigned getSCForRMW32(AtomicOrdering Ordering) {
   switch (Ordering) {
   default:
     llvm_unreachable("Unexpected AtomicOrdering");
   case AtomicOrdering::Monotonic:
     return RISCV::SC_W;
   case AtomicOrdering::Acquire:
     return RISCV::SC_W;
   case AtomicOrdering::Release:
     return RISCV::SC_W_RL;
   case AtomicOrdering::AcquireRelease:
     return RISCV::SC_W_RL;
   case AtomicOrdering::SequentiallyConsistent:
     return RISCV::SC_W_AQ_RL;
   }
 }

 static unsigned getLRForRMW64(AtomicOrdering Ordering) {
   switch (Ordering) {
   default:
     llvm_unreachable("Unexpected AtomicOrdering");
   case AtomicOrdering::Monotonic:
     return RISCV::LR_D;
   case AtomicOrdering::Acquire:
     return RISCV::LR_D_AQ;
   case AtomicOrdering::Release:
     return RISCV::LR_D;
   case AtomicOrdering::AcquireRelease:
     return RISCV::LR_D_AQ;
   case AtomicOrdering::SequentiallyConsistent:
     return RISCV::LR_D_AQ_RL;
   }
 }

 static unsigned getSCForRMW64(AtomicOrdering Ordering) {
   switch (Ordering) {
   default:
     llvm_unreachable("Unexpected AtomicOrdering");
   case AtomicOrdering::Monotonic:
     return RISCV::SC_D;
   case AtomicOrdering::Acquire:
     return RISCV::SC_D;
   case AtomicOrdering::Release:
     return RISCV::SC_D_RL;
   case AtomicOrdering::AcquireRelease:
     return RISCV::SC_D_RL;
   case AtomicOrdering::SequentiallyConsistent:
     return RISCV::SC_D_AQ_RL;
   }
 }

 static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
   if (Width == 32)
     return getLRForRMW32(Ordering);
   if (Width == 64)
     return getLRForRMW64(Ordering);
   llvm_unreachable("Unexpected LR width\n");
 }

 static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
   if (Width == 32)
     return getSCForRMW32(Ordering);
   if (Width == 64)
     return getSCForRMW64(Ordering);
   llvm_unreachable("Unexpected SC width\n");
 }

 static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
                                    DebugLoc DL, MachineBasicBlock *ThisMBB,
                                    MachineBasicBlock *LoopMBB,
                                    MachineBasicBlock *DoneMBB,
                                    AtomicRMWInst::BinOp BinOp, int Width) {
   Register DestReg = MI.getOperand(0).getReg();
   Register ScratchReg = MI.getOperand(1).getReg();
   Register AddrReg = MI.getOperand(2).getReg();
   Register IncrReg = MI.getOperand(3).getReg();
   AtomicOrdering Ordering =
       static_cast<AtomicOrdering>(MI.getOperand(4).getImm());

   // .loop:
   //   lr.[w|d] dest, (addr)
   //   binop scratch, dest, val
   //   sc.[w|d] scratch, scratch, (addr)
   //   bnez scratch, loop
   BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
       .addReg(AddrReg);
   switch (BinOp) {
   default:
     llvm_unreachable("Unexpected AtomicRMW BinOp");
   case AtomicRMWInst::Nand:
     BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
         .addReg(DestReg)
         .addReg(IncrReg);
     BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
         .addReg(ScratchReg)
         .addImm(-1);
     break;
   }
   BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
       .addReg(AddrReg)
       .addReg(ScratchReg);
   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
       .addReg(ScratchReg)
       .addReg(RISCV::X0)
       .addMBB(LoopMBB);
 }

 static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
                               MachineBasicBlock *MBB, Register DestReg,
                               Register OldValReg, Register NewValReg,
                               Register MaskReg, Register ScratchReg) {
   assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
   assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
   assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");

   // We select bits from newval and oldval using:
   // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
   // r = oldval ^ ((oldval ^ newval) & masktargetdata);
   BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
       .addReg(OldValReg)
       .addReg(NewValReg);
   BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
       .addReg(ScratchReg)
       .addReg(MaskReg);
   BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
       .addReg(OldValReg)
       .addReg(ScratchReg);
 }

 static void doMaskedAtomicBinOpExpansion(
     const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
     MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
     MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
   assert(Width == 32 && "Should never need to expand masked 64-bit operations");
   Register DestReg = MI.getOperand(0).getReg();
   Register ScratchReg = MI.getOperand(1).getReg();
   Register AddrReg = MI.getOperand(2).getReg();
   Register IncrReg = MI.getOperand(3).getReg();
   Register MaskReg = MI.getOperand(4).getReg();
   AtomicOrdering Ordering =
       static_cast<AtomicOrdering>(MI.getOperand(5).getImm());

   // .loop:
   //   lr.w destreg, (alignedaddr)
   //   binop scratch, destreg, incr
   //   xor scratch, destreg, scratch
   //   and scratch, scratch, masktargetdata
   //   xor scratch, destreg, scratch
   //   sc.w scratch, scratch, (alignedaddr)
   //   bnez scratch, loop
   BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
       .addReg(AddrReg);
   switch (BinOp) {
   default:
     llvm_unreachable("Unexpected AtomicRMW BinOp");
   case AtomicRMWInst::Xchg:
     BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
         .addReg(IncrReg)
         .addImm(0);
     break;
   case AtomicRMWInst::Add:
     BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
         .addReg(DestReg)
         .addReg(IncrReg);
     break;
   case AtomicRMWInst::Sub:
     BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
         .addReg(DestReg)
         .addReg(IncrReg);
     break;
   case AtomicRMWInst::Nand:
     BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
         .addReg(DestReg)
         .addReg(IncrReg);
     BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
         .addReg(ScratchReg)
         .addImm(-1);
     break;
   }

   insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
                     ScratchReg);

   BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
       .addReg(AddrReg)
       .addReg(ScratchReg);
   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
       .addReg(ScratchReg)
       .addReg(RISCV::X0)
       .addMBB(LoopMBB);
 }

 bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
     MachineBasicBlock::iterator &NextMBBI) {
   MachineInstr &MI = *MBBI;
   DebugLoc DL = MI.getDebugLoc();

   MachineFunction *MF = MBB.getParent();
   auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

   // Insert new MBBs.
   MF->insert(++MBB.getIterator(), LoopMBB);
   MF->insert(++LoopMBB->getIterator(), DoneMBB);

   // Set up successors and transfer remaining instructions to DoneMBB.
   LoopMBB->addSuccessor(LoopMBB);
   LoopMBB->addSuccessor(DoneMBB);
   DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
   DoneMBB->transferSuccessors(&MBB);
   MBB.addSuccessor(LoopMBB);

   if (!IsMasked)
     doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
   else
     doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
                                  Width);

   NextMBBI = MBB.end();
   MI.eraseFromParent();

   LivePhysRegs LiveRegs;
   computeAndAddLiveIns(LiveRegs, *LoopMBB);
   computeAndAddLiveIns(LiveRegs, *DoneMBB);

   return true;
 }

 static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
                        MachineBasicBlock *MBB, Register ValReg,
                        Register ShamtReg) {
   BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
       .addReg(ValReg)
       .addReg(ShamtReg);
   BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
       .addReg(ValReg)
       .addReg(ShamtReg);
 }

 bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
     MachineBasicBlock::iterator &NextMBBI) {
   assert(IsMasked == true &&
          "Should only need to expand masked atomic max/min");
   assert(Width == 32 && "Should never need to expand masked 64-bit operations");

   MachineInstr &MI = *MBBI;
   DebugLoc DL = MI.getDebugLoc();
   MachineFunction *MF = MBB.getParent();
   auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

   // Insert new MBBs.
   MF->insert(++MBB.getIterator(), LoopHeadMBB);
   MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
   MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
   MF->insert(++LoopTailMBB->getIterator(), DoneMBB);

   // Set up successors and transfer remaining instructions to DoneMBB.
   LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
   LoopHeadMBB->addSuccessor(LoopTailMBB);
   LoopIfBodyMBB->addSuccessor(LoopTailMBB);
   LoopTailMBB->addSuccessor(LoopHeadMBB);
   LoopTailMBB->addSuccessor(DoneMBB);
   DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
   DoneMBB->transferSuccessors(&MBB);
   MBB.addSuccessor(LoopHeadMBB);

   Register DestReg = MI.getOperand(0).getReg();
   Register Scratch1Reg = MI.getOperand(1).getReg();
   Register Scratch2Reg = MI.getOperand(2).getReg();
   Register AddrReg = MI.getOperand(3).getReg();
   Register IncrReg = MI.getOperand(4).getReg();
   Register MaskReg = MI.getOperand(5).getReg();
   bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
   AtomicOrdering Ordering =
       static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());

   //
   // .loophead:
   //   lr.w destreg, (alignedaddr)
   //   and scratch2, destreg, mask
   //   mv scratch1, destreg
   //   [sext scratch2 if signed min/max]
   //   ifnochangeneeded scratch2, incr, .looptail
   BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
       .addReg(AddrReg);
   BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
       .addReg(DestReg)
       .addReg(MaskReg);
   BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
       .addReg(DestReg)
       .addImm(0);

   switch (BinOp) {
   default:
     llvm_unreachable("Unexpected AtomicRMW BinOp");
   case AtomicRMWInst::Max: {
     insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
         .addReg(Scratch2Reg)
         .addReg(IncrReg)
         .addMBB(LoopTailMBB);
     break;
   }
   case AtomicRMWInst::Min: {
     insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
         .addReg(IncrReg)
         .addReg(Scratch2Reg)
         .addMBB(LoopTailMBB);
     break;
   }
   case AtomicRMWInst::UMax:
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
         .addReg(Scratch2Reg)
         .addReg(IncrReg)
         .addMBB(LoopTailMBB);
     break;
   case AtomicRMWInst::UMin:
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
         .addReg(IncrReg)
         .addReg(Scratch2Reg)
         .addMBB(LoopTailMBB);
     break;
   }

   // .loopifbody:
   //   xor scratch1, destreg, incr
   //   and scratch1, scratch1, mask
   //   xor scratch1, destreg, scratch1
   insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
                     MaskReg, Scratch1Reg);

   // .looptail:
   //   sc.w scratch1, scratch1, (addr)
   //   bnez scratch1, loop
   BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
       .addReg(AddrReg)
       .addReg(Scratch1Reg);
   BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
       .addReg(Scratch1Reg)
       .addReg(RISCV::X0)
       .addMBB(LoopHeadMBB);

   NextMBBI = MBB.end();
   MI.eraseFromParent();

   LivePhysRegs LiveRegs;
   computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
   computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
   computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
   computeAndAddLiveIns(LiveRegs, *DoneMBB);

   return true;
 }

 bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
     int Width, MachineBasicBlock::iterator &NextMBBI) {
   MachineInstr &MI = *MBBI;
   DebugLoc DL = MI.getDebugLoc();
   MachineFunction *MF = MBB.getParent();
   auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
   auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

   // Insert new MBBs.
   MF->insert(++MBB.getIterator(), LoopHeadMBB);
   MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
   MF->insert(++LoopTailMBB->getIterator(), DoneMBB);

   // Set up successors and transfer remaining instructions to DoneMBB.
   LoopHeadMBB->addSuccessor(LoopTailMBB);
   LoopHeadMBB->addSuccessor(DoneMBB);
   LoopTailMBB->addSuccessor(DoneMBB);
   LoopTailMBB->addSuccessor(LoopHeadMBB);
   DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
   DoneMBB->transferSuccessors(&MBB);
   MBB.addSuccessor(LoopHeadMBB);

   Register DestReg = MI.getOperand(0).getReg();
   Register ScratchReg = MI.getOperand(1).getReg();
   Register AddrReg = MI.getOperand(2).getReg();
   Register CmpValReg = MI.getOperand(3).getReg();
   Register NewValReg = MI.getOperand(4).getReg();
   AtomicOrdering Ordering =
       static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());

   if (!IsMasked) {
     // .loophead:
     //   lr.[w|d] dest, (addr)
     //   bne dest, cmpval, done
     BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
         .addReg(AddrReg);
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
         .addReg(DestReg)
         .addReg(CmpValReg)
         .addMBB(DoneMBB);
     // .looptail:
     //   sc.[w|d] scratch, newval, (addr)
     //   bnez scratch, loophead
     BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
         .addReg(AddrReg)
         .addReg(NewValReg);
     BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
         .addReg(ScratchReg)
         .addReg(RISCV::X0)
         .addMBB(LoopHeadMBB);
   } else {
     // .loophead:
     //   lr.w dest, (addr)
     //   and scratch, dest, mask
     //   bne scratch, cmpval, done
     Register MaskReg = MI.getOperand(5).getReg();
     BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
         .addReg(AddrReg);
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
         .addReg(DestReg)
         .addReg(MaskReg);
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
         .addReg(ScratchReg)
         .addReg(CmpValReg)
         .addMBB(DoneMBB);

     // .looptail:
     //   xor scratch, dest, newval
     //   and scratch, scratch, mask
     //   xor scratch, dest, scratch
     //   sc.w scratch, scratch, (adrr)
     //   bnez scratch, loophead
     insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
                       MaskReg, ScratchReg);
     BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
         .addReg(AddrReg)
         .addReg(ScratchReg);
     BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
         .addReg(ScratchReg)
         .addReg(RISCV::X0)
         .addMBB(LoopHeadMBB);
   }

   NextMBBI = MBB.end();
   MI.eraseFromParent();

   LivePhysRegs LiveRegs;
   computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
   computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
   computeAndAddLiveIns(LiveRegs, *DoneMBB);

   return true;
 }

 } // end of anonymous namespace

 INITIALIZE_PASS(RISCVExpandAtomicPseudo, "riscv-expand-atomic-pseudo",
                 RISCV_EXPAND_ATOMIC_PSEUDO_NAME, false, false)

 namespace llvm {

 FunctionPass *createRISCVExpandAtomicPseudoPass() {
   return new RISCVExpandAtomicPseudo();
 }

 } // end of namespace llvm
	//===-- RISCVExpandAtomicPseudoInsts.cpp - Expand atomic pseudo instrs. ---===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file contains a pass that expands atomic pseudo instructions into
	// target instructions. This pass should be run at the last possible moment,
	// avoiding the possibility for other passes to break the requirements for
	// forward progress in the LR/SC block.
	//
	//===----------------------------------------------------------------------===//

	#include "RISCV.h"
	#include "RISCVInstrInfo.h"
	#include "RISCVTargetMachine.h"

	#include "llvm/CodeGen/LivePhysRegs.h"
	#include "llvm/CodeGen/MachineFunctionPass.h"
	#include "llvm/CodeGen/MachineInstrBuilder.h"

	using namespace llvm;

	#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME \
	"RISCV atomic pseudo instruction expansion pass"

	namespace {

	class RISCVExpandAtomicPseudo : public MachineFunctionPass {
	public:
	const RISCVInstrInfo *TII;
	static char ID;

	RISCVExpandAtomicPseudo() : MachineFunctionPass(ID) {
	initializeRISCVExpandAtomicPseudoPass(*PassRegistry::getPassRegistry());
	}

	bool runOnMachineFunction(MachineFunction &MF) override;

	StringRef getPassName() const override {
	return RISCV_EXPAND_ATOMIC_PSEUDO_NAME;
	}

	private:
	bool expandMBB(MachineBasicBlock &MBB);
	bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandAtomicBinOp(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
	bool IsMasked, int Width,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	AtomicRMWInst::BinOp, bool IsMasked, int Width,
	MachineBasicBlock::iterator &NextMBBI);
	bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI, bool IsMasked,
	int Width, MachineBasicBlock::iterator &NextMBBI);
	};

	char RISCVExpandAtomicPseudo::ID = 0;

	bool RISCVExpandAtomicPseudo::runOnMachineFunction(MachineFunction &MF) {
	TII = static_cast<const RISCVInstrInfo *>(MF.getSubtarget().getInstrInfo());
	bool Modified = false;
	for (auto &MBB : MF)
	Modified \|= expandMBB(MBB);
	return Modified;
	}

	bool RISCVExpandAtomicPseudo::expandMBB(MachineBasicBlock &MBB) {
	bool Modified = false;

	MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
	while (MBBI != E) {
	MachineBasicBlock::iterator NMBBI = std::next(MBBI);
	Modified \|= expandMI(MBB, MBBI, NMBBI);
	MBBI = NMBBI;
	}

	return Modified;
	}

	bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
	MachineBasicBlock::iterator MBBI,
	MachineBasicBlock::iterator &NextMBBI) {
	// RISCVInstrInfo::getInstSizeInBytes expects that the total size of the
	// expanded instructions for each pseudo is correct in the Size field of the
	// tablegen definition for the pseudo.
	switch (MBBI->getOpcode()) {
	case RISCV::PseudoAtomicLoadNand32:
	return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
	NextMBBI);
	case RISCV::PseudoAtomicLoadNand64:
	return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
	NextMBBI);
	case RISCV::PseudoMaskedAtomicSwap32:
	return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
	NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadAdd32:
	return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, true, 32, NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadSub32:
	return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, true, 32, NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadNand32:
	return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, true, 32,
	NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadMax32:
	return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, true, 32,
	NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadMin32:
	return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, true, 32,
	NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadUMax32:
	return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, true, 32,
	NextMBBI);
	case RISCV::PseudoMaskedAtomicLoadUMin32:
	return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, true, 32,
	NextMBBI);
	case RISCV::PseudoCmpXchg32:
	return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
	case RISCV::PseudoCmpXchg64:
	return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
	case RISCV::PseudoMaskedCmpXchg32:
	return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
	}

	return false;
	}

	static unsigned getLRForRMW32(AtomicOrdering Ordering) {
	switch (Ordering) {
	default:
	llvm_unreachable("Unexpected AtomicOrdering");
	case AtomicOrdering::Monotonic:
	return RISCV::LR_W;
	case AtomicOrdering::Acquire:
	return RISCV::LR_W_AQ;
	case AtomicOrdering::Release:
	return RISCV::LR_W;
	case AtomicOrdering::AcquireRelease:
	return RISCV::LR_W_AQ;
	case AtomicOrdering::SequentiallyConsistent:
	return RISCV::LR_W_AQ_RL;
	}
	}

	static unsigned getSCForRMW32(AtomicOrdering Ordering) {
	switch (Ordering) {
	default:
	llvm_unreachable("Unexpected AtomicOrdering");
	case AtomicOrdering::Monotonic:
	return RISCV::SC_W;
	case AtomicOrdering::Acquire:
	return RISCV::SC_W;
	case AtomicOrdering::Release:
	return RISCV::SC_W_RL;
	case AtomicOrdering::AcquireRelease:
	return RISCV::SC_W_RL;
	case AtomicOrdering::SequentiallyConsistent:
	return RISCV::SC_W_AQ_RL;
	}
	}

	static unsigned getLRForRMW64(AtomicOrdering Ordering) {
	switch (Ordering) {
	default:
	llvm_unreachable("Unexpected AtomicOrdering");
	case AtomicOrdering::Monotonic:
	return RISCV::LR_D;
	case AtomicOrdering::Acquire:
	return RISCV::LR_D_AQ;
	case AtomicOrdering::Release:
	return RISCV::LR_D;
	case AtomicOrdering::AcquireRelease:
	return RISCV::LR_D_AQ;
	case AtomicOrdering::SequentiallyConsistent:
	return RISCV::LR_D_AQ_RL;
	}
	}

	static unsigned getSCForRMW64(AtomicOrdering Ordering) {
	switch (Ordering) {
	default:
	llvm_unreachable("Unexpected AtomicOrdering");
	case AtomicOrdering::Monotonic:
	return RISCV::SC_D;
	case AtomicOrdering::Acquire:
	return RISCV::SC_D;
	case AtomicOrdering::Release:
	return RISCV::SC_D_RL;
	case AtomicOrdering::AcquireRelease:
	return RISCV::SC_D_RL;
	case AtomicOrdering::SequentiallyConsistent:
	return RISCV::SC_D_AQ_RL;
	}
	}

	static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
	if (Width == 32)
	return getLRForRMW32(Ordering);
	if (Width == 64)
	return getLRForRMW64(Ordering);
	llvm_unreachable("Unexpected LR width\n");
	}

	static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
	if (Width == 32)
	return getSCForRMW32(Ordering);
	if (Width == 64)
	return getSCForRMW64(Ordering);
	llvm_unreachable("Unexpected SC width\n");
	}

	static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
	DebugLoc DL, MachineBasicBlock *ThisMBB,
	MachineBasicBlock *LoopMBB,
	MachineBasicBlock *DoneMBB,
	AtomicRMWInst::BinOp BinOp, int Width) {
	Register DestReg = MI.getOperand(0).getReg();
	Register ScratchReg = MI.getOperand(1).getReg();
	Register AddrReg = MI.getOperand(2).getReg();
	Register IncrReg = MI.getOperand(3).getReg();
	AtomicOrdering Ordering =
	static_cast<AtomicOrdering>(MI.getOperand(4).getImm());

	// .loop:
	// lr.[w\|d] dest, (addr)
	// binop scratch, dest, val
	// sc.[w\|d] scratch, scratch, (addr)
	// bnez scratch, loop
	BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
	.addReg(AddrReg);
	switch (BinOp) {
	default:
	llvm_unreachable("Unexpected AtomicRMW BinOp");
	case AtomicRMWInst::Nand:
	BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
	.addReg(DestReg)
	.addReg(IncrReg);
	BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
	.addReg(ScratchReg)
	.addImm(-1);
	break;
	}
	BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
	.addReg(AddrReg)
	.addReg(ScratchReg);
	BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
	.addReg(ScratchReg)
	.addReg(RISCV::X0)
	.addMBB(LoopMBB);
	}

	static void insertMaskedMerge(const RISCVInstrInfo *TII, DebugLoc DL,
	MachineBasicBlock *MBB, Register DestReg,
	Register OldValReg, Register NewValReg,
	Register MaskReg, Register ScratchReg) {
	assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
	assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
	assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");

	// We select bits from newval and oldval using:
	// https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
	// r = oldval ^ ((oldval ^ newval) & masktargetdata);
	BuildMI(MBB, DL, TII->get(RISCV::XOR), ScratchReg)
	.addReg(OldValReg)
	.addReg(NewValReg);
	BuildMI(MBB, DL, TII->get(RISCV::AND), ScratchReg)
	.addReg(ScratchReg)
	.addReg(MaskReg);
	BuildMI(MBB, DL, TII->get(RISCV::XOR), DestReg)
	.addReg(OldValReg)
	.addReg(ScratchReg);
	}

	static void doMaskedAtomicBinOpExpansion(
	const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
	MachineBasicBlock ThisMBB, MachineBasicBlock LoopMBB,
	MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
	assert(Width == 32 && "Should never need to expand masked 64-bit operations");
	Register DestReg = MI.getOperand(0).getReg();
	Register ScratchReg = MI.getOperand(1).getReg();
	Register AddrReg = MI.getOperand(2).getReg();
	Register IncrReg = MI.getOperand(3).getReg();
	Register MaskReg = MI.getOperand(4).getReg();
	AtomicOrdering Ordering =
	static_cast<AtomicOrdering>(MI.getOperand(5).getImm());

	// .loop:
	// lr.w destreg, (alignedaddr)
	// binop scratch, destreg, incr
	// xor scratch, destreg, scratch
	// and scratch, scratch, masktargetdata
	// xor scratch, destreg, scratch
	// sc.w scratch, scratch, (alignedaddr)
	// bnez scratch, loop
	BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
	.addReg(AddrReg);
	switch (BinOp) {
	default:
	llvm_unreachable("Unexpected AtomicRMW BinOp");
	case AtomicRMWInst::Xchg:
	BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
	.addReg(IncrReg)
	.addImm(0);
	break;
	case AtomicRMWInst::Add:
	BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
	.addReg(DestReg)
	.addReg(IncrReg);
	break;
	case AtomicRMWInst::Sub:
	BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
	.addReg(DestReg)
	.addReg(IncrReg);
	break;
	case AtomicRMWInst::Nand:
	BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
	.addReg(DestReg)
	.addReg(IncrReg);
	BuildMI(LoopMBB, DL, TII->get(RISCV::XORI), ScratchReg)
	.addReg(ScratchReg)
	.addImm(-1);
	break;
	}

	insertMaskedMerge(TII, DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
	ScratchReg);

	BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
	.addReg(AddrReg)
	.addReg(ScratchReg);
	BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
	.addReg(ScratchReg)
	.addReg(RISCV::X0)
	.addMBB(LoopMBB);
	}

	bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
	MachineBasicBlock::iterator &NextMBBI) {
	MachineInstr &MI = *MBBI;
	DebugLoc DL = MI.getDebugLoc();

	MachineFunction *MF = MBB.getParent();
	auto LoopMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

	// Insert new MBBs.
	MF->insert(++MBB.getIterator(), LoopMBB);
	MF->insert(++LoopMBB->getIterator(), DoneMBB);

	// Set up successors and transfer remaining instructions to DoneMBB.
	LoopMBB->addSuccessor(LoopMBB);
	LoopMBB->addSuccessor(DoneMBB);
	DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
	DoneMBB->transferSuccessors(&MBB);
	MBB.addSuccessor(LoopMBB);

	if (!IsMasked)
	doAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp, Width);
	else
	doMaskedAtomicBinOpExpansion(TII, MI, DL, &MBB, LoopMBB, DoneMBB, BinOp,
	Width);

	NextMBBI = MBB.end();
	MI.eraseFromParent();

	LivePhysRegs LiveRegs;
	computeAndAddLiveIns(LiveRegs, *LoopMBB);
	computeAndAddLiveIns(LiveRegs, *DoneMBB);

	return true;
	}

	static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
	MachineBasicBlock *MBB, Register ValReg,
	Register ShamtReg) {
	BuildMI(MBB, DL, TII->get(RISCV::SLL), ValReg)
	.addReg(ValReg)
	.addReg(ShamtReg);
	BuildMI(MBB, DL, TII->get(RISCV::SRA), ValReg)
	.addReg(ValReg)
	.addReg(ShamtReg);
	}

	bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
	AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
	MachineBasicBlock::iterator &NextMBBI) {
	assert(IsMasked == true &&
	"Should only need to expand masked atomic max/min");
	assert(Width == 32 && "Should never need to expand masked 64-bit operations");

	MachineInstr &MI = *MBBI;
	DebugLoc DL = MI.getDebugLoc();
	MachineFunction *MF = MBB.getParent();
	auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

	// Insert new MBBs.
	MF->insert(++MBB.getIterator(), LoopHeadMBB);
	MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
	MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
	MF->insert(++LoopTailMBB->getIterator(), DoneMBB);

	// Set up successors and transfer remaining instructions to DoneMBB.
	LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
	LoopHeadMBB->addSuccessor(LoopTailMBB);
	LoopIfBodyMBB->addSuccessor(LoopTailMBB);
	LoopTailMBB->addSuccessor(LoopHeadMBB);
	LoopTailMBB->addSuccessor(DoneMBB);
	DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
	DoneMBB->transferSuccessors(&MBB);
	MBB.addSuccessor(LoopHeadMBB);

	Register DestReg = MI.getOperand(0).getReg();
	Register Scratch1Reg = MI.getOperand(1).getReg();
	Register Scratch2Reg = MI.getOperand(2).getReg();
	Register AddrReg = MI.getOperand(3).getReg();
	Register IncrReg = MI.getOperand(4).getReg();
	Register MaskReg = MI.getOperand(5).getReg();
	bool IsSigned = BinOp == AtomicRMWInst::Min \|\| BinOp == AtomicRMWInst::Max;
	AtomicOrdering Ordering =
	static_cast<AtomicOrdering>(MI.getOperand(IsSigned ? 7 : 6).getImm());

	//
	// .loophead:
	// lr.w destreg, (alignedaddr)
	// and scratch2, destreg, mask
	// mv scratch1, destreg
	// [sext scratch2 if signed min/max]
	// ifnochangeneeded scratch2, incr, .looptail
	BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
	.addReg(AddrReg);
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), Scratch2Reg)
	.addReg(DestReg)
	.addReg(MaskReg);
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), Scratch1Reg)
	.addReg(DestReg)
	.addImm(0);

	switch (BinOp) {
	default:
	llvm_unreachable("Unexpected AtomicRMW BinOp");
	case AtomicRMWInst::Max: {
	insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
	.addReg(Scratch2Reg)
	.addReg(IncrReg)
	.addMBB(LoopTailMBB);
	break;
	}
	case AtomicRMWInst::Min: {
	insertSext(TII, DL, LoopHeadMBB, Scratch2Reg, MI.getOperand(6).getReg());
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
	.addReg(IncrReg)
	.addReg(Scratch2Reg)
	.addMBB(LoopTailMBB);
	break;
	}
	case AtomicRMWInst::UMax:
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
	.addReg(Scratch2Reg)
	.addReg(IncrReg)
	.addMBB(LoopTailMBB);
	break;
	case AtomicRMWInst::UMin:
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
	.addReg(IncrReg)
	.addReg(Scratch2Reg)
	.addMBB(LoopTailMBB);
	break;
	}

	// .loopifbody:
	// xor scratch1, destreg, incr
	// and scratch1, scratch1, mask
	// xor scratch1, destreg, scratch1
	insertMaskedMerge(TII, DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
	MaskReg, Scratch1Reg);

	// .looptail:
	// sc.w scratch1, scratch1, (addr)
	// bnez scratch1, loop
	BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
	.addReg(AddrReg)
	.addReg(Scratch1Reg);
	BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
	.addReg(Scratch1Reg)
	.addReg(RISCV::X0)
	.addMBB(LoopHeadMBB);

	NextMBBI = MBB.end();
	MI.eraseFromParent();

	LivePhysRegs LiveRegs;
	computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
	computeAndAddLiveIns(LiveRegs, *LoopIfBodyMBB);
	computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
	computeAndAddLiveIns(LiveRegs, *DoneMBB);

	return true;
	}

	bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
	MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
	int Width, MachineBasicBlock::iterator &NextMBBI) {
	MachineInstr &MI = *MBBI;
	DebugLoc DL = MI.getDebugLoc();
	MachineFunction *MF = MBB.getParent();
	auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
	auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

	// Insert new MBBs.
	MF->insert(++MBB.getIterator(), LoopHeadMBB);
	MF->insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
	MF->insert(++LoopTailMBB->getIterator(), DoneMBB);

	// Set up successors and transfer remaining instructions to DoneMBB.
	LoopHeadMBB->addSuccessor(LoopTailMBB);
	LoopHeadMBB->addSuccessor(DoneMBB);
	LoopTailMBB->addSuccessor(DoneMBB);
	LoopTailMBB->addSuccessor(LoopHeadMBB);
	DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
	DoneMBB->transferSuccessors(&MBB);
	MBB.addSuccessor(LoopHeadMBB);

	Register DestReg = MI.getOperand(0).getReg();
	Register ScratchReg = MI.getOperand(1).getReg();
	Register AddrReg = MI.getOperand(2).getReg();
	Register CmpValReg = MI.getOperand(3).getReg();
	Register NewValReg = MI.getOperand(4).getReg();
	AtomicOrdering Ordering =
	static_cast<AtomicOrdering>(MI.getOperand(IsMasked ? 6 : 5).getImm());

	if (!IsMasked) {
	// .loophead:
	// lr.[w\|d] dest, (addr)
	// bne dest, cmpval, done
	BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
	.addReg(AddrReg);
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
	.addReg(DestReg)
	.addReg(CmpValReg)
	.addMBB(DoneMBB);
	// .looptail:
	// sc.[w\|d] scratch, newval, (addr)
	// bnez scratch, loophead
	BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
	.addReg(AddrReg)
	.addReg(NewValReg);
	BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
	.addReg(ScratchReg)
	.addReg(RISCV::X0)
	.addMBB(LoopHeadMBB);
	} else {
	// .loophead:
	// lr.w dest, (addr)
	// and scratch, dest, mask
	// bne scratch, cmpval, done
	Register MaskReg = MI.getOperand(5).getReg();
	BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
	.addReg(AddrReg);
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
	.addReg(DestReg)
	.addReg(MaskReg);
	BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
	.addReg(ScratchReg)
	.addReg(CmpValReg)
	.addMBB(DoneMBB);

	// .looptail:
	// xor scratch, dest, newval
	// and scratch, scratch, mask
	// xor scratch, dest, scratch
	// sc.w scratch, scratch, (adrr)
	// bnez scratch, loophead
	insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
	MaskReg, ScratchReg);
	BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
	.addReg(AddrReg)
	.addReg(ScratchReg);
	BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
	.addReg(ScratchReg)
	.addReg(RISCV::X0)
	.addMBB(LoopHeadMBB);
	}

	NextMBBI = MBB.end();
	MI.eraseFromParent();

	LivePhysRegs LiveRegs;
	computeAndAddLiveIns(LiveRegs, *LoopHeadMBB);
	computeAndAddLiveIns(LiveRegs, *LoopTailMBB);
	computeAndAddLiveIns(LiveRegs, *DoneMBB);

	return true;
	}

	} // end of anonymous namespace

	INITIALIZE_PASS(RISCVExpandAtomicPseudo, "riscv-expand-atomic-pseudo",
	RISCV_EXPAND_ATOMIC_PSEUDO_NAME, false, false)

	namespace llvm {

	FunctionPass *createRISCVExpandAtomicPseudoPass() {
	return new RISCVExpandAtomicPseudo();
	}

	} // end of namespace llvm