PerfApi.cpp

// Copyright 2024 Tenstorrent Corporation.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <algorithm>
#include <cinttypes>
#include <iomanip>
#include <ranges>
#include "PerfApi.hpp"

using namespace TT_PERF;

using CSRN = WdRiscv::CsrNumber;
using std::cerr;

// Exclude the rounding/flag fields of the combined FP/vector CSRs (fcsr = frm|fflags,
// vcsr = vxrm|vxsat) from speculative producer-consumer CSR forwarding. A forwarded rounding field
// gets re-injected into architectural state when a younger FP/vector op writes the sticky flag
// (fflags/vxsat): that puts the combined CSR in lastWrittenRegs_, so execute()'s "undo" re-pokes it
// from a prevValue still holding the forwarded value. An older retiring csrrs then reads the leaked
// value and diverges from the reference trace. Standalone CSRs (VL/VTYPE/VSTART/VLENB) lack this
// coupling and are forwarded normally.
static inline bool isNonForwardableCsr(unsigned csrNum)
{
  switch (WdRiscv::CsrNumber(csrNum))
    {
    case CSRN::FFLAGS: case CSRN::FRM:   case CSRN::FCSR:   // fcsr group
    case CSRN::VXSAT:  case CSRN::VXRM:  case CSRN::VCSR:   // vcsr group
      return true;
    default:
      return false;
    }
}

// True if hardware can update csrNum without an instruction writing it, so a read at speculative
// execute and a read at retire may legitimately differ. See the use site in retire().
//
// Do not add: MTOPEI/STOPEI/VSTOPEI (reading them *claims* the interrupt -- not passive reads);
// SEED (fresh entropy per read unless --reproducibleseed); MSTATUS/SSTATUS (FS/VS/SD do
// drift, but both are written often and a forced rd would mask a write's real divergence).
static inline bool isAsyncUpdatedCsr(unsigned csrNum)
{
  using CN = WdRiscv::CsrNumber;
  auto inRange = [&](CN lo, CN hi) { return csrNum >= unsigned(lo) and csrNum <= unsigned(hi); };

  switch (WdRiscv::CsrNumber(csrNum))
    {
    case CN::MIP:   case CN::SIP:   case CN::VSIP:      // interrupt-pending, set by hardware
    case CN::HIP:   case CN::HVIP:  case CN::MVIP:
    case CN::MTOPI: case CN::STOPI: case CN::VSTOPI:    // AIA top interrupt, recomputed
    case CN::SCOUNTOVF:                                 // Sscofpmf overflow (sets MIP.LCOFIP)
      return true;
    default:                                            // free-running counters, one block each
      return inRange(CN::CYCLE,   CN::HPMCOUNTER31)  or   // 0xc00-0xc1f  cycle/time/instret/hpm
             inRange(CN::CYCLEH,  CN::HPMCOUNTER31H) or   // 0xc80-0xc9f  RV32 high halves
             inRange(CN::MCYCLE,  CN::MHPMCOUNTER31) or   // 0xb00-0xb1f  machine
             inRange(CN::MCYCLEH, CN::MHPMCOUNTER31H);    // 0xb80-0xb9f  machine, RV32 high
    }
}

// True if csrNum is an AIA/Smcsrind indirect data window (MIREG*/SIREG*/VSIREG*): a CSR with no
// storage of its own that aliases whichever register its addressing context selects.
static inline bool isIndirectCsrWindow(unsigned csrNum)
{
  using CN = WdRiscv::CsrNumber;
  auto inRange = [&](CN lo, CN hi) { return csrNum >= unsigned(lo) and csrNum <= unsigned(hi); };
  return inRange(CN::MIREG,  CN::MIREG6)  or
         inRange(CN::SIREG,  CN::SIREG6)  or
         inRange(CN::VSIREG, CN::VSIREG6);
}

template <typename URV>
PerfApi<URV>::PerfApi(SystemType& system)
  : system_(system)
{
  unsigned n = system.hartCount();

  traceFiles_.resize(n);

  hartPacketMaps_.resize(n);
  hartStoreMaps_.resize(n);
  hartLastRetired_.resize(n, initHartLastRetired);
  hartRegProducers_.resize(n);
  hartSpecLrs_.resize(n);
  hartSpecCsrs_.resize(n);

  for (auto& producers : hartRegProducers_)
    producers.resize(totalRegCount_);

  hartRawPtrs_.reserve(n);
  for (unsigned i = 0; i < n; ++i)
    hartRawPtrs_.push_back(system_.ithHart(i).get());
}


template <typename URV>
std::shared_ptr<typename PerfApi<URV>::HartType>
PerfApi<URV>::checkHart(const char* caller, unsigned hartIx)
{
  auto hart = getHart(hartIx);
  if (not hart)
    {
      cerr << caller << ": Bad hart index: " << hartIx << '\n';
      assert(0 && "Error: Assertion failed -- bad hart index");
    }
  return hart;
}


template <typename URV>
typename PerfApi<URV>::HartType*
PerfApi<URV>::checkHartRaw(const char* caller, unsigned hartIx)
{
  if (hartIx >= hartRawPtrs_.size()) [[unlikely]]
    {
      cerr << caller << ": Bad hart index: " << hartIx << '\n';
      assert(0 && "Error: Assertion failed -- bad hart index");
      return nullptr;
    }
  return hartRawPtrs_[hartIx];
}


template <typename URV>
std::shared_ptr<InstrPac>
PerfApi<URV>::checkTag(const char* caller, unsigned hartIx, uint64_t tag)
{
  auto packetPtr = hartPacketMaps_[hartIx].findShared(tag);
  if (packetPtr)
    return packetPtr;
  cerr << caller << ": Unknown tag (never fetched): " << tag << '\n';
  assert(0 && "Error: Assertion failed -- unknown tag");
  return nullptr;
}


template <typename URV>
InstrPac*
PerfApi<URV>::checkTagRaw(const char* caller, unsigned hartIx, uint64_t tag)
{
  auto* packet = hartPacketMaps_[hartIx].find(tag);
  if (packet)
    return packet;
  cerr << caller << ": Unknown tag (never fetched): " << tag << '\n';
  assert(0 && "Error: Assertion failed -- unknown tag");
  return nullptr;
}


template <typename URV>
bool
PerfApi<URV>::checkTime(const char* caller, uint64_t time)
{
  if (time < time_)
    {
      cerr << caller << ": Bad time: " << time << '\n';
      assert(0 && "Error: Assertion failed -- bad time value");
      return false;
    }
  time_ = time;
  return true;
}


template <typename URV>
bool
PerfApi<URV>::fetch(unsigned hartIx, uint64_t time, uint64_t tag, uint64_t vpc,
                    bool& trap, ExceptionCause& cause, uint64_t& trapPc)
{
  if (commandLog_) [[unlikely]]
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_fetch %" PRIu64 " 0x%" PRIx64 "\n",
            hartIx, time, tag, vpc);

  auto* hart = checkHartRaw("Fetch", hartIx);
  if (not hart) [[unlikely]]
    return false;

  if (not checkTime("Fetch", time)) [[unlikely]]
    return false;

  if (tag == 0) [[unlikely]]
    {
      cerr << "Error: PerfApi::fetch: Hart-ix=" << hartIx << " tag=" << tag
                << " zero tag is reserved.\n";
      assert(0 && "Error: Assertion failed -- zero tag");
      return false;
    }

  auto& packetMap = hartPacketMaps_[hartIx];
  if (packetMap.wouldBeOutOfOrder(tag)) [[unlikely]]
    {
      cerr << "Error: PerfApi::fetch: Hart-ix=" << hartIx << "tag=" << tag
                << " tag is not in increasing order.\n";
      assert(0 && "Error: Assertion failed -- tag out of order");
      return false;
    }

  auto* existingPacket = getInstructionPacketRaw(hartIx, tag);
  if (existingPacket) [[unlikely]]
    {
      cerr << "Error: PerfApi::fetch: Hart-ix=" << hartIx << "tag=" << tag
                << " tag is already fetched.\n";
      return false;   // Tag already fetched.
    }

  auto prev = this->prevFetch_;
  if (prev and not prev->predicted() and not prev->trapped() and not prev->executed())
    {
      bool sequential = prev->instrVa() + prev->decodedInst().instSize() == vpc;
      if (not prev->decodedInst().isBranch())
        {
          if (not sequential)
            prev->predictBranch(true, vpc);
        }
      else
        {
          if (sequential)
            prev->predictBranch(false, vpc);
        }
    }

  uint64_t ppc = 0, ppc2 = 0;  // Physical pc.
  uint32_t opcode = 0;
  uint64_t gpc = 0; // Guest physical pc.
  cause = hart->fetchInstNoTrap(vpc, ppc, ppc2, gpc, opcode);  // hart is now raw pointer

  auto packet = std::make_shared<InstrPac>(tag, vpc, ppc, ppc2);
  assert(packet);
  packet->fetched_ = true;
  packet->opcode_= opcode;
  insertPacket(hartIx, tag, packet);
  if (not decode(hartIx, time, tag))
    assert(0 && "Error: Assertion failed -- failed to decode");
  prevFetch_ = packet;

  trap = cause != ExceptionCause::NONE;
  packet->trap_ = trap;
  packet->trapCause_ = uint64_t(cause);

  if (prev and not prev->trapped() and prev->executed() and prev->nextIva_ != vpc)
    {
      packet->shouldFlush_ = true;
      packet->flushVa_ = prev->nextIva_;
    }

  if (trap)
    {
      prevFetch_ = nullptr;
      trapPc = 0;
    }

  return true;
}


template <typename URV>
bool
PerfApi<URV>::decode(unsigned hartIx, uint64_t time, uint64_t tag)
{
  if (commandLog_) [[unlikely]]
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_decode %" PRIu64 "\n",
            hartIx, time, tag);

  if (not checkTime("Decode", time)) [[unlikely]]
    return false;

  auto* hartPtr = checkHartRaw("Decode", hartIx);
  if (not hartPtr) [[unlikely]]
    return false;
  auto& hart = *hartPtr;

  const auto packetSharedPtr = hartPacketMaps_[hartIx].findShared(tag);
  if (not packetSharedPtr) [[unlikely]]
    {
      cerr << "Decode: Unknown tag (never fetched): " << tag << '\n';
      assert(0 && "Error: Assertion failed -- unknown tag");
      return false;
    }

  auto& packet = *packetSharedPtr;

  if (packet.decoded())
    return true;

  hart.decode(packet.instrVa(), packet.instrPa(), packet.opcode_, packet.di_);
  packet.decoded_ = true;

  using OM = WdRiscv::OperandMode;
  using OT = WdRiscv::OperandType;

  determineExplicitOperands(packet);

  // Determine effective group multiplier of vector operands. We do this before adding
  // explicit operands as we may be producing vtype which affects LMUL.
  if (packet.di_.isVector())
    getVectorOperandsLmul(hart, packet);

  determineImplicitOperands(packet);

  // Collect producers of operands of this instruction.
  auto& producers = hartRegProducers_[hartIx];
  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  const unsigned operandCount = packet.operandCount_;
  for (unsigned i = 0; i < operandCount; ++i)
    {
      auto& op = packet.operands_[i];
      auto mode = op.mode;
      auto type = op.type;

      if (type == OT::Imm)
        continue;

      assert(type != OT::None and mode != OM::None);
      if (mode == OM::None)
        continue;

      unsigned regNum = op.number;
      unsigned gri = globalRegIx(type, regNum);

      if (type != OT::VecReg)
        packet.opProducers_[i].scalar = producers[gri];
      else
        {
          auto lmul = op.lmul;
          assert(lmul != 0 and lmul <= maxEffLmul_);
          auto& vec = packet.opProducers_[i].vec;
          vec.reserve(lmul);
          for (unsigned n = 0; n < lmul; ++n)
            {
              unsigned vgri = gri + n;
              vec.push_back(producers[vgri]);
            }
        }
    }

  // Mark this instruction as the producer of each of its destination registers.
  for (unsigned i = 0; i < operandCount; ++i)
    {
      auto& op = packet.operands_[i];
      auto mode = op.mode;
      if (mode == OM::Write or mode == OM::ReadWrite)
        {
          unsigned regNum = op.number;
          auto type = op.type;
          unsigned gri = globalRegIx(type, regNum);
          if (regNum == 0 and type == OT::IntReg)
            continue;  // Reg X0 has no producer

          if (type != OT::VecReg)
            producers[gri] = packetSharedPtr;
          else
            {
              auto lmul = op.lmul;
              assert(lmul <= maxEffLmul_);
              for (unsigned n = 0; n < lmul; ++n)
                {
                  unsigned vgri = gri + n;
                  producers[vgri] = packetSharedPtr;
                }
            }
        }
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)

  return true;
}


template <typename URV>
std::vector<typename PerfApi<URV>::CsrFwdSave>
PerfApi<URV>::pushSpecCsrContext(HartType& hart, unsigned hartIx, uint64_t tag,
                                 const InstrPac* packet)
{
  std::vector<CsrFwdSave> saves;

  const auto& specCsrs = hartSpecCsrs_.at(hartIx);
  if (specCsrs.empty())
    return saves;

  auto isPacketOperand = [packet](unsigned csrNum) {
    if (not packet)
      return false;
    for (unsigned i = 0; i < packet->operandCount_; ++i)
      {
        const auto& op = packet->operands_[i];  // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index)
        if (op.type == OperandType::CsReg and op.number == csrNum)
          return true;
      }
    return false;
  };

  saves.reserve(specCsrs.size());
  for (const auto& entry : specCsrs)
    {
      if (entry.tag >= tag) continue;                  // forward only OLDER in-flight writes
      if (isPacketOperand(entry.csrNum)) continue;     // owned by set/restoreHartValues
      if (isNonForwardableCsr(entry.csrNum)) continue; // FP/vector-format CSRs: see isNonForwardableCsr
      URV prev{};
      bool ok = hart.peekCsr(CSRN(entry.csrNum), prev);
      saves.push_back({entry.csrNum, prev, ok});
      if (ok) hart.pokeCsr(CSRN(entry.csrNum), URV(entry.newVal));
    }

  return saves;
}


template <typename URV>
void
PerfApi<URV>::popSpecCsrContext(HartType& hart, const std::vector<CsrFwdSave>& saves)
{
  for (auto it = saves.rbegin(); it != saves.rend(); ++it)
    if (it->ok)
      hart.pokeCsr(CSRN(it->csrNum), it->prev);
}


template <typename URV>
bool
PerfApi<URV>::execute(unsigned hartIx, uint64_t time, uint64_t tag)
{
  if (commandLog_) [[unlikely]]
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_execute %" PRIu64 "\n",
            hartIx, time, tag);

  if (not checkTime("Execute", time)) [[unlikely]]
    return false;

  auto* hartPtr = checkHartRaw("Execute", hartIx);
  if (not hartPtr) [[unlikely]]
    return false;

  auto& hart = *hartPtr;

  // Use findPacket to get both raw pointer and iterator, avoiding redundant lookups
  auto lookupResult = findPacket(hartIx, tag);
  if (not lookupResult) [[unlikely]]
    {
      cerr << "execute: Unknown tag (never fetched): " << tag << '\n';
      assert(0 && "Error: Assertion failed -- unknown tag");
      return false;
    }

  auto* pacPtr = lookupResult.packet;
  auto& packet = *pacPtr;
  const auto& di = packet.decodedInst();

#if 0
  if (di.isLr())
    return true;   // LR is executed and retired at PerApi::retire.
#endif

  auto& packetMap = hartPacketMaps_[hartIx];

  if (packet.executed()) [[unlikely]]
    {
      // Instruction is being re-executed. Must be load/store. Every instruction that
      // depends on it must be re-executed.
      if (not di.isLoad() and not di.isStore())
        assert(0 && "Error: Assertion failed -- re-executed instruction is not ld/store");
      // Iterate over packets starting from current tag
      for (auto iter = packetMap.iteratorAt(tag); iter != packetMap.end(); ++iter)
        {
          auto* succ = iter.packet();
          if (succ->dependsOn(packet))
            succ->executed_ = false;
        }
    }

  // Indirect-CSR (AIA/Smcsrind) context pre-forwarding. MIREG*/SIREG*/VSIREG* alias whichever
  // register their addressing context (the paired *ISELECT, plus hstatus.VGEIN for VSIREG*)
  // selects. execute() forwards older in-flight CSR writes, but only after collectOperandValues()
  // and its operand save -- too late for a window operand, which then resolves under a stale
  // context and trips the save/set assert. So when an indirect window is accessed, forward the
  // older-spec-CSR context now across both calls and undo it afterwards; other instructions are
  // unaffected.
  bool hasIndirectWindow = std::ranges::any_of(packet.operands_ | std::views::take(packet.operandCount_),
                                               [](const auto& op) {
                                                 return op.type == OperandType::CsReg and isIndirectCsrWindow(op.number);
                                               });
  SpecCsrContext ctxFwds(hart, hasIndirectWindow
                                 ? pushSpecCsrContext(hart, hartIx, packet.tag_, &packet)
                                 : std::vector<CsrFwdSave>{});

  // Collect register operand values. Some values come from in-flight instructions
  // (register renaming).
  bool peekOk = collectOperandValues(hart, packet);

  // Execute the instruction: Poke source register values, execute, recover destination
  // register values.
  if (not execute(hartIx, packet))
    assert(0 && "Error: Assertion failed -- failed to execute instruction");

  ctxFwds.pop();

  // We should not fail to read an operand value unless there is an exception.
  if (not peekOk)
    assert(packet.trap_);

  packet.executed_ = true;
  packet.execTime_ = time;
  if (packet.predicted_)
    packet.mispredicted_ = packet.prTarget_ != packet.nextIva_;

  if (packet.isBranch())
    {
      // Check if the next instruction in program order is at the right PC.
      auto* nextPacket = packetMap.find(tag + 1);
      if (nextPacket)
        {
          auto& next = *nextPacket;
          if (next.iva_ != packet.nextIva_)
            {
              next.shouldFlush_ = true;
              next.flushVa_ = packet.nextIva_;
            }
          if (next.executed_)
            {
              packet.shouldFlush_ = true;
              packet.flushVa_ = packet.iva_;
            }
        }
    }

  return true;
}


template <typename URV>
bool
PerfApi<URV>::execute(unsigned hartIx, InstrPac& packet)
{
  assert(packet.decoded());

  auto* hartPtr = getHartRaw(hartIx);
  assert(hartPtr);
  auto& hart = *hartPtr;

  uint64_t prevPc = hart.peekPc();
  uint64_t prevInstrCount = hart.getInstructionCount();

  hart.pokePc(packet.instrVa());
  hart.setInstructionCount(packet.tag_ - 1);

  // FIX: handle MSTATUSH
  URV prevMstatus = 0;
  if (not hart.peekCsr(CSRN::MSTATUS, prevMstatus))
    assert(0 && "Error: Assertion failed");

  // Architectural operand values; must precede the spec-CSR overlay.
  std::array<OpVal, 9> prevVal;
  bool saveOk = saveHartValues(hart, packet, prevVal);

  // Before setHartValues: pokeCsr ignores VTYPE/VL/VSTART while VS is Off.
  SpecCsrContext csrFwdSaves(hart, pushSpecCsrContext(hart, hartIx, packet.tag_, &packet));

  // Install packet operand values (some obtained from previous in-flight instructions)
  // into the hart registers.
  bool setOk = setHartValues(hart, packet);

  assert(hartLastRetired_[hartIx] == initHartLastRetired or packet.tag_ > hartLastRetired_[hartIx]);

  // Only make the time adjustment when the hart has retired at least one instruction.
  if (hartLastRetired_[hartIx] != initHartLastRetired)
    {
      auto itag = std::bit_cast<int64_t>(packet.tag_);
      auto ilast = std::bit_cast<int64_t>(hartLastRetired_[hartIx]);
      assert(itag >= 0 and ilast >= 0);  // Check overflow
      hart.adjustTime(itag - ilast - 1);
    }

  const auto& di = packet.decodedInst();

  unsigned imsicId = 0, imsicGuest = 0;
  if (di.isCsr())
    saveImsicTopei(hart, CSRN(di.ithOperand(2)), imsicId, imsicGuest);

  packet.privMode_ = hart.privilegeMode();
  packet.virtMode_ = hart.virtMode();

  // Save current reservation state before executing an LR so we can restore it after.
  if (di.isLr())
    packet.hadPriorLr_ = hart.getLr(packet.savedLrAddr_, packet.savedLrSize_);

  // Before speculative SC execution, set the reservation from the paired LR (which may
  // differ from what is currently in Memory due to other speculative LR executions).
  uint64_t scPrevLrAddr = 0;
  unsigned scPrevLrSize = 0;
  bool scHadPrevLr = false;
  if (di.isSc())
    {
      scHadPrevLr = hart.getLr(scPrevLrAddr, scPrevLrSize);
      auto& specLrs = hartSpecLrs_[hartIx];
      for (auto it = specLrs.rbegin(); it != specLrs.rend(); ++it)
	if (it->tag < packet.tag_)
	  {
	    hart.makeLr(it->addr, it->size);
	    break;
	  }
    }

  // Execute
  uint64_t execPreLrAddr = 0; unsigned execPreLrSize = 0;
  bool execPreHadLr = hart.getLr(execPreLrAddr, execPreLrSize);

  skipIoLoad_ = true;   // Load from IO space takes effect at retire.
  hart.singleStep();
  skipIoLoad_ = false;

  bool trap = hart.lastInstructionTrapped();
  packet.trap_ = trap;
  packet.interrupt_ = hart.lastInstructionInterrupted();

  // After speculative LR execution, save its reservation to the spec list and then undo
  // the makeLr so that Memory always reflects the architectural (retired) reservation.
  if (di.isLr() and not trap)
    {
      uint64_t lrAddr = 0;
      unsigned lrSize = 0;
      if (hart.getLr(lrAddr, lrSize))
	hartSpecLrs_[hartIx].push_back({packet.tag_, lrAddr, lrSize});
      if (packet.hadPriorLr_)
	hart.makeLr(packet.savedLrAddr_, packet.savedLrSize_);
      else
	hart.cancelLr(WdRiscv::CancelLrCause::FLUSH);
    }

  // For non-LR/non-SC instructions, singleStep may still mutate the hart's LR
  // reservation (notably Zwrs WRS.NTO, which clears the reservation as part of its
  // architectural semantics). Without restoring, the speculative reservation clear
  // leaks into architectural state and a subsequent SC retire spuriously fails.
  // LR/SC already manage the reservation themselves via the blocks above.
  if (not di.isLr() and not di.isSc()) {
    if (execPreHadLr) hart.makeLr(execPreLrAddr, execPreLrSize);
    else hart.cancelLr(WdRiscv::CancelLrCause::FLUSH);
  }

  // After speculative SC execution, capture the outcome on the packet so the timing
  // model can dispatch a store-buffer entry with the predicted result, then restore
  // the previous reservation state so Memory reflects only architectural (retired) state.
  if (di.isSc())
    {
      if (not trap)
        recordScOutcome(hart, packet);

      if (scHadPrevLr)
	hart.makeLr(scPrevLrAddr, scPrevLrSize);
      else
	hart.cancelLr(WdRiscv::CancelLrCause::FLUSH);
    }

  if (trap)
    packet.trapCause_ = hart.lastTrapCause();

  // If save fails or set fails, there must be a trap.
  if (not saveOk or not setOk)
    {
      if (not trap)
        cerr << "PerfApi::execute: save/set operands failed for instruction tag " << packet.tag_ << '\n';
      assert(trap);
    }

  // Record PC of subsequent packet.
  packet.nextIva_ = hart.peekPc();

  recordExecutionResults(hart, packet);

  // CSR-write speculative outcome capture. Snapshot rd and the new CSR value while
  // the hart is still in the post-singleStep state (before restoreHartValues wipes
  // the CSR back to its pre-execute value). peekCsr gives the post-execute CSR
  // value; rd already lives in destValues_. Done unconditionally for every
  // non-trapping CSR op; retire() reconciles it for writes, the model decides
  // whether to demote the write via FNSR or rely on the speculation path.
  if (di.isCsr() and not trap)
    {
      const unsigned csrNum = di.ithOperand(2);
      packet.csrExecuted_ = true;
      packet.csrNum_      = csrNum;
      URV csrVal{};
      if (hart.peekCsr(CSRN(csrNum), csrVal))
        packet.csrExecNewVal_ = csrVal;
      if (di.op0() != 0)
        packet.csrExecRd_ = packet.destValues_.at(0).second.scalar;
    }

  // Undo changes to the hart.

  if (hartLastRetired_[hartIx] != initHartLastRetired)
    hart.adjustTime(-(int64_t)(packet.tag_ - hartLastRetired_[hartIx]));  // Restore timer value.
  else
    hart.adjustTime(-1); // untick one cycle for the first instruction

  // Restore CSRs modified by the instruction or trap. TODO: For vector ld/st we have to
  // restore partially modified vectors.
  if (di.isXRet() and not trap)
    {   // For an MRET/SRET/... Privilege may have been lowered. Restore it before restoring CSRs.
      hart.setVirtualMode(hart.lastVirtMode());
      hart.setPrivilegeMode(hart.lastPrivMode());
    }

  // Restore CSR changes due to a trap or to mret/sret or to side effects from
  // vector/fp instructions (MSTATUS.VS, MSTATUS.FS, FCSR, etc...).
  static thread_local std::vector<CSRN> csrns;
  csrns.clear();
  hart.lastCsr(csrns);
  for (auto csrn : csrns)
    {
      URV value = hart.lastCsrValue(csrn);
      if (not hart.pokeCsr(csrn, value))
        assert(0);
      if (trap or di.isXRet())
        recordChangedCsr(hart, packet, csrn);
    }

  if (trap)
    {  // Privilege raised.  Restore it after restoring CSRs.
      hart.setVirtualMode(hart.lastVirtMode());
      hart.setPrivilegeMode(hart.lastPrivMode());
    }

  // Restore hart registers that we changed before single step.
  restoreHartValues(hart, packet, prevVal);

  URV mstatus = 0;
  if (not hart.peekCsr(CSRN::MSTATUS, mstatus))
    assert(0 && "Error: Assertion failed");
  if (mstatus != prevMstatus)
    hart.pokeCsr(CSRN::MSTATUS, prevMstatus);

  if (di.isCsr())
    restoreImsicTopei(hart, CSRN(di.ithOperand(2)), imsicId, imsicGuest);

  // Back to architectural CSR state for the rest of the recovery below.
  csrFwdSaves.pop();

  // If this instruction is a CSR *write* that captured a predicted outcome, push
  // it onto hartSpecCsrs_ so younger speculative consumers see the forwarded
  // value. Read-only CSR ops (csrrs/c with rs1=x0) don't modify the CSR — pushing
  // them would forward an irrelevant value (e.g. the current TIME) and cause
  // spurious CsrMispredict at retire. The CSR operand at index 2 is ReadWrite for
  // writes (csrrw and csrrs/c with rs1!=0) and Read for read-only forms.
  if (di.isCsr() and not trap and packet.csrExecuted_
      and di.effectiveIthOperandMode(2) == WdRiscv::OperandMode::ReadWrite
      and not isNonForwardableCsr(packet.csrNum_))
    {
      // Insert in tag order: pushSpecCsrContext pokes in list order and needs the youngest
      // write to a CSR last, and execute is out of order, so appending is not enough.
      auto& specCsrs = hartSpecCsrs_[hartIx];
      auto pos = std::upper_bound(specCsrs.begin(), specCsrs.end(), packet.tag_,
                                  [](uint64_t t, const SpecCsrEntry& e) { return t < e.tag; });
      specCsrs.insert(pos, {packet.tag_, packet.csrNum_, packet.csrExecNewVal_});
    }

  hart.setTargetProgramFinished(false);
  hart.pokePc(prevPc);
  hart.setInstructionCount(prevInstrCount);

  // Collect the page table walks only when tracing is enabled.
  if (traceFiles_[hartIx])
    {
      packet.fetchWalks_ = hart.virtMem().getFetchWalks();
      packet.dataWalks_ = hart.virtMem().getDataWalks();
    }

  hart.clearTraceData();

  return true;
}


template <typename URV>
void
PerfApi<URV>::recordScOutcome(HartType& hart, InstrPac& packet)
{
  uint64_t sva = 0, spa1 = 0, spa2 = 0, sval = 0;
  unsigned ssize = hart.lastStore(sva, spa1, spa2, sval);
  packet.scExecuted_    = true;
  packet.scExecSuccess_ = (ssize != 0);  // lastStore reports zero size for failed SC
  if (packet.scExecSuccess_)
    {
      packet.scStoreVal_ = sval;
      packet.scPa1_      = spa1;
      packet.scPa2_      = spa2;
      packet.scSize_     = ssize;
    }
}


template <typename URV>
void
PerfApi<URV>::recordChangedCsr(HartType& hart, InstrPac& packet, CSRN csrn)
{
  Operand op;
  op.type = OperandType::CsReg;
  op.mode = OperandMode::Write; // Arbitrary.
  op.number = unsigned(csrn);
  URV csrVal{};
  if (not hart.peekCsr(csrn, csrVal))
    assert(0);
  op.value.scalar = csrVal;
  packet.changedCsrs_.at(packet.changedCsrCount_++) = op;
}


template <typename URV>
bool
PerfApi<URV>::retire(unsigned hartIx, uint64_t time, uint64_t tag,
                     TT_PERF::RetireResult* status)
{
  using TT_PERF::RetireResult;

  // Helper to set status and return false
  auto fail = [&](RetireResult result) {
    if (status)
      *status = result;
    return false;
  };

  if (commandLog_) [[unlikely]]
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_retire %" PRIu64 "\n",
            hartIx, time, tag);

  if (not checkTime("Retire", time)) [[unlikely]]
    return fail(RetireResult::InvalidTime);

  auto* hartPtr = checkHartRaw("Retire", hartIx);
  if (not hartPtr) [[unlikely]]
    return fail(RetireResult::InvalidHart);

  // Use findPacket to get both raw pointer and iterator for later erase
  auto lookupResult = findPacket(hartIx, tag);
  if (not lookupResult) [[unlikely]]
    {
      cerr << "Retire: Unknown tag (never fetched): " << tag << '\n';
      return fail(RetireResult::UnknownTag);
    }

  auto* pacPtr = lookupResult.packet;
  auto& packet = *pacPtr;
  auto& hart = *hartPtr;

  if (hartLastRetired_[hartIx] != initHartLastRetired and tag <= hartLastRetired_[hartIx])
    {
      cerr << "Error: Hart=" << hartIx << " time=" << time << " tag=" << tag
                << " Out of order retire\n";
      return fail(RetireResult::OutOfOrder);
    }
  const auto prevLastRetired = hartLastRetired_[hartIx];
  hartLastRetired_[hartIx] = tag;

  if (packet.retired())
    {
      cerr << "Error: Hart=" << hartIx << " time=" << time << " tag=" << tag
                << " Tag retired more than once\n";
      return fail(RetireResult::AlreadyRetired);
    }

  if (packet.instrVa() != hart.peekPc())
    {
      cerr << "Error: Hart=" << hartIx << " time=" << time << " tag=" << tag << std::hex
                << " Wrong pc at retire: 0x" << packet.instrVa() << " expecting 0x"
                << hart.peekPc() << '\n' << std::dec;
      return fail(RetireResult::WrongPc);
    }

  // A packet never executed (a client that does not speculate) is executed by this step.
  const bool execAtRetire = not packet.executed();
  bool peekOk = true;
  if (execAtRetire)
    {
      peekOk = collectOperandValues(hart, packet);
      packet.privMode_ = hart.privilegeMode();
      packet.virtMode_ = hart.virtMode();
    }

  hart.setInstructionCount(tag - 1);
  auto* traceFile = traceFiles_[hartIx];

  hart.singleStep(nullptr);

  const auto& di = packet.decodedInst();

  if (execAtRetire)
    {
      bool trap = hart.lastInstructionTrapped();
      packet.trap_ = trap;
      if (not peekOk)
        assert(trap);
      if (di.isSc() and not trap)
        recordScOutcome(hart, packet);
      packet.nextIva_ = hart.peekPc();
      recordExecutionResults(hart, packet);  // Before the CSR rd poke below reads destValues_.

      if (trap or di.isXRet())
        {
          std::vector<CSRN> csrns;
          hart.lastCsr(csrns);
          for (auto csrn : csrns)
            recordChangedCsr(hart, packet, csrn);
        }

      packet.executed_ = true;
      packet.execTime_ = time;
      if (packet.predicted_)
        packet.mispredicted_ = packet.prTarget_ != packet.nextIva_;
    }

  // SC reconciliation: capture the retire-time outcome from singleStep and overwrite
  // the packet's SC fields so drainStore commits the architectural data. On a
  // mismatch with the execute-time prediction, raise ScMispredict so the timing
  // model flushes younger packets and reconciles its store buffer.
  bool scMispredict = false;
  if (di.isSc() and packet.scExecuted_ and not packet.trap_)
    {
      uint64_t sva = 0, spa1 = 0, spa2 = 0, sval = 0;
      unsigned retireSize = hart.lastStore(sva, spa1, spa2, sval);
      bool retireSuccess = (retireSize != 0);
      if (retireSuccess != packet.scExecSuccess_)
        {
          scMispredict = true;
          packet.shouldFlush_ = true;
          packet.flushVa_     = packet.nextIva_;  // refetch from instruction after SC
        }
      packet.scExecSuccess_ = retireSuccess;
      packet.scSize_        = retireSize;       // zero on failure
      if (retireSuccess)
        {
          packet.scStoreVal_ = sval;
          packet.scPa1_      = spa1;
          packet.scPa2_      = spa2;
        }
    }

  using CN = WdRiscv::CsrNumber;
  if (di.isCsr() and di.op0() != 0)
    {
      unsigned csrNum = di.op2();

      // Hardware updates some CSRs on its own, so retire()'s re-read yields a different rd than
      // execute() sampled. Force the execute-time value: both samples are architecturally valid
      // (the read simply happened earlier), and a stale rd is otherwise invisible here --
      // checkExecVsRetire() exempts CSR ops -- until it aborts the run on the first dependent
      // non-CSR instruction. OpenSBI pmu_ctr_start_hw() reads mip and branches on LCOFIP two
      // instructions later, tripping the compare on the intervening slli.
      //
      // Read forms only (csrrs/csrrc rs1=x0, csrrsi/csrrci imm=0): on a write the execute-time
      // value is a prediction whose divergence is a real CsrMispredict, reconciled below.
      // FFLAGS/FCSR (flags accumulate from other FP ops) and MCYCLE/TIME predate this and stay
      // forced for every form.
      //
      // TODO(revisit): this makes the execute-time sample authoritative, a perf-api contract
      // choice -- a client sampling these non-speculatively wants the retire-time value instead.
      const bool readForm = di.effectiveIthOperandMode(2) != WdRiscv::OperandMode::ReadWrite;
      if (csrNum == unsigned(CN::MCYCLE) or csrNum == unsigned(CN::TIME) or
          csrNum == unsigned(CN::FFLAGS) or csrNum == unsigned(CN::FCSR) or
          (readForm and isAsyncUpdatedCsr(csrNum)))
        {
          hart.pokeIntReg(di.op0(), URV(packet.destValues_.at(0).second.scalar));
        }
    }

  // CSR reconciliation for writes: compare the retire-time CSR new value and rd
  // against the execute-time capture; on divergence raise CsrMispredict so the
  // model can flush younger packets. The captured fields are then overwritten with
  // the retire-time values for any downstream consumer. Read-only forms (csrrs/c
  // with rs1=x0; csrrsi/csrrci with imm=0) are skipped via effectiveIthOperandMode
  // — they don't modify the CSR, so there's nothing to reconcile, and
  // time-advancing CSRs like TIME/MCYCLE would otherwise trip the comparison
  // spuriously.
  bool csrMispredict = false;
  if (di.isCsr() and packet.csrExecuted_ and not packet.trap_
      and di.effectiveIthOperandMode(2) == WdRiscv::OperandMode::ReadWrite)
    {
      URV retireCsrVal{};
      bool csrOk = hart.peekCsr(CSRN(packet.csrNum_), retireCsrVal);
      uint64_t retireRd = 0;
      if (di.op0() != 0)
        retireRd = hart.peekIntReg(di.op0());

      const bool csrValueDiverged = csrOk and (uint64_t(retireCsrVal) != packet.csrExecNewVal_);
      const bool rdDiverged       = (di.op0() != 0) and (retireRd != packet.csrExecRd_);
      if (csrValueDiverged or rdDiverged)
        {
          csrMispredict = true;
          packet.shouldFlush_ = true;
          packet.flushVa_     = packet.nextIva_;  // refetch from instruction after the CSR write
        }

      if (csrOk) packet.csrExecNewVal_ = retireCsrVal;
      packet.csrExecRd_ = retireRd;
    }

  if (traceFile)
    {
      if (not execAtRetire)
        {
          hart.virtMem().setFetchWalks(packet.fetchWalks_);  // We print the walk from execute.
          hart.virtMem().setDataWalks(packet.dataWalks_);
        }
      hart.printInstCsvTrace(packet.di_, traceFile);
    }

  // Sanity check. Results at execute and retire must match.
#if 1
  if (not execAtRetire and not checkExecVsRetire(hart, packet))
    {
      // An async interrupt was taken at commit: singleStep above took the trap instead of
      // committing this instruction (hart PC is now the handler; it re-executes after
      // mret). It did NOT retire. Undo the retire-pointer advance so its tag is not
      // consumed, and report the preemption. The packet is left un-retired so the model
      // can flush it (still flushable) and redirect fetch to the handler.
      if (hart.lastInstructionInterrupted() and not packet.trap_)
        {
          hartLastRetired_[hartIx] = prevLastRetired;
          return fail(RetireResult::InterruptPreempted);
        }
      return fail(RetireResult::ExecRetireMismatch);
    }
#endif

  // Undo renaming of destination registers.
  undoDestRegRename(hartIx, packet);

  bool trap = hart.lastInstructionTrapped();
  packet.trap_ = packet.trap_ or trap;
  packet.interrupt_ = hart.lastInstructionInterrupted();
  if (packet.trap_)
    packet.trapCause_ = hart.lastTrapCause();

#if 0
  if (di.isLr())
    {
      // Record PC of subsequent packet.
      packet.nextIva_ = hart.peekPc();

      recordExecutionResults(hart, packet);
      packet.executed_ = true;
    }
#endif

  packet.retired_ = true;

  // AMO under multi-hart drains inline here (legacy, pending AMO's own speculation
  // pass). SC always drains via the unified drainStore path using the retire-time
  // outcome captured on the packet above; the paired speculative-LR entry is popped
  // there too so the single-hart and multi-hart paths share one cleanup site.
  bool amoInlineDrain = packet.isAmo() and system_.hartCount() > 1;
  if (amoInlineDrain)
    {
      uint64_t sva = 0, spa1 = 0, spa2 = 0, sval = 0;
      unsigned size = hart.lastStore(sva, spa1, spa2, sval);
      if (size != 0)
        if (not commitMemoryWrite(hart, spa1, spa2, size, packet.stData_))
          assert(0 && "Error: Assertion failed -- could not commit AMO data to memory");
      auto& storeMap = hartStoreMaps_[hartIx];
      packet.drained_ = true;
      storeMap.erase(packet.tag());
    }

  // Pop the matching speculative-CSR entry for a retired CSR op: the architectural
  // commit subsumes the speculative entry.
  if (di.isCsr() and packet.csrExecuted_)
    {
      auto& specCsrs = hartSpecCsrs_[hartIx];
      for (auto it = specCsrs.begin(); it != specCsrs.end(); ++it)
        if (it->tag == tag) { specCsrs.erase(it); break; }
    }

  // Clear dependency on other packets to expedite release of packet memory.
  for (auto& producer : packet.opProducers_)
    producer.clear();

  // Erase packet from packet map. Stores erased at drain time.
  auto& pacMap = hartPacketMaps_[hartIx];

  if (amoInlineDrain)
    pacMap.erase(tag);
  else
    {
      bool store = packet.isStore() or di.isVectorStore() or di.isCbo_zero() or di.isAmo();
      if (not store)
        pacMap.erase(tag);  // Stores erased at drain stage.
    }

  if (status)
    *status = csrMispredict ? RetireResult::CsrMispredict
              : scMispredict ? RetireResult::ScMispredict
              : RetireResult::Success;
  return true;
}


template <typename URV>
bool
PerfApi<URV>::checkExecVsRetire(const HartType& hart, const InstrPac& packet)
{
  unsigned hartIx = hart.sysHartIndex();
  auto tag = packet.tag_;

  // SC: the only meaningful execute-vs-retire divergence is success/fail, which is
  // handled by the ScMispredict path in retire(). Comparing rd (0 or 1) here would
  // false-trip when the predicted outcome differs from the architectural outcome;
  // the mispredict signal is the right channel for that.
  if (packet.isSc())
    return true;

  // CSR writes: legitimate divergence between exec and retire (e.g. source operand
  // or implicit side-effect drift while waiting for ROB head under NonSpecExec) is
  // handled by the CsrMispredict path in retire(). The rd comparison here would
  // false-trip on every divergence; skip it whenever execute captured an outcome.
  if (packet.decodedInst().isCsr() and packet.csrExecuted_)
    return true;

  bool retireTrap = hart.lastInstructionTrapped();
  if (packet.trap_ != retireTrap)
    {
      cerr << "Error: Hart=" << hartIx << " tag=" << tag << " trap on execute/retire "
           << "differ: " << packet.trap_ << '/' << retireTrap << '\n';
      return false;
    }

  if (packet.nextPc() != hart.peekPc())
    {
      cerr << "Hart=" << hartIx << " tag=" << tag << " execute and retire PCs differ:"
           << " 0x" << std::hex << packet.nextPc() << " & 0x" << hart.peekPc() << std::dec << '\n';
      return false;
    }

  if (packet.trap_)
     return true;

  if (int reg = hart.lastIntReg(); reg > 0)
    {
      uint64_t val = hart.peekIntReg(reg);
      uint64_t execVal = packet.destValues_.at(0).second.scalar;
      if (val != execVal)

        cerr << "Error: Hart=" << hartIx << " tag=" << tag << " retire & exec vals "
             << "differ for x" << reg << ": " << " 0x" << std::hex << val
             << " & 0x" << execVal << std::dec << '\n';
      return val == execVal;
    }

  if (int reg = hart.lastFpReg(); reg >= 0)
    {
      uint64_t val = 0;
      if (not hart.peekFpReg(reg, val))
        assert(0 && "Error: Assertion failed");
      uint64_t execVal = packet.destValues_.at(0).second.scalar;
      if (val != execVal)
        cerr << "Error: Hart=" << hartIx << " tag=" << tag << " exec & retire vals "
             << "differ for f" << reg << ": " << " 0x" << std::hex << val
             << " & 0x" << execVal << std::dec << '\n';
      return val == execVal;
    }

  unsigned group = 0;
  int vr = hart.lastVecReg(packet.di_, group);
  if (vr >= 0)
    {
      static thread_local std::vector<uint8_t> retire;
      retire.clear();
      hart.peekVecRegLsb(vr, retire);
      const auto& exec = packet.destValues_.at(0).second.vec;
      assert(retire.size() <= exec.size());
      for (unsigned i = 0; i < retire.size(); ++i)
        if (retire.at(i) != exec.at(i))
          {
            cerr << "Error: Hart=" << hartIx << " tag=" << tag << " lmul=" << group
                 << " vd=" << vr << " byte-ccount=" << retire.size()
                 << " exec & retire vec vals differ\n";
            cerr << "  retire: 0x" << std::hex;
            cerr << std::hex;
            unsigned count = retire.size();
            for (unsigned j = 0; j < count; ++j)
              cerr << std::setw(2) << std::setfill('0') << unsigned(retire.at(count-1-j));
            cerr << "\n";
            cerr << "  exec:   0x" << std::hex;
            for (unsigned j = 0; j < count; ++j)
              cerr << std::setw(2) << std::setfill('0') << unsigned(exec.at(count-1-j));
            cerr << "\n";
            return false;
          }
    }

  return true;
}


template <typename URV>
const char*
PerfApi<URV>::transAccessName(TransAccess access)
{
  switch (access)
    {
    case TransAccess::Fetch: return "translate-instr-addr";
    case TransAccess::Load:  return "translate-load-addr";
    case TransAccess::Store: return "translate-store-addr";
    }
  return "translate-addr";
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateAddr(unsigned hartIx, uint64_t tag, uint64_t va, uint64_t& pa,
                            TransAccess access, std::vector<Walk>* walks)
{
  bool x = access == TransAccess::Fetch;
  bool r = access == TransAccess::Load;
  bool w = access == TransAccess::Store;

  auto* hart = checkHartRaw(transAccessName(access), hartIx);
  hart->clearPageTableWalk();
  pa = va;

  // Clear up front, not just where a walk is produced: on the no-op paths below a caller
  // reusing one vector would otherwise read the previous instruction's walk back as this one's.
  if (walks)
    walks->clear();

  // Privilege/virtual mode of the requesting instruction, which is what its walk must use.
  // See translateInstrAddrForTag for the fallback when the packet has no recorded mode.
  auto pm = hart->privilegeMode();
  bool vm = hart->virtMode();
  if (tag != noSpecContext)
    if (const auto* packet = hartPacketMaps_.at(hartIx).find(tag); packet and packet->executed())
      {
        pm = packet->privlegeMode();
        vm = packet->virtMode();
      }

  if (pm == WdRiscv::PrivilegeMode::Machine or not hart->isRvs())
    return WdRiscv::ExceptionCause::NONE;

  // An S-level CSR aliases to its VS counterpart per the hart's virtual mode, as does the
  // translation config that poking SATP/VSATP/HGATP re-derives, so the forwarding below has to
  // run in the instruction's mode rather than the hart's. A no-op when they already agree.
  VirtModeGuard vmGuard(vm == hart->virtMode()? nullptr : hart, vm);

  // MSTATUS/SSTATUS carry SUM and MXR, so a walk without the forwarded writes can fault on a
  // page the instruction legitimately accessed. After vmGuard: CSRs are put back before it is.
  SpecCsrContext csrFwdSaves(*hart, tag == noSpecContext
                                      ? std::vector<CsrFwdSave>{}
                                      : pushSpecCsrContext(*hart, hartIx, tag, nullptr));

  WdRiscv::ExceptionCause cause = WdRiscv::ExceptionCause::NONE;
  {
    // A TLB hit has no walk to report, and callers asking for walks need one.
    WalkTraceGuard traceGuard(walks? &hart->virtMem() : nullptr);
    cause = hart->transAddrNoUpdate(va, pm, vm, r, w, x, pa, not walks /*useTlb*/);
  }

  if (walks)
    *walks = x? hart->getFetchPageTableWalks() : hart->getDataPageTableWalks();

  return cause;
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateInstrAddr(unsigned hartIx, uint64_t va, uint64_t& pa)
{
  return translateAddr(hartIx, noSpecContext, va, pa, TransAccess::Fetch, nullptr);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateLoadAddr(unsigned hartIx, uint64_t va, uint64_t& pa)
{
  return translateAddr(hartIx, noSpecContext, va, pa, TransAccess::Load, nullptr);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateStoreAddr(unsigned hartIx, uint64_t va, uint64_t& pa)
{
  return translateAddr(hartIx, noSpecContext, va, pa, TransAccess::Store, nullptr);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateInstrAddr(unsigned hartIx, uint64_t va, uint64_t& pa,
                            std::vector<Walk>& walks)
{
  return translateAddr(hartIx, noSpecContext, va, pa, TransAccess::Fetch, &walks);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateLoadAddr(unsigned hartIx, uint64_t va, uint64_t& pa,
                           std::vector<Walk>& walks)
{
  return translateAddr(hartIx, noSpecContext, va, pa, TransAccess::Load, &walks);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateStoreAddr(unsigned hartIx, uint64_t va, uint64_t& pa,
                            std::vector<Walk>& walks)
{
  return translateAddr(hartIx, noSpecContext, va, pa, TransAccess::Store, &walks);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateInstrAddrForTag(unsigned hartIx, uint64_t tag, uint64_t va, uint64_t& pa,
                                       std::vector<Walk>& walks)
{
  return translateAddr(hartIx, tag, va, pa, TransAccess::Fetch, &walks);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateLoadAddrForTag(unsigned hartIx, uint64_t tag, uint64_t va, uint64_t& pa,
                                      std::vector<Walk>& walks)
{
  return translateAddr(hartIx, tag, va, pa, TransAccess::Load, &walks);
}


template <typename URV>
WdRiscv::ExceptionCause
PerfApi<URV>::translateStoreAddrForTag(unsigned hartIx, uint64_t tag, uint64_t va, uint64_t& pa,
                                       std::vector<Walk>& walks)
{
  return translateAddr(hartIx, tag, va, pa, TransAccess::Store, &walks);
}


template <typename URV>
bool
PerfApi<URV>::drainStore(unsigned hartIx, uint64_t time, uint64_t tag)
{
  if (commandLog_) [[unlikely]]
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_drain_store %" PRIu64 "\n",
            hartIx, time, tag);

  if (not checkTime("Drain-store", time)) [[unlikely]]
    return false;

  auto* hartPtr = checkHartRaw("Drain-store", hartIx);

  // Use findPacket for combined lookup - need iterator for later erase
  auto lookupResult = findPacket(hartIx, tag);

  if (not hartPtr or not lookupResult or not lookupResult.packet->retired()) [[unlikely]]
    {
      assert(0 && "Error: Assertion failed");
      return false;
    }

  auto& hart = *hartPtr;
  auto* pacPtr = lookupResult.packet;
  auto& packet = *pacPtr;

  auto& di = packet.di_;
  if (not di.isStore() and not di.isVectorStore() and not di.isCbo_zero() and not di.isAmo() and not di.isSc()) [[unlikely]]
    {
      cerr << "Error: Hart=" << hartIx << " time=" << time << " tag=" << tag
                << " Draining a non-store instruction\n";
      return false;
    }

  // AMO under multi-hart drains inline in retire (legacy); SC always drains here.
  bool skipDrain = packet.isAmo() and system_.hartCount() > 1;

  if (skipDrain)
    assert(packet.drained());   // AMO must already be drained at retire.
  else
    {
      if (packet.drained())
        {
          cerr << "Error: Hart=" << hartIx << " time=" << time << " tag=" << tag
                    << " Instruction drained more than once\n";
          assert(0 && "Error: Assertion failed");
        }

      if (packet.isSc())
        {
          // Use the retire-time outcome captured on the packet (scExecSuccess_/scPa1_/
          // scPa2_/scSize_/scStoreVal_). On a successful SC commit the bytes; on a failed
          // SC nothing to write. Always cancel the reservation to match SC architectural
          // semantics. NOTE: a mispredict at retire has already overwritten these fields
          // to reflect the architectural outcome (see retire()).
          if (packet.scExecSuccess_)
            if (not commitMemoryWrite(hart, packet.scPa1_, packet.scPa2_,
                                       packet.scSize_, packet.scStoreVal_))
              assert(0 && "Error: Assertion failed -- could not commit SC data to memory");
          hart.cancelLr(WdRiscv::CancelLrCause::SC);
        }
      else if (packet.dsize_ and not commitMemoryWrite(hart, packet))
        assert(0 && "Error: Assertion failed");

      packet.drained_ = true;
    }

  // When SC drains, remove the paired LR's speculative reservation entry.
  if (packet.isSc())
    {
      auto& specLrs = hartSpecLrs_[hartIx];
      for (auto it = specLrs.rbegin(); it != specLrs.rend(); ++it)
	if (it->tag < tag)
	  {
	    specLrs.erase(std::next(it).base());
	    break;
	  }
    }

  // Clear dependency on other packets to expedite release of packet memory.
  for (auto& producer : packet.opProducers_)
    producer.clear();

  hartPacketMaps_[hartIx].erase(tag);
  hartStoreMaps_[hartIx].erase(tag);

  return true;
}


template <typename URV>
bool
PerfApi<URV>::getLoadData(unsigned hartIx, uint64_t tag, uint64_t va, uint64_t pa1,
                     uint64_t pa2, unsigned size, uint64_t& data, unsigned elemIx,
                     unsigned field)
{
  auto* hart = checkHartRaw("Get-load-data", hartIx);
  auto* packet = checkTagRaw("Get-load-Data", hartIx, tag);

  bool isLoad = ( packet->di_.isLoad() or packet->di_.isAmo() or
                  packet->di_.isVectorLoad() );
  if (not isLoad)
    assert(0 && "Error: Assertion failed");

  // Scalar instruction should not have trapped. Vector may trap on a later element.
  if (not packet->di_.isVectorLoad())
    assert(not packet->trapped());

  // If AMO destination register is x0, we lose the loaded value: redo the read for AMOs
  // to avoid that case. AMOs should not have a discrepancy between early read and read at
  // retire, so redoing the read is ok.
  bool amoRedo = packet->isAmo() and packet->di_.op0() == 0;

  if (packet->executed() and not amoRedo)
    {
      data = packet->executedDestVal(*hart, size, elemIx, field);
      return true;
    }

  data = 0;
  if (hart->isDeviceAddr(pa1))
    {
      if (skipIoLoad_)
        return true;  // Load from IO space happens at execute.
      hart->deviceRead(pa1, size, data);

      return true;
    }

  if (uint64_t toHost = 0; hart->getToHostAddress(toHost) and toHost == pa1)
    return true;  // Reading from toHost yields 0.

  auto& storeMap =  hartStoreMaps_[hartIx];

  unsigned mask = (1 << size) - 1;  // One bit per byte of load data.
  unsigned forwarded = 0;            // One bit per forwarded byte.

  unsigned size1 = size;
  if (pa1 != pa2 and pageNum(pa1) != pageNum(pa2))
    size1 = offsetToNextPage(pa1);

  // Use lower_bound to get end iterator - only iterate over stores with tags < current tag
  // Iterate in reverse order: newer stores have priority (they override older stores),
  // so we can exit early once all bytes are forwarded.
  auto storeEnd = storeMap.lower_bound(tag);

  for (auto it = std::make_reverse_iterator(storeEnd); it != storeMap.rend(); ++it)
    {
      auto* stPac = it->second.get();
      if (not stPac->executed()) [[unlikely]]
        continue;

      // SC: only forward from an SC that speculatively succeeded. A speculatively-failed
      // SC has no store data; skip. A predicted-success SC that later mispredicts will
      // be caught at retire and the model will flush/replay younger packets including
      // this load — for forwarding purposes the predicted outcome governs.
      if (stPac->isSc() and (not stPac->scExecuted_ or not stPac->scExecSuccess_)) [[unlikely]]
        continue;

      const auto& stMap = stPac->stDataMap_;

      // For small scalar stores, check VA-based overlap first
      if (not stPac->isVectorStore() and not stPac->isCbo_zero())
        {
          uint64_t stAddr = stPac->dataVa();
          unsigned stSize = stPac->dataSize();
          if (stAddr + stSize <= va or va + size <= stAddr)
            continue;  // No VA overlap - skip this store
        }

      // Only check bytes that haven't been forwarded yet
      unsigned remaining = mask & ~forwarded;
      if (remaining == 0) [[unlikely]]
        break;  // All bytes forwarded

      for (unsigned i = 0; remaining != 0; ++i, remaining >>= 1)
        {
          if ((remaining & 1) == 0)
            continue;  // This byte already forwarded

          uint64_t byteAddr = i < size1 ? pa1 + i : pa2 + (i - size1);

          auto iter = stMap.find(byteAddr);
          if (iter == stMap.end())
            continue;

          data &= ~(0xffLL << (i * 8));     // Clear byte location in data
          uint8_t byte = iter->second;
          data |= uint64_t(byte) << (i*8);  // Insert forwarded value instead
          forwarded |= (1u << i);
        }
    }

  if (forwarded == mask)
    return true;

  // Non-forward bytes are read from memory.
  for (unsigned i = 0; i < size; ++i)
    if (not (forwarded & (1 << i)))
      {
        uint8_t byte = 0;
        uint64_t pa = i < size1 ? pa1 + i : pa2 + (i - size1);
        if (not hart->peekMemory(pa, byte, true))
          assert(0 && "Error: Assertion failed");
        data |= uint64_t(byte) << (i*8);
      }

  return true;
}


template <typename URV>
bool
PerfApi<URV>::setStoreData(unsigned hartIx, uint64_t tag, uint64_t pa1, uint64_t pa2,
                      unsigned size, uint64_t value)
{
  auto* hartPtr = checkHartRaw("Set-store-data", hartIx);
  auto* packetPtr = checkTagRaw("Set-store-data", hartIx, tag);
  if (not packetPtr or not hartPtr)
    {
      assert(0 && "Error: Assertion failed");
      return false;
    }

  auto& packet = *packetPtr;
  if (not (packet.isStore() or packet.isAmo() or packet.isVectorStore() or packet.isCbo_zero()))
    {
      assert(0 && "Error: Assertion failed");
      return false;
    }

  auto& hart = *hartPtr;

  if (pa1 != pa2)
    {
      bool isDev = hart.isAclintAddr(pa1) or hart.isImsicAddr(pa1) or hart.isPciAddr(pa1);
      if (isDev)
        assert(0 && "Error: Assertion failed");

      isDev = hart.isAclintAddr(pa2) or hart.isImsicAddr(pa2) or hart.isPciAddr(pa2);
      if (isDev)
        assert(0 && "Error: Assertion failed");
    }

  packet.dpa_ = pa1;
  packet.dpa2_ = pa2;
  packet.stData_ = value;
  packet.dsize_ = size;

  unsigned size1 = size;
  if (pa1 != pa2 and pageNum(pa1) != pageNum(pa2))
    size1 = offsetToNextPage(pa1);

  for (unsigned i = 0; i < size; ++i, value >>= 8)
    {
      uint64_t pa = i < size1 ? pa1 + i : pa2 + (i - size1);
      uint8_t byte = value;
      packet.stDataMap_[pa] = byte;
    }

  return true;
}


template <typename URV>
bool
PerfApi<URV>::commitMemoryWrite(HartType& hart, uint64_t pa1, uint64_t pa2, unsigned size, uint64_t value)
{
  if (hart.isToHostAddr(pa1))
    {
      hart.handleStoreToHost(pa1, value);
      return true;
    }

  auto commit = [] (HartType& hart, uint64_t pa, unsigned sz, uint64_t val) -> bool {
    switch (sz)
      {
      case 1:  return hart.pokeMemory(pa, uint8_t(val), true);
      case 2:  return hart.pokeMemory(pa, uint16_t(val), true);
      case 4:  return hart.pokeMemory(pa, uint32_t(val), true);
      case 8:  return hart.pokeMemory(pa, val, true);
      default:
        {
          bool ok = true;
          for (unsigned i = 0; i < sz; ++i, val >>= 8)
            {
              uint8_t byte = val;
              ok = hart.pokeMemory(pa + i, byte, true) and ok;
            }
          return ok;
        }
      }
    return true;
  };

  // Recover reservation to re-make it after commit. Commit uses memory poke which drops
  // the reservation.
  uint64_t prevLrAddr = 0;
  unsigned prevLrSize = 0;
  bool hasLr = hart.getLr(prevLrAddr, prevLrSize);

  bool ok = true;

  if (pa1 == pa2 or pageNum(pa1) == pageNum(pa2))
    ok = commit(hart, pa1, size, value);
  else
    {
      unsigned size1 = offsetToNextPage(pa1);
      ok = commit(hart, pa1, size1, value);

      value = value >> size1*8;
      ok = commit(hart, pa2, size - size1, value) and ok;
    }

  if (hasLr)
    hart.makeLr(prevLrAddr, prevLrSize);

  return ok;
}


template <typename URV>
bool
PerfApi<URV>::commitMemoryWrite(HartType& hart, const InstrPac& packet)
{
  if (not packet.isVectorStore() and not packet.isCbo_zero())
    return commitMemoryWrite(hart, packet.dpa_, packet.dpa2_, packet.dsize_, packet.stData_);

  auto ok = true;

  // Recover reservation to re-make it after commit. Commit uses memory poke which drops
  // the reservation.
  uint64_t prevLrAddr = 0;
  unsigned prevLrSize = 0;
  bool hasLr = hart.getLr(prevLrAddr, prevLrSize);

  for (auto [addr, val] : packet.stDataMap_)
    ok = hart.pokeMemory(addr, val, true) and ok;

  if (hasLr)
    hart.makeLr(prevLrAddr, prevLrSize);

  return ok;
}


template <typename URV>
bool
PerfApi<URV>::flush(unsigned hartIx, uint64_t time, uint64_t tag)
{
  if (commandLog_) [[unlikely]]
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_flush %" PRIu64 "\n",
            hartIx, time, tag);

  if (not checkTime("Flush", time)) [[unlikely]]
    return false;

  if (not checkHartRaw("Flush", hartIx)) [[unlikely]]
    return false;

  auto& packetMap = hartPacketMaps_[hartIx];
  auto& storeMap =  hartStoreMaps_[hartIx];
  auto& producers = hartRegProducers_[hartIx];

  // Flush tag and all newer packets. Flush in reverse order to undo register renaming.
  for (auto iter = packetMap.rbegin(); iter != packetMap.rend(); ++iter)
    {
      auto* pacPtr = iter.packet();
      if (pacPtr->tag_ < tag) [[likely]]
        break;

      if (pacPtr->retired()) [[unlikely]]
        {
          assert(0 && "Error: Assertion failed: Flushing retired instruction");
          return false;
        }

      auto& packet = *pacPtr;
      const size_t operandCount = packet.operandCount_;
      // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
      for (size_t i = 0; i < operandCount; ++i)
        {
          using OM = WdRiscv::OperandMode;
          using OT = WdRiscv::OperandType;

          auto& op = packet.operands_[i];
          if (op.mode == OM::Write or op.mode == OM::ReadWrite)
            {
              unsigned regNum = op.number;
              unsigned gri = globalRegIx(op.type, regNum);
              if (gri != 0)
                assert(producers[gri]->tag_ == packet.tag_);

              const auto& iop = packet.opProducers_[i];

              if (op.type != OT::VecReg)
                {
                  const auto* prevRaw = iop.scalar.get();
                  if (prevRaw and not prevRaw->retired_)
                    producers[gri] = iop.scalar;
                  else
                    producers[gri] = nullptr;
                }
              else
                {
                  const size_t vecSize = iop.vec.size();
                  assert(vecSize <= maxEffLmul_);
                  for (unsigned n = 0; n < vecSize; ++n)
                    {
                      const auto* prevRaw = iop.vec[n].get();
                      if (prevRaw and not prevRaw->retired_)
                        producers[gri+n] = iop.vec[n];
                      else
                        producers[gri+n] = nullptr;
                    }
                }
            }
        }
      // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)
    }

  // Delete input tag and all newer instructions
  packetMap.eraseFrom(tag);

  auto eraseIt = storeMap.lower_bound(tag);
  storeMap.erase(eraseIt, storeMap.end());

  // Remove speculative LR entries for flushed instructions.
  auto& specLrs = hartSpecLrs_[hartIx];
  std::erase_if(specLrs, [tag](const SpecLrEntry& e) { return e.tag >= tag; });

  // Remove speculative CSR entries for flushed instructions.
  auto& specCsrs = hartSpecCsrs_[hartIx];
  std::erase_if(specCsrs, [tag](const SpecCsrEntry& e) { return e.tag >= tag; });

  if (prevFetch_ and prevFetch_->tag_ > tag)
    prevFetch_ = nullptr;

  return true;
}


template <typename URV>
bool
PerfApi<URV>::shouldFlush(unsigned hartIx, uint64_t time, uint64_t tag, bool& flush,
                     uint64_t& addr)
{
  flush = false;
  addr = 0;

  if (commandLog_)
    fprintf(commandLog_, "hart=%" PRIu32 " time=%" PRIu64 " perf_model_should_flush %" PRIu64 "\n",
            hartIx, time, tag);

  if (not checkTime("Flush", time))
    return false;

  if (not checkHartRaw("Flush", hartIx))
    return false;

  auto* pacPtr = checkTagRaw("Retire", hartIx, tag);
  if (not pacPtr)
    return false;
  auto& packet = *pacPtr;

  if (packet.shouldFlush())
    {
      flush = true;
      addr = packet.flushVa_;
    }
  else
    {
      // If on the wrong path after a branch, then we should flush.
      // Iterate backwards through packets with tag < current tag
      auto& packetMap = hartPacketMaps_[hartIx];
      for (uint64_t prevTag = tag; prevTag > packetMap.minTag(); )
        {
          --prevTag;
          auto* prevPacket = packetMap.find(prevTag);
          if (not prevPacket)
            continue;
          if (prevPacket->mispredicted_)
            {
              flush = true;
              if (prevPacket->di_.isBranch())
                addr = prevPacket->nextIva_;
              else
                addr = prevPacket->iva_;
            }
        }
    }

  return true;
}


unsigned
InstrPac::getSourceOperands(std::array<Operand, 3>& ops) const
{
  assert(decoded_);
  if (not decoded_)
    return 0;

  // Return explicit operands. Skip implicit.
  const unsigned limit = std::min(di_.operandCount(), operandCount_);
  unsigned count = 0;

  using OM = WdRiscv::OperandMode;
  using CN = WdRiscv::CsrNumber;

  for (unsigned i = 0; i < limit; ++i)
    {
      const auto& op = operands_[i];  // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index)
      if (op.mode == OM::Read or op.mode == OM::ReadWrite or op.type == OperandType::Imm)
        {
          // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
          auto& tgt = ops[count];
          tgt = op;
          count++;

          // We map FRM/FFLAGS to FCSR. Recover the value of FRM/FFLAGS. Same for
          // VXRM/VXSAT and VCSR. This is useful for comparing trace-driven traces to
          // execution-driven traces.
          auto iopn = di_.ithOperand(i);  // Ith operand number,
          if (op.type != OperandType::CsReg or op.number == iopn)
            continue;  // Not a CSR or not remapped.

          tgt.number = iopn;   // Unmap register number

          // And fix its value.
          auto val = tgt.value.scalar;

          if (iopn == unsigned(CN::FRM))
            tgt.value.scalar = WdRiscv::FcsrFields{val}.bits_.FRM;
          else if (iopn == unsigned(CN::FFLAGS))
            tgt.value.scalar = WdRiscv::FcsrFields{val}.bits_.FFLAGS;
          else if (iopn == unsigned(CN::VXRM))
            tgt.value.scalar = WdRiscv::VcsrFields{val}.bits_.VXRM;
          else if (iopn == unsigned(CN::VXSAT))
            tgt.value.scalar = WdRiscv::VcsrFields{val}.bits_.VXSAT;
          else
            assert(0);
        }
    }

  return count;
}


unsigned
InstrPac::getDestOperands(std::array<Operand, 2>& ops) const
{
  assert(decoded_);
  if (not decoded_)
    return 0;

  // Return explicit operands. Skip implicit.
  const unsigned limit = std::min(di_.operandCount(), operandCount_);
  unsigned count = 0;

  using OM = WdRiscv::OperandMode;
  using CN = WdRiscv::CsrNumber;

  for (unsigned i = 0; i < limit; ++i)
    {
      const auto& op = operands_.at(i);
      if (op.mode == OM::Write or op.mode == OM::ReadWrite)
        {
          auto& tgt = ops.at(count);
          tgt = op;
          tgt.value = destValues_.at(count).second;
          count++;

          // We map FRM/FFLAGS to FCSR. Recover the value of FRM/FFLAGS. Same for
          // VXRM/VXSAT and VCSR. This is useful for comparing trace-driven traces to
          // execution-driven traces.
          auto iopn = di_.ithOperand(i);  // Ith operand number,
          if (op.type != OperandType::CsReg or op.number == iopn)
            continue;  // Not a CSR or not remapped.

          tgt.number = iopn;   // Unmap register number

          // And fix its value.
          auto val = tgt.value.scalar;

          if (iopn == unsigned(CN::FRM))
            tgt.value.scalar = WdRiscv::FcsrFields{val}.bits_.FRM;
          else if (iopn == unsigned(CN::FFLAGS))
            tgt.value.scalar = WdRiscv::FcsrFields{val}.bits_.FFLAGS;
          else if (iopn == unsigned(CN::VXRM))
            tgt.value.scalar = WdRiscv::VcsrFields{val}.bits_.VXRM;
          else if (iopn == unsigned(CN::VXSAT))
            tgt.value.scalar = WdRiscv::VcsrFields{val}.bits_.VXSAT;
          else
            assert(0);
        }
    }

  return count;
}


unsigned
InstrPac::getImplicitDestOperands(std::array<Operand, 4>& ops) const
{
  assert(decoded_);
  if (not decoded_)
    return 0;

  assert(di_.operandCount() <= operandCount_);

  using OM = WdRiscv::OperandMode;

  unsigned explicitDests = 0;
  const unsigned diOpCount = di_.operandCount();

  for (unsigned i = 0; i < diOpCount; ++i)
    {
      const auto& op = operands_.at(i);
      if (op.mode == OM::Write or op.mode == OM::ReadWrite)
        ++explicitDests;
    }

  unsigned count = 0;  // Count of implicit destinations.

  const unsigned start = diOpCount;
  const unsigned opCount = operandCount_;

  for (unsigned i = start; i < opCount; ++i)
    {
      const auto& op = operands_.at(i);
      if (op.mode == OM::Write or op.mode == OM::ReadWrite)
        {
          auto& tgt = ops.at(count);
          tgt = op;
          tgt.value = destValues_.at(explicitDests + count).second;
          ++count;
        }
    }

  return count;
}


unsigned
InstrPac::getImplicitSrcOperands(std::array<Operand, 8>& impOps) const
{
  assert(decoded_);
  if (not decoded_)
    return 0;

  assert(di_.operandCount() <= operandCount_);

  using OM = WdRiscv::OperandMode;

  unsigned count = 0;  // Count of implicit sources.

  const unsigned start = di_.operandCount();
  const unsigned opCount = operandCount_;

  for (unsigned i = start; i < opCount; ++i)
    {
      const auto& op = operands_.at(i);
      if (op.mode == OM::Read or op.mode == OM::ReadWrite)
        impOps[count++] = op;  // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index)
    }

  return count;
}


unsigned
InstrPac::getChangedCsrs(std::array<Operand, 8>& ops) const
{
  const unsigned count = changedCsrCount_;
  for (unsigned i = 0; i < count; ++i)
    ops.at(i) = changedCsrs_.at(i);
  return count;
}


uint64_t
InstrPac::branchTargetFromDecode() const
{
  if (not isBranch()) return 0;

  using WdRiscv::InstId;
  switch (di_.instEntry()->instId())
    {
    case InstId::jal:
    case InstId::c_jal:
    case InstId::c_j:
      return instrVa() + di_.op1As<int64_t>();

    case InstId::beq:
    case InstId::bne:
    case InstId::blt:
    case InstId::bge:
    case InstId::bltu:
    case InstId::bgeu:
    case InstId::c_beqz:
    case InstId::c_bnez:
      return instrVa() + di_.op2As<int64_t>();

    default:
      return 0;
    }
}


template <typename URV>
uint64_t
InstrPac::executedDestVal(const WdRiscv::Hart<URV>& hart, unsigned size, unsigned elemIx, unsigned field) const
{
  assert(executed());

  const OpVal& destVal = destValues_.at(0).second;

  if (operands_[0].type != WdRiscv::OperandType::VecReg)
    {
      assert(size == dataSize());
      return destVal.scalar;
    }

  const std::vector<uint8_t>& vec = destVal.vec;   // Vector register value.

  const auto& info = hart.getLastVectorMemory();
  unsigned elemSize = info.elemSize_;

  unsigned offset = elemSize * elemIx;
  if (info.isSegmented_)
    {
      if (info.fields_ > 0)
        {
          assert(field < info.fields_);
          unsigned bytesPerReg = hart.vecRegs().bytesPerRegister();
          assert(info.group_ > 0);
          offset += field*bytesPerReg*info.group_;
        }
      else
        assert(field == 0);
    }

  assert(offset + size <= vec.size());

  uint64_t val = 0;
  for (unsigned i = 0; i < size; ++i)
    {
      uint64_t byte = vec[offset + i];
      val |= byte << i*8;
    }
  return val;
}


size_t InstrPac::getPacketSize() const
{
  size_t totalSize = 0;

  totalSize += sizeof(*this); // Get the size of the object itself

  // stDataMap_: unordered_map<uint64_t, uint8_t>
  totalSize += stDataMap_.size() * (sizeof(uint64_t) + sizeof(uint8_t));  // key-value pairs

  // vecAddrs_: vector<VaPaSkip> where VaPaSkip = tuple<uint64_t, uint64_t, bool>
  totalSize += vecAddrs_.size() * (sizeof(uint64_t) + sizeof(uint64_t) + sizeof(bool));  // tuple elements

  // fetchWalks_: vector<vector<WalkEntry>>
  for (const auto& walk : fetchWalks_) {
    totalSize += 2*walk.size()*sizeof(uint64_t);  // Inner address & pte vectors.
    totalSize += sizeof(walk);  // Walk objects
  }

  // dataWalks_: vector<vector<WalkEntry>>
  for (const auto& walk : dataWalks_) {
    totalSize += 2*walk.size()*sizeof(uint64_t);  // Inner address & pte vectors.
    totalSize += sizeof(walk);  // Walk objects
  }

  return totalSize;
}


template <typename URV>
bool
PerfApi<URV>::saveHartValues(HartType& hart, const InstrPac& packet,
                        std::array<OpVal, 9>& prevVal)
{
  using OM = WdRiscv::OperandMode;
  using OT = WdRiscv::OperandType;

  bool ok = true;
  const unsigned opCount = packet.operandCount_;

  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  for (unsigned i = 0; i < opCount; ++i)
    {
      const auto& op = packet.operands_[i];
      auto mode = op.mode;
      if (mode == OM::None)
        continue;

      auto type = op.type;
      uint32_t number = op.number;

      switch (type)
        {
        case OT::IntReg:
          {
            URV tmp{};
            if (not hart.peekIntReg(number, tmp))
              assert(0 && "Error: Assertion failed");
            prevVal[i].scalar = tmp;
          }
          break;

        case OT::FpReg:
          ok = hart.peekFpReg(number, prevVal[i].scalar) and ok;
          break;

        case OT::CsReg:
          {
            URV tmp{};
            ok = hart.peekCsr(CSRN(number), tmp) and ok;
            prevVal[i].scalar = tmp;
          }
          break;

        case OT::VecReg:
          ok = peekVecRegGroup(hart, number, op.lmul, prevVal[i]) and ok;
          break;

        case OT::Imm:
          break;

        default:
          assert(0 && "Error: Assertion failed");
          break;
        }
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)

  return ok;
}


template <typename URV>
void
PerfApi<URV>::saveImsicTopei(HartType& hart, CSRN csrn, unsigned& id, unsigned& guest)
{
  id = 0;
  guest = 0;

  auto imsic = hart.imsic();
  if (not imsic)
    return;

  if (csrn == CSRN::MTOPEI)
    {
      id = imsic->machineTopId();
    }
  else if (csrn == CSRN::STOPEI)
    {
      id = imsic->supervisorTopId();
    }
  else if (csrn == CSRN::VSTOPEI)
    {
      URV hs = 0;
      if (hart.peekCsr(CSRN::HSTATUS, hs))
        {
          WdRiscv::HstatusFields<URV> hsf(hs);
          unsigned gg = hsf.bits_.VGEIN;
          if (gg > 0 and gg < imsic->guestCount())
            {
              guest = gg;
              imsic->guestTopId(gg);
            }
        }
    }
}



template <typename URV>
void
PerfApi<URV>::restoreImsicTopei(HartType& hart, CSRN csrn, unsigned id, unsigned guest)
{
  auto imsic = hart.imsic();
  if (not imsic)
    return;

  if (id == 0)
    return;

  if (csrn == CSRN::MTOPEI)
    {
      imsic->setMachinePending(id, true);
    }
  else if (csrn == CSRN::STOPEI)
    {
      imsic->setSupervisorPending(id, true);
    }
  else if (csrn == CSRN::VSTOPEI)
    {
      if (guest > 0 and guest < imsic->guestCount())
        imsic->setGuestPending(guest, id, true);
    }
}


template <typename URV>
void
PerfApi<URV>::restoreHartValues(HartType& hart, const InstrPac& packet,
                                const std::array<OpVal, 9>& prevVal)
{
  using OM = WdRiscv::OperandMode;
  using OT = WdRiscv::OperandType;

  const unsigned opCount = packet.operandCount_;

  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  for (unsigned i = 0; i < opCount; ++i)
    {
      const auto& op = packet.operands_[i];
      auto mode = op.mode;
      auto type = op.type;
      uint32_t number = op.number;
      uint64_t prev = prevVal[i].scalar;
      const std::vector<uint8_t>& vec = prevVal[i].vec;
      if (mode == OM::None)
        continue;

      switch (type)
        {
        case OT::IntReg:
          if (not hart.pokeIntReg(number, URV(prev)))
            assert(0 && "Error: Assertion failed");
          break;

        case OT::FpReg:
          if (not hart.pokeFpReg(number, prev))
            assert(0 && "Error: Assertion failed");
          break;

        case OT::CsReg:
          {
            auto csrn = CSRN(number);
            hart.pokeCsr(csrn, URV(prev));  // May fail because of privilege. It's ok: handled at caller.
          }
          break;

        case OT::VecReg:
          {
            size_t bytesPerReg = hart.vecRegs().bytesPerRegister();
            size_t count = vec.size() / bytesPerReg;
            assert(count * bytesPerReg == vec.size());
            for (unsigned j = 0; j < count; ++j)
              {
                const auto* pokeData = &vec[j*bytesPerReg];
                if (not hart.pokeVecRegLsb(number + j, std::span(pokeData, bytesPerReg)))
                  assert(packet.trap_);
              }
          }
          break;

        default:
          assert(0 && "Error: Assertion failed");
          break;
        }
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)
}


template <typename URV>
bool
PerfApi<URV>::setHartValues(HartType& hart, const InstrPac& packet)
{
  bool ok = true;
  const unsigned opCount = packet.operandCount_;

  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  for (unsigned i = 0; i < opCount; ++i)
    {
      const auto& op = packet.operands_[i];
      if (op.mode == WdRiscv::OperandMode::None)
        continue;

      auto type = op.type;
      uint32_t regNum = op.number;
      ok = pokeRegister(hart, type, regNum, op.value) and ok;
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)

  return ok;
}


template <typename URV>
bool
PerfApi<URV>::peekRegister(HartType& hart, WdRiscv::OperandType type, unsigned regNum,
                      OpVal& value)
{
  using OT = WdRiscv::OperandType;
  switch(type)
    {
    case OT::IntReg:
      {
        URV tmp{};
        if (hart.peekIntReg(regNum, tmp))
          {
            value.scalar = tmp;
            return true;
          }
        return false;
      }

    case OT::FpReg:
      return hart.peekFpReg(regNum, value.scalar);

    case OT::CsReg:
      {
        URV tmp{};
        if (hart.peekCsr(WdRiscv::CsrNumber(regNum), tmp))
          {
            value.scalar = tmp;
            return true;
          }
        return false;
      }

    case OT::VecReg:
      return hart.peekVecRegLsb(regNum, value.vec);

    case OT::Imm:
    case OT::None:
      assert(0 && "Error: Assertion failed");
      return false;
    }

  return false;
}


template <typename URV>
bool
PerfApi<URV>::pokeRegister(HartType& hart, WdRiscv::OperandType type, unsigned regNum,
                      const OpVal& value)
{
  using OT = WdRiscv::OperandType;

  uint64_t scalar = value.scalar;
  const std::vector<uint8_t>& vecVal = value.vec;

  switch (type)
    {
    case OT::IntReg:
      if (hart.pokeIntReg(regNum, URV(scalar)))
        return true;
      assert(0 && "Error: Assertion failed");
      return false;

    case OT::FpReg:
      return hart.pokeFpReg(regNum, scalar);

    case OT::CsReg:
      return hart.pokeCsr(CSRN(regNum), URV(scalar));

    case OT::VecReg:
      {
        bool ok = true;

        size_t bytesPerReg = hart.vecRegs().bytesPerRegister();
        size_t count = vecVal.size() / bytesPerReg;
        assert(count * bytesPerReg == vecVal.size());

        for (unsigned i = 0; i < count; ++i)
          {
            const auto* pokeData = &vecVal[i*bytesPerReg];
            ok = hart.pokeVecRegLsb(regNum + i, std::span(pokeData, bytesPerReg)) and ok;
          }

        return ok;
      }

    case OT::Imm:
    default:
      break;
    }

  assert(0 && "Error: Assertion failed");
  return false;
}


template <typename URV>
bool
PerfApi<URV>::peekVecRegGroup(HartType& hart, unsigned regNum, unsigned lmul, OpVal& value)
{
  std::vector<uint8_t>& data = value.vec;
  static thread_local std::vector<uint8_t> vecVal;

  bool ok = true;

  for (unsigned n = 0; n < lmul; ++n)
    {
      vecVal.clear();
      ok = hart.peekVecRegLsb(regNum + n, vecVal) and ok;

      // Append single vector value to data.
      data.insert(data.end(), vecVal.begin(), vecVal.end());
    }

  return ok;
}


template <typename URV>
void
PerfApi<URV>::updatePacketDataAddress(HartType& hart, InstrPac& packet)
{
  const auto& di = packet.decodedInst();

  auto hartIx = hart.sysHartIndex();

  if (di.isLoad())
    {
      hart.lastLdStAddress(packet.dva_, packet.dpa_, packet.dpa2_);
      packet.dsize_ = di.loadSize();
      uint64_t dpa = packet.dpa_;
      packet.deviceAccess_ = ( hart.isAclintMtimeAddr(dpa) or hart.isImsicAddr(dpa) or
                               hart.isPciAddr(dpa) or hart.isHtifAddr(dpa) );
    }
  else if (di.isVectorLoad())
    {
      const auto& info = hart.getLastVectorMemory();
      const size_t elemCount = info.elems_.size();
      packet.vecAddrs_.reserve(elemCount);
      for (const auto& elem : info.elems_)
        {
          InstrPac::VaPaSkip skip{elem.va_, elem.pa_, elem.skip_};
          packet.vecAddrs_.emplace_back(skip);
        }
      packet.dsize_ = info.elemSize_;
    }
  else if (di.isStore() or di.isAmo() or di.isVectorStore() or di.isCbo_zero() or di.isCmo())
    {
      uint64_t sva = 0, spa1 = 0, spa2 = 0, sval = 0;
      unsigned ssize = hart.lastStore(sva, spa1, spa2, sval);
      if (ssize == 0 and (di.isStore() or di.isAmo()) and not di.isSc())
        {    // sc or vec-store may have 0 size
          cerr << "Error: Hart=" << hartIx << " tag=" << packet.tag_
                    << " store/AMO with zero size\n";
          assert(0 && "Error: Assertion failed");
        }

      packet.dva_ = sva;
      packet.dpa_ = spa1;  // FIX TODO : handle page crossing
      packet.dpa2_ = spa2;
      packet.dsize_ = ssize;
      assert(ssize == packet.dsize_);

      auto& storeMap =  hartStoreMaps_[hartIx];
      auto tag = packet.tag();

      // AMO under multi-hart still skips the storeMap (pending follow-up). SC is
      // inserted whenever execute reported success, regardless of hart count.
      bool amoSkipStoreMap = packet.isAmo() and system_.hartCount() > 1;
      bool scInsert = packet.isSc() and (ssize != 0);

      if (di.isCbo_zero())
        {
          storeMap[tag] = hartPacketMaps_[hartIx].findShared(tag);
        }
      else if (di.isVectorStore())
        {
          const auto& info = hart.getLastVectorMemory();
          const size_t elemCount = info.elems_.size();
          packet.vecAddrs_.reserve(elemCount);
          for (const auto& elem : info.elems_)
            {
              InstrPac::VaPaSkip skip{elem.va_, elem.pa_, elem.skip_};
              packet.vecAddrs_.emplace_back(skip);
            }

          packet.dsize_ = info.elemSize_;

          storeMap[tag] = hartPacketMaps_[hartIx].findShared(tag);
          // FIX What to do about device access? Do we allow mixed device/non-device
          // access?
        }
      else if (scInsert or (di.isAmo() and not amoSkipStoreMap) or (di.isStore() and not di.isSc()))
        {
          storeMap[tag] = hartPacketMaps_[hartIx].findShared(tag);
          uint64_t dpa = packet.dpa_;
          packet.deviceAccess_ = ( hart.isAclintMtimeAddr(dpa) or hart.isImsicAddr(dpa) or
                                   hart.isPciAddr(dpa) or hart.isHtifAddr(dpa) );
        }
    }
}


template <typename URV>
void
PerfApi<URV>::recordExecutionResults(HartType& hart, InstrPac& packet)
{
  if (not packet.trap_)
    updatePacketDataAddress(hart, packet);

  if (hart.hasTargetProgramFinished())
    packet.nextIva_ = haltPc;

  if (packet.di_.isBranch())
    packet.taken_ = hart.lastBranchTaken();

  // Record the values of the destination register.
  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  unsigned destIx = 0;
  const unsigned opCount = packet.operandCount_;
  for (unsigned i = 0; i < opCount; ++i)
    {
      using OM = WdRiscv::OperandMode;
      using OT = WdRiscv::OperandType;

      auto& op = packet.operands_[i];

      if (op.mode == OM::Write or op.mode == OM::ReadWrite)
        {
          unsigned regNum = op.number;
          unsigned gri = globalRegIx(op.type, regNum);
          OpVal destVal;
          if (op.type != OT::VecReg)
            {
              if (not peekRegister(hart, op.type, regNum, destVal))
                assert(packet.trap_);
            }
          else
            {
              if (not peekVecRegGroup(hart, regNum, op.lmul, destVal))
                assert(packet.trap_);
            }
          packet.destValues_[destIx] = InstrPac::DestValue(gri, std::move(destVal));
          destIx++;
        }
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)

  // Memory should not have changed.
}


template <typename URV>
void
PerfApi<URV>::getVectorOperandsLmul(HartType& hart, InstrPac& packet)
{
  auto di = packet.decodedInst();
  if (not di.isVector())
    return;

  using CN = WdRiscv::CsrNumber;
  using OT = WdRiscv::OperandType;

  auto hartIx = hart.sysHartIndex();
  // pokeCsr ignores VTYPE while VS is Off.
  SpecCsrContext csrFwd(hart, pushSpecCsrContext(hart, hartIx, packet.tag_, &packet));

  // 1. Set vtype value if it is in-flight.
  auto& producers = hartRegProducers_[hartIx];
  auto vtypeGri = globalRegIx(OT::CsReg, unsigned(CN::VTYPE));
  auto producer = producers[vtypeGri];  // Producer of vtype

  URV prevMstatus = hart.peekCsr(CSRN::MSTATUS);

  URV prevVal = 0;
  if (producer)
    {
      if (not hart.peekCsr(CN::VTYPE, prevVal))
        assert(0 && "Error: Assertion failed");

      OpVal vtypeVal;
      if (not getDestValue(*producer, OT::CsReg, unsigned(CN::VTYPE), vtypeVal))
        assert(0);
      hart.pokeCsr(CN::VTYPE, URV(vtypeVal.scalar));
    }

  // 2. Determine the operands LMUL
  getVecOpsLmul(hart, packet);

  // 3. Restore vtype if it was set.
  if (producer)
    {
      hart.pokeCsr(CN::VTYPE, prevVal);  // This may change MSTATUS.VS

      URV mstatus = hart.peekCsr(CSRN::MSTATUS);
      if (mstatus != prevMstatus)
        hart.pokeCsr(CSRN::MSTATUS, prevMstatus);
    }
}


template <typename URV>
void
PerfApi<URV>::getVecOpsLmul(HartType& hart, InstrPac& packet)
{
  const auto& vecRegs = hart.vecRegs();

  auto groupX8 = vecRegs.groupMultiplierX8();
  unsigned effLmul = groupX8 <= 8 ? 1 : groupX8 / 8;

  auto wideX8 = 2*groupX8;
  unsigned effWideLmul = wideX8 <= 8 ? 1 : wideX8 / 8;

  for (unsigned i = 0; i < 3; ++i)
    packet.operands_[i].lmul = effLmul;  // NOLINT(cppcoreguidelines-pro-bounds-constant-array-index)

  using InstId = WdRiscv::InstId;

  auto di = packet.decodedInst();

  if (di.isVectorLoad() or di.isVectorStore())
    {
      unsigned fields = di.vecFieldCount();

      if (di.isVectorLoadIndexed() or di.isVectorStoreIndexed())
        {
          auto ig8 = groupX8 * hart.vecLdStIndexElemSize(di) / vecRegs.elemWidthInBytes();
          auto dg8 = groupX8;

          unsigned dmul = dg8 <= 8 ? 1 : dg8 / 8;   // Data reg effective lmul
          unsigned imul = ig8 <= 8 ? 1 : ig8 / 8;   // Index reg effective lmul

          if (fields > 0)
            dmul *= fields;

          packet.operands_[0].lmul = dmul;
          packet.operands_[2].lmul = imul;
        }
      else
        {
          auto id = di.instId();
          if ( (id >= InstId::vlre8_v and id <= InstId::vlre64_v) or
               (id >= InstId::vs1r_v and id <= InstId::vs8r_v) )
            packet.operands_[0].lmul = fields;
          else
            {
              auto dg8 = groupX8 * hart.vecLdStElemSize(di) / vecRegs.elemWidthInBytes();
              unsigned dmul = dg8 <= 8 ? 1 : dg8 / 8;   // Data reg effective lmul
              if (fields)
                dmul *= fields;             // Segment load/store
              packet.operands_[0].lmul = dmul;
            }
        }

      return;
    }

  switch(di.instId())
    {
    case InstId::vmv1r_v:
      packet.operands_[0].lmul = packet.operands_[1].lmul = 1;
      break;
    case InstId::vmv2r_v:
      packet.operands_[0].lmul = packet.operands_[1].lmul = 2;
      break;
    case InstId::vmv4r_v:
      packet.operands_[0].lmul = packet.operands_[1].lmul = 4;
      break;
    case InstId::vmv8r_v:
      packet.operands_[0].lmul = packet.operands_[1].lmul = 8;
      break;

    case InstId::vmv_x_s:
    case InstId::vmv_s_x:
    case InstId::vfmv_f_s:
    case InstId::vfmv_s_f:
      packet.operands_[0].lmul = packet.operands_[1].lmul = 1;
      break;

    case InstId::vwaddu_vv:
    case InstId::vwaddu_vx:
    case InstId::vwsubu_vv:
    case InstId::vwsubu_vx:
    case InstId::vwadd_vv:
    case InstId::vwadd_vx:
    case InstId::vwsub_vv:
    case InstId::vwsub_vx:
    case InstId::vwmulu_vv:
    case InstId::vwmulu_vx:
    case InstId::vwmul_vv:
    case InstId::vwmul_vx:
    case InstId::vwmulsu_vv:
    case InstId::vwmulsu_vx:
    case InstId::vwmaccu_vv:
    case InstId::vwmaccu_vx:
    case InstId::vwmacc_vv:
    case InstId::vwmacc_vx:
    case InstId::vwmaccsu_vv:
    case InstId::vwmaccsu_vx:
    case InstId::vwmaccus_vx:
    case InstId::vwsll_vv:
    case InstId::vwsll_vx:
    case InstId::vwsll_vi:
    case InstId::vfwcvt_xu_f_v:
    case InstId::vfwcvt_x_f_v:
    case InstId::vfwcvt_rtz_xu_f_v:
    case InstId::vfwcvt_rtz_x_f_v:
    case InstId::vfwcvt_f_xu_v:
    case InstId::vfwcvt_f_x_v:
    case InstId::vfwcvt_f_f_v:
    case InstId::vfwcvtbf16_f_f_v:
    case InstId::vfwmaccbf16_vv:
    case InstId::vfwmaccbf16_vf:
    case InstId::vwabda_vv:
    case InstId::vwabda_vx:
    case InstId::vwabdau_vv:
    case InstId::vwabdau_vx:
    case InstId::vzip_vv:
      packet.operands_[0].lmul = effWideLmul;
      break;

    case InstId::vunzipe_v:
    case InstId::vunzipo_v:
      packet.operands_.at(1).lmul = effWideLmul;
      break;

    case InstId::vwaddu_wv:
    case InstId::vwaddu_wx:
    case InstId::vwsubu_wv:
    case InstId::vwsubu_wx:
    case InstId::vwadd_wv:
    case InstId::vwadd_wx:
    case InstId::vwsub_wv:
    case InstId::vwsub_wx:
    case InstId::vfwadd_wv:
    case InstId::vfwadd_wf:
    case InstId::vfwsub_wv:
    case InstId::vfwsub_wf:
      packet.operands_[0].lmul = effWideLmul;
      packet.operands_[1].lmul = effWideLmul;
      break;

    case InstId::vnsrl_wv:
    case InstId::vnsrl_wx:
    case InstId::vnsrl_wi:
    case InstId::vnsra_wv:
    case InstId::vnsra_wx:
    case InstId::vnsra_wi:
    case InstId::vnclipu_wv:
    case InstId::vnclipu_wx:
    case InstId::vnclipu_wi:
    case InstId::vnclip_wv:
    case InstId::vnclip_wx:
    case InstId::vnclip_wi:

    case InstId::vfncvt_xu_f_w:
    case InstId::vfncvt_x_f_w:
    case InstId::vfncvt_rtz_xu_f_w:
    case InstId::vfncvt_rtz_x_f_w :
    case InstId::vfncvt_f_xu_w:
    case InstId::vfncvt_f_x_w :
    case InstId::vfncvt_f_f_w:
    case InstId::vfncvt_rod_f_f_w:
    case InstId::vfncvtbf16_f_f_w:
      packet.operands_[1].lmul = effWideLmul;
      break;

    case InstId::vsext_vf2:
    case InstId::vzext_vf2:
      packet.operands_[1].lmul = effLmul < 2 ? 1 : effLmul / 2;
      break;

    case InstId::vsext_vf4:
    case InstId::vzext_vf4:
      packet.operands_[1].lmul = effLmul < 4 ? 1 : effLmul / 4;
      break;

    case InstId::vsext_vf8:
    case InstId::vzext_vf8:
      packet.operands_[1].lmul = effLmul < 8 ? 1 : effLmul / 8;
      break;

    case InstId::vmseq_vv:
    case InstId::vmseq_vx:
    case InstId::vmseq_vi:
    case InstId::vmsne_vv:
    case InstId::vmsne_vx:
    case InstId::vmsne_vi:
    case InstId::vmsltu_vv:
    case InstId::vmsltu_vx:
    case InstId::vmslt_vv:
    case InstId::vmslt_vx:
    case InstId::vmsleu_vv:
    case InstId::vmsleu_vx:
    case InstId::vmsleu_vi:
    case InstId::vmsle_vv:
    case InstId::vmsle_vx:
    case InstId::vmsle_vi:
    case InstId::vmsgtu_vx:
    case InstId::vmsgtu_vi:
    case InstId::vmsgt_vx:
    case InstId::vmsgt_vi:
    // Vector floating-point compares also produce a mask (single register)
    // regardless of the effective LMUL of their source operands.
    case InstId::vmfeq_vv:
    case InstId::vmfeq_vf:
    case InstId::vmfne_vv:
    case InstId::vmfne_vf:
    case InstId::vmflt_vv:
    case InstId::vmflt_vf:
    case InstId::vmfle_vv:
    case InstId::vmfle_vf:
    case InstId::vmfgt_vf:
    case InstId::vmfge_vf:
      packet.operands_[0].lmul = 1;
      break;

    case InstId::vredsum_vs:
    case InstId::vredand_vs:
    case InstId::vredor_vs:
    case InstId::vredxor_vs:
    case InstId::vredminu_vs:
    case InstId::vredmin_vs:
    case InstId::vredmaxu_vs:
    case InstId::vredmax_vs:
    case InstId::vfredusum_vs:
    case InstId::vfredosum_vs:
    case InstId::vfredmin_vs:
    case InstId::vfredmax_vs:

    case InstId::vwredsumu_vs:
    case InstId::vwredsum_vs:
    case InstId::vfwredusum_vs:
    case InstId::vfwredosum_vs:
      packet.operands_[0].lmul = 1;  // Destination vec operand has lmul of 1.
      packet.operands_[2].lmul = 1;  // Third vec operand has lmul of 1.
      break;

    case InstId::vmand_mm:
    case InstId::vmnand_mm:
    case InstId::vmandn_mm:
    case InstId::vmxor_mm:
    case InstId::vmor_mm:
    case InstId::vmnor_mm:
    case InstId::vmorn_mm:
    case InstId::vmxnor_mm:
    case InstId::vcpop_m:
    case InstId::vfirst_m:
    case InstId::vmsbf_m:
    case InstId::vmsif_m:
    case InstId::vmsof_m:
      packet.operands_[0].lmul = packet.operands_[1].lmul = packet.operands_[2].lmul = 1;
      break;
    case InstId::viota_m:
      packet.operands_.at(1).lmul = packet.operands_.at(2).lmul = 1;
      break;

    case InstId::vrgatherei16_vv:
      {
        unsigned op2g8 = (16*groupX8) / vecRegs.elemWidthInBits();
        unsigned op2Lmul = op2g8 <= 8 ? 1 : op2g8 / 8;
        packet.operands_[2].lmul = op2Lmul;
      }
      break;

    case InstId::vaesdf_vs:
    case InstId::vaesef_vs:
    case InstId::vaesem_vs:
    case InstId::vaesdm_vs:
    case InstId::vaesz_vs:
    case InstId::vsm4r_vs:
      packet.operands_[1].lmul = 1;
      break;

    case InstId::vqwdotau_vv:
    case InstId::vqwdotas_vv:
    case InstId::vqwbdotau_vv:
    case InstId::vqwbdotas_vv:
    case InstId::vfbdota_vv:
    case InstId::vfwdota_vv:
    case InstId::vfqwdota_vv:
    case InstId::vfqwbdota_vv:
    case InstId::vfwbdota_vv:
      packet.operands_[0].lmul = 1;
      break;

    default:
      break;
    }
}


template <typename URV>
void
PerfApi<URV>::undoDestRegRename(unsigned hartIx, const InstrPac& packet)
{
  auto& producers = hartRegProducers_[hartIx];
  const size_t opCount = packet.operandCount_;
  const uint64_t packetTag = packet.tag();

  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  for (size_t i = 0; i < opCount; ++i)
    {
      using OM = WdRiscv::OperandMode;
      using OT = WdRiscv::OperandType;

      const auto& op = packet.operands_[i];

      auto mode = op.mode;
      if (mode == OM::Write or mode == OM::ReadWrite)
        {
          unsigned regNum = op.number;
          auto type = op.type;
          unsigned gri = globalRegIx(type, regNum);
          if (type != OT::VecReg)
            {
              auto& producer = producers[gri];
              if (producer and producer->tag() == packetTag)
                producer = nullptr;
            }
          else
            {
              assert(op.lmul <= maxEffLmul_);
              for (unsigned n = 0; n < op.lmul; ++n)
                {
                  auto& producer = producers[gri + n];
                  if (producer and producer->tag() == packetTag)
                    producer = nullptr;
                }
            }
        }
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)
}


template <typename URV>
bool
PerfApi<URV>::collectOperandValues(HartType& hart, InstrPac& packet)
{
  bool peekOk = true;

  auto hartIx = hart.sysHartIndex();
  auto tag = packet.tag();

  using OT = WdRiscv::OperandType;

  unsigned vecRegSize = hart.vecRegSize();
  const unsigned opCount = packet.operandCount_;

  // NOLINTBEGIN(cppcoreguidelines-pro-bounds-constant-array-index)
  for (unsigned i = 0; i < opCount; ++i)
    {
      auto& op = packet.operands_[i];

      auto type = op.type;
      if (type == OT::Imm) [[unlikely]]
        continue;  // Value obtained at decode.

      assert(op.mode != WdRiscv::OperandMode::None);

      unsigned regNum = op.number;
      OpVal opVal;

      auto& iop = packet.opProducers_[i];   // Ith operand producer

      if (type != OT::VecReg) [[likely]]
        {
          auto* producer = iop.scalar.get();
          if (producer)
            {
              if (not producer->executed()) [[unlikely]]
                {
                  cerr << "Error: PerfApi::execute: Hart-ix=" << hartIx << "tag=" << tag
                       << " depends on tag=" << producer->tag_ << " which is not yet executed.\n";
                  assert(0 && "Error: Assertion failed");
                  return false;
                }
              if (not getDestValue(*producer, type, regNum, opVal))
                assert(0);
            }
          else
            peekOk = peekRegister(hart, type, regNum, opVal) and peekOk;
        }
      else
        {
          const size_t vecSize = iop.vec.size();
          assert(vecSize <= maxEffLmul_);
          opVal.vec.reserve(vecSize * vecRegSize);
          for (unsigned n = 0; n < vecSize; ++n)
            {
              OpVal val;
              auto* producer = iop.vec[n].get();
              if (producer)
                {
                  if (not producer->executed()) [[unlikely]]
                    {
                      cerr << "Error: PerfApi::execute: Hart-ix=" << hartIx << "tag=" << tag
                           << " depends on tag=" << producer->tag_ << " which is not yet executed.\n";
                      assert(0 && "Error: Assertion failed");
                      return false;
                    }
                  peekOk = getVecDestValue(*producer, regNum + n, vecRegSize, val) and peekOk;
                }
              else
                peekOk = peekRegister(hart, type, regNum+n, val) and peekOk;

              // Append val to opVal.
              opVal.vec.insert(opVal.vec.end(), val.vec.begin(), val.vec.end());
            }
        }

      op.value = std::move(opVal);
    }
  // NOLINTEND(cppcoreguidelines-pro-bounds-constant-array-index)

  return peekOk;
}


template <typename URV>
void
PerfApi<URV>::determineExplicitOperands(InstrPac& packet)
{
  using OM = WdRiscv::OperandMode;
  using OT = WdRiscv::OperandType;

  const auto& di = packet.decodedInst();

  using enum WdRiscv::InstId;
  auto id = di.instId();
  bool twoDest = (id == amocas_q) or (id == amocas_d and sizeof(URV) == 4);

  packet.operandCount_ = 0;
  const unsigned diOpCount = di.operandCount();

  for (unsigned i = 0; i < diOpCount; ++i)
    {
      auto mode = di.effectiveIthOperandMode(i);
      auto type = di.ithOperandType(i);

      if (mode == OM::None)
        assert(type == OT::Imm);

      // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
      auto& op = packet.operands_[packet.operandCount_++];
      op.type = type;
      op.mode = mode;
      op.number = di.ithOperand(i);     // Irrelevant for immediate ops.

      if (twoDest and i == 0)
        {
          // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
          auto& op2 = packet.operands_[packet.operandCount_++];
          op2.type = op.type;
          op2.mode = op.mode;
          op2.number = op.number+1;
        }

      if (op.type == OT::CsReg)
        {
          using CN = WdRiscv::CsrNumber;
          if (op.number == unsigned(CN::FRM) or op.number == unsigned(CN::FFLAGS))
            op.number = unsigned(CN::FCSR);
          if (op.number == unsigned(CN::VXRM) or op.number == unsigned(CN::VXSAT))
            op.number = unsigned(CN::VCSR);
        }
      else if (op.type == OT::Imm)
        op.value.scalar = di.ithOperand(i);
    }
}


template <typename URV>
void
PerfApi<URV>::determineImplicitOperands(InstrPac& packet)
{
  using OM = WdRiscv::OperandMode;
  using OT = WdRiscv::OperandType;

  const auto& di = packet.decodedInst();

  // Determine implicit operands. Vtype is an implicit source for all vector instructions.
  // It is an implicit destination for the vsetvl/vsetvli/vsetivli. Same for VL.
  if (di.isVector())
    {
      using WdRiscv::InstId;
      auto id = di.instId();
      bool isVset = (id == InstId::vsetvl or id == InstId::vsetvli or id == InstId::vsetivli);

      if (di.isMasked() and not isVset)
        {
          // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
          auto& v0Op = packet.operands_[packet.operandCount_++];
          v0Op.type = OT::VecReg;
          v0Op.mode = OM::Read;
          v0Op.number = 0;
          v0Op.lmul = 1;
        }

      // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
      auto& vtOp = packet.operands_[packet.operandCount_++];
      vtOp.type = OT::CsReg;
      vtOp.mode = isVset? OM::Write : OM::Read;
      vtOp.number = unsigned(CSRN::VTYPE);

      // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
      auto& vlOp = packet.operands_[packet.operandCount_++];
      vlOp.type = OT::CsReg;
      if (di.isVectorLoadFaultFirst())
        vlOp.mode = OM::ReadWrite;
      else
        vlOp.mode = isVset ? OM::Write : OM::Read;
      vlOp.number = unsigned(CSRN::VL);

      // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
      auto& vsOp = packet.operands_[packet.operandCount_++];
      vsOp.type = OT::CsReg;
      vsOp.mode = OM::ReadWrite;
      vsOp.number = unsigned(CSRN::VSTART);
      if (di.isVectorFp())   // FCSR implicit for FP instrs.
        {
          // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
          auto& fcsrOp = packet.operands_[packet.operandCount_++];
          fcsrOp.type = OT::CsReg;
          fcsrOp.mode = OM::ReadWrite;
          fcsrOp.number = unsigned(CSRN::FCSR);
        }
      else if (di.isVectorFixedPoint())  // VCSR implicit for fixed point instrs.
        {
          // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
          auto& vcsrOp = packet.operands_[packet.operandCount_++];
          vcsrOp.type = OT::CsReg;
          vcsrOp.mode = OM::ReadWrite;
          vcsrOp.number = unsigned(CSRN::VCSR);
        }
    }
  else if (di.isFp() and (di.modifiesFflags() or di.hasDynamicRoundingMode()))
    {
      // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index)
      auto& op = packet.operands_[packet.operandCount_++];
      op.type = OT::CsReg;
      op.mode = OM::Read;
      op.number = unsigned(CSRN::FCSR);
      if (di.modifiesFflags())
        op.mode = di.hasDynamicRoundingMode() ? OM::ReadWrite : OM::Write;
    }
}


template <typename URV>
void
PerfApi<URV>::flattenOperand(const Operand& op, std::vector<Operand>& flat) const
{
  flat.clear();

  auto hart0 = system_.ithHart(0);
  unsigned vecSize = hart0->vecRegs().bytesPerRegister();

  if (op.type != OperandType::VecReg or op.value.vec.size() <= vecSize)
    {
      flat.push_back(op);
      return;
    }

  unsigned remains = op.value.vec.size();  // Count of remaining bytes to flatten.
  unsigned vecIx = op.number;
  unsigned offset = 0;  // Offset into data of original operand.

  while (remains > 0)
    {
      Operand flatOp;
      flatOp.type = op.type;
      flatOp.mode = op.mode;
      flatOp.number = vecIx++;
      flatOp.lmul = 1;

      unsigned chunk = std::min(remains, vecSize);
      remains -= chunk;

      auto& dest = flatOp.value.vec;
      const auto& src = op.value.vec;
      dest.insert(dest.end(),  src.begin() + offset,  src.begin() + offset + chunk);
      offset += chunk;

      flat.push_back(flatOp);
    }
}


template <typename URV>
bool
PerfApi<URV>::getDestValue(const InstrPac& producer, WdRiscv::OperandType regType,
                      unsigned regNum, OpVal& val)
{
  assert(producer.executed());

  auto gri = globalRegIx(regType, regNum);
  for (const auto& p : producer.destValues_)
    if (p.first == gri)
      {
        val = p.second;
        return true;
      }

  assert(0 && "Error: Assertion failed");
  return false;
}

// Explicit template instantiations for RV32 and RV64
template class TT_PERF::PerfApi<uint32_t>;
template class TT_PERF::PerfApi<uint64_t>;

template uint64_t TT_PERF::InstrPac::executedDestVal<uint32_t>(const WdRiscv::Hart<uint32_t>&, unsigned, unsigned, unsigned) const;
template uint64_t TT_PERF::InstrPac::executedDestVal<uint64_t>(const WdRiscv::Hart<uint64_t>&, unsigned, unsigned, unsigned) const;