diff --git a/contrib/offline-replay/run_offline_replay_backtest.sh b/contrib/offline-replay/run_offline_replay_backtest.sh index 688c9dfcf4d..312d92a2ecd 100755 --- a/contrib/offline-replay/run_offline_replay_backtest.sh +++ b/contrib/offline-replay/run_offline_replay_backtest.sh @@ -316,7 +316,11 @@ build_firedancer() { EXTRAS=offline-replay make -j OBJDIR=$(EXTRAS=offline-replay make --silent --no-print-directory objdir 2>/dev/null || true) - : "${OBJDIR:?cannot determine OBJDIR (make objdir failed)}" + if [ -z "$OBJDIR" ] || [ ! -x "$OBJDIR/bin/firedancer-dev" ]; then + send_slack_message "@here offline-replay: firedancer-dev not found at \`${OBJDIR:-}/bin\` after build (OBJDIR/build-config mismatch, not a ledger failure). Exiting." + exit 1 + fi + export OBJDIR } # Convert the downloaded rocksdb into a shredcap capture covering the replay @@ -614,6 +618,7 @@ replay_until_clean() { # config, permissions, ...) — that is an infrastructure problem, not # a ledger mismatch, so there is nothing to minimize. if [ "$status" -ne 0 ] && [ ! -s "$TEMP_LOG" ]; then + sudo -E $OBJDIR/bin/firedancer-dev configure fini all --config $LEDGER_DIR/offline_replay.toml &> /dev/null || true send_slack_message "@here Backtest produced no log output; firedancer-dev likely failed at startup. See the configure output in the harness log. Exiting." exit 1 fi diff --git a/src/flamenco/runtime/tests/run_ledger_backtest.sh b/src/flamenco/runtime/tests/run_ledger_backtest.sh index b35b90e0a92..2fe09ad3edb 100755 --- a/src/flamenco/runtime/tests/run_ledger_backtest.sh +++ b/src/flamenco/runtime/tests/run_ledger_backtest.sh @@ -294,6 +294,8 @@ fi echo_notice "Running backtest for $LEDGER" +[ -x "$OBJDIR/bin/firedancer-dev" ] || { echo "error: firedancer-dev not found at \"$OBJDIR/bin\" -- build it, or set OBJDIR/EXTRAS" >&2; exit 1; } + sudo killall firedancer-dev &> /dev/null || true set -x @@ -325,6 +327,9 @@ if [ "$status" -eq 0 ]; then exit 0 fi +if [ -z "$CI" ] && grep -q "Unable to set the thread affinity" "$LOG"; then + $OBJDIR/bin/firedancer-dev configure fini all --config ${DUMP_DIR}/${LEDGER}_backtest.toml &> /dev/null || true +fi tail -n 10 $LOG echo "Failed with status: $status"