diff --git a/app/mining/page.tsx b/app/mining/page.tsx index f885f1c3..c65ee270 100644 --- a/app/mining/page.tsx +++ b/app/mining/page.tsx @@ -13,7 +13,7 @@ import { XAxis, YAxis, CartesianGrid, ResponsiveContainer, Legend, } from 'recharts'; import { ChartTooltip as Tooltip } from '@/components/charts/ChartTooltip'; -import { getApiUrl } from '@/lib/api-config'; +import { getApiUrl, NETWORK } from '@/lib/api-config'; import { useTheme } from '@/contexts/ThemeContext'; import { categoryColor } from '@/lib/category-colors'; import { getChartColors } from '@/lib/chart-theme'; @@ -119,7 +119,7 @@ function PeriodSelector({ value, onChange }: { value: Period; onChange: (p: Peri function DistributionSection() { const { theme } = useTheme(); - const [period, setPeriod] = useState('7d'); + const [period, setPeriod] = useState('30d'); const [data, setData] = useState([]); const [total, setTotal] = useState(0); const [loading, setLoading] = useState(true); @@ -169,7 +169,7 @@ function DistributionSection() { function RankingSection() { const { theme } = useTheme(); - const [period, setPeriod] = useState('7d'); + const [period, setPeriod] = useState('30d'); const [ranking, setRanking] = useState([]); const [total, setTotal] = useState(0); const [loading, setLoading] = useState(true); @@ -316,7 +316,7 @@ function ChartModeToggle({ mode, onChange }: { mode: ChartMode; onChange: (m: Ch function HashrateShareSection() { const { theme } = useTheme(); const colors = getChartColors(theme); - const [period, setPeriod] = useState('all'); + const [period, setPeriod] = useState('30d'); const [chartMode, setChartMode] = useState('line'); const [series, setSeries] = useState([]); const [allPools, setAllPools] = useState([]); @@ -514,25 +514,41 @@ function HashrateShareSection() { function MinerBehaviorSection() { const { theme } = useTheme(); const colors = getChartColors(theme); - const [period, setPeriod] = useState('all'); + const [period, setPeriod] = useState('30d'); const [series, setSeries] = useState([]); const [summary, setSummary] = useState(null); const [message, setMessage] = useState(null); + const [coverage, setCoverage] = useState<{ firstDate: string; lastDate: string; refreshedAt: string | null; freshness: 'current' | 'stale' | 'unknown' } | null>(null); const [loading, setLoading] = useState(true); + const currency = NETWORK === 'mainnet' ? 'ZEC' : 'TAZ'; useEffect(() => { + const controller = new AbortController(); setLoading(true); - fetch(`${getApiUrl()}/v1/mining/miner-behavior?period=${period}`) - .then(r => r.ok ? readApiData(r) : null) - .then(res => { - if (res) { - setSeries(res.series || []); - setSummary(res.summary || null); - setMessage(res.message || null); - } - setLoading(false); + setSeries([]); + setSummary(null); + setMessage(null); + setCoverage(null); + fetch(`${getApiUrl()}/v1/mining/miner-behavior?period=${period}`, { signal: controller.signal }) + .then(async response => { + if (!response.ok) throw new Error('Miner behavior unavailable'); + return readApiData(response); }) - .catch(() => setLoading(false)); + .then(data => { + if (controller.signal.aborted) return; + if (!data || !Array.isArray(data.series)) throw new Error('Invalid miner behavior response'); + setSeries(data.series); + setSummary(data.series.length ? data.summary || null : null); + setCoverage(data.coverage || null); + setMessage(data.series.length ? null : data.message || 'No miner reward spending snapshots are available for this period.'); + }) + .catch(() => { + if (!controller.signal.aborted) setMessage('Miner behavior data is temporarily unavailable. Try again later.'); + }) + .finally(() => { + if (!controller.signal.aborted) setLoading(false); + }); + return () => controller.abort(); }, [period]); const chartData = series.map(p => ({ @@ -560,8 +576,8 @@ function MinerBehaviorSection() {

- How much of their block rewards miners move vs leave unspent. A high moved ratio means rewards changed address - quickly; it does not prove they were sold. The{' '} + Spending of attributed miner payout outputs, grouped by the date the reward was earned. A movement means + an output was spent; it does not prove a sale. Amounts reflect current indexed spending when refreshed, rather than spending on the reward date. Unattributed payouts are excluded. The{' '} ZODL leaderboard breaks each pool's spending down by destination (shielded vs. exchange vs. transparent), showing observed transfers rather than proving sales.

@@ -572,7 +588,7 @@ function MinerBehaviorSection() {

Total Earned

- {formatZec(summary.totalEarnedZat)} ZEC + {formatZec(summary.totalEarnedZat)} {currency}

@@ -580,7 +596,7 @@ function MinerBehaviorSection() {

Total Moved

- {formatZec(summary.totalSpentZat)} ZEC + {formatZec(summary.totalSpentZat)} {currency}

@@ -588,13 +604,13 @@ function MinerBehaviorSection() {

Still Held

- {formatZec(summary.totalHeldZat)} ZEC + {formatZec(summary.totalHeldZat)} {currency}

-

Sell Ratio

+

Moved Ratio

{(summary.overallSellRatio * 100).toFixed(1)}%

@@ -610,11 +626,10 @@ function MinerBehaviorSection() { > {loading ? ( - ) : message ? ( + ) : message || !chartData.length ? (
-

{message}

-

Run the snapshot job to populate this data.

+

{message || 'No miner reward spending snapshots are available for this period.'}

) : ( @@ -634,7 +649,7 @@ function MinerBehaviorSection() { stroke={colors.axis} tick={{ fill: colors.axis, fontSize: 12 }} tickFormatter={(v: number) => `${v.toFixed(0)}`} - label={{ value: 'ZEC', angle: -90, position: 'insideLeft', fill: colors.axis, fontSize: 12 }} + label={{ value: currency, angle: -90, position: 'insideLeft', fill: colors.axis, fontSize: 12 }} /> { - const label = name === 'earned' ? 'Earned' : name === 'spent' ? 'Moved/Sold' : 'Held'; - return [`${Number(value).toFixed(2)} ZEC`, label]; + const label = name === 'earned' ? 'Earned' : name === 'spent' ? 'Moved' : 'Held'; + return [`${Number(value).toFixed(2)} ${currency}`, label]; }} labelFormatter={(label) => String(label)} /> value === 'earned' ? 'Earned' : value === 'spent' ? 'Moved/Sold' : 'Held'} + formatter={(value) => value === 'earned' ? 'Earned' : value === 'spent' ? 'Moved' : 'Held'} /> @@ -664,6 +679,13 @@ function MinerBehaviorSection() { )} + {coverage && series.length > 0 && ( +

+ Available reward dates: {coverage.firstDate}–{coverage.lastDate}. + {coverage.refreshedAt && ` Oldest snapshot refresh: ${new Date(coverage.refreshedAt).toLocaleString(undefined, { timeZone: 'UTC' })} UTC.`} + {coverage.freshness === 'stale' && ' Some snapshots are more than 48 hours old.'} +

+ )} ); } diff --git a/components/mining/MiningSoftwareSection.tsx b/components/mining/MiningSoftwareSection.tsx index 1e68b7c2..af289be5 100644 --- a/components/mining/MiningSoftwareSection.tsx +++ b/components/mining/MiningSoftwareSection.tsx @@ -63,7 +63,7 @@ export function MiningSoftwareSection() { const { theme } = useTheme(); const colors = getChartColors(theme); const palette = getMiningSoftwareColors(theme); - const [period, setPeriod] = useState("all"); + const [period, setPeriod] = useState("30d"); const [range, setRange] = useState<{ from: string; to: string } | null>(null); const [bucket, setBucket] = useState("auto"); const [data, setData] = useState(null); diff --git a/components/mining/OrphanRateSection.tsx b/components/mining/OrphanRateSection.tsx index b887c645..668b7a28 100644 --- a/components/mining/OrphanRateSection.tsx +++ b/components/mining/OrphanRateSection.tsx @@ -43,7 +43,7 @@ function utc(value: string) { } export function OrphanRateSection() { - const [period, setPeriod] = useState('7d'); + const [period, setPeriod] = useState('30d'); const { data, loading, error } = useApiQuery('/v1/mining/orphan-rates', undefined, { refreshInterval: 60_000 }); const supported = data?.method === 'observed-indexer-reorg-v1' || data?.method === 'observed-block-capture-v2'; const selected = supported ? data.periods.find(window => window.period === period) : null; diff --git a/components/network/MiningMetricsChart.tsx b/components/network/MiningMetricsChart.tsx index 7686a5d9..47d117b1 100644 --- a/components/network/MiningMetricsChart.tsx +++ b/components/network/MiningMetricsChart.tsx @@ -9,6 +9,7 @@ import { useTheme } from '@/contexts/ThemeContext'; import { getChartColors } from '@/lib/chart-theme'; import { formatDifficulty, formatHashrate } from '@/lib/format-numbers'; import { useApiQuery } from '@/hooks/useApiQuery'; +import { CURRENCY } from '@/lib/config'; import { ChartCard } from './ChartCard'; type MetricKey = 'solrate' | 'difficulty' | 'blockTime' | 'txFees' | 'txCount'; @@ -17,7 +18,7 @@ const METRICS: { key: MetricKey; label: string; color: string; format: (v: numbe { key: 'solrate', label: 'Solrate', color: 'gold', format: (v) => formatHashrate(v) }, { key: 'difficulty', label: 'Difficulty', color: 'yellow', format: (v) => formatDifficulty(v) }, { key: 'blockTime', label: 'Block time', color: 'green', format: (v) => `~${Math.round(v)}s` }, - { key: 'txFees', label: 'TX fees', color: 'purple', format: (v) => `${v.toFixed(6)} ZEC` }, + { key: 'txFees', label: 'TX fees', color: 'purple', format: (v) => `${v.toFixed(6)} ${CURRENCY}` }, { key: 'txCount', label: 'TX count', color: 'gold', format: (v) => v.toFixed(1) }, ]; @@ -25,6 +26,7 @@ interface MiningMetricsData { points?: ({ height: number } & Record)[]; latest?: Partial>; window?: number; + solrateMethod?: string; } export function MiningMetricsChart() { @@ -33,7 +35,7 @@ export function MiningMetricsChart() { const [active, setActive] = useState('solrate'); const window = 20; - const { data } = useApiQuery( + const { data, loading, error } = useApiQuery( '/v1/mining/metrics', { window, limit: 120 }, ); @@ -42,6 +44,11 @@ export function MiningMetricsChart() { const metric = METRICS.find((m) => m.key === active)!; const stroke = metric.color === 'gold' ? colors.gold : metric.color === 'yellow' ? colors.yellow : metric.color === 'green' ? colors.orchard : colors.purple; + const hasValues = points.some((point) => point[active] != null && Number.isFinite(point[active])); + const elapsedSolrate = active === 'solrate' && data?.solrateMethod === 'difficulty-work-over-elapsed-v1'; + const emptyMessage = loading ? 'Loading mining metrics…' + : error ? 'Mining metrics are temporarily unavailable.' + : `No complete ${window}-block window is available for ${metric.label.toLowerCase()}.`; return (
@@ -60,14 +67,18 @@ export function MiningMetricsChart() {

{value != null ? m.format(value) : '—'}

-

{window} blk avg

+

{window} blk {m.key === 'solrate' && data?.solrateMethod === 'difficulty-work-over-elapsed-v1' ? 'estimate' : 'avg'}

); })}
-

{window}-block trailing averages. Timing uses block-header timestamps; unavailable values remain gaps.

+

{elapsedSolrate + ? `${window}-block estimates use total difficulty and elapsed header time; unavailable values remain gaps.` + : `${window}-block trailing averages. Timing uses block-header timestamps; unavailable values remain gaps.`}

+ {error && hasValues &&

Refresh unavailable. Showing the last received mining metrics.

} + {!hasValues ?
{emptyMessage}
: @@ -93,9 +104,9 @@ export function MiningMetricsChart() { labelFormatter={(h) => `Block ${h}`} formatter={(value) => [value == null ? 'Unavailable' : metric.format(Number(value)), metric.label]} /> - + - + } ); diff --git a/components/network/NetworkAccounting.tsx b/components/network/NetworkAccounting.tsx index 5d6c2834..1e2082d1 100644 --- a/components/network/NetworkAccounting.tsx +++ b/components/network/NetworkAccounting.tsx @@ -14,12 +14,13 @@ type Accounting = { schedule: { network: string; nu7Height: number | null } | nu block: { height: number; feesPaidZat: number | string | null; feesToNsmZat: number | string | null; minerFeeAllocationZat: number | string | null; minerSubsidyZat: number | string | null; minerReceiptsZat: number | string | null; reissuanceZat: number | string | null } | null }; -type Totals = { blocks: number; feeBlocks: number; nsmSamples: number; missingBlocks: number; feesPaidZat: string | null; +type Totals = { blocks: number; feeBlocks: number; receiptBlocks?: number; nsmSamples: number; missingBlocks: number; feesPaidZat: string | null; feesToNsmZat: string | null; minerFeeAllocationZat: string | null; firstHeight: number | null; lastHeight: number | null; firstTimestamp: number | null; lastTimestamp: number | null }; -type Point = { timestamp: number; firstHeight: number; lastHeight: number; blocks: number; feeBlocks: number; nsmSamples: number; +type Point = { timestamp: number; firstHeight: number; lastHeight: number; blocks: number; feeBlocks: number; receiptBlocks?: number; nsmSamples: number; missingBlocks: number; feesPaidZat: string | null; feesToNsmZat: string | null; minerFeeAllocationZat: string | null; - minerReceiptsZat: string | null; nsmBalanceZat: string | null; cumulativeRemovalZat: string | null }; + minerReceiptsZat: string | null; nsmBalanceZat: string | null; cumulativeRemovalZat: string | null; + nsmSampleHeight?: number | null; nsmSampleHash?: string | null; nsmSampleTimestamp?: number | null }; type Period = '1d' | '7d' | '30d' | 'all'; type History = { period: Period; schedule: Accounting['schedule']; nodeHeight: number; indexedHeight: number | null; bucketSeconds: number; points: Point[]; totals: Totals; selected: Totals; @@ -54,20 +55,25 @@ export function NetworkAccounting() { Number.isSafeInteger(activationHeight) && activationHeight > 0 && activationHeight <= 499_999_999 && Number.isSafeInteger(data.nodeHeight) && data.nodeHeight >= activationHeight && data.nodeHeight <= 499_999_999; const [metric, setMetric] = useState('fees'); - const [period, setPeriod] = useState('1d'); + const [period, setPeriod] = useState('30d'); const history = useApiQuery('/v1/network/accounting/history', { period }, { enabled: nu7Active, refreshInterval: 30_000 }); // The hook retains previous query data. Never label an old range or network // as the newly selected period while its request is still in flight. const historySnapshot = history.data?.schedule?.network === expectedChain && history.data.schedule.nu7Height === activationHeight ? history.data : null; const selected = historySnapshot?.period === period ? historySnapshot : null; const points = (selected?.points ?? []).flatMap((point, i, all) => { - const plotted = { ...point, ...Object.fromEntries(Object.entries(point).filter(([key]) => key.endsWith('Zat')) + const plotted = { ...point, chartTimestamp: metric === 'reserve' ? point.nsmSampleTimestamp ?? point.timestamp : point.timestamp, + ...Object.fromEntries(Object.entries(point).filter(([key]) => key.endsWith('Zat')) .map(([key, value]) => [key + 'Display', value == null || !Number.isFinite(Number(value)) ? null : Number(value) / 1e8])) }; return i > 0 && point.timestamp - all[i-1].timestamp > selected!.bucketSeconds - ? [{ timestamp: all[i-1].timestamp + selected!.bucketSeconds }, plotted] : [plotted]; + ? [{ chartTimestamp: all[i-1].timestamp + selected!.bucketSeconds }, plotted] : [plotted]; }); const series = historySeries[metric]; const hasValues = selected?.points.some(point => series.some(([key]) => point[key] != null)); + const completeReceiptBuckets = selected?.points.filter(point => point.minerReceiptsZat != null && Number.isFinite(Number(point.minerReceiptsZat))).length ?? 0; + const reserveBuckets = selected?.points.filter(point => point.nsmBalanceZat != null).length ?? 0; + const receiptBlocks = selected?.selected.receiptBlocks; + const hasReceiptCoverage = Number.isSafeInteger(receiptBlocks) && receiptBlocks! >= 0 && receiptBlocks! <= (selected?.selected.blocks ?? 0); if (!nu7Active) return null; const block = data.block; const bucketLabel = selected ? selected.bucketSeconds < 3600 ? `${selected.bucketSeconds/60}-minute` : selected.bucketSeconds < 86400 @@ -113,7 +119,7 @@ export function NetworkAccounting() { {hasValues ? - utc(Number(v), period === '1d')} + utc(Number(v), period === '1d')} minTickGap={45} padding={{ left: 12, right: 32 }} tick={{ fill: colors.axis, fontSize: 12 }} tickLine={false} axisLine={false} /> Number(v).toLocaleString('en-US', metric === 'reserve' ? { maximumFractionDigits: 8 } : { maximumSignificantDigits: 4 })} @@ -121,11 +127,18 @@ export function NetworkAccounting() {
{utc(Number(timestamp))} UTC {payload?.[0]?.payload?.firstHeight != null &&
Blocks {payload[0].payload.firstHeight.toLocaleString()}–{payload[0].payload.lastHeight.toLocaleString()} {metric === 'fees' &&
Fees paid: {zec(payload[0].payload.feesPaidZat)}
}
} + {metric === 'reserve' && payload?.[0]?.payload?.nsmSampleHeight != null &&
+ Recorded at block {payload[0].payload.nsmSampleHeight.toLocaleString()} +
{payload[0].payload.nsmSampleHash}
+
Block-header time: {new Date(payload[0].payload.nsmSampleTimestamp * 1000).toISOString()}
+ {payload[0].payload.lastHeight > payload[0].payload.nsmSampleHeight &&
{payload[0].payload.lastHeight - payload[0].payload.nsmSampleHeight} blocks before this bucket’s last block
} +
}
} formatter={(_value, name, item) => [zec(item.payload[String(item.dataKey).replace(/Display$/, '')]), name]} /> - {series.map(([key, name, color]) => metric === 'fees' - ? - : )} + {series.map(([key, name, color]) => metric === 'fees' || metric === 'receipts' + ? + : )} :

{history.loading || (!selected && !history.error) ? 'Loading selected accounting period…' : history.error ? 'Accounting history unavailable.' : 'No complete observations for this metric in this period.'}

} @@ -133,10 +146,12 @@ export function NetworkAccounting() { {series.map(([key, name, color]) =>
    • )} }

    {metric === 'fees' ? 'Total column height = fees paid.' : metric === 'cumulative' ? 'Cumulative minimum removal starts at NU7 activation, including earlier blocks outside the selected range.' - : metric === 'reserve' ? 'Reserve balance at the last block sampled in each bucket.' : 'Actual miner receipts summed within each bucket.'} Amounts in {CURRENCY}; exact values in tooltips.

    + : metric === 'reserve' ? 'Last recorded reserve observation within each bucket, plotted at its block-header time. Sample block and time are in tooltips.' : 'Each column sums actual miner receipts in a complete bucket.'} Amounts in {CURRENCY}; exact values in tooltips.

    {selected &&

    {bucketLabel} buckets · Block-header times in UTC · {selected.selected.firstTimestamp != null ? `${utc(selected.selected.firstTimestamp)}–${utc(selected.selected.lastTimestamp!)}` : 'No blocks in this period'}.

    -

    {selected.selected.feeBlocks.toLocaleString()} of {selected.selected.blocks.toLocaleString()} blocks have complete fees · {selected.selected.nsmSamples.toLocaleString()} reserve samples{selected.selected.missingBlocks ? ` · ${selected.selected.missingBlocks.toLocaleString()} missing blocks` : ''}. Missing observations remain gaps.

    + {metric === 'receipts' ?

    {hasReceiptCoverage ? `${receiptBlocks!.toLocaleString()} of ${selected.selected.blocks.toLocaleString()} blocks have miner receipts` : 'Block receipt coverage unavailable'} · {completeReceiptBuckets.toLocaleString()} of {selected.points.length.toLocaleString()} reported buckets have complete receipts{selected.selected.missingBlocks ? ` · ${selected.selected.missingBlocks.toLocaleString()} missing blocks` : ''}. Incomplete receipt buckets remain gaps.

    + : metric === 'reserve' ?

    {selected.selected.nsmSamples.toLocaleString()} recorded reserve samples · {reserveBuckets.toLocaleString()} of {selected.points.length.toLocaleString()} reported buckets have a reserve observation{selected.selected.missingBlocks ? ` · ${selected.selected.missingBlocks.toLocaleString()} missing blocks` : ''}. Buckets without observations remain gaps; values are not interpolated.

    + :

    {selected.selected.feeBlocks.toLocaleString()} of {selected.selected.blocks.toLocaleString()} blocks have complete fees{selected.selected.missingBlocks ? ` · ${selected.selected.missingBlocks.toLocaleString()} missing blocks` : ''}. Incomplete fee observations remain gaps.

    } {selected.selected.firstHeight === activationHeight && period !== 'all' &&

    Available history begins at NU7 activation; this period is not yet fully elapsed.

    }
    } {(history.error || (selected?.indexedHeight != null && selected.nodeHeight > selected.indexedHeight)) &&

    {history.error ? 'History refresh unavailable. Displayed data may be stale.' : `History is ${selected!.nodeHeight - selected!.indexedHeight!} blocks behind the node.`}

    } @@ -156,7 +171,7 @@ export function NetworkAccounting() {

    All ranges include only blocks at or after the serving node’s NU7 activation height. Since NU7 includes the full available activation history. Amounts require complete indexed transactions; missing blocks or fees make the affected total unavailable.

    Actual miner receipts exclude funding and founders’ payouts. Subsidy allocations are shown once in Issuance below.

    The signed NSM reserve is separate from circulating supply and cumulative fees. Reserve growth subtracts the hash-matched sample immediately before activation from the latest indexed block sample. Missing baseline or end samples make growth unavailable. The current node reserve can be newer than the history.

    -

    Reserve and miner-receipt samples begin when collection is enabled. Missing bucket-end reserve samples remain gaps. The RPC does not separately report reissuance.

    +

    Reserve history uses the last recorded canonical observation within each bucket and selected period. Sample-free buckets or missing canonical blocks remain gaps. Each point identifies its actual sample block and header time; no values are interpolated or carried into empty buckets. Miner receipts use complete indexed coinbase payouts and retained or historical node subsidy allocations. The RPC does not separately report reissuance.

    ; diff --git a/components/network/NetworkHashrateChart.tsx b/components/network/NetworkHashrateChart.tsx index d3f227bd..96daeea6 100644 --- a/components/network/NetworkHashrateChart.tsx +++ b/components/network/NetworkHashrateChart.tsx @@ -51,7 +51,7 @@ function HashratePeriodSelector({ value, onChange }: { value: Period; onChange: export function NetworkHashrateChart() { const { theme } = useTheme(); const colors = getChartColors(theme); - const [period, setPeriod] = useState('all'); + const [period, setPeriod] = useState('30d'); const [window, setWindow] = useState('24h'); const stats = useApiQuery('/v1/network/stats', undefined, { refreshInterval: 30_000 }); diff --git a/components/network/NetworkHistoryCharts.tsx b/components/network/NetworkHistoryCharts.tsx index bae819f1..212564fc 100644 --- a/components/network/NetworkHistoryCharts.tsx +++ b/components/network/NetworkHistoryCharts.tsx @@ -27,9 +27,9 @@ export function NetworkHistoryCharts({ initialData, initialFetchedAt }: { initia const { theme } = useTheme(); const colors = getChartColors(theme); - const { data, loading } = useApiQuery( + const { data, loading, error } = useApiQuery( '/v1/network/chain-size-history', - { period: 'all' }, + { period: '30d' }, { refreshInterval: 300_000, initialFetchedAt, initialData: initialData ?? undefined }, ); const sizePoints = useMemo( @@ -42,7 +42,7 @@ export function NetworkHistoryCharts({ initialData, initialFetchedAt }: { initia return ( -

    Explorer node disk snapshots · GiB (1,024³ bytes). Storage depends on the node implementation.

    +

    Last 30 days · Explorer node disk snapshots · GiB (1,024³ bytes). Storage depends on the node implementation.

    {loading ? ( ) : sizePoints.length > 0 ? ( @@ -63,7 +63,7 @@ export function NetworkHistoryCharts({ initialData, initialFetchedAt }: { initia ) : ( - + )}     ); diff --git a/lib/generated/api-reference.json b/lib/generated/api-reference.json index 8c1b4916..5ad63657 100644 --- a/lib/generated/api-reference.json +++ b/lib/generated/api-reference.json @@ -11,7 +11,7 @@ ], "listKey": null, "zatoshiFields": [], - "note": "All Zat amounts are exact decimal strings or null. Bounded to 120 blocks by default, at most 1000; before is an exclusive height cursor. Fee removal is floor(aggregate block fees * 3 / 5) per active block, never rounded after aggregation. Reserves, supply, pools and subsidy use retained hash-matched node observations; missed/pre-collection tips remain null. NSM net change requires consecutive observations and is neither gross removals nor reissuance. Orphan observations are retained but excluded by a canonical height/hash join. Reissuance remains unavailable. Requires database migration 027 and the accounting observer. Optional period=1d|7d|30d|all selects NU7-only UTC block-header aggregation instead of block pagination; all means since activation. Returns at most approximately 241 time buckets, exact selected-period and since-activation fee totals (per-block rounding first), cumulative minimum removal, and net reserve growth from the preactivation hash-matched sample. Cannot combine period with limit/before. Missing fee/indexed ranges make affected totals and cumulative values null; missing bucket-end reserve samples stay null. Reserve growth is not cumulative fees. Responses are cached internally for at most 60 seconds and canonical anchors revalidated on every read.", + "note": "All Zat amounts are exact decimal strings or null. Bounded to 120 blocks by default, at most 1000; before is an exclusive height cursor. Fee removal is floor(aggregate block fees * 3 / 5) per active block, never rounded after aggregation. Reserves, supply and pools use retained hash-matched node observations; missed/pre-collection tips remain null. Actual miner receipts use indexed coinbase creation minus authoritative founders/funding allocations, preferring the independent hash/height/network-matched block_subsidy_allocations cache (migration 036), with validated retained subsidy observations as fallback. Historical allocation retrieval never fabricates NSM, pool or supply samples. receiptBlocks counts known actual receipts in each period point, totals, selected totals and legacy coverage; one missing receipt keeps its bucket sum null. NSM net change requires consecutive observations and is neither gross removals nor reissuance. Orphan observations are retained but excluded by a canonical height/hash join. Reissuance remains unavailable. Requires database migration 027 and the accounting observer. Optional period=1d|7d|30d|all selects NU7-only UTC block-header aggregation instead of block pagination; all means since activation. Returns at most approximately 241 time buckets, exact selected-period and since-activation fee totals (per-block rounding first), cumulative minimum removal, and net reserve growth from the preactivation hash-matched sample. Cannot combine period with limit/before. Missing fee/indexed ranges make affected totals and cumulative values null. Period nsmBalanceZat is the last recorded non-null canonical observation within its selected UTC bucket, not a bucket-end balance. nsmSampleHeight, nsmSampleHash and nsmSampleTimestamp identify that exact recorded block; the timestamp is epoch seconds from its block header, not collection time. All sample fields and balance remain null when the bucket has missing canonical blocks or no recorded NSM sample; no earlier out-of-period sample is carried forward. The latest-block reserve and preactivation-baseline reserve-growth requirements remain strict. Reserve growth is not cumulative fees. Responses are cached internally for at most 60 seconds and canonical anchors revalidated on every read.", "ownershipToken": false, "pagination": null, "parameters": [ @@ -1487,7 +1487,7 @@ ], "listKey": null, "zatoshiFields": [], - "note": "Returns up to limit displayed points after loading window additional predecessor blocks for complete trailing averages. Window is clamped to 5–100 and limit to 20–500. Timing uses consecutive block-header timestamps; nonpositive intervals make solrate unavailable. Points and latest retain null for unavailable or incomplete-window metrics, including an empty chain. Solrate is the rolling average of per-block difficulty × 8192 / positive header interval, distinct from the target-work 24h network estimate. Fees are ZEC and transaction counts include coinbase.", + "note": "Returns up to limit displayed points after loading window additional predecessor blocks for complete trailing windows. Window is clamped to 5–100 and limit to 20–500. Solrate uses summed difficulty × 8192 divided by the full window elapsed header time, identified by solrateMethod=difficulty-work-over-elapsed-v1; it is distinct from the target-work 24h network estimate. Equal or backwards individual timestamps do not invalidate a positive complete window. Missing heights or timestamps, invalid difficulty, incomplete history, and nonpositive total elapsed time leave solrate null. Block time is the signed mean consecutive header interval. Other metrics retain trailing averages and null for unavailable or incomplete windows, including an empty chain. Fees are ZEC (TAZ on testnet) and transaction counts include coinbase.", "ownershipToken": false, "pagination": null, "parameters": [ @@ -3516,14 +3516,14 @@ { "method": "GET", "path": "/v1/mining/miner-behavior", - "description": "Miner behavior heuristics.", + "description": "Attributed miner payout reward-date cohorts with exact earned, moved and unspent zatoshis.", "category": "mining", "query": [ "period" ], "listKey": null, "zatoshiFields": [], - "note": null, + "note": "Completed UTC reward dates; spendingBasis=current_indexed_canonical_spends describes indexed canonical spending at snapshot refresh, not historical day-close holdings or proof of sales. Unattributed payouts are excluded. Available responses include decimal-string zatoshi totals and coverage (first/last date, tracked blocks/outputs, oldest refreshedAt, 48-hour freshness threshold). Missing/empty snapshots return status=unavailable, reason=snapshots_not_configured/no_snapshots_in_period, series=[], summary=null and no-store; backend failures return 503. Legacy sellRatio fields describe moved-output ratios.", "ownershipToken": false, "pagination": null, "parameters": [ diff --git a/package.json b/package.json index 356b9bfe..3ed0efbb 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,8 @@ "test:route-cache-build": "node --test server/tests/next-route-cache-build.test.js", "test:inbox-scan": "node --test server/tests/inbox-scan.test.js server/tests/scan-worker-pool.test.js server/api/test/scan-pipeline.test.js server/api/test/scan-cache.test.js server/api/test/scan-stream.test.js && cargo test --manifest-path wasm/Cargo.toml --lib", "test:live-refresh": "node --test server/tests/live-refresh.test.js server/tests/relative-time.test.js server/tests/api-query-freshness.test.js server/tests/network-snapshot.test.js server/tests/network-accounting-visibility.test.js server/tests/privacy-trend-dates.test.js", - "test:frontend": "node --test server/tests/detail-link-prefetch.test.js server/tests/orphan-ssr-replacement.test.js server/tests/crosslink-observations-ui.test.js server/tests/valuation-context.test.js server/tests/address-pagination-client.test.js server/tests/newsletter-security.test.js server/tests/name-rendering.test.js server/tests/http-status-boundaries.test.js server/tests/chart-sharing.test.js server/tests/redesign-data-boundaries.test.js server/tests/chart-catalog.test.js server/tests/block-timing.test.js server/tests/coinbase-message.test.js server/tests/network-upgrades.test.js server/tests/browser-decoder.test.js server/tests/hashrate-display.test.js server/tests/reorg-display.test.js && npm run test:live-refresh && npm run test:sitemap-security && npm run test:performance && npm run test:ttfb-probe && npm run test:isr-warmup && npm run test:revalidate-security && NODE_PATH=$PWD/server/api/node_modules node --test server/tests/address-labels-resilience.test.js && npm run test:brand && npm run test:v1-frontend && node --test server/tests/address-connections.test.js && npm run test:attestations && npm run test:nu7-results && npm run test:governance && node --test server/tests/network-issuance-ui.test.js && node --test server/tests/launch-copy-privacy.test.js", - "test:server-regressions": "NODE_PATH=$PWD/server/api/node_modules node --test server/tests/crosslink-roster.test.js server/tests/hashrate.test.js server/tests/reorg-api.test.js server/tests/orphan-rates.test.js server/tests/blocks-orphan-batch.test.js server/tests/tx-detail-parallel.test.js server/tests/tx-mempool-cache.test.js server/tests/migration-activity-summary.test.js server/tests/blend-check-batch.test.js server/tests/db-pool-deadlines.test.js server/tests/safe-log.test.js server/tests/ironwood-accounting.test.js server/tests/exposure-consumers.test.js server/tests/network-halving.test.js server/tests/network-issuance.test.js server/tests/supply-history-backfill.test.js server/tests/braces-security.test.js server/tests/network-schedule.test.js server/tests/network-block-time.test.js server/tests/mining-metrics.test.js server/tests/network-issuance-curve.test.js server/tests/network-fee-history.test.js server/tests/network-readiness.test.js server/tests/network-accounting-period.test.js server/tests/rpc-cookie-rotation.test.js server/tests/peer-client.test.js server/tests/node-census.test.js server/tests/node-census-postgres.test.js server/tests/privacy-score.test.js server/tests/privacy-stats-snapshot.test.js server/tests/utxo-age.test.js server/tests/signal-telegram.test.js server/tests/analytics-history.test.js server/tests/mining-attribution-repair.test.js server/tests/analytics-history-routes.test.js server/tests/crosschain-v2.test.js server/tests/crosschain-date-pages.test.js server/tests/crosschain-maintenance-budget.test.js server/tests/near-provider-budget.test.js server/tests/address-pagination.test.js server/tests/address-cursor-postgres.test.js && npm run test:error-message-leakage", + "test:frontend": "node --test server/tests/detail-link-prefetch.test.js server/tests/orphan-ssr-replacement.test.js server/tests/crosslink-observations-ui.test.js server/tests/valuation-context.test.js server/tests/address-pagination-client.test.js server/tests/newsletter-security.test.js server/tests/name-rendering.test.js server/tests/http-status-boundaries.test.js server/tests/chart-sharing.test.js server/tests/redesign-data-boundaries.test.js server/tests/chart-catalog.test.js server/tests/block-timing.test.js server/tests/coinbase-message.test.js server/tests/network-upgrades.test.js server/tests/browser-decoder.test.js server/tests/hashrate-display.test.js server/tests/mining-metrics-ui.test.js server/tests/reorg-display.test.js && npm run test:live-refresh && npm run test:sitemap-security && npm run test:performance && npm run test:ttfb-probe && npm run test:isr-warmup && npm run test:revalidate-security && NODE_PATH=$PWD/server/api/node_modules node --test server/tests/address-labels-resilience.test.js && npm run test:brand && npm run test:v1-frontend && node --test server/tests/address-connections.test.js && npm run test:attestations && npm run test:nu7-results && npm run test:governance && node --test server/tests/network-issuance-ui.test.js && node --test server/tests/launch-copy-privacy.test.js", + "test:server-regressions": "NODE_PATH=$PWD/server/api/node_modules node --test server/tests/crosslink-roster.test.js server/tests/hashrate.test.js server/tests/reorg-api.test.js server/tests/orphan-rates.test.js server/tests/blocks-orphan-batch.test.js server/tests/tx-detail-parallel.test.js server/tests/tx-mempool-cache.test.js server/tests/migration-activity-summary.test.js server/tests/blend-check-batch.test.js server/tests/db-pool-deadlines.test.js server/tests/safe-log.test.js server/tests/ironwood-accounting.test.js server/tests/exposure-consumers.test.js server/tests/network-halving.test.js server/tests/network-issuance.test.js server/tests/supply-history-backfill.test.js server/tests/braces-security.test.js server/tests/network-schedule.test.js server/tests/network-block-time.test.js server/tests/mining-metrics.test.js server/tests/miner-behavior.test.js server/tests/network-issuance-curve.test.js server/tests/network-fee-history.test.js server/tests/network-readiness.test.js server/tests/network-accounting-period.test.js server/tests/subsidy-allocations.test.js server/tests/subsidy-allocations-maintenance.test.js server/tests/subsidy-allocation-state.test.js server/tests/subsidy-allocation-health.test.js server/tests/rpc-cookie-rotation.test.js server/tests/peer-client.test.js server/tests/node-census.test.js server/tests/node-census-postgres.test.js server/tests/privacy-score.test.js server/tests/privacy-stats-snapshot.test.js server/tests/utxo-age.test.js server/tests/signal-telegram.test.js server/tests/analytics-history.test.js server/tests/mining-attribution-repair.test.js server/tests/analytics-history-routes.test.js server/tests/crosschain-v2.test.js server/tests/crosschain-date-pages.test.js server/tests/crosschain-maintenance-budget.test.js server/tests/near-provider-budget.test.js server/tests/address-pagination.test.js server/tests/address-cursor-postgres.test.js && npm run test:error-message-leakage", "verify:fast": "npm run lint && tsc --noEmit", "verify:frontend": "npm run verify:fast && npm run test:frontend && CIPHERSCAN_ALLOW_BUILD_UPSTREAM_FALLBACK=1 npm run build && npm run test:route-cache-build", "perf:analyze": "node scripts/perf/analyze-slow-pages.mjs", diff --git a/public/openapi-v1.json b/public/openapi-v1.json index 61b6de32..00950ef7 100644 --- a/public/openapi-v1.json +++ b/public/openapi-v1.json @@ -16136,7 +16136,7 @@ "/v1/mining/miner-behavior": { "get": { "operationId": "get_mining_Miner-behavior", - "summary": "Miner behavior heuristics.", + "summary": "Attributed miner payout reward-date cohorts with exact earned, moved and unspent zatoshis.", "tags": [ "mining" ], @@ -16157,7 +16157,7 @@ "x-cipherscan-v1-status": "adapter", "responses": { "200": { - "description": "Miner behavior heuristics.", + "description": "Attributed miner payout reward-date cohorts with exact earned, moved and unspent zatoshis.", "content": { "application/json": { "schema": { diff --git a/server/api/lib/accounting-history.js b/server/api/lib/accounting-history.js index d14c2491..9e0a385c 100644 --- a/server/api/lib/accounting-history.js +++ b/server/api/lib/accounting-history.js @@ -1,15 +1,18 @@ const { blockAccounting, signedZat } = require('./network-accounting'); const { supplyZat } = require('./network-issuance'); +const { allocationJoin, allocationColumns } = require('./subsidy-allocations'); // Bound work by height/index, not an unbounded timestamp or transaction scan. -const HISTORY_SQL = `WITH recent AS MATERIALIZED ( +function accountingHistorySql(withAllocations = false) { + return `WITH recent AS MATERIALIZED ( SELECT height, hash, timestamp, size, transaction_count FROM blocks WHERE height <= $1 AND ($2::bigint IS NULL OR height < $2) ORDER BY height DESC LIMIT $3 ) SELECT b.*, a.observed_at, a.chain, a.nsm_balance_zat::text, - a.circulating_supply_zat::text, a.pool_balances_zat, a.subsidy, + a.circulating_supply_zat::text, a.pool_balances_zat, a.subsidy, ${allocationColumns(withAllocations)}, t.* FROM recent b - LEFT JOIN node_accounting_observations a ON a.height=b.height AND a.hash=b.hash + LEFT JOIN node_accounting_observations a ON a.height=b.height AND a.hash=b.hash AND a.chain=$4 + ${allocationJoin(withAllocations, '$4')} LEFT JOIN LATERAL ( SELECT count(txid) AS tx_count, count(*) FILTER (WHERE is_coinbase) AS coinbases, @@ -22,11 +25,18 @@ const HISTORY_SQL = `WITH recent AS MATERIALIZED ( sum(orchard_actions) AS orchard_actions, sum(ironwood_actions) AS ironwood_actions FROM transactions WHERE block_height=b.height AND block_hash=b.hash ) t ON true ORDER BY b.height ASC`; +} +const HISTORY_SQL = accountingHistorySql(); function historyPoints(rows, schedule, chain) { return rows.map((row, index) => { const matched = row.chain === chain; - const accounting = blockAccounting(row, matched ? row.subsidy : null, schedule); + const accounting = blockAccounting(row, matched ? row.subsidy : null, schedule, { + founders: row.allocation_founders_zat, funding: row.allocation_funding_streams_zat, miner: row.allocation_miner_zat, + }); + const cachedAllocations = ['allocation_founders_zat', 'allocation_funding_streams_zat'].every(key => supplyZat(row[key]) !== null); + const receiptSource = accounting.minerReceiptsZat === null ? null : cachedAllocations + ? 'indexed-coinbase-and-historical-node-subsidy' : 'indexed-coinbase-and-retained-tip-subsidy'; const previous = rows[index - 1]; const nsm = matched ? signedZat(row.nsm_balance_zat) : null; const previousNsm = previous?.chain === chain ? signedZat(previous.nsm_balance_zat) : null; @@ -35,7 +45,7 @@ function historyPoints(rows, schedule, chain) { const complete = accounting.feesPaidZat !== null; return { ...Object.fromEntries(Object.entries(accounting).map(([key, value]) => [key, key.endsWith('Zat') && value !== null ? String(value) : value])), - timestamp: Number(row.timestamp), sizeBytes: row.size == null ? null : Number(row.size), transactionCount: Number(row.transaction_count), + receiptSource, timestamp: Number(row.timestamp), sizeBytes: row.size == null ? null : Number(row.size), transactionCount: Number(row.transaction_count), nsmBalanceZat: nsm, nsmNetChangeZat: delta, circulatingSupplyZat: matched && supplyZat(row.circulating_supply_zat) !== null ? String(row.circulating_supply_zat) : null, poolBalancesZat: matched ? row.pool_balances_zat ?? null : null, @@ -47,4 +57,4 @@ function historyPoints(rows, schedule, chain) { }; }); } -module.exports = { HISTORY_SQL, historyPoints }; +module.exports = { HISTORY_SQL, accountingHistorySql, historyPoints }; diff --git a/server/api/lib/accounting-period.js b/server/api/lib/accounting-period.js index a3d16633..e39ebe1d 100644 --- a/server/api/lib/accounting-period.js +++ b/server/api/lib/accounting-period.js @@ -1,11 +1,15 @@ // Period charts aggregate exact block amounts in PostgreSQL, before returning // at most ~241 UTC buckets. Do not round 60% of an already aggregated fee sum. -const PERIOD_SQL = `WITH canonical AS MATERIALIZED ( +const { allocationJoin, allocationColumns, sampledSubsidyZatSql } = require('./subsidy-allocations'); + +function accountingPeriodSql(withAllocations = false) { + return `WITH canonical AS MATERIALIZED ( SELECT b.height, b.hash, b.timestamp, b.transaction_count, a.nsm_balance_zat, - a.subsidy, t.*, + a.subsidy, ${allocationColumns(withAllocations)}, t.*, b.height - lag(b.height, 1, $2::bigint - 1) OVER (ORDER BY b.height) - 1 AS missing_before FROM blocks b LEFT JOIN node_accounting_observations a ON a.height=b.height AND a.hash=b.hash AND a.chain=$3 + ${allocationJoin(withAllocations, '$3')} LEFT JOIN LATERAL ( SELECT count(txid) AS tx_count, count(*) FILTER (WHERE is_coinbase) AS coinbases, count(*) FILTER (WHERE NOT is_coinbase AND (fee IS NULL OR fee < 0)) AS invalid_fees, @@ -14,17 +18,19 @@ const PERIOD_SQL = `WITH canonical AS MATERIALIZED ( FILTER (WHERE is_coinbase) AS coinbase_value FROM transactions WHERE block_height=b.height AND block_hash=b.hash ) t ON true WHERE b.height BETWEEN $2 AND $1 +), allocations AS MATERIALIZED ( + SELECT *, coalesce(allocation_founders_zat::numeric, ${sampledSubsidyZatSql('founders')}) AS founders_zat, + coalesce(allocation_funding_streams_zat::numeric, ${sampledSubsidyZatSql('fundingstreamstotal')}) AS funding_streams_zat + FROM canonical ), amounts AS MATERIALIZED ( SELECT *, CASE WHEN tx_count=transaction_count AND coinbases=1 AND invalid_fees=0 AND fees BETWEEN 0 AND 2100000000000000 THEN fees END AS paid, CASE WHEN tx_count=transaction_count AND coinbases=1 AND invalid_fees=0 AND coinbase_value BETWEEN 0 AND 2100000000000000 - AND (subsidy->>'founders')::numeric * 100000000 = trunc((subsidy->>'founders')::numeric * 100000000) - AND (subsidy->>'fundingstreamstotal')::numeric * 100000000 = trunc((subsidy->>'fundingstreamstotal')::numeric * 100000000) - AND (subsidy->>'founders')::numeric >= 0 AND (subsidy->>'fundingstreamstotal')::numeric >= 0 - AND coinbase_value >= ((subsidy->>'founders')::numeric + (subsidy->>'fundingstreamstotal')::numeric)*100000000 - THEN coinbase_value - ((subsidy->>'founders')::numeric + (subsidy->>'fundingstreamstotal')::numeric)*100000000 END AS receipts - FROM canonical + AND founders_zat IS NOT NULL AND funding_streams_zat IS NOT NULL + AND coinbase_value >= founders_zat + funding_streams_zat + THEN coinbase_value - founders_zat - funding_streams_zat END AS receipts + FROM allocations ), exact AS MATERIALIZED ( SELECT *, floor(paid*3/5) AS removed FROM amounts ), bounds AS ( @@ -32,20 +38,20 @@ const PERIOD_SQL = `WITH canonical AS MATERIALIZED ( ), buckets AS ( SELECT floor(e.timestamp / bounds.bucket_seconds)::bigint * bounds.bucket_seconds AS timestamp, min(height) AS first_height, max(height) AS last_height, count(*)::int AS blocks, - count(paid)::int AS fee_blocks, count(nsm_balance_zat)::int AS nsm_samples, + count(paid)::int AS fee_blocks, count(receipts)::int AS receipt_blocks, count(nsm_balance_zat)::int AS nsm_samples, sum(missing_before)::int AS missing_blocks, CASE WHEN count(paid)=count(*) AND sum(missing_before)=0 THEN sum(paid)::text END AS fees_paid_zat, CASE WHEN count(paid)=count(*) AND sum(missing_before)=0 THEN sum(removed)::text END AS fees_to_nsm_zat, CASE WHEN count(paid)=count(*) AND sum(missing_before)=0 THEN sum(paid-removed)::text END AS miner_fee_allocation_zat, CASE WHEN count(receipts)=count(*) AND sum(missing_before)=0 THEN sum(trunc(receipts))::text END AS miner_receipts_zat, - (array_agg(nsm_balance_zat ORDER BY height DESC))[1]::text AS nsm_balance_zat, + (array_agg(nsm_balance_zat ORDER BY height DESC) FILTER (WHERE nsm_balance_zat IS NOT NULL))[1]::text AS nsm_balance_zat, sum(removed) AS removed_sum, count(*)-count(paid)+sum(missing_before) AS unknown_fees FROM exact e CROSS JOIN bounds GROUP BY 1 ), cumulative AS ( SELECT *, CASE WHEN sum(unknown_fees) OVER (ORDER BY timestamp)=0 THEN (sum(removed_sum) OVER (ORDER BY timestamp))::text END AS cumulative_removal_zat FROM buckets ), totals AS ( - SELECT jsonb_build_object('blocks',count(*),'feeBlocks',count(paid),'nsmSamples',count(nsm_balance_zat), + SELECT jsonb_build_object('blocks',count(*),'feeBlocks',count(paid),'receiptBlocks',count(receipts),'nsmSamples',count(nsm_balance_zat), 'missingBlocks',coalesce(sum(missing_before),0), 'feesPaidZat',CASE WHEN count(*)>0 AND count(paid)=count(*) AND sum(missing_before)=0 THEN sum(paid)::text END, 'feesToNsmZat',CASE WHEN count(*)>0 AND count(paid)=count(*) AND sum(missing_before)=0 THEN sum(removed)::text END, @@ -53,7 +59,7 @@ const PERIOD_SQL = `WITH canonical AS MATERIALIZED ( 'firstHeight',min(height),'lastHeight',max(height),'firstTimestamp',min(timestamp),'lastTimestamp',max(timestamp)) AS value FROM exact ), selected AS ( - SELECT jsonb_build_object('blocks',count(*),'feeBlocks',count(paid),'nsmSamples',count(nsm_balance_zat), + SELECT jsonb_build_object('blocks',count(*),'feeBlocks',count(paid),'receiptBlocks',count(receipts),'nsmSamples',count(nsm_balance_zat), 'missingBlocks',coalesce(sum(missing_before),0), 'feesPaidZat',CASE WHEN count(*)>0 AND count(paid)=count(*) AND sum(missing_before)=0 THEN sum(paid)::text END, 'feesToNsmZat',CASE WHEN count(*)>0 AND count(paid)=count(*) AND sum(missing_before)=0 THEN sum(removed)::text END, @@ -65,13 +71,16 @@ const PERIOD_SQL = `WITH canonical AS MATERIALIZED ( -- cumulative value still includes all earlier NU7 blocks. SELECT floor(e.timestamp / bounds.bucket_seconds)::bigint * bounds.bucket_seconds AS timestamp, min(height) AS first_height, max(height) AS last_height, count(*)::int AS blocks, - count(paid)::int AS fee_blocks, count(nsm_balance_zat)::int AS nsm_samples, + count(paid)::int AS fee_blocks, count(receipts)::int AS receipt_blocks, count(nsm_balance_zat)::int AS nsm_samples, sum(missing_before)::int AS missing_blocks, CASE WHEN count(paid)=count(*) AND sum(missing_before)=0 THEN sum(paid)::text END AS fees_paid_zat, CASE WHEN count(paid)=count(*) AND sum(missing_before)=0 THEN sum(removed)::text END AS fees_to_nsm_zat, CASE WHEN count(paid)=count(*) AND sum(missing_before)=0 THEN sum(paid-removed)::text END AS miner_fee_allocation_zat, CASE WHEN count(receipts)=count(*) AND sum(missing_before)=0 THEN sum(trunc(receipts))::text END AS miner_receipts_zat, - (array_agg(nsm_balance_zat ORDER BY height DESC))[1]::text AS nsm_balance_zat + (array_agg(nsm_balance_zat ORDER BY height DESC) FILTER (WHERE nsm_balance_zat IS NOT NULL))[1]::text AS nsm_balance_zat, + (array_agg(height ORDER BY height DESC) FILTER (WHERE nsm_balance_zat IS NOT NULL))[1] AS nsm_sample_height, + (array_agg(hash ORDER BY height DESC) FILTER (WHERE nsm_balance_zat IS NOT NULL))[1] AS nsm_sample_hash, + (array_agg(e.timestamp ORDER BY height DESC) FILTER (WHERE nsm_balance_zat IS NOT NULL))[1] AS nsm_sample_timestamp FROM exact e CROSS JOIN bounds WHERE $4::bigint IS NULL OR e.timestamp >= $4 GROUP BY 1 ), baseline AS ( SELECT b.height,b.hash,a.nsm_balance_zat::text AS balance FROM blocks b @@ -83,12 +92,17 @@ SELECT (SELECT value FROM totals) AS totals, (SELECT value FROM selected) AS sel (SELECT row_to_json(baseline) FROM baseline) AS baseline, (SELECT row_to_json(tip) FROM tip) AS tip, coalesce((SELECT jsonb_agg(jsonb_build_object('timestamp',s.timestamp,'firstHeight',s.first_height, - 'lastHeight',s.last_height,'blocks',s.blocks,'feeBlocks',s.fee_blocks,'nsmSamples',s.nsm_samples, + 'lastHeight',s.last_height,'blocks',s.blocks,'feeBlocks',s.fee_blocks,'receiptBlocks',s.receipt_blocks,'nsmSamples',s.nsm_samples, 'missingBlocks',s.missing_blocks,'feesPaidZat',s.fees_paid_zat,'feesToNsmZat',s.fees_to_nsm_zat, 'minerFeeAllocationZat',s.miner_fee_allocation_zat,'minerReceiptsZat',s.miner_receipts_zat, 'nsmBalanceZat',CASE WHEN s.missing_blocks=0 THEN s.nsm_balance_zat END, + 'nsmSampleHeight',CASE WHEN s.missing_blocks=0 THEN s.nsm_sample_height END, + 'nsmSampleHash',CASE WHEN s.missing_blocks=0 THEN s.nsm_sample_hash END, + 'nsmSampleTimestamp',CASE WHEN s.missing_blocks=0 THEN s.nsm_sample_timestamp END, 'cumulativeRemovalZat',c.cumulative_removal_zat) ORDER BY s.timestamp) FROM selected_buckets s JOIN cumulative c USING(timestamp)), '[]'::jsonb) AS points`; +} +const PERIOD_SQL = accountingPeriodSql(); function periodPayload(row, period, schedule, info, observedAt) { const baseline = row.baseline ?? null; @@ -102,8 +116,8 @@ function periodPayload(row, period, schedule, info, observedAt) { growthSinceNu7Zat: baseline?.balance != null && balance != null ? (BigInt(balance)-BigInt(baseline.balance)).toString() : null }, source: 'complete-canonical-indexed-transactions-and-hash-matched-node-observations', timeBasis: 'block-header-timestamps-UTC', - sampling: 'Reserve is the last block sample in each bucket; missing bucket-end samples remain null. No interpolation.', + sampling: 'Reserve is the last recorded canonical observation within each selected bucket, identified by nsmSampleHeight, nsmSampleHash and its block-header nsmSampleTimestamp. Buckets without a sample or with missing canonical blocks remain null. No interpolation or carry-forward from outside the bucket or selected period. Latest reserve and growth still require the exact end-block sample.', nsmChangeMeaning: 'Net reserve growth from the preactivation sample, not cumulative fees or gross removal.' }; } -module.exports = { PERIOD_SQL, periodPayload }; +module.exports = { PERIOD_SQL, accountingPeriodSql, periodPayload }; diff --git a/server/api/lib/miner-behavior.js b/server/api/lib/miner-behavior.js new file mode 100644 index 00000000..79f6eb06 --- /dev/null +++ b/server/api/lib/miner-behavior.js @@ -0,0 +1,49 @@ +'use strict'; + +const SCOPE = 'attributed_payout_outputs'; + +function unavailable(period, reason) { + return { + period, status: 'unavailable', reason, series: [], summary: null, coverage: null, + message: reason === 'snapshots_not_configured' + ? 'Miner reward spending snapshots are not available on this network.' + : 'No miner reward spending snapshots are available for this period.', + generatedAt: new Date().toISOString(), + }; +} + +function summarize(period, rows, now = Date.now()) { + if (!rows.length) return unavailable(period, 'no_snapshots_in_period'); + const byDate = new Map(); + let refreshedAt = null, trackedBlocks = 0, trackedOutputs = 0; + for (const row of rows) { + const earned = BigInt(row.earned_zat), spent = BigInt(row.spent_zat), held = BigInt(row.held_zat); + if (earned < 0n || spent < 0n || held < 0n || spent + held !== earned) throw new Error('Invalid miner behavior accounting'); + const date = row.date; + if (!byDate.has(date)) byDate.set(date, { date, earned: 0n, spent: 0n, held: 0n, pools: {} }); + const point = byDate.get(date); + point.earned += earned; point.spent += spent; point.held += held; + point.pools[row.pool_name] = { earned: earned.toString(), spent: spent.toString(), held: held.toString(), blocks: Number(row.blocks_mined) }; + trackedBlocks += Number(row.blocks_mined); + trackedOutputs += Number(row.outputs_total); + if (row.updated_at) { + const time = new Date(row.updated_at).toISOString(); + if (!refreshedAt || time < refreshedAt) refreshedAt = time; + } + } + const series = [...byDate.values()].map(point => ({ + date: point.date, earnedZat: point.earned.toString(), spentZat: point.spent.toString(), heldZat: point.held.toString(), + sellRatio: point.earned > 0n ? Number(point.spent * 10000n / point.earned) / 10000 : 0, + pools: point.pools, + })); + const earned = series.reduce((sum, point) => sum + BigInt(point.earnedZat), 0n); + const spent = series.reduce((sum, point) => sum + BigInt(point.spentZat), 0n); + return { + period, status: 'available', series, + summary: { totalEarnedZat: earned.toString(), totalSpentZat: spent.toString(), totalHeldZat: (earned - spent).toString(), overallSellRatio: earned > 0n ? Number(spent * 10000n / earned) / 10000 : 0 }, + coverage: { scope: SCOPE, spendingBasis: 'current_indexed_canonical_spends', freshness: !refreshedAt || Date.parse(refreshedAt) > now + 300000 ? 'unknown' : now - Date.parse(refreshedAt) > 172800000 ? 'stale' : 'current', staleAfterSeconds: 172800, firstDate: series[0].date, lastDate: series.at(-1).date, days: series.length, trackedBlocks, trackedOutputs, refreshedAt }, + generatedAt: new Date().toISOString(), + }; +} + +module.exports = { unavailable, summarize }; diff --git a/server/api/lib/network-accounting.js b/server/api/lib/network-accounting.js index fdba8bbb..a0fd574c 100644 --- a/server/api/lib/network-accounting.js +++ b/server/api/lib/network-accounting.js @@ -9,7 +9,7 @@ function signedZat(value) { return n >= -(1n << 63n) && n < (1n << 63n) ? n.toString() : null; } -function blockAccounting(row, subsidy, schedule) { +function blockAccounting(row, subsidy, schedule, allocations = {}) { const height = Number(row?.height); const fees = supplyZat(row?.fees); const complete = row && Number(row.tx_count) === Number(row.transaction_count) && Number(row.coinbases) === 1 && Number(row.invalid_fees) === 0; @@ -17,9 +17,11 @@ function blockAccounting(row, subsidy, schedule) { const active = schedule ? schedule.nu7Height !== null && height >= schedule.nu7Height : null; const recycled = paid === null || active === null ? null : active ? Number(BigInt(paid) * 3n / 5n) : 0; const minerFees = recycled === null ? null : paid - recycled; - const minerSubsidy = subsidyZat(subsidy?.miner); - const founders = subsidyZat(subsidy?.founders); - const funding = subsidyZat(subsidy?.fundingstreamstotal); + const validAllocations = ['founders', 'funding'].every(key => supplyZat(allocations[key]) !== null); + const cachedMiner = validAllocations ? supplyZat(allocations.miner) : null; + const minerSubsidy = cachedMiner ?? subsidyZat(subsidy?.miner); + const founders = validAllocations ? supplyZat(allocations.founders) : subsidyZat(subsidy?.founders); + const funding = validAllocations ? supplyZat(allocations.funding) : subsidyZat(subsidy?.fundingstreamstotal); // Coinbase transparent outputs plus net shielded creation, less compulsory // non-miner payouts. Lockbox allocations are not spendable coinbase outputs. const coinbase = supplyZat(row?.coinbase_value); diff --git a/server/api/lib/subsidy-allocation-health.js b/server/api/lib/subsidy-allocation-health.js new file mode 100644 index 00000000..752abd16 --- /dev/null +++ b/server/api/lib/subsidy-allocation-health.js @@ -0,0 +1,20 @@ +'use strict'; +const fs = require('node:fs/promises'); + +// Public health never exposes local paths, raw errors, credentials or retry data. +async function readAllocationHealth(filename, network, { readFile = fs.readFile, now = Date.now } = {}) { + try { + const raw = await readFile(filename, 'utf8'); + if (raw.length > 16384) return { status: 'unavailable' }; + const report = JSON.parse(raw); + const checked = typeof report.checkedAt === 'string' ? Date.parse(report.checkedAt) : NaN; + const age = now() - checked; + if (!Number.isFinite(age) || report.network !== network) return { status: 'unavailable' }; + if (age < -30000 || age > 300000) return { status: 'stale' }; + return { status: report.status === 'healthy' ? 'up' : 'unhealthy', age_seconds: Math.max(0, Math.floor(age / 1000)) }; + } catch { + return { status: 'unavailable' }; + } +} + +module.exports = { readAllocationHealth }; diff --git a/server/api/lib/subsidy-allocations.js b/server/api/lib/subsidy-allocations.js new file mode 100644 index 00000000..667e9115 --- /dev/null +++ b/server/api/lib/subsidy-allocations.js @@ -0,0 +1,69 @@ +// Historical subsidy allocations are deterministic node RPC results, separate +// from real-time NSM/pool samples. Missing tables are supported during rollout. +const availability = new WeakMap(); +const MAX_ZAT = '2100000000000000'; + +async function hasSubsidyAllocations(pool) { + const now = Date.now(); + let cached = availability.get(pool); + if (!cached || cached.expires <= now) { + cached = { expires: now + 30000, promise: pool.query( + "SELECT to_regclass('public.block_subsidy_allocations') IS NOT NULL AS available", + ).then(({ rows }) => rows[0]?.available === true) }; + availability.set(pool, cached); + cached.promise.catch(() => availability.delete(pool)); + } + return cached.promise; +} + +function allocationJoin(enabled, chainParameter) { + if (!enabled) return ''; + return `LEFT JOIN public.block_subsidy_allocations sa ON sa.height=b.height AND sa.hash=b.hash + AND sa.chain=${chainParameter} AND sa.source='getblocksubsidy' + AND sa.founders_zat BETWEEN 0 AND ${MAX_ZAT} + AND sa.funding_streams_zat BETWEEN 0 AND ${MAX_ZAT} + AND (sa.miner_zat IS NULL OR sa.miner_zat BETWEEN 0 AND ${MAX_ZAT})`; +} + +function allocationColumns(enabled) { + return ['founders', 'funding_streams', 'miner'].map(key => + `${enabled ? `sa.${key}_zat::text` : 'NULL::text'} AS allocation_${key}_zat`).join(', '); +} + +// Match the JS decoder: only exact, nonnegative JSON-number amounts are valid. +function sampledSubsidyZatSql(key, alias = '') { + const subsidy = alias ? `${alias}.subsidy` : 'subsidy'; + return `CASE WHEN jsonb_typeof(${subsidy}->'${key}')='number' THEN + CASE WHEN (${subsidy}->>'${key}')::numeric BETWEEN 0 AND 21000000 + AND (${subsidy}->>'${key}')::numeric*100000000=trunc((${subsidy}->>'${key}')::numeric*100000000) + THEN (${subsidy}->>'${key}')::numeric*100000000 END END`; +} + +function sampledAllocationsSql(alias = 'a') { + return { founders: sampledSubsidyZatSql('founders', alias), + funding: sampledSubsidyZatSql('fundingstreamstotal', alias), miner: sampledSubsidyZatSql('miner', alias) }; +} + +/** Parse RPC coin units without rounding or accepting fractional zatoshis. */ +function exactCoinZat(value) { + if (typeof value === 'number') { + if (!Number.isFinite(value) || value < 0 || value > 21000000) return null; + const fixed = value.toFixed(8); + if (Number(fixed) !== value) return null; + value = fixed; + } + if (typeof value !== 'string' || !/^(?:0|[1-9]\d*)(?:\.\d{1,8})?$/.test(value)) return null; + const [whole, fraction = ''] = value.split('.'); + const zat = BigInt(whole) * 100000000n + BigInt(fraction.padEnd(8, '0')); + return zat <= BigInt(MAX_ZAT) ? zat.toString() : null; +} + +function parseRpcSubsidyAllocations(subsidy) { + const foundersZat = exactCoinZat(subsidy?.founders); + const fundingStreamsZat = exactCoinZat(subsidy?.fundingstreamstotal); + if (foundersZat === null || fundingStreamsZat === null) return null; + return { foundersZat, fundingStreamsZat, minerZat: exactCoinZat(subsidy?.miner) }; +} + +module.exports = { hasSubsidyAllocations, allocationJoin, allocationColumns, + sampledSubsidyZatSql, sampledAllocationsSql, exactCoinZat, parseRpcSubsidyAllocations }; diff --git a/server/api/openapi/v1.yaml b/server/api/openapi/v1.yaml index 00258101..926d9cde 100644 --- a/server/api/openapi/v1.yaml +++ b/server/api/openapi/v1.yaml @@ -1910,7 +1910,10 @@ paths: /v1/system/health/deep: get: operationId: get_system_Health_Deep - summary: Deep health check including DB connectivity. + summary: >- + Uncached deep health check including DB connectivity and, when configured, historical + subsidy allocation repair freshness and coverage. An unavailable, stale or unhealthy repair + monitor returns HTTP 503. tags: - system x-cipherscan-legacy-path: /health/deep @@ -1919,7 +1922,10 @@ paths: x-cipherscan-v1-status: adapter responses: '200': - description: Deep health check including DB connectivity. + description: >- + Uncached deep health check including DB connectivity and, when configured, historical + subsidy allocation repair freshness and coverage. An unavailable, stale or unhealthy + repair monitor returns HTTP 503. content: application/json: schema: @@ -10607,7 +10613,7 @@ paths: /v1/mining/miner-behavior: get: operationId: get_mining_Miner-behavior - summary: Miner behavior heuristics. + summary: Attributed miner payout reward-date cohorts with exact earned, moved and unspent zatoshis. tags: - mining parameters: @@ -10623,7 +10629,9 @@ paths: x-cipherscan-v1-status: adapter responses: '200': - description: Miner behavior heuristics. + description: >- + Attributed miner payout reward-date cohorts with exact earned, moved and unspent + zatoshis. content: application/json: schema: diff --git a/server/api/routes/blocks.js b/server/api/routes/blocks.js index ab14e1dd..8029aac0 100644 --- a/server/api/routes/blocks.js +++ b/server/api/routes/blocks.js @@ -12,6 +12,7 @@ const { decodeCoinbaseText } = require('../coinbase-data'); const { applyListCacheHeaders, createListCache } = require('../list-cache'); const { parseSafeListPagination, offsetExceededError } = require('../lib/pagination'); const { logSafeError } = require('../lib/safe-log'); +const { readAllocationHealth } = require('../lib/subsidy-allocation-health'); const disabledListCache = createListCache({ enabled: false }); @@ -96,6 +97,7 @@ router.get('/health', (req, res) => { // Deep health check — external monitoring (Better Stack, status page) router.get('/health/deep', async (req, res) => { + res.set('Cache-Control', 'no-store'); const checks = {}; let critical = false; let degraded = false; @@ -216,6 +218,11 @@ router.get('/health/deep', async (req, res) => { critical = true; } + if (process.env.SUBSIDY_ALLOCATION_HEALTH_FILE) { + checks.subsidy_allocations = await readAllocationHealth(process.env.SUBSIDY_ALLOCATION_HEALTH_FILE, networkName()); + if (checks.subsidy_allocations.status !== 'up') critical = true; + } + const status = critical ? 'unhealthy' : degraded ? 'degraded' : 'healthy'; const httpCode = critical ? 503 : 200; diff --git a/server/api/routes/mining.js b/server/api/routes/mining.js index 4f654e35..ccf72cb0 100644 --- a/server/api/routes/mining.js +++ b/server/api/routes/mining.js @@ -12,6 +12,7 @@ const router = express.Router(); const { POOL_BY_ADDRESS, POOL_BY_TAG, getPoolTagSql, getPoolName } = require('../mining-pools'); const { logSafeError } = require('../lib/safe-log'); const { orphanRates } = require('../lib/orphan-rates'); +const minerBehavior = require('../lib/miner-behavior'); let pool; let redisClient; @@ -370,97 +371,32 @@ router.get('/api/mining/rewards', async (req, res) => { // ============================================================================ router.get('/api/mining/miner-behavior', async (req, res) => { try { - const period = req.query.period || '90d'; - const cacheKey = `mining:history-v3:miner-behavior:${period}`; - + const period = req.query.period || '30d'; + if (!['24h','3d','7d','30d','90d','6m','1y','all'].includes(period)) return res.status(400).json({ error: 'Invalid period' }); + const cacheKey = `mining:history-v4:miner-behavior:${period}`; const cached = await getFromCache(cacheKey); if (cached) return res.json(cached); const interval = parsePeriod(period); const whereClause = interval - ? `WHERE date >= CURRENT_DATE - INTERVAL '${interval}'` - : ''; - - // Check if snapshot table exists - const tableCheck = await pool.query(` - SELECT EXISTS ( - SELECT FROM information_schema.tables - WHERE table_name = 'mining_behavior_daily' - ) as exists - `); - + ? `WHERE date >= (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date - INTERVAL '${interval}' AND date < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date` + : `WHERE date < (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')::date`; + const tableCheck = await pool.query(`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema='public' AND table_name='mining_behavior_daily') AS exists`); if (!tableCheck.rows[0].exists) { - return res.json({ - period, - series: [], - summary: null, - message: 'Miner behavior data is being computed. Check back soon.', - generatedAt: new Date().toISOString(), - }); + return res.set('Cache-Control', 'no-store').json(minerBehavior.unavailable(period, 'snapshots_not_configured')); } - const result = await pool.query(` - SELECT - date::text AS date, - pool_name, - earned_zat, - spent_zat, - held_zat, - blocks_mined, - outputs_spent, - outputs_total - FROM mining_behavior_daily - ${whereClause} - ORDER BY date, pool_name + SELECT date::text AS date, pool_name, earned_zat, spent_zat, held_zat, + blocks_mined, outputs_spent, outputs_total, updated_at + FROM mining_behavior_daily ${whereClause} ORDER BY date, pool_name `); - - // Group by date for aggregate view - const byDate = new Map(); - for (const row of result.rows) { - const dateKey = row.date; - if (!byDate.has(dateKey)) { - byDate.set(dateKey, { date: dateKey, earned: BigInt(0), spent: BigInt(0), held: BigInt(0), pools: {} }); - } - const entry = byDate.get(dateKey); - const earned = BigInt(row.earned_zat); - const spent = BigInt(row.spent_zat); - const held = BigInt(row.held_zat); - entry.earned += earned; - entry.spent += spent; - entry.held += held; - entry.pools[row.pool_name] = { - earned: row.earned_zat, - spent: row.spent_zat, - held: row.held_zat, - blocks: parseInt(row.blocks_mined), - }; - } - - const series = [...byDate.values()].map(entry => ({ - date: entry.date, - earnedZat: entry.earned.toString(), - spentZat: entry.spent.toString(), - heldZat: entry.held.toString(), - sellRatio: entry.earned > 0n ? Number((entry.spent * 10000n) / entry.earned) / 10000 : 0, - pools: entry.pools, - })); - - // Compute overall summary - const totalEarned = series.reduce((s, r) => s + BigInt(r.earnedZat), 0n); - const totalSpent = series.reduce((s, r) => s + BigInt(r.spentZat), 0n); - const summary = { - totalEarnedZat: totalEarned.toString(), - totalSpentZat: totalSpent.toString(), - totalHeldZat: (totalEarned - totalSpent).toString(), - overallSellRatio: totalEarned > 0n ? Number((totalSpent * 10000n) / totalEarned) / 10000 : 0, - }; - - const response = { period, series, summary, generatedAt: new Date().toISOString() }; - await setCache(cacheKey, response, 900); // 15 min cache + const response = minerBehavior.summarize(period, result.rows); + if (response.status === 'available') await setCache(cacheKey, response, 900); + else res.set('Cache-Control', 'no-store'); res.json(response); } catch (error) { logSafeError('Error fetching miner behavior:', error); - res.status(500).json({ error: 'Failed to fetch miner behavior' }); + res.status(503).set('Cache-Control', 'no-store').json({ error: 'Miner reward spending snapshots are temporarily unavailable' }); } }); diff --git a/server/api/routes/network-analytics.js b/server/api/routes/network-analytics.js index 0fc2a891..498fc1a3 100644 --- a/server/api/routes/network-analytics.js +++ b/server/api/routes/network-analytics.js @@ -88,6 +88,13 @@ function rollingAverage(values, window) { return out; } +function miningNumber(value) { + if ((typeof value !== 'number' && typeof value !== 'string') + || (typeof value === 'string' && value.trim() === '')) return null; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; +} + async function discoverNextHalving(callZebraRPC, currentHeight, chainInfo) { if (!Number.isSafeInteger(currentHeight) || currentHeight < 1) throw new Error('Invalid chain height'); const info = chainInfo ?? await callZebraRPC('getblockchaininfo'); @@ -195,22 +202,42 @@ function registerNetworkAnalyticsRoutes(router) { ); const rows = result.rows.reverse(); - const intervals = rows.map((r, i) => { + const timestamps = rows.map((r) => { + const timestamp = miningNumber(r.timestamp); + return Number.isSafeInteger(timestamp) && timestamp >= 0 ? timestamp : null; + }); + const heights = rows.map((r) => { + const height = miningNumber(r.height); + return Number.isSafeInteger(height) && height >= 0 ? height : null; + }); + const intervals = rows.map((_, i) => { if (i === 0) return null; - const prev = rows[i - 1]; - const delta = parseInt(r.timestamp, 10) - parseInt(prev.timestamp, 10); - return Number(r.height) === Number(prev.height) + 1 ? delta : null; + if (timestamps[i] === null || timestamps[i - 1] === null + || heights[i] === null || heights[i - 1] === null + || heights[i] !== heights[i - 1] + 1) return null; + return timestamps[i] - timestamps[i - 1]; }); - const difficulties = rows.map((r) => parseFloat(r.difficulty) || 0); - // Same 2^13 Equihash constant as /api/network/stats — see comment there. - const solrates = difficulties.map((d, i) => intervals[i] > 0 ? (d * 8192) / intervals[i] : null); + const difficulties = rows.map((r) => { + const difficulty = miningNumber(r.difficulty); + return Number.isFinite(difficulty) && difficulty > 0 ? difficulty : null; + }); const fees = rows.map((r) => (parseInt(r.total_fees, 10) || 0) / 1e8); const txCounts = rows.map((r) => parseInt(r.transaction_count, 10) || 0); const rollDiff = rollingAverage(difficulties, window); - const rollSolrate = rollingAverage(solrates, window); const rollBlockTime = rollingAverage(intervals, window); + // Estimate the complete window's work over its elapsed header time, not + // the mean of instantaneous rates. Canonical headers can share a second + // or move backwards; those intervals do not invalidate a positive full + // span. rollingAverage still requires every consecutive interval and + // every contributing difficulty, so missing data remains unavailable. + // The 2^13 difficulty conversion is an estimate, distinct from nBits work. + const rollSolrate = rollDiff.map((difficulty, i) => { + if (difficulty === null || !(rollBlockTime[i] > 0)) return null; + const estimate = difficulty * 8192 / rollBlockTime[i]; + return Number.isFinite(estimate) ? estimate : null; + }); const rollFees = rollingAverage(fees, window); const rollTx = rollingAverage(txCounts, window); @@ -228,6 +255,7 @@ function registerNetworkAnalyticsRoutes(router) { res.json({ success: true, window, + solrateMethod: 'difficulty-work-over-elapsed-v1', latest: { solrate: latest.solrate ?? null, difficulty: latest.difficulty ?? null, diff --git a/server/api/routes/network-readiness.js b/server/api/routes/network-readiness.js index ea7f9079..1807e01e 100644 --- a/server/api/routes/network-readiness.js +++ b/server/api/routes/network-readiness.js @@ -3,8 +3,9 @@ const { cadencePoints } = require('../lib/block-cadence'); const { signedZat, blockAccounting } = require('../lib/network-accounting'); const { supplyZat } = require('../lib/network-issuance'); const { logSafeError } = require('../lib/safe-log'); -const { HISTORY_SQL, historyPoints } = require('../lib/accounting-history'); -const { PERIOD_SQL, periodPayload } = require('../lib/accounting-period'); +const { accountingHistorySql, historyPoints } = require('../lib/accounting-history'); +const { accountingPeriodSql, periodPayload } = require('../lib/accounting-period'); +const { hasSubsidyAllocations } = require('../lib/subsidy-allocations'); const cadenceCache = new WeakMap(); const accountingPeriodCache = new WeakMap(); @@ -70,7 +71,8 @@ function registerNetworkReadinessRoutes(router) { const observedAt = new Date().toISOString(); const seconds = { '1d': 86400, '7d': 604800, '30d': 2592000 }[period]; const since = seconds ? Math.floor(Date.now()/1000)-seconds : null; - cached = { expires: Date.now()+60000, promise: pool.query(PERIOD_SQL, [info.blocks, schedule.nu7Height, info.chain, since]) + cached = { expires: Date.now()+60000, promise: hasSubsidyAllocations(pool) + .then(enabled => pool.query(accountingPeriodSql(enabled), [info.blocks, schedule.nu7Height, info.chain, since])) .then(({ rows }) => ({ row: rows[0], observedAt })) }; cache.set(key, cached); cached.promise.catch(() => cache.delete(key)); @@ -101,7 +103,8 @@ function registerNetworkReadinessRoutes(router) { const { pool, callZebraRPC } = req.app.locals; const info = await callZebraRPC('getblockchaininfo'); const schedule = networkSchedule(info); - const { rows } = await pool.query(HISTORY_SQL, [info.blocks, before, limit]); + const withAllocations = await hasSubsidyAllocations(pool); + const { rows } = await pool.query(accountingHistorySql(withAllocations), [info.blocks, before, limit, info.chain]); const last = rows.at(-1); if (last && await callZebraRPC('getblockhash', [Number(last.height)]) !== last.hash) { return res.status(503).json({ success: false, error: 'Indexed history is reconciling with the node' }); @@ -111,6 +114,7 @@ function registerNetworkReadinessRoutes(router) { observedAt: new Date().toISOString(), schedule, points, nextBefore: rows.length === limit ? Number(rows[0].height) : null, coverage: { blocks: points.length, feeBlocks: points.filter(p => p.feesPaidZat !== null).length, + receiptBlocks: points.filter(p => p.minerReceiptsZat !== null).length, nsmSamples: points.filter(p => p.nsmBalanceZat !== null).length }, source: 'canonical-indexed-transactions-and-hash-matched-node-observations', nsmChangeMeaning: 'Net reserve change between consecutive observed blocks, not gross fee removal or reissuance.', diff --git a/server/api/test/subsidy-allocation-health.test.js b/server/api/test/subsidy-allocation-health.test.js new file mode 100644 index 00000000..5efd0e37 --- /dev/null +++ b/server/api/test/subsidy-allocation-health.test.js @@ -0,0 +1,51 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const express = require('express'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { readAllocationHealth } = require('../lib/subsidy-allocation-health'); + +test('public allocation health requires a recent healthy same-network report', async () => { + const now = 1792146000000; + const read = report => readAllocationHealth('/private/never-published', 'testnet', { now: () => now, readFile: async () => JSON.stringify(report) }); + const valid = { checkedAt: new Date(now - 60000).toISOString(), network: 'testnet', status: 'healthy' }; + assert.deepEqual(await read(valid), { status: 'up', age_seconds: 60 }); + assert.equal((await read({ ...valid, checkedAt: new Date(now - 300001).toISOString() })).status, 'stale'); + assert.equal((await read({ ...valid, checkedAt: new Date(now + 30001).toISOString() })).status, 'stale'); + for (const change of [{ network: 'mainnet' }, { checkedAt: 'bad' }, { checkedAt: null }]) assert.equal((await read({ ...valid, ...change })).status, 'unavailable'); + for (const status of ['unhealthy', 'error', undefined]) assert.equal((await read({ ...valid, status })).status, 'unhealthy'); + assert.deepEqual(await readAllocationHealth('/private/missing', 'testnet', { readFile: async () => { throw new Error('sensitive path'); } }), { status: 'unavailable' }); + assert.deepEqual(await readAllocationHealth('/private/large', 'testnet', { readFile: async () => 'x'.repeat(16385) }), { status: 'unavailable' }); +}); + +test('configured allocation monitor failure makes deep health 503 and healthy recovery clears it', async () => { + const previous = { network: process.env.NETWORK, file: process.env.SUBSIDY_ALLOCATION_HEALTH_FILE }; + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'subsidy-health-')); + const filename = path.join(directory, 'health.json'); + process.env.NETWORK = 'testnet'; + delete process.env.SUBSIDY_ALLOCATION_HEALTH_FILE; + const app = express(); + app.locals.pool = { query: async () => ({ rows: [] }) }; + app.locals.chainTip = { height: 4500000 }; + app.locals.callZebraRPC = async () => 4500000; + app.use(require('../routes/blocks')); + const server = await new Promise(resolve => { const listening = app.listen(0, '127.0.0.1', () => resolve(listening)); }); + const get = async () => { const response = await fetch(`http://127.0.0.1:${server.address().port}/health/deep`); assert.equal(response.headers.get('cache-control'), 'no-store'); return { code: response.status, body: await response.json() }; }; + try { + let result = await get(); + assert.equal(result.code, 200); assert.equal(result.body.checks.subsidy_allocations, undefined); + process.env.SUBSIDY_ALLOCATION_HEALTH_FILE = filename; + result = await get(); assert.equal(result.code, 503); assert.equal(result.body.checks.subsidy_allocations.status, 'unavailable'); + await fs.writeFile(filename, JSON.stringify({ network: 'testnet', checkedAt: new Date().toISOString(), status: 'unhealthy', reason: 'do not publish this' })); + result = await get(); assert.equal(result.code, 503); assert.equal(result.body.checks.subsidy_allocations.status, 'unhealthy'); + assert.equal(JSON.stringify(result.body).includes('do not publish'), false); + await fs.writeFile(filename, JSON.stringify({ network: 'testnet', checkedAt: new Date().toISOString(), status: 'healthy' })); + result = await get(); assert.equal(result.code, 200); assert.equal(result.body.status, 'healthy'); + } finally { + await new Promise(resolve => server.close(resolve)); + await fs.rm(directory, { recursive: true, force: true }); + if (previous.network === undefined) delete process.env.NETWORK; else process.env.NETWORK = previous.network; + if (previous.file === undefined) delete process.env.SUBSIDY_ALLOCATION_HEALTH_FILE; else process.env.SUBSIDY_ALLOCATION_HEALTH_FILE = previous.file; + } +}); diff --git a/server/api/test/v1/network-readiness.test.js b/server/api/test/v1/network-readiness.test.js index 3c16c66d..82d42a14 100644 --- a/server/api/test/v1/network-readiness.test.js +++ b/server/api/test/v1/network-readiness.test.js @@ -13,11 +13,15 @@ test('redesign network timing and accounting retain source semantics through v1' const timestamp = Math.floor(Date.now() / 1000); const rows = Array.from({ length: 125 }, (_, i) => ({ height: height - i, timestamp: timestamp - i * 25 })); let unavailable = false; - source.locals.pool = { query: async sql => ({ rows: sql.includes('WITH canonical AS MATERIALIZED') ? [{ - totals:{blocks:2,feeBlocks:2,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4'}, - selected:{blocks:2,feeBlocks:2,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4'}, - bucket_seconds:'300',points:[{timestamp,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4',cumulativeRemovalZat:'2'}], + source.locals.pool = { query: async sql => ({ rows: sql.includes('to_regclass') ? [{ available: true }] : sql.includes('WITH canonical AS MATERIALIZED') ? [{ + totals:{blocks:2,feeBlocks:2,receiptBlocks:2,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4'}, + selected:{blocks:2,feeBlocks:2,receiptBlocks:2,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4'}, + bucket_seconds:'300',points:[{timestamp,blocks:2,receiptBlocks:2,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4',cumulativeRemovalZat:'2'}], baseline:{height:3600000-1,hash:'canonical',balance:'9007199254740993'},tip:{height,hash:'canonical',balance:'9007199254740995'}, + }] : sql.includes('WITH recent AS MATERIALIZED') ? [{ + height, hash: 'canonical', transaction_count: 3, tx_count: 3, coinbases: 1, + invalid_fees: 0, fees: '3', coinbase_value: '100000001', + allocation_founders_zat: '0', allocation_funding_streams_zat: '20000000', allocation_miner_zat: '80000000', }] : sql.includes('WITH tip') ? [{ height, hash: 'canonical', transaction_count: 3, tx_count: 3, coinbases: 1, invalid_fees: 0, fees: '3', coinbase_value: '100000004', @@ -69,9 +73,18 @@ test('redesign network timing and accounting retain source semantics through v1' const payload = await history.json(); assert.equal(payload.data.period,period); assert.equal(payload.data.totals.feesToNsmZat,'2'); + assert.equal(payload.data.totals.receiptBlocks,2); + assert.equal(payload.data.selected.receiptBlocks,2); + assert.equal(payload.data.points[0].receiptBlocks,2); assert.equal(payload.data.reserve.growthSinceNu7Zat,'2'); assert.equal(payload.data.points[0].cumulativeRemovalZat,'2'); } + const legacy = await (await fetch(`${base}/accounting/history?limit=1`)).json(); + assert.equal(legacy.data.coverage.receiptBlocks, 1); + assert.equal(legacy.data.points[0].minerReceiptsZat, '80000001'); + assert.equal(legacy.data.points[0].minerSubsidyZat, '80000000'); + assert.equal(legacy.data.points[0].nsmBalanceZat, null); + assert.equal(legacy.data.points[0].observedAt, null); assert.equal((await fetch(`${base}/accounting/history?period=all&limit=120`)).status,400); unavailable = true; const failed = await fetch(`${base}/accounting`); diff --git a/server/api/v1/inventory/manifest.js b/server/api/v1/inventory/manifest.js index 8214fdf1..06eff07a 100644 --- a/server/api/v1/inventory/manifest.js +++ b/server/api/v1/inventory/manifest.js @@ -36,7 +36,7 @@ /** @typedef {{ next: (pagination: object) => object|null, prev: (pagination: object) => object|null }} CursorMap */ const MANIFEST = [ - { method: 'GET', legacyPath: '/api/network/accounting/history', file: 'server/api/routes/network-readiness.js', classification: 'public', domain: 'network', auth: 'none', description: 'Canonical block fee allocations, shielded activity and sampled NSM/supply history.', v1: { path: '/v1/network/accounting/history', status: 'adapter', shape: 'passthrough', notes: 'All Zat amounts are exact decimal strings or null. Bounded to 120 blocks by default, at most 1000; before is an exclusive height cursor. Fee removal is floor(aggregate block fees * 3 / 5) per active block, never rounded after aggregation. Reserves, supply, pools and subsidy use retained hash-matched node observations; missed/pre-collection tips remain null. NSM net change requires consecutive observations and is neither gross removals nor reissuance. Orphan observations are retained but excluded by a canonical height/hash join. Reissuance remains unavailable. Requires database migration 027 and the accounting observer. Optional period=1d|7d|30d|all selects NU7-only UTC block-header aggregation instead of block pagination; all means since activation. Returns at most approximately 241 time buckets, exact selected-period and since-activation fee totals (per-block rounding first), cumulative minimum removal, and net reserve growth from the preactivation hash-matched sample. Cannot combine period with limit/before. Missing fee/indexed ranges make affected totals and cumulative values null; missing bucket-end reserve samples stay null. Reserve growth is not cumulative fees. Responses are cached internally for at most 60 seconds and canonical anchors revalidated on every read.' } }, + { method: 'GET', legacyPath: '/api/network/accounting/history', file: 'server/api/routes/network-readiness.js', classification: 'public', domain: 'network', auth: 'none', description: 'Canonical block fee allocations, shielded activity and sampled NSM/supply history.', v1: { path: '/v1/network/accounting/history', status: 'adapter', shape: 'passthrough', notes: 'All Zat amounts are exact decimal strings or null. Bounded to 120 blocks by default, at most 1000; before is an exclusive height cursor. Fee removal is floor(aggregate block fees * 3 / 5) per active block, never rounded after aggregation. Reserves, supply and pools use retained hash-matched node observations; missed/pre-collection tips remain null. Actual miner receipts use indexed coinbase creation minus authoritative founders/funding allocations, preferring the independent hash/height/network-matched block_subsidy_allocations cache (migration 036), with validated retained subsidy observations as fallback. Historical allocation retrieval never fabricates NSM, pool or supply samples. receiptBlocks counts known actual receipts in each period point, totals, selected totals and legacy coverage; one missing receipt keeps its bucket sum null. NSM net change requires consecutive observations and is neither gross removals nor reissuance. Orphan observations are retained but excluded by a canonical height/hash join. Reissuance remains unavailable. Requires database migration 027 and the accounting observer. Optional period=1d|7d|30d|all selects NU7-only UTC block-header aggregation instead of block pagination; all means since activation. Returns at most approximately 241 time buckets, exact selected-period and since-activation fee totals (per-block rounding first), cumulative minimum removal, and net reserve growth from the preactivation hash-matched sample. Cannot combine period with limit/before. Missing fee/indexed ranges make affected totals and cumulative values null. Period nsmBalanceZat is the last recorded non-null canonical observation within its selected UTC bucket, not a bucket-end balance. nsmSampleHeight, nsmSampleHash and nsmSampleTimestamp identify that exact recorded block; the timestamp is epoch seconds from its block header, not collection time. All sample fields and balance remain null when the bucket has missing canonical blocks or no recorded NSM sample; no earlier out-of-period sample is carried forward. The latest-block reserve and preactivation-baseline reserve-growth requirements remain strict. Reserve growth is not cumulative fees. Responses are cached internally for at most 60 seconds and canonical anchors revalidated on every read.' } }, { method: 'POST', legacyPath: null, file: 'server/api/v1/routes/ask.js', classification: 'public', domain: 'ask', auth: 'none', description: 'Contextual mainnet questions with reviewed public sources, supported language selection and bounded recent questions. Paid answers require bot verification and shared spend limits.', v1: { path: '/v1/ask/chat', status: 'native', nativeKey: 'ask', shape: 'passthrough' } }, { method: 'POST', legacyPath: null, file: 'server/api/v1/routes/ask.js', classification: 'public', domain: 'ask', auth: 'none', description: 'Explain a mainnet analysis using server-fetched public evidence. Requires configured AI and an exact fingerprint match with the chart observations. Private, no-store.', v1: { path: '/v1/ask/explain', status: 'native', nativeKey: 'ask', shape: 'passthrough' } }, { method: 'GET', legacyPath: null, file: 'server/api/v1/routes/ask.js', classification: 'public', domain: 'ask', auth: 'none', description: 'Mainnet Ask capability: guided starter analyses or configured AI interpretation. Private, no-store.', v1: { path: '/v1/ask', status: 'native', nativeKey: 'ask', shape: 'passthrough' } }, @@ -60,7 +60,7 @@ const MANIFEST = [ { method: 'GET', legacyPath: '/health/deep', file: 'server/api/routes/blocks.js', classification: 'ops', domain: 'system', auth: 'none', - description: 'Deep health check including DB connectivity.', + description: 'Uncached deep health check including DB connectivity and, when configured, historical subsidy allocation repair freshness and coverage. An unavailable, stale or unhealthy repair monitor returns HTTP 503.', v1: { path: '/v1/system/health/deep', status: 'adapter', shape: 'passthrough' }, }, { @@ -140,7 +140,7 @@ const MANIFEST = [ // network-analytics.js // --------------------------------------------------------------------- { method: 'GET', legacyPath: '/api/network/halving', file: 'server/api/routes/network-analytics.js', classification: 'public', domain: 'network', auth: 'none', description: 'Node-observed halving schedule and cadence-based estimates.', v1: { path: '/v1/network/halving', status: 'adapter', shape: 'passthrough', notes: 'Halving boundaries use the public-network consensus subsidy clock, integrating 150/75/25-second target-spacing eras from the serving node activation schedule. Unscheduled future upgrades can change the forecast. Observed cadence scales elapsed-time estimates across the known eras. Unknown schedules and missing observations are unavailable. Future state-dependent rewards may be null even when the halving boundary is known. Complete estimates may be cached for 300 seconds.' } }, - { method: 'GET', legacyPath: '/api/network/mining-metrics', file: 'server/api/routes/network-analytics.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Mining metrics summary.', v1: { path: '/v1/mining/metrics', status: 'adapter', shape: 'passthrough', notes: 'Returns up to limit displayed points after loading window additional predecessor blocks for complete trailing averages. Window is clamped to 5–100 and limit to 20–500. Timing uses consecutive block-header timestamps; nonpositive intervals make solrate unavailable. Points and latest retain null for unavailable or incomplete-window metrics, including an empty chain. Solrate is the rolling average of per-block difficulty × 8192 / positive header interval, distinct from the target-work 24h network estimate. Fees are ZEC and transaction counts include coinbase.' } }, + { method: 'GET', legacyPath: '/api/network/mining-metrics', file: 'server/api/routes/network-analytics.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Mining metrics summary.', v1: { path: '/v1/mining/metrics', status: 'adapter', shape: 'passthrough', notes: 'Returns up to limit displayed points after loading window additional predecessor blocks for complete trailing windows. Window is clamped to 5–100 and limit to 20–500. Solrate uses summed difficulty × 8192 divided by the full window elapsed header time, identified by solrateMethod=difficulty-work-over-elapsed-v1; it is distinct from the target-work 24h network estimate. Equal or backwards individual timestamps do not invalidate a positive complete window. Missing heights or timestamps, invalid difficulty, incomplete history, and nonpositive total elapsed time leave solrate null. Block time is the signed mean consecutive header interval. Other metrics retain trailing averages and null for unavailable or incomplete windows, including an empty chain. Fees are ZEC (TAZ on testnet) and transaction counts include coinbase.' } }, { method: 'GET', legacyPath: '/api/mining/orphan-rates', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Observed orphan rates and pool breakdowns for 24h, 7d and 30d.', v1: { path: '/v1/mining/orphan-rates', status: 'adapter', shape: 'passthrough', notes: 'Observed unique non-canonical blocks from indexer reorg archives and node-validated competing-block capture / (current canonical blocks + observed losses), using shared half-open UTC block-header timestamp windows and one database snapshot. External reports, explicitly invalid archives, and hashes currently canonical are excluded. Hashes are deduplicated across both sources; capture at heights not yet indexed is excluded. Method observed-block-capture-v2 adds capture startedAt, heartbeatAt, observers, healthyObservers and status; these report observer availability, never network-wide completeness. Pool labels group public payout/tag attribution; Unknown remains included. Coverage is unverified, collectionStart is null, and uptimeVerified is false: this is not a complete network-wide or true per-pool orphan rate. Rates are fractions or null when canonical history is incomplete or there are no canonical blocks; observed zero stays zero. Includes asOf and indexed tip height/hash/header timestamp. HTTP cache lifetime is 30 seconds; read failures return 503/no-store.' } }, { method: 'GET', legacyPath: '/api/network/hashrate-history', file: 'server/api/routes/network-analytics.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Work-based hashrate in Sol/s over full trailing 24h or 7d windows, sampled at midnight UTC. Current estimates and as-of times are shared through network/stats. Missing samples return null.', v1: { path: '/v1/mining/hashrate-history', status: 'adapter', shape: 'passthrough' } }, { method: 'GET', legacyPath: '/api/network/pool-history', file: 'server/api/routes/network-analytics.js', classification: 'public', domain: 'shielded-pools', auth: 'none', description: 'Daily shielded and transparent pool balances from chain state.', v1: { path: '/v1/shielded-pools/history', status: 'adapter', shape: 'passthrough' } }, @@ -201,7 +201,7 @@ const MANIFEST = [ { method: 'GET', legacyPath: '/api/mining/pool-ranking', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Ranked mining pools.', v1: { path: '/v1/mining/pool-ranking', status: 'adapter', shape: 'passthrough' } }, { method: 'GET', legacyPath: '/api/mining/hashrate-share', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Hashrate share over time.', v1: { path: '/v1/mining/hashrate-share', status: 'adapter', shape: 'passthrough' } }, { method: 'GET', legacyPath: '/api/mining/rewards', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Block reward accounting.', v1: { path: '/v1/mining/rewards', status: 'adapter', shape: 'passthrough' } }, - { method: 'GET', legacyPath: '/api/mining/miner-behavior', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Miner behavior heuristics.', v1: { path: '/v1/mining/miner-behavior', status: 'adapter', shape: 'passthrough' } }, + { method: 'GET', legacyPath: '/api/mining/miner-behavior', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'Attributed miner payout reward-date cohorts with exact earned, moved and unspent zatoshis.', v1: { path: '/v1/mining/miner-behavior', status: 'adapter', shape: 'passthrough', notes: 'Completed UTC reward dates; spendingBasis=current_indexed_canonical_spends describes indexed canonical spending at snapshot refresh, not historical day-close holdings or proof of sales. Unattributed payouts are excluded. Available responses include decimal-string zatoshi totals and coverage (first/last date, tracked blocks/outputs, oldest refreshedAt, 48-hour freshness threshold). Missing/empty snapshots return status=unavailable, reason=snapshots_not_configured/no_snapshots_in_period, series=[], summary=null and no-store; backend failures return 503. Legacy sellRatio fields describe moved-output ratios.' } }, { method: 'GET', legacyPath: '/api/mining/zodl-leaderboard', file: 'server/api/routes/mining.js', classification: 'public', domain: 'mining', auth: 'none', description: 'ZODL miner leaderboard.', v1: { path: '/v1/mining/zodl-leaderboard', status: 'adapter', shape: 'passthrough' } }, // --------------------------------------------------------------------- diff --git a/server/deploy/NU7-RECEIPTS-REPAIR.md b/server/deploy/NU7-RECEIPTS-REPAIR.md new file mode 100644 index 00000000..1a8773e1 --- /dev/null +++ b/server/deploy/NU7-RECEIPTS-REPAIR.md @@ -0,0 +1,82 @@ +# Historical miner-receipts repair + +This release separates historical subsidy allocations from sampled NSM state. +Actual receipts remain indexed coinbase creation minus authoritative founders +and funding-stream payouts. It does not replace actual receipts with the maximum +permitted reward, and does not reconstruct historical NSM/supply/pool snapshots. + +## Release order + +1. Reconcile the intended explorer/API commit with serving main/testnet sources. + Apply **036_block_subsidy_allocations.sql**, then + **037_block_subsidy_allocation_privileges.sql** from the canonical Rust migration + directory (the API release includes identical deployment copies). Check both + ledger entries and effective API/job role grants: PostgreSQL default ACLs can + grant more than 036 explicitly requests. Migration 037 removes all non-owner + cache rights, including version-specific MAINTAIN, then restores existing + named-role SELECT/INSERT only. Source-table/default ACLs and data are unchanged. + No Rust binary or node update + is required. The API supports an absent allocation table during rollout. +2. Deploy the API and frontend from the tested commit. `receiptBlocks` is additive + on period points, selected/lifetime totals and legacy coverage. Receipts use + cached allocations only when height, canonical hash, network, source and + integer amounts match. The frontend supports older API responses explicitly. +3. On the **active official-testnet writer only**, configure the job using the + existing writer DB and local node credentials. `/etc/cipherscan/subsidy-allocations.env` + must set `NETWORK=testnet` and the applicable `DB_*`, `ZEBRA_RPC_URL` and cookie + variables. Protect it as mode 0600. Adjust the unit's checkout/runtime paths + to the verified host; do not infer testnet paths from mainnet defaults. +4. Run one bounded canary: + `node server/jobs/maintain-subsidy-allocations.js`. + Default batch is 100 (configurable 1–500); each run has a 45-second work budget, + 5-second DB statement deadlines and at most 8 seconds per RPC. The systemd + unit has a separate 60-second deadline and 5-second stop deadline. Per-height + RPC failures do not block later candidates; database/identity failures stop the + run. Prior immutable inserts remain. Private atomic state persists retries with + exponential backoff from 1 to 15 minutes. Repeat canaries until coverage reconciles. +5. Verify both sides of halving 4,497,948, actual receipts/underclaims, canonical + joins, fee totals, unchanged NSM sample counts and all chart periods. Period + caches last at most 60 seconds; allocation-table discovery caches 30 seconds. + Only then install/enable the timer for continuous gap maintenance. +6. Enable the separate health timer. It requires the maintenance timer to be + active and enabled, a successful run within five minutes, matching node/database + identity, a matching indexed-tip canonical anchor and no persistent canonical + allocation gaps older than 24 blocks. + Backed-off hashes remain visible to this check. State and health reports live + in `/var/lib/cipherscan-subsidy-allocations` with private file permissions. + After a healthy report exists, set the API `SUBSIDY_ALLOCATION_HEALTH_FILE` to + its `health.json`. The uncached deep-health endpoint returns HTTP 503 for a + missing, invalid, stale or unhealthy configured report. Confirm an external + monitor polls this endpoint before claiming notification coverage. + +The worker verifies DB is writable, expected network, node-announced NU7 and +node/database genesis identity, and takes a session advisory lock. A separate +ownership session serializes the complete state lifecycle before any state +read/write; overlapping invocations leave state untouched, and session loss +terminates the maintenance CLI. The CLI also has a 60-second wall-clock watchdog. It checks the +canonical node hash before and after each historical subsidy RPC, then inserts +only if the same block is still canonical in the database. A later reorg can +retain an orphan cache entry; canonical hash/height/network joins exclude it. +The worker has no reason to update canonical block/transaction rows and never +writes migration-027 observations. + +Keep the **passive standby writer and this timer inactive**. An unscheduled or +preactivation mainnet does no historical allocation work. Do not install a +scheduled writer on the passive standby as part of file/source synchronization. + +## Verification and rollback + +Run the focused accounting/worker/UI/v1 tests, including native PostgreSQL with +`TEST_ACCOUNTING_DATABASE_URL` pointing to a disposable database. Include the +real sparse-range regression: isolated receipt buckets must remain visible, and +missing data must never become zero or a connected line. + +Rollback the API/frontend and disable both new timers if needed. Remove the API +health-file setting when rolling back the monitor. The additive +cache and its narrowed grants can remain: old readers do not reference it, no original observations are +rewritten, and valid historical records are idempotent by hash. Preserve release +and migration evidence; do not delete the table to hide an unexplained mismatch. + +Local implementation and replay verification do **not** establish public +installation, migration application, a completed backfill or Vercel readiness. +Record those separately after an actual rollout. diff --git a/server/deploy/cipherscan-subsidy-allocations-health.service b/server/deploy/cipherscan-subsidy-allocations-health.service new file mode 100644 index 00000000..f4031d25 --- /dev/null +++ b/server/deploy/cipherscan-subsidy-allocations-health.service @@ -0,0 +1,18 @@ +[Unit] +Description=Verify historical subsidy allocation maintenance and canonical coverage +After=network-online.target postgresql.service +[Service] +Type=oneshot +WorkingDirectory=/root/cipherscan +Environment=NODE_PATH=/root/cipherscan/server/api/node_modules +EnvironmentFile=/etc/cipherscan/subsidy-allocations.env +StateDirectory=cipherscan-subsidy-allocations +StateDirectoryMode=0700 +ExecStart=/usr/bin/node server/jobs/check-subsidy-allocations.js +TimeoutStartSec=60 +TimeoutStopSec=5 +Nice=15 +CPUWeight=10 +IOWeight=10 +MemoryMax=256M +UMask=0077 diff --git a/server/deploy/cipherscan-subsidy-allocations-health.timer b/server/deploy/cipherscan-subsidy-allocations-health.timer new file mode 100644 index 00000000..adba3e3d --- /dev/null +++ b/server/deploy/cipherscan-subsidy-allocations-health.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Check subsidy allocation maintenance health every minute +[Timer] +OnBootSec=90 +OnUnitInactiveSec=60 +AccuracySec=5 +[Install] +WantedBy=timers.target diff --git a/server/deploy/cipherscan-subsidy-allocations.service b/server/deploy/cipherscan-subsidy-allocations.service new file mode 100644 index 00000000..c9264c9b --- /dev/null +++ b/server/deploy/cipherscan-subsidy-allocations.service @@ -0,0 +1,19 @@ +[Unit] +Description=Maintain canonical historical subsidy allocations for actual miner receipts +After=network-online.target postgresql.service +ConditionPathExists=/etc/cipherscan/subsidy-allocations.env +[Service] +Type=oneshot +WorkingDirectory=/root/cipherscan +Environment=NODE_PATH=/root/cipherscan/server/api/node_modules +EnvironmentFile=/etc/cipherscan/subsidy-allocations.env +StateDirectory=cipherscan-subsidy-allocations +StateDirectoryMode=0700 +ExecStart=/usr/bin/node server/jobs/maintain-subsidy-allocations.js +TimeoutStartSec=60 +TimeoutStopSec=5 +Nice=15 +CPUWeight=10 +IOWeight=10 +MemoryMax=256M +UMask=0077 diff --git a/server/deploy/cipherscan-subsidy-allocations.timer b/server/deploy/cipherscan-subsidy-allocations.timer new file mode 100644 index 00000000..b76146af --- /dev/null +++ b/server/deploy/cipherscan-subsidy-allocations.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Fill missing canonical historical subsidy allocations every minute +[Timer] +OnBootSec=60 +OnUnitInactiveSec=60 +AccuracySec=5 +[Install] +WantedBy=timers.target diff --git a/server/deploy/cipherscan-testnet-mining-behavior.service b/server/deploy/cipherscan-testnet-mining-behavior.service new file mode 100644 index 00000000..9d6e1a91 --- /dev/null +++ b/server/deploy/cipherscan-testnet-mining-behavior.service @@ -0,0 +1,18 @@ +[Unit] +Description=Refresh 30 completed UTC dates of attributed testnet miner payouts +After=network-online.target postgresql.service +ConditionPathExists=/etc/cipherscan/subsidy-allocations.env +[Service] +Type=oneshot +WorkingDirectory=/root/cipherscan +Environment=NODE_PATH=/root/cipherscan/server/api/node_modules +EnvironmentFile=/etc/cipherscan/subsidy-allocations.env +Environment=NETWORK=testnet +ExecStart=/usr/bin/node server/jobs/snapshot-mining-behavior.js --days=30 +TimeoutStartSec=600 +TimeoutStopSec=5 +Nice=15 +CPUWeight=10 +IOWeight=10 +MemoryMax=256M +UMask=0077 diff --git a/server/deploy/cipherscan-testnet-mining-behavior.timer b/server/deploy/cipherscan-testnet-mining-behavior.timer new file mode 100644 index 00000000..79faa129 --- /dev/null +++ b/server/deploy/cipherscan-testnet-mining-behavior.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Keep testnet miner reward spending snapshots refreshed hourly +[Timer] +OnBootSec=120 +OnCalendar=*-*-* *:20:00 UTC +RandomizedDelaySec=30 +Persistent=true +[Install] +WantedBy=timers.target diff --git a/server/deploy/sql/036_block_subsidy_allocations.sql b/server/deploy/sql/036_block_subsidy_allocations.sql new file mode 100644 index 00000000..45f7d3cd --- /dev/null +++ b/server/deploy/sql/036_block_subsidy_allocations.sql @@ -0,0 +1,32 @@ +-- Deterministic historical subsidy allocations, separate from real tip samples. +-- Readers must match network, height AND canonical hash. Orphan records may remain. +BEGIN; +SET LOCAL lock_timeout = '2s'; +SET LOCAL statement_timeout = '120s'; +CREATE TABLE IF NOT EXISTS public.block_subsidy_allocations ( + hash text PRIMARY KEY CHECK (hash ~ '^[0-9a-f]{64}$'), + height bigint NOT NULL CHECK (height BETWEEN 0 AND 499999999), + chain text NOT NULL CHECK (chain IN ('main', 'test')), + founders_zat bigint NOT NULL CHECK (founders_zat BETWEEN 0 AND 2100000000000000), + funding_streams_zat bigint NOT NULL CHECK (funding_streams_zat BETWEEN 0 AND 2100000000000000), + miner_zat bigint CHECK (miner_zat BETWEEN 0 AND 2100000000000000), + retrieved_at timestamptz NOT NULL DEFAULT now(), + source text NOT NULL CHECK (source = 'getblocksubsidy') +); +COMMENT ON TABLE public.block_subsidy_allocations IS +'Historical getblocksubsidy allocations in exact zatoshis. No NSM, pool or supply snapshot is inferred. Canonical hash, height and network must match when reading.'; +DO $$ +DECLARE r record; +BEGIN + FOR r IN SELECT DISTINCT grantee FROM information_schema.role_table_grants + WHERE table_schema='public' AND table_name='blocks' AND grantee <> 'PUBLIC' + LOOP + IF has_table_privilege(r.grantee,'public.blocks','SELECT') THEN + EXECUTE format('GRANT SELECT ON public.block_subsidy_allocations TO %I', r.grantee); + END IF; + IF has_table_privilege(r.grantee,'public.blocks','INSERT') THEN + EXECUTE format('GRANT INSERT ON public.block_subsidy_allocations TO %I', r.grantee); + END IF; + END LOOP; +END $$; +COMMIT; diff --git a/server/deploy/sql/037_block_subsidy_allocation_privileges.sql b/server/deploy/sql/037_block_subsidy_allocation_privileges.sql new file mode 100644 index 00000000..159aad0c --- /dev/null +++ b/server/deploy/sql/037_block_subsidy_allocation_privileges.sql @@ -0,0 +1,61 @@ +-- Default ACLs can give a newly created cache more than its SELECT/INSERT API. +-- Restrict only this table; leave its owner, source tables and defaults alone. +BEGIN; +SET LOCAL lock_timeout = '2s'; +SET LOCAL statement_timeout = '120s'; + +DO $$ +DECLARE + saved_grants jsonb; + r record; +BEGIN + -- Snapshot before CASCADE can remove grants delegated by another grantee. + -- Read the actual ACL, including PUBLIC (OID 0) and inherited group roles. + SELECT COALESCE(jsonb_agg(to_jsonb(grants)), '[]'::jsonb) + INTO saved_grants + FROM ( + SELECT acl.grantee, + bool_or(acl.privilege_type = 'SELECT') AS can_select, + bool_or(acl.privilege_type = 'INSERT') AS can_insert, + bool_or(acl.privilege_type = 'SELECT' AND acl.is_grantable) AS select_grantable, + bool_or(acl.privilege_type = 'INSERT' AND acl.is_grantable) AS insert_grantable + FROM pg_class AS t + CROSS JOIN LATERAL aclexplode(COALESCE(t.relacl, acldefault('r', t.relowner))) AS acl + WHERE t.oid = 'public.block_subsidy_allocations'::regclass + AND acl.grantee <> t.relowner + GROUP BY acl.grantee + ) AS grants; + + FOR r IN SELECT * FROM jsonb_to_recordset(saved_grants) AS grants(grantee oid) + LOOP + IF r.grantee = 0 THEN + REVOKE ALL PRIVILEGES ON TABLE public.block_subsidy_allocations FROM PUBLIC CASCADE; + ELSE + -- ALL also removes version-specific rights such as PG17+ MAINTAIN. + EXECUTE format('REVOKE ALL PRIVILEGES ON TABLE public.block_subsidy_allocations FROM %I CASCADE', + pg_get_userbyid(r.grantee)); + END IF; + END LOOP; + + FOR r IN SELECT * FROM jsonb_to_recordset(saved_grants) AS grants( + grantee oid, can_select boolean, can_insert boolean, + select_grantable boolean, insert_grantable boolean + ) + LOOP + -- Never restore PUBLIC access, including a default public writer grant. + IF r.grantee <> 0 THEN + IF r.can_select THEN + EXECUTE format('GRANT SELECT ON TABLE public.block_subsidy_allocations TO %I%s', + pg_get_userbyid(r.grantee), + CASE WHEN r.select_grantable THEN ' WITH GRANT OPTION' ELSE '' END); + END IF; + IF r.can_insert THEN + EXECUTE format('GRANT INSERT ON TABLE public.block_subsidy_allocations TO %I%s', + pg_get_userbyid(r.grantee), + CASE WHEN r.insert_grantable THEN ' WITH GRANT OPTION' ELSE '' END); + END IF; + END IF; + END LOOP; +END $$; + +COMMIT; diff --git a/server/deploy/testnet-mining-behavior.sql b/server/deploy/testnet-mining-behavior.sql new file mode 100644 index 00000000..b241b1ae --- /dev/null +++ b/server/deploy/testnet-mining-behavior.sql @@ -0,0 +1,50 @@ +-- Scoped provisioning of the existing miner-behavior summary on official testnet. +-- No raw chain rows, source-table privileges, default ACLs or other summaries change. +BEGIN; +SET LOCAL lock_timeout = '2s'; +SET LOCAL statement_timeout = '10s'; +DO $$ BEGIN + IF current_database() <> 'zcash_explorer_testnet' + OR pg_is_in_recovery() + OR (SELECT hash FROM public.blocks WHERE height=0) IS DISTINCT FROM + '05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38' THEN + RAISE EXCEPTION 'Official testnet primary database required'; + END IF; +END $$; +CREATE TABLE IF NOT EXISTS public.mining_behavior_daily ( + date date NOT NULL, + pool_name text NOT NULL, + miner_address text NOT NULL, + earned_zat bigint NOT NULL DEFAULT 0, + spent_zat bigint NOT NULL DEFAULT 0, + held_zat bigint NOT NULL DEFAULT 0, + blocks_mined integer NOT NULL DEFAULT 0, + outputs_spent integer NOT NULL DEFAULT 0, + outputs_total integer NOT NULL DEFAULT 0, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (date, pool_name) +); +DO $$ +DECLARE r record; +BEGIN + IF (SELECT relowner FROM pg_class WHERE oid='public.mining_behavior_daily'::regclass) <> current_user::regrole THEN + RAISE EXCEPTION 'Summary owner requires review'; + END IF; + -- Remove any rights supplied by default ACLs, including future table rights. + FOR r IN + SELECT DISTINCT a.grantee, bool_or(a.privilege_type='SELECT') OVER (PARTITION BY a.grantee) AS can_select + FROM pg_class t CROSS JOIN LATERAL aclexplode(COALESCE(t.relacl,acldefault('r',t.relowner))) a + WHERE t.oid='public.mining_behavior_daily'::regclass AND a.grantee<>t.relowner + LOOP + IF r.grantee=0 THEN + REVOKE ALL PRIVILEGES ON public.mining_behavior_daily FROM PUBLIC CASCADE; + ELSE + EXECUTE format('REVOKE ALL PRIVILEGES ON public.mining_behavior_daily FROM %I CASCADE',pg_get_userbyid(r.grantee)); + IF r.can_select THEN EXECUTE format('GRANT SELECT ON public.mining_behavior_daily TO %I',pg_get_userbyid(r.grantee)); END IF; + END IF; + END LOOP; +END $$; +GRANT SELECT, INSERT, DELETE ON public.mining_behavior_daily TO zcash_user; +COMMENT ON TABLE public.mining_behavior_daily IS +'Attributed miner payout output cohorts by completed UTC reward date. Exact zatoshis; current indexed spending at recomputation time, not historical spending-day flows or proof of sales. Missing/unattributed payouts are excluded.'; +COMMIT; diff --git a/server/jobs/check-subsidy-allocations.js b/server/jobs/check-subsidy-allocations.js new file mode 100644 index 00000000..04e5ceb4 --- /dev/null +++ b/server/jobs/check-subsidy-allocations.js @@ -0,0 +1,47 @@ +#!/usr/bin/env node +'use strict'; + +const { execFile } = require('node:child_process'); +const { promisify } = require('node:util'); +const path = require('node:path'); +const { loadEnv } = require('../lib/job-utils'); +loadEnv(__dirname); +const { getPool } = require('../lib/db-pool'); +const { callZebraRPC } = require('../lib/zebra-rpc'); +const { DEFAULT_STATE_FILE, readAllocationState, writePrivateJson } = require('../lib/subsidy-allocation-state'); +const { checkAllocationHealth } = require('../lib/subsidy-allocation-health'); + +const execute = promisify(execFile); +const pool = getPool({ max: 1, connectionTimeoutMillis: 5000, statement_timeout: 5000, query_timeout: 6000 }); +const stateFile = process.env.SUBSIDY_ALLOCATION_STATE_FILE || DEFAULT_STATE_FILE; +const healthFile = process.env.SUBSIDY_ALLOCATION_HEALTH_FILE || path.join(path.dirname(stateFile), 'health.json'); + +async function systemdProperties(unit, properties) { + const { stdout } = await execute('systemctl', ['show', unit, `--property=${properties.join(',')}`], { timeout: 3000, maxBuffer: 16384 }); + return Object.fromEntries(stdout.trim().split('\n').map(line => { + const equals = line.indexOf('='); + return [line.slice(0, equals), line.slice(equals + 1)]; + })); +} + +async function main() { + let health; + try { + const [state, timer, service] = await Promise.all([ + readAllocationState(stateFile, process.env.NETWORK), + systemdProperties('cipherscan-subsidy-allocations.timer', ['ActiveState', 'UnitFileState']), + systemdProperties('cipherscan-subsidy-allocations.service', ['Result']), + ]); + health = await checkAllocationHealth({ pool, rpc: callZebraRPC, network: process.env.NETWORK, state, + systemd: { timerActive: timer.ActiveState, timerEnabled: timer.UnitFileState, serviceResult: service.Result } }); + } catch { + health = { version: 1, network: process.env.NETWORK ?? null, checkedAt: new Date().toISOString(), status: 'unhealthy', + reason: 'monitor_unavailable' }; + } + await writePrivateJson(healthFile, health); + console.log(JSON.stringify(health)); + process.exitCode = health.status === 'healthy' ? 0 : 1; +} + +main().catch(() => { console.error('Allocation health status could not be persisted'); process.exitCode = 1; }) + .finally(() => pool.end()); diff --git a/server/jobs/maintain-subsidy-allocations.js b/server/jobs/maintain-subsidy-allocations.js new file mode 100644 index 00000000..64f7daa0 --- /dev/null +++ b/server/jobs/maintain-subsidy-allocations.js @@ -0,0 +1,27 @@ +#!/usr/bin/env node +'use strict'; + +const { loadEnv } = require('../lib/job-utils'); +loadEnv(__dirname); +const { getPool } = require('../lib/db-pool'); +const { callZebraRPC } = require('../lib/zebra-rpc'); +const { maintainAllocations } = require('../lib/subsidy-allocation-maintenance'); +const { DEFAULT_STATE_FILE, runMaintenanceWithState, sanitizeMessage } = require('../lib/subsidy-allocation-state'); +const watchdog = setTimeout(() => { + console.error('Allocation maintenance exceeded its wall-clock deadline'); + process.exit(1); +}, 60000); +watchdog.unref(); +const pool = getPool({ max: 2, connectionTimeoutMillis: 5000, statement_timeout: 5000, query_timeout: 6000 }); +runMaintenanceWithState({ + filePath: process.env.SUBSIDY_ALLOCATION_STATE_FILE || DEFAULT_STATE_FILE, + network: process.env.NETWORK, + ownershipPool: pool, + onOwnershipLost: () => { console.error('Allocation state ownership was lost'); process.exit(1); }, + maintain: ({ excludeHashes }) => maintainAllocations(pool, callZebraRPC, { + network: process.env.NETWORK, excludeHashes, + limit: process.env.SUBSIDY_ALLOCATION_BATCH === undefined ? 100 : Number(process.env.SUBSIDY_ALLOCATION_BATCH), + }), +}).then(({ result, exitCode }) => { console.log(JSON.stringify(result)); process.exitCode = exitCode; }) + .catch(error => { console.error(sanitizeMessage(error.message)); process.exitCode = 1; }) + .finally(async () => { await pool.end(); clearTimeout(watchdog); }); diff --git a/server/jobs/snapshot-mining-behavior.js b/server/jobs/snapshot-mining-behavior.js index 78daedd6..e897208f 100644 --- a/server/jobs/snapshot-mining-behavior.js +++ b/server/jobs/snapshot-mining-behavior.js @@ -2,14 +2,16 @@ /** * Mining Behavior Snapshot Job * - * Pre-computes daily miner sell/hold metrics by checking whether coinbase + * Pre-computes daily attributed-payout movement/unspent metrics by checking whether coinbase * outputs have been spent (via transaction_inputs.prev_txid lookup). * * Maintains the mining_behavior_daily table. * * Modes: * node snapshot-mining-behavior.js — incremental (last 7 days) - * node snapshot-mining-behavior.js --backfill — full history (slow, hours) + * node snapshot-mining-behavior.js --backfill — explicit full attributed reward history + * node snapshot-mining-behavior.js --days=30 — 30 completed UTC reward dates + * node snapshot-mining-behavior.js --from=DATE --to=DATE — bounded UTC dates * * Cron: * 0 5 * * * cd /root/cipherscan/server/jobs && node snapshot-mining-behavior.js >> /var/log/mining-behavior.log 2>&1 @@ -20,19 +22,14 @@ if (require.main === module) loadEnv(__dirname); const { getPool, getReadPool } = require('../lib/db-pool'); -const pool = require.main === module ? getPool({ max: 3 }) : null; -// The per-day work below is read (SELECT aggregate) then write (DELETE + -// INSERT), as two separate, already-autocommitted statement groups — there -// is no explicit BEGIN/COMMIT spanning them (withAdvisoryLock only takes a -// session-level pg_advisory_lock, not a SQL transaction), so the read half -// can safely move to the replica without ever mixing a single transaction -// across primary/replica. -const readPool = require.main === module ? getReadPool({ max: 3 }) : null; +const pool = require.main === module ? getPool({ max: 2, connectionTimeoutMillis: 5000, statement_timeout: 10000, query_timeout: 11000 }) : null; +// Each day is read under one repeatable-read snapshot, then replaced +// atomically on the writer after checking its canonical source anchor. +const readPool = require.main === module ? getReadPool({ max: 1, connectionTimeoutMillis: 5000, statement_timeout: 10000, query_timeout: 11000 }) : null; const LOCK_ID = 839275; -const BACKFILL_MODE = process.argv.includes('--backfill'); -const DAYS_FLAG = process.argv.find(a => a.startsWith('--days=')); -const INCREMENTAL_DAYS = DAYS_FLAG ? parseInt(DAYS_FLAG.split('=')[1]) : 7; +const { selectDays, parseDay } = require('../lib/utxo-age'); +const { setTimeout: pause } = require('node:timers/promises'); // Share the API registry so snapshots cannot retain obsolete pool identities. const { getPoolName } = require('../api/mining-pools'); @@ -50,9 +47,9 @@ async function readDay(reader, dateStr) { const result = await reader.query(` WITH coinbase_txs AS MATERIALIZED ( - SELECT b.miner_address, t.txid + SELECT b.height, b.miner_address, t.txid FROM blocks b - JOIN transactions t ON t.block_height = b.height AND t.is_coinbase = true + JOIN transactions t ON t.block_height = b.height AND t.block_hash = b.hash AND t.is_coinbase = true WHERE b.timestamp >= $1 AND b.timestamp < $2 AND b.miner_address IS NOT NULL AND b.miner_address NOT IN ('t3cFfPt1Bcvgez9ZbMBFWeZsskxTkPzGCow', 't2HifwjUj9uyxr9bknR8LFuQbc98c3vkXtu') @@ -63,34 +60,26 @@ async function readDay(reader, dateStr) { WHERE txo.txid IN (SELECT txid FROM coinbase_txs) ), day_coinbase AS ( - SELECT ct.miner_address, o.txid as coinbase_txid, o.vout_index, o.value + SELECT ct.height, ct.miner_address, o.txid as coinbase_txid, o.vout_index, o.value FROM coinbase_txs ct - JOIN all_cb_outputs o ON o.txid = ct.txid AND o.address = ct.miner_address + JOIN all_cb_outputs o ON o.txid = ct.txid AND o.address = ct.miner_address AND o.value > 0 + ), + observed AS MATERIALIZED ( + SELECT dc.*, EXISTS ( + SELECT 1 FROM transaction_inputs ti + JOIN transactions spending ON spending.txid=ti.txid + JOIN blocks spending_block ON spending_block.height=spending.block_height AND spending_block.hash=spending.block_hash + WHERE ti.prev_txid=dc.coinbase_txid AND ti.prev_vout=dc.vout_index + ) AS is_spent + FROM day_coinbase dc ) - SELECT dc.miner_address, COUNT(*) as output_count, - SUM(dc.value) as total_earned, - SUM(CASE WHEN ti.txid IS NOT NULL THEN dc.value ELSE 0 END) as total_spent, - SUM(CASE WHEN ti.txid IS NOT NULL THEN 1 ELSE 0 END) as spent_count - FROM day_coinbase dc - LEFT JOIN transaction_inputs ti - ON ti.prev_txid = dc.coinbase_txid AND ti.prev_vout = dc.vout_index - GROUP BY dc.miner_address - `, [dayStart, dayEnd]); - - const blockCounts = await reader.query(` - SELECT miner_address, COUNT(*) as blocks - FROM blocks - WHERE timestamp >= $1 AND timestamp < $2 - AND miner_address IS NOT NULL - AND miner_address NOT IN ('t3cFfPt1Bcvgez9ZbMBFWeZsskxTkPzGCow', 't2HifwjUj9uyxr9bknR8LFuQbc98c3vkXtu') - GROUP BY miner_address + SELECT miner_address, COUNT(*) as output_count, COUNT(DISTINCT height)::int as blocks, + SUM(value) as total_earned, + SUM(CASE WHEN is_spent THEN value ELSE 0 END) as total_spent, + SUM(CASE WHEN is_spent THEN 1 ELSE 0 END) as spent_count + FROM observed GROUP BY miner_address `, [dayStart, dayEnd]); - const blockMap = {}; - for (const row of blockCounts.rows) { - blockMap[row.miner_address] = parseInt(row.blocks); - } - const poolAgg = {}; for (const row of result.rows) { const poolName = getPoolNameForAddress(row.miner_address); @@ -107,7 +96,7 @@ async function readDay(reader, dateStr) { const entry = poolAgg[poolName]; entry.earned += BigInt(row.total_earned || 0); entry.spent += BigInt(row.total_spent || 0); - entry.blocks += blockMap[row.miner_address] || 0; + entry.blocks += parseInt(row.blocks || 0); entry.outputsSpent += parseInt(row.spent_count || 0); entry.outputsTotal += parseInt(row.output_count || 0); } @@ -144,46 +133,91 @@ async function computeDay(client, dateStr, reader = readPool) { return writeDay(client, dateStr, await readDay(reader, dateStr)); } -async function run() { - const client = await pool.connect(); - try { - await withAdvisoryLock(client, LOCK_ID, async (client) => { - log(`Starting mining behavior snapshot (${BACKFILL_MODE ? 'BACKFILL' : 'incremental'})...`); - - let startDate; - if (BACKFILL_MODE) { - const earliest = await readPool.query( - `SELECT MIN(date_trunc('day', to_timestamp(timestamp)))::date as min_date FROM blocks WHERE miner_address IS NOT NULL` - ); - startDate = earliest.rows[0]?.min_date || new Date('2016-10-28'); - } else { - startDate = new Date(); - startDate.setDate(startDate.getDate() - INCREMENTAL_DAYS); - } - - const endDate = new Date(); - endDate.setDate(endDate.getDate() - 1); - - let current = new Date(startDate); - let daysProcessed = 0; - - while (current <= endDate) { - const dateStr = current.toISOString().slice(0, 10); - const poolCount = await computeDay(client, dateStr); - daysProcessed++; - - if (daysProcessed % 30 === 0 || !BACKFILL_MODE) { - log(` ${dateStr}: ${poolCount} pools`); - } +async function processDates(client, readers, dates, { network = 'mainnet', pauseMs = 100, continueOnError = false } = {}) { + const expectedGenesis = { + mainnet: '00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08', + testnet: '05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38', + }[network]; + if (!expectedGenesis) throw new Error('Unsupported mining snapshot network'); + const guard = (await client.query('SELECT current_database() AS database, pg_is_in_recovery() AS recovery, (SELECT hash FROM blocks WHERE height=0) AS genesis')).rows[0]; + if (guard?.recovery !== false || guard.genesis !== expectedGenesis || (network === 'testnet' && guard.database !== 'zcash_explorer_testnet')) throw new Error('Mining snapshot database/network mismatch'); + let processed = 0; + const failedDates = []; + for (const date of dates) { + try { + await processDate(client, readers, date); + processed++; + } catch (error) { + if (!continueOnError) throw error; + failedDates.push(date); + log(`ERROR: snapshot ${date} unavailable`); + } + await pause(pauseMs); + } + return { processed, failedDates }; +} - current.setDate(current.getDate() + 1); +async function processDate(client, readers, date) { + const reader = await readers.connect(); + let anchor, data; + try { + await reader.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'); + await reader.query("SET LOCAL statement_timeout='10s'; SET LOCAL work_mem='16MB'; SET LOCAL max_parallel_workers_per_gather=0; SET LOCAL TIME ZONE 'UTC'"); + anchor = (await reader.query('SELECT height,hash,timestamp FROM blocks ORDER BY height DESC LIMIT 1')).rows[0]; + if (!anchor || Number(anchor.timestamp) < Date.parse(date+'T00:00:00Z')/1000+86400) throw new Error('Incomplete source day'); + const counts = (await reader.query(` + SELECT COUNT(*)::int AS blocks, COUNT(*) FILTER (WHERE cb.n <> 1)::int AS incomplete_coinbases + FROM blocks b CROSS JOIN LATERAL ( + SELECT COUNT(*)::int n FROM transactions t + WHERE t.block_hash=b.hash AND t.block_height=b.height AND t.is_coinbase=true + ) cb WHERE b.timestamp >= $1 AND b.timestamp < $2 + `, [Date.parse(date+'T00:00:00Z')/1000, Date.parse(date+'T00:00:00Z')/1000+86400])).rows[0]; + if (!counts || counts.incomplete_coinbases !== 0) throw new Error('Incomplete canonical coinbase source'); + data = await readDay(reader, date); + for (const row of Object.values(data)) { + if (row.earned < 0n || row.spent < 0n || row.spent > row.earned) throw new Error('Invalid mining snapshot accounting'); } + await reader.query('COMMIT'); + } catch (error) { await reader.query('ROLLBACK'); throw error; } + finally { reader.release(); } + await client.query('BEGIN'); + try { + await client.query("SET LOCAL lock_timeout='2s'; SET LOCAL statement_timeout='5s'; SET LOCAL TIME ZONE 'UTC'"); + if (!(await client.query('SELECT 1 FROM blocks WHERE height=$1 AND hash=$2', [anchor.height,anchor.hash])).rowCount) throw new Error('Source chain changed'); + const pools = await writeDay(client,date,data); + if (!(await client.query('SELECT 1 FROM blocks WHERE height=$1 AND hash=$2', [anchor.height,anchor.hash])).rowCount) throw new Error('Source chain changed'); + await client.query('COMMIT'); + log(`Completed ${date}: ${pools} tracked payout groups`); + } catch (error) { await client.query('ROLLBACK'); throw error; } +} - log(`Done. Processed ${daysProcessed} days.`); +function selectSnapshotDates(args, now = new Date(), earliest = null) { + if (!args.includes('--backfill')) return selectDays(args, now); + if (args.length !== 1) throw new Error('--backfill cannot be combined with bounded date options'); + if (earliest === null) return []; + const first = parseDay(earliest), last = Math.floor(now.getTime()/86400000)-1; + return Array.from({length:Math.max(0,last-first+1)},(_,i)=>new Date((first+i)*86400000).toISOString().slice(0,10)); +} + +async function run() { + const client = await pool.connect(); + try { + // Existing mainnet cron defaults to seven completed days. Testnet's scoped + // unit explicitly refreshes 30 days; no genesis reconstruction is implicit. + const args = process.argv.slice(2); + const earliest = args.includes('--backfill') + ? (await readPool.query("SELECT MIN((to_timestamp(timestamp) AT TIME ZONE 'UTC')::date)::text AS date FROM blocks WHERE miner_address IS NOT NULL")).rows[0]?.date ?? null + : null; + const dates = selectSnapshotDates(args, new Date(), earliest); + const network = process.env.NETWORK || 'mainnet'; + await withAdvisoryLock(client, LOCK_ID, async () => { + const result = await processDates(client, readPool, network === 'testnet' ? dates.reverse() : dates, { network, continueOnError: network === 'testnet' }); + if (result.failedDates.length) process.exitCode = 1; + log(`Finished: ${result.processed} dates refreshed, ${result.failedDates.length} unavailable`); }); } catch (error) { - log(`ERROR: ${error.message}`); - console.error(error); + log('ERROR: mining behavior snapshot failed'); + console.error(error.code || 'snapshot_failed'); process.exitCode = 1; } finally { client.release(); @@ -193,4 +227,4 @@ async function run() { } if (require.main === module) run(); -module.exports = { computeDay, readDay, writeDay }; +module.exports = { computeDay, readDay, writeDay, processDates, selectSnapshotDates }; diff --git a/server/lib/subsidy-allocation-health.js b/server/lib/subsidy-allocation-health.js new file mode 100644 index 00000000..4cc95288 --- /dev/null +++ b/server/lib/subsidy-allocation-health.js @@ -0,0 +1,64 @@ +'use strict'; + +const { networkSchedule } = require('../api/lib/network-schedule'); +const { CANDIDATES_SQL } = require('./subsidy-allocation-maintenance'); + +const MAX_AGE_MS = 5 * 60_000; +const GAP_GRACE_BLOCKS = 24; + +async function checkAllocationHealth({ pool, rpc, network, state, systemd, now = Date.now() }) { + const health = { version: 1, network, checkedAt: new Date(now).toISOString(), status: 'unhealthy', reason: '' }; + const fail = reason => ({ ...health, reason }); + const chain = { mainnet: 'main', testnet: 'test' }[network]; + if (!chain) return fail('network_unconfigured'); + if (systemd?.timerActive !== 'active' || systemd?.timerEnabled !== 'enabled') return fail('timer_inactive'); + if (systemd.serviceResult !== 'success') return fail('job_failed'); + if (!state || state.network !== network || state.version !== 1 + || !['running', 'complete', 'budget', 'inactive', 'busy', 'degraded', 'error'].includes(state.status)) return fail('state_unavailable'); + for (const [field, label] of [['lastFinished', 'completion'], ['lastSuccess', 'success']]) { + const timestamp = Date.parse(state[field]); + if (!Number.isFinite(timestamp) || timestamp > now + 30_000 || now - timestamp > MAX_AGE_MS) { + return fail(`job_${label}_stale`); + } + } + if (['error', 'degraded'].includes(state.status)) return fail('job_failed'); + if (state.status === 'running' && (!Number.isFinite(Date.parse(state.lastAttempt)) || now - Date.parse(state.lastAttempt) > 75_000)) { + return fail('job_hung'); + } + try { + const recovery = await pool.query('SELECT pg_is_in_recovery() AS recovery'); + if (recovery.rows[0]?.recovery !== false) return fail('database_passive'); + const info = await rpc('getblockchaininfo', [], { timeout: 8000 }); + if (info?.chain !== chain) return fail('network_mismatch'); + const schedule = networkSchedule(info); + if (!schedule || !Number.isSafeInteger(info.blocks) || info.blocks < 0 || info.blocks > 499999999) { + return fail('schedule_unavailable'); + } + const genesis = await pool.query('SELECT hash FROM public.blocks WHERE height=0'); + const hash = genesis.rows[0]?.hash; + if (!/^[0-9a-f]{64}$/.test(hash ?? '') || await rpc('getblockhash', [0], { timeout: 8000 }) !== hash) { + return fail('genesis_mismatch'); + } + const anchor = (await pool.query('SELECT height, hash FROM public.blocks WHERE height <= $1 ORDER BY height DESC LIMIT 1', [info.blocks])).rows[0]; + const anchorHeight = Number(anchor?.height); + if (!Number.isSafeInteger(anchorHeight) || anchorHeight < 0 || anchorHeight > info.blocks + || (info.blocks > 0 && anchorHeight === 0) || !/^[0-9a-f]{64}$/.test(anchor?.hash ?? '')) return fail('anchor_unavailable'); + if (schedule.nu7Height !== null && info.blocks - GAP_GRACE_BLOCKS >= schedule.nu7Height) { + const missing = await pool.query(CANDIDATES_SQL, [schedule.nu7Height, info.blocks - GAP_GRACE_BLOCKS, chain, 1, []]); + if (missing.rows.length) { + const height = Number(missing.rows[0].height); + return { ...fail('allocation_gap'), ...(Number.isSafeInteger(height) && height >= 0 ? { missingHeight: height } : {}) }; + } + } + const currentAnchor = (await pool.query('SELECT hash FROM public.blocks WHERE height=$1', [anchorHeight])).rows[0]?.hash; + if (currentAnchor !== anchor.hash) return fail('anchor_changed'); + try { + if (await rpc('getblockhash', [anchorHeight], { timeout: 8000 }) !== anchor.hash) return fail('anchor_mismatch'); + } catch { return fail('anchor_unavailable'); } + return { ...health, status: 'healthy', reason: 'ok' }; + } catch { + return fail('coverage_unavailable'); + } +} + +module.exports = { MAX_AGE_MS, GAP_GRACE_BLOCKS, checkAllocationHealth }; diff --git a/server/lib/subsidy-allocation-maintenance.js b/server/lib/subsidy-allocation-maintenance.js new file mode 100644 index 00000000..ad4122f7 --- /dev/null +++ b/server/lib/subsidy-allocation-maintenance.js @@ -0,0 +1,120 @@ +'use strict'; + +const { networkSchedule } = require('../api/lib/network-schedule'); +const { sampledAllocationsSql, parseRpcSubsidyAllocations } = require('../api/lib/subsidy-allocations'); + +const LOCK_ID = 360036; +const HASH = /^[0-9a-f]{64}$/; +const sampled = sampledAllocationsSql('a'); +const CANDIDATES_SQL = `SELECT b.height, b.hash FROM public.blocks b + LEFT JOIN public.node_accounting_observations a + ON a.height=b.height AND a.hash=b.hash AND a.chain=$3 + WHERE b.height BETWEEN $1 AND $2 + AND (${sampled.founders} IS NULL OR ${sampled.funding} IS NULL) + AND NOT EXISTS (SELECT 1 FROM public.block_subsidy_allocations c + WHERE c.hash=b.hash AND c.height=b.height AND c.chain=$3) + AND NOT (b.hash = ANY($5::text[])) + ORDER BY b.height LIMIT $4`; + +// One statement rechecks DB identity; a later reorg can retain an orphan cache +// record, but canonical hash/height/network joins can never expose it as current. +const INSERT_SQL = `INSERT INTO public.block_subsidy_allocations + (hash,height,chain,founders_zat,funding_streams_zat,miner_zat,source) + SELECT b.hash,b.height,$3,$4,$5,$6,'getblocksubsidy' + FROM public.blocks b WHERE b.hash=$1 AND b.height=$2 + ON CONFLICT (hash) DO NOTHING RETURNING hash`; + +class AllocationBudgetError extends Error { + constructor() { + super('Allocation maintenance time budget exhausted'); + this.name = 'AllocationBudgetError'; + } +} + +async function maintainAllocations(pool, rpc, { + network, limit = 100, timeBudgetMs = 45000, now = Date.now, excludeHashes = [], +} = {}) { + const expectedChain = { mainnet: 'main', testnet: 'test' }[network]; + if (!expectedChain) throw new Error('Explicit NETWORK=mainnet or testnet is required'); + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500) throw new Error('Allocation batch limit must be 1–500'); + if (!Number.isSafeInteger(timeBudgetMs) || timeBudgetMs < 1 || timeBudgetMs > 60000) throw new Error('Allocation time budget must be 1–60000 ms'); + if (!Array.isArray(excludeHashes) || excludeHashes.length > 5000 || excludeHashes.some(hash => typeof hash !== 'string' || !HASH.test(hash))) { + throw new Error('Allocation exclusions must contain at most 5000 lowercase hexadecimal block hashes'); + } + const exclusions = [...new Set(excludeHashes)]; + const deadline = now() + timeBudgetMs; + const client = await pool.connect(); + let locked = false; + const result = { status: 'complete', candidates: 0, processed: 0, written: 0, skipped: 0, + failures: [], writtenHashes: [] }; + const call = async (method, params = []) => { + const remaining = deadline - now(); + if (remaining <= 0) throw new AllocationBudgetError(); + return rpc(method, params, { timeout: Math.min(8000, remaining) }); + }; + try { + const recovery = await client.query('SELECT pg_is_in_recovery() AS recovery'); + if (recovery.rows[0]?.recovery !== false) throw new Error('Refusing allocation writes on a passive or unknown database'); + const lock = await client.query('SELECT pg_try_advisory_lock($1) AS acquired', [LOCK_ID]); + locked = lock.rows[0]?.acquired === true; + if (!locked) return { ...result, status: 'busy' }; + + const info = await call('getblockchaininfo'); + if (info?.chain !== expectedChain) throw new Error('Serving node network does not match configured network'); + const schedule = networkSchedule(info); + if (!schedule || !Number.isSafeInteger(info.blocks) || info.blocks > 499999999) throw new Error('Serving node schedule or height is unavailable'); + if (schedule.nu7Height === null || info.blocks < schedule.nu7Height) return { ...result, status: 'inactive' }; + const genesis = await client.query('SELECT hash FROM public.blocks WHERE height=0'); + const hash = genesis.rows[0]?.hash; + if (!HASH.test(hash ?? '') || await call('getblockhash', [0]) !== hash) throw new Error('Database and serving node genesis do not match'); + + const { rows } = await client.query(CANDIDATES_SQL, [schedule.nu7Height, info.blocks, expectedChain, limit, exclusions]); + result.candidates = rows.length; + for (const row of rows) { + if (now() >= deadline) { result.status = 'budget'; break; } + const height = Number(row.height); + if (!Number.isSafeInteger(height) || height < schedule.nu7Height || height > info.blocks || !HASH.test(row.hash ?? '')) throw new Error('Invalid canonical allocation candidate'); + result.processed++; + let allocations; + try { + if (await call('getblockhash', [height]) !== row.hash) { result.skipped++; continue; } + const subsidy = await call('getblocksubsidy', [height]); + if (now() >= deadline) throw new AllocationBudgetError(); + allocations = parseRpcSubsidyAllocations(subsidy); + if (!allocations) { + result.failures.push({ height, hash: row.hash, message: 'Historical subsidy allocations unavailable.' }); + result.status = 'partial'; + continue; + } + if (await call('getblockhash', [height]) !== row.hash) { result.skipped++; continue; } + } catch (error) { + // A deadline-limited timeout says nothing about the block. Leave it + // eligible next run instead of teaching the caller to back it off. + if (error instanceof AllocationBudgetError || now() >= deadline) { + result.status = 'budget'; + break; + } + result.failures.push({ height, hash: row.hash, message: 'Historical subsidy RPC unavailable.' }); + result.status = 'partial'; + continue; + } + if (now() >= deadline) { result.status = 'budget'; break; } + // Database errors remain fatal: an allocation RPC failure must never + // hide a broken writer, schema, connection or canonical insert guard. + const written = await client.query(INSERT_SQL, [row.hash, height, expectedChain, + allocations.foundersZat, allocations.fundingStreamsZat, allocations.minerZat]); + result.written += written.rowCount; + if (written.rowCount) result.writtenHashes.push(row.hash); + else result.skipped++; + } + return result; + } catch (error) { + if (error instanceof AllocationBudgetError) return { ...result, status: 'budget' }; + throw error; + } finally { + if (locked) await client.query('SELECT pg_advisory_unlock($1)', [LOCK_ID]).catch(() => {}); + client.release(); + } +} + +module.exports = { maintainAllocations, CANDIDATES_SQL, INSERT_SQL }; diff --git a/server/lib/subsidy-allocation-state.js b/server/lib/subsidy-allocation-state.js new file mode 100644 index 00000000..768d02e1 --- /dev/null +++ b/server/lib/subsidy-allocation-state.js @@ -0,0 +1,170 @@ +'use strict'; + +const fs = require('node:fs/promises'); +const path = require('node:path'); +const { randomUUID } = require('node:crypto'); + +const DEFAULT_STATE_FILE = '/var/lib/cipherscan-subsidy-allocations/state.json'; +const MAX_RETRIES = 5000; +// Always acquire this session lock before the worker's 360036 lock. The CLI +// pool has two connections; the outer session remains checked out throughout. +const ALLOCATION_STATE_LOCK_ID = 360037; +const HASH = /^[0-9a-f]{64}$/; +const NETWORKS = new Set(['mainnet', 'testnet']); +const STATUSES = new Set(['new', 'running', 'complete', 'budget', 'inactive', 'busy', 'degraded', 'error']); + +function sanitizeMessage(value) { + return String(value ?? 'Allocation maintenance failed') + .replace(/[a-z][a-z0-9+.-]*:\/\/\S+/gi, '[url]') + .replace(/(password|token|secret|cookie|authorization|api[-_ ]?key)\s*[:=]\s*\S+/gi, '$1=[redacted]') + .replace(/[\r\n\x00-\x1f\x7f]/g, ' ').slice(0, 240); +} + +function initialState(network) { + if (!NETWORKS.has(network)) throw new Error('Explicit NETWORK=mainnet or testnet is required'); + return { version: 1, network, status: 'new', lastAttempt: null, lastFinished: null, + lastSuccess: null, lastError: null, result: null, retries: {} }; +} + +async function readAllocationState(filePath, network) { + const empty = initialState(network); + let input; + try { + if ((await fs.stat(filePath)).size > 4 * 1024 * 1024) throw new Error('Allocation state exceeds size limit'); + input = JSON.parse(await fs.readFile(filePath, 'utf8')); + } catch (error) { + if (error.code === 'ENOENT') return empty; + throw new Error('Allocation state is unreadable or malformed'); + } + if (input?.version !== 1 || input.network !== network || !STATUSES.has(input.status) || !input.retries || Array.isArray(input.retries) + || typeof input.retries !== 'object' || Object.keys(input.retries).length > MAX_RETRIES) { + throw new Error('Allocation state identity or retry entries are invalid'); + } + for (const [hash, retry] of Object.entries(input.retries)) { + if (!HASH.test(hash) || !Number.isSafeInteger(retry?.height) || retry.height < 0 || retry.height > 499999999 + || !Number.isSafeInteger(retry.attempts) || retry.attempts < 1 || retry.attempts > 32 + || !Number.isFinite(Date.parse(retry.nextRetryAt)) || !Number.isFinite(Date.parse(retry.lastFailedAt))) { + throw new Error('Allocation retry entry is invalid'); + } + } + return { ...empty, ...input }; +} + +async function writePrivateJson(filePath, value, { beforePublish = () => {} } = {}) { + await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 }); + const temporary = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + let handle; + try { + handle = await fs.open(temporary, 'wx', 0o600); + await handle.writeFile(`${JSON.stringify(value)}\n`); + await handle.sync(); + await handle.close(); + handle = null; + beforePublish(); + await fs.rename(temporary, filePath); + } finally { + await handle?.close().catch(() => {}); + await fs.unlink(temporary).catch(() => {}); + } +} + +function deferredHashes(state, now) { + return Object.entries(state.retries).filter(([, retry]) => Date.parse(retry.nextRetryAt) > now) + .map(([hash]) => hash).slice(0, MAX_RETRIES); +} + +function finishState(state, result, finishedAt) { + const retries = { ...state.retries }; + for (const hash of result.writtenHashes ?? []) if (HASH.test(hash)) delete retries[hash]; + const failures = (result.failures ?? []).map(failure => ({ height: failure.height, hash: failure.hash, + message: sanitizeMessage(failure.message) })); + for (const failure of failures) { + if (!HASH.test(failure.hash) || !Number.isSafeInteger(failure.height) || failure.height < 0 || failure.height > 499999999) { + throw new Error('Maintenance returned an invalid failed candidate'); + } + const attempts = Math.min(32, (retries[failure.hash]?.attempts ?? 0) + 1); + const delay = Math.min(15 * 60_000, 60_000 * 2 ** Math.min(attempts - 1, 4)); + retries[failure.hash] = { height: failure.height, attempts, + lastFailedAt: new Date(finishedAt).toISOString(), nextRetryAt: new Date(finishedAt + delay).toISOString(), + message: failure.message }; + } + const bounded = Object.fromEntries(Object.entries(retries) + .sort((a, b) => Date.parse(b[1].lastFailedAt) - Date.parse(a[1].lastFailedAt)).slice(0, MAX_RETRIES)); + const successful = failures.length === 0 && ['complete', 'budget', 'inactive'].includes(result.status); + return { ...state, status: failures.length ? 'degraded' : result.status, + lastFinished: new Date(finishedAt).toISOString(), + lastSuccess: successful ? new Date(finishedAt).toISOString() : state.lastSuccess, + lastError: failures.length ? `${failures.length} historical allocation candidate(s) failed` : null, + result: { ...result, failures }, retries: bounded }; +} + +async function withStateOwnership(pool, work, onOwnershipLost = () => {}) { + if (!pool || typeof pool.connect !== 'function') throw new Error('Allocation state requires a dedicated ownership session'); + const client = await pool.connect(); + let locked = false; + let lost = null; + const assertOwned = () => { if (lost) throw lost; }; + const onLost = () => { + if (lost) return; + lost = new Error('Allocation state ownership connection was lost'); + onOwnershipLost(); + }; + client.on('error', onLost); + client.on('end', onLost); + try { + const result = await client.query('SELECT pg_try_advisory_lock($1) AS acquired', [ALLOCATION_STATE_LOCK_ID]); + assertOwned(); + locked = result.rows[0]?.acquired === true; + if (!locked) return { state: null, exitCode: 0, + result: { status: 'busy', candidates: 0, processed: 0, written: 0, skipped: 0, failures: [], writtenHashes: [] } }; + return await work(assertOwned); + } finally { + if (locked && !lost) { + try { await client.query('SELECT pg_advisory_unlock($1)', [ALLOCATION_STATE_LOCK_ID]); } + catch { lost = new Error('Allocation state ownership could not be released'); } + } + client.off('error', onLost); + client.off('end', onLost); + client.release(lost); + } +} + +async function runOwnedMaintenance({ filePath, network, maintain, now }, assertOwned) { + const read = async () => { + assertOwned(); + const state = await readAllocationState(filePath, network); + assertOwned(); + return state; + }; + const write = async state => { + assertOwned(); + await writePrivateJson(filePath, state, { beforePublish: assertOwned }); + assertOwned(); + }; + const started = now(); + const previous = await read(); + const running = { ...previous, status: 'running', lastAttempt: new Date(started).toISOString(), lastError: null }; + await write(running); + try { + const result = await maintain({ excludeHashes: deferredHashes(previous, started) }); + const state = finishState(await read(), result, now()); + await write(state); + return { result: state.result, state, exitCode: state.result.failures.length ? 1 : 0 }; + } catch (error) { + assertOwned(); + const current = await read().catch(() => running); + await write({ ...current, status: 'error', + lastFinished: new Date(now()).toISOString(), lastError: sanitizeMessage(error.message) }); + throw error; + } +} + +async function runMaintenanceWithState({ filePath = DEFAULT_STATE_FILE, network, maintain, now = Date.now, + ownershipPool, onOwnershipLost } = {}) { + initialState(network); + return withStateOwnership(ownershipPool, + assertOwned => runOwnedMaintenance({ filePath, network, maintain, now }, assertOwned), onOwnershipLost); +} + +module.exports = { DEFAULT_STATE_FILE, MAX_RETRIES, ALLOCATION_STATE_LOCK_ID, initialState, readAllocationState, writePrivateJson, + sanitizeMessage, deferredHashes, finishState, withStateOwnership, runMaintenanceWithState }; diff --git a/server/tests/fixtures/mining-metrics-testnet-header-window.json b/server/tests/fixtures/mining-metrics-testnet-header-window.json new file mode 100644 index 00000000..28213cf2 --- /dev/null +++ b/server/tests/fixtures/mining-metrics-testnet-header-window.json @@ -0,0 +1,540 @@ +{ + "network": "testnet", + "source": "https://api.testnet.zecblock.com/api/blocks/list?cursor=4508003&limit=100", + "capturedDate": "2026-10-16", + "note": "Public canonical indexed header timestamps; same-second heights 4507966/4507967 do not imply missing blocks. Values normalized only to JSON integers for height/timestamp.", + "rows": [ + { + "height": 4508002, + "timestamp": 1792147560, + "difficulty": "10962.9501822337", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4508001, + "timestamp": 1792147525, + "difficulty": "10899.3311667693", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4508000, + "timestamp": 1792147522, + "difficulty": "10890.8739984247", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507999, + "timestamp": 1792147496, + "difficulty": "10882.4264976306", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507998, + "timestamp": 1792147476, + "difficulty": "10857.6427605598", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507997, + "timestamp": 1792147406, + "difficulty": "10870.3523603221", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507996, + "timestamp": 1792147334, + "difficulty": "10858.4799917328", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507995, + "timestamp": 1792147331, + "difficulty": "10838.6886929256", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507994, + "timestamp": 1792147316, + "difficulty": "10823.539605838", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507993, + "timestamp": 1792147309, + "difficulty": "10816.7623736786", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507992, + "timestamp": 1792147303, + "difficulty": "10842.5809602339", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507991, + "timestamp": 1792147260, + "difficulty": "10867.8974346647", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507990, + "timestamp": 1792147240, + "difficulty": "10851.8748662058", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507989, + "timestamp": 1792147230, + "difficulty": "10852.7420591979", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507988, + "timestamp": 1792147206, + "difficulty": "10870.0153440775", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507987, + "timestamp": 1792147205, + "difficulty": "10862.8535343971", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507986, + "timestamp": 1792147182, + "difficulty": "10908.9547089923", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507985, + "timestamp": 1792147150, + "difficulty": "10868.6846779821", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507984, + "timestamp": 1792147134, + "difficulty": "10902.7273997097", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507983, + "timestamp": 1792147099, + "difficulty": "10866.6703875985", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507982, + "timestamp": 1792147066, + "difficulty": "10875.9505728287", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507981, + "timestamp": 1792147032, + "difficulty": "10852.3615740622", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507980, + "timestamp": 1792146987, + "difficulty": "10902.5751810476", + "transaction_count": 5, + "total_fees": "70000" + }, + { + "height": 4507979, + "timestamp": 1792146966, + "difficulty": "10906.7731299753", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507978, + "timestamp": 1792146931, + "difficulty": "10935.4420325348", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507977, + "timestamp": 1792146917, + "difficulty": "10938.6379330489", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507976, + "timestamp": 1792146901, + "difficulty": "10945.628072554", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507975, + "timestamp": 1792146892, + "difficulty": "11069.7247336604", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507974, + "timestamp": 1792146814, + "difficulty": "11066.950804423", + "transaction_count": 7, + "total_fees": "130000" + }, + { + "height": 4507973, + "timestamp": 1792146801, + "difficulty": "11072.5285990411", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507972, + "timestamp": 1792146775, + "difficulty": "11036.0717993883", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507971, + "timestamp": 1792146771, + "difficulty": "11041.8030163434", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507970, + "timestamp": 1792146766, + "difficulty": "11034.1509040479", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507969, + "timestamp": 1792146650, + "difficulty": "11005.3399570993", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507968, + "timestamp": 1792146639, + "difficulty": "10997.7629236148", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507967, + "timestamp": 1792146633, + "difficulty": "11065.7104736852", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507966, + "timestamp": 1792146633, + "difficulty": "11112.4247315253", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507965, + "timestamp": 1792146624, + "difficulty": "11103.6445352163", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507964, + "timestamp": 1792146622, + "difficulty": "11116.1277046886", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507963, + "timestamp": 1792146612, + "difficulty": "11141.3265542234", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507962, + "timestamp": 1792146601, + "difficulty": "11153.4999899371", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507961, + "timestamp": 1792146537, + "difficulty": "11156.8137680809", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507960, + "timestamp": 1792146464, + "difficulty": "11125.8498539961", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507959, + "timestamp": 1792146436, + "difficulty": "11137.8667190501", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507958, + "timestamp": 1792146405, + "difficulty": "11141.1350875492", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507957, + "timestamp": 1792146368, + "difficulty": "11187.2729661989", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507956, + "timestamp": 1792146351, + "difficulty": "11207.5217774072", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507955, + "timestamp": 1792146315, + "difficulty": "11232.0135151568", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507954, + "timestamp": 1792146313, + "difficulty": "11221.1851304165", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507953, + "timestamp": 1792146299, + "difficulty": "11181.0295665553", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507952, + "timestamp": 1792146288, + "difficulty": "11179.8144614685", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507951, + "timestamp": 1792146242, + "difficulty": "11190.6760952376", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507950, + "timestamp": 1792146192, + "difficulty": "11205.6430642573", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507949, + "timestamp": 1792146164, + "difficulty": "11299.0116006702", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507948, + "timestamp": 1792146158, + "difficulty": "11308.19665523", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507947, + "timestamp": 1792146101, + "difficulty": "11263.9744873138", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507946, + "timestamp": 1792146087, + "difficulty": "11246.6127544218", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507945, + "timestamp": 1792146064, + "difficulty": "11255.7238429687", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507944, + "timestamp": 1792146045, + "difficulty": "11230.0274811456", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507943, + "timestamp": 1792145953, + "difficulty": "11196.4814923918", + "transaction_count": 3, + "total_fees": "85000" + }, + { + "height": 4507942, + "timestamp": 1792145939, + "difficulty": "11180.2073408066", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507941, + "timestamp": 1792145933, + "difficulty": "11164.2887627244", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507940, + "timestamp": 1792145932, + "difficulty": "11161.3693506399", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507939, + "timestamp": 1792145919, + "difficulty": "11188.5734895363", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507938, + "timestamp": 1792145906, + "difficulty": "11176.436111552", + "transaction_count": 3, + "total_fees": "40000" + }, + { + "height": 4507937, + "timestamp": 1792145882, + "difficulty": "11164.2887627244", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507936, + "timestamp": 1792145871, + "difficulty": "11139.672210403", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507935, + "timestamp": 1792145857, + "difficulty": "11184.4034131635", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507934, + "timestamp": 1792145842, + "difficulty": "11189.9289784468", + "transaction_count": 4, + "total_fees": "105000" + }, + { + "height": 4507933, + "timestamp": 1792145799, + "difficulty": "11235.0508253558", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507932, + "timestamp": 1792145797, + "difficulty": "11227.1029393986", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507931, + "timestamp": 1792145796, + "difficulty": "11210.7616610487", + "transaction_count": 1, + "total_fees": "0" + }, + { + "height": 4507930, + "timestamp": 1792145777, + "difficulty": "11225.1956528415", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507929, + "timestamp": 1792145721, + "difficulty": "11301.4161217091", + "transaction_count": 2, + "total_fees": "20000" + }, + { + "height": 4507928, + "timestamp": 1792145670, + "difficulty": "11275.3962446929", + "transaction_count": 4, + "total_fees": "55000" + }, + { + "height": 4507927, + "timestamp": 1792145580, + "difficulty": "11293.9086374572", + "transaction_count": 1, + "total_fees": "0" + } + ] +} diff --git a/server/tests/miner-behavior.test.js b/server/tests/miner-behavior.test.js new file mode 100644 index 00000000..edf21ec4 --- /dev/null +++ b/server/tests/miner-behavior.test.js @@ -0,0 +1,115 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { summarize } = require('../api/lib/miner-behavior'); +const { processDates, selectSnapshotDates } = require('../jobs/snapshot-mining-behavior'); + +test('bounded UTC days and explicit legacy backfill retain their distinct selection', () => { + const now=new Date('2026-10-16T00:01:00Z'); + assert.equal(selectSnapshotDates(['--days=30'],now).length,30); + assert.deepEqual(selectSnapshotDates(['--backfill'],now,'2026-10-14'),['2026-10-14','2026-10-15']); + assert.deepEqual(selectSnapshotDates(['--backfill'],now,null),[]); + assert.throws(()=>selectSnapshotDates(['--backfill','--days=30'],now,'2026-10-14')); +}); + +test('empty snapshots stay unavailable rather than claiming zero earnings or active computation', () => { + const data = summarize('30d', []); + assert.equal(data.status, 'unavailable'); + assert.equal(data.summary, null); + assert.equal(data.coverage, null); + assert.doesNotMatch(data.message, /comput|snapshot job/i); +}); + +test('cohort totals preserve exact zatoshis and disclose tracked coverage and oldest refresh', () => { + const data = summarize('30d', [ + { date:'2026-10-14',pool_name:'Other',earned_zat:'9007199254740993',spent_zat:'1',held_zat:'9007199254740992',blocks_mined:2,outputs_total:3,updated_at:'2026-10-16T10:00:00Z' }, + { date:'2026-10-15',pool_name:'Other',earned_zat:'7',spent_zat:'2',held_zat:'5',blocks_mined:1,outputs_total:1,updated_at:'2026-10-16T11:00:00Z' }, + ], Date.parse('2026-10-16T12:00:00Z')); + assert.equal(data.summary.totalEarnedZat, '9007199254741000'); + assert.equal(data.summary.totalSpentZat, '3'); + assert.equal(data.summary.totalHeldZat, '9007199254740997'); + assert.deepEqual(data.coverage, { scope:'attributed_payout_outputs', spendingBasis:'current_indexed_canonical_spends', freshness:'current', staleAfterSeconds:172800, firstDate:'2026-10-14',lastDate:'2026-10-15',days:2,trackedBlocks:3,trackedOutputs:4,refreshedAt:'2026-10-16T10:00:00.000Z' }); + assert.throws(() => summarize('30d', [{date:'2026-10-15',earned_zat:'1',spent_zat:'2',held_zat:'0'}])); +}); + +async function api(query, fn) { + const express = require('express'), app = express(); + app.locals.pool = { query }; app.locals.redisClient = null; + app.use(require('../api/routes/mining')); + const server = await new Promise(resolve => { const s=app.listen(0,'127.0.0.1',()=>resolve(s)); }); + try { await fn(`http://127.0.0.1:${server.address().port}`); } + finally { server.closeAllConnections(); await new Promise(resolve=>server.close(resolve)); } +} + +test('API distinguishes missing, empty and failed snapshots without claiming a running job', async () => { + for (const exists of [false, true]) { + await api(async sql => ({ rows: sql.includes('information_schema') ? [{exists}] : [] }), async base => { + const response = await fetch(base+'/api/mining/miner-behavior?period=30d'); + const data = await response.json(); + assert.equal(response.status, 200); assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.equal(data.summary, null); assert.deepEqual(data.series, []); + assert.equal(data.reason, exists ? 'no_snapshots_in_period' : 'snapshots_not_configured'); + assert.doesNotMatch(data.message, /comput|snapshot job/i); + }); + } + await api(async () => { throw new Error('private database detail'); }, async base => { + const response=await fetch(base+'/api/mining/miner-behavior?period=30d'); + assert.equal(response.status,503); assert.equal(response.headers.get('cache-control'),'no-store'); + assert.doesNotMatch(JSON.stringify(await response.json()),/private database detail/); + }); +}); + +test('testnet snapshot refuses a different database or missing canonical coinbase before publication', async () => { + for (const database of ['zcash_crosslink', 'zcash_explorer_testnet']) { + const writes=[]; + const writer={query:async sql => { writes.push(sql); return {rows:[{database,recovery:false,genesis:'05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38'}]}; }}; + const reads=[]; + const reader={query:async sql => { reads.push(sql); if(sql.includes('ORDER BY height'))return {rows:[{height:2,hash:'tip',timestamp:1792108800}]};if(sql.includes('incomplete_coinbases'))return {rows:[{blocks:1,incomplete_coinbases:1}]};return {rows:[]}; },release(){}}; + await assert.rejects(processDates(writer,{connect:async()=>reader},['2026-10-14'],{network:'testnet',pauseMs:0}), database==='zcash_crosslink' ? /database\/network mismatch/ : /Incomplete canonical coinbase/); + assert.ok(!writes.some(sql=>sql.includes('DELETE')||sql.includes('INSERT'))); + if(database==='zcash_explorer_testnet')assert.ok(reads.includes('ROLLBACK')); + else assert.equal(reads.length,0); + } +}); + +test('one unavailable day does not starve other testnet reward dates', async () => { + const inserted=[]; + const writer={query:async(sql,args)=>{ + if(sql.includes('pg_is_in_recovery'))return {rows:[{database:'zcash_explorer_testnet',recovery:false,genesis:'05a60a92d99d85997cce3b87616c089f6124d7342af37106edc76126334a2c38'}]}; + if(sql.includes('INSERT INTO mining_behavior_daily'))inserted.push(args[0]); + return {rowCount:1,rows:[]}; + }}; + const reader={query:async(sql,args)=>{ + if(sql.includes('ORDER BY height'))return {rows:[{height:2,hash:'tip',timestamp:1792108800}]}; + if(sql.includes('incomplete_coinbases'))return {rows:[{blocks:1,incomplete_coinbases:args[0]===Date.parse('2026-10-13T00:00:00Z')/1000?1:0}]}; + if(sql.includes('output_count'))return {rows:[{miner_address:'test-miner',blocks:1,output_count:1,total_earned:'7',total_spent:'0',spent_count:0}]}; + return {rows:[]}; + },release(){}}; + const result=await processDates(writer,{connect:async()=>reader},['2026-10-13','2026-10-14'],{network:'testnet',pauseMs:0,continueOnError:true}); + assert.deepEqual(result,{processed:1,failedDates:['2026-10-13']}); + assert.deepEqual(inserted,['2026-10-14']); +}); + +const databaseUrl=process.env.TEST_ANALYTICS_DATABASE_URL||process.env.TEST_UTXO_DATABASE_URL; +test('native PostgreSQL snapshot replaces one cohort atomically and rolls back failed publication', {skip:!databaseUrl}, async () => { + const {Client}=require('pg'), client=new Client({connectionString:databaseUrl}); await client.connect(); + try { + await client.query(`CREATE TEMP TABLE blocks(height bigint PRIMARY KEY,hash text,timestamp bigint,miner_address text); + CREATE TEMP TABLE transactions(txid text PRIMARY KEY,block_height bigint,block_hash text,is_coinbase boolean); + CREATE TEMP TABLE transaction_outputs(txid text,vout_index int,value bigint,address text); + CREATE TEMP TABLE transaction_inputs(txid text,prev_txid text,prev_vout int); + CREATE TEMP TABLE mining_behavior_daily(date date,pool_name text,miner_address text,earned_zat bigint,spent_zat bigint,held_zat bigint,blocks_mined int,outputs_spent int,outputs_total int,updated_at timestamptz DEFAULT now()); + INSERT INTO blocks VALUES(0,'00040fe8ec8471911baa1db1266ea15dd06b4a8a5c453883c000b031973dce08',0,NULL),(1,'canonical',1791936001,'test-miner'),(2,'tip',1792108800,NULL); + INSERT INTO transactions VALUES('coinbase',1,'canonical',true),('orphan',1,'orphan-hash',true),('spend',2,'tip',false),('orphan-spend',1,'orphan-hash',false); + INSERT INTO transaction_outputs VALUES('coinbase',0,9007199254740993,'test-miner'),('coinbase',1,7,'test-miner'),('orphan',0,42,'test-miner'); + INSERT INTO transaction_inputs VALUES('spend','coinbase',1),('spend','coinbase',1),('orphan-spend','coinbase',0);`); + const readers={connect:async()=>({query:client.query.bind(client),release(){}})}; + await processDates(client,readers,['2026-10-14'],{pauseMs:0}); + const before=(await client.query('SELECT * FROM mining_behavior_daily')).rows; + assert.equal(before.length,1); assert.equal(before[0].earned_zat,'9007199254741000'); + assert.equal(before[0].spent_zat,'7');assert.equal(before[0].held_zat,'9007199254740993'); + const failing={query:async(sql,args)=>{if(sql.includes('INSERT INTO mining_behavior_daily'))throw new Error('Injected write failure');return client.query(sql,args);}}; + await assert.rejects(processDates(failing,readers,['2026-10-14'],{pauseMs:0}),/Injected write failure/); + assert.deepEqual((await client.query('SELECT * FROM mining_behavior_daily')).rows,before); + } finally {await client.end();} +}); diff --git a/server/tests/mining-metrics-ui.test.js b/server/tests/mining-metrics-ui.test.js new file mode 100644 index 00000000..bfb48453 --- /dev/null +++ b/server/tests/mining-metrics-ui.test.js @@ -0,0 +1,110 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const { createRequire } = require('node:module'); +const ts = require('typescript'); + +function render(data, { active = 'solrate', loading = false, error = null, currency = 'TAZ' } = {}) { + const exports = {}; + const jsx = (type, props) => ({ type, props }); + vm.runInNewContext(ts.transpileModule(fs.readFileSync('components/network/MiningMetricsChart.tsx', 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, jsx: ts.JsxEmit.ReactJSX }, + }).outputText, { exports, require(name) { + if (name === 'react/jsx-runtime') return { jsx, jsxs: jsx }; + if (name === 'react') return { useState: () => [active, () => {}] }; + if (name === 'recharts') return Object.fromEntries(['LineChart', 'Line', 'XAxis', 'YAxis', 'CartesianGrid', 'ResponsiveContainer'].map(key => [key, key])); + if (name === '@/components/charts/ChartTooltip') return { ChartTooltip: 'Tooltip' }; + if (name === '@/contexts/ThemeContext') return { useTheme: () => ({ theme: 'dark' }) }; + if (name === '@/lib/chart-theme') return { getChartColors: () => ({ gold: '#d4a44a' }) }; + if (name === '@/lib/format-numbers') return { formatHashrate: String, formatDifficulty: String }; + if (name === '@/hooks/useApiQuery') return { useApiQuery: () => ({ data, loading, error }) }; + if (name === '@/lib/config') return { CURRENCY: currency }; + if (name === './ChartCard') return { ChartCard: 'ChartCard' }; + throw new Error(`Unexpected import ${name}`); + } }); + return exports.MiningMetricsChart(); +} + +function nodes(tree, type) { + if (!tree || typeof tree !== 'object') return []; + if (Array.isArray(tree)) return tree.flatMap(item => nodes(item, type)); + return [...(tree.type === type ? [tree] : []), ...nodes(tree.props?.children, type)]; +} + +function textContent(tree) { + if (tree == null || typeof tree === 'boolean') return ''; + if (Array.isArray(tree)) return tree.map(textContent).join(''); + if (typeof tree === 'object') return textContent(tree.props?.children); + return String(tree); +} + +const sparse = { + solrateMethod: 'difficulty-work-over-elapsed-v1', + latest: { solrate: null, txFees: 0.001 }, + points: [null, 1000, null].map((solrate, i) => ({ height: 1000 + i, solrate, txFees: 0.001 })), +}; + +test('known isolated mining points render real dots while unknown points remain gaps', () => { + const tree = render(sparse); + const chart = nodes(tree, 'LineChart')[0]; + const line = nodes(tree, 'Line')[0]; + assert.equal(line.props.type, 'linear'); + assert.equal(line.props.connectNulls, false); + assert.deepEqual(Array.from(chart.props.data, p => p.solrate), [null, 1000, null]); + + // Use Recharts' real coordinate calculator and SVG primitives. A single + // known value between nulls must have area even though no line can join it. + const chartRequire = createRequire(require.resolve('recharts/package.json')); + const React = chartRequire('react'); + const { renderToStaticMarkup } = chartRequire('react-dom/server'); + const { computeLinePoints } = chartRequire('./lib/cartesian/Line'); + const { Dot, Curve } = chartRequire('.'); + const points = computeLinePoints({ + layout: 'horizontal', dataKey: line.props.dataKey, bandSize: 0, displayedData: chart.props.data, + xAxis: { type: 'category', allowDuplicatedCategory: true }, + xAxisTicks: [{ coordinate: 0 }, { coordinate: 100 }, { coordinate: 200 }], yAxisTicks: [], + yAxis: { type: 'number', scale: { map: value => 250 - Number(value) / 10 } }, + }); + const svg = renderToStaticMarkup(React.createElement('svg', {}, [ + React.createElement(Curve, { key: 'curve', points, type: line.props.type, connectNulls: line.props.connectNulls }), + ...points.map((point, i) => React.createElement(Dot, { + ...line.props.dot, key: i, cx: point.x, cy: point.y ?? undefined, + })), + ])); + assert.equal((svg.match(/]+>/)[0]; + for (const attribute of ['cx="100"', 'cy="150"', 'r="2"']) assert.ok(circle.includes(attribute)); + assert.doesNotMatch(svg, /L/, 'unknown points are not bridged by a line'); +}); + +test('empty, incomplete, failed and loading charts explain why no series is displayed', () => { + for (const [data, options, expected] of [ + [null, { loading: true }, /Loading mining metrics/], + [null, { error: 'HTTP 503' }, /temporarily unavailable/], + [{ points: [] }, {}, /No complete 20-block window/], + [{ points: [{ height: 1, solrate: null }] }, {}, /No complete 20-block window/], + ]) { + const tree = render(data, options); + assert.equal(nodes(tree, 'LineChart').length, 0); + assert.match(textContent(tree), expected); + assert.ok(nodes(tree, 'div').some(node => node.props.role === 'status')); + } +}); + +test('refresh errors retain valid points with an explicit stale-data message', () => { + const tree = render(sparse, { error: 'Request timed out' }); + assert.equal(nodes(tree, 'LineChart').length, 1); + assert.match(textContent(tree), /Showing the last received mining metrics/); +}); + +test('method copy remains accurate during rolling API upgrades and testnet fees use TAZ', () => { + assert.match(textContent(render(sparse)), /total difficulty and elapsed header time/); + assert.doesNotMatch(textContent(render({ ...sparse, solrateMethod: undefined })), /total difficulty and elapsed header time/); + for (const currency of ['TAZ', 'ZEC']) { + const tree = render(sparse, { currency, active: 'txFees' }); + assert.match(textContent(tree), new RegExp(`0.001000 ${currency}`)); + const tooltip = nodes(tree, 'Tooltip')[0]; + assert.equal(tooltip.props.formatter(0.001)[0], `0.001000 ${currency}`); + } +}); diff --git a/server/tests/mining-metrics.test.js b/server/tests/mining-metrics.test.js index a99c4cfb..18bd1b83 100644 --- a/server/tests/mining-metrics.test.js +++ b/server/tests/mining-metrics.test.js @@ -1,5 +1,6 @@ const test = require('node:test'); const assert = require('node:assert/strict'); +const fs = require('node:fs'); const { registerNetworkAnalyticsRoutes } = require('../api/routes/network-analytics'); const blocks = (length = 200) => Array.from({ length }, (_, i) => ({ @@ -47,17 +48,82 @@ test('missing heights preserve timing gaps and recover once the gap leaves the w assert.equal(body.points.find(p => p.height === 971).blockTime, 75); }); -test('nonpositive timestamp deltas remain unavailable for solrate, never filled with zero', async () => { +test('equal or backwards individual timestamps retain positive complete-window solrate', async () => { for (const delta of [0, -5]) { const rows = blocks(); rows[0].timestamp = rows[1].timestamp + delta; const { body } = await metrics(rows); - assert.equal(body.latest.solrate, null); - assert.equal(body.points.at(-1).solrate, null); + assert.equal(body.solrateMethod, 'difficulty-work-over-elapsed-v1'); + assert.ok(Math.abs(body.latest.solrate - 20 * 100 * 8192 / (19 * 75 + delta)) < 1e-8); + assert.equal(body.points.at(-1).solrate, body.latest.solrate); assert.equal(body.latest.blockTime, (19 * 75 + delta) / 20); } }); +test('solrate divides all contributing work by elapsed time rather than averaging instantaneous rates', async () => { + const rows = blocks(6); + const intervals = [5, 10, 20, 40, 80]; + const difficulties = [2, 3, 5, 7, 11]; + const chronological = rows.slice().reverse(); + chronological[0].difficulty = 1_000_000; + for (let i = 1; i < chronological.length; i++) { + chronological[i].timestamp = chronological[i - 1].timestamp + intervals[i - 1]; + chronological[i].difficulty = difficulties[i - 1]; + } + const { body } = await metrics(rows, { window: 5 }); + assert.equal(body.latest.solrate, difficulties.reduce((a, b) => a + b, 0) * 8192 / 155); + assert.equal(body.latest.blockTime, 31); + assert.equal(body.latest.difficulty, 28 / 5, 'predecessor difficulty is outside the numerator'); +}); + +test('nonpositive complete-window elapsed time remains unavailable for solrate', async () => { + for (const elapsed of [0, -5]) { + const rows = blocks(21); + rows[0].timestamp = rows[20].timestamp + elapsed; + const { body } = await metrics(rows); + assert.equal(body.latest.solrate, null); + assert.equal(body.latest.blockTime, elapsed / 20, 'signed block-time diagnostic is unchanged'); + } +}); + +test('unknown timestamps and invalid difficulty never become zero work or a complete timing window', async () => { + for (const invalid of [null, undefined, '', ' ', 'bad', '100bad', NaN, Infinity, true]) { + const rows = blocks(21); + rows[10].timestamp = invalid; + let { body } = await metrics(rows); + assert.equal(body.latest.solrate, null); + assert.equal(body.latest.blockTime, null); + rows[10].timestamp = blocks(21)[10].timestamp; + rows[10].difficulty = invalid; + ({ body } = await metrics(rows)); + assert.equal(body.latest.solrate, null); + assert.equal(body.latest.difficulty, null); + assert.equal(body.latest.blockTime, 75); + } + for (const difficulty of [0, -1]) { + const rows = blocks(21); + rows[10].difficulty = difficulty; + assert.equal((await metrics(rows)).body.latest.solrate, null); + } +}); + +test('captured testnet equal-timestamp blocks have complete positive windows and no artificial solrate gap', async () => { + const fixture = JSON.parse(fs.readFileSync('server/tests/fixtures/mining-metrics-testnet-header-window.json', 'utf8')); + const { body } = await metrics(fixture.rows, { window: 20, limit: fixture.rows.length - 20 }); + const chronological = fixture.rows.slice().reverse(); + assert.equal(chronological.find(r => r.height === 4507966).timestamp, 1792146633); + assert.equal(chronological.find(r => r.height === 4507967).timestamp, 1792146633); + assert.equal(body.points.length, fixture.rows.length - 20); + assert.ok(body.points.every(point => point.solrate > 0)); + for (const point of body.points) { + const index = chronological.findIndex(r => r.height === point.height); + const span = chronological[index].timestamp - chronological[index - 20].timestamp; + const difficulty = chronological.slice(index - 19, index + 1).reduce((sum, row) => sum + Number(row.difficulty), 0); + assert.ok(Math.abs(point.solrate - difficulty * 8192 / span) < 1e-7); + } + assert.ok(body.points.filter(p => p.height >= 4507967 && p.height <= 4507986).every(p => p.solrate > 0)); +}); + test('insufficient history and empty data do not invent full-window averages', async () => { for (const rows of [[], blocks(4)]) { const { body } = await metrics(rows); diff --git a/server/tests/network-accounting-period.test.js b/server/tests/network-accounting-period.test.js index f82b3f05..3377b3f1 100644 --- a/server/tests/network-accounting-period.test.js +++ b/server/tests/network-accounting-period.test.js @@ -3,20 +3,23 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const express = require('express'); const { registerNetworkReadinessRoutes } = require('../api/routes/network-readiness'); -const { periodPayload, PERIOD_SQL } = require('../api/lib/accounting-period'); +const { periodPayload, PERIOD_SQL, accountingPeriodSql } = require('../api/lib/accounting-period'); +const { accountingHistorySql, historyPoints } = require('../api/lib/accounting-history'); const info = { chain: 'test', blocks: 4465027, bestblockhash: 'tip', upgrades: { 'blossom': { name: 'Blossom', activationheight: 584000 }, '77190ad9': { name: 'NU7', activationheight: 4465026, status: 'active' }, } }; -const totals = { blocks: 2, feeBlocks: 2, missingBlocks: 0, nsmSamples: 2, +const totals = { blocks: 2, feeBlocks: 2, receiptBlocks: 2, missingBlocks: 0, nsmSamples: 2, feesPaidZat: '6', feesToNsmZat: '2', minerFeeAllocationZat: '4', firstHeight: 4465026, lastHeight: 4465027 }; const row = { totals, selected: totals, bucket_seconds: '300', points: [], baseline: { height: 4465025, hash: 'baseline', balance: '9007199254740993' }, tip: { height: 4465027, hash: 'tip', balance: '9007199254740995' } }; -async function withRoute({ chainInfo = info, result = row, canonical = true } = {}, fn) { +async function withRoute({ chainInfo = info, result = row, canonical = true, allocationsAvailable = false } = {}, fn) { const queries = []; const rpc = []; const app = express(); const router = express.Router(); - app.locals.pool = { query: async (sql, params) => { queries.push({ sql, params }); return { rows: [result] }; } }; + app.locals.pool = { query: async (sql, params) => { + if (sql.includes('to_regclass')) return { rows: [{ available: allocationsAvailable }] }; + queries.push({ sql, params }); return { rows: [result] }; } }; app.locals.callZebraRPC = async (method, params) => { rpc.push({method, params}); if (method === 'getblockchaininfo') return chainInfo; return (typeof canonical === 'function' ? canonical() : canonical) ? params[0] === 4465025 ? 'baseline' : 'tip' : 'orphan'; @@ -33,6 +36,7 @@ test('period mode exposes exact values, full activation bounds and separate rese assert.equal(response.status, 200); const data = await response.json(); assert.equal(data.period, period); assert.equal(data.totals.feesToNsmZat, '2'); + assert.equal(data.totals.receiptBlocks, 2); assert.equal(data.selected.receiptBlocks, 2); assert.equal(data.reserve.growthSinceNu7Zat, '2'); assert.equal(data.reserve.baselineZat, '9007199254740993'); } assert.equal(queries.length, 4); @@ -41,6 +45,13 @@ test('period mode exposes exact values, full activation bounds and separate rese assert.equal(rpc.filter(r => r.method === 'getblockhash').length, 8); }); }); +test('period uses historical allocations only when the independent cache exists', async () => { + for (const allocationsAvailable of [false, true]) await withRoute({ allocationsAvailable }, async (base, queries) => { + const response = await fetch(`${base}/api/network/accounting/history?period=all`); + assert.equal(response.status, 200); + assert.equal(queries[0].sql.includes('LEFT JOIN public.block_subsidy_allocations'), allocationsAvailable); + }); +}); test('ambiguous pagination/period combinations and unknown periods are rejected before querying', async () => { await withRoute({}, async (base, queries) => { for (const query of ['period=30', 'period=1d&limit=120', 'period=all&before=4465027', 'period=all&period=1d']) { @@ -88,14 +99,19 @@ test('PostgreSQL buckets retain exact rounding, selected boundaries, gaps and ca await client.connect(); try { await client.query(`BEGIN; - CREATE TEMP TABLE blocks(height bigint,hash text,timestamp bigint,transaction_count int); + CREATE TEMP TABLE blocks(height bigint,hash text,timestamp bigint,transaction_count int,size int); CREATE TEMP TABLE transactions(txid text,block_height bigint,block_hash text,is_coinbase boolean,fee bigint, - total_output bigint,value_balance_sapling bigint,value_balance_orchard bigint,value_balance_ironwood bigint); - CREATE TEMP TABLE node_accounting_observations(height bigint,hash text,chain text,nsm_balance_zat bigint,subsidy jsonb); - INSERT INTO blocks VALUES(99,'base',599,1),(100,'a',600,2),(101,'b',610,2),(102,'c',920,1); - INSERT INTO transactions VALUES('ca',100,'a',true,0,100,0,0,0),('ta',100,'a',false,3,0,0,0,0), + total_output bigint,value_balance_sapling bigint,value_balance_orchard bigint,value_balance_ironwood bigint, + version int DEFAULT 6,sapling_spend_count int DEFAULT 0,sapling_output_count int DEFAULT 0, + orchard_actions int DEFAULT 0,ironwood_actions int DEFAULT 0); + CREATE TEMP TABLE node_accounting_observations(height bigint,hash text,chain text,nsm_balance_zat bigint,subsidy jsonb, + observed_at timestamptz,circulating_supply_zat bigint,pool_balances_zat jsonb); + CREATE TEMP TABLE block_subsidy_allocations(height bigint,hash text,chain text,founders_zat bigint, + funding_streams_zat bigint,miner_zat bigint,source text); + INSERT INTO blocks(height,hash,timestamp,transaction_count) VALUES(99,'base',599,1),(100,'a',600,2),(101,'b',610,2),(102,'c',920,1); + INSERT INTO transactions(txid,block_height,block_hash,is_coinbase,fee,total_output,value_balance_sapling,value_balance_orchard,value_balance_ironwood) VALUES('ca',100,'a',true,0,100,0,0,0),('ta',100,'a',false,3,0,0,0,0), ('cb',101,'b',true,0,100,0,0,0),('tb',101,'b',false,3,0,0,0,0),('cc',102,'c',true,0,100,0,0,0); - INSERT INTO node_accounting_observations VALUES(99,'base','test',9007199254740993,'{"founders":0,"fundingstreamstotal":0}'), + INSERT INTO node_accounting_observations(height,hash,chain,nsm_balance_zat,subsidy) VALUES(99,'base','test',9007199254740993,'{"founders":0,"fundingstreamstotal":0}'), (100,'a','test',9007199254740994,'{"founders":0,"fundingstreamstotal":0}'), (101,'b','test',9007199254740995,'{"founders":0,"fundingstreamstotal":0}'), (102,'orphan','test',1,'{"founders":0,"fundingstreamstotal":0}');`); @@ -103,8 +119,54 @@ test('PostgreSQL buckets retain exact rounding, selected boundaries, gaps and ca assert.equal(result.totals.feesPaidZat,'6'); assert.equal(result.totals.feesToNsmZat,'2'); assert.equal(result.points[0].minerReceiptsZat,'200'); assert.equal(result.points[0].nsmBalanceZat,'9007199254740995'); + assert.equal(result.points[0].nsmSampleHeight,101); + assert.equal(result.points[0].nsmSampleHash,'b'); + assert.equal(result.points[0].nsmSampleTimestamp,610); assert.equal(result.points[1].nsmBalanceZat,null); assert.equal(result.tip.balance,null); + for (const key of ['nsmSampleHeight','nsmSampleHash','nsmSampleTimestamp']) assert.equal(result.points[1][key],null); + // A missed bucket-end observation does not erase an earlier real sample. + // The exact latest reserve and selected-period boundaries remain strict. + await client.query("UPDATE node_accounting_observations SET nsm_balance_zat=NULL WHERE hash='b'"); + const lastRecorded = (await client.query(PERIOD_SQL, [102,100,'test',null])).rows[0]; + assert.equal(lastRecorded.points[0].nsmBalanceZat,'9007199254740994'); + assert.equal(lastRecorded.points[0].nsmSampleHeight,100); + assert.equal(lastRecorded.points[0].nsmSampleHash,'a'); + assert.equal(lastRecorded.points[0].nsmSampleTimestamp,600); + assert.equal(lastRecorded.tip.balance,null); + const bounded = (await client.query(PERIOD_SQL, [102,100,'test',605])).rows[0]; + assert.equal(bounded.points[0].nsmBalanceZat,null, 'never carry a sample from before the selected period'); + for (const key of ['nsmSampleHeight','nsmSampleHash','nsmSampleTimestamp']) assert.equal(bounded.points[0][key],null); + await client.query("UPDATE node_accounting_observations SET nsm_balance_zat=9007199254740995 WHERE hash='b'"); assert.equal(result.points[1].cumulativeRemovalZat,'2'); + assert.equal(result.totals.receiptBlocks, 2); assert.equal(result.selected.receiptBlocks, 2); + assert.equal(result.points[1].receiptBlocks, 0); assert.equal(result.points[1].minerReceiptsZat, null); + // Only the fixture namespace changes; production SQL joins the public cache. + const cachedSql = accountingPeriodSql(true).replaceAll('public.block_subsidy_allocations', 'pg_temp.block_subsidy_allocations'); + await client.query("INSERT INTO block_subsidy_allocations VALUES(102,'c','main',0,0,101,'getblocksubsidy');"); + result = (await client.query(cachedSql, [102,100,'test',null])).rows[0]; + assert.equal(result.points[1].minerReceiptsZat, null, 'wrong-network cache is excluded'); + await client.query("UPDATE block_subsidy_allocations SET chain='test',hash='orphan'"); + result = (await client.query(cachedSql, [102,100,'test',null])).rows[0]; + assert.equal(result.points[1].minerReceiptsZat, null, 'orphan cache is excluded'); + await client.query("UPDATE block_subsidy_allocations SET hash='c'"); + result = (await client.query(cachedSql, [102,100,'test',null])).rows[0]; + assert.equal(result.totals.receiptBlocks, 3); assert.equal(result.selected.receiptBlocks, 3); + assert.equal(result.points[1].receiptBlocks, 1); assert.equal(result.points[1].minerReceiptsZat, '100', 'actual receipts preserve one-zatoshi underclaim'); + assert.equal(result.points[1].nsmBalanceZat, null); assert.equal(result.tip.balance, null); + await client.query(`INSERT INTO block_subsidy_allocations VALUES(100,'a','test',-1,0,101,'getblocksubsidy'); + UPDATE node_accounting_observations SET subsidy='{"founders":"malformed","fundingstreamstotal":0}' WHERE hash='orphan';`); + result = (await client.query(cachedSql, [102,100,'test',null])).rows[0]; + assert.equal(result.points[0].minerReceiptsZat, '200', 'invalid cache falls back to valid samples'); + // Legacy reads share the same allocation authority without creating NSM samples. + const historySql = accountingHistorySql(true).replaceAll('public.block_subsidy_allocations', 'pg_temp.block_subsidy_allocations'); + const history = historyPoints((await client.query(historySql, [102,null,3,'test'])).rows, {nu7Height:100}, 'test'); + assert.equal(history[2].minerReceiptsZat, '100'); assert.equal(history[2].minerSubsidyZat, '101'); + assert.equal(history[2].nsmBalanceZat, null); assert.equal(history[2].observedAt, null); + // Non-numeric sample JSON is unavailable rather than a numeric-cast crash. + await client.query(`UPDATE node_accounting_observations SET subsidy='{"founders":"malformed","fundingstreamstotal":0}' WHERE hash='b';`); + result = (await client.query(PERIOD_SQL, [102,100,'test',null])).rows[0]; + assert.equal(result.points[0].minerReceiptsZat, null); assert.equal(result.points[0].receiptBlocks, 1); + await client.query(`UPDATE node_accounting_observations SET subsidy='{"founders":0,"fundingstreamstotal":0}' WHERE hash='b';`); result = (await client.query(PERIOD_SQL, [102,100,'test',605])).rows[0]; assert.equal(result.selected.feesPaidZat,'3'); assert.equal(result.points[0].feesToNsmZat,'1'); assert.equal(result.points[0].cumulativeRemovalZat,'2'); @@ -116,10 +178,11 @@ test('PostgreSQL buckets retain exact rounding, selected boundaries, gaps and ca result = (await client.query(PERIOD_SQL, [102,100,'test',null])).rows[0]; assert.equal(result.totals.missingBlocks,1); assert.equal(result.totals.feesPaidZat,null); assert.equal(result.points[1].cumulativeRemovalZat,null); + for (const key of ['nsmBalanceZat','nsmSampleHeight','nsmSampleHash','nsmSampleTimestamp']) assert.equal(result.points[1][key],null); await client.query(`TRUNCATE blocks,transactions,node_accounting_observations; - INSERT INTO blocks SELECT h,'block-'||h,100000+h*25,2 FROM generate_series(100,6100) h; - INSERT INTO transactions SELECT 'coinbase-'||h,h,'block-'||h,true,0,100,0,0,0 FROM generate_series(100,6100) h; - INSERT INTO transactions SELECT 'tx-'||h,h,'block-'||h,false,3,0,0,0,0 FROM generate_series(100,6100) h;`); + INSERT INTO blocks(height,hash,timestamp,transaction_count) SELECT h,'block-'||h,100000+h*25,2 FROM generate_series(100,6100) h; + INSERT INTO transactions(txid,block_height,block_hash,is_coinbase,fee,total_output,value_balance_sapling,value_balance_orchard,value_balance_ironwood) SELECT 'coinbase-'||h,h,'block-'||h,true,0,100,0,0,0 FROM generate_series(100,6100) h; + INSERT INTO transactions(txid,block_height,block_hash,is_coinbase,fee,total_output,value_balance_sapling,value_balance_orchard,value_balance_ironwood) SELECT 'tx-'||h,h,'block-'||h,false,3,0,0,0,0 FROM generate_series(100,6100) h;`); result = (await client.query(PERIOD_SQL, [6100,100,'test',null])).rows[0]; assert.equal(result.totals.blocks,6001); assert.equal(result.totals.feesToNsmZat,'6001'); assert.ok(result.points.length <= 241); assert.equal(result.points.at(-1).cumulativeRemovalZat,'6001'); diff --git a/server/tests/network-accounting-visibility.test.js b/server/tests/network-accounting-visibility.test.js index a6e30d22..448d6956 100644 --- a/server/tests/network-accounting-visibility.test.js +++ b/server/tests/network-accounting-visibility.test.js @@ -1,10 +1,11 @@ const test = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); +const { createRequire } = require('node:module'); const vm = require('node:vm'); const ts = require('typescript'); -function render(data, { network = 'mainnet', error = null, historyData = undefined, metric = 'fees', period = '1d' } = {}) { +function render(data, { network = 'mainnet', error = null, historyData = undefined, metric = 'fees', period = '30d' } = {}) { const exports = {}; const calls = []; const jsx = (type, props) => ({ type, props }); @@ -12,13 +13,13 @@ function render(data, { network = 'mainnet', error = null, historyData = undefin compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, jsx: ts.JsxEmit.ReactJSX }, }).outputText, { exports, require(name) { if (name === 'react/jsx-runtime') return { jsx, jsxs: jsx }; - if (name === 'react') return { useState: value => [value === 'fees' ? metric : value === '1d' ? period : value, () => {}] }; + if (name === 'react') return { useState: value => [value === 'fees' ? metric : value === '30d' ? period : value, () => {}] }; if (name === 'recharts') return Object.fromEntries(['ComposedChart','Bar','Line','XAxis','YAxis','ResponsiveContainer','CartesianGrid'].map(key => [key,key])); if (name === 'next/link') return { default: 'a' }; if (name === '@/components/ui/SectionHeader') return { SectionHeader: 'SectionHeader' }; if (name === '@/components/charts/ChartTooltip') return { ChartTooltip: 'Tooltip' }; if (name === '@/contexts/ThemeContext') return { useTheme: () => ({ theme: 'dark' }) }; - if (name === '@/lib/chart-theme') return { getChartColors: () => ({}) }; + if (name === '@/lib/chart-theme') return { getChartColors: () => ({ gold: '#d4a44a' }) }; if (name === '@/lib/config') return { NETWORK: network, CURRENCY: 'ZEC' }; if (name === '@/components/ui/Card') return { Card: 'Card', CardBody: 'CardBody' }; if (name === '@/hooks/useApiQuery') return { useApiQuery(url, params, options) { @@ -79,7 +80,7 @@ function nodes(tree, type) { if (Array.isArray(tree)) return tree.flatMap(item => nodes(item, type)); return [...(tree.type === type ? [tree] : []), ...nodes(tree.props?.children, type)]; } -const periodHistory = { period: '1d', schedule: { network: 'test', nu7Height: 4400000 }, +const periodHistory = { period: '30d', schedule: { network: 'test', nu7Height: 4400000 }, nodeHeight: 4400001, indexedHeight: 4400001, bucketSeconds: 300, totals: { firstHeight:4400000, lastHeight:4400001, feesToNsmZat:'2' }, selected: {blocks:2,feeBlocks:2,nsmSamples:2,missingBlocks:0,feesPaidZat:'6',feesToNsmZat:'2',minerFeeAllocationZat:'4',firstHeight:4400000,lastHeight:4400001,firstTimestamp:600,lastTimestamp:610}, @@ -88,7 +89,7 @@ const periodHistory = { period: '1d', schedule: { network: 'test', nu7Height: 44 test('fee columns contain only the two allocations and the canonical block tag is in the header', () => { const data = {...snapshot(4400001,4400000,'test'),block:{height:4400001}}; const result = render(data,{network:'testnet',historyData:periodHistory}); - assert.equal(result.historyParams.period,'1d'); + assert.equal(result.historyParams.period,'30d'); const bars = nodes(result.tree,'Bar'); assert.deepEqual(bars.map(node=>node.props.dataKey),['feesToNsmZatDisplay','minerFeeAllocationZatDisplay']); assert.ok(bars.every(node=>node.props.stackId==='fees')); @@ -97,11 +98,121 @@ test('fee columns contain only the two allocations and the canonical block tag i }); test('range changes never plot retained data under a new period, and reserve/cumulative use steps', () => { const data = snapshot(4400001,4400000,'test'); - const stale = render(data,{network:'testnet',historyData:periodHistory,period:'30d'}); - assert.equal(stale.historyParams.period,'30d'); assert.equal(nodes(stale.tree,'ComposedChart').length,0); + const stale = render(data,{network:'testnet',historyData:periodHistory,period:'1d'}); + assert.equal(stale.historyParams.period,'1d'); assert.equal(nodes(stale.tree,'ComposedChart').length,0); for(const metric of ['reserve','cumulative']) { const result = render(data,{network:'testnet',historyData:periodHistory,metric}); assert.equal(nodes(result.tree,'Line')[0].props.type,'stepAfter'); assert.equal(nodes(result.tree,'Line')[0].props.connectNulls,false); } }); + +test('reserve points use their recorded sample time, show provenance, and keep empty buckets null', () => { + const historyData = { ...periodHistory, selected: { ...periodHistory.selected, nsmSamples: 1 }, + points: [ + { ...periodHistory.points[0], nsmSampleHeight: 4400000, nsmSampleHash: 'canonical-sample-hash', nsmSampleTimestamp: 605 }, + { ...periodHistory.points[0], timestamp: 900, nsmBalanceZat: null, nsmSampleHeight: null, nsmSampleHash: null, nsmSampleTimestamp: null }, + ] }; + const data = snapshot(4400001,4400000,'test'); + const result = render(data,{network:'testnet',historyData,metric:'reserve'}); + const chart = nodes(result.tree,'ComposedChart')[0]; + assert.deepEqual(Array.from(chart.props.data, point => point.chartTimestamp), [605,900]); + assert.deepEqual(Array.from(chart.props.data, point => point.nsmBalanceZatDisplay), [0.00000102,null]); + assert.equal(nodes(result.tree,'XAxis')[0].props.dataKey,'chartTimestamp'); + const line = nodes(result.tree,'Line')[0]; + assert.equal(line.props.connectNulls,false); + assert.ok(line.props.dot.r > 0, 'isolated real reserve observations must be visible'); + const tooltip = nodes(result.tree,'Tooltip')[0].props; + const label = textContent(tooltip.labelFormatter(605,[{payload:historyData.points[0]}])); + assert.match(label,/Recorded at block 4,400,000/); + assert.match(label,/canonical-sample-hash/); + assert.match(label,/1970-01-01T00:10:05.000Z/); + assert.match(label,/1 blocks before this bucket/); + assert.match(textContent(result.tree),/1 of 2 reported buckets have a reserve observation/); + const fees = render(data,{network:'testnet',historyData}); + assert.deepEqual(Array.from(nodes(fees.tree,'ComposedChart')[0].props.data, point => point.chartTimestamp),[600,900]); +}); + +function textContent(tree) { + if (tree == null || typeof tree === 'boolean') return ''; + if (Array.isArray(tree)) return tree.map(textContent).join(''); + if (typeof tree === 'object') return textContent(tree.props?.children); + return String(tree); +} + +// Complete fees do not imply complete coinbase-receipt observations. The middle +// receipt bucket is isolated between unknown totals, as in the real 7D incident. +const sparseReceiptHistory = { + ...periodHistory, period: '7d', bucketSeconds: 2700, nodeHeight: 4400005, indexedHeight: 4400005, + selected: { ...periodHistory.selected, blocks: 6, feeBlocks: 6, receiptBlocks: 3, nsmSamples: 6, + lastHeight: 4400005, lastTimestamp: 6010 }, + points: [null, '2500000000', null].map((minerReceiptsZat, i) => ({ + ...periodHistory.points[0], timestamp: 600 + i * 2700, + firstHeight: 4400000 + i * 2, lastHeight: 4400001 + i * 2, + blocks: 2, feeBlocks: 2, receiptBlocks: [1, 2, 0][i], nsmSamples: 2, + missingBlocks: 0, minerReceiptsZat, + })), +}; + +test('isolated receipt buckets render real Recharts bars while unknown totals remain absent', () => { + const result = render(snapshot(4400005, 4400000, 'test'), { + network: 'testnet', historyData: sparseReceiptHistory, period: '7d', metric: 'receipts', + }); + const chart = nodes(result.tree, 'ComposedChart')[0]; + const bars = nodes(result.tree, 'Bar'); + assert.equal(bars.length, 1); + assert.equal(nodes(result.tree, 'Line').length, 0); + assert.equal(bars[0].props.dataKey, 'minerReceiptsZatDisplay'); + assert.equal(bars[0].props.stackId, undefined); + assert.deepEqual(Array.from(chart.props.data, point => point.minerReceiptsZatDisplay), [null, 25, null]); + + // Recharts' complete chart needs browser layout effects. Exercise its actual + // bar geometry and SVG renderer here, rather than only checking mocked props. + const chartRequire = createRequire(require.resolve('recharts/package.json')); + const React = chartRequire('react'); + const { renderToStaticMarkup } = chartRequire('react-dom/server'); + const { computeBarRectangles } = chartRequire('./lib/cartesian/Bar'); + const { Rectangle } = chartRequire('.'); + const rectangles = computeBarRectangles({ + layout: 'horizontal', barSettings: { ...bars[0].props, minPointSize: 0, hasCustomShape: false }, + pos: { offset: 0, size: bars[0].props.maxBarSize }, bandSize: 100, + xAxis: { type: 'category' }, + yAxis: { type: 'number', scale: { domain: () => [0, 50], map: value => value == null ? undefined : 250 - Number(value) * 5 } }, + xAxisTicks: chart.props.data.map((_, i) => ({ coordinate: i * 100 })), yAxisTicks: [], + displayedData: chart.props.data, offset: { top: 0, left: 0, width: 300, height: 250 }, dataStartIndex: 0, + }); + assert.equal(rectangles.length, 1, 'unknown receipt buckets must not become zero-filled bars'); + assert.equal(rectangles[0].payload.timestamp, 3300); + assert.ok(rectangles[0].width > 0 && rectangles[0].height > 0, 'isolated known bucket has visible area'); + const svg = renderToStaticMarkup(React.createElement('svg', null, rectangles.map((rectangle, i) => React.createElement(Rectangle, { + key: i, x: rectangle.x, y: rectangle.y, width: rectangle.width, height: rectangle.height, fill: bars[0].props.fill, + })))); + assert.match(svg, /class="recharts-rectangle"/); + assert.match(svg, /fill="#d4a44a"/); + assert.equal((svg.match(/ { + const data = snapshot(4400005, 4400000, 'test'); + const options = { network: 'testnet', historyData: sparseReceiptHistory, period: '7d', metric: 'receipts' }; + const text = textContent(render(data, options).tree); + assert.match(text, /3 of 6 blocks have miner receipts/); + assert.match(text, /1 of 3 reported buckets have complete receipts/); + assert.match(text, /Incomplete receipt buckets remain gaps/); + assert.doesNotMatch(text, /blocks have complete fees/); + + for (const receiptBlocks of [undefined, null, -1, 7, '3']) { + const historyData = { ...sparseReceiptHistory, selected: { ...sparseReceiptHistory.selected, receiptBlocks } }; + const fallback = textContent(render(data, { ...options, historyData }).tree); + assert.match(fallback, /Block receipt coverage unavailable/); + assert.match(fallback, /1 of 3 reported buckets have complete receipts/); + } + + const historyData = { ...sparseReceiptHistory, + points: sparseReceiptHistory.points.map(point => ({ ...point, minerReceiptsZat: null })) }; + const empty = render(data, { ...options, historyData }); + assert.equal(nodes(empty.tree, 'ComposedChart').length, 0); + assert.match(textContent(empty.tree), /No complete observations for this metric in this period/); + assert.match(textContent(empty.tree), /0 of 3 reported buckets have complete receipts/); +}); diff --git a/server/tests/network-readiness.test.js b/server/tests/network-readiness.test.js index 10d45726..e0d32ae2 100644 --- a/server/tests/network-readiness.test.js +++ b/server/tests/network-readiness.test.js @@ -43,6 +43,7 @@ test('history preserves exact reserve values, misses and decreases without infer assert.equal(points[1].nsmNetChangeZat, '-1'); for (const i of [0, 2, 3, 4]) assert.equal(points[i].nsmNetChangeZat, null); assert.equal(points[0].minerReceiptsZat, '80000004'); + assert.equal(points[0].receiptSource, 'indexed-coinbase-and-retained-tip-subsidy'); assert.equal(points[0].reissuanceZat, null); assert.equal(historyPoints([base], schedule, 'main')[0].minerReceiptsZat, null); }); @@ -53,3 +54,21 @@ test('block rounding occurs before historical aggregation, including across acti assert.equal(points[0].nsmBalanceZat, null); assert.match(points[0].accountingUnavailableReason, /No matching/); }); + +test('canonical historical allocations recover actual receipts while NSM samples remain missing', () => { + const cached = { ...row, coinbase_value: '100000001', + allocation_founders_zat: '0', allocation_funding_streams_zat: '20000000', allocation_miner_zat: '80000000' }; + const [point] = historyPoints([cached], schedule, 'test'); + assert.equal(point.minerReceiptsZat, '80000001'); + assert.equal(point.receiptSource, 'indexed-coinbase-and-historical-node-subsidy'); + assert.equal(point.minerSubsidyZat, '80000000'); + assert.equal(point.minerFeeAllocationZat, '2'); + assert.notEqual(BigInt(point.minerReceiptsZat), BigInt(point.minerSubsidyZat) + BigInt(point.minerFeeAllocationZat), 'actual coinbase underclaim is preserved'); + assert.equal(point.nsmBalanceZat, null); assert.equal(point.nsmNetChangeZat, null); + assert.equal(point.observedAt, null); assert.match(point.accountingUnavailableReason, /No matching/); + assert.equal(historyPoints([{ ...cached, allocation_miner_zat: null }], schedule, 'test')[0].minerReceiptsZat, '80000001'); + assert.equal(historyPoints([{ ...cached, allocation_founders_zat: '-1' }], schedule, 'test')[0].minerReceiptsZat, null); + assert.equal(historyPoints([{ ...cached, allocation_founders_zat: '-1' }], schedule, 'test')[0].receiptSource, null); + const sampled = { ...cached, allocation_founders_zat: '-1', chain: 'test', subsidy: { miner: 0.8, founders: 0, fundingstreamstotal: 0.2 } }; + assert.equal(historyPoints([sampled], schedule, 'test')[0].minerReceiptsZat, '80000001', 'invalid cache falls back to the matching real sample'); +}); diff --git a/server/tests/subsidy-allocation-health.test.js b/server/tests/subsidy-allocation-health.test.js new file mode 100644 index 00000000..5e448841 --- /dev/null +++ b/server/tests/subsidy-allocation-health.test.js @@ -0,0 +1,133 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const { checkAllocationHealth } = require('../lib/subsidy-allocation-health'); +const { initialState } = require('../lib/subsidy-allocation-state'); + +const now = Date.parse('2026-10-16T03:00:00Z'); +const genesis = '0'.repeat(64); +const anchorHash = 'd'.repeat(64); +const activation = 4465026; +const state = { ...initialState('testnet'), status: 'complete', lastAttempt: new Date(now - 10_000).toISOString(), + lastFinished: new Date(now - 9000).toISOString(), lastSuccess: new Date(now - 9000).toISOString() }; +const systemd = { timerActive: 'active', timerEnabled: 'enabled', serviceResult: 'success' }; + +function harness(options = {}) { + const queries = []; + const pool = { async query(sql, params) { + queries.push({ sql, params }); + if (options.databaseError) throw new Error('password=private database inaccessible'); + if (/pg_is_in_recovery/.test(sql)) return { rows: [{ recovery: options.passive === true }] }; + if (/height=0/.test(sql)) return { rows: [{ hash: genesis }] }; + if (/SELECT height, hash/.test(sql)) return { rows: options.anchorMissing ? [] : [{ height: options.anchorHeight ?? options.height ?? activation + 100, hash: anchorHash }] }; + if (/WHERE height=\$1/.test(sql)) return { rows: [{ hash: options.anchorChanged ? 'e'.repeat(64) : anchorHash }] }; + return { rows: options.missingHeight == null ? [] : [{ height: options.missingHeight }] }; + } }; + const rpc = async (method, params = []) => { + if (options.rpcError) throw new Error('https://user:private@example.invalid failed'); + if (method === 'getblockhash') { + if (params[0] === 0) return options.genesis ?? genesis; + if (options.anchorRpcError) throw new Error('Node checkpoint unavailable'); + return options.nodeAnchor ?? anchorHash; + } + return options.info ?? { chain: 'test', blocks: options.height ?? activation + 100, upgrades: { + blossom: { name: 'Blossom', activationheight: 584000 }, nu7: { name: 'NU7', activationheight: activation }, + } }; + }; + return { queries, check: extra => checkAllocationHealth({ pool, rpc, network: 'testnet', state, systemd, now, ...extra }) }; +} + +test('health verifies recent execution and canonical allocation gaps with the serving-node grace window', async () => { + const h = harness(); + const health = await h.check(); + assert.deepEqual(health, { version: 1, network: 'testnet', checkedAt: new Date(now).toISOString(), status: 'healthy', reason: 'ok' }); + const candidates = h.queries.find(query => /ORDER BY b.height/.test(query.sql)); + assert.deepEqual(candidates.params, [activation, activation + 76, 'test', 1, []]); + assert.ok(h.queries.every(query => /^SELECT/.test(query.sql)), 'monitor never writes observations or cache records'); +}); + +test('old allocation gaps are unhealthy even when a fresh successful worker is deferring the failed hash', async () => { + const h = harness({ missingHeight: activation + 1 }); + const health = await h.check({ state: { ...state, retries: { ['a'.repeat(64)]: { nextRetryAt: new Date(now + 900000).toISOString() } } } }); + assert.equal(health.status, 'unhealthy'); + assert.equal(health.reason, 'allocation_gap'); + assert.equal(health.missingHeight, activation + 1); + assert.deepEqual(h.queries.find(query => /ORDER BY b.height/.test(query.sql)).params[4], [], 'monitor must not hide deferred failed heights'); +}); + +test('missing state, wrong network, stale completion and stale success remain unhealthy', async () => { + for (const [changed, reason] of [ + [null, 'state_unavailable'], [initialState('testnet'), 'state_unavailable'], + [{ ...state, status: 'unknown' }, 'state_unavailable'], + [{ ...state, network: 'mainnet' }, 'state_unavailable'], + [{ ...state, lastFinished: new Date(now - 300001).toISOString() }, 'job_completion_stale'], + [{ ...state, lastSuccess: new Date(now - 300001).toISOString() }, 'job_success_stale'], + [{ ...state, lastSuccess: new Date(now + 30001).toISOString() }, 'job_success_stale'], + ]) { + const h = harness(); + const health = await h.check({ state: changed }); + assert.equal(health.reason, reason); + assert.equal(health.status, 'unhealthy'); + assert.equal(h.queries.length, 0); + } +}); + +test('disabled timers, failed services and hung or failed jobs cannot report healthy', async () => { + for (const extra of [ + { systemd: { ...systemd, timerActive: 'inactive' } }, + { systemd: { ...systemd, timerEnabled: 'disabled' } }, + { systemd: { ...systemd, serviceResult: 'timeout' } }, + { state: { ...state, status: 'error' } }, + { state: { ...state, status: 'degraded' } }, + { state: { ...state, status: 'running', lastAttempt: new Date(now - 75001).toISOString() } }, + ]) assert.equal((await harness().check(extra)).status, 'unhealthy'); + assert.equal((await harness().check({ state: { ...state, status: 'running' } })).status, 'healthy', 'a bounded in-progress run may retain a recent successful heartbeat'); +}); + +test('database, network, genesis and RPC failures return only stable public-safe reason codes', async () => { + for (const options of [{ passive: true }, { databaseError: true }, { rpcError: true }, + { genesis: 'a'.repeat(64) }, { info: { chain: 'main', blocks: activation } }, + { info: { chain: 'test', blocks: activation, upgrades: {} } }]) { + const health = await harness(options).check(); + assert.equal(health.status, 'unhealthy'); + assert.match(health.reason, /^[a-z_]+$/); + assert.doesNotMatch(JSON.stringify(health), /password|private|https:|example\.invalid/); + } +}); + +test('a recovered worker and repaired canonical gap return healthy on the next check', async () => { + assert.equal((await harness({ missingHeight: activation }).check()).reason, 'allocation_gap'); + assert.equal((await harness().check({ state: { ...state, lastSuccess: new Date(now).toISOString() } })).status, 'healthy'); +}); + +test('an equal-height node reorg cannot report healthy for stale database allocations', async () => { + assert.equal((await harness({ nodeAnchor: 'e'.repeat(64) }).check()).reason, 'anchor_mismatch'); + assert.equal((await harness({ anchorChanged: true }).check()).reason, 'anchor_changed'); + assert.equal((await harness({ anchorMissing: true }).check()).reason, 'anchor_unavailable'); + assert.equal((await harness({ anchorRpcError: true }).check()).reason, 'anchor_unavailable'); + assert.equal((await harness({ anchorHeight: activation + 101 }).check()).reason, 'anchor_unavailable'); + assert.equal((await harness().check()).status, 'healthy', 'matching canonical checkpoint recovers on the next check'); +}); + +test('the 24-block grace includes its oldest boundary and never fabricates a preactivation range', async () => { + const recent = harness({ height: activation + 23, missingHeight: activation }); + assert.equal((await recent.check()).status, 'healthy'); + assert.equal(recent.queries.some(query => /ORDER BY b.height/.test(query.sql)), false); + assert.equal((await harness({ height: activation + 24, missingHeight: activation }).check()).reason, 'allocation_gap'); +}); + +test('systemd units bound hangs and retain private state without a notification sender', () => { + for (const name of ['cipherscan-subsidy-allocations', 'cipherscan-subsidy-allocations-health']) { + const service = fs.readFileSync(`server/deploy/${name}.service`, 'utf8'); + assert.match(service, /StateDirectory=cipherscan-subsidy-allocations\n/); + assert.match(service, /StateDirectoryMode=0700\n/); + assert.match(service, /TimeoutStartSec=60\n/); + assert.match(service, /TimeoutStopSec=5\n/); + assert.doesNotMatch(service, /OnFailure|sendMessage/); + const timer = fs.readFileSync(`server/deploy/${name}.timer`, 'utf8'); + assert.match(timer, /OnUnitInactiveSec=60\n/); + assert.match(timer, /WantedBy=timers.target/); + } +}); diff --git a/server/tests/subsidy-allocation-state.test.js b/server/tests/subsidy-allocation-state.test.js new file mode 100644 index 00000000..964ebed6 --- /dev/null +++ b/server/tests/subsidy-allocation-state.test.js @@ -0,0 +1,191 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs/promises'); +const os = require('node:os'); +const path = require('node:path'); +const { EventEmitter } = require('node:events'); +const { initialState, readAllocationState, writePrivateJson, deferredHashes, finishState, + runMaintenanceWithState: runWithState } = require('../lib/subsidy-allocation-state'); + +function ownershipHarness() { + let owner = null; + const clients = []; + const pool = { async connect() { + const client = new EventEmitter(); + client.dead = false; + client.query = async sql => { + if (client.dead) throw new Error('Ownership session disconnected'); + if (/pg_try_advisory_lock/.test(sql)) { + const acquired = owner === null; + if (acquired) owner = client; + return { rows: [{ acquired }] }; + } + if (owner === client) owner = null; + return { rows: [] }; + }; + client.release = error => { client.released = true; client.releaseError = error; if (owner === client) owner = null; }; + clients.push(client); + return client; + } }; + return { pool, clients, disconnectOwner() { + const previous = owner; + owner = null; + previous.dead = true; + previous.emit('error', new Error('Owner connection lost')); + } }; +} + +function runMaintenanceWithState(options) { + return runWithState({ ownershipPool: ownershipHarness().pool, ...options }); +} + +const baseTime = Date.parse('2026-10-16T03:00:00Z'); +const hash = 'a'.repeat(64); +const otherHash = 'b'.repeat(64); +const success = { status: 'complete', candidates: 1, processed: 1, written: 1, skipped: 0, + writtenHashes: [hash], failures: [] }; +const partial = { ...success, status: 'partial', writtenHashes: [otherHash], + failures: [{ height: 4465026, hash, message: 'Historical subsidy RPC unavailable.' }] }; + +async function stateFile(t) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'subsidy-state-')); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + return path.join(directory, 'private', 'state.json'); +} + +test('state replacement is atomic and private, with no leftover temporary files', async t => { + const filePath = await stateFile(t); + await writePrivateJson(filePath, initialState('testnet')); + await writePrivateJson(filePath, { ...initialState('testnet'), status: 'running' }); + assert.equal((await readAllocationState(filePath, 'testnet')).status, 'running'); + assert.equal((await fs.stat(filePath)).mode & 0o777, 0o600); + assert.equal((await fs.stat(path.dirname(filePath))).mode & 0o777, 0o700); + assert.deepEqual(await fs.readdir(path.dirname(filePath)), ['state.json']); +}); + +test('partial progress persists with a failed exit, and deferred hashes do not block other work', async t => { + const filePath = await stateFile(t); + const first = await runMaintenanceWithState({ filePath, network: 'testnet', now: () => baseTime, + maintain: async ({ excludeHashes }) => { assert.deepEqual(excludeHashes, []); return partial; } }); + assert.equal(first.exitCode, 1); + assert.equal(first.state.status, 'degraded'); + assert.equal(first.state.lastSuccess, null); + assert.equal(first.state.result.written, 1); + assert.equal(Date.parse(first.state.retries[hash].nextRetryAt), baseTime + 60_000); + const second = await runMaintenanceWithState({ filePath, network: 'testnet', now: () => baseTime + 30_000, + maintain: async ({ excludeHashes }) => { assert.deepEqual(excludeHashes, [hash]); return { ...success, writtenHashes: [otherHash] }; } }); + assert.equal(second.exitCode, 0); + assert.equal(second.state.lastSuccess, new Date(baseTime + 30_000).toISOString()); + assert.equal(second.state.retries[hash].attempts, 1); + assert.deepEqual(deferredHashes(second.state, baseTime + 60_000), [], 'the failed height becomes eligible again'); +}); + +test('per-hash retry delays grow from one minute to fifteen and clear after successful recovery', () => { + let state = initialState('testnet'); + let currentTime = baseTime; + for (const delay of [60_000, 120_000, 240_000, 480_000, 900_000, 900_000]) { + state = finishState(state, partial, currentTime); + assert.equal(Date.parse(state.retries[hash].nextRetryAt) - currentTime, delay); + currentTime += delay; + } + const recovered = finishState(state, success, currentTime); + assert.equal(recovered.status, 'complete'); + assert.equal(recovered.lastSuccess, new Date(currentTime).toISOString()); + assert.deepEqual(recovered.retries, {}); +}); + +test('the retry registry stays bounded while newly failed heights remain eligible for backoff', () => { + const state = initialState('testnet'); + for (let i = 0; i < 5000; i++) state.retries[i.toString(16).padStart(64, '0')] = { + height: i, attempts: 1, lastFailedAt: new Date(baseTime - i).toISOString(), nextRetryAt: new Date(baseTime + 60_000).toISOString(), + }; + const next = finishState(state, partial, baseTime + 1); + assert.equal(Object.keys(next.retries).length, 5000); + assert.ok(next.retries[hash]); + assert.equal(deferredHashes(next, baseTime + 1).length, 5000); +}); + +test('fatal errors preserve earlier progress and record a finished failed attempt without leaking credentials', async t => { + const filePath = await stateFile(t); + await runMaintenanceWithState({ filePath, network: 'testnet', now: () => baseTime, maintain: async () => success }); + await assert.rejects(runMaintenanceWithState({ filePath, network: 'testnet', now: () => baseTime + 60_000, + maintain: async () => { throw new Error('RPC https://user:password@example.invalid/path token=private\nfailed'); } })); + const state = await readAllocationState(filePath, 'testnet'); + assert.equal(state.status, 'error'); + assert.equal(state.lastSuccess, new Date(baseTime).toISOString()); + assert.equal(state.lastFinished, new Date(baseTime + 60_000).toISOString()); + assert.equal(state.result.written, 1, 'previous successful progress is retained'); + assert.doesNotMatch(state.lastError, /user:password|private|\n/); +}); + +test('missing state initializes safely, but corrupt or wrong-network state cannot start maintenance', async t => { + const filePath = await stateFile(t); + assert.equal((await readAllocationState(filePath, 'testnet')).status, 'new'); + await writePrivateJson(filePath, initialState('mainnet')); + let calls = 0; + await assert.rejects(runMaintenanceWithState({ filePath, network: 'testnet', maintain: async () => { calls++; return success; } }), /identity/); + assert.equal(calls, 0); + await fs.writeFile(filePath, '{broken'); + await assert.rejects(readAllocationState(filePath, 'testnet'), /malformed/); + await writePrivateJson(filePath, { ...initialState('testnet'), status: 'unknown' }); + await assert.rejects(readAllocationState(filePath, 'testnet'), /identity/); + await assert.rejects(readAllocationState(filePath, 'crosslink'), /Explicit NETWORK/); +}); + +test('busy and interrupted attempts cannot manufacture a fresh successful heartbeat', async t => { + const filePath = await stateFile(t); + const state = { ...initialState('testnet'), status: 'running', lastAttempt: new Date(baseTime).toISOString(), + lastSuccess: new Date(baseTime - 60_000).toISOString() }; + await writePrivateJson(filePath, state); + const result = await runMaintenanceWithState({ filePath, network: 'testnet', now: () => baseTime + 30_000, + maintain: async () => ({ ...success, status: 'busy', written: 0, writtenHashes: [] }) }); + assert.equal(result.state.lastSuccess, state.lastSuccess); +}); + +test('a concurrent busy wrapper never reads or changes the active owner state', async t => { + const filePath = await stateFile(t); + const ownership = ownershipHarness(); + let entered; + const started = new Promise(resolve => { entered = resolve; }); + let finish; + const first = runMaintenanceWithState({ filePath, network: 'testnet', ownershipPool: ownership.pool, + now: () => baseTime, maintain: () => { entered(); return new Promise(resolve => { finish = resolve; }); } }); + await started; + const beforeBusy = await fs.readFile(filePath, 'utf8'); + let calls = 0; + const busy = await runMaintenanceWithState({ filePath, network: 'testnet', ownershipPool: ownership.pool, + now: () => baseTime + 1, maintain: async () => { calls++; return success; } }); + assert.equal(busy.result.status, 'busy'); + assert.equal(busy.state, null); + assert.equal(calls, 0); + assert.equal(await fs.readFile(filePath, 'utf8'), beforeBusy); + finish(partial); + assert.equal((await first).state.retries[hash].attempts, 1); + assert.ok(ownership.clients.every(client => client.released)); +}); + +test('an abandoned ownership session recovers and the disconnected process cannot overwrite its successor', async t => { + const filePath = await stateFile(t); + const ownership = ownershipHarness(); + let entered; + const started = new Promise(resolve => { entered = resolve; }); + let finish; + let lost = 0; + const abandoned = runMaintenanceWithState({ filePath, network: 'testnet', ownershipPool: ownership.pool, + now: () => baseTime, onOwnershipLost: () => { lost++; }, + maintain: () => { entered(); return new Promise(resolve => { finish = resolve; }); } }); + await started; + ownership.disconnectOwner(); + assert.equal(lost, 1); + const recovered = await runMaintenanceWithState({ filePath, network: 'testnet', ownershipPool: ownership.pool, + now: () => baseTime + 60_000, maintain: async () => success }); + assert.equal(recovered.state.status, 'complete'); + const successorState = await fs.readFile(filePath, 'utf8'); + finish(partial); + await assert.rejects(abandoned, /ownership connection was lost/); + assert.equal(await fs.readFile(filePath, 'utf8'), successorState); + assert.ok(ownership.clients[0].released); + assert.ok(ownership.clients[0].releaseError); +}); diff --git a/server/tests/subsidy-allocations-maintenance.test.js b/server/tests/subsidy-allocations-maintenance.test.js new file mode 100644 index 00000000..33e55dbd --- /dev/null +++ b/server/tests/subsidy-allocations-maintenance.test.js @@ -0,0 +1,308 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { maintainAllocations } = require('../lib/subsidy-allocation-maintenance'); + +const genesis = '0'.repeat(64); +const hash = 'a'.repeat(64); +const otherHash = 'b'.repeat(64); +const activationHeight = 4_465_026; +const subsidy = { founders: 0, fundingstreamstotal: 0.04166666, miner: 0.41666668 }; +const info = { + chain: 'test', blocks: activationHeight + 10, bestblockhash: 'c'.repeat(64), + upgrades: { + blossom: { name: 'Blossom', activationheight: 584_000 }, + nu7: { name: 'NU7', activationheight: activationHeight, status: 'active' }, + }, +}; + +// This fake models the external safety boundaries: the serving chain can +// reorganize before/after its height-pinned RPC, or before the guarded insert. +function harness(options = {}) { + const queries = []; + const rpcCalls = []; + const inserts = []; + let released = false; + let allocationHashReads = 0; + const candidates = options.candidates ?? [{ height: activationHeight, hash }]; + const client = { + async query(sql, params = []) { + queries.push({ sql, params }); + if (/pg_try_advisory_lock/.test(sql)) return { rows: [{ acquired: options.locked !== true }] }; + if (/pg_advisory_unlock/.test(sql)) return { rows: [{ pg_advisory_unlock: true }] }; + if (/pg_is_in_recovery/.test(sql)) return { rows: [{ recovery: options.recovery === true }] }; + if (/INSERT INTO (?:public\.)?block_subsidy_allocations/.test(sql)) { + if (options.insertError) throw options.insertError; + inserts.push({ sql, params }); + options.onInsert?.(); + return { rows: [], rowCount: options.inserted === false ? 0 : 1 }; + } + if (/FROM (?:public\.)?blocks/.test(sql) && /height\s*=\s*0/.test(sql)) { + return { rows: [{ hash: options.databaseGenesis ?? genesis }] }; + } + if (/FROM (?:public\.)?blocks/.test(sql)) { + const excluded = new Set(params[4] ?? []); + return { rows: candidates.filter(row => !excluded.has(row.hash)).slice(0, params[3]) }; + } + throw new Error(`Unexpected database query: ${sql}`); + }, + release() { released = true; }, + }; + const pool = { connect: async () => client }; + const rpc = async (method, params = [], rpcOptions = {}) => { + rpcCalls.push({ method, params, options: rpcOptions }); + options.onRpc?.(method, params); + if (method === 'getblockchaininfo') { + if (options.infoError) throw options.infoError; + return options.info ?? info; + } + if (method === 'getblockhash') { + if (Number(params[0]) === 0) return options.nodeGenesis ?? genesis; + allocationHashReads++; + if (options.hashes) return options.hashes[allocationHashReads - 1] ?? hash; + return candidates.find(row => Number(row.height) === Number(params[0]))?.hash ?? hash; + } + if (method === 'getblocksubsidy') { + const error = typeof options.rpcError === 'function' ? options.rpcError(params[0]) : options.rpcError; + if (error) throw error; + return typeof options.subsidy === 'function' ? options.subsidy(params[0]) + : options.subsidy === undefined ? subsidy : options.subsidy; + } + throw new Error(`Unexpected RPC method: ${method}`); + }; + return { pool, rpc, queries, rpcCalls, inserts, get released() { return released; } }; +} + +test('maintenance stores exact historical allocations without manufacturing reserve observations', async () => { + const h = harness(); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.processed, 1); + assert.equal(result.written, 1); + assert.deepEqual(result.writtenHashes, [hash]); + assert.deepEqual(result.failures, []); + assert.equal(h.inserts.length, 1); + const values = h.inserts[0].params.map(String); + for (const expected of [hash, String(activationHeight), 'test', '0', '4166666', '41666668']) { + assert.ok(values.includes(expected), `missing exact allocation value ${expected}`); + } + assert.match(h.inserts[0].sql, /ON CONFLICT[^;]*DO NOTHING/is); + assert.ok(h.queries.every(query => !/INSERT INTO (?:public\.)?node_accounting_observations/.test(query.sql))); + assert.ok(h.queries.every(query => !/FOR (?:UPDATE|SHARE)/.test(query.sql)), 'job does not require UPDATE privilege on blocks'); + assert.equal(h.released, true); +}); + +test('zero founders and funding allocations after the halving are authoritative values', async () => { + const h = harness({ subsidy: { founders: 0, fundingstreamstotal: 0, miner: 0.26041666 } }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.written, 1); + assert.ok(h.inserts[0].params.map(String).includes('26041666')); +}); + +test('a reorg before or after subsidy retrieval never persists the old allocation', async () => { + for (const hashes of [[otherHash], [hash, otherHash]]) { + const h = harness({ hashes }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.written, 0); + assert.equal(h.inserts.length, 0); + assert.equal(h.released, true); + if (hashes.length === 1) assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 0); + } +}); + +test('the guarded insert may lose a canonical race or find an existing immutable hash', async () => { + const h = harness({ inserted: false }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.processed, 1); + assert.equal(result.written, 0); + assert.deepEqual(result.writtenHashes, []); + assert.equal(h.released, true); +}); + +test('malformed, missing, negative and fractional-zatoshi RPC amounts remain missing', async () => { + for (const response of [ + null, + { ...subsidy, founders: undefined }, + { ...subsidy, founders: -1 }, + { ...subsidy, fundingstreamstotal: 0.041666661 }, + { ...subsidy, fundingstreamstotal: 21_000_001 }, + { ...subsidy, founders: 'not reported' }, + { ...subsidy, founders: Number.NaN }, + ]) { + const h = harness({ subsidy: response }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.status, 'partial'); + assert.deepEqual(result.failures, [{ height: activationHeight, hash, message: 'Historical subsidy allocations unavailable.' }]); + assert.equal(h.inserts.length, 0); + assert.equal(h.released, true); + } +}); + +test('missing optional miner subsidy does not hide known compulsory allocations', async () => { + const h = harness({ subsidy: { founders: 0, fundingstreamstotal: 0.04166666 } }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.written, 1); + assert.ok(h.inserts[0].params.includes(null)); + assert.ok(h.inserts[0].params.map(String).includes('4166666')); +}); + +test('standby, wrong-network and genesis mismatches cannot write allocations', async () => { + for (const options of [ + { recovery: true }, + { info: { ...info, chain: 'main' } }, + { databaseGenesis: otherHash }, + { nodeGenesis: otherHash }, + ]) { + const h = harness(options); + await assert.rejects(maintainAllocations(h.pool, h.rpc, { network: 'testnet' })); + assert.equal(h.inserts.length, 0); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 0); + assert.equal(h.released, true); + } +}); + +test('a second maintenance run exits when another worker owns its advisory lock', async () => { + const h = harness({ locked: true }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.written, 0); + assert.equal(h.inserts.length, 0); + assert.equal(h.released, true); +}); + +test('an unscheduled mainnet does not backfill arbitrary historical allocations', async () => { + const h = harness({ info: { ...info, chain: 'main', upgrades: { + blossom: { name: 'Blossom', activationheight: 653_600 }, + } } }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'mainnet' }); + assert.equal(result.written, 0); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 0); + assert.equal(h.inserts.length, 0); + assert.equal(h.released, true); +}); + +test('an exhausted wall-clock budget starts no historical subsidy work', async () => { + const h = harness(); + let ticks = 0; + const result = await maintainAllocations(h.pool, h.rpc, { + network: 'testnet', timeBudgetMs: 10, now: () => ticks++ === 0 ? 0 : 11, + }); + assert.equal(result.status, 'budget'); + assert.deepEqual(result.failures, []); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 0); + assert.equal(h.released, true); +}); + +test('maintenance stops between rows when its budget expires', async () => { + let currentTime = 0; + const h = harness({ candidates: [ + { height: activationHeight, hash }, + { height: activationHeight + 1, hash: otherHash }, + ], onInsert: () => { currentTime = 11; } }); + const result = await maintainAllocations(h.pool, h.rpc, { + network: 'testnet', timeBudgetMs: 10, now: () => currentTime, + }); + assert.equal(result.status, 'budget'); + assert.equal(result.processed, 1); + assert.equal(result.written, 1); + assert.deepEqual(result.failures, []); + assert.deepEqual(result.writtenHashes, [hash]); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 1); + assert.equal(h.released, true); +}); + +test('the requested candidate limit bounds historical RPC and insert work', async () => { + const h = harness({ candidates: [ + { height: activationHeight, hash }, + { height: activationHeight + 1, hash: otherHash }, + ] }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet', limit: 1 }); + assert.equal(result.processed, 1); + assert.equal(result.written, 1); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 1); +}); + +test('invalid invocation cannot connect to the database or node', async () => { + const h = harness(); + for (const options of [ + {}, { network: 'crosslink' }, { network: 'testnet', limit: 0 }, + { network: 'testnet', limit: 501 }, { network: 'testnet', limit: 1.5 }, + { network: 'testnet', timeBudgetMs: 0 }, { network: 'testnet', timeBudgetMs: 60_001 }, + { network: 'testnet', excludeHashes: null }, { network: 'testnet', excludeHashes: 'bad' }, + { network: 'testnet', excludeHashes: ['bad'] }, { network: 'testnet', excludeHashes: [hash.toUpperCase()] }, + { network: 'testnet', excludeHashes: Array(5001).fill(hash) }, + ]) await assert.rejects(maintainAllocations(h.pool, h.rpc, options)); + assert.equal(h.queries.length, 0); + assert.equal(h.rpcCalls.length, 0); +}); + +test('insertion and network discovery failures remain fatal and release the worker session', async () => { + for (const options of [{ infoError: new Error('node unavailable') }, { insertError: new Error('database unavailable') }]) { + const h = harness(options); + await assert.rejects(maintainAllocations(h.pool, h.rpc, { network: 'testnet' })); + assert.equal(h.released, true); + assert.ok(h.queries.some(query => /pg_advisory_unlock/.test(query.sql))); + } +}); + +test('an unavailable or malformed earliest height does not block later valid receipts', async () => { + for (const options of [ + { rpcError: height => height === activationHeight ? new Error('upstream response contains secret material') : null }, + { subsidy: height => height === activationHeight ? { ...subsidy, founders: 'secret material' } : subsidy }, + ]) { + const h = harness({ ...options, candidates: [ + { height: activationHeight, hash }, + { height: activationHeight + 1, hash: otherHash }, + ] }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet' }); + assert.equal(result.status, 'partial'); + assert.equal(result.processed, 2); + assert.equal(result.written, 1); + assert.deepEqual(result.writtenHashes, [otherHash]); + assert.equal(result.failures.length, 1); + assert.equal(result.failures[0].height, activationHeight); + assert.equal(result.failures[0].hash, hash); + assert.match(result.failures[0].message, /^Historical subsidy (RPC|allocations) unavailable\.$/); + assert.doesNotMatch(JSON.stringify(result), /secret material/); + assert.equal(h.released, true); + } +}); + +test('excluded failed hashes leave the bounded batch available to later gaps', async () => { + const h = harness({ candidates: [ + { height: activationHeight, hash }, + { height: activationHeight + 1, hash: otherHash }, + ] }); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet', limit: 1, excludeHashes: [hash] }); + assert.equal(result.candidates, 1); + assert.equal(result.processed, 1); + assert.deepEqual(result.writtenHashes, [otherHash]); + assert.deepEqual(result.failures, []); + assert.ok(h.queries.some(query => /ANY\(\$5::text\[\]\)/.test(query.sql) && query.params[4]?.[0] === hash)); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy' && call.params[0] === activationHeight).length, 0); +}); + +test('the exclusion bound accepts 5000 valid hashes without unnecessary RPC work', async () => { + const h = harness(); + const result = await maintainAllocations(h.pool, h.rpc, { network: 'testnet', excludeHashes: Array(5000).fill(hash) }); + assert.equal(result.processed, 0); + assert.equal(result.written, 0); + assert.equal(h.rpcCalls.filter(call => call.method === 'getblocksubsidy').length, 0); +}); + +test('deadline exhaustion during a candidate never marks that height for backoff', async () => { + for (const rpcError of [undefined, new Error('RPC timeout')]) { + let currentTime = 0; + const h = harness({ rpcError, onRpc: method => { + if (method === 'getblocksubsidy') currentTime = 11; + } }); + const result = await maintainAllocations(h.pool, h.rpc, { + network: 'testnet', timeBudgetMs: 10, now: () => currentTime, + }); + assert.equal(result.status, 'budget'); + assert.equal(result.written, 0); + assert.deepEqual(result.failures, []); + assert.deepEqual(result.writtenHashes, []); + assert.equal(h.inserts.length, 0); + assert.equal(h.released, true); + } +}); diff --git a/server/tests/subsidy-allocations.test.js b/server/tests/subsidy-allocations.test.js new file mode 100644 index 00000000..743aa25f --- /dev/null +++ b/server/tests/subsidy-allocations.test.js @@ -0,0 +1,38 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { exactCoinZat, parseRpcSubsidyAllocations, hasSubsidyAllocations, sampledAllocationsSql } = require('../api/lib/subsidy-allocations'); + +test('node allocation parsing preserves exact zatoshis across the real testnet halving', () => { + assert.deepEqual(parseRpcSubsidyAllocations({ founders: 0, fundingstreamstotal: 0.04166666, miner: 0.41666668 }), + { foundersZat: '0', fundingStreamsZat: '4166666', minerZat: '41666668' }); + assert.deepEqual(parseRpcSubsidyAllocations({ founders: '0.00000000', fundingstreamstotal: '0', miner: '0.26041666' }), + { foundersZat: '0', fundingStreamsZat: '0', minerZat: '26041666' }); + assert.deepEqual(parseRpcSubsidyAllocations({ founders: 0, fundingstreamstotal: 0 }), + { foundersZat: '0', fundingStreamsZat: '0', minerZat: null }); + assert.equal(exactCoinZat(0.00000001), '1'); + for (const value of [null, undefined, '', '-1', '1e-8', '0.000000001', 0.000000001, Infinity, NaN, 21000001, '21000000.00000001']) { + assert.equal(exactCoinZat(value), null, `invalid amount ${value}`); + assert.equal(parseRpcSubsidyAllocations({ founders: 0, fundingstreamstotal: value }), null); + } +}); + +test('optional allocation-table detection coalesces concurrent reads and propagates query failures', async () => { + let calls = 0; + const pool = { query: async () => { calls++; return { rows: [{ available: true }] }; } }; + assert.deepEqual(await Promise.all(Array.from({ length: 5 }, () => hasSubsidyAllocations(pool))), [true, true, true, true, true]); + assert.equal(calls, 1); + assert.equal(await hasSubsidyAllocations({ query: async () => ({ rows: [{ available: false }] }) }), false); + let fail = true; + const recovering = { query: async () => { if (fail) throw new Error('unavailable'); return { rows: [{ available: true }] }; } }; + await assert.rejects(hasSubsidyAllocations(recovering), /unavailable/); + fail = false; + assert.equal(await hasSubsidyAllocations(recovering), true); +}); + +test('worker and receipts share guarded numeric sample allocation expressions', () => { + const fields = sampledAllocationsSql('sample'); + assert.deepEqual(Object.keys(fields), ['founders', 'funding', 'miner']); + assert.ok(fields.founders.includes("jsonb_typeof(sample.subsidy->'founders')='number'")); + assert.ok(fields.funding.includes("sample.subsidy->>'fundingstreamstotal'")); + assert.ok(fields.miner.includes("sample.subsidy->>'miner'")); +});