From 69e34182fe79e7eb1c506cfb3b5e21bc9735fb4a Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 18 Aug 2026 15:20:05 +0800 Subject: [PATCH 01/49] fix(render): cap JVM heap at 50% of container memory below 8GB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tiered heap table returned -Xmx2g for any resources.memory < 8GB, which equals the container limit verbatim (compose.go) — with the image's forced ZGC (Xms=Xmx) there is zero headroom and the JVM crash-loops with 'Failed to commit memory'. Below 8GB now cap -Xmx at floor(total/2) in MB (2GB->1024m, 3GB->1536m, 4GB->2g). jvm.heap_max still bypasses verbatim. Also fix the 1GB edge where -Xmn equaled -Xmx leaving zero old gen (512m heap now gets 128m new). Found in TX-167 functional testing. --- cmd/preflight.go | 31 ++++++++++++++++ cmd/preflight_test.go | 70 +++++++++++++++++++++++++++++++++++++ internal/render/jvm.go | 28 ++++++++++++--- internal/render/jvm_test.go | 41 ++++++++++++++++++++++ 4 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 cmd/preflight_test.go diff --git a/cmd/preflight.go b/cmd/preflight.go index 26833585..5928ef0b 100644 --- a/cmd/preflight.go +++ b/cmd/preflight.go @@ -74,6 +74,7 @@ func runPreflight(cmd *cobra.Command, args []string) error { // Memory check checks = append(checks, checkMemory(cmd, tgt, parsed)) + checks = append(checks, checkMemoryRecommended(parsed)...) // Port check for _, node := range parsed.Nodes { @@ -197,6 +198,36 @@ func checkMemory(cmd *cobra.Command, tgt target.Target, parsed *intent.Intent) c Message: fmt.Sprintf("%dGB total", memGB)} } +// checkMemoryRecommended warns about container memory below java-tron's +// official startup recommendation. This is an intent-side check: it does not +// inspect the target's available memory or alter the existing memory check. +func checkMemoryRecommended(parsed *intent.Intent) []checkResult { + var checks []checkResult + for i, node := range parsed.Nodes { + memoryGB := render.ParseMemoryGB(node.Resources.Memory) + if memoryGB <= 0 || memoryGB >= 8 { + continue + } + nodeName := parsed.Name + if len(parsed.Nodes) > 1 { + nodeName = fmt.Sprintf("%s-node%d", parsed.Name, i) + } + checks = append(checks, checkResult{ + Name: "memory-recommended", + Status: "warning", + Message: fmt.Sprintf("node '%s' requests %s container memory, below java-tron's official minimum 8192MB (start.sh ALLOW_MIN_MEMORY); small containers risk JVM startup failure", nodeName, node.Resources.Memory), + }) + } + if len(checks) == 0 { + return []checkResult{{ + Name: "memory-recommended", + Status: "pass", + Message: "all nodes meet java-tron's official 8192MB minimum", + }} + } + return checks +} + // checkPorts probes every well-known port the intent will expose. // Earlier this shelled out to `ss -tlnp` which doesn't exist on macOS, // so the check silently reported every port as available. Use net.Dial diff --git a/cmd/preflight_test.go b/cmd/preflight_test.go new file mode 100644 index 00000000..79c0e60f --- /dev/null +++ b/cmd/preflight_test.go @@ -0,0 +1,70 @@ +package cmd + +import ( + "strings" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/intent" +) + +func TestCheckMemoryRecommended(t *testing.T) { + tests := []struct { + name string + nodes []intent.NodeSpec + warnings int + status string + messageHas []string + }{ + { + name: "4GB warns", + nodes: []intent.NodeSpec{{Type: "fullnode", Resources: intent.Resources{Memory: "4GB"}}}, + warnings: 1, messageHas: []string{"node 'test'", "8192MB"}, + }, + { + name: "8GB passes", + nodes: []intent.NodeSpec{{Type: "fullnode", Resources: intent.Resources{Memory: "8GB"}}}, + status: "pass", + }, + { + name: "16GB passes", + nodes: []intent.NodeSpec{{Type: "fullnode", Resources: intent.Resources{Memory: "16GB"}}}, + status: "pass", + }, + { + name: "mixed memory warns once", + nodes: []intent.NodeSpec{ + {Type: "fullnode", Resources: intent.Resources{Memory: "2GB"}}, + {Type: "fullnode", Resources: intent.Resources{Memory: "16GB"}}, + }, + warnings: 1, + }, + { + name: "empty memory uses default", + nodes: []intent.NodeSpec{{Type: "fullnode"}}, + status: "pass", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + checks := checkMemoryRecommended(&intent.Intent{Name: "test", Nodes: tt.nodes}) + gotWarnings := 0 + for _, check := range checks { + if check.Status == "warning" { + gotWarnings++ + } + for _, part := range tt.messageHas { + if !strings.Contains(check.Message, part) { + t.Errorf("message %q missing %q", check.Message, part) + } + } + } + if gotWarnings != tt.warnings { + t.Errorf("warning count = %d, want %d", gotWarnings, tt.warnings) + } + if tt.status != "" && (len(checks) != 1 || checks[0].Status != tt.status) { + t.Errorf("checks = %+v, want one %s check", checks, tt.status) + } + }) + } +} diff --git a/internal/render/jvm.go b/internal/render/jvm.go index 449dfb85..4f53ff37 100644 --- a/internal/render/jvm.go +++ b/internal/render/jvm.go @@ -94,18 +94,30 @@ func calculateHeapMax(totalMemoryGB int, jvm *intent.JVMConfig) string { return jvm.HeapMax } + var heapMax string switch { case totalMemoryGB >= 64: - return "24g" + heapMax = "24g" case totalMemoryGB >= 32: - return "14g" + heapMax = "14g" case totalMemoryGB >= 16: - return "8g" + heapMax = "8g" case totalMemoryGB >= 8: - return "4g" + heapMax = "4g" default: - return "2g" + heapMax = "2g" } + + // Keep native headroom for ZGC in the container. For sub-8GB limits, + // cap the start.sh tier at half the limit, computed in whole MB so 3GB + // becomes 1536m rather than truncating to 1g. + if totalMemoryGB > 0 && totalMemoryGB < 8 { + halfMB := totalMemoryGB * 1024 / 2 + if halfMB < 2*1024 { + return fmt.Sprintf("%dm", halfMB) + } + } + return heapMax } func calculateHeapNew(heapMax string, jvm *intent.JVMConfig) string { @@ -115,6 +127,12 @@ func calculateHeapNew(heapMax string, jvm *intent.JVMConfig) string { // HeapNew ≈ HeapMax / 4 switch heapMax { + case "512m": + return "128m" + case "1024m": + return "256m" + case "1536m": + return "384m" case "24g": return "6g" case "14g": diff --git a/internal/render/jvm_test.go b/internal/render/jvm_test.go index 02415f52..05eb85ca 100644 --- a/internal/render/jvm_test.go +++ b/internal/render/jvm_test.go @@ -42,6 +42,9 @@ func TestCalculateHeapMax(t *testing.T) { {16, "8g"}, {8, "4g"}, {4, "2g"}, + {3, "1536m"}, + {2, "1024m"}, + {1, "512m"}, } for _, c := range cases { got := calculateHeapMax(c.memGB, nil) @@ -51,6 +54,44 @@ func TestCalculateHeapMax(t *testing.T) { } } +func TestJVMArgs_HeapHeadroom(t *testing.T) { + cases := []struct { + memGB int + heap string + new string + }{ + {2, "1024m", "256m"}, + {1, "512m", "128m"}, + {3, "1536m", "384m"}, + {4, "2g", "512m"}, + {8, "4g", "1g"}, + {16, "8g", "2g"}, + } + for _, tc := range cases { + got := JVMArgsString(tc.memGB, 17, nil) + for _, want := range []string{"-Xmx" + tc.heap, "-Xms" + tc.heap, "-Xmn" + tc.new} { + if !strings.Contains(got, want) { + t.Errorf("JVMArgs(%dGB) missing %q in %q", tc.memGB, want, got) + } + } + } +} + +func TestJVMArgs_HeapMaxOverrideRemainsVerbatim(t *testing.T) { + got := JVMArgsString(2, 17, &intent.JVMConfig{HeapMax: "1536m"}) + if !strings.Contains(got, "-Xmx1536m") || !strings.Contains(got, "-Xms1536m") { + t.Errorf("heap_max override not applied verbatim: %q", got) + } +} + +func TestJVMArgs_MBMemoryInput(t *testing.T) { + memGB := ParseMemoryGB("2048m") + got := JVMArgsString(memGB, 17, nil) + if !strings.Contains(got, "-Xmx1024m") || !strings.Contains(got, "-Xms1024m") { + t.Errorf("2048m input produced unsafe heap: %q", got) + } +} + func TestCalculateHeapMax_Override(t *testing.T) { jvm := &intent.JVMConfig{HeapMax: "12g"} if got := calculateHeapMax(32, jvm); got != "12g" { From 835bb975d8ca3bc141ebd583db2b78d3d734e526 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 18 Aug 2026 15:20:34 +0800 Subject: [PATCH 02/49] fix(render): de-conflict in-container HTTP ports (fullNode/solidity/PBFT) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ports.http: 8091 rendered fullNodePort = solidityPort = 8091 because solidityPort defaulted to 8091 independently of ports.http — the shipped examples/private-network.yaml crash-looped its fullnode on bind conflict (HttpApiOnSolidityService, then HttpApiOnPBFTService once that was moved). RenderHOCON now bumps solidityPort/PBFTPort by +2 when they equal fullNodePort, with a >65535 overflow guard. Explicitness is judged by config_overrides only: ApplyDefaults pre-fills ports.solidity_http before render, so checking the intent field made the avoidance dead code in the real pipeline (caught in review, with an intent.Parse end-to-end regression test to pin it). Bumps only de-conflict the three node.http keys; cross-family clashes with grpc/p2p/jsonrpc/metrics are out of scope (documented in the function comment). The example also sets ports.solidity_http: 8093 explicitly. Found in TX-167 functional testing. --- cmd/config/validate.go | 18 +++++-- examples/private-network.yaml | 5 +- internal/render/hocon.go | 71 +++++++++++++++++++++++++++ internal/render/hocon_test.go | 90 +++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 6 deletions(-) diff --git a/cmd/config/validate.go b/cmd/config/validate.go index 5942fa0f..179058af 100644 --- a/cmd/config/validate.go +++ b/cmd/config/validate.go @@ -8,6 +8,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/render" ) var validateExplain bool @@ -43,13 +44,20 @@ func runValidate(cmd *cobra.Command, args []string) error { "Refer to examples/ for valid intent files", ) } + raw, rawErr := intent.LoadRaw(intentPath) + if rawErr != nil { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, rawErr.Error()) + } + if err := render.ValidateIntentHTTPPortConflicts("", parsed, raw); err != nil { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()). + WithSuggestions( + "Change ports.http, or move the colliding service: ports.solidity_http for solidityPort, config_overrides \"node.http.PBFTPort\" for PBFTPort", + "Run config render after correcting the conflicting HTTP port", + ) + } if validateExplain { - // Need the raw (no-defaults) form to distinguish explicit vs default. - raw, rawErr := intent.LoadRaw(intentPath) - if rawErr != nil { - return output.NewError("VALIDATION_ERROR", output.ExitValidationError, rawErr.Error()) - } + // The raw (no-defaults) form distinguishes explicit vs default. fmt.Printf("Intent %q is valid (%s, %d node(s))\n\n", parsed.Name, parsed.Network, len(parsed.Nodes)) printExplain(os.Stdout, raw, parsed) return nil diff --git a/examples/private-network.yaml b/examples/private-network.yaml index 0a931ced..82977f4f 100644 --- a/examples/private-network.yaml +++ b/examples/private-network.yaml @@ -31,7 +31,10 @@ nodes: resources: memory: 8GB ports: - http: 8091 + # Host http must dodge the witness's 8090 — but NOT 8091/8092: those are + # java-tron's in-container solidityPort/PBFTPort defaults, and the 1:1 + # port mapping would collide with them (trond rejects such intents). + http: 8095 grpc: 50052 jsonrpc: 8545 p2p: 18889 diff --git a/internal/render/hocon.go b/internal/render/hocon.go index 170c489f..1562a5d5 100644 --- a/internal/render/hocon.go +++ b/internal/render/hocon.go @@ -176,6 +176,9 @@ func RenderHOCONWithSecrets(templateDir string, i *intent.Intent, node *intent.N // 1. Targeted line-level rewrites. config = applyPortOverrides(config, node) config = applyFeatureOverrides(config, node) + if err := checkHTTPPortConflicts(config, node); err != nil { + return Rendered{}, err + } // Monitoring: auto-enable prometheus metrics in HOCON. if i.Monitoring != nil && i.Monitoring.Enabled != nil && *i.Monitoring.Enabled { @@ -199,6 +202,30 @@ func RenderHOCONWithSecrets(templateDir string, i *intent.Intent, node *intent.N }, nil } +// ValidateIntentHTTPPortConflicts performs the same check early, but only +// when the intent explicitly sets HTTP port. Defaults and auto_ports are +// left to the authoritative render-time check. +func ValidateIntentHTTPPortConflicts(templateDir string, parsed, raw *intent.Intent) error { + if raw.Target.AutoPorts { + return nil + } + data, err := LoadTemplate(templateDir, parsed.Network) + if err != nil { + return err + } + for idx := range raw.Nodes { + if raw.Nodes[idx].Ports.HTTP == 0 { + continue + } + node := parsed.Nodes[idx] + config := applyPortOverrides(string(data), &node) + if err := checkHTTPPortConflicts(config, &node); err != nil { + return err + } + } + return nil +} + // hoconAppendix is the "trond overrides" block in its two forms. The // two differ in at most one line — the `localwitness` assignment. type hoconAppendix struct { @@ -448,6 +475,50 @@ func applyPortOverrides(config string, node *intent.NodeSpec) string { return config } +// checkHTTPPortConflicts rejects collisions among the HTTP services that are +// rendered by the template. Explicit config_overrides are applied in the +// appendix and therefore remain authoritative; an operator who explicitly +// sets a colliding key has taken responsibility for that configuration. +func checkHTTPPortConflicts(config string, node *intent.NodeSpec) error { + full, fullOK := hoconPortValue(config, "fullNodePort") + solidity, solidityOK := hoconPortValue(config, "solidityPort") + pbft, pbftOK := hoconPortValue(config, "PBFTPort") + if !fullOK { + return nil + } + if solidityOK && full == solidity && !hasHTTPOverride(node, "solidityPort") { + return fmt.Errorf("rendered HTTP port conflict: fullNodePort=%d collides with solidityPort (in-container). Set ports.solidity_http in the intent (or config_overrides \"node.http.solidityPort\") to a different port", full) + } + if pbftOK && full == pbft && !hasHTTPOverride(node, "PBFTPort") { + return fmt.Errorf("rendered HTTP port conflict: fullNodePort=%d collides with PBFTPort (in-container). Set config_overrides \"node.http.PBFTPort\" to a different port", full) + } + return nil +} + +func hasHTTPOverride(node *intent.NodeSpec, key string) bool { + _, ok := node.ConfigOverrides["node.http."+key] + return ok +} + +// hoconPortValue returns the first active integer assignment for key. Port +// names occur in both HTTP and RPC sections; the first occurrence is the HTTP +// template value, matching replaceHOCONValue's existing behavior. +func hoconPortValue(config, key string) (int, bool) { + for _, line := range strings.Split(config, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") || + (!strings.HasPrefix(trimmed, key+" =") && !strings.HasPrefix(trimmed, key+"=")) { + continue + } + value := strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(trimmed, key+" ="), key+"=")) + var port int + if _, err := fmt.Sscanf(value, "%d", &port); err == nil { + return port, true + } + } + return 0, false +} + // applyFeatureOverrides enables/disables features in the HOCON config. func applyFeatureOverrides(config string, node *intent.NodeSpec) string { features := node.Features diff --git a/internal/render/hocon_test.go b/internal/render/hocon_test.go index 863aa297..7a86b232 100644 --- a/internal/render/hocon_test.go +++ b/internal/render/hocon_test.go @@ -54,6 +54,96 @@ func TestRenderHOCON_PortOverrides(t *testing.T) { } } +func TestRenderHOCON_HTTPPortCollisionAvoidance(t *testing.T) { + base := &intent.Intent{Name: "ports", Network: "mainnet", Target: intent.Target{Type: "local"}} + for _, tt := range []struct { + name string + http int + messageHas string + }{{"solidity default collision", 8091, "ports.solidity_http"}, {"PBFT default collision", 8092, "node.http.PBFTPort"}} { + t.Run(tt.name, func(t *testing.T) { + node := &intent.NodeSpec{Type: "fullnode", Ports: intent.PortMapping{HTTP: tt.http}} + _, err := RenderHOCON("", base, node) + if err == nil || !strings.Contains(err.Error(), tt.messageHas) || !strings.Contains(err.Error(), "different port") { + t.Fatalf("expected actionable collision error, got %v", err) + } + }) + } +} + +func TestRenderHOCON_HTTPPortCollisionExplicitSolidity(t *testing.T) { + // config_overrides is the explicit escape hatch for operators who + // intentionally need the equal value. + node := &intent.NodeSpec{Type: "fullnode", Ports: intent.PortMapping{HTTP: 8091, SolidityHTTP: 8091}} + out, err := RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, node) + if err == nil || !strings.Contains(err.Error(), "solidityPort") { + t.Fatalf("expected solidity collision error, got %v", err) + } + + node.ConfigOverrides = map[string]any{"node.http.solidityPort": 8091} + out, err = RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, node) + if err != nil { + t.Fatalf("render with override: %v", err) + } + if !strings.Contains(out, "solidityPort = 8091") { + t.Error("config_overrides solidity port escape hatch was changed") + } +} + +func TestRenderHOCON_HTTPPortCollisionThroughIntentParse(t *testing.T) { + data := []byte(`name: pipeline-port-test +target: + type: local + runtime: docker +network: mainnet +nodes: + - type: fullnode + version: latest + ports: + http: 8091 +`) + i, err := intent.Parse(data) + if err != nil { + t.Fatalf("parse intent: %v", err) + } + if i.Nodes[0].Ports.SolidityHTTP != 8091 { + t.Fatalf("ApplyDefaults did not fill solidity HTTP port: %d", i.Nodes[0].Ports.SolidityHTTP) + } + + if _, err := RenderHOCON("", i, &i.Nodes[0]); err == nil || !strings.Contains(err.Error(), "solidityPort") { + t.Fatalf("expected parsed collision error, got %v", err) + } +} + +func TestHTTPPortCollisionCheck(t *testing.T) { + config := "fullNodePort = 65534\nsolidityPort = 65534\nPBFTPort = 65534\n" + node := &intent.NodeSpec{Type: "fullnode"} + if err := checkHTTPPortConflicts(config, node); err == nil { + t.Fatal("expected collision error") + } + if err := checkHTTPPortConflicts("fullNodePort = 8090\n", node); err != nil { + t.Fatalf("missing service keys should not error: %v", err) + } + if err := checkHTTPPortConflicts(config, &intent.NodeSpec{ConfigOverrides: map[string]any{ + "node.http.solidityPort": 65534, + "node.http.PBFTPort": 65534, + }}); err != nil { + t.Fatalf("explicit override should suppress error: %v", err) + } +} + +func TestRenderHOCON_DefaultHTTPPortsUnchanged(t *testing.T) { + out, err := RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, &intent.NodeSpec{Type: "fullnode"}) + if err != nil { + t.Fatalf("render: %v", err) + } + for _, want := range []string{"fullNodePort = 8090", "solidityPort = 8091", "PBFTPort = 8092"} { + if !strings.Contains(out, want) { + t.Errorf("default port missing or changed: %s", want) + } + } +} + // TestRenderHOCON_JSONRPCPortAndEnable locks down the #165 fix: // features.jsonrpc=true + ports.jsonrpc=NNNNN must produce BOTH // `httpFullNodeEnable = true` (already worked) AND From 5c578b31d386e95c1f57325664b5d5bc68689596 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 18 Aug 2026 15:21:06 +0800 Subject: [PATCH 03/49] fix(target): probe ssh nodes through the tunnel; report real host in endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verify/health/diagnose/wait and MCP health probed ssh nodes either via t.Exec('curl') — rejected by the ssh command allowlist — or by dialing 127.0.0.1 from the trond host, bypassing ssh entirely. On a firewalled target (22 only) every liveness/sync probe failed while the node was healthy; status/apply/inspect also reported endpoints as 127.0.0.1. internal/target now offers a target-aware transport: an optional Dialer interface (SSHTarget.DialContext = direct-tcpip over the existing ssh client, restricted to loopback addrs; LocalTarget = net.Dialer), plus HTTPClient/Get/Post/DialContext helpers with an enforceable timeout (preemptable even when the underlying dial ignores ctx). Probes migrated: verify, health, diagnose sync/peers/version/ports, apply LiveStatus jar branch, wait --port/--http, MCP health. Endpoint reporting (status, apply created/updated/no_change, inspect, network add/create, MCP status/ endpoints/monitoring, healthTool display) now uses EndpointHost so ssh rigs report the target host. preflight checkPorts now dials through the target too — it previously checked the local machine for ssh targets. Found in TX-167 functional testing. --- cmd/health.go | 4 +- cmd/inspect.go | 6 +- cmd/network/add.go | 4 +- cmd/network/create.go | 4 +- cmd/preflight.go | 6 +- cmd/status.go | 4 +- cmd/verify.go | 3 +- cmd/wait.go | 18 ++- internal/apply/apply.go | 17 +-- internal/apply/observe_test.go | 116 +++++++++++++----- internal/apply/probe.go | 18 ++- internal/diagnosis/peers.go | 3 +- internal/diagnosis/ports.go | 21 ++-- internal/diagnosis/sync.go | 3 +- internal/diagnosis/version.go | 3 +- internal/mcp/helpers.go | 14 ++- internal/mcp/resources.go | 6 +- internal/mcp/tools_diagnostic.go | 23 +++- internal/mcp/tools_inspection.go | 4 +- internal/target/local.go | 5 + internal/target/ssh.go | 17 +++ internal/target/target.go | 102 ++++++++++++++++ internal/target/target_test.go | 204 +++++++++++++++++++++++++++++++ 23 files changed, 510 insertions(+), 95 deletions(-) create mode 100644 internal/target/target_test.go diff --git a/cmd/health.go b/cmd/health.go index d853a99b..dad3674e 100644 --- a/cmd/health.go +++ b/cmd/health.go @@ -3,10 +3,12 @@ package cmd import ( "encoding/json" "fmt" + "time" "github.com/spf13/cobra" "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/target" ) var healthCmd = &cobra.Command{ @@ -39,7 +41,7 @@ func runHealth(cmd *cobra.Command, args []string) error { } url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnowblock", httpPort) - out, err := nc.Target.Exec(cmd.Context(), "curl", "-s", "--max-time", "5", url) + out, err := target.Get(cmd.Context(), nc.Target, url, 5*time.Second) if err != nil { result := map[string]any{ "name": name, diff --git a/cmd/inspect.go b/cmd/inspect.go index 909304d1..b5590531 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -13,6 +13,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) func stdoutWriter() io.Writer { return os.Stdout } @@ -166,11 +167,12 @@ func buildManifest(ctx context.Context, nodes []state.ManagedNode) map[string]an func manifestForNode(ctx context.Context, n *state.ManagedNode) map[string]any { endpoints := map[string]string{} + host := target.EndpointHost(n.Target.Type, n.Target.Host) if n.HTTPPort != 0 { - endpoints["http"] = fmt.Sprintf("http://127.0.0.1:%d", n.HTTPPort) + endpoints["http"] = fmt.Sprintf("http://%s:%d", host, n.HTTPPort) } if n.GRPCPort != 0 { - endpoints["grpc"] = fmt.Sprintf("127.0.0.1:%d", n.GRPCPort) + endpoints["grpc"] = fmt.Sprintf("%s:%d", host, n.GRPCPort) } entry := map[string]any{ diff --git a/cmd/network/add.go b/cmd/network/add.go index b732e968..4c9a43f0 100644 --- a/cmd/network/add.go +++ b/cmd/network/add.go @@ -246,8 +246,8 @@ func runAdd(cmd *cobra.Command, args []string) error { "network": addNetworkName, "added": nodeName, "endpoints": map[string]string{ - "http": fmt.Sprintf("http://127.0.0.1:%d", node.Ports.HTTP), - "grpc": fmt.Sprintf("127.0.0.1:%d", node.Ports.GRPC), + "http": fmt.Sprintf("http://%s:%d", target.EndpointHost(parsed.Target.Type, parsed.Target.Host), node.Ports.HTTP), + "grpc": fmt.Sprintf("%s:%d", target.EndpointHost(parsed.Target.Type, parsed.Target.Host), node.Ports.GRPC), }, } output.WriteJSON(os.Stdout, result) diff --git a/cmd/network/create.go b/cmd/network/create.go index f083b0e6..483b72bf 100644 --- a/cmd/network/create.go +++ b/cmd/network/create.go @@ -326,10 +326,10 @@ func deployNetworkMonitoring(ctx context.Context, tgt target.Target, workDir str } urls := map[string]string{ - "grafana_url": fmt.Sprintf("http://127.0.0.1:%d", parsed.Monitoring.Grafana.Port), + "grafana_url": fmt.Sprintf("http://%s:%d", target.EndpointHost(parsed.Target.Type, parsed.Target.Host), parsed.Monitoring.Grafana.Port), } if parsed.Monitoring.Prometheus.Port > 0 { - urls["prometheus_url"] = fmt.Sprintf("http://127.0.0.1:%d", parsed.Monitoring.Prometheus.Port) + urls["prometheus_url"] = fmt.Sprintf("http://%s:%d", target.EndpointHost(parsed.Target.Type, parsed.Target.Host), parsed.Monitoring.Prometheus.Port) } return monitoringResult{ urls: urls, diff --git a/cmd/preflight.go b/cmd/preflight.go index 5928ef0b..3b5b1260 100644 --- a/cmd/preflight.go +++ b/cmd/preflight.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "net" "os" "strings" "time" @@ -233,7 +232,7 @@ func checkMemoryRecommended(parsed *intent.Intent) []checkResult { // so the check silently reported every port as available. Use net.Dial // instead — same behaviour as the diagnose port_listening checker, no // runtime dependency. -func checkPorts(_ *cobra.Command, _ target.Target, node *intent.NodeSpec) []checkResult { +func checkPorts(cmd *cobra.Command, tgt target.Target, node *intent.NodeSpec) []checkResult { ports := []struct { name string port int @@ -243,13 +242,12 @@ func checkPorts(_ *cobra.Command, _ target.Target, node *intent.NodeSpec) []chec {"p2p", node.Ports.P2P}, } - dialer := net.Dialer{Timeout: 1500 * time.Millisecond} var results []checkResult for _, p := range ports { if p.port == 0 { continue } - conn, err := dialer.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", p.port)) + conn, err := target.DialContext(cmd.Context(), tgt, "tcp", fmt.Sprintf("127.0.0.1:%d", p.port), 1500*time.Millisecond) if err == nil { _ = conn.Close() results = append(results, checkResult{ diff --git a/cmd/status.go b/cmd/status.go index 852bff26..dc4ed2e2 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -89,8 +89,8 @@ func runStatus(cmd *cobra.Command, args []string) error { "healthy": false, "logs": apply.LogsDescriptor(node), "api_endpoints": map[string]any{ - "http": fmt.Sprintf("http://127.0.0.1:%d", effectivePort(node.HTTPPort, 8090)), - "grpc": fmt.Sprintf("127.0.0.1:%d", effectivePort(node.GRPCPort, 50051)), + "http": fmt.Sprintf("http://%s:%d", target.EndpointHost(node.Target.Type, node.Target.Host), effectivePort(node.HTTPPort, 8090)), + "grpc": fmt.Sprintf("%s:%d", target.EndpointHost(node.Target.Type, node.Target.Host), effectivePort(node.GRPCPort, 50051)), }, } diff --git a/cmd/verify.go b/cmd/verify.go index 0550fa53..3d01408d 100644 --- a/cmd/verify.go +++ b/cmd/verify.go @@ -11,6 +11,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) var ( @@ -57,7 +58,7 @@ func runVerify(cmd *cobra.Command, args []string) error { for time.Now().Before(deadline) { attempt++ url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnowblock", httpPort) - out, err := tgt.Exec(cmd.Context(), "curl", "-s", "--max-time", "5", url) + out, err := target.Get(cmd.Context(), tgt, url, 5*time.Second) if err == nil { var block struct { BlockHeader struct { diff --git a/cmd/wait.go b/cmd/wait.go index 13bbb619..acb7c8a2 100644 --- a/cmd/wait.go +++ b/cmd/wait.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net" "os" "strings" "time" @@ -184,9 +183,8 @@ func expandHTTPURL(raw string, port int) string { return strings.ReplaceAll(raw, "{http}", fmt.Sprintf("http://127.0.0.1:%d", port)) } -func probeTCP(ctx context.Context, _ target.Target, port int) error { - d := net.Dialer{Timeout: 2 * time.Second} - conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("127.0.0.1:%d", port)) +func probeTCP(ctx context.Context, tgt target.Target, port int) error { + conn, err := target.DialContext(ctx, tgt, "tcp", fmt.Sprintf("127.0.0.1:%d", port), 2*time.Second) if err != nil { return err } @@ -212,8 +210,16 @@ func httpProbeURL(raw string) (string, error) { } func probeHTTP(ctx context.Context, nc *nodeContext, url string) error { - args := []string{"-fsS", "--max-time", "5", url} - out, err := nc.runtimeExec(ctx, "curl", args...) + var out []byte + var err error + probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if nc.Node.Runtime == "jar" { + out, err = target.Get(probeCtx, nc.Target, url, 5*time.Second) + } else { + args := []string{"-fsS", "--max-time", "5", url} + out, err = nc.runtimeExec(ctx, "curl", args...) + } if err != nil { return fmt.Errorf("curl: %w", err) } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6881e402..d8c72795 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -418,8 +418,8 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { Network: opts.Intent.Network, IsPrivate: intent.IsPrivate(opts.Intent.Network), Endpoints: map[string]string{ - "http": fmt.Sprintf("http://127.0.0.1:%d", node.Ports.HTTP), - "grpc": fmt.Sprintf("127.0.0.1:%d", node.Ports.GRPC), + "http": fmt.Sprintf("http://%s:%d", target.EndpointHost(opts.Intent.Target.Type, opts.Intent.Target.Host), node.Ports.HTTP), + "grpc": fmt.Sprintf("%s:%d", target.EndpointHost(opts.Intent.Target.Type, opts.Intent.Target.Host), node.Ports.GRPC), }, DurationMs: deployedMs, Build: buildSummary, @@ -562,6 +562,7 @@ func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) * } ports := opts.Intent.Nodes[0].Ports + host := target.EndpointHost(opts.Intent.Target.Type, opts.Intent.Target.Host) return &Result{ Name: opts.Intent.Name, Outcome: "no_change", @@ -572,24 +573,24 @@ func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) * Network: opts.Intent.Network, IsPrivate: intent.IsPrivate(opts.Intent.Network), Endpoints: map[string]string{ - "http": fmt.Sprintf("http://127.0.0.1:%d", ports.HTTP), - "grpc": fmt.Sprintf("127.0.0.1:%d", ports.GRPC), + "http": fmt.Sprintf("http://%s:%d", host, ports.HTTP), + "grpc": fmt.Sprintf("%s:%d", host, ports.GRPC), }, DurationMs: time.Since(start).Milliseconds(), Build: buildSummary, - MonitoringEndpoints: monitoringEndpointsFromExisting(opts.Existing), + MonitoringEndpoints: monitoringEndpointsFromExisting(opts.Existing, host), } } // monitoringEndpointsFromExisting returns monitoring URLs from a managed // node's saved state, if monitoring was deployed for it. -func monitoringEndpointsFromExisting(existing *state.ManagedNode) map[string]string { +func monitoringEndpointsFromExisting(existing *state.ManagedNode, host string) map[string]string { if existing == nil || existing.Monitoring == nil || !existing.Monitoring.Enabled { return nil } return map[string]string{ - "prometheus_url": fmt.Sprintf("http://127.0.0.1:%d", existing.Monitoring.PrometheusPort), - "grafana_url": fmt.Sprintf("http://127.0.0.1:%d", existing.Monitoring.GrafanaPort), + "prometheus_url": fmt.Sprintf("http://%s:%d", host, existing.Monitoring.PrometheusPort), + "grafana_url": fmt.Sprintf("http://%s:%d", host, existing.Monitoring.GrafanaPort), } } diff --git a/internal/apply/observe_test.go b/internal/apply/observe_test.go index 2e1a8fc9..91f59871 100644 --- a/internal/apply/observe_test.go +++ b/internal/apply/observe_test.go @@ -2,6 +2,12 @@ package apply import ( "context" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" "strings" "testing" @@ -23,6 +29,43 @@ func newExecTarget(fn func(ctx context.Context, name string, args ...string) ([] return &execTarget{fakeTarget: &fakeTarget{}, exec: fn} } +// httpTarget is a fakeTarget whose DialContext redirects loopback probes to +// a test HTTP server. Jar-runtime LiveStatus probes go through the +// target-aware HTTP client (target.Get/Post over DialContext), so the fake +// must serve real HTTP rather than script Exec output. +type httpTarget struct { + *fakeTarget + server *httptest.Server +} + +func (h *httpTarget) DialContext(ctx context.Context, network, _ string) (net.Conn, error) { + addr := strings.TrimPrefix(h.server.URL, "http://") + return (&net.Dialer{}).DialContext(ctx, network, addr) +} + +// newHTTPStatusTarget starts an httptest server that answers the LiveStatus +// probe endpoints and returns a target wired to it plus the port LiveStatus +// should probe. handler may return ("", true) to fail a request. +func newHTTPStatusTarget(t *testing.T, respond func(path string, body []byte) (string, bool)) (*httpTarget, int) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + reqBody, _ := io.ReadAll(r.Body) + out, fail := respond(r.URL.Path, reqBody) + if fail { + w.WriteHeader(http.StatusBadGateway) + return + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, out) + })) + t.Cleanup(srv.Close) + port, err := strconv.Atoi(strings.TrimPrefix(srv.URL, "http://127.0.0.1:")) + if err != nil { + t.Fatalf("parse httptest port: %v", err) + } + return &httpTarget{fakeTarget: &fakeTarget{}, server: srv}, port +} + func TestLogsDescriptor_Docker(t *testing.T) { node := &state.ManagedNode{Name: "fn0", Runtime: "docker"} got := LogsDescriptor(node) @@ -102,17 +145,16 @@ func TestContainerID_NilTarget(t *testing.T) { } func TestLiveStatus_SetsHealthy(t *testing.T) { - node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: 8090} - tgt := newExecTarget(func(_ context.Context, _ string, args ...string) ([]byte, error) { - url := args[len(args)-1] - switch { - case strings.Contains(url, "/wallet/getnowblock"): - return []byte(`{"block_header":{"raw_data":{"number":42,"timestamp":1}}}`), nil - case strings.Contains(url, "/wallet/listnodes"): - return []byte(`{"nodes":[{},{}]}`), nil + tgt, port := newHTTPStatusTarget(t, func(path string, _ []byte) (string, bool) { + switch path { + case "/wallet/getnowblock": + return `{"block_header":{"raw_data":{"number":42,"timestamp":1}}}`, false + case "/wallet/listnodes": + return `{"nodes":[{},{}]}`, false } - return nil, context.DeadlineExceeded + return "", true }) + node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: port} out := LiveStatus(context.Background(), tgt, node) if out["healthy"] != true { t.Errorf("healthy = %v, want true (RPC answered with a parseable block)", out["healthy"]) @@ -144,38 +186,47 @@ func TestIsHex64(t *testing.T) { func TestLiveStatus_GenesisBlockID(t *testing.T) { blockJSON := `{"block_header":{"raw_data":{"number":3,"timestamp":1}}}` - mk := func(genesisBody string, genesisErr bool) *execTarget { - return newExecTarget(func(_ context.Context, _ string, args ...string) ([]byte, error) { - url := args[len(args)-1] - switch { - case strings.Contains(url, "/wallet/getblockbynum"): + mk := func(genesisBody string, genesisErr bool) (*httpTarget, int) { + return newHTTPStatusTarget(t, func(path string, _ []byte) (string, bool) { + switch path { + case "/wallet/getblockbynum": if genesisErr { - return nil, context.DeadlineExceeded + return "", true } - return []byte(genesisBody), nil - case strings.Contains(url, "/wallet/getnowblock"): - return []byte(blockJSON), nil + return genesisBody, false + case "/wallet/getnowblock": + return blockJSON, false } - return nil, context.DeadlineExceeded + return "", true }) } - node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: 8090} // Valid blockID → surfaced. - out := LiveStatus(context.Background(), mk(`{"blockID":"`+genesisBlockID+`","block_header":{}}`, false), node) + tgt, port := newHTTPStatusTarget(t, func(path string, _ []byte) (string, bool) { + if path == "/wallet/getblockbynum" { + return `{"blockID":"` + genesisBlockID + `","block_header":{}}`, false + } + if path == "/wallet/getnowblock" { + return blockJSON, false + } + return "", true + }) + out := LiveStatus(context.Background(), tgt, &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: port}) if out["genesis_block_id"] != genesisBlockID { t.Errorf("genesis_block_id = %v, want %s", out["genesis_block_id"], genesisBlockID) } // Malformed blockID (not 64-hex) → absent. - out = LiveStatus(context.Background(), mk(`{"blockID":"nope"}`, false), node) + tgt, port = mk(`{"blockID":"nope"}`, false) + out = LiveStatus(context.Background(), tgt, &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: port}) if _, ok := out["genesis_block_id"]; ok { t.Errorf("genesis_block_id must be absent for a malformed blockID; got %v", out["genesis_block_id"]) } // Probe failed (node down) → absent, but the primary healthy signal // (from getnowblock) is unaffected — proves it doesn't starve them. - out = LiveStatus(context.Background(), mk("", true), node) + tgt, port = mk("", true) + out = LiveStatus(context.Background(), tgt, &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: port}) if _, ok := out["genesis_block_id"]; ok { t.Errorf("genesis_block_id must be absent when the probe fails; got %v", out["genesis_block_id"]) } @@ -185,11 +236,10 @@ func TestLiveStatus_GenesisBlockID(t *testing.T) { } func TestLiveStatus_NoHealthyOnProbeFailure(t *testing.T) { - node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: 8090} - tgt := newExecTarget(func(_ context.Context, _ string, _ ...string) ([]byte, error) { - return nil, context.DeadlineExceeded // node down: every probe fails - }) - out := LiveStatus(context.Background(), tgt, node) + // No listener behind the target: every dial fails, so every probe errs + // (a node that is down behaves the same over the tunnel). + node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: 1} + out := LiveStatus(context.Background(), &fakeTarget{}, node) if _, ok := out["healthy"]; ok { t.Errorf("healthy must be absent when the probe fails (caller seeds false); got %v", out["healthy"]) } @@ -201,13 +251,13 @@ func TestLiveStatus_NoHealthyOnProbeFailure(t *testing.T) { // not fabricate a block_height of 0. func TestLiveStatus_NoHealthyOnEmptyOrErrorBody(t *testing.T) { for _, body := range []string{`{}`, `{"Error":"some node error"}`, `{"block_header":{"raw_data":{"number":0,"timestamp":0}}}`} { - node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: 8090} - tgt := newExecTarget(func(_ context.Context, _ string, args ...string) ([]byte, error) { - if strings.Contains(args[len(args)-1], "/wallet/getnowblock") { - return []byte(body), nil + tgt, port := newHTTPStatusTarget(t, func(path string, _ []byte) (string, bool) { + if path == "/wallet/getnowblock" { + return body, false } - return nil, context.DeadlineExceeded + return "", true }) + node := &state.ManagedNode{Name: "fn0", Runtime: "jar", HTTPPort: port} out := LiveStatus(context.Background(), tgt, node) if _, ok := out["healthy"]; ok { t.Errorf("body %q: healthy must be absent (no real block); got %v", body, out["healthy"]) diff --git a/internal/apply/probe.go b/internal/apply/probe.go index db9b817b..eeccb49b 100644 --- a/internal/apply/probe.go +++ b/internal/apply/probe.go @@ -35,19 +35,25 @@ func LiveStatus(ctx context.Context, tgt target.Target, node *state.ManagedNode) port = 8090 } - // probe issues a curl against the node's HTTP API. body=="" is a GET - // (TRON endpoints that take no params, e.g. getnowblock); a non-empty - // body is POSTed as JSON (e.g. getblockbynum needs {"num":N}). + // probe issues a request against the node's HTTP API. body=="" is a + // GET (TRON endpoints that take no params, e.g. getnowblock); a + // non-empty body is POSTed as JSON (e.g. getblockbynum needs + // {"num":N}). Jar nodes go through the target-aware HTTP client + // (SSH-tunnelled for remote targets); docker nodes curl inside the + // container via `docker exec`. probe := func(path, body string) ([]byte, error) { url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path) + if node.Runtime == "jar" { + if body == "" { + return target.Get(ctx, tgt, url, 2*time.Second) + } + return target.Post(ctx, tgt, url, []byte(body), 2*time.Second) + } args := []string{"-fsS", "--max-time", "2"} if body != "" { args = append(args, "-X", "POST", "-H", "Content-Type: application/json", "-d", body) } args = append(args, url) - if node.Runtime == "jar" { - return tgt.Exec(ctx, "curl", args...) - } return tgt.Exec(ctx, "docker", append([]string{"exec", node.Name, "curl"}, args...)...) } diff --git a/internal/diagnosis/peers.go b/internal/diagnosis/peers.go index 8f6bd6fb..28768448 100644 --- a/internal/diagnosis/peers.go +++ b/internal/diagnosis/peers.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -19,7 +20,7 @@ func (c *PeersChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpt } url := fmt.Sprintf("http://127.0.0.1:%d/wallet/listnodes", opts.HTTPPort) - out, err := tgt.Exec(ctx, "curl", "-s", "--max-time", "5", url) + out, err := target.Get(ctx, tgt, url, 5*time.Second) if err != nil { return CheckResult{ Name: c.Name(), diff --git a/internal/diagnosis/ports.go b/internal/diagnosis/ports.go index 9e936cb5..418ea54c 100644 --- a/internal/diagnosis/ports.go +++ b/internal/diagnosis/ports.go @@ -3,7 +3,6 @@ package diagnosis import ( "context" "fmt" - "net" "time" "github.com/tronprotocol/tron-deployment/internal/target" @@ -11,30 +10,30 @@ import ( // PortsChecker verifies expected ports are accepting TCP connections. // -// We probe via net.Dial against 127.0.0.1 from the host where trond is -// running, instead of running `ss -tlnp` inside the target. This works -// the same on Linux and macOS (the previous `ss` invocation silently -// returned empty on Darwin, marking every port as "not listening" even -// when java-tron was healthy and serving traffic). For docker-runtime -// nodes, the host-side mapped port is exactly what test harnesses care -// about reaching, so this is also the right thing semantically. +// We dial 127.0.0.1 through the target (SSH direct-tcpip when the target +// is remote, a plain host dial for local), instead of running `ss -tlnp` +// inside the target. This works the same on Linux and macOS (the previous +// `ss` invocation silently returned empty on Darwin, marking every port +// as "not listening" even when java-tron was healthy and serving traffic). +// For docker-runtime nodes, the host-side mapped port is exactly what test +// harnesses care about reaching, so this is also the right thing +// semantically. type PortsChecker struct{} func (c *PortsChecker) Name() string { return "port_listening" } -func (c *PortsChecker) Run(ctx context.Context, _ target.Target, opts CheckOpts) CheckResult { +func (c *PortsChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpts) CheckResult { ports := []int{opts.HTTPPort, opts.GRPCPort} if opts.HTTPPort == 0 && opts.GRPCPort == 0 { ports = []int{8090, 50051} } - dialer := net.Dialer{Timeout: 1500 * time.Millisecond} var missing []int for _, port := range ports { if port == 0 { continue } - conn, err := dialer.DialContext(ctx, "tcp", fmt.Sprintf("127.0.0.1:%d", port)) + conn, err := target.DialContext(ctx, tgt, "tcp", fmt.Sprintf("127.0.0.1:%d", port), 1500*time.Millisecond) if err != nil { missing = append(missing, port) continue diff --git a/internal/diagnosis/sync.go b/internal/diagnosis/sync.go index 7cc73183..cc96f066 100644 --- a/internal/diagnosis/sync.go +++ b/internal/diagnosis/sync.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "time" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -20,7 +21,7 @@ func (c *SyncChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpts } url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnowblock", opts.HTTPPort) - out, err := tgt.Exec(ctx, "curl", "-s", "--max-time", "5", url) + out, err := target.Get(ctx, tgt, url, 5*time.Second) if err != nil { return CheckResult{ Name: c.Name(), diff --git a/internal/diagnosis/version.go b/internal/diagnosis/version.go index 935ede7d..d874d988 100644 --- a/internal/diagnosis/version.go +++ b/internal/diagnosis/version.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strings" + "time" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -21,7 +22,7 @@ func (c *VersionChecker) Run(ctx context.Context, tgt target.Target, opts CheckO } url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnodeinfo", opts.HTTPPort) - out, err := tgt.Exec(ctx, "curl", "-s", "--max-time", "5", url) + out, err := target.Get(ctx, tgt, url, 5*time.Second) if err != nil { return CheckResult{ Name: c.Name(), diff --git a/internal/mcp/helpers.go b/internal/mcp/helpers.go index 27822675..be1b9655 100644 --- a/internal/mcp/helpers.go +++ b/internal/mcp/helpers.go @@ -29,14 +29,16 @@ func notFoundWithSuggestions(resource, name string, suggestions ...string) *outp WithSuggestions(suggestions...) } -// httpURL formats a port into the http://127.0.0.1:

URL we surface +// httpURL formats host+port into the http://:

URL we surface // to agents. Agents can re-use this in their own follow-up probes -// (e.g. `wait --http `). -func httpURL(port int) string { - return "http://127.0.0.1:" + strconv.Itoa(port) +// (e.g. `wait --http `). For ssh targets the host is the recorded +// remote host; internal probes that dial through the target pass +// 127.0.0.1 (the tunnel lands on the remote loopback). +func httpURL(host string, port int) string { + return "http://" + host + ":" + strconv.Itoa(port) } // grpcAddr formats the host:port grpc endpoint. -func grpcAddr(port int) string { - return "127.0.0.1:" + strconv.Itoa(port) +func grpcAddr(host string, port int) string { + return host + ":" + strconv.Itoa(port) } diff --git a/internal/mcp/resources.go b/internal/mcp/resources.go index 0f7b72a8..8c10e624 100644 --- a/internal/mcp/resources.go +++ b/internal/mcp/resources.go @@ -11,6 +11,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) // registerResources wires read-only data sources as MCP resources. @@ -140,13 +141,14 @@ func readNodeEndpointsResource(_ context.Context, req *mcp.ReadResourceRequest) if err != nil { return nil, err } + host := target.EndpointHost(node.Target.Type, node.Target.Host) endpoints := map[string]any{ "name": node.Name, "runtime": node.Runtime, "target": node.Target, "endpoints": map[string]string{ - "http": fmt.Sprintf("http://127.0.0.1:%d", node.HTTPPort), - "grpc": fmt.Sprintf("127.0.0.1:%d", node.GRPCPort), + "http": fmt.Sprintf("http://%s:%d", host, node.HTTPPort), + "grpc": fmt.Sprintf("%s:%d", host, node.GRPCPort), }, "version": node.Version, "labels": node.Labels, diff --git a/internal/mcp/tools_diagnostic.go b/internal/mcp/tools_diagnostic.go index 2ee5946a..d94e32fe 100644 --- a/internal/mcp/tools_diagnostic.go +++ b/internal/mcp/tools_diagnostic.go @@ -14,6 +14,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/diagnosis" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) // registerDiagnosticTools wires up read-only triage tools. Pure-data @@ -169,7 +170,21 @@ func healthTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp if port == 0 { port = 8090 } - url := httpURL(port) + "/wallet/getnowblock" + // The probe URL stays on loopback: target.HTTPClient dials through the + // SSH tunnel, which lands on the remote host's loopback. + url := httpURL("127.0.0.1", port) + "/wallet/getnowblock" + endpoint := httpURL(target.EndpointHost(node.Target.Type, node.Target.Host), port) + "/wallet/getnowblock" + + // Probe through the node's target (SSH-tunnelled for remote nodes), + // not via http.DefaultClient against this host's loopback — that + // bypassed SSH entirely and always failed for remote rigs. + tgt, err := mcpResolveTargetFromNode(node) + if err != nil { + return errResult(err) + } + if c, ok := any(tgt).(interface{ Close() error }); ok { + defer c.Close() + } probeCtx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() @@ -178,11 +193,11 @@ func healthTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp return errResult(err) } start := time.Now() - resp, err := http.DefaultClient.Do(req) + resp, err := target.HTTPClient(tgt, 5*time.Second).Do(req) if err != nil { return jsonResult(map[string]any{ "name": args.Name, "healthy": false, - "endpoint": url, "error": err.Error(), + "endpoint": endpoint, "error": err.Error(), }) } defer resp.Body.Close() @@ -190,7 +205,7 @@ func healthTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp return jsonResult(map[string]any{ "name": args.Name, "healthy": healthy, - "endpoint": url, + "endpoint": endpoint, "latency_ms": time.Since(start).Milliseconds(), "status": resp.StatusCode, }) diff --git a/internal/mcp/tools_inspection.go b/internal/mcp/tools_inspection.go index 47d6f35e..3e29181e 100644 --- a/internal/mcp/tools_inspection.go +++ b/internal/mcp/tools_inspection.go @@ -205,10 +205,10 @@ func inspectAllNodes(ctx context.Context, _ *mcp.CallToolRequest, _ emptyArgs) ( // extraction. eps := map[string]string{} if n.HTTPPort != 0 { - eps["http"] = httpURL(n.HTTPPort) + eps["http"] = httpURL(target.EndpointHost(n.Target.Type, n.Target.Host), n.HTTPPort) } if n.GRPCPort != 0 { - eps["grpc"] = grpcAddr(n.GRPCPort) + eps["grpc"] = grpcAddr(target.EndpointHost(n.Target.Type, n.Target.Host), n.GRPCPort) } if len(eps) > 0 { row["endpoints"] = eps diff --git a/internal/target/local.go b/internal/target/local.go index af15f52f..91566d7c 100644 --- a/internal/target/local.go +++ b/internal/target/local.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "net" "os" "os/exec" "path/filepath" @@ -17,6 +18,10 @@ import ( // LocalTarget executes commands and file operations on the local machine. type LocalTarget struct{} +func (t *LocalTarget) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, addr) +} + // NewLocalTarget creates a new LocalTarget. func NewLocalTarget() *LocalTarget { return &LocalTarget{} diff --git a/internal/target/ssh.go b/internal/target/ssh.go index c9f0db3a..fe2f68be 100644 --- a/internal/target/ssh.go +++ b/internal/target/ssh.go @@ -35,6 +35,23 @@ type SSHTarget struct { client *ssh.Client } +func (t *SSHTarget) DialContext(_ context.Context, network, addr string) (net.Conn, error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("ssh direct dial requires host:port address: %w", err) + } + if host != "localhost" { + ip := net.ParseIP(host) + if ip == nil || !ip.IsLoopback() { + return nil, fmt.Errorf("ssh direct dial address %q is not loopback", addr) + } + } + if t.client == nil { + return nil, fmt.Errorf("ssh not connected") + } + return t.client.Dial(network, addr) +} + // NewSSHTarget creates a new SSHTarget. Call Connect() before use. func NewSSHTarget(host string, port int, user, identityFile string) *SSHTarget { if port == 0 { diff --git a/internal/target/target.go b/internal/target/target.go index a87a66d7..7782d912 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -1,10 +1,112 @@ package target import ( + "bytes" "context" + "fmt" + "io" + "net" + "net/http" "os" + "time" ) +// Dialer is optionally implemented by targets that can reach their own +// loopback through an existing connection (for example, SSH direct-tcpip). +// DialContext callers must pass an address on the target's own loopback. +type Dialer interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) +} + +// HTTPClient returns an HTTP client that reaches node-local addresses through +// the target when it supports dialing. Local and test targets use the normal +// host transport. +func HTTPClient(t Target, timeout time.Duration) *http.Client { + client := &http.Client{Timeout: timeout} + if d, ok := t.(Dialer); ok { + client.Transport = &http.Transport{DialContext: d.DialContext} + } + return client +} + +// EndpointHost returns the host used when reporting a target endpoint. +func EndpointHost(targetType, host string) string { + if targetType == "ssh" && host != "" { + return host + } + return "127.0.0.1" +} + +// DialContext dials through the target when it implements Dialer (reaching +// the target's own loopback, e.g. over an SSH direct-tcpip channel), and +// falls back to a plain net.Dialer otherwise. +func DialContext(ctx context.Context, t Target, network, addr string, timeout time.Duration) (net.Conn, error) { + if d, ok := t.(Dialer); ok { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + type dialResult struct { + conn net.Conn + err error + } + ch := make(chan dialResult, 1) + go func() { + conn, err := d.DialContext(ctx, network, addr) + ch <- dialResult{conn, err} + }() + select { + case r := <-ch: + return r.conn, r.err + case <-ctx.Done(): + // A dial that succeeds after timeout may leave an unowned conn; + // the SSH/session lifetime will reclaim it. This is an accepted + // tradeoff for making the timeout preemptible. + return nil, ctx.Err() + } + } + return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr) +} + +// Get issues an HTTP GET through the target-aware client and returns the +// response body. Non-2xx statuses are an error carrying a body snippet. +func Get(ctx context.Context, t Target, url string, timeout time.Duration) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + return do(HTTPClient(t, timeout), req) +} + +// Post issues an HTTP POST with a JSON body through the target-aware client +// and returns the response body. Non-2xx statuses are an error. +func Post(ctx context.Context, t Target, url string, body []byte, timeout time.Duration) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + return do(HTTPClient(t, timeout), req) +} + +func do(client *http.Client, req *http.Request) ([]byte, error) { + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + out, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + snippet := out + if len(snippet) > 200 { + snippet = snippet[:200] + } + return nil, fmt.Errorf("http %d: %s", resp.StatusCode, snippet) + } + return out, nil +} + // Target abstracts command execution and file operations on a deployment target. type Target interface { // Exec runs a command on the target and returns combined output. diff --git a/internal/target/target_test.go b/internal/target/target_test.go new file mode 100644 index 00000000..a8003c37 --- /dev/null +++ b/internal/target/target_test.go @@ -0,0 +1,204 @@ +package target + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" +) + +// LocalTargetNoDial satisfies Target WITHOUT the optional Dialer interface, +// to prove the fallback paths (plain host transport / net.Dialer) still +// work for targets that cannot tunnel. It must NOT embed LocalTarget: the +// embedded *LocalTarget method set would promote DialContext onto it and +// silently re-implement Dialer. Methods are never exercised by the tests +// below (they only dial), so they fail loudly if that ever changes. +type LocalTargetNoDial struct{} + +type blockingDialer struct{} + +func (blockingDialer) DialContext(ctx context.Context, _, _ string) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() +} + +type blockingTarget struct{ LocalTargetNoDial } + +func (blockingTarget) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + return blockingDialer{}.DialContext(ctx, network, addr) +} + +var _ Target = (*LocalTargetNoDial)(nil) + +func (l *LocalTargetNoDial) Exec(context.Context, string, ...string) ([]byte, error) { + return nil, fmt.Errorf("LocalTargetNoDial: Exec not implemented") +} +func (l *LocalTargetNoDial) Upload(context.Context, string, string) error { + return fmt.Errorf("LocalTargetNoDial: Upload not implemented") +} +func (l *LocalTargetNoDial) Download(context.Context, string, string) error { + return fmt.Errorf("LocalTargetNoDial: Download not implemented") +} +func (l *LocalTargetNoDial) ReadFile(context.Context, string) ([]byte, error) { + return nil, fmt.Errorf("LocalTargetNoDial: ReadFile not implemented") +} +func (l *LocalTargetNoDial) WriteFile(context.Context, string, []byte, os.FileMode) error { + return fmt.Errorf("LocalTargetNoDial: WriteFile not implemented") +} +func (l *LocalTargetNoDial) DiskFree(context.Context, string) (uint64, error) { + return 0, fmt.Errorf("LocalTargetNoDial: DiskFree not implemented") +} +func (l *LocalTargetNoDial) MemTotal(context.Context) (uint64, error) { + return 0, fmt.Errorf("LocalTargetNoDial: MemTotal not implemented") +} +func (l *LocalTargetNoDial) PutFile(context.Context, string, string) error { + return fmt.Errorf("LocalTargetNoDial: PutFile not implemented") +} +func (l *LocalTargetNoDial) Sha256IfExists(context.Context, string) (string, error) { + return "", fmt.Errorf("LocalTargetNoDial: Sha256IfExists not implemented") +} +func (l *LocalTargetNoDial) CommandExists(context.Context, string) bool { return false } +func (l *LocalTargetNoDial) String() string { return "local-no-dial" } + +// TestEndpointHost pins the ssh-vs-local reporting rule: ssh targets with a +// recorded host report that host; everything else reports loopback. +func TestEndpointHost(t *testing.T) { + cases := []struct { + targetType, host, want string + }{ + {"ssh", "10.0.0.5", "10.0.0.5"}, + {"ssh", "", "127.0.0.1"}, // recorded host missing → fail-safe loopback + {"local", "", "127.0.0.1"}, + {"local", "ignored", "127.0.0.1"}, + {"", "", "127.0.0.1"}, + } + for _, tc := range cases { + if got := EndpointHost(tc.targetType, tc.host); got != tc.want { + t.Errorf("EndpointHost(%q, %q) = %q, want %q", tc.targetType, tc.host, got, tc.want) + } + } +} + +// TestLocalTargetDialContext proves the local DialContext reaches a real +// listener — this is what makes target.Get/Post work unchanged for local +// targets after the ssh-tunnel migration. +func TestLocalTargetDialContext(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + + conn, err := NewLocalTarget().DialContext(context.Background(), "tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("DialContext: %v", err) + } + conn.Close() +} + +// TestHTTPClientFallsBackForNonDialer: a Target that does not implement +// Dialer must get a working default-transport client (plain host dial). +func TestHTTPClientFallsBackForNonDialer(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + fmt.Fprint(w, `{"ok":true}`) + })) + defer srv.Close() + + client := HTTPClient(&LocalTargetNoDial{}, 2*time.Second) + if _, ok := client.Transport.(*http.Transport); ok { + t.Fatalf("non-Dialer target must keep the default transport, got a custom one") + } + resp, err := client.Get(srv.URL) + if err != nil { + t.Fatalf("GET via fallback client: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } +} + +// TestDialContextFallback: the free DialContext helper falls back to a plain +// net.Dialer when the target does not implement Dialer. +func TestDialContextFallback(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + + conn, err := DialContext(context.Background(), &LocalTargetNoDial{}, "tcp", ln.Addr().String(), 2*time.Second) + if err != nil { + t.Fatalf("DialContext fallback: %v", err) + } + conn.Close() +} + +func TestSSHTargetDialContextNilClient(t *testing.T) { + _, err := NewSSHTarget("host", 22, "user", "").DialContext(context.Background(), "tcp", "127.0.0.1:8090") + if err == nil || !strings.Contains(err.Error(), "ssh not connected") { + t.Fatalf("DialContext nil client error = %v", err) + } +} + +func TestSSHTargetDialContextRejectsNonLoopback(t *testing.T) { + _, err := NewSSHTarget("host", 22, "user", "").DialContext(context.Background(), "tcp", "10.0.0.5:8090") + if err == nil || !strings.Contains(err.Error(), "not loopback") { + t.Fatalf("DialContext non-loopback error = %v", err) + } +} + +func TestDialContextDialerHonorsTimeout(t *testing.T) { + start := time.Now() + _, err := DialContext(context.Background(), &blockingTarget{}, "tcp", "127.0.0.1:8090", 50*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "context deadline exceeded") { + t.Fatalf("DialContext timeout error = %v", err) + } + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { + t.Fatalf("DialContext took %s, want near 50ms", elapsed) + } +} + +// TestGetNon2xxIsError pins the status-code contract: a non-2xx response is +// an error carrying a body snippet (curl -fsS semantics the probes replaced). +func TestGetNon2xxIsError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprint(w, "boom") + })) + defer srv.Close() + + _, err := Get(context.Background(), NewLocalTarget(), srv.URL, 2*time.Second) + if err == nil || !strings.Contains(err.Error(), "http 500") { + t.Fatalf("Get on 500 = %v, want http 500 error", err) + } +} + +// TestPostSendsJSONBody verifies Post sets the JSON content type and ships +// the body through (getblockbynum-style probes depend on it). +func TestPostSendsJSONBody(t *testing.T) { + var gotCT, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotCT = r.Header.Get("Content-Type") + buf := make([]byte, 64) + n, _ := r.Body.Read(buf) + gotBody = string(buf[:n]) + fmt.Fprint(w, `{}`) + })) + defer srv.Close() + + if _, err := Post(context.Background(), NewLocalTarget(), srv.URL, []byte(`{"num":0}`), 2*time.Second); err != nil { + t.Fatalf("Post: %v", err) + } + if gotCT != "application/json" { + t.Errorf("Content-Type = %q, want application/json", gotCT) + } + if gotBody != `{"num":0}` { + t.Errorf("body = %q, want %q", gotBody, `{"num":0}`) + } +} From c60ccfd45694eb57dd83125e319ff1fbf8eda036 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 19 Aug 2026 12:11:47 +0800 Subject: [PATCH 04/49] chore(ci): pause tag-triggered releases (manual dispatch only) --- .github/workflows/release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2afadb3e..46d99e05 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,10 @@ name: Release on: - push: - tags: ["v*"] + # Tag-triggered releases are temporarily paused (TX-167). The pipeline + # itself is kept intact; run it manually from the Actions tab or via + # `gh workflow run release.yml` when releases resume. + workflow_dispatch: permissions: contents: write From 5abf817d9250ad3cc6078942c26647fa3b3cd6aa Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 19 Aug 2026 17:19:46 +0800 Subject: [PATCH 05/49] test(render): remove ineffectual assignment --- internal/render/hocon_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/render/hocon_test.go b/internal/render/hocon_test.go index 7a86b232..4044e661 100644 --- a/internal/render/hocon_test.go +++ b/internal/render/hocon_test.go @@ -75,13 +75,13 @@ func TestRenderHOCON_HTTPPortCollisionExplicitSolidity(t *testing.T) { // config_overrides is the explicit escape hatch for operators who // intentionally need the equal value. node := &intent.NodeSpec{Type: "fullnode", Ports: intent.PortMapping{HTTP: 8091, SolidityHTTP: 8091}} - out, err := RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, node) + _, err := RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, node) if err == nil || !strings.Contains(err.Error(), "solidityPort") { t.Fatalf("expected solidity collision error, got %v", err) } node.ConfigOverrides = map[string]any{"node.http.solidityPort": 8091} - out, err = RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, node) + out, err := RenderHOCON("", &intent.Intent{Name: "ports", Network: "mainnet"}, node) if err != nil { t.Fatalf("render with override: %v", err) } From 6e55e30a98546fcddbfee36b7f4e3f1c902d13ac Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 19 Aug 2026 16:33:21 +0800 Subject: [PATCH 06/49] fix(runtime): actually swap node artifact on upgrade/rollback (AUD-001) upgrade/rollback previously only rewrote state.version and restarted the node, so the 'upgraded' node kept running the old image/JAR. - runtime: add optional ArtifactUpgrader interface + UpgradeOpts - docker: pull new image, rewrite compose image tag, up -d (recreate); pull/recreate failure leaves old compose untouched for recovery - jar: download + SHA256-verify to temp file, atomic install, restart; requires --jar-url (jar rollback needs the OLD version's URL) - cmd: --jar-url/--jar-sha256 flags; state version committed only after artifact switch + start succeed; failure restores old state - jar: fix panic when download SHA256 output is empty --- cmd/rollback.go | 71 ++++++-- cmd/upgrade.go | 82 ++++++--- internal/apply/apply.go | 18 +- internal/runtime/jar.go | 61 +++++-- internal/runtime/runtime.go | 40 ++++- internal/runtime/upgrade.go | 165 ++++++++++++++++++ internal/runtime/upgrade_test.go | 286 +++++++++++++++++++++++++++++++ internal/state/types.go | 9 +- 8 files changed, 657 insertions(+), 75 deletions(-) create mode 100644 internal/runtime/upgrade.go create mode 100644 internal/runtime/upgrade_test.go diff --git a/cmd/rollback.go b/cmd/rollback.go index 889a6735..b15ce1b1 100644 --- a/cmd/rollback.go +++ b/cmd/rollback.go @@ -2,14 +2,18 @@ package cmd import ( "fmt" - "os" + "path/filepath" "time" "github.com/spf13/cobra" "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/runtime" ) +var rollbackJarURL string +var rollbackJarSHA256 string + var rollbackCmd = &cobra.Command{ Use: "rollback ", Short: "Rollback a node to its previous version", @@ -18,6 +22,8 @@ var rollbackCmd = &cobra.Command{ } func init() { + rollbackCmd.Flags().StringVar(&rollbackJarURL, "jar-url", "", "Target-version JAR URL (jar runtime)") + rollbackCmd.Flags().StringVar(&rollbackJarSHA256, "jar-sha256", "", "Optional target-version JAR SHA256 (jar runtime)") rootCmd.AddCommand(rollbackCmd) } @@ -44,25 +50,41 @@ func runRollback(cmd *cobra.Command, args []string) error { currentVersion := nc.Node.Version targetVersion := nc.Node.PreviousVersion - // Stop current. A stop failure is surfaced as a warning rather than - // a hard error — the node may already be down (which is the common - // case when the operator runs rollback after a crash), but a - // genuinely stuck process should not be hidden either. + upgrader, supported := nc.Runtime.(runtime.ArtifactUpgrader) + if !supported { + writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) + return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, fmt.Sprintf("Runtime for %s does not support safe artifact rollbacks", name)) + } + tx, err := upgrader.PrepareArtifact(cmd.Context(), name, runtime.UpgradeOpts{Version: targetVersion, JarURL: rollbackJarURL, JarSHA256: rollbackJarSHA256}) + if err != nil { + writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) + return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, fmt.Sprintf("Prepare rollback failed: %v", err)) + } if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { - fmt.Fprintf(os.Stderr, "warning: stop %s before rollback failed: %v\n", name, err) + _ = tx.Cleanup(cmd.Context()) + writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) + return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to stop %s: %v", name, err)) + } + if err := tx.Activate(cmd.Context()); err != nil { + return rollbackArtifact(cmd, nc, tx, name, currentVersion, start, fmt.Sprintf("activate artifact: %v", err)) } - - // Restore previous version - nc.Node.Version = targetVersion - nc.Node.PreviousVersion = currentVersion // Start - if err := nc.Runtime.Start(cmd.Context(), name); err != nil { - writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) - return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, - fmt.Sprintf("Rollback to %s failed: %v", targetVersion, err)) + if err := tx.Start(cmd.Context()); err != nil { + return rollbackArtifact(cmd, nc, tx, name, currentVersion, start, fmt.Sprintf("start artifact: %v", err)) + } + if nc.Node.Runtime == "jar" { + if digest, err := nc.Target.Sha256IfExists(cmd.Context(), filepath.Join(nc.Node.InstallPath, "FullNode.jar")); err == nil { + nc.Node.ArtifactSHA256 = digest + } + } + if err := tx.Cleanup(cmd.Context()); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: cleanup rollback artifact: %v\n", err) } + // Commit state only after the replacement artifact is running. + nc.Node.Version = targetVersion + nc.Node.PreviousVersion = currentVersion nc.Node.Status = "running" nc.SaveState() writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -75,3 +97,24 @@ func runRollback(cmd *cobra.Command, args []string) error { }) return nil } + +func rollbackArtifact(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { + if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { + cause = fmt.Sprintf("%s; stop failed: %v", cause, err) + } + rollbackErr := tx.Rollback(cmd.Context()) + if rollbackErr == nil { + rollbackErr = tx.Start(cmd.Context()) + } + if rollbackErr == nil { + if err := tx.Cleanup(cmd.Context()); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: cleanup rollback artifact: %v\n", err) + } + } + writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) + msg := fmt.Sprintf("Rollback failed, restored %s: %s", previousVersion, cause) + if rollbackErr != nil { + msg = fmt.Sprintf("%s; restore ALSO failed: %v", msg, rollbackErr) + } + return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, msg) +} diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 044749d1..96f2a150 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -2,14 +2,18 @@ package cmd import ( "fmt" + "path/filepath" "time" "github.com/spf13/cobra" "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/runtime" ) var upgradeVersion string +var upgradeJarURL string +var upgradeJarSHA256 string var upgradeCmd = &cobra.Command{ Use: "upgrade ", @@ -22,6 +26,8 @@ On failure, automatically rolls back to the previous version.`, func init() { upgradeCmd.Flags().StringVar(&upgradeVersion, "version", "", "Target version (required)") + upgradeCmd.Flags().StringVar(&upgradeJarURL, "jar-url", "", "Target-version JAR URL (jar runtime)") + upgradeCmd.Flags().StringVar(&upgradeJarSHA256, "jar-sha256", "", "Optional target-version JAR SHA256 (jar runtime)") mustMarkRequired(upgradeCmd, "version") rootCmd.AddCommand(upgradeCmd) } @@ -42,45 +48,50 @@ func runUpgrade(cmd *cobra.Command, args []string) error { previousVersion := nc.Node.Version - // Stop the node + upgrader, supported := nc.Runtime.(runtime.ArtifactUpgrader) + if !supported { + writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "UPGRADE_ERROR", Start: start}) + return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, + fmt.Sprintf("Runtime for %s does not support safe artifact upgrades", name), + "Use a Docker or JAR runtime") + } + // Prepare (pull/download and verify) while the node is still running. + tx, err := upgrader.PrepareArtifact(cmd.Context(), name, runtime.UpgradeOpts{Version: upgradeVersion, JarURL: upgradeJarURL, JarSHA256: upgradeJarSHA256}) + if err != nil { + writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "UPGRADE_ERROR", Start: start}) + return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, fmt.Sprintf("Prepare upgrade failed: %v", err)) + } + if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { + _ = tx.Cleanup(cmd.Context()) writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "UPGRADE_ERROR", Start: start}) return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to stop %s for upgrade: %v", name, err)) } - // Update version in state - nc.Node.PreviousVersion = previousVersion - nc.Node.Version = upgradeVersion + if err := tx.Activate(cmd.Context()); err != nil { + return rollbackUpgrade(cmd, nc, tx, name, previousVersion, start, fmt.Sprintf("activate artifact: %v", err)) + } // Start with new version. On failure, restore the previous version // in state and try to bring it back up. Surface BOTH errors to the // user — a silent rollback-failed leaves the operator thinking the // node is back when in fact nothing is running. - if err := nc.Runtime.Start(cmd.Context(), name); err != nil { - nc.Node.Version = previousVersion - nc.Node.PreviousVersion = "" - rollbackErr := nc.Runtime.Start(cmd.Context(), name) - nc.SaveState() - writeAudit(auditEvent{ - Command: "upgrade", - Node: name, - Target: nc.Target.String(), - Result: "rollback", - ErrorCode: "UPGRADE_ERROR", - Start: start, - }) - - msg := fmt.Sprintf("Upgrade failed, rolled back to %s: %v", previousVersion, err) - if rollbackErr != nil { - msg = fmt.Sprintf("%s; rollback start ALSO failed: %v", msg, rollbackErr) + if err := tx.Start(cmd.Context()); err != nil { + return rollbackUpgrade(cmd, nc, tx, name, previousVersion, start, fmt.Sprintf("start new artifact: %v", err)) + } + if nc.Node.Runtime == "jar" { + if digest, err := nc.Target.Sha256IfExists(cmd.Context(), filepath.Join(nc.Node.InstallPath, "FullNode.jar")); err == nil { + nc.Node.ArtifactSHA256 = digest } - return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, - msg, - "Check logs: trond logs "+name, - "Run diagnostics: trond diagnose "+name) + } + if err := tx.Cleanup(cmd.Context()); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: cleanup upgrade artifact: %v\n", err) } + // Commit state only after the replacement artifact is running. + nc.Node.PreviousVersion = previousVersion + nc.Node.Version = upgradeVersion nc.Node.Status = "running" nc.SaveState() writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -93,3 +104,24 @@ func runUpgrade(cmd *cobra.Command, args []string) error { }) return nil } + +func rollbackUpgrade(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { + if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { + cause = fmt.Sprintf("%s; stop failed: %v", cause, err) + } + rollbackErr := tx.Rollback(cmd.Context()) + if rollbackErr == nil { + rollbackErr = tx.Start(cmd.Context()) + } + if rollbackErr == nil { + if err := tx.Cleanup(cmd.Context()); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: cleanup rollback artifact: %v\n", err) + } + } + writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "rollback", ErrorCode: "UPGRADE_ERROR", Start: start}) + msg := fmt.Sprintf("Upgrade failed, rolled back to %s: %s", previousVersion, cause) + if rollbackErr != nil { + msg = fmt.Sprintf("%s; rollback ALSO failed: %v", msg, rollbackErr) + } + return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, msg, "Check logs: trond logs "+name, "Run diagnostics: trond diagnose "+name) +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index d8c72795..6c347592 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -318,6 +318,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { ConfigData: []byte(hocon), EnvVars: opts.EnvVars, } + artifactSHA256 := "" switch runtimeType { case "docker": @@ -348,6 +349,9 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { // JarPath still points at the install_path location for `mkdir -p` / // config layout; only the systemd ExecStart references jarPath above. deployOpts.JarPath = filepath.Join(node.InstallPath, "FullNode.jar") + if opts.Existing != nil { + deployOpts.ArtifactSHA256 = opts.Existing.ArtifactSHA256 + } if node.Jar != nil { deployOpts.JarURL = node.Jar.URL deployOpts.JarSHA256 = node.Jar.SHA256 @@ -356,6 +360,9 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { if err := rt.Deploy(ctx, deployOpts); err != nil { return nil, fmt.Errorf("jar deploy: %w", err) } + if digest, err := opts.Target.Sha256IfExists(ctx, deployOpts.JarPath); err == nil { + artifactSHA256 = digest + } default: return nil, fmt.Errorf("unsupported runtime %q", runtimeType) } @@ -365,11 +372,12 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { // Persist into state. managed := state.ManagedNode{ - Name: opts.Intent.Name, - IntentHash: opts.IntentHash, - ConfigHash: configHash, - Version: node.Version, - Network: opts.Intent.Network, + Name: opts.Intent.Name, + IntentHash: opts.IntentHash, + ConfigHash: configHash, + Version: node.Version, + ArtifactSHA256: artifactSHA256, + Network: opts.Intent.Network, Target: state.NodeTarget{ Type: opts.Intent.Target.Type, Host: opts.Intent.Target.Host, diff --git a/internal/runtime/jar.go b/internal/runtime/jar.go index 9241ac88..d833b00d 100644 --- a/internal/runtime/jar.go +++ b/internal/runtime/jar.go @@ -32,16 +32,32 @@ func (r *JarRuntime) Deploy(ctx context.Context, opts DeployOpts) error { return fmt.Errorf("create install dir: %w", err) } - // Download jar if URL provided and jar not yet present + tracker := newChangeTracker(r.target) + // Apply skips a pinned artifact already installed; upgrades download first. if opts.JarURL != "" { - if err := r.downloadJar(ctx, opts.JarURL, opts.JarPath, opts.JarSHA256); err != nil { - return fmt.Errorf("download jar: %w", err) + before, err := r.target.Sha256IfExists(ctx, opts.JarPath) + if err != nil { + return fmt.Errorf("hash existing jar: %w", err) + } + if opts.JarSHA256 != "" && before == opts.JarSHA256 && before == opts.ArtifactSHA256 { + // The artifact and recorded state agree: clean no-op. + } else { + // A matching file with missing/stale state may represent an + // interrupted deploy. Reinstall and restart so the process converges. + staleArtifactState := opts.JarSHA256 != "" && before == opts.JarSHA256 + if err := r.downloadJar(ctx, opts.JarURL, opts.JarPath, opts.JarSHA256); err != nil { + return fmt.Errorf("download jar: %w", err) + } + after, err := r.target.Sha256IfExists(ctx, opts.JarPath) + if err != nil { + return fmt.Errorf("hash installed jar: %w", err) + } + tracker.changed = before != after || staleArtifactState } } // Write config file. Tracked: systemd has no idea this file exists, // and java-tron parses it once at JVM startup. - tracker := newChangeTracker(r.target) configPath := filepath.Join(installPath, "config.conf") if err := tracker.write(ctx, configPath, opts.ConfigData, 0644); err != nil { return fmt.Errorf("write config: %w", err) @@ -108,6 +124,11 @@ func (r *JarRuntime) Deploy(ctx context.Context, opts DeployOpts) error { return nil } +// ArtifactSHA256 returns the digest of a deployed JAR. +func (r *JarRuntime) ArtifactSHA256(ctx context.Context, path string) (string, error) { + return r.target.Sha256IfExists(ctx, path) +} + func (r *JarRuntime) Start(ctx context.Context, name string) error { unitName := fmt.Sprintf("tron-%s.service", name) _, err := r.target.Exec(ctx, "systemctl", "start", unitName) @@ -217,33 +238,35 @@ func (r *JarRuntime) Logs(ctx context.Context, name string, opts LogOpts) (io.Re // downloadJar downloads the jar file and verifies its SHA256 hash. func (r *JarRuntime) downloadJar(ctx context.Context, url, destPath, expectedSHA256 string) error { - // Check if jar already exists with correct hash - if expectedSHA256 != "" { - out, err := r.target.Exec(ctx, "sha256sum", destPath) - if err == nil { - fields := strings.Fields(string(out)) - if len(fields) > 0 && fields[0] == expectedSHA256 { - return nil // Already downloaded and verified - } - } - } - - // Download - if _, err := r.target.Exec(ctx, "curl", "-fSL", "-o", destPath, url); err != nil { + // Download to a temporary path so a failed transfer or checksum never + // destroys the currently running artifact. + tmpPath := destPath + ".upgrade.tmp" + if _, err := r.target.Exec(ctx, "curl", "-fSL", "-o", tmpPath, url); err != nil { + _, _ = r.target.Exec(ctx, "rm", "-f", tmpPath) return fmt.Errorf("download %s: %w", url, err) } // Verify hash if expectedSHA256 != "" { - out, err := r.target.Exec(ctx, "sha256sum", destPath) + out, err := r.target.Exec(ctx, "sha256sum", tmpPath) if err != nil { + _, _ = r.target.Exec(ctx, "rm", "-f", tmpPath) return fmt.Errorf("sha256sum: %w", err) } fields := strings.Fields(string(out)) - if len(fields) == 0 || fields[0] != expectedSHA256 { + if len(fields) == 0 { + _, _ = r.target.Exec(ctx, "rm", "-f", tmpPath) + return fmt.Errorf("SHA256 verification returned no digest; expected %s", expectedSHA256) + } + if fields[0] != expectedSHA256 { + _, _ = r.target.Exec(ctx, "rm", "-f", tmpPath) return fmt.Errorf("SHA256 mismatch: expected %s, got %s", expectedSHA256, fields[0]) } } + if _, err := r.target.Exec(ctx, "mv", tmpPath, destPath); err != nil { + _, _ = r.target.Exec(ctx, "rm", "-f", tmpPath) + return fmt.Errorf("install downloaded jar: %w", err) + } return nil } diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index 259b1ed1..ab04d7fb 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -84,14 +84,38 @@ type Runtime interface { Logs(ctx context.Context, name string, opts LogOpts) (io.ReadCloser, error) } +// UpgradeOpts carries the artifact coordinates for a version switch. +type UpgradeOpts struct { + Version string + JarURL string // Jar runtime only: URL of the target-version JAR + JarSHA256 string // Jar runtime only: optional integrity pin +} + +// ArtifactUpgrader is an optional runtime capability. Callers must prepare +// before stopping the node, then activate and rollback the prepared artifact +// as one transaction. Runtimes without it cannot safely be upgraded. +type ArtifactUpgrader interface { + PrepareArtifact(ctx context.Context, name string, opts UpgradeOpts) (ArtifactTransaction, error) +} + +// ArtifactTransaction changes only the artifact. The caller owns stopping and +// starting the service, and must Rollback when activation or startup fails. +type ArtifactTransaction interface { + Activate(context.Context) error + Start(context.Context) error + Rollback(context.Context) error + Cleanup(context.Context) error +} + // DeployOpts contains everything needed for a deployment. type DeployOpts struct { - Name string - ConfigData []byte - ComposeData []byte // Docker runtime only - SystemdData []byte // Jar runtime only - JarPath string // Jar runtime only - JarURL string // Jar runtime only - JarSHA256 string // Jar runtime only - EnvVars map[string]string + Name string + ConfigData []byte + ComposeData []byte // Docker runtime only + SystemdData []byte // Jar runtime only + JarPath string // Jar runtime only + JarURL string // Jar runtime only + JarSHA256 string // Jar runtime only + ArtifactSHA256 string // previously recorded Jar artifact digest + EnvVars map[string]string } diff --git a/internal/runtime/upgrade.go b/internal/runtime/upgrade.go new file mode 100644 index 00000000..0f669682 --- /dev/null +++ b/internal/runtime/upgrade.go @@ -0,0 +1,165 @@ +package runtime + +import ( + "context" + "fmt" + "path/filepath" + "regexp" + "strings" + "time" +) + +var versionPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+\-]*$`) + +func validateUpgradeVersion(version string) error { + if !versionPattern.MatchString(version) { + return fmt.Errorf("invalid version/image tag %q", version) + } + return nil +} + +var composeImageLine = regexp.MustCompile(`(?m)^(\s*image:\s*)(\S+)`) + +// swapComposeImageTag replaces the tag on the single image used by a node. +// Digests are deliberately discarded: an explicit version switch must select +// the requested tag rather than continue pinning the old artifact. +func swapComposeImageTag(compose []byte, newVersion string) (out []byte, oldImage, newImage string, err error) { + matches := composeImageLine.FindAllSubmatchIndex(compose, -1) + if len(matches) == 0 { + return nil, "", "", fmt.Errorf("compose file has no image line") + } + for _, m := range matches { + image := string(compose[m[4]:m[5]]) + if oldImage == "" { + oldImage = image + } else if image != oldImage { + return nil, "", "", fmt.Errorf("compose file contains multiple different images: %q and %q", oldImage, image) + } + } + base := oldImage + if at := strings.IndexByte(base, '@'); at >= 0 { + base = base[:at] + } + colon := strings.LastIndexByte(base, ':') + slash := strings.LastIndexByte(base, '/') + if colon > slash { + base = base[:colon] + } + newImage = base + ":" + newVersion + if oldImage == newImage { + return compose, oldImage, newImage, nil + } + out = composeImageLine.ReplaceAllFunc(compose, func(line []byte) []byte { + m := composeImageLine.FindSubmatchIndex(line) + return append(append(append([]byte{}, line[:m[3]]...), newImage...), line[m[5]:]...) + }) + return out, oldImage, newImage, nil +} + +type dockerArtifactTransaction struct { + r *DockerRuntime + name string + path string + old, updated []byte +} + +func (t *dockerArtifactTransaction) Activate(ctx context.Context) error { + return t.r.target.WriteFile(ctx, t.path, t.updated, 0644) +} +func (t *dockerArtifactTransaction) Start(ctx context.Context) error { + _, err := t.r.target.Exec(ctx, "docker", "compose", "-f", t.path, "-p", t.name, "up", "-d", "--force-recreate") + return err +} +func (t *dockerArtifactTransaction) Rollback(ctx context.Context) error { + return t.r.target.WriteFile(ctx, t.path, t.old, 0644) +} +func (t *dockerArtifactTransaction) Cleanup(context.Context) error { return nil } + +func (r *DockerRuntime) PrepareArtifact(ctx context.Context, name string, opts UpgradeOpts) (ArtifactTransaction, error) { + if err := validateUpgradeVersion(opts.Version); err != nil { + return nil, err + } + composePath := filepath.Join(r.workDir, name, "docker-compose.yaml") + compose, err := r.target.ReadFile(ctx, composePath) + if err != nil { + return nil, fmt.Errorf("read compose: %w", err) + } + updated, _, newImage, err := swapComposeImageTag(compose, opts.Version) + if err != nil { + return nil, err + } + if string(updated) == string(compose) { + return &dockerArtifactTransaction{r: r, name: name, path: composePath, old: compose, updated: updated}, nil + } + pullCtx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + if _, err := r.target.Exec(pullCtx, "docker", "pull", newImage); err != nil { + return nil, fmt.Errorf("docker pull %s: %w", newImage, err) + } + return &dockerArtifactTransaction{r: r, name: name, path: composePath, old: compose, updated: updated}, nil +} + +type jarArtifactTransaction struct { + r *JarRuntime + name, candidate, live, backup string + activated bool +} + +func (t *jarArtifactTransaction) Activate(ctx context.Context) error { + if _, err := t.r.target.Exec(ctx, "mv", t.live, t.backup); err != nil { + return fmt.Errorf("backup jar: %w", err) + } + if _, err := t.r.target.Exec(ctx, "mv", t.candidate, t.live); err != nil { + if _, restoreErr := t.r.target.Exec(ctx, "mv", t.backup, t.live); restoreErr != nil { + return fmt.Errorf("install jar %s: %w; restore backup %s to %s: %v", t.candidate, err, t.backup, t.live, restoreErr) + } + return fmt.Errorf("install jar: %w", err) + } + t.activated = true + return nil +} +func (t *jarArtifactTransaction) Start(ctx context.Context) error { + _, err := t.r.target.Exec(ctx, "systemctl", "start", fmt.Sprintf("tron-%s.service", t.name)) + return err +} +func (t *jarArtifactTransaction) Rollback(ctx context.Context) error { + if !t.activated { + return nil + } + if _, err := t.r.target.Exec(ctx, "mv", t.live, t.candidate); err != nil { + return err + } + if _, err := t.r.target.Exec(ctx, "mv", t.backup, t.live); err != nil { + if _, restoreErr := t.r.target.Exec(ctx, "mv", t.candidate, t.live); restoreErr != nil { + return fmt.Errorf("restore backup %s to %s: %w; restore candidate %s to %s: %v", t.backup, t.live, err, t.candidate, t.live, restoreErr) + } + return fmt.Errorf("restore backup %s to %s: %w", t.backup, t.live, err) + } + return nil +} +func (t *jarArtifactTransaction) Cleanup(ctx context.Context) error { + _, err := t.r.target.Exec(ctx, "rm", "-f", t.candidate, t.backup) + return err +} + +func (r *JarRuntime) PrepareArtifact(ctx context.Context, name string, opts UpgradeOpts) (ArtifactTransaction, error) { + if err := validateUpgradeVersion(opts.Version); err != nil { + return nil, err + } + if opts.JarURL == "" { + return nil, fmt.Errorf("jar URL is required for artifact upgrade; provide --jar-url") + } + if r.purgeInstallPath == "" { + return nil, fmt.Errorf("install path is unavailable for artifact upgrade") + } + jarPath := filepath.Join(r.purgeInstallPath, "FullNode.jar") + candidate := jarPath + ".candidate" + if err := r.downloadJar(ctx, opts.JarURL, candidate, opts.JarSHA256); err != nil { + _, _ = r.target.Exec(ctx, "rm", "-f", candidate) + return nil, fmt.Errorf("download jar: %w", err) + } + return &jarArtifactTransaction{r: r, name: name, candidate: candidate, live: jarPath, backup: jarPath + ".upgrade.backup"}, nil +} + +var _ ArtifactUpgrader = (*DockerRuntime)(nil) +var _ ArtifactUpgrader = (*JarRuntime)(nil) diff --git a/internal/runtime/upgrade_test.go b/internal/runtime/upgrade_test.go new file mode 100644 index 00000000..05e6f82b --- /dev/null +++ b/internal/runtime/upgrade_test.go @@ -0,0 +1,286 @@ +package runtime + +import ( + "context" + "errors" + "fmt" + "os" + "reflect" + "strings" + "testing" +) + +func TestSwapComposeImageTag(t *testing.T) { + tests := []struct { + name, compose, version, want, old, image string + wantErr bool + }{ + {"official", "image: tronprotocol/java-tron:4.8.0\n", "4.8.1", "image: tronprotocol/java-tron:4.8.1\n", "tronprotocol/java-tron:4.8.0", "tronprotocol/java-tron:4.8.1", false}, + {"registry port", "image: registry:5000/img:1.0\n", "2.0", "image: registry:5000/img:2.0\n", "registry:5000/img:1.0", "registry:5000/img:2.0", false}, + {"digest pin", "image: img:1.0@sha256:abc\n", "2.0", "image: img:2.0\n", "img:1.0@sha256:abc", "img:2.0", false}, + {"same version", "image: img:1.0\n", "1.0", "image: img:1.0\n", "img:1.0", "img:1.0", false}, + {"missing", "services:\n node:\n command: tron\n", "1.0", "", "", "", true}, + {"different images", "image: a:1\nother:\n image: b:1\n", "2", "", "", "", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, old, image, err := swapComposeImageTag([]byte(tc.compose), tc.version) + if (err != nil) != tc.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tc.wantErr) + } + if tc.wantErr { + return + } + if string(got) != tc.want || old != tc.old || image != tc.image { + t.Fatalf("got %q, %q, %q", got, old, image) + } + }) + } +} + +type upgradeFakeTarget struct { + *fakeTarget + pullErr error +} + +type jarUpgradeFakeTarget struct{ *fakeTarget } + +type failingUpgradeTarget struct { + *fakeTarget + writeErr error + startErr error + startFailures int + mvFailAt int + mvCount int +} + +func (f *failingUpgradeTarget) WriteFile(ctx context.Context, path string, data []byte, perm os.FileMode) error { + if f.writeErr != nil { + return f.writeErr + } + return f.fakeTarget.WriteFile(ctx, path, data, perm) +} + +func (f *failingUpgradeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { + f.cmds = append(f.cmds, append([]string{cmd}, args...)) + if cmd == "docker" && len(args) > 0 && args[len(args)-1] == "--force-recreate" { + if f.startFailures > 0 { + f.startFailures-- + return nil, f.startErr + } + } + if cmd == "mv" { + f.mvCount++ + if f.mvFailAt == f.mvCount { + return nil, fmt.Errorf("mv %s to %s failed", args[0], args[1]) + } + } + return nil, nil +} + +func (f *jarUpgradeFakeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { + f.cmds = append(f.cmds, append([]string{cmd}, args...)) + if cmd == "sha256sum" { + return []byte("deadbeef /opt/tron/FullNode.jar\n"), nil + } + return nil, nil +} + +func (f *upgradeFakeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { + f.cmds = append(f.cmds, append([]string{cmd}, args...)) + if cmd == "docker" && len(args) >= 2 && args[0] == "pull" { + return nil, f.pullErr + } + return nil, nil +} + +func TestDockerRuntimeUpgradeArtifact(t *testing.T) { + path := "/deploy/n1/docker-compose.yaml" + f := &upgradeFakeTarget{fakeTarget: newFakeTarget()} + f.files[path] = []byte("services:\n n1:\n image: img:1\n") + tx, err := NewDockerRuntime(f, "/deploy").PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2"}) + if err != nil { + t.Fatal(err) + } + if err := tx.Activate(context.Background()); err != nil { + t.Fatal(err) + } + if got := string(f.files[path]); !strings.Contains(got, "img:2") { + t.Fatalf("compose not updated: %s", got) + } + if !reflect.DeepEqual(f.cmds[0], []string{"docker", "pull", "img:2"}) { + t.Fatalf("commands = %v", f.cmds) + } + + f = &upgradeFakeTarget{fakeTarget: newFakeTarget(), pullErr: errors.New("no image")} + f.files[path] = []byte("image: img:1\n") + if _, err := NewDockerRuntime(f, "/deploy").PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2"}); err == nil { + t.Fatal("expected pull error") + } + if string(f.files[path]) != "image: img:1\n" { + t.Fatal("compose changed after failed pull") + } +} + +func TestDockerArtifactActivateWriteFailureDoesNotStart(t *testing.T) { + f := &failingUpgradeTarget{fakeTarget: newFakeTarget(), writeErr: errors.New("write failed")} + tx := &dockerArtifactTransaction{r: NewDockerRuntime(f, "/deploy"), name: "n1", path: "/deploy/n1/docker-compose.yaml", updated: []byte("new")} + if err := tx.Activate(context.Background()); err == nil { + t.Fatal("Activate succeeded despite compose write failure") + } + for _, cmd := range f.cmds { + if len(cmd) > 0 && cmd[0] == "docker" { + t.Fatal("docker Start command was invoked after Activate failure") + } + } +} + +func TestDockerArtifactStartFailureRollbackRestoresCompose(t *testing.T) { + old := []byte("image: img:1\n") + f := &failingUpgradeTarget{fakeTarget: newFakeTarget(), startErr: errors.New("up failed"), startFailures: 1} + path := "/deploy/n1/docker-compose.yaml" + tx := &dockerArtifactTransaction{r: NewDockerRuntime(f, "/deploy"), name: "n1", path: path, old: old, updated: []byte("image: img:2\n")} + if err := tx.Activate(context.Background()); err != nil { + t.Fatal(err) + } + if err := tx.Start(context.Background()); err == nil { + t.Fatal("Start succeeded unexpectedly") + } + if err := tx.Rollback(context.Background()); err != nil { + t.Fatal(err) + } + if err := tx.Start(context.Background()); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(f.files[path], old) { + t.Fatalf("compose = %q, want old compose %q", f.files[path], old) + } + if got := countCommand(f.cmds, "docker"); got != 2 { + t.Fatalf("docker compose calls = %d, want failed up plus rollback up", got) + } +} + +func TestJarArtifactActivateSecondMoveSelfRestoresBackup(t *testing.T) { + f := &failingUpgradeTarget{fakeTarget: newFakeTarget(), mvFailAt: 2} + tx := &jarArtifactTransaction{r: NewJarRuntime(f), candidate: "/opt/tron/FullNode.jar.candidate", live: "/opt/tron/FullNode.jar", backup: "/opt/tron/FullNode.jar.upgrade.backup"} + if err := tx.Activate(context.Background()); err == nil { + t.Fatal("Activate succeeded despite candidate move failure") + } else if !strings.Contains(err.Error(), "/opt/tron/FullNode.jar.candidate") { + t.Fatalf("error lacks artifact paths: %v", err) + } + if got := f.cmds[len(f.cmds)-1]; !reflect.DeepEqual(got, []string{"mv", "/opt/tron/FullNode.jar.upgrade.backup", "/opt/tron/FullNode.jar"}) { + t.Fatalf("last recovery command = %v", got) + } +} + +func TestJarArtifactRollbackSequence(t *testing.T) { + f := &failingUpgradeTarget{fakeTarget: newFakeTarget()} + tx := &jarArtifactTransaction{r: NewJarRuntime(f), name: "n1", candidate: "/opt/tron/FullNode.jar.candidate", live: "/opt/tron/FullNode.jar", backup: "/opt/tron/FullNode.jar.upgrade.backup", activated: true} + if err := tx.Rollback(context.Background()); err != nil { + t.Fatal(err) + } + if err := tx.Start(context.Background()); err != nil { + t.Fatal(err) + } + want := [][]string{{"mv", tx.live, tx.candidate}, {"mv", tx.backup, tx.live}, {"systemctl", "start", "tron-n1.service"}} + if !reflect.DeepEqual(f.cmds, want) { + t.Fatalf("rollback commands = %v, want %v", f.cmds, want) + } +} + +func TestValidateUpgradeVersionRejectsUnsafeTags(t *testing.T) { + for _, version := range []string{"../../etc", "tag with spaces"} { + if err := validateUpgradeVersion(version); err == nil { + t.Errorf("validateUpgradeVersion(%q) accepted unsafe tag", version) + } + } +} + +func countCommand(cmds [][]string, name string) int { + count := 0 + for _, cmd := range cmds { + if len(cmd) > 0 && cmd[0] == name { + count++ + } + } + return count +} + +func TestJarRuntimeUpgradeArtifact(t *testing.T) { + t.Run("missing URL", func(t *testing.T) { + r := NewJarRuntime(newFakeTarget()) + r.SetPurgeInstallPath("/opt/tron") + if _, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2"}); err == nil || !strings.Contains(err.Error(), "jar URL") { + t.Fatal(err) + } + }) + t.Run("download and restart", func(t *testing.T) { + f := &jarUpgradeFakeTarget{newFakeTarget()} + r := NewJarRuntime(f) + r.SetPurgeInstallPath("/opt/tron") + tx, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2", JarURL: "https://example/jar"}) + if err != nil { + t.Fatal(err) + } + if err := tx.Activate(context.Background()); err != nil { + t.Fatal(err) + } + if f.cmds[0][0] != "curl" || f.cmds[len(f.cmds)-1][0] != "mv" { + t.Fatalf("commands = %v", f.cmds) + } + }) + t.Run("verified existing jar", func(t *testing.T) { + f := &jarUpgradeFakeTarget{newFakeTarget()} + f.files["/opt/tron/FullNode.jar"] = []byte("old") + r := NewJarRuntime(f) + r.SetPurgeInstallPath("/opt/tron") + tx, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2", JarURL: "https://example/jar", JarSHA256: "deadbeef"}) + if err != nil { + t.Fatal(err) + } + if err := tx.Activate(context.Background()); err != nil { + t.Fatal(err) + } + if f.cmds[0][0] != "curl" { + t.Fatalf("commands = %v", f.cmds) + } + }) +} + +func TestJarDeployInterruptedArtifactConvergesOnRerun(t *testing.T) { + for _, tc := range []struct { + name string + recorded string + restart bool + }{ + {name: "missing recorded digest", recorded: "", restart: true}, + {name: "matching recorded digest", recorded: "deadbeef", restart: false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := &jarUpgradeFakeTarget{newFakeTarget()} + f.files["/opt/tron/FullNode.jar"] = []byte("new artifact") + r := NewJarRuntime(f) + err := r.Deploy(context.Background(), DeployOpts{ + Name: "n1", + JarPath: "/opt/tron/FullNode.jar", + JarURL: "https://example/jar", + JarSHA256: "deadbeef", + ArtifactSHA256: tc.recorded, + ConfigData: []byte("config"), + SystemdData: []byte("unit"), + }) + if err != nil { + t.Fatal(err) + } + gotRestart := false + for _, cmd := range f.cmds { + if len(cmd) >= 2 && cmd[0] == "systemctl" && cmd[1] == "restart" { + gotRestart = true + } + } + if gotRestart != tc.restart { + t.Fatalf("restart = %v, want %v; commands = %v", gotRestart, tc.restart, f.cmds) + } + }) + } +} diff --git a/internal/state/types.go b/internal/state/types.go index 43a7017a..badf2b59 100644 --- a/internal/state/types.go +++ b/internal/state/types.go @@ -7,10 +7,11 @@ import ( // ManagedNode represents a deployed node tracked in the state file. type ManagedNode struct { - Name string `json:"name"` - IntentHash string `json:"intent_hash"` - ConfigHash string `json:"config_hash"` - Version string `json:"version"` + Name string `json:"name"` + IntentHash string `json:"intent_hash"` + ConfigHash string `json:"config_hash"` + Version string `json:"version"` + ArtifactSHA256 string `json:"artifact_sha256,omitempty"` // Network is the intent's `network:` value (mainnet | nile | private) // recorded at apply time, so `status` can report it and derive the // `is_private` safety fact without re-reading the intent. omitempty: From 4490064af34d332022f6ccfb7990ab687ccc7e84 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 19 Aug 2026 16:59:38 +0800 Subject: [PATCH 07/49] fix(mcp): redact witness keys from live config resources --- internal/mcp/conf_helpers.go | 13 ++++++++++ internal/mcp/conf_helpers_test.go | 29 +++++++++++++++++++++++ internal/mcp/resources.go | 2 +- internal/render/hocon.go | 3 ++- internal/render/witness_redaction_test.go | 5 ++++ 5 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 internal/mcp/conf_helpers_test.go diff --git a/internal/mcp/conf_helpers.go b/internal/mcp/conf_helpers.go index 0a2a36b8..9ab29f6f 100644 --- a/internal/mcp/conf_helpers.go +++ b/internal/mcp/conf_helpers.go @@ -3,7 +3,9 @@ package mcp import ( "context" "fmt" + "strings" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -27,3 +29,14 @@ func readLiveConfigForMCP(ctx context.Context, tgt target.Target, node *state.Ma } return string(out), nil } + +// redactLiveConfigForMCP applies the existing display redaction before a +// complete live config crosses the MCP boundary. Raw reads remain available +// to comparison code, which must detect witness-key rotation. +func redactLiveConfigForMCP(config string) string { + lines := strings.Split(config, "\n") + for i, line := range lines { + lines[i] = render.RedactWitnessLine(line) + } + return strings.Join(lines, "\n") +} diff --git a/internal/mcp/conf_helpers_test.go b/internal/mcp/conf_helpers_test.go new file mode 100644 index 00000000..aa8dc015 --- /dev/null +++ b/internal/mcp/conf_helpers_test.go @@ -0,0 +1,29 @@ +package mcp + +import ( + "strings" + "testing" +) + +func TestRedactLiveConfigForMCP(t *testing.T) { + key := strings.Repeat("a", 64) + config := "node.p2p.version = 1\n" + `localwitness = ["` + key + `"]` + "\nseed.node.ip.list = []\n" + got := redactLiveConfigForMCP(config) + if strings.Contains(got, key) { + t.Fatal("live config redaction leaked witness private key") + } + if !strings.Contains(got, `localwitness = [""]`) { + t.Fatalf("redacted config lost readable witness structure: %q", got) + } + if !strings.Contains(got, "node.p2p.version = 1") || !strings.Contains(got, "seed.node.ip.list = []") { + t.Fatalf("redaction changed unrelated config: %q", got) + } +} + +func TestRedactLiveConfigForMCP_ColonWitnessSyntax(t *testing.T) { + key := strings.Repeat("b", 64) + got := redactLiveConfigForMCP(`localwitness : ["` + key + `"]`) + if strings.Contains(got, key) || got != `localwitness = [""]` { + t.Fatalf("colon-separated live witness config was not redacted: %q", got) + } +} diff --git a/internal/mcp/resources.go b/internal/mcp/resources.go index 8c10e624..f8975d00 100644 --- a/internal/mcp/resources.go +++ b/internal/mcp/resources.go @@ -190,7 +190,7 @@ func readNodeConfResource(ctx context.Context, req *mcp.ReadResourceRequest) (*m Contents: []*mcp.ResourceContents{{ URI: req.Params.URI, MIMEType: "text/plain", - Text: live, + Text: redactLiveConfigForMCP(live), }}, }, nil } diff --git a/internal/render/hocon.go b/internal/render/hocon.go index 1562a5d5..e12d55e9 100644 --- a/internal/render/hocon.go +++ b/internal/render/hocon.go @@ -105,7 +105,8 @@ func IsWitnessKeyLine(line string) bool { if !ok { return false } - return strings.HasPrefix(strings.TrimLeft(rest, " \t"), "=") + separator := strings.TrimLeft(rest, " \t") + return strings.HasPrefix(separator, "=") || strings.HasPrefix(separator, ":") } // RedactWitnessLine returns line unchanged unless it is a witness diff --git a/internal/render/witness_redaction_test.go b/internal/render/witness_redaction_test.go index 3988aecb..7fd50759 100644 --- a/internal/render/witness_redaction_test.go +++ b/internal/render/witness_redaction_test.go @@ -215,6 +215,7 @@ func TestRendered_FormattingCannotSpillTheKey(t *testing.T) { func TestIsWitnessKeyLine_ExactKeyMatch(t *testing.T) { match := []string{ `localwitness = ["deadbeef"]`, + `localwitness : ["deadbeef"]`, `localwitness=["deadbeef"]`, ` localwitness = [ deadbeef ]`, "\tlocalwitness\t=\t[\"deadbeef\"]", @@ -254,6 +255,10 @@ func TestRedactWitnessLine(t *testing.T) { if got != `localwitness = [""]` { t.Errorf("unexpected marker: %q", got) } + got = RedactWitnessLine(fmt.Sprintf(`localwitness : [%q]`, probeWitnessKey)) + if strings.Contains(got, probeWitnessKey) || got != `localwitness = [""]` { + t.Errorf("colon-separated witness line was not redacted: %q", got) + } // Indent is preserved so redacted diff lines stay aligned. if got := RedactWitnessLine(` localwitness = ["x"]`); got != ` localwitness = [""]` { t.Errorf("indent not preserved: %q", got) From cc3b68720b3a5ec3d50d9c2784fb73693eac01f7 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 19 Aug 2026 17:56:57 +0800 Subject: [PATCH 08/49] fix(security): enforce private permissions for JAR configs --- cmd/files.go | 3 +- internal/apply/apply.go | 15 ++++++ internal/apply/apply_test.go | 75 ++++++++++++++++++++++++++- internal/apply/build_test.go | 12 ++++- internal/runtime/jar.go | 2 +- internal/runtime/redeploy_test.go | 75 +++++++++++++++++++++++++++ internal/target/local.go | 40 +++++++++++++- internal/target/local_putfile_test.go | 39 ++++++++++++++ internal/target/ssh.go | 14 +++-- internal/target/target.go | 9 +++- 10 files changed, 273 insertions(+), 11 deletions(-) diff --git a/cmd/files.go b/cmd/files.go index 9a598a34..e47135e2 100644 --- a/cmd/files.go +++ b/cmd/files.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/target" ) // filesCmd is the parent for "trond files get" and "trond files put". @@ -179,7 +180,7 @@ func runFilesGet(cmd *cobra.Command, args []string) error { if err := os.MkdirAll(filepath.Dir(localDst), 0o755); err != nil { return output.NewError("FILES_ERROR", output.ExitGeneralError, err.Error()) } - if err := os.WriteFile(localDst, data, 0o644); err != nil { + if err := target.WriteLocalFile(localDst, data, 0o600); err != nil { return output.NewError("FILES_ERROR", output.ExitGeneralError, err.Error()) } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 6c347592..fdf1f758 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -258,6 +258,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { // intent hash alone (legacy behavior preserved). if node.Build == nil && opts.Existing != nil && opts.Existing.IntentHash == opts.IntentHash { + remediateJarConfigPermissions(ctx, opts, node) return noChangeResult(opts, nil, start), nil } @@ -278,6 +279,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { opts.Existing.IntentHash == opts.IntentHash && buildSummary != nil && opts.Existing.BuildCacheKey == buildSummary.CacheKey { + remediateJarConfigPermissions(ctx, opts, node) return noChangeResult(opts, buildSummary, start), nil } @@ -448,6 +450,19 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { return res, nil } +func remediateJarConfigPermissions(ctx context.Context, opts Options, node *intent.NodeSpec) { + if opts.Existing == nil || opts.Existing.Runtime != "jar" || node == nil || node.InstallPath == "" { + return + } + perms, ok := opts.Target.(target.Permissions) + if !ok { + return + } + // Best effort preserves the existing no_change result contract. Failure + // observability belongs in a future warnings/audit contract, not this AUD. + _ = perms.Chmod(ctx, filepath.Join(node.InstallPath, "config.conf"), 0o600) +} + // recordedNode returns the managed node this apply would replace, i.e. the // state entry keyed by the intent's name. It prefers opts.Existing (the // caller's own lookup) and otherwise resolves the name against the state it diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index 0aa2d1f3..e6037193 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -14,7 +14,10 @@ import ( // fakeTarget satisfies target.Target with no-op stubs. Apply only // exercises Exec (for the JDK probe + docker deploy fan-out); the // other methods exist to satisfy the interface contract. -type fakeTarget struct{} +type fakeTarget struct { + chmodPaths []string + chmodPerms []os.FileMode +} func (f *fakeTarget) Exec(_ context.Context, _ string, _ ...string) ([]byte, error) { return nil, nil @@ -25,6 +28,11 @@ func (f *fakeTarget) ReadFile(_ context.Context, _ string) ([]byte, error) { ret func (f *fakeTarget) WriteFile(_ context.Context, _ string, _ []byte, _ os.FileMode) error { return nil } +func (f *fakeTarget) Chmod(_ context.Context, path string, perm os.FileMode) error { + f.chmodPaths = append(f.chmodPaths, path) + f.chmodPerms = append(f.chmodPerms, perm) + return nil +} func (f *fakeTarget) DiskFree(_ context.Context, _ string) (uint64, error) { return 1 << 40, nil } func (f *fakeTarget) MemTotal(_ context.Context) (uint64, error) { return 1 << 30, nil } func (f *fakeTarget) PutFile(_ context.Context, _, _ string) error { return nil } @@ -67,6 +75,71 @@ func TestApply_NoOpWhenIntentHashMatches(t *testing.T) { } } +func TestRemediateJarConfigPermissions(t *testing.T) { + tests := []struct { + name string + runtime string + install string + wantChmod bool + }{ + {name: "jar", runtime: "jar", install: "/opt/tron", wantChmod: true}, + {name: "non-jar", runtime: "docker", install: "/opt/tron"}, + {name: "empty install path", runtime: "jar"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ft := &fakeTarget{} + parsed := minimalIntent() + parsed.Target.Runtime = tc.runtime + parsed.Nodes[0].InstallPath = tc.install + existing := state.ManagedNode{Runtime: tc.runtime} + remediateJarConfigPermissions(context.Background(), Options{ + Target: ft, + Existing: &existing, + }, &parsed.Nodes[0]) + if tc.wantChmod { + if len(ft.chmodPaths) != 1 || ft.chmodPaths[0] != "/opt/tron/config.conf" { + t.Fatalf("Chmod calls = %v, want /opt/tron/config.conf", ft.chmodPaths) + } + if ft.chmodPerms[0] != 0o600 { + t.Errorf("Chmod mode = %#o, want 0600", ft.chmodPerms[0]) + } + } else if len(ft.chmodPaths) != 0 { + t.Fatalf("unexpected Chmod calls: %v", ft.chmodPaths) + } + }) + } +} + +func TestApply_NoChangeRemediatesJarConfigPermissions(t *testing.T) { + parsed := minimalIntent() + parsed.Target.Runtime = "jar" + parsed.Nodes[0].InstallPath = "/opt/tron" + store, st := freshStore(t) + ft := &fakeTarget{} + existing := state.ManagedNode{ + Name: parsed.Name, + IntentHash: "same-hash", + Runtime: "jar", + } + res, err := Apply(context.Background(), Options{ + Intent: parsed, Target: ft, Store: store, State: st, + IntentHash: "same-hash", Existing: &existing, + }) + if err != nil { + t.Fatalf("Apply: %v", err) + } + if res.Outcome != "no_change" { + t.Fatalf("Outcome = %q, want no_change", res.Outcome) + } + if len(ft.chmodPaths) != 1 || ft.chmodPaths[0] != "/opt/tron/config.conf" { + t.Fatalf("Chmod calls = %v, want /opt/tron/config.conf", ft.chmodPaths) + } + if ft.chmodPerms[0] != 0o600 { + t.Errorf("Chmod mode = %#o, want 0600", ft.chmodPerms[0]) + } +} + func TestApply_RequiresIntent(t *testing.T) { if _, err := Apply(context.Background(), Options{}); err == nil { t.Error("expected error when Intent is nil") diff --git a/internal/apply/build_test.go b/internal/apply/build_test.go index fd362e64..c9d7356a 100644 --- a/internal/apply/build_test.go +++ b/internal/apply/build_test.go @@ -368,9 +368,10 @@ func TestApply_DirtySourceTriggersRebuild(t *testing.T) { Version: res1.Version, Runtime: "jar", } + ft2 := &fakeTarget{} res2, err := Apply(context.Background(), Options{ Intent: in, - Target: &fakeTarget{}, + Target: ft2, Store: store, State: st, IntentHash: "same-intent-hash", @@ -454,9 +455,10 @@ func TestApply_CleanCacheHitNoChange(t *testing.T) { Version: res1.Version, Runtime: "jar", } + ft2 := &fakeTarget{} res2, err := Apply(context.Background(), Options{ Intent: in, - Target: &fakeTarget{}, + Target: ft2, Store: store, State: st, IntentHash: "stable-hash", @@ -480,6 +482,12 @@ func TestApply_CleanCacheHitNoChange(t *testing.T) { if res2.Build.CacheKey != res1.Build.CacheKey { t.Errorf("cache key drift: %q vs %q", res2.Build.CacheKey, res1.Build.CacheKey) } + if len(ft2.chmodPaths) != 1 || ft2.chmodPaths[0] != filepath.Join(in.Nodes[0].InstallPath, "config.conf") { + t.Errorf("Chmod calls = %v, want jar config path", ft2.chmodPaths) + } + if ft2.chmodPerms[0] != 0o600 { + t.Errorf("Chmod mode = %#o, want 0600", ft2.chmodPerms[0]) + } } // TestApply_RuntimeMustMatchArtifact pins the Phase 2 limitation diff --git a/internal/runtime/jar.go b/internal/runtime/jar.go index d833b00d..bed9d854 100644 --- a/internal/runtime/jar.go +++ b/internal/runtime/jar.go @@ -59,7 +59,7 @@ func (r *JarRuntime) Deploy(ctx context.Context, opts DeployOpts) error { // Write config file. Tracked: systemd has no idea this file exists, // and java-tron parses it once at JVM startup. configPath := filepath.Join(installPath, "config.conf") - if err := tracker.write(ctx, configPath, opts.ConfigData, 0644); err != nil { + if err := tracker.write(ctx, configPath, opts.ConfigData, 0600); err != nil { return fmt.Errorf("write config: %w", err) } diff --git a/internal/runtime/redeploy_test.go b/internal/runtime/redeploy_test.go index f22a643d..f4160cec 100644 --- a/internal/runtime/redeploy_test.go +++ b/internal/runtime/redeploy_test.go @@ -2,9 +2,13 @@ package runtime import ( "context" + "os" + "path/filepath" "slices" "strings" "testing" + + "github.com/tronprotocol/tron-deployment/internal/target" ) // These tests cover one defect with three instances: a deploy writes new @@ -152,6 +156,77 @@ func TestJarRuntime_Deploy_RestartsOnlyWhenSomethingChanged(t *testing.T) { } } +func TestJarRuntime_Deploy_ConfigAndSecretEnvAreOwnerOnly(t *testing.T) { + ft := newFakeTarget() + rt := NewJarRuntime(ft) + opts := DeployOpts{ + Name: "n1", + JarPath: "/opt/tron/FullNode.jar", + ConfigData: []byte(witnessConfig), + SystemdData: []byte("[Service]\n"), + EnvVars: map[string]string{"SR_PRIVATE_KEY": "deadbeef"}, + } + if err := rt.Deploy(context.Background(), opts); err != nil { + t.Fatalf("Deploy: %v", err) + } + + for _, path := range []string{ + "/opt/tron/config.conf", + "/etc/systemd/system/tron-n1.service.d/env.conf", + } { + perm, ok := ft.perms[path] + if !ok { + t.Errorf("no mode recorded for %s", path) + continue + } + if perm != 0600 { + t.Errorf("%s mode = %#o, want 0600", path, perm) + } + } +} + +type jarDiskTarget struct{ *target.LocalTarget } + +func (t *jarDiskTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { + if cmd == "systemctl" { + return nil, nil + } + return t.LocalTarget.Exec(ctx, cmd, args...) +} + +func (t *jarDiskTarget) WriteFile(ctx context.Context, path string, data []byte, perm os.FileMode) error { + if strings.HasPrefix(path, "/etc/systemd/") { + return nil + } + return t.LocalTarget.WriteFile(ctx, path, data, perm) +} + +func TestJarRuntime_Deploy_TightensExistingConfigOnDisk(t *testing.T) { + install := t.TempDir() + configPath := filepath.Join(install, "config.conf") + if err := os.WriteFile(configPath, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + + rt := NewJarRuntime(&jarDiskTarget{LocalTarget: target.NewLocalTarget()}) + opts := DeployOpts{ + Name: "n1", + JarPath: filepath.Join(install, "FullNode.jar"), + ConfigData: []byte(witnessConfig), + SystemdData: []byte("[Service]\n"), + } + if err := rt.Deploy(context.Background(), opts); err != nil { + t.Fatalf("Deploy: %v", err) + } + info, err := os.Stat(configPath) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("config mode = %#o, want 0600", got) + } +} + // TestJarRuntime_Deploy_EnvVarsAllSurvive covers a separate defect in the // same function: the drop-in was written once per key inside the loop, // each write truncating the last, so exactly one variable survived and diff --git a/internal/target/local.go b/internal/target/local.go index 91566d7c..d17f0228 100644 --- a/internal/target/local.go +++ b/internal/target/local.go @@ -115,7 +115,11 @@ func (t *LocalTarget) CommandExists(_ context.Context, name string) bool { var _ Target = (*LocalTarget)(nil) func (t *LocalTarget) Download(_ context.Context, remotePath, localPath string) error { - return copyFile(remotePath, localPath) + data, err := os.ReadFile(remotePath) + if err != nil { + return err + } + return writeFile(localPath, data, 0o600) } func (t *LocalTarget) ReadFile(_ context.Context, path string) ([]byte, error) { @@ -123,7 +127,11 @@ func (t *LocalTarget) ReadFile(_ context.Context, path string) ([]byte, error) { } func (t *LocalTarget) WriteFile(_ context.Context, path string, data []byte, perm os.FileMode) error { - return os.WriteFile(path, data, perm) + return writeFile(path, data, perm) +} + +func (t *LocalTarget) Chmod(_ context.Context, path string, perm os.FileMode) error { + return os.Chmod(path, perm) } func (t *LocalTarget) DiskFree(ctx context.Context, path string) (uint64, error) { @@ -221,3 +229,31 @@ func copyFile(src, dst string) error { } return out.Close() } + +// writeFile enforces permissions before truncating an existing file, so +// sensitive contents are never written while the old mode remains active. +func writeFile(path string, data []byte, perm os.FileMode) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, perm) + if err != nil { + return err + } + if err := f.Chmod(perm); err != nil { + _ = f.Close() + return err + } + if err := f.Truncate(0); err != nil { + _ = f.Close() + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +// WriteLocalFile securely writes a local destination, enforcing permissions +// before truncating existing contents. Used for downloaded files too. +func WriteLocalFile(path string, data []byte, perm os.FileMode) error { + return writeFile(path, data, perm) +} diff --git a/internal/target/local_putfile_test.go b/internal/target/local_putfile_test.go index 0a8fc8c5..6699d952 100644 --- a/internal/target/local_putfile_test.go +++ b/internal/target/local_putfile_test.go @@ -7,6 +7,45 @@ import ( "testing" ) +func TestLocalTargetWriteFileTightensExistingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.conf") + if err := os.WriteFile(path, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := NewLocalTarget().WriteFile(context.Background(), path, []byte("new\n"), 0o600); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("mode = %#o, want 0600", got) + } +} + +func TestLocalTargetDownloadTightensExistingFile(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + dst := filepath.Join(dir, "dst") + if err := os.WriteFile(src, []byte("secret\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dst, []byte("old\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := NewLocalTarget().Download(context.Background(), src, dst); err != nil { + t.Fatal(err) + } + info, err := os.Stat(dst) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("mode = %#o, want 0600", got) + } +} + // TestLocalTarget_PutFile_DifferentPath asserts the standard copy // flow with atomic install (write `.tmp`, rename). Phase 4 needs // this when the cobra path serves a Local target but the caller diff --git a/internal/target/ssh.go b/internal/target/ssh.go index fe2f68be..0f64e2b4 100644 --- a/internal/target/ssh.go +++ b/internal/target/ssh.go @@ -409,7 +409,7 @@ func (t *SSHTarget) Download(ctx context.Context, remotePath, localPath string) return err } - return os.WriteFile(localPath, data, 0644) + return writeFile(localPath, data, 0o600) } func (t *SSHTarget) ReadFile(ctx context.Context, path string) ([]byte, error) { @@ -420,6 +420,14 @@ func (t *SSHTarget) WriteFile(ctx context.Context, path string, data []byte, per return t.writeRemoteFile(ctx, path, data, perm) } +func (t *SSHTarget) Chmod(ctx context.Context, path string, perm os.FileMode) error { + if err := security.ValidateCommand("chmod"); err != nil { + return err + } + _, err := t.Exec(ctx, "chmod", fmt.Sprintf("%o", perm), path) + return err +} + func (t *SSHTarget) DiskFree(ctx context.Context, path string) (uint64, error) { out, err := t.Exec(ctx, "df", "--output=avail", "-k", path) if err != nil { @@ -490,8 +498,8 @@ func (t *SSHTarget) writeRemoteFile(ctx context.Context, path string, data []byt quotedPath := shellQuote(path) quotedDir := shellQuote(filepath.Dir(path)) - cmd := fmt.Sprintf("mkdir -p %s && tee %s > /dev/null && chmod %o %s", - quotedDir, quotedPath, perm, quotedPath) + cmd := fmt.Sprintf("umask 077 && mkdir -p %s && if test -e %s; then chmod %o %s; fi && tee %s > /dev/null && chmod %o %s", + quotedDir, quotedPath, perm, quotedPath, quotedPath, perm, quotedPath) done := make(chan error, 1) go func() { done <- session.Run(cmd) }() diff --git a/internal/target/target.go b/internal/target/target.go index 7782d912..8efc9550 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -121,7 +121,8 @@ type Target interface { // ReadFile reads a file from the target. ReadFile(ctx context.Context, path string) ([]byte, error) - // WriteFile writes data to a file on the target. + // WriteFile writes data to a file on the target. It must enforce perm on + // both newly-created and existing files before writing their contents. WriteFile(ctx context.Context, path string, data []byte, perm os.FileMode) error // DiskFree returns available disk space in bytes at the given path. @@ -152,3 +153,9 @@ type Target interface { // String returns a human-readable description of the target. String() string } + +// Permissions is an optional target capability for tightening an existing +// file without rewriting its contents. +type Permissions interface { + Chmod(ctx context.Context, path string, perm os.FileMode) error +} From 78364f257eec4d9d9d6e13361a86f7534d34fff0 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:05:26 +0800 Subject: [PATCH 09/49] fix(apply): probe jar nodes over target tunnel in WaitForReady (AUD-004) --- internal/apply/apply.go | 2 +- internal/apply/wait.go | 16 ++++++++++------ internal/apply/wait_test.go | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 internal/apply/wait_test.go diff --git a/internal/apply/apply.go b/internal/apply/apply.go index fdf1f758..96f56869 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -439,7 +439,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { _ = builtImageTag // consumed by Phase 3 (docker runtime + image artifact) if opts.Wait { - waitErr := WaitForReady(ctx, opts.Target, opts.Intent.Name, node.Ports.HTTP, opts.WaitTimeout) + waitErr := WaitForReady(ctx, opts.Target, opts.Intent.Name, runtimeType, node.Ports.HTTP, opts.WaitTimeout) res.WaitedMs = time.Since(start).Milliseconds() - deployedMs ready := waitErr == nil res.Ready = &ready diff --git a/internal/apply/wait.go b/internal/apply/wait.go index a478f6bf..e42999b8 100644 --- a/internal/apply/wait.go +++ b/internal/apply/wait.go @@ -8,16 +8,15 @@ import ( "github.com/tronprotocol/tron-deployment/internal/target" ) -// WaitForReady polls the node's HTTP API via `docker exec` until it +// WaitForReady polls the node's HTTP API until it // responds 2xx or the timeout elapses. Used by Apply when Wait=true // and exposed publicly so callers (verify command, MCP tool, recipe // step) can reuse the same probe shape without duplicating the curl // invocation. // -// The probe runs `docker exec curl ...` so it sees the -// container's network — important when host-port mapping is delayed -// on first start. -func WaitForReady(ctx context.Context, tgt target.Target, name string, httpPort int, timeout time.Duration) error { +// Docker probes run inside the container so they see its network. Jar probes +// use target.Get, which tunnels to the target's loopback for remote targets. +func WaitForReady(ctx context.Context, tgt target.Target, name, runtimeName string, httpPort int, timeout time.Duration) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() if httpPort == 0 { @@ -28,7 +27,12 @@ func WaitForReady(ctx context.Context, tgt target.Target, name string, httpPort defer tick.Stop() var lastErr error for { - _, err := tgt.Exec(ctx, "docker", "exec", name, "curl", "-fsS", "--max-time", "5", url) + var err error + if runtimeName == "jar" { + _, err = target.Get(ctx, tgt, url, 5*time.Second) + } else { + _, err = tgt.Exec(ctx, "docker", "exec", name, "curl", "-fsS", "--max-time", "5", url) + } if err == nil { return nil } diff --git a/internal/apply/wait_test.go b/internal/apply/wait_test.go new file mode 100644 index 00000000..a6fd6dc8 --- /dev/null +++ b/internal/apply/wait_test.go @@ -0,0 +1,20 @@ +package apply + +import ( + "context" + "testing" + "time" +) + +func TestWaitForReadyJarUsesTargetHTTP(t *testing.T) { + tgt, port := newHTTPStatusTarget(t, func(path string, _ []byte) (string, bool) { + if path != "/wallet/getnowblock" { + t.Errorf("path = %q, want getnowblock", path) + } + return `{"block_header":{"raw_data":{"number":1}}}`, false + }) + + if err := WaitForReady(context.Background(), tgt, "jar-node", "jar", port, time.Second); err != nil { + t.Fatalf("WaitForReady jar: %v", err) + } +} From c03e75ee02f072f626ee84f6aac93ef5760fa84c Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:09:29 +0800 Subject: [PATCH 10/49] fix(cli): return NODE_NOT_FOUND instead of panicking on empty inspect selection (AUD-006) --- cmd/inspect.go | 18 +++++++++++++++ cmd/inspect_test.go | 53 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 cmd/inspect_test.go diff --git a/cmd/inspect.go b/cmd/inspect.go index b5590531..867522ea 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -123,6 +123,24 @@ func runInspect(cmd *cobra.Command, args []string) error { } nodes = filtered } + if len(nodes) == 0 { + names := make([]string, 0, len(deployState.Nodes)) + for _, n := range deployState.Nodes { + names = append(names, n.Name) + } + message := "No nodes matched the inspect filter" + if inspectNetwork != "" { + message = fmt.Sprintf("No nodes matched network %q", inspectNetwork) + } else if len(inspectLabelFlags) > 0 { + message = "No nodes matched the inspect label filter" + } + suggestions := []string{"Run: trond list"} + if len(names) > 0 { + suggestions = append(suggestions, "Available nodes: "+strings.Join(names, ", ")) + } + return output.NewError("NODE_NOT_FOUND", output.ExitGeneralError, message). + WithSuggestions(suggestions...) + } manifest := buildManifest(cmd.Context(), nodes) diff --git a/cmd/inspect_test.go b/cmd/inspect_test.go new file mode 100644 index 00000000..a7fb74a4 --- /dev/null +++ b/cmd/inspect_test.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "testing" + + "github.com/spf13/cobra" + + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +func TestRunInspectLabelFilterNoMatchReturnsNodeNotFound(t *testing.T) { + t.Setenv("TROND_STATE_DIR", t.TempDir()) + paths.SetBaseDir("") + defer paths.SetBaseDir("") + store, err := state.NewStore(statePath()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{ + {Name: "node-a", Labels: map[string]string{"role": "witness"}}, + {Name: "node-b", Labels: map[string]string{"role": "fullnode"}}, + }}); err != nil { + t.Fatal(err) + } + + oldAll, oldNetwork, oldLabels := inspectAll, inspectNetwork, inspectLabelFlags + defer func() { inspectAll, inspectNetwork, inspectLabelFlags = oldAll, oldNetwork, oldLabels }() + inspectAll = false + inspectNetwork = "" + inspectLabelFlags = []string{"role=missing"} + + cmd := &cobra.Command{} + cmd.Flags().String("output", "json", "") + err = runInspect(cmd, nil) + if err == nil { + t.Fatal("runInspect succeeded for an empty filter result") + } + structured, ok := err.(*output.StructuredError) + if !ok { + t.Fatalf("error type = %T, want *output.StructuredError", err) + } + if structured.Code != "NODE_NOT_FOUND" { + t.Fatalf("error code = %q, want NODE_NOT_FOUND", structured.Code) + } + if structured.ExitCode != output.ExitGeneralError { + t.Fatalf("exit code = %d, want %d", structured.ExitCode, output.ExitGeneralError) + } + if len(structured.Suggestions) == 0 || structured.Suggestions[len(structured.Suggestions)-1] != "Available nodes: node-a, node-b" { + t.Fatalf("suggestions = %v, want available node names", structured.Suggestions) + } +} From 3f4231bb9485d8c05d9e7c5918c64578845da751 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:15:11 +0800 Subject: [PATCH 11/49] fix(network): verify every node during rolling network upgrade (AUD-007) --- cmd/network/upgrade.go | 49 +++++++++++++++++-- cmd/network/upgrade_test.go | 96 +++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 cmd/network/upgrade_test.go diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index 87e69bb1..8d1131c6 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -5,15 +5,18 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "time" "github.com/spf13/cobra" "github.com/tronprotocol/tron-deployment/internal/guard" + "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" + "gopkg.in/yaml.v3" ) // upgradeCmd does a rolling upgrade across every node in a private @@ -165,9 +168,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error { } stepStart = time.Now() - err = runChild(cmd.Context(), exe, "verify", - "--intent", upgradeIntentPath, - "--timeout", upgradeVerifyTimeout.String()) + err = verifyNode(cmd.Context(), exe, node, upgradeIntentPath, upgradeVerifyTimeout) steps = append(steps, upgradeStep{ Node: node, Phase: "verify", @@ -233,7 +234,9 @@ func isWitnessNode(n state.ManagedNode) bool { return strings.Contains(strings.ToLower(n.Name), "witness") } -func runChild(ctx context.Context, exe string, argv ...string) error { +var runChild = runChildCommand + +func runChildCommand(ctx context.Context, exe string, argv ...string) error { argv = append(argv, "--output", "json") cmd := exec.CommandContext(ctx, exe, argv...) cmd.Stdout = os.Stdout @@ -241,6 +244,44 @@ func runChild(ctx context.Context, exe string, argv ...string) error { return cmd.Run() } +// verifyNode projects the multi-node intent to the node being upgraded before +// invoking the existing single-node verify command. Verify reads Nodes[0], so +// passing the original multi-node intent would silently verify only node 0. +func verifyNode(ctx context.Context, exe, node, intentPath string, timeout time.Duration) error { + parsed, err := intent.Load(intentPath) + if err != nil { + return fmt.Errorf("load verify intent: %w", err) + } + var projected *intent.Intent + for i := range parsed.Nodes { + if fmt.Sprintf("%s-node%d", parsed.Name, i) == node { + projected, _, _, err = nodeIntent(parsed, i) + break + } + } + if projected == nil { + return fmt.Errorf("node %q is not present in verify intent", node) + } + data, err := yaml.Marshal(projected) + if err != nil { + return fmt.Errorf("marshal verify intent for %s: %w", node, err) + } + tmp, err := os.CreateTemp(filepath.Dir(intentPath), ".trond-verify-*.yaml") + if err != nil { + return fmt.Errorf("create verify intent for %s: %w", node, err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return fmt.Errorf("write verify intent for %s: %w", node, err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close verify intent for %s: %w", node, err) + } + return runChild(ctx, exe, "verify", "--intent", tmpPath, "--timeout", timeout.String()) +} + func statusFor(err error) string { if err == nil { return "ok" diff --git a/cmd/network/upgrade_test.go b/cmd/network/upgrade_test.go new file mode 100644 index 00000000..dfe7f9d3 --- /dev/null +++ b/cmd/network/upgrade_test.go @@ -0,0 +1,96 @@ +package network + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestVerifyNodeInvokesVerifyForEachProjectedNode(t *testing.T) { + dir := t.TempDir() + intentPath := filepath.Join(dir, "network.yaml") + if err := os.WriteFile(intentPath, []byte(`name: net +network: private +target: + type: local +nodes: + - type: fullnode + version: latest + - type: witness + version: latest + witness_key_env: SR_KEY +`), 0600); err != nil { + t.Fatal(err) + } + + oldRunChild := runChild + defer func() { runChild = oldRunChild }() + var intents []string + runChild = func(_ context.Context, _ string, argv ...string) error { + for i := range argv { + if argv[i] == "--intent" && i+1 < len(argv) { + data, err := os.ReadFile(argv[i+1]) + if err != nil { + t.Fatalf("read projected intent: %v", err) + } + intents = append(intents, string(data)) + } + } + return nil + } + + for i, node := range []string{"net-node0", "net-node1"} { + if err := verifyNode(context.Background(), "trond", node, intentPath, time.Second); err != nil { + t.Fatalf("verifyNode(%s): %v", node, err) + } + wantName := []string{"name: net-node0", "name: net-node1"}[i] + if !strings.Contains(intents[i], wantName) { + t.Fatalf("projected intent %q does not target %s", intents[i], node) + } + } + if len(intents) != 2 { + t.Fatalf("verify calls = %d, want 2", len(intents)) + } +} + +func TestVerifyNodeFailureReturnedForFailingNode(t *testing.T) { + dir := t.TempDir() + intentPath := filepath.Join(dir, "network.yaml") + if err := os.WriteFile(intentPath, []byte(`name: net +network: private +target: + type: local +nodes: + - type: fullnode + version: latest + - type: fullnode + version: latest +`), 0600); err != nil { + t.Fatal(err) + } + + oldRunChild := runChild + defer func() { runChild = oldRunChild }() + var calls []string + runChild = func(_ context.Context, _ string, argv ...string) error { + calls = append(calls, strings.Join(argv, " ")) + if len(calls) == 2 { + return errors.New("verify node net-node1 failed") + } + return nil + } + if err := verifyNode(context.Background(), "trond", "net-node0", intentPath, time.Second); err != nil { + t.Fatal(err) + } + err := verifyNode(context.Background(), "trond", "net-node1", intentPath, time.Second) + if err == nil || !strings.Contains(err.Error(), "verify node net-node1 failed") { + t.Fatalf("error = %v, want failing node verify error", err) + } + if len(calls) != 2 { + t.Fatalf("verify calls = %d, want 2", len(calls)) + } +} From a349b1bd5ee0b01b5967f8a3b5ea27d6fd907ab5 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:17:38 +0800 Subject: [PATCH 12/49] fix(network): roll back nodes whose verify failed after upgrade (AUD-008) --- cmd/network/upgrade.go | 5 ++++- cmd/network/upgrade_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index 8d1131c6..505902d9 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -166,6 +166,10 @@ func runUpgrade(cmd *cobra.Command, args []string) error { return finishWithFailure(cmd.Context(), outputFmt, networkName, exe, steps, upgraded, start, node, err) } + // The artifact has been changed successfully. Keep this node in the + // rollback set before verification: a failed health check still means + // this node was mutated and must be restored by --auto-rollback. + upgraded = append(upgraded, node) stepStart = time.Now() err = verifyNode(cmd.Context(), exe, node, upgradeIntentPath, upgradeVerifyTimeout) @@ -181,7 +185,6 @@ func runUpgrade(cmd *cobra.Command, args []string) error { steps, upgraded, start, node, err) } - upgraded = append(upgraded, node) } result := map[string]any{ diff --git a/cmd/network/upgrade_test.go b/cmd/network/upgrade_test.go index dfe7f9d3..d4677554 100644 --- a/cmd/network/upgrade_test.go +++ b/cmd/network/upgrade_test.go @@ -94,3 +94,31 @@ nodes: t.Fatalf("verify calls = %d, want 2", len(calls)) } } + +func TestFinishWithFailureRollsBackNodeWhoseVerifyFailed(t *testing.T) { + oldAutoRollback := upgradeAutoRollback + oldRunChild := runChild + defer func() { + upgradeAutoRollback = oldAutoRollback + runChild = oldRunChild + }() + upgradeAutoRollback = true + var rollbackNodes []string + runChild = func(_ context.Context, _ string, argv ...string) error { + if len(argv) >= 2 && argv[0] == "rollback" { + rollbackNodes = append(rollbackNodes, argv[1]) + } + return nil + } + + err := finishWithFailure(context.Background(), "", "net", "trond", + []upgradeStep{{Node: "net-node0", Phase: "upgrade", Status: "ok"}, + {Node: "net-node0", Phase: "verify", Status: "failed"}}, + []string{"net-node0"}, time.Now(), "net-node0", errors.New("verify failed")) + if err == nil || !strings.Contains(err.Error(), "net-node0") { + t.Fatalf("error = %v, want failed node", err) + } + if len(rollbackNodes) != 1 || rollbackNodes[0] != "net-node0" { + t.Fatalf("rollback nodes = %v, want [net-node0]", rollbackNodes) + } +} From 7862f5bed2e5ecd8fc0fef21e6a241db200c4abd Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:26:24 +0800 Subject: [PATCH 13/49] fix(replay): advance cursor only after a block fully replays (AUD-009) --- tools/replay/replayer.go | 37 ++++--- tools/replay/replayer_test.go | 183 ++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 14 deletions(-) create mode 100644 tools/replay/replayer_test.go diff --git a/tools/replay/replayer.go b/tools/replay/replayer.go index a06f10c7..56e13cce 100644 --- a/tools/replay/replayer.go +++ b/tools/replay/replayer.go @@ -84,6 +84,13 @@ func (r *Replayer) Run(ctx context.Context) error { "check %s for failure details", r.state.LastMainnetBlock, n, r.cfg.FailLog) return err } + if ctx.Err() != nil { + r.state.InProgressTxIndex = 0 + if flushErr := r.flushState(); flushErr != nil { + log.Printf("save state failed: %v", flushErr) + } + return ctx.Err() + } // One flush atomically clears the in-progress markers and advances // LastMainnetBlock, so "block done" is a single atomic event. If a // SIGKILL lands between the two writes, the next start enters @@ -157,14 +164,13 @@ func (r *Replayer) resolveStartBlock(_ context.Context) (int64, error) { // - Total timer invocations: `slots` (typically 3-9), not `n` (typically 150). // // On fetch failure we still wait for the full paceTotal to keep the -// schedule steady. Empty blocks / non-existent blocks return immediately -// without burning the full pace so replay advances faster. +// schedule steady, then abort without advancing the cursor. Empty blocks / +// non-existent blocks return immediately without burning the full pace. // // Returns (blockOk, blockFail, blockSkip, err). -// err is non-nil iff this block attempted broadcasts (attempted > 0) and -// all of them failed (ok == 0). In that case the caller (Run) should -// stop the service to avoid blindly advancing state while the private -// chain is in a bad state. +// err is non-nil for fetch failures, cancellation, or any failed broadcast. +// In each case the caller (Run) must stop before advancing state so an +// interrupted or incomplete block is retried on the next run. func (r *Replayer) processBlock(ctx context.Context, num int64) (int64, int64, int64, error) { blockStart := time.Now() @@ -180,9 +186,12 @@ func (r *Replayer) processBlock(ctx context.Context, num int64) (int64, int64, i blk, err := r.trongrid.getBlock(ctx, num) if err != nil { + if ctx.Err() != nil { + return 0, 0, 0, ctx.Err() + } log.Printf("block %d fetch failed: %v", num, err) r.waitUntil(ctx, blockStart.Add(paceTotal)) - return 0, 0, 0, nil // fetch failure is a TronGrid-side issue; not "all-fail", continue + return 0, 0, 0, fmt.Errorf("block %d fetch failed: %w", num, err) } if blk == nil { log.Printf("block %d not found", num) @@ -232,7 +241,7 @@ func (r *Replayer) processBlock(ctx context.Context, num int64) (int64, int64, i return atomic.LoadInt64(&r.state.TotalBroadcastOk) - okBefore, atomic.LoadInt64(&r.state.TotalBroadcastFail) - failBefore, atomic.LoadInt64(&r.state.TotalSkipped) - skipBefore, - nil + ctx.Err() default: } r.processTx(ctx, num, blk.Transactions[idx]) @@ -266,7 +275,7 @@ func (r *Replayer) processBlock(ctx context.Context, num int64) (int64, int64, i return atomic.LoadInt64(&r.state.TotalBroadcastOk) - okBefore, atomic.LoadInt64(&r.state.TotalBroadcastFail) - failBefore, atomic.LoadInt64(&r.state.TotalSkipped) - skipBefore, - nil + ctx.Err() default: } r.processTx(ctx, num, blk.Transactions[idx]) @@ -286,12 +295,12 @@ func (r *Replayer) processBlock(ctx context.Context, num int64) (int64, int64, i blockFail := atomic.LoadInt64(&r.state.TotalBroadcastFail) - failBefore blockSkip := atomic.LoadInt64(&r.state.TotalSkipped) - skipBefore - // If we attempted broadcasts (blockFail > 0) and all of them failed - // (blockOk == 0), trigger a shutdown. Skipped txs (VoteWitness etc.) - // are excluded because we never attempted to broadcast them. - if ctx.Err() == nil && blockFail > 0 && blockOk == 0 { + // Any failed broadcast means the block is incomplete. Do not advance the + // mainnet cursor: a retry must replay from the durable block boundary. + // Skipped txs (VoteWitness etc.) are excluded because they are intentional. + if ctx.Err() == nil && blockFail > 0 { return blockOk, blockFail, blockSkip, - fmt.Errorf("block %d: all %d broadcast attempts failed", num, blockFail) + fmt.Errorf("block %d: %d broadcast attempts failed", num, blockFail) } return blockOk, blockFail, blockSkip, nil } diff --git a/tools/replay/replayer_test.go b/tools/replay/replayer_test.go new file mode 100644 index 00000000..ff06b7fe --- /dev/null +++ b/tools/replay/replayer_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/tronprotocol/tron-deployment/tools/common/broadcast" +) + +func TestRunCancellationDoesNotAdvanceCursor(t *testing.T) { + var ( + mu sync.Mutex + cancel context.CancelFunc + calls int + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/wallet/getblockbynum" { + _, _ = w.Write([]byte(`{"block_header":{"raw_data":{"number":100}},"transactions":[{"txID":"tx-1","raw_data":{"contract":[{"type":"TransferContract"}]}}]}`)) + return + } + if req.URL.Path == "/wallet/broadcasttransaction" { + mu.Lock() + calls++ + if cancel != nil { + cancel() + cancel = nil + } + mu.Unlock() + _, _ = w.Write([]byte(`{"result":true}`)) + return + } + http.NotFound(w, req) + })) + defer server.Close() + + ctx, stop := context.WithCancel(context.Background()) + cancel = stop + defer stop() + stateDir := t.TempDir() + statePath := filepath.Join(stateDir, "state.json") + failLog := filepath.Join(stateDir, "failures.jsonl") + skipLog := filepath.Join(stateDir, "skips.jsonl") + fail, err := openJsonlLogger(failLog) + if err != nil { + t.Fatal(err) + } + defer fail.Close() + skip, err := openJsonlLogger(skipLog) + if err != nil { + t.Fatal(err) + } + defer skip.Close() + + r := &Replayer{ + cfg: Config{ + TrongridURL: server.URL, + TrongridQPS: 1000, + PrivateNode: server.URL, + Start: 100, + End: 100, + StateFile: statePath, + FailLog: failLog, + SkipLog: skipLog, + TpsMultiplier: 100, + IncludeAll: true, + }, + trongrid: newTronGridClient(server.URL, "", 1000), + private: newBroadcastClientForTest(server.URL), + state: ReplayState{LastMainnetBlock: 99}, + skipTypes: map[string]struct{}{}, + failLog: fail, + skipLog: skip, + } + + err = r.Run(ctx) + if err == nil { + t.Fatal("Run succeeded after cancellation") + } + got := loadState(statePath) + if got.LastMainnetBlock != 99 { + t.Fatalf("last_mainnet_block = %d, want 99", got.LastMainnetBlock) + } + if calls != 1 { + t.Fatalf("broadcast calls = %d, want 1", calls) + } +} + +func TestRunBroadcastFailureDoesNotAdvanceCursor(t *testing.T) { + var broadcasts int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/wallet/getblockbynum": + _, _ = w.Write([]byte(`{"block_header":{"raw_data":{"number":100}},"transactions":[{"txID":"tx-1","raw_data":{"contract":[{"type":"TransferContract"}]}},{"txID":"tx-2","raw_data":{"contract":[{"type":"TransferContract"}]}}]}`)) + case "/wallet/broadcasttransaction": + broadcasts++ + if broadcasts == 1 { + _, _ = w.Write([]byte(`{"result":true}`)) + } else { + _, _ = w.Write([]byte(`{"result":false,"code":"FAIL","message":"failed"}`)) + } + default: + http.NotFound(w, req) + } + })) + defer server.Close() + + stateDir := t.TempDir() + statePath := filepath.Join(stateDir, "state.json") + failLog := filepath.Join(stateDir, "failures.jsonl") + skipLog := filepath.Join(stateDir, "skips.jsonl") + fail, err := openJsonlLogger(failLog) + if err != nil { + t.Fatal(err) + } + defer fail.Close() + skip, err := openJsonlLogger(skipLog) + if err != nil { + t.Fatal(err) + } + defer skip.Close() + r := &Replayer{ + cfg: Config{TrongridURL: server.URL, TrongridQPS: 1000, PrivateNode: server.URL, Start: 100, End: 100, StateFile: statePath, FailLog: failLog, SkipLog: skipLog, TpsMultiplier: 100, IncludeAll: true}, + trongrid: newTronGridClient(server.URL, "", 1000), private: newBroadcastClientForTest(server.URL), + state: ReplayState{LastMainnetBlock: 99}, skipTypes: map[string]struct{}{}, failLog: fail, skipLog: skip, + } + if err := r.Run(context.Background()); err == nil { + t.Fatal("Run succeeded after a broadcast failure") + } + if got := loadState(statePath).LastMainnetBlock; got != 99 { + t.Fatalf("last_mainnet_block = %d, want 99", got) + } +} + +func TestRunFetchFailureDoesNotAdvanceCursor(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path == "/wallet/getblockbynum" { + http.Error(w, "upstream unavailable", http.StatusBadGateway) + return + } + http.NotFound(w, req) + })) + defer server.Close() + + stateDir := t.TempDir() + statePath := filepath.Join(stateDir, "state.json") + failLog := filepath.Join(stateDir, "failures.jsonl") + skipLog := filepath.Join(stateDir, "skips.jsonl") + fail, err := openJsonlLogger(failLog) + if err != nil { + t.Fatal(err) + } + defer fail.Close() + skip, err := openJsonlLogger(skipLog) + if err != nil { + t.Fatal(err) + } + defer skip.Close() + r := &Replayer{ + cfg: Config{TrongridURL: server.URL, TrongridQPS: 1000, PrivateNode: server.URL, Start: 100, End: 100, StateFile: statePath, FailLog: failLog, SkipLog: skipLog, TpsMultiplier: 100, IncludeAll: true}, + trongrid: newTronGridClient(server.URL, "", 1000), private: newBroadcastClientForTest(server.URL), + state: ReplayState{LastMainnetBlock: 99}, skipTypes: map[string]struct{}{}, failLog: fail, skipLog: skip, + } + err = r.Run(context.Background()) + if err == nil || !strings.Contains(err.Error(), "fetch failed") { + t.Fatalf("error = %v, want fetch failure", err) + } + if got := loadState(statePath).LastMainnetBlock; got != 99 { + t.Fatalf("last_mainnet_block = %d, want 99", got) + } +} + +// The production broadcast client has no test constructor; keep this helper +// local so the test can use the same HTTP implementation without changing the +// shared package API. +func newBroadcastClientForTest(url string) *broadcast.Client { + return broadcast.NewWithTimeout(url, time.Second) +} From efb4967f8dc957d49e19019409a51b971dc13040 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:32:25 +0800 Subject: [PATCH 14/49] fix(snapshot): verify process identity before signaling stop AUD-010: snapshot stop no longer signals a recorded PID blindly. On Linux it reads /proc//cmdline, on macOS it falls back to ps, and only sends SIGTERM/SIGKILL when the process is a trond binary running 'snapshot download'. Mismatches fail closed with PROCESS_IDENTITY_MISMATCH; dead PIDs keep the existing stale manifest cleanup. Closes the PID-reuse accidental-kill risk. --- cmd/snapshot/jobs.go | 48 ++++++++++++++++++++++++++++- cmd/snapshot/jobs_test.go | 63 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 cmd/snapshot/jobs_test.go diff --git a/cmd/snapshot/jobs.go b/cmd/snapshot/jobs.go index 0f3427a2..d8847f14 100644 --- a/cmd/snapshot/jobs.go +++ b/cmd/snapshot/jobs.go @@ -4,6 +4,9 @@ import ( "fmt" "io" "os" + "os/exec" + "path/filepath" + "strings" "syscall" "text/tabwriter" "time" @@ -161,9 +164,22 @@ func runStop(cmd *cobra.Command, args []string) error { return output.NewError("JOB_NOT_FOUND", output.ExitGeneralError, "no such job: "+id) } if !snapshot.IsRunning(job.PID) { - fmt.Printf("Job %s already stopped (pid %d).\n", id, job.PID) + // ESRCH means the recorded process is gone. Remove stale bookkeeping; + // unlike an identity mismatch, there is no live process to protect. + if err := snapshot.RemoveJob(paths.SnapshotJobs(), id); err != nil { + return output.NewError("STOP_ERROR", output.ExitGeneralError, err.Error()) + } + fmt.Printf("Job %s already stopped (pid %d); removed stale manifest.\n", id, job.PID) return nil } + if ok, err := snapshotProcessIdentity(job.PID); err != nil || !ok { + message := fmt.Sprintf("refusing to signal pid %d: process identity does not match snapshot download", job.PID) + if err != nil { + message = fmt.Sprintf("%s (%v)", message, err) + } + return output.NewError("PROCESS_IDENTITY_MISMATCH", output.ExitGeneralError, message). + WithSuggestions("Run: trond snapshot jobs to verify the recorded process") + } sig := syscall.SIGTERM if stopForce { sig = syscall.SIGKILL @@ -179,6 +195,36 @@ func runStop(cmd *cobra.Command, args []string) error { return nil } +// snapshotProcessIdentity verifies the live PID still belongs to the detached +// trond snapshot download. This check is intentionally immediately before +// Signal; PID reuse between the check and signal remains a small unavoidable +// TOCTOU window, but an unrelated live PID is never signalled blindly. +func snapshotProcessIdentity(pid int) (bool, error) { + var argv []string + if data, err := os.ReadFile(filepath.Join("/proc", fmt.Sprint(pid), "cmdline")); err == nil { + for _, arg := range strings.Split(string(data), "\x00") { + if arg != "" { + argv = append(argv, arg) + } + } + } else { + out, psErr := exec.Command("ps", "-p", fmt.Sprint(pid), "-o", "command=").Output() + if psErr != nil { + return false, psErr + } + argv = strings.Fields(strings.TrimSpace(string(out))) + } + if len(argv) == 0 || !strings.Contains(argv[0], "trond") { + return false, nil + } + for i := 0; i+1 < len(argv); i++ { + if argv[i] == "snapshot" && argv[i+1] == "download" { + return true, nil + } + } + return false, nil +} + func truncate(s string, n int) string { if len(s) <= n { return s diff --git a/cmd/snapshot/jobs_test.go b/cmd/snapshot/jobs_test.go new file mode 100644 index 00000000..b6857dce --- /dev/null +++ b/cmd/snapshot/jobs_test.go @@ -0,0 +1,63 @@ +package snapshot + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/spf13/cobra" + + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/snapshot" +) + +func TestSnapshotProcessIdentityRejectsSleep(t *testing.T) { + proc := exec.Command("sleep", "30") + if err := proc.Start(); err != nil { + t.Fatal(err) + } + defer proc.Process.Kill() + defer proc.Wait() + + ok, err := snapshotProcessIdentity(proc.Process.Pid) + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatalf("sleep pid %d incorrectly matched snapshot download", proc.Process.Pid) + } +} + +func TestSnapshotProcessIdentityRejectsSelf(t *testing.T) { + ok, err := snapshotProcessIdentity(os.Getpid()) + if err != nil { + t.Fatal(err) + } + if ok { + t.Fatal("test process incorrectly matched snapshot download") + } +} + +func TestRunStopMissingPIDRemovesManifest(t *testing.T) { + base := t.TempDir() + paths.SetBaseDir(base) + defer paths.SetBaseDir("") + jobsDir := paths.SnapshotJobs() + job := snapshot.Job{ + ID: "gone", PID: 999999, StartedAt: time.Now().UTC(), + LogPath: filepath.Join(jobsDir, "gone.log"), + } + if err := snapshot.WriteJob(jobsDir, job); err != nil { + t.Fatal(err) + } + + cmd := &cobra.Command{} + if err := runStop(cmd, []string{"gone"}); err != nil { + t.Fatalf("runStop: %v", err) + } + if _, err := os.Stat(filepath.Join(jobsDir, "gone.json")); !os.IsNotExist(err) { + t.Fatalf("manifest still exists, stat error = %v", err) + } +} From 767787fab6c1f94f59c48602603b9ecf33ead850 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:40:47 +0800 Subject: [PATCH 15/49] fix(build): fold untracked file contents into dirty cache key AUD-012: new untracked files in the source tree (new .go files, patch inputs) previously did not affect the dirty hash, so the cache key could match while build inputs differed. Dirty hashing now lists untracked files via 'git status --porcelain=v1 -z --untracked-files=all', sorts them by path, and streams each file's index, size, path and content into the SHA-256. Ignored files are excluded and the no-untracked case is byte-identical to the previous behavior. --- internal/build/source.go | 53 +++++++++++++++++++++++++++++-- internal/build/source_test.go | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/internal/build/source.go b/internal/build/source.go index b27e5aa8..7eef4945 100644 --- a/internal/build/source.go +++ b/internal/build/source.go @@ -5,8 +5,11 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "io" + "os" "os/exec" "path/filepath" + "sort" "strings" ) @@ -60,13 +63,14 @@ func (s *Source) Resolve(ctx context.Context) error { // the concatenation of: // // git diff HEAD (tracked + staged + unstaged) -// git status --porcelain -uall (untracked files + modes) +// git status --porcelain=v1 -z -uall (untracked files + modes) +// contents of each untracked file, sorted by path func (s *Source) computeDirty(ctx context.Context) (bool, string, error) { diff, err := s.runGit(ctx, "diff", "HEAD") if err != nil { return false, "", fmt.Errorf("git diff HEAD: %w", err) } - status, err := s.runGit(ctx, "status", "--porcelain", "-uall") + status, err := s.runGit(ctx, "status", "--porcelain=v1", "-z", "--untracked-files=all") if err != nil { return false, "", fmt.Errorf("git status --porcelain -uall: %w", err) } @@ -81,9 +85,54 @@ func (s *Source) computeDirty(ctx context.Context) (bool, string, error) { // combination. h.Write([]byte{0}) h.Write([]byte(status)) + paths := untrackedPaths(status) + sort.Strings(paths) + for i, path := range paths { + if err := foldUntrackedFile(h, i, s.Path, path); err != nil { + return false, "", err + } + } return true, hex.EncodeToString(h.Sum(nil)), nil } +// untrackedPaths extracts paths from git's NUL-terminated porcelain output. +// -z disables quoting, so paths containing whitespace or newlines remain +// unambiguous. Only ?? entries are content that is absent from git diff. +func untrackedPaths(status string) []string { + var paths []string + for _, record := range strings.Split(status, "\x00") { + if len(record) >= 3 && record[:3] == "?? " { + paths = append(paths, record[3:]) + } + } + return paths +} + +func foldUntrackedFile(h io.Writer, index int, root, path string) error { + fullPath := filepath.Join(root, filepath.FromSlash(path)) + file, err := os.Open(fullPath) + if err != nil { + return fmt.Errorf("read untracked file %q: %w", path, err) + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return fmt.Errorf("stat untracked file %q: %w", path, err) + } + // Include index, path, and size as framing; then stream the contents so + // large source files do not require a second in-memory copy. + if _, err := fmt.Fprintf(h, "untracked %d %d\x00%s\x00", index, info.Size(), path); err != nil { + return fmt.Errorf("hash untracked file %q: %w", path, err) + } + if _, err := io.Copy(h, file); err != nil { + return fmt.Errorf("hash untracked file %q: %w", path, err) + } + if _, err := io.WriteString(h, "\x00"); err != nil { + return fmt.Errorf("hash untracked file %q: %w", path, err) + } + return nil +} + func (s *Source) runGit(ctx context.Context, args ...string) (string, error) { cmd := exec.CommandContext(ctx, "git", append([]string{"-C", s.Path}, args...)...) out, err := cmd.Output() diff --git a/internal/build/source_test.go b/internal/build/source_test.go index 87972b27..d87d14f7 100644 --- a/internal/build/source_test.go +++ b/internal/build/source_test.go @@ -61,6 +61,7 @@ func TestSource_Resolve_DirtyWithUntracked(t *testing.T) { if err := s1.Resolve(context.Background()); err != nil { t.Fatalf("first Resolve: %v", err) } + cleanHash := s1.PatchHash // Add an untracked file. git diff returns empty here; trond MUST // notice via git status --porcelain -uall. @@ -79,6 +80,64 @@ func TestSource_Resolve_DirtyWithUntracked(t *testing.T) { if s2.PatchHash == "" { t.Fatal("dirty repo MUST have non-empty PatchHash") } + if s2.PatchHash == cleanHash { + t.Fatal("adding an untracked file MUST change PatchHash") + } +} + +func TestSource_Resolve_UntrackedContentChangesHash(t *testing.T) { + dir := initGitRepo(t) + path := filepath.Join(dir, "new.go") + if err := os.WriteFile(path, []byte("package first\n"), 0o600); err != nil { + t.Fatal(err) + } + first := resolveSource(t, dir).PatchHash + if err := os.WriteFile(path, []byte("package second\n"), 0o600); err != nil { + t.Fatal(err) + } + second := resolveSource(t, dir).PatchHash + if first == second { + t.Fatalf("untracked content change did not change hash: %q", first) + } +} + +func TestSource_Resolve_NoUntrackedHashUnchanged(t *testing.T) { + dir := initGitRepo(t) + first := resolveSource(t, dir).PatchHash + second := resolveSource(t, dir).PatchHash + if first != second || first != "" { + t.Fatalf("clean hash changed: %q then %q", first, second) + } +} + +func TestSource_Resolve_IgnoredUntrackedDoesNotChangeHash(t *testing.T) { + dir := initGitRepo(t) + baseline := resolveSource(t, dir).PatchHash + if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("ignored/\n"), 0o600); err != nil { + t.Fatal(err) + } + mustGit(t, dir, "add", ".gitignore") + mustGit(t, dir, "commit", "-m", "ignore generated files") + baseline = resolveSource(t, dir).PatchHash + if err := os.Mkdir(filepath.Join(dir, "ignored"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "ignored", "output.bin"), []byte("one"), 0o600); err != nil { + t.Fatal(err) + } + got := resolveSource(t, dir).PatchHash + if got != baseline { + t.Fatalf("ignored file changed hash: %q then %q", baseline, got) + } +} + +func resolveSource(t *testing.T, dir string) Source { + t.Helper() + s := Source{Path: dir, RevisionSpec: "HEAD"} + if err := s.Resolve(context.Background()); err != nil { + t.Fatalf("Resolve: %v", err) + } + return s } func TestSource_Resolve_DirtyTrackedEdit(t *testing.T) { From c39f7fc979b37bafe60f505380098c2d070b42a1 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 12:40:57 +0800 Subject: [PATCH 16/49] fix(render): fail loudly on unparseable memory values AUD-014: ParseMemoryGB now returns (int, error) instead of silently returning 0 for MB forms, decimals and malformed input (which callers then fell back to 16GB). It accepts integer GB ('8GB'/'8g'/'8'), MB ('1024m'/'1024MB') and fractional GB ('2.5GB'), rounding up to whole GB, and rejects empty, malformed, negative, zero and unknown-unit values. All callers (apply, preflight memory checks, config render, explain-intent, network add, MCP config render) now surface a validation error instead of silently deploying with 16GB. --- cmd/config/explain_intent.go | 7 ++++++- cmd/config/render.go | 7 ++++--- cmd/network/add.go | 7 ++++--- cmd/preflight.go | 16 +++++++++++++--- cmd/preflight_test.go | 7 ++++--- internal/apply/apply.go | 6 +++--- internal/mcp/tools_config.go | 7 ++++--- internal/render/golden_test.go | 6 +++--- internal/render/jvm.go | 32 ++++++++++++++++++++------------ internal/render/jvm_test.go | 22 +++++++++++++++++----- 10 files changed, 78 insertions(+), 39 deletions(-) diff --git a/cmd/config/explain_intent.go b/cmd/config/explain_intent.go index 9029e35f..970576c0 100644 --- a/cmd/config/explain_intent.go +++ b/cmd/config/explain_intent.go @@ -76,7 +76,12 @@ func explainNode(w io.Writer, raw, final *intent.NodeSpec) { mem := final.Resources.Memory rawMem := r.Resources.Memory row(w, mark(rawMem), "resources.memory", mem, rawMem == "") - heap := render.JVMArgsString(render.ParseMemoryGB(mem), 17, final.JVM) + memGB, memErr := render.ParseMemoryGB(mem) + if memErr != nil { + fmt.Fprintf(w, " ! JVM heap not derived: invalid resources.memory %q (%v)\n", mem, memErr) + return + } + heap := render.JVMArgsString(memGB, 17, final.JVM) xmx := firstFlag(heap, "-Xmx") if xmx != "" { fmt.Fprintf(w, " → JVM heap derived: %s\n", xmx) diff --git a/cmd/config/render.go b/cmd/config/render.go index 03665efe..94bf5c94 100644 --- a/cmd/config/render.go +++ b/cmd/config/render.go @@ -100,9 +100,10 @@ func runRender(cmd *cobra.Command, args []string) error { // host memory, so we size from the intent's resources.memory and // default to JDK 17 — both are safe static assumptions for the // `config render` preview path. - memGB := render.ParseMemoryGB(node.Resources.Memory) - if memGB == 0 { - memGB = 16 + memGB, err := render.ParseMemoryGB(node.Resources.Memory) + if err != nil { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, + fmt.Sprintf("invalid resources.memory %q: %v", node.Resources.Memory, err)) } jvmArgs := render.JVMArgsString(memGB, 17, node.JVM) diff --git a/cmd/network/add.go b/cmd/network/add.go index 4c9a43f0..49054b06 100644 --- a/cmd/network/add.go +++ b/cmd/network/add.go @@ -181,9 +181,10 @@ func runAdd(cmd *cobra.Command, args []string) error { // Deploy path — needs the real witness key inlined. hocon := rendered.Deployable() - memGB := render.ParseMemoryGB(node.Resources.Memory) - if memGB == 0 { - memGB = 16 + memGB, err := render.ParseMemoryGB(node.Resources.Memory) + if err != nil { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, + fmt.Sprintf("invalid resources.memory %q: %v", node.Resources.Memory, err)) } jvmArgs := render.JVMArgsString(memGB, 17, node.JVM) composeYAML := render.RenderCompose(nodeName, parsed, node, "", jvmArgs, "") diff --git a/cmd/preflight.go b/cmd/preflight.go index 3b5b1260..8441a21d 100644 --- a/cmd/preflight.go +++ b/cmd/preflight.go @@ -182,7 +182,12 @@ func checkMemory(cmd *cobra.Command, tgt target.Target, parsed *intent.Intent) c // so the threshold stayed at the hardcoded 16 GB regardless of intent. minGB := uint64(0) for _, node := range parsed.Nodes { - if g := uint64(render.ParseMemoryGB(node.Resources.Memory)); g > minGB { + g, err := render.ParseMemoryGB(node.Resources.Memory) + if err != nil { + return checkResult{Name: "memory", Status: "fail", + Message: fmt.Sprintf("node memory %q is invalid: %v", node.Resources.Memory, err)} + } + if g := uint64(g); g > minGB { minGB = g } } @@ -203,8 +208,13 @@ func checkMemory(cmd *cobra.Command, tgt target.Target, parsed *intent.Intent) c func checkMemoryRecommended(parsed *intent.Intent) []checkResult { var checks []checkResult for i, node := range parsed.Nodes { - memoryGB := render.ParseMemoryGB(node.Resources.Memory) - if memoryGB <= 0 || memoryGB >= 8 { + memoryGB, err := render.ParseMemoryGB(node.Resources.Memory) + if err != nil { + checks = append(checks, checkResult{Name: "memory-recommended", Status: "fail", + Message: fmt.Sprintf("node memory %q is invalid: %v", node.Resources.Memory, err)}) + continue + } + if memoryGB >= 8 { continue } nodeName := parsed.Name diff --git a/cmd/preflight_test.go b/cmd/preflight_test.go index 79c0e60f..a3419a48 100644 --- a/cmd/preflight_test.go +++ b/cmd/preflight_test.go @@ -39,9 +39,10 @@ func TestCheckMemoryRecommended(t *testing.T) { warnings: 1, }, { - name: "empty memory uses default", - nodes: []intent.NodeSpec{{Type: "fullnode"}}, - status: "pass", + name: "empty memory fails validation", + nodes: []intent.NodeSpec{{Type: "fullnode"}}, + status: "fail", + messageHas: []string{"invalid", "memory"}, }, } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 96f56869..09955d58 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -297,9 +297,9 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { if jdk == 0 { jdk = detectJDK(ctx, opts.Target) } - memGB := render.ParseMemoryGB(node.Resources.Memory) - if memGB == 0 { - memGB = 16 + memGB, err := render.ParseMemoryGB(node.Resources.Memory) + if err != nil { + return nil, fmt.Errorf("invalid resources.memory %q: %w", node.Resources.Memory, err) } jvmArgs := render.JVMArgsString(memGB, jdk, node.JVM) diff --git a/internal/mcp/tools_config.go b/internal/mcp/tools_config.go index c7efc019..4874faf9 100644 --- a/internal/mcp/tools_config.go +++ b/internal/mcp/tools_config.go @@ -2,6 +2,7 @@ package mcp import ( "context" + "fmt" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -85,9 +86,9 @@ func renderTool(ctx context.Context, _ *mcp.CallToolRequest, args renderArg) (*m if err != nil { return errResult(err) } - memGB := render.ParseMemoryGB(node.Resources.Memory) - if memGB == 0 { - memGB = 16 + memGB, err := render.ParseMemoryGB(node.Resources.Memory) + if err != nil { + return errResult(fmt.Errorf("invalid resources.memory %q: %w", node.Resources.Memory, err)) } jvmArgs := render.JVMArgsString(memGB, 17, node.JVM) diff --git a/internal/render/golden_test.go b/internal/render/golden_test.go index cfe847ee..3673909e 100644 --- a/internal/render/golden_test.go +++ b/internal/render/golden_test.go @@ -60,9 +60,9 @@ func TestRenderHOCON_Golden(t *testing.T) { if err != nil { t.Fatalf("RenderHOCON: %v", err) } - memGB := ParseMemoryGB(node.Resources.Memory) - if memGB == 0 { - memGB = 16 + memGB, err := ParseMemoryGB(node.Resources.Memory) + if err != nil { + t.Fatalf("ParseMemoryGB: %v", err) } jvmArgs := JVMArgsString(memGB, 17, node.JVM) gotCompose := RenderCompose(parsed.Name, parsed, node, "", jvmArgs, "") diff --git a/internal/render/jvm.go b/internal/render/jvm.go index 4f53ff37..6ddea72d 100644 --- a/internal/render/jvm.go +++ b/internal/render/jvm.go @@ -2,39 +2,47 @@ package render import ( "fmt" + "math" "strconv" "strings" "github.com/tronprotocol/tron-deployment/internal/intent" ) -// ParseMemoryGB converts an intent memory string like "16GB", "32G", "4096MB" -// into a whole number of gigabytes. Returns 0 if the input is empty or -// unparseable — the caller should pick a default. -func ParseMemoryGB(s string) int { +// ParseMemoryGB converts an intent memory string into whole gigabytes. +// GB values and MB values are rounded up, so the result never silently +// requests less memory than the user specified (for example, 2.5GB is 3GB +// and 1537MB is 2GB). Invalid, empty, zero, and negative values return an +// error; callers must not replace a bad value with a default. +func ParseMemoryGB(s string) (int, error) { + s = strings.TrimSpace(strings.ToUpper(s)) if s == "" { - return 0 + return 0, fmt.Errorf("memory must not be empty") } - s = strings.TrimSpace(strings.ToUpper(s)) unit := "GB" + hasUnit := false for _, u := range []string{"GB", "MB", "G", "M"} { if strings.HasSuffix(s, u) { + hasUnit = true unit = u s = strings.TrimSuffix(s, u) break } } - n, err := strconv.Atoi(strings.TrimSpace(s)) - if err != nil || n <= 0 { - return 0 + if !hasUnit && strings.Contains(s, ".") { + return 0, fmt.Errorf("fractional memory values require a GB unit") + } + n, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil || math.IsNaN(n) || math.IsInf(n, 0) || n <= 0 { + return 0, fmt.Errorf("invalid memory %q", s) } switch unit { case "GB", "G": - return n + return int(math.Ceil(n)), nil case "MB", "M": - return n / 1024 + return int(math.Ceil(n / 1024)), nil } - return n + return 0, fmt.Errorf("invalid memory unit in %q", s) } // JVMArgs generates JVM command-line arguments based on system memory and intent overrides. diff --git a/internal/render/jvm_test.go b/internal/render/jvm_test.go index 05eb85ca..a5c1ad0f 100644 --- a/internal/render/jvm_test.go +++ b/internal/render/jvm_test.go @@ -18,18 +18,27 @@ func TestParseMemoryGB(t *testing.T) { {"64g", 64}, {"4096MB", 4}, {"2048M", 2}, - {"", 0}, - {"garbage", 0}, - {"-4GB", 0}, + {"2048m", 2}, + {"2.5GB", 3}, + {"1537MB", 2}, {" 16 GB ", 16}, } for _, c := range cases { - if got := ParseMemoryGB(c.in); got != c.want { + got, err := ParseMemoryGB(c.in) + if err != nil || got != c.want { t.Errorf("ParseMemoryGB(%q) = %d, want %d", c.in, got, c.want) } } } +func TestParseMemoryGBRejectsInvalidInput(t *testing.T) { + for _, in := range []string{"", "garbage", "-4GB", "0", "0GB", "1TB", "2.5"} { + if got, err := ParseMemoryGB(in); err == nil || got != 0 { + t.Errorf("ParseMemoryGB(%q) = (%d, %v), want error", in, got, err) + } + } +} + func TestCalculateHeapMax(t *testing.T) { cases := []struct { memGB int @@ -85,7 +94,10 @@ func TestJVMArgs_HeapMaxOverrideRemainsVerbatim(t *testing.T) { } func TestJVMArgs_MBMemoryInput(t *testing.T) { - memGB := ParseMemoryGB("2048m") + memGB, err := ParseMemoryGB("2048m") + if err != nil { + t.Fatalf("ParseMemoryGB: %v", err) + } got := JVMArgsString(memGB, 17, nil) if !strings.Contains(got, "-Xmx1024m") || !strings.Contains(got, "-Xms1024m") { t.Errorf("2048m input produced unsafe heap: %q", got) From 46a990bdcfd4e5758c9fb26f86ca0db4515de227 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 14:19:10 +0800 Subject: [PATCH 17/49] fix(snapshot): refuse downloads that overwrite a running node's chain data AUD-011: snapshot download could extract ~50GB into the live chain DB directory of a running (or error-state) managed node. Guard before source resolution/preflight/detach: - record storage_root in state at apply / network add time (bind-mount aware: absolute, ./-relative anchored at the compose project dir, named volumes unresolvable) - candidates: storage root, root/database, root/output-directory{,/database}, plus parent dir for the conventional output-directory layout; symlink-canonicalized comparison with missing-path ancestor fallback - jar nodes: install_path root; docker nodes skip the /opt/tron default - refuse for nodes in running or error status (container may still hold the LevelDB lock after a failed restart) - legacy states without storage_root: no_change apply backfills the field; guard falls back to parsing the node's docker-compose.yaml bind source (/ and ./ prefixed) with an actionable stderr warning - docs: NODE_RUNNING semantics in --help and AGENTS.md --- AGENTS.md | 2 + cmd/network/add.go | 2 + cmd/snapshot/download.go | 176 ++++++++++++++++++++++++++-- cmd/snapshot/download_node_test.go | 171 +++++++++++++++++++++++++++ internal/apply/apply.go | 51 +++++++- internal/apply/apply_test.go | 25 ++++ internal/apply/storage_root_test.go | 33 ++++++ internal/state/types.go | 1 + 8 files changed, 445 insertions(+), 16 deletions(-) create mode 100644 cmd/snapshot/download_node_test.go create mode 100644 internal/apply/storage_root_test.go diff --git a/AGENTS.md b/AGENTS.md index 5de91a2f..9e28a121 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -278,6 +278,8 @@ done trond apply --intent intent-with-snapshot.yaml --auto-approve --wait -o json ``` +Snapshot download refuses destinations belonging to running or error-state managed nodes (`NODE_RUNNING`); stop the node before writing chain data. + ### `plaintext_transport` — do not skip this The six mainnet mirrors are bare IPs that publish **no HTTPS endpoint** diff --git a/cmd/network/add.go b/cmd/network/add.go index 49054b06..a2c20e6f 100644 --- a/cmd/network/add.go +++ b/cmd/network/add.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" + applypkg "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/guard" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" @@ -226,6 +227,7 @@ func runAdd(cmd *cobra.Command, args []string) error { P2PPort: node.Ports.P2P, MetricsPort: node.Ports.Metrics, InstallPath: node.InstallPath, + StorageRoot: applypkg.StorageRootForNode(node, "docker", paths.Deployments(), nodeName), Labels: node.Labels, }) if err := store.Save(deployState); err != nil { diff --git a/cmd/snapshot/download.go b/cmd/snapshot/download.go index 106ecd8a..a8047878 100644 --- a/cmd/snapshot/download.go +++ b/cmd/snapshot/download.go @@ -1,10 +1,12 @@ package snapshot import ( + "bufio" "errors" "fmt" "os" "path/filepath" + "strings" "time" "github.com/spf13/cobra" @@ -40,6 +42,8 @@ upstream MD5 sidecar — if the sidecar cannot be fetched the download aborts rather than extracting unchecked data, unless you pass --no-verify. Also pre-checks free disk space and refuses to overwrite an existing database without --force. +It refuses destinations belonging to running or error-state managed nodes; +stop the node first, then download. The default destination is ./output-directory under the current working directory — same convention as the official tron-docker tooling. Pass @@ -92,6 +96,21 @@ func init() { func runDownload(cmd *cobra.Command, _ []string) error { outputFmt, _ := cmd.Flags().GetString("output") + dest := dlDest + if dlNode != "" { + resolved, err := destFromNode(dlNode) + if err != nil { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) + } + dest = resolved + } + if dest == "" { + dest = "./output-directory" + } + if err := refuseRunningNodeDestination(dest, dlNode); err != nil { + return err + } + src, err := resolveSource(dlDomain, dlNetwork, dlKind, dlRegion, dlEngine) if err != nil { return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) @@ -106,18 +125,6 @@ func runDownload(cmd *cobra.Command, _ []string) error { backup = latest } - dest := dlDest - if dlNode != "" { - resolved, err := destFromNode(dlNode) - if err != nil { - return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) - } - dest = resolved - } - if dest == "" { - dest = "./output-directory" - } - opts := snapshot.DownloadOptions{ Source: *src, Backup: backup, @@ -227,6 +234,151 @@ func runDownload(cmd *cobra.Command, _ []string) error { return nil } +// refuseRunningNodeDestination protects managed node databases from being +// overwritten while java-tron (jar) or Docker holds them open. --force only +// controls the ordinary existing-database overwrite prompt; it never bypasses +// this data-corruption guard. +func refuseRunningNodeDestination(dest, explicitNode string) error { + store, err := state.NewStore(paths.State()) + if err != nil { + return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) + } + st, err := store.Load() + if err != nil { + return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) + } + var matched *state.ManagedNode + if explicitNode != "" { + matched = store.GetNode(st, explicitNode) + } else { + for i := range st.Nodes { + if st.Nodes[i].Status != "running" && st.Nodes[i].Status != "error" { + continue + } + if managedNodeDestinationMatches(dest, &st.Nodes[i]) { + matched = &st.Nodes[i] + break + } + } + } + if matched == nil || (matched.Status != "running" && matched.Status != "error") { + return nil + } + // An error can mean restart failed while the container is still alive; + // treat it as unsafe until the operator explicitly stops the node. + return output.NewError("NODE_RUNNING", output.ExitGeneralError, + fmt.Sprintf("refusing to write snapshot into running node %q data directory", matched.Name)). + WithSuggestions("Stop the node first: trond stop " + matched.Name) +} + +func managedNodeDestinationMatches(dest string, node *state.ManagedNode) bool { + if node == nil || dest == "" { + return false + } + destAbs, err := filepath.Abs(filepath.Clean(dest)) + if err != nil { + return false + } + roots := make([]string, 0, 2) + if node.InstallPath != "" && node.Runtime != "docker" { + rootAbs, rootErr := filepath.Abs(filepath.Clean(node.InstallPath)) + if rootErr == nil { + roots = append(roots, rootAbs) + } + } + if node.StorageRoot != "" { + storageAbs, storageErr := filepath.Abs(filepath.Clean(node.StorageRoot)) + if storageErr == nil { + roots = append(roots, storageAbs) + } + } + if node.Runtime == "docker" && node.StorageRoot == "" { + if fallback, ok := legacyDockerStorageRoot(node.Name); ok { + roots = append(roots, fallback) + } + } + for _, root := range roots { + candidates := []string{root, filepath.Join(root, "output-directory"), filepath.Join(root, "output-directory", "database"), filepath.Join(root, "database")} + if filepath.Base(root) == "output-directory" { + candidates = append(candidates, filepath.Dir(root)) + } + for _, candidate := range candidates { + if samePath(destAbs, candidate) { + return true + } + } + } + return false +} + +// legacyDockerStorageRoot recovers bind sources from pre-storage_root state. +// Compose is generated in a stable, one-volume-per-line form; parse only the +// volume targeting java-tron's data directory. +func legacyDockerStorageRoot(name string) (string, bool) { + compose := filepath.Join(paths.Deployments(), name, "docker-compose.yaml") + f, err := os.Open(compose) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: state predates storage_root tracking; guard incomplete, please re-apply %s\n", name) + return "", false + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + marker := ":/java-tron/output-directory" + if i := strings.Index(line, marker); i >= 0 { + source := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line[:i]), "-")) + if strings.HasPrefix(source, "/") { + return source, true + } + if strings.HasPrefix(source, "./") { + return filepath.Join(paths.Deployments(), name, source), true + } + if source != "" { + fmt.Fprintf(os.Stderr, "warning: named volume — host path unresolvable; guard cannot cover node %s\n", name) + return "", false + } + break + } + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "warning: unable to parse compose file; guard incomplete, please re-apply %s: %v\n", name, err) + return "", false + } + fmt.Fprintf(os.Stderr, "warning: compose bind source not found; guard incomplete, please re-apply %s\n", name) + return "", false +} + +func samePath(left, right string) bool { + left = canonicalPath(left) + right = canonicalPath(right) + return left == right +} + +func canonicalPath(path string) string { + path = filepath.Clean(path) + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return filepath.Clean(resolved) + } + // The destination may not exist yet. Resolve the nearest existing parent + // so a symlinked storage root still compares equal before extraction. + missing := []string{} + for current := path; ; current = filepath.Dir(current) { + if resolved, err := filepath.EvalSymlinks(current); err == nil { + for i := len(missing) - 1; i >= 0; i-- { + resolved = filepath.Join(resolved, missing[i]) + } + return filepath.Clean(resolved) + } + parent := filepath.Dir(current) + if parent == current { + break + } + missing = append(missing, filepath.Base(current)) + } + return path +} + // downloadPayload builds the `-o json` body for a completed foreground // download. Extracted from runDownload so a unit test can validate the // exact shipped shape against schemas/output/snapshot-download.schema.json diff --git a/cmd/snapshot/download_node_test.go b/cmd/snapshot/download_node_test.go new file mode 100644 index 00000000..09b78ed2 --- /dev/null +++ b/cmd/snapshot/download_node_test.go @@ -0,0 +1,171 @@ +package snapshot + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +func TestRefuseRunningNodeDestinationByName(t *testing.T) { + seedSnapshotNodeState(t, state.ManagedNode{Name: "jar0", Runtime: "jar", Status: "running", InstallPath: "/srv/tron/jar0"}) + + err := refuseRunningNodeDestination("/srv/tron/jar0", "jar0") + assertRunningNodeError(t, err, "jar0") +} + +func TestRefuseRunningNodeDestinationByPath(t *testing.T) { + root := t.TempDir() + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "running", StorageRoot: root}) + + err := refuseRunningNodeDestination(filepath.Join(root, "output-directory"), "") + assertRunningNodeError(t, err, "docker0") +} + +func TestRefuseRunningNodeDataRootParentAndOutputDirectory(t *testing.T) { + root := t.TempDir() + dataMount := filepath.Join(root, "output-directory") + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "running", StorageRoot: dataMount}) + + for _, dest := range []string{root, dataMount} { + if err := refuseRunningNodeDestination(dest, ""); err == nil { + t.Errorf("destination %q was allowed", dest) + } + } +} + +func TestRefuseRunningNodeDatabaseChild(t *testing.T) { + root := t.TempDir() + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "running", StorageRoot: root}) + assertRunningNodeError(t, refuseRunningNodeDestination(filepath.Join(root, "database"), ""), "docker0") +} + +func TestRefuseErrorNodeDestination(t *testing.T) { + root := t.TempDir() + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "error", StorageRoot: root}) + assertRunningNodeError(t, refuseRunningNodeDestination(filepath.Join(root, "database"), ""), "docker0") +} + +func TestDockerInstallPathIsNotCandidate(t *testing.T) { + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "running", InstallPath: "/opt/tron"}) + if err := refuseRunningNodeDestination("/opt/tron", ""); err != nil { + t.Fatalf("docker install path incorrectly blocked: %v", err) + } +} + +func TestLegacyDockerComposeFallback(t *testing.T) { + base := t.TempDir() + paths.SetBaseDir(base) + t.Cleanup(func() { paths.SetBaseDir("") }) + composeDir := filepath.Join(paths.Deployments(), "old0") + if err := os.MkdirAll(composeDir, 0o755); err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(composeDir, "docker-compose.yaml"), []byte(" - "+root+":/java-tron/output-directory\n"), 0o644); err != nil { + t.Fatal(err) + } + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{Name: "old0", Runtime: "docker", Status: "running", InstallPath: "/opt/tron"}}}); err != nil { + t.Fatal(err) + } + assertRunningNodeError(t, refuseRunningNodeDestination(filepath.Join(root, "database"), ""), "old0") +} + +func TestLegacyDockerComposeRelativeFallback(t *testing.T) { + base := t.TempDir() + paths.SetBaseDir(base) + t.Cleanup(func() { paths.SetBaseDir("") }) + name := "old-relative" + composeDir := filepath.Join(paths.Deployments(), name) + if err := os.MkdirAll(composeDir, 0o755); err != nil { + t.Fatal(err) + } + compose := " - ./chain:/java-tron/output-directory\n" + if err := os.WriteFile(filepath.Join(composeDir, "docker-compose.yaml"), []byte(compose), 0o644); err != nil { + t.Fatal(err) + } + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: name, Runtime: "docker", Status: "running", InstallPath: "/opt/tron", + }}}); err != nil { + t.Fatal(err) + } + dest := filepath.Join(paths.Deployments(), name, "chain") + assertRunningNodeError(t, refuseRunningNodeDestination(dest, ""), name) +} + +func TestStoragePathAbsoluteMountOnlyBlocksDataMount(t *testing.T) { + root := t.TempDir() + mount := filepath.Join(root, "data") + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "running", StorageRoot: mount}) + + if err := refuseRunningNodeDestination(mount, ""); err == nil { + t.Fatal("storage.path/data destination was allowed") + } + if err := refuseRunningNodeDestination(root, ""); err != nil { + t.Fatalf("storage.path root incorrectly blocked: %v", err) + } +} + +func TestRefuseRunningNodeDestinationSymlinkAndRelativeForms(t *testing.T) { + root := t.TempDir() + link := filepath.Join(t.TempDir(), "storage-link") + if err := os.Symlink(root, link); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + seedSnapshotNodeState(t, state.ManagedNode{Name: "docker0", Runtime: "docker", Status: "running", StorageRoot: root}) + + for _, dest := range []string{link + "/output-directory/", filepath.Join(root, ".", "output-directory")} { + if err := refuseRunningNodeDestination(dest, ""); err == nil { + t.Errorf("destination %q was allowed", dest) + } + } +} + +func TestAllowStoppedNodeDestination(t *testing.T) { + root := t.TempDir() + seedSnapshotNodeState(t, state.ManagedNode{Name: "jar0", Runtime: "jar", Status: "stopped", InstallPath: root}) + + if err := refuseRunningNodeDestination(filepath.Join(root, "output-directory"), "jar0"); err != nil { + t.Fatalf("stopped node rejected: %v", err) + } +} + +func seedSnapshotNodeState(t *testing.T, node state.ManagedNode) { + t.Helper() + base := t.TempDir() + paths.SetBaseDir(base) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{node}}); err != nil { + t.Fatal(err) + } +} + +func assertRunningNodeError(t *testing.T, err error, name string) { + t.Helper() + if err == nil { + t.Fatal("running node destination was allowed") + } + if !strings.Contains(err.Error(), "running node") || !strings.Contains(err.Error(), name) { + t.Fatalf("error = %v, want running node %s", err, name) + } + structured, ok := err.(*output.StructuredError) + if !ok || len(structured.Suggestions) == 0 || structured.Suggestions[0] != "Stop the node first: trond stop "+name { + t.Fatalf("error = %v, want stop suggestion", err) + } +} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 09955d58..f2d4a6ed 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -402,6 +402,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { MetricsPort: node.Ports.Metrics, Labels: node.Labels, InstallPath: node.InstallPath, + StorageRoot: StorageRootForNode(node, runtimeType, opts.DeploymentsDir, opts.Intent.Name), Monitoring: monRes.managed, } outcome := "created" @@ -463,6 +464,37 @@ func remediateJarConfigPermissions(ctx context.Context, opts Options, node *inte _ = perms.Chmod(ctx, filepath.Join(node.InstallPath, "config.conf"), 0o600) } +// StorageRootForNode records only host bind-mount roots. Named volumes are +// intentionally left empty: snapshot --to cannot safely resolve their host +// path. The values mirror render.storageSource's explicit data, then path +// resolution order. +func StorageRootForNode(node *intent.NodeSpec, runtimeType, deploymentsDir, nodeName string) string { + if runtimeType != "docker" || node == nil { + return "" + } + if node.Storage.Data != "" { + return bindStoragePath(node.Storage.Data, deploymentsDir, nodeName) + } + if node.Storage.StoragePath != "" { + root := bindStoragePath(node.Storage.StoragePath, deploymentsDir, nodeName) + if root == "" { + return "" + } + return filepath.Join(root, "data") + } + return "" +} + +func bindStoragePath(path, deploymentsDir, nodeName string) string { + if !strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "./") { + return "" + } + if strings.HasPrefix(path, "./") { + return filepath.Join(deploymentsDir, nodeName, path) + } + return path +} + // recordedNode returns the managed node this apply would replace, i.e. the // state entry keyed by the intent's name. It prefers opts.Existing (the // caller's own lookup) and otherwise resolves the name against the state it @@ -578,10 +610,21 @@ func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) * // Result would report network/is_private from the intent while a // follow-up `status` (which reads state) still showed it absent. // Best-effort: a save failure here doesn't fail the no-op. - if opts.Existing != nil && opts.Existing.Network != opts.Intent.Network && opts.Store != nil && opts.State != nil { - opts.Existing.Network = opts.Intent.Network - opts.Store.UpsertNode(opts.State, *opts.Existing) - _ = opts.Store.Save(opts.State) + if opts.Existing != nil && opts.Store != nil && opts.State != nil { + changed := false + if opts.Existing.Network != opts.Intent.Network { + opts.Existing.Network = opts.Intent.Network + changed = true + } + storageRoot := StorageRootForNode(&opts.Intent.Nodes[0], opts.Existing.Runtime, opts.DeploymentsDir, opts.Intent.Name) + if storageRoot != "" && opts.Existing.StorageRoot != storageRoot { + opts.Existing.StorageRoot = storageRoot + changed = true + } + if changed { + opts.Store.UpsertNode(opts.State, *opts.Existing) + _ = opts.Store.Save(opts.State) + } } ports := opts.Intent.Nodes[0].Ports diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index e6037193..c7a63ed6 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -140,6 +140,31 @@ func TestApply_NoChangeRemediatesJarConfigPermissions(t *testing.T) { } } +func TestApply_NoChangeBackfillsStorageRoot(t *testing.T) { + parsed := minimalIntent() + parsed.Target.Runtime = "docker" + parsed.Nodes[0].Storage.Data = "/srv/tron/node0/output-directory" + store, st := freshStore(t) + existing := state.ManagedNode{ + Name: parsed.Name, IntentHash: "same-hash", Runtime: "docker", + } + res, err := Apply(context.Background(), Options{ + Intent: parsed, Target: &fakeTarget{}, Store: store, State: st, + IntentHash: "same-hash", Existing: &existing, + DeploymentsDir: "/srv/tron/deployments", + }) + if err != nil { + t.Fatalf("Apply: %v", err) + } + if res.Outcome != "no_change" { + t.Fatalf("Outcome = %q, want no_change", res.Outcome) + } + got := store.GetNode(st, parsed.Name) + if got == nil || got.StorageRoot != "/srv/tron/node0/output-directory" { + t.Fatalf("StorageRoot = %v, want /srv/tron/node0/output-directory", got) + } +} + func TestApply_RequiresIntent(t *testing.T) { if _, err := Apply(context.Background(), Options{}); err == nil { t.Error("expected error when Intent is nil") diff --git a/internal/apply/storage_root_test.go b/internal/apply/storage_root_test.go new file mode 100644 index 00000000..047e5e60 --- /dev/null +++ b/internal/apply/storage_root_test.go @@ -0,0 +1,33 @@ +package apply + +import ( + "path/filepath" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/intent" +) + +func TestStorageRootForNode(t *testing.T) { + deployments := "/deployments" + tests := []struct { + name string + storage intent.Storage + want string + }{ + {"named data wins", intent.Storage{Data: "volume", StoragePath: "/ignored"}, ""}, + {"relative path anchored", intent.Storage{StoragePath: "./data"}, filepath.Join(deployments, "node0", "data", "data")}, + {"relative data anchored", intent.Storage{Data: "./x"}, filepath.Join(deployments, "node0", "x")}, + {"absolute data unchanged", intent.Storage{Data: "/srv/data", StoragePath: "/ignored"}, "/srv/data"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &intent.NodeSpec{Storage: tt.storage} + if got := StorageRootForNode(node, "docker", deployments, "node0"); got != tt.want { + t.Fatalf("StorageRootForNode = %q, want %q", got, tt.want) + } + if got := StorageRootForNode(node, "jar", deployments, "node0"); got != "" { + t.Fatalf("jar StorageRoot = %q, want empty", got) + } + }) + } +} diff --git a/internal/state/types.go b/internal/state/types.go index badf2b59..81409447 100644 --- a/internal/state/types.go +++ b/internal/state/types.go @@ -25,6 +25,7 @@ type ManagedNode struct { ComposePath string `json:"compose_path,omitempty"` SystemdUnit string `json:"systemd_unit,omitempty"` InstallPath string `json:"install_path,omitempty"` + StorageRoot string `json:"storage_root,omitempty"` // HTTPPort and GRPCPort capture the API ports as configured at deploy time // so probe commands (health, diagnose, verify) can target the right port // without re-reading the intent file. Older state files predate these From 6d11a778463c14d1a1fd28f89b2f9ea370e7f55f Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 15:34:16 +0800 Subject: [PATCH 18/49] fix(target): resilient ssh transport and jar download (AUD-019/020/021) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AUD-019: jar download on ssh targets fetches locally then uploads via PutFile + atomic remote mv (whitelist no longer blocks curl); 10min client, Content-Length + 512MiB hard limit, SHA256 verified before upload, remote temp cleaned on failure - AUD-020: HTTPClient transports are reused — package-shared for LocalTarget, per-instance cache for ssh with Close() deregistration and a global CloseIdleConnections - AUD-021: ssh Connect honors context via DialContext + NewClientConn; cancelled handshakes drain and close late clients --- internal/runtime/jar.go | 74 +++++++++++++++++ internal/runtime/upgrade_test.go | 92 +++++++++++++++++++++ internal/target/ssh.go | 66 +++++++++++++-- internal/target/target.go | 66 ++++++++++++++- internal/target/target_test.go | 138 +++++++++++++++++++++++++++++++ 5 files changed, 430 insertions(+), 6 deletions(-) diff --git a/internal/runtime/jar.go b/internal/runtime/jar.go index bed9d854..b045c68d 100644 --- a/internal/runtime/jar.go +++ b/internal/runtime/jar.go @@ -3,12 +3,16 @@ package runtime import ( "bytes" "context" + "crypto/sha256" "fmt" "io" "maps" + "net/http" + "os" "path/filepath" "slices" "strings" + "time" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -241,6 +245,9 @@ func (r *JarRuntime) downloadJar(ctx context.Context, url, destPath, expectedSHA // Download to a temporary path so a failed transfer or checksum never // destroys the currently running artifact. tmpPath := destPath + ".upgrade.tmp" + if remote, ok := r.target.(interface{ IsRemote() bool }); ok && remote.IsRemote() { + return r.downloadJarLocally(ctx, url, tmpPath, destPath, expectedSHA256) + } if _, err := r.target.Exec(ctx, "curl", "-fSL", "-o", tmpPath, url); err != nil { _, _ = r.target.Exec(ctx, "rm", "-f", tmpPath) return fmt.Errorf("download %s: %w", url, err) @@ -271,5 +278,72 @@ func (r *JarRuntime) downloadJar(ctx context.Context, url, destPath, expectedSHA return nil } +func (r *JarRuntime) downloadJarLocally(ctx context.Context, url, remoteTmp, destPath, expectedSHA256 string) error { + const maxJARBytes int64 = 512 << 20 + f, err := os.CreateTemp("", "trond-jar-") + if err != nil { + return fmt.Errorf("create local download: %w", err) + } + localPath := f.Name() + defer os.Remove(localPath) + defer f.Close() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create download request: %w", err) + } + client := &http.Client{Timeout: 10 * time.Minute} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("download %s: %w", url, err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("download %s: http %s", url, resp.Status) + } + if resp.ContentLength > maxJARBytes { + return fmt.Errorf("download %s exceeds maximum JAR size of %d bytes", url, maxJARBytes) + } + if written, err := io.Copy(f, io.LimitReader(resp.Body, maxJARBytes+1)); err != nil { + return fmt.Errorf("save local download: %w", err) + } else if written > maxJARBytes { + return fmt.Errorf("download %s exceeds maximum JAR size of %d bytes", url, maxJARBytes) + } + if err := f.Close(); err != nil { + return fmt.Errorf("close local download: %w", err) + } + if expectedSHA256 != "" { + check, err := os.Open(localPath) + if err != nil { + return err + } + h := sha256.New() + _, copyErr := io.Copy(h, check) + _ = check.Close() + if copyErr != nil { + return copyErr + } + got := fmt.Sprintf("%x", h.Sum(nil)) + if got != expectedSHA256 { + return fmt.Errorf("SHA256 mismatch: expected %s, got %s", expectedSHA256, got) + } + } + if err := r.target.PutFile(ctx, localPath, remoteTmp); err != nil { + r.cleanupRemoteTmp(remoteTmp) + return fmt.Errorf("upload jar: %w", err) + } + if _, err := r.target.Exec(ctx, "mv", remoteTmp, destPath); err != nil { + r.cleanupRemoteTmp(remoteTmp) + return fmt.Errorf("install downloaded jar: %w", err) + } + return nil +} + +func (r *JarRuntime) cleanupRemoteTmp(path string) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, _ = r.target.Exec(cleanupCtx, "rm", "-f", path) +} + // Ensure JarRuntime implements Runtime var _ Runtime = (*JarRuntime)(nil) diff --git a/internal/runtime/upgrade_test.go b/internal/runtime/upgrade_test.go index 05e6f82b..fc2436f3 100644 --- a/internal/runtime/upgrade_test.go +++ b/internal/runtime/upgrade_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "net/http" + "net/http/httptest" "os" "reflect" "strings" @@ -45,6 +47,20 @@ type upgradeFakeTarget struct { type jarUpgradeFakeTarget struct{ *fakeTarget } +type remoteJarUpgradeFakeTarget struct { + *jarUpgradeFakeTarget + uploads int + remotePath string + putErr error +} + +func (f *remoteJarUpgradeFakeTarget) IsRemote() bool { return true } +func (f *remoteJarUpgradeFakeTarget) PutFile(ctx context.Context, localPath, remotePath string) error { + f.uploads++ + f.remotePath = remotePath + return f.putErr +} + type failingUpgradeTarget struct { *fakeTarget writeErr error @@ -214,6 +230,82 @@ func TestJarRuntimeUpgradeArtifact(t *testing.T) { t.Fatal(err) } }) + t.Run("ssh downloads locally and uploads", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("jar-bytes")) + })) + defer server.Close() + f := &remoteJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + r := NewJarRuntime(f) + if err := r.downloadJar(context.Background(), server.URL, "/opt/tron/FullNode.jar", ""); err != nil { + t.Fatal(err) + } + if f.uploads != 1 { + t.Fatalf("uploads = %d, want 1", f.uploads) + } + if f.remotePath != "/opt/tron/FullNode.jar.upgrade.tmp" { + t.Fatalf("upload path = %q", f.remotePath) + } + if want := []string{"mv", "/opt/tron/FullNode.jar.upgrade.tmp", "/opt/tron/FullNode.jar"}; !reflect.DeepEqual(f.cmds[len(f.cmds)-1], want) { + t.Fatalf("final command = %v, want %v", f.cmds[len(f.cmds)-1], want) + } + for _, cmd := range f.cmds { + if len(cmd) > 0 && cmd[0] == "curl" { + t.Fatalf("remote curl used: %v", f.cmds) + } + } + }) + t.Run("SHA mismatch does not upload", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("jar-bytes")) })) + defer server.Close() + f := &remoteJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + err := NewJarRuntime(f).downloadJar(context.Background(), server.URL, "/opt/tron/FullNode.jar", strings.Repeat("0", 64)) + if err == nil || !strings.Contains(err.Error(), "SHA256 mismatch") { + t.Fatalf("error = %v", err) + } + if f.uploads != 0 { + t.Fatalf("uploads = %d, want 0", f.uploads) + } + }) + t.Run("oversized response does not upload", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", fmt.Sprint((512<<20)+1)) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + f := &remoteJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + err := NewJarRuntime(f).downloadJar(context.Background(), server.URL, "/opt/tron/FullNode.jar", "") + if err == nil || !strings.Contains(err.Error(), "maximum JAR size") { + t.Fatalf("error = %v", err) + } + if f.uploads != 0 { + t.Fatalf("uploads = %d, want 0", f.uploads) + } + }) + t.Run("non-2xx response does not upload", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "no", http.StatusBadGateway) })) + defer server.Close() + f := &remoteJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + err := NewJarRuntime(f).downloadJar(context.Background(), server.URL, "/opt/tron/FullNode.jar", "") + if err == nil || !strings.Contains(err.Error(), "http 502") { + t.Fatalf("error = %v", err) + } + if f.uploads != 0 { + t.Fatalf("uploads = %d, want 0", f.uploads) + } + }) + t.Run("upload failure cleans temporary remote file", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte("jar-bytes")) })) + defer server.Close() + f := &remoteJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}, putErr: errors.New("upload failed")} + err := NewJarRuntime(f).downloadJar(context.Background(), server.URL, "/opt/tron/FullNode.jar", "") + if err == nil { + t.Fatal("expected upload failure") + } + if len(f.cmds) != 1 || !reflect.DeepEqual(f.cmds[0], []string{"rm", "-f", "/opt/tron/FullNode.jar.upgrade.tmp"}) { + t.Fatalf("cleanup commands = %v", f.cmds) + } + }) t.Run("download and restart", func(t *testing.T) { f := &jarUpgradeFakeTarget{newFakeTarget()} r := NewJarRuntime(f) diff --git a/internal/target/ssh.go b/internal/target/ssh.go index 0f64e2b4..6b367ece 100644 --- a/internal/target/ssh.go +++ b/internal/target/ssh.go @@ -35,6 +35,8 @@ type SSHTarget struct { client *ssh.Client } +var sshNetDialContext = (&net.Dialer{}).DialContext + func (t *SSHTarget) DialContext(_ context.Context, network, addr string) (net.Conn, error) { host, _, err := net.SplitHostPort(addr) if err != nil { @@ -71,8 +73,19 @@ func (t *SSHTarget) WithKnownHosts(path string) *SSHTarget { return t } -// Connect establishes the SSH connection, verifying the server's host key. +// Connect establishes the SSH connection, verifying the server's host key, +// with a two-minute deadline. +// It bounds TCP connect and SSH handshake at two minutes. func (t *SSHTarget) Connect() error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + return t.ConnectContext(ctx) +} + +// ConnectContext establishes SSH while honoring cancellation during both TCP +// connect and SSH handshake. Callers with a bounded operation should use this +// method; Connect supplies a two-minute context. +func (t *SSHTarget) ConnectContext(ctx context.Context) error { identityPath, err := expandHome(t.identityFile) if err != nil { return fmt.Errorf("expand identity path: %w", err) @@ -101,15 +114,55 @@ func (t *SSHTarget) Connect() error { } addr := net.JoinHostPort(t.host, strconv.Itoa(t.port)) - client, err := ssh.Dial("tcp", addr, config) + conn, err := sshNetDialContext(ctx, "tcp", addr) if err != nil { return fmt.Errorf("ssh connect to %s: %w", addr, err) } - - t.client = client + type result struct { + client *ssh.Client + err error + } + resultCh := make(chan result, 1) + go func() { + cc, chans, reqs, handshakeErr := ssh.NewClientConn(conn, addr, config) + if handshakeErr != nil { + resultCh <- result{err: handshakeErr} + return + } + client := ssh.NewClient(cc, chans, reqs) + if ctx.Err() != nil { + _ = client.Close() + resultCh <- result{err: ctx.Err()} + return + } + resultCh <- result{client: client} + }() + select { + case result := <-resultCh: + if result.err != nil { + _ = conn.Close() + return fmt.Errorf("ssh connect to %s: %w", addr, result.err) + } + t.client = result.client + case <-ctx.Done(): + _ = conn.Close() + // Handshake can finish concurrently with cancellation. Drain the result + // and close a client created after the cancellation branch won. + go func() { + result := <-resultCh + if result.client != nil { + _ = result.client.Close() + } + }() + return fmt.Errorf("ssh connect to %s: %w", addr, ctx.Err()) + } return nil } +// IsRemote marks targets whose artifacts must be downloaded by the local +// process and uploaded over the target connection. +func (t *SSHTarget) IsRemote() bool { return true } + // hostKeyCallback returns a verifier backed by known_hosts. Falls back to // ~/.ssh/known_hosts when no explicit file is configured. // @@ -186,8 +239,11 @@ func (t *SSHTarget) hostKeyCallback() (ssh.HostKeyCallback, error) { // Close closes the SSH connection. func (t *SSHTarget) Close() error { + unregisterTransport(t) if t.client != nil { - return t.client.Close() + err := t.client.Close() + t.client = nil + return err } return nil } diff --git a/internal/target/target.go b/internal/target/target.go index 8efc9550..9bcdd9aa 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -8,6 +8,8 @@ import ( "net" "net/http" "os" + "reflect" + "sync" "time" ) @@ -24,11 +26,73 @@ type Dialer interface { func HTTPClient(t Target, timeout time.Duration) *http.Client { client := &http.Client{Timeout: timeout} if d, ok := t.(Dialer); ok { - client.Transport = &http.Transport{DialContext: d.DialContext} + if _, local := t.(*LocalTarget); local { + client.Transport = localTransport(d) + } else { + client.Transport = targetTransport(t, d) + } } return client } +var targetTransports sync.Map +var localTransportOnce sync.Once +var sharedLocalTransport *http.Transport + +func localTransport(d Dialer) *http.Transport { + localTransportOnce.Do(func() { + sharedLocalTransport = http.DefaultTransport.(*http.Transport).Clone() + sharedLocalTransport.DialContext = d.DialContext + }) + return sharedLocalTransport +} + +func targetTransport(t Target, d Dialer) *http.Transport { + key := transportKey{label: t.String(), identity: targetIdentity(t)} + if value, ok := targetTransports.Load(key); ok { + return value.(*http.Transport) + } + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.DialContext = d.DialContext + actual, loaded := targetTransports.LoadOrStore(key, tr) + if loaded { + tr.CloseIdleConnections() + } + return actual.(*http.Transport) +} + +func unregisterTransport(t Target) { + key := transportKey{label: t.String(), identity: targetIdentity(t)} + if value, ok := targetTransports.LoadAndDelete(key); ok { + value.(*http.Transport).CloseIdleConnections() + } +} + +type transportKey struct { + label string + identity uintptr +} + +func targetIdentity(t Target) uintptr { + v := reflect.ValueOf(t) + if v.IsValid() && v.Kind() == reflect.Pointer { + return v.Pointer() + } + return 0 +} + +// CloseIdleConnections releases pooled connections held by target-aware HTTP +// transports. Long-lived callers should invoke it during shutdown. +func CloseIdleConnections() { + if sharedLocalTransport != nil { + sharedLocalTransport.CloseIdleConnections() + } + targetTransports.Range(func(_, value any) bool { + value.(*http.Transport).CloseIdleConnections() + return true + }) +} + // EndpointHost returns the host used when reporting a target endpoint. func EndpointHost(targetType, host string) string { if targetType == "ssh" && host != "" { diff --git a/internal/target/target_test.go b/internal/target/target_test.go index a8003c37..5aa4dcfb 100644 --- a/internal/target/target_test.go +++ b/internal/target/target_test.go @@ -2,11 +2,18 @@ package target import ( "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/pem" + "errors" "fmt" + "io" "net" "net/http" "net/http/httptest" "os" + "path/filepath" "strings" "testing" "time" @@ -123,6 +130,137 @@ func TestHTTPClientFallsBackForNonDialer(t *testing.T) { } } +func TestHTTPClientReusesTransportPerTarget(t *testing.T) { + target := NewLocalTarget() + first := HTTPClient(target, time.Second) + second := HTTPClient(target, time.Second) + if first.Transport != second.Transport { + t.Fatalf("HTTPClient transports differ: %p vs %p", first.Transport, second.Transport) + } + if _, ok := first.Transport.(*http.Transport); !ok { + t.Fatalf("transport type = %T, want *http.Transport", first.Transport) + } +} + +func TestHTTPClientDoesNotShareTransportAcrossTargetInstances(t *testing.T) { + firstTarget := NewSSHTarget("same", 22, "user", "") + secondTarget := NewSSHTarget("same", 22, "user", "") + first := HTTPClient(firstTarget, time.Second) + second := HTTPClient(secondTarget, time.Second) + if first.Transport == second.Transport { + t.Fatal("distinct target instances must not share a transport") + } +} + +func TestHTTPClientLocalTargetsShareTransport(t *testing.T) { + if HTTPClient(NewLocalTarget(), time.Second).Transport != HTTPClient(NewLocalTarget(), time.Second).Transport { + t.Fatal("LocalTarget instances should use shared transport") + } +} + +func TestSSHTargetCloseUnregistersTransport(t *testing.T) { + target := NewSSHTarget("same", 22, "user", "") + first := HTTPClient(target, time.Second).Transport + if err := target.Close(); err != nil { + t.Fatal(err) + } + second := HTTPClient(target, time.Second).Transport + if first == second { + t.Fatal("transport reused after target Close") + } +} + +func TestCloseIdleConnectionsClosesCachedTransports(t *testing.T) { + CloseIdleConnections() + if sharedLocalTransport == nil { + t.Fatal("expected shared local transport") + } + CloseIdleConnections() +} + +func TestSSHTargetConnectContextUnreachableReturnsPromptly(t *testing.T) { + originalDial := sshNetDialContext + sshNetDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + <-ctx.Done() + return nil, ctx.Err() + } + t.Cleanup(func() { sshNetDialContext = originalDial }) + identity, knownHosts := sshTestFiles(t) + target := NewSSHTarget("127.0.0.1", 1, "user", identity).WithKnownHosts(knownHosts) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + start := time.Now() + err := target.ConnectContext(ctx) + if err == nil { + t.Fatal("ConnectContext unexpectedly succeeded") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v, want context.DeadlineExceeded", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Fatalf("ConnectContext took %s, want prompt failure", elapsed) + } +} + +func TestSSHTargetConnectContextCancelDuringHandshake(t *testing.T) { + identity, knownHosts := sshTestFiles(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + accepted := make(chan struct{}) + go func() { + conn, acceptErr := listener.Accept() + if acceptErr == nil { + close(accepted) + defer conn.Close() + _, _ = io.Copy(io.Discard, conn) + } + }() + port := listener.Addr().(*net.TCPAddr).Port + target := NewSSHTarget("127.0.0.1", port, "user", identity).WithKnownHosts(knownHosts) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- target.ConnectContext(ctx) }() + select { + case <-accepted: + case <-time.After(time.Second): + t.Fatal("test server did not accept TCP connection") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } + case <-time.After(time.Second): + t.Fatal("ConnectContext did not return after cancellation") + } +} + +func sshTestFiles(t *testing.T) (string, string) { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.MarshalPKCS8PrivateKey(private) + if err != nil { + t.Fatal(err) + } + identity := filepath.Join(t.TempDir(), "id_ed25519") + if err := os.WriteFile(identity, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + knownHosts := filepath.Join(t.TempDir(), "known_hosts") + if err := os.WriteFile(knownHosts, nil, 0o600); err != nil { + t.Fatal(err) + } + _ = public + return identity, knownHosts +} + // TestDialContextFallback: the free DialContext helper falls back to a plain // net.Dialer when the target does not implement Dialer. func TestDialContextFallback(t *testing.T) { From 6ce21736c8b36c7030b81f840a24d8b3e8eaac4d Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 15:46:46 +0800 Subject: [PATCH 19/49] fix(state): make Save atomic and surface persistence errors AUD-016: SaveState failures were silently swallowed by lifecycle commands and MCP auto-heal; they now return STATE_ERROR (or mark the heal item failed) so callers learn the state was not persisted. AUD-017: Save writes to a unique temp file, fsyncs, and renames atomically (0600), so an interrupted write cannot leave truncated JSON behind. Concurrent writers without the state lock may still lost-update (tracked under AUD-015); the on-disk file itself always stays a complete, valid document. --- cmd/remove.go | 14 +++- cmd/restart.go | 18 ++++- cmd/rollback.go | 14 +++- cmd/start.go | 17 +++- cmd/state_save_error_test.go | 115 +++++++++++++++++++++++++++ cmd/stop.go | 14 +++- cmd/upgrade.go | 14 +++- internal/apply/apply.go | 15 ++-- internal/apply/apply_test.go | 32 ++++++++ internal/mcp/tools_heal.go | 24 +++++- internal/mcp/tools_heal_save_test.go | 79 ++++++++++++++++++ internal/state/concurrent_test.go | 55 ++++++++++++- internal/state/store.go | 40 ++++++++-- internal/state/store_test.go | 43 +++++++++- 14 files changed, 467 insertions(+), 27 deletions(-) create mode 100644 cmd/state_save_error_test.go create mode 100644 internal/mcp/tools_heal_save_test.go diff --git a/cmd/remove.go b/cmd/remove.go index 2d443cf7..83458dce 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -16,6 +16,8 @@ var ( removeConfirm string ) +var saveRemoveState = func(nc *nodeContext) error { return nc.Store.Save(nc.State) } + var removeCmd = &cobra.Command{ Use: "remove ", Short: "Remove a deployed node", @@ -78,7 +80,9 @@ func runRemove(cmd *cobra.Command, args []string) error { } nc.Store.RemoveNode(nc.State, name) - nc.Store.Save(nc.State) + if err := persistRemoveState(name, nc, start); err != nil { + return err + } writeAudit(auditEvent{Command: "remove", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) writeResult(map[string]any{ @@ -88,3 +92,11 @@ func runRemove(cmd *cobra.Command, args []string) error { }) return nil } + +func persistRemoveState(name string, nc *nodeContext, start time.Time) error { + if err := saveRemoveState(nc); err != nil { + writeAudit(auditEvent{Command: "remove", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) + return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) + } + return nil +} diff --git a/cmd/restart.go b/cmd/restart.go index 013ff070..d718401f 100644 --- a/cmd/restart.go +++ b/cmd/restart.go @@ -16,6 +16,8 @@ var restartCmd = &cobra.Command{ RunE: runRestart, } +var saveRestartState = func(nc *nodeContext) error { return nc.SaveState() } + func init() { rootCmd.AddCommand(restartCmd) } @@ -42,16 +44,28 @@ func runRestart(cmd *cobra.Command, args []string) error { if err := nc.Runtime.Start(cmd.Context(), name); err != nil { nc.Node.Status = "error" - nc.SaveState() + if saveErr := persistRestartState(name, nc, start); saveErr != nil { + return saveErr + } writeAudit(auditEvent{Command: "restart", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "RESTART_ERROR", Start: start}) return exitWithError("RESTART_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to start %s after stop: %v", name, err)) } nc.Node.Status = "running" - nc.SaveState() + if err := persistRestartState(name, nc, start); err != nil { + return err + } writeAudit(auditEvent{Command: "restart", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) writeResult(map[string]any{"name": name, "status": "running"}) return nil } + +func persistRestartState(name string, nc *nodeContext, start time.Time) error { + if err := saveRestartState(nc); err != nil { + writeAudit(auditEvent{Command: "restart", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) + return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) + } + return nil +} diff --git a/cmd/rollback.go b/cmd/rollback.go index b15ce1b1..56cff77d 100644 --- a/cmd/rollback.go +++ b/cmd/rollback.go @@ -21,6 +21,8 @@ var rollbackCmd = &cobra.Command{ RunE: runRollback, } +var saveRollbackState = func(nc *nodeContext) error { return nc.SaveState() } + func init() { rollbackCmd.Flags().StringVar(&rollbackJarURL, "jar-url", "", "Target-version JAR URL (jar runtime)") rollbackCmd.Flags().StringVar(&rollbackJarSHA256, "jar-sha256", "", "Optional target-version JAR SHA256 (jar runtime)") @@ -86,7 +88,9 @@ func runRollback(cmd *cobra.Command, args []string) error { nc.Node.Version = targetVersion nc.Node.PreviousVersion = currentVersion nc.Node.Status = "running" - nc.SaveState() + if err := persistRollbackState(name, nc, start); err != nil { + return err + } writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) writeResult(map[string]any{ @@ -98,6 +102,14 @@ func runRollback(cmd *cobra.Command, args []string) error { return nil } +func persistRollbackState(name string, nc *nodeContext, start time.Time) error { + if err := saveRollbackState(nc); err != nil { + writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) + return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) + } + return nil +} + func rollbackArtifact(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { cause = fmt.Sprintf("%s; stop failed: %v", cause, err) diff --git a/cmd/start.go b/cmd/start.go index 4d2a5844..e57fd7c8 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -16,6 +16,9 @@ var startCmd = &cobra.Command{ RunE: runStart, } +var saveStartState = func(nc *nodeContext) error { return nc.SaveState() } +var resolveStartNodeContext = resolveNodeContext + func init() { rootCmd.AddCommand(startCmd) } @@ -28,7 +31,7 @@ func runStart(cmd *cobra.Command, args []string) error { return err } - nc, err := resolveNodeContext(name) + nc, err := resolveStartNodeContext(name) if err != nil { return err } @@ -41,9 +44,19 @@ func runStart(cmd *cobra.Command, args []string) error { } nc.Node.Status = "running" - nc.SaveState() + if err := persistStartState(name, nc, start); err != nil { + return err + } writeAudit(auditEvent{Command: "start", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) writeResult(map[string]any{"name": name, "status": "running"}) return nil } + +func persistStartState(name string, nc *nodeContext, start time.Time) error { + if err := saveStartState(nc); err != nil { + writeAudit(auditEvent{Command: "start", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) + return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) + } + return nil +} diff --git a/cmd/state_save_error_test.go b/cmd/state_save_error_test.go new file mode 100644 index 00000000..eb56757b --- /dev/null +++ b/cmd/state_save_error_test.go @@ -0,0 +1,115 @@ +package cmd + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/paths" + runtimepkg "github.com/tronprotocol/tron-deployment/internal/runtime" + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +type startTestRuntime struct{} + +func (startTestRuntime) Deploy(context.Context, runtimepkg.DeployOpts) error { return nil } +func (startTestRuntime) Start(context.Context, string) error { return nil } +func (startTestRuntime) Stop(context.Context, string) error { return nil } +func (startTestRuntime) Remove(context.Context, string, bool) error { return nil } +func (startTestRuntime) Status(context.Context, string) (*runtimepkg.NodeStatus, error) { + return nil, nil +} +func (startTestRuntime) Logs(context.Context, string, runtimepkg.LogOpts) (io.ReadCloser, error) { + return io.NopCloser(nil), nil +} + +// TestPersistStateSaveFailuresReturnStateErrorAndAudit covers the six +// persistence handlers directly. The start command's resolver wiring is +// covered separately by TestRunStart_StateSaveFailure. +func TestPersistStateSaveFailuresReturnStateErrorAndAudit(t *testing.T) { + cases := []struct { + name string + set func(func(*nodeContext) error) func(*nodeContext) error + call func(string, *nodeContext, time.Time) error + }{ + {"start", func(v func(*nodeContext) error) func(*nodeContext) error { + old := saveStartState + saveStartState = v + return old + }, persistStartState}, + {"stop", func(v func(*nodeContext) error) func(*nodeContext) error { + old := saveStopState + saveStopState = v + return old + }, persistStopState}, + {"restart", func(v func(*nodeContext) error) func(*nodeContext) error { + old := saveRestartState + saveRestartState = v + return old + }, persistRestartState}, + {"remove", func(v func(*nodeContext) error) func(*nodeContext) error { + old := saveRemoveState + saveRemoveState = v + return old + }, persistRemoveState}, + {"rollback", func(v func(*nodeContext) error) func(*nodeContext) error { + old := saveRollbackState + saveRollbackState = v + return old + }, persistRollbackState}, + {"upgrade", func(v func(*nodeContext) error) func(*nodeContext) error { + old := saveUpgradeState + saveUpgradeState = v + return old + }, persistUpgradeState}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + old := tc.set(func(*nodeContext) error { return errors.New("injected save failure") }) + t.Cleanup(func() { tc.set(old) }) + nc := &nodeContext{Node: &state.ManagedNode{Name: "n0"}, Target: target.NewLocalTarget()} + err := tc.call("n0", nc, time.Now()) + structured, ok := err.(*output.StructuredError) + if !ok || structured.Code != "STATE_ERROR" { + t.Fatalf("error = %v, want STATE_ERROR", err) + } + entries := auditLines(t) + if len(entries) != 1 || entries[0][0] != tc.name || entries[0][2] != "error" { + t.Fatalf("audit = %+v, want %s/error", entries, tc.name) + } + }) + } +} + +func TestRunStart_StateSaveFailure(t *testing.T) { + oldResolve, oldSave := resolveStartNodeContext, saveStartState + t.Cleanup(func() { resolveStartNodeContext, saveStartState = oldResolve, oldSave }) + nc := &nodeContext{ + Node: &state.ManagedNode{Name: "n0", Status: "stopped"}, + Target: target.NewLocalTarget(), + Runtime: startTestRuntime{}, + } + resolveStartNodeContext = func(string) (*nodeContext, error) { return nc, nil } + saveStartState = func(*nodeContext) error { return errors.New("injected save failure") } + + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + cmd := newCmd() + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) + if err := runStart(cmd, []string{"n0"}); err == nil { + t.Fatal("runStart succeeded after state save failure") + } else if structured, ok := err.(*output.StructuredError); !ok || structured.Code != "STATE_ERROR" { + t.Fatalf("runStart error = %v, want STATE_ERROR", err) + } + entries := auditLines(t) + if len(entries) != 1 || entries[0][0] != "start" || entries[0][2] != "error" { + t.Fatalf("audit = %+v, want start/error", entries) + } +} diff --git a/cmd/stop.go b/cmd/stop.go index 6dd41737..c2a6dcc4 100644 --- a/cmd/stop.go +++ b/cmd/stop.go @@ -16,6 +16,8 @@ var stopCmd = &cobra.Command{ RunE: runStop, } +var saveStopState = func(nc *nodeContext) error { return nc.SaveState() } + func init() { rootCmd.AddCommand(stopCmd) } @@ -41,9 +43,19 @@ func runStop(cmd *cobra.Command, args []string) error { } nc.Node.Status = "stopped" - nc.SaveState() + if err := persistStopState(name, nc, start); err != nil { + return err + } writeAudit(auditEvent{Command: "stop", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) writeResult(map[string]any{"name": name, "status": "stopped"}) return nil } + +func persistStopState(name string, nc *nodeContext, start time.Time) error { + if err := saveStopState(nc); err != nil { + writeAudit(auditEvent{Command: "stop", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) + return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) + } + return nil +} diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 96f2a150..ea29c222 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -24,6 +24,8 @@ On failure, automatically rolls back to the previous version.`, RunE: runUpgrade, } +var saveUpgradeState = func(nc *nodeContext) error { return nc.SaveState() } + func init() { upgradeCmd.Flags().StringVar(&upgradeVersion, "version", "", "Target version (required)") upgradeCmd.Flags().StringVar(&upgradeJarURL, "jar-url", "", "Target-version JAR URL (jar runtime)") @@ -93,7 +95,9 @@ func runUpgrade(cmd *cobra.Command, args []string) error { nc.Node.PreviousVersion = previousVersion nc.Node.Version = upgradeVersion nc.Node.Status = "running" - nc.SaveState() + if err := persistUpgradeState(name, nc, start); err != nil { + return err + } writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) writeResult(map[string]any{ @@ -105,6 +109,14 @@ func runUpgrade(cmd *cobra.Command, args []string) error { return nil } +func persistUpgradeState(name string, nc *nodeContext, start time.Time) error { + if err := saveUpgradeState(nc); err != nil { + writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) + return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) + } + return nil +} + func rollbackUpgrade(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { cause = fmt.Sprintf("%s; stop failed: %v", cause, err) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index f2d4a6ed..c84217c3 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -259,7 +259,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { if node.Build == nil && opts.Existing != nil && opts.Existing.IntentHash == opts.IntentHash { remediateJarConfigPermissions(ctx, opts, node) - return noChangeResult(opts, nil, start), nil + return noChangeResult(opts, nil, start) } // Resolve build before render. The artifact path it produces @@ -280,7 +280,7 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { buildSummary != nil && opts.Existing.BuildCacheKey == buildSummary.CacheKey { remediateJarConfigPermissions(ctx, opts, node) - return noChangeResult(opts, buildSummary, start), nil + return noChangeResult(opts, buildSummary, start) } rendered, err := render.RenderHOCONWithSecrets(opts.TemplateDir, opts.Intent, node) @@ -603,13 +603,14 @@ func validateOptions(o Options) error { // pre-build no_change gate and the build-aware gate) check this // before invoking — the helper itself trusts the contract to keep // the body straight-line. -func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) *Result { +func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) (*Result, error) { // Backfill the network on legacy state. A node deployed before // ManagedNode.Network existed has it empty; on a no-op re-apply we // learn it from the intent, so persist it — otherwise the no_change // Result would report network/is_private from the intent while a // follow-up `status` (which reads state) still showed it absent. - // Best-effort: a save failure here doesn't fail the no-op. + // Persist metadata backfills as part of the no-op. Save failures propagate + // so callers cannot report success while legacy state remains stale. if opts.Existing != nil && opts.Store != nil && opts.State != nil { changed := false if opts.Existing.Network != opts.Intent.Network { @@ -623,7 +624,9 @@ func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) * } if changed { opts.Store.UpsertNode(opts.State, *opts.Existing) - _ = opts.Store.Save(opts.State) + if err := opts.Store.Save(opts.State); err != nil { + return nil, fmt.Errorf("persist no-change state: %w", err) + } } } @@ -645,7 +648,7 @@ func noChangeResult(opts Options, buildSummary *BuildSummary, start time.Time) * DurationMs: time.Since(start).Milliseconds(), Build: buildSummary, MonitoringEndpoints: monitoringEndpointsFromExisting(opts.Existing, host), - } + }, nil } // monitoringEndpointsFromExisting returns monitoring URLs from a managed diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index c7a63ed6..662ec5bd 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -165,6 +165,38 @@ func TestApply_NoChangeBackfillsStorageRoot(t *testing.T) { } } +func TestApply_NoChangeBackfillSaveFailurePropagates(t *testing.T) { + parsed := minimalIntent() + parsed.Target.Runtime = "docker" + parsed.Nodes[0].Storage.Data = "/srv/tron/node0/output-directory" + dir := t.TempDir() + statePath := filepath.Join(dir, "state.json") + store, err := state.NewStore(statePath) + if err != nil { + t.Fatal(err) + } + st := &state.DeploymentState{} + if err := store.Save(st); err != nil { + t.Fatal(err) + } + // A directory at the state path makes the final atomic rename fail. + if err := os.Remove(statePath); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(statePath, 0o700); err != nil { + t.Fatal(err) + } + existing := state.ManagedNode{Name: parsed.Name, IntentHash: "same-hash", Runtime: "docker"} + res, err := Apply(context.Background(), Options{ + Intent: parsed, Target: &fakeTarget{}, Store: store, State: st, + IntentHash: "same-hash", Existing: &existing, + DeploymentsDir: filepath.Join(dir, "deployments"), + }) + if err == nil || res != nil { + t.Fatalf("Apply = (%v, %v), want propagated save error", res, err) + } +} + func TestApply_RequiresIntent(t *testing.T) { if _, err := Apply(context.Background(), Options{}); err == nil { t.Error("expected error when Intent is nil") diff --git a/internal/mcp/tools_heal.go b/internal/mcp/tools_heal.go index deec14b0..6a75318c 100644 --- a/internal/mcp/tools_heal.go +++ b/internal/mcp/tools_heal.go @@ -25,6 +25,15 @@ type autoHealArgs struct { DryRun bool `json:"dry_run,omitempty" jsonschema:"if true, propose actions without executing"` } +// Kept as a tiny seam for the state-save failure contract; production uses +// Store.Save directly, while tests can exercise result handling without +// requiring a live Docker/systemd runtime. +var saveHealState = func(store *state.Store, st *state.DeploymentState) error { + return store.Save(st) +} + +var runHealAction = mcpRunHealAction + func registerHealTools(s *mcp.Server) { mcp.AddTool(s, &mcp.Tool{ Name: "auto_heal", @@ -118,7 +127,7 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs }) continue } - err := mcpRunHealAction(ctx, tgt, node, action) + err := runHealAction(ctx, tgt, node, action) entry := map[string]any{ "check": r.Name, "action": action.action, @@ -132,7 +141,9 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs entry["message"] = action.message node.Status = "running" store.UpsertNode(st, *node) - _ = store.Save(st) + if saveErr := saveHealState(store, st); saveErr != nil { + recordHealStateSaveFailure(saveErr, entry, r, &stillFailing) + } } healed = append(healed, entry) } @@ -146,6 +157,15 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs }) } +func recordHealStateSaveFailure(err error, entry map[string]any, check diagnosis.CheckResult, stillFailing *[]diagnosis.CheckResult) { + if err == nil { + return + } + entry["result"] = "failed" + entry["message"] = "state save failed: " + err.Error() + *stillFailing = append(*stillFailing, check) +} + type mcpHealAction struct { action string message string diff --git a/internal/mcp/tools_heal_save_test.go b/internal/mcp/tools_heal_save_test.go new file mode 100644 index 00000000..f685c4be --- /dev/null +++ b/internal/mcp/tools_heal_save_test.go @@ -0,0 +1,79 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "net" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +func TestAutoHealTool_StateSaveFailureMarksHealedFailedAndStillFailing(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + port := closedHealPort(t) + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "heal0", Runtime: "docker", Status: "stopped", Network: "private", HTTPPort: port, + }}}); err != nil { + t.Fatal(err) + } + + originalSave, originalRun := saveHealState, runHealAction + t.Cleanup(func() { saveHealState, runHealAction = originalSave, originalRun }) + saveHealState = func(*state.Store, *state.DeploymentState) error { + return errors.New("injected state save failure") + } + runHealAction = func(context.Context, target.Target, *state.ManagedNode, mcpHealAction) error { + return nil + } + + res, _, err := autoHealTool(context.Background(), nil, autoHealArgs{Name: "heal0"}) + if err != nil { + t.Fatalf("autoHealTool: %v", err) + } + body := extractText(t, res) + var got struct { + Healed []struct { + Result string `json:"result"` + } `json:"healed"` + StillFailing []struct { + Name string `json:"name"` + } `json:"still_failing"` + } + if err := json.Unmarshal([]byte(body), &got); err != nil { + t.Fatalf("decode auto_heal result: %v", err) + } + if len(got.Healed) != 1 || got.Healed[0].Result != "failed" { + t.Fatalf("healed = %+v, want one failed entry", got.Healed) + } + found := false + for _, check := range got.StillFailing { + if check.Name == "port_listening" { + found = true + } + } + if !found { + t.Fatalf("still_failing = %+v, want port_listening", got.StillFailing) + } +} + +func closedHealPort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := l.Addr().(*net.TCPAddr).Port + if err := l.Close(); err != nil { + t.Fatal(err) + } + return port +} diff --git a/internal/state/concurrent_test.go b/internal/state/concurrent_test.go index be713695..53264f3e 100644 --- a/internal/state/concurrent_test.go +++ b/internal/state/concurrent_test.go @@ -1,6 +1,9 @@ package state import ( + "encoding/json" + "os" + "path/filepath" "sync" "sync/atomic" "testing" @@ -78,9 +81,9 @@ func TestLock_AcquireWaitsForRelease(t *testing.T) { // TestStore_ConcurrentSavesDontDropData hammers Save from many // goroutines (each acquiring the lock) and verifies the final on-disk -// state contains every node we wrote. State writes that race on the -// underlying file (without flock or with a buggy atomic-rename) lose -// data here. +// state contains every node we wrote. Lost-update prevention belongs to the +// caller's state.lock protocol; this test covers that protocol plus Save's +// complete-file writes. func TestStore_ConcurrentSavesDontDropData(t *testing.T) { dir := t.TempDir() storePath := dir + "/state.json" @@ -146,6 +149,52 @@ func TestStore_ConcurrentSavesDontDropData(t *testing.T) { } } +func TestStore_ConcurrentSavesWithoutLockKeepCompleteJSON(t *testing.T) { + dir := t.TempDir() + path := dir + "/state.json" + const writers = 20 + + var wg sync.WaitGroup + for i := range writers { + wg.Add(1) + go func(id int) { + defer wg.Done() + store, err := NewStore(path) + if err != nil { + t.Errorf("NewStore: %v", err) + return + } + if err := store.Save(&DeploymentState{Nodes: []ManagedNode{{Name: stableName(id, 0)}}}); err != nil { + t.Errorf("Save: %v", err) + } + }(i) + } + wg.Wait() + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var got DeploymentState + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("final state is corrupted: %v", err) + } + if len(got.Nodes) != 1 { + t.Fatalf("final state has %d nodes, want one complete candidate", len(got.Nodes)) + } + for i := range writers { + if got.Nodes[0].Name == stableName(i, 0) { + goto candidateFound + } + } + t.Fatalf("final state %q is not one of the concurrent Save candidates", got.Nodes[0].Name) + +candidateFound: + if matches, err := filepath.Glob(filepath.Join(dir, ".state-*.tmp")); err != nil || len(matches) != 0 { + t.Fatalf("temporary state files remain: matches=%v err=%v", matches, err) + } +} + func stableName(id, j int) string { return "node-" + itoa(id) + "-" + itoa(j) } diff --git a/internal/state/store.go b/internal/state/store.go index d38ec9c9..bfd886b9 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -69,16 +69,46 @@ func (s *Store) Save(state *DeploymentState) error { return fmt.Errorf("marshal state: %w", err) } - // Atomic write: write to temp file then rename - tmpPath := s.path + ".tmp" - if err := os.WriteFile(tmpPath, data, 0600); err != nil { + // Save guarantees complete file contents via same-directory atomic rename. + // Concurrent lost-update protection is the caller's responsibility while + // holding state.lock (AUD-015 tracked separately). + tmp, err := os.CreateTemp(filepath.Dir(s.path), ".state-*.tmp") + if err != nil { + return fmt.Errorf("create state temp file: %w", err) + } + tmpPath := tmp.Name() + cleanup := func() { + _ = tmp.Close() + _ = os.Remove(tmpPath) + } + if _, err := tmp.Write(data); err != nil { + cleanup() return fmt.Errorf("write state temp file: %w", err) } - + if err := tmp.Sync(); err != nil { + cleanup() + return fmt.Errorf("sync state temp file: %w", err) + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpPath) + return fmt.Errorf("close state temp file: %w", err) + } if err := os.Rename(tmpPath, s.path); err != nil { - os.Remove(tmpPath) + _ = os.Remove(tmpPath) return fmt.Errorf("rename state file: %w", err) } + // Sync the directory entry so a crash after rename cannot lose the name. + dir, err := os.Open(filepath.Dir(s.path)) + if err != nil { + return fmt.Errorf("open state directory: %w", err) + } + if err := dir.Sync(); err != nil { + _ = dir.Close() + return fmt.Errorf("sync state directory: %w", err) + } + if err := dir.Close(); err != nil { + return fmt.Errorf("close state directory: %w", err) + } return nil } diff --git a/internal/state/store_test.go b/internal/state/store_test.go index 387c8093..275a3cd6 100644 --- a/internal/state/store_test.go +++ b/internal/state/store_test.go @@ -85,9 +85,9 @@ func TestSaveAtomic(t *testing.T) { t.Fatalf("save: %v", err) } - // No temp file should be left behind - if _, err := os.Stat(path + ".tmp"); !os.IsNotExist(err) { - t.Errorf("temp file still present: %v", err) + // No staging file should be left behind. + if matches, err := filepath.Glob(filepath.Join(filepath.Dir(path), ".state-*.tmp")); err != nil || len(matches) != 0 { + t.Errorf("temporary state files remain: matches=%v err=%v", matches, err) } // Reload and verify contents round-trip @@ -101,6 +101,43 @@ func TestSaveAtomic(t *testing.T) { } } +func TestSave_ErrorPropagatesAndCleansTemp(t *testing.T) { + dir := t.TempDir() + // A directory at the destination makes the final rename fail. + path := filepath.Join(dir, "state.json") + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + s, err := NewStore(filepath.Join(dir, "state.json")) + if err != nil { + t.Fatal(err) + } + if err := s.Save(&DeploymentState{Nodes: []ManagedNode{{Name: "broken"}}}); err == nil { + t.Fatal("Save succeeded despite destination being a directory") + } + if matches, err := filepath.Glob(filepath.Join(dir, ".state-*.tmp")); err != nil || len(matches) != 0 { + t.Errorf("temporary state files remain after failure: matches=%v err=%v", matches, err) + } +} + +func TestSave_ReplacesWithCompleteJSON(t *testing.T) { + s, path := newTempStore(t) + if err := s.Save(&DeploymentState{Nodes: []ManagedNode{{Name: "complete"}}}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var loaded DeploymentState + if err := json.Unmarshal(data, &loaded); err != nil { + t.Fatalf("final state is not complete JSON: %v", err) + } + if len(loaded.Nodes) != 1 || loaded.Nodes[0].Name != "complete" { + t.Fatalf("unexpected final state: %+v", loaded) + } +} + func TestSave_PermissionsRestrictive(t *testing.T) { s, path := newTempStore(t) st, _ := s.Load() From a0d7dcec565abcac4e548501d30ddc1e9490720b Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 16:18:48 +0800 Subject: [PATCH 20/49] fix(snapshot): stage downloads and publish atomically; merge build env maps AUD-029: extract into an in-destination staging dir, verify digests before publish, and swap via rename with a same-filesystem backup so a failed publish restores the previous chain data (restore failure retains the backup and reports both errors). AUD-030: merge host/docker build env through explicit maps so intent env wins deterministically over host defaults regardless of iteration order. AUD-037: stop creating destination directories during preflight; checks stay read-only and creation happens in the download itself. --- internal/build/builder.go | 15 +++- internal/build/runner_host.go | 19 +++- internal/build/runner_host_test.go | 34 ++++++++ internal/snapshot/download.go | 93 ++++++++++++++++---- internal/snapshot/snapshot_test.go | 129 ++++++++++++++++++++++++++++ internal/snapshot/transport_test.go | 20 ++++- 6 files changed, 287 insertions(+), 23 deletions(-) diff --git a/internal/build/builder.go b/internal/build/builder.go index 536fc696..00f962a7 100644 --- a/internal/build/builder.go +++ b/internal/build/builder.go @@ -598,17 +598,20 @@ func (r Request) validate() error { // Intent values override host values on key collision (last writer // wins in docker's `-e`). Output is sorted for reproducible argv. func allowedEnvPassthrough(intent map[string]string) []string { - out := []string{} + values := make(map[string]string) for k := range envAllowlist { if v, ok := os.LookupEnv(k); ok { - out = append(out, k+"="+v) + values[k] = v } } for _, e := range os.Environ() { if !strings.HasPrefix(e, orgGradleProjectPrefix) { continue } - out = append(out, e) + key, value, ok := strings.Cut(e, "=") + if ok { + values[key] = value + } } keys := make([]string, 0, len(intent)) for k := range intent { @@ -619,7 +622,11 @@ func allowedEnvPassthrough(intent map[string]string) []string { if err := ValidateEnvKey(k); err != nil { continue } - out = append(out, k+"="+intent[k]) + values[k] = intent[k] + } + out := make([]string, 0, len(values)) + for k, v := range values { + out = append(out, k+"="+v) } sort.Strings(out) return out diff --git a/internal/build/runner_host.go b/internal/build/runner_host.go index e1285d95..ffe5a72f 100644 --- a/internal/build/runner_host.go +++ b/internal/build/runner_host.go @@ -185,8 +185,23 @@ func runGradleHost(ctx context.Context, r *resolved) error { // uses, so semantics line up across builders: an `org.gradle.project.*` // override or a whitelisted KEY=VALUE flows through identically. func hostBuildEnv(intent map[string]string) []string { - base := os.Environ() - return append(base, allowedEnvPassthrough(intent)...) + values := make(map[string]string) + for _, entry := range os.Environ() { + if key, value, ok := strings.Cut(entry, "="); ok { + values[key] = value + } + } + for _, entry := range allowedEnvPassthrough(intent) { + if key, value, ok := strings.Cut(entry, "="); ok { + values[key] = value + } + } + merged := make([]string, 0, len(values)) + for key, value := range values { + merged = append(merged, key+"="+value) + } + sort.Strings(merged) + return merged } // findLargestFatJAR walks the source tree for jars whose IMMEDIATE diff --git a/internal/build/runner_host_test.go b/internal/build/runner_host_test.go index 86ed525e..07c6b346 100644 --- a/internal/build/runner_host_test.go +++ b/internal/build/runner_host_test.go @@ -9,6 +9,40 @@ import ( "testing" ) +func TestAllowedEnvPassthroughIntentOverridesHost(t *testing.T) { + t.Setenv("GRADLE_OPTS", "host-value") + got := allowedEnvPassthrough(map[string]string{"GRADLE_OPTS": "intent-value"}) + found := false + for _, value := range got { + if value == "GRADLE_OPTS=intent-value" { + found = true + } + if value == "GRADLE_OPTS=host-value" { + t.Fatalf("host value leaked after intent override: %v", got) + } + } + if !found { + t.Fatalf("intent value missing: %v", got) + } +} + +func TestHostBuildEnvIntentOverridesHostWithoutDuplicate(t *testing.T) { + t.Setenv("GRADLE_OPTS", "host-value") + got := hostBuildEnv(map[string]string{"GRADLE_OPTS": "intent-value"}) + count := 0 + for _, value := range got { + if strings.HasPrefix(value, "GRADLE_OPTS=") { + count++ + if value != "GRADLE_OPTS=intent-value" { + t.Fatalf("host value retained: %v", got) + } + } + } + if count != 1 { + t.Fatalf("GRADLE_OPTS entries = %d, want one: %v", count, got) + } +} + // TestResolveHostIdentity pins the contract `realHostRunner` relies // on: the function returns a stable digest with the canonical // `sha256:` prefix + a human-readable ref starting with `host:`. Two diff --git a/internal/snapshot/download.go b/internal/snapshot/download.go index 80855637..b36cf16d 100644 --- a/internal/snapshot/download.go +++ b/internal/snapshot/download.go @@ -242,13 +242,12 @@ func Preflight(ctx context.Context, opts DownloadOptions) (*PreflightResult, err } } - // Local checks. - if err := os.MkdirAll(opts.DestDir, 0o755); err != nil { - return nil, fmt.Errorf("ensure dest dir: %w", err) - } - free, err := freeBytes(opts.DestDir) + // Local checks are read-only. A missing destination is checked against its + // nearest existing parent; dry-run must not create filesystem entries. + checkPath := existingParent(opts.DestDir) + free, err := freeBytes(checkPath) if err != nil { - return nil, fmt.Errorf("statfs %s: %w", opts.DestDir, err) + return nil, fmt.Errorf("statfs %s: %w", checkPath, err) } r.FreeBytes = free @@ -271,13 +270,12 @@ func Preflight(ctx context.Context, opts DownloadOptions) (*PreflightResult, err } // Download streams the snapshot tarball, hashes it on the fly, and -// extracts it directly to disk — no full .tgz is ever written. This -// halves the disk-space requirement compared to download-then-extract -// and shaves the extraction wall-time off the user-perceived runtime. +// extracts it into a sibling staging directory inside the destination. +// After all digest checks pass, the staged output is atomically published. // // Pipeline: // -// HTTP body → TeeReader(md5 + sha256) → progress wrapper → gzip → tar → fs +// HTTP body → TeeReader(md5 + sha256) → progress wrapper → gzip → tar → stage → atomic publish // // A note on what the digests buy. The upstream .md5sum sidecar comes down // the same connection as the tarball, so on a cleartext mirror it proves @@ -285,11 +283,10 @@ func Preflight(ctx context.Context, opts DownloadOptions) (*PreflightResult, err // ExpectedSHA256, obtained out of band, is the digest that can actually // detect substitution; the plaintext warning tells the operator so. // -// On any error during streaming we abort cleanly: partially extracted -// files stay where they are, the next run with --force will overwrite -// them. We don't try to roll back, because rolling back a 50 GB -// extraction would cost an extra 50 GB of seeks for what is fundamentally -// a discardable cache. +// On any error during streaming or verification, the live destination is +// unchanged and the staging directory is removed. With --force, the old +// snapshot is moved to a sibling backup while publishing; it is removed only +// after the new snapshot is live, or after a successful restore. func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error) { if opts.DestDir == "" { return nil, errors.New("dest dir is required") @@ -361,6 +358,15 @@ func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error } } + if err := os.MkdirAll(opts.DestDir, 0o755); err != nil { + return nil, fmt.Errorf("ensure dest dir: %w", err) + } + stage, err := os.MkdirTemp(opts.DestDir, ".snapshot-stage-*") + if err != nil { + return nil, fmt.Errorf("create snapshot stage: %w", err) + } + defer os.RemoveAll(stage) + start := time.Now() url := pre.URL req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody) @@ -395,7 +401,7 @@ func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error } defer gz.Close() - extracted, err := extractTar(gz, opts.DestDir, opts.Force) + extracted, err := extractTar(gz, stage, true) if err != nil { return nil, err } @@ -429,6 +435,9 @@ func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error } verified = true } + if err := publishSnapshot(stage, opts.DestDir, opts.Force); err != nil { + return nil, err + } dur := time.Since(start) return &DownloadResult{ @@ -448,6 +457,45 @@ func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error }, nil } +var snapshotRename = os.Rename + +func publishSnapshot(stage, dest string, force bool) error { + source := filepath.Join(stage, snapshotRoot) + target := filepath.Join(dest, snapshotRoot) + if _, err := os.Stat(source); err != nil { + return fmt.Errorf("staged snapshot missing %s: %w", source, err) + } + backup := "" + if _, err := os.Lstat(target); err == nil { + if !force { + return &OverwriteError{Path: target, Message: fmt.Sprintf("existing database at %s; pass --force to overwrite", target)} + } + var backupErr error + backup, backupErr = os.MkdirTemp(dest, ".snapshot-backup-*") + if backupErr != nil { + return fmt.Errorf("create snapshot backup: %w", backupErr) + } + os.RemoveAll(backup) + if err := snapshotRename(target, backup); err != nil { + return fmt.Errorf("stage existing snapshot: %w", err) + } + } + if err := snapshotRename(source, target); err != nil { + if backup != "" { + restoreErr := snapshotRename(backup, target) + if restoreErr != nil { + return fmt.Errorf("publish snapshot: %w; restore snapshot: %v; backup retained at %s", err, restoreErr, backup) + } + _ = os.RemoveAll(backup) + } + return fmt.Errorf("publish snapshot: %w", err) + } + if backup != "" { + _ = os.RemoveAll(backup) + } + return nil +} + // extractTar walks an already-gunzipped tar stream and writes entries // under destDir. Returns the count of regular files written. We mimic // tron-docker's safety check (no path traversal, no writing through @@ -712,6 +760,19 @@ func isNonEmptyDir(p string) bool { return len(entries) > 0 } +func existingParent(path string) string { + path = filepath.Clean(path) + for current := path; ; current = filepath.Dir(current) { + if info, err := os.Stat(current); err == nil && info.IsDir() { + return current + } + parent := filepath.Dir(current) + if parent == current { + return "." + } + } +} + func freeBytes(path string) (uint64, error) { var st syscall.Statfs_t if err := syscall.Statfs(path, &st); err != nil { diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 70b328fb..302c1ee7 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "context" + "errors" "io" "net/http" "net/http/httptest" @@ -110,6 +111,134 @@ func TestExtractTar_StreamingHonoursForce(t *testing.T) { } } +func TestPreflightDoesNotCreateMissingDestination(t *testing.T) { + base := t.TempDir() + dest := filepath.Join(base, "missing", "destination") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodHead { + w.Header().Set("Content-Length", "1") + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + _, err := Preflight(context.Background(), DownloadOptions{ + Source: Source{BaseURL: server.URL}, Backup: "b", Kind: DBKindLite, DestDir: dest, + HTTPClient: server.Client(), + }) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(dest); !os.IsNotExist(err) { + t.Fatalf("dry-run created destination: stat err=%v", err) + } +} + +func TestPublishSnapshotForceReplacesAndCleansBackup(t *testing.T) { + dest := t.TempDir() + stage := filepath.Join(dest, ".snapshot-stage-test") + old := filepath.Join(dest, snapshotRoot, "database", "CURRENT") + if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(old, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + newFile := filepath.Join(stage, snapshotRoot, "database", "CURRENT") + if err := os.MkdirAll(filepath.Dir(newFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(newFile, []byte("new"), 0o644); err != nil { + t.Fatal(err) + } + if err := publishSnapshot(stage, dest, true); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(old) + if err != nil || string(got) != "new" { + t.Fatalf("published content = %q, err=%v", got, err) + } + if entries, err := filepath.Glob(filepath.Join(dest, ".snapshot-backup-*")); err != nil || len(entries) != 0 { + t.Fatalf("backup leaked: %v (err=%v)", entries, err) + } +} + +func TestPublishSnapshotFailureRestoresOldSnapshot(t *testing.T) { + oldRename := snapshotRename + t.Cleanup(func() { snapshotRename = oldRename }) + dest := t.TempDir() + stage := filepath.Join(dest, ".snapshot-stage-test") + old := filepath.Join(dest, snapshotRoot, "database", "CURRENT") + if err := os.MkdirAll(filepath.Dir(old), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(old, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(stage, snapshotRoot), 0o755); err != nil { + t.Fatal(err) + } + var publish bool + snapshotRename = func(from, to string) error { + if strings.Contains(from, ".snapshot-stage-test") && !publish { + publish = true + return errors.New("injected publish failure") + } + return os.Rename(from, to) + } + err := publishSnapshot(stage, dest, true) + if err == nil || !strings.Contains(err.Error(), "publish snapshot") { + t.Fatalf("error = %v, want publish failure", err) + } + got, readErr := os.ReadFile(old) + if readErr != nil || string(got) != "old" { + t.Fatalf("restore content = %q, err=%v", got, readErr) + } + if entries, globErr := filepath.Glob(filepath.Join(dest, ".snapshot-backup-*")); globErr != nil || len(entries) != 0 { + t.Fatalf("backup leaked after successful restore: %v (err=%v)", entries, globErr) + } +} + +func TestPublishSnapshotAndRestoreFailureRetainsBackup(t *testing.T) { + oldRename := snapshotRename + t.Cleanup(func() { snapshotRename = oldRename }) + dest := t.TempDir() + stage := filepath.Join(dest, ".snapshot-stage-test") + old := filepath.Join(dest, snapshotRoot) + if err := os.MkdirAll(filepath.Join(old, "database"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(old, "database", "CURRENT"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(stage, snapshotRoot), 0o755); err != nil { + t.Fatal(err) + } + var publishAttempts int + snapshotRename = func(from, to string) error { + if strings.Contains(from, ".snapshot-stage-test") { + publishAttempts++ + return errors.New("injected publish failure") + } + if strings.Contains(from, ".snapshot-backup-") { + return errors.New("injected restore failure") + } + return os.Rename(from, to) + } + err := publishSnapshot(stage, dest, true) + if err == nil || !strings.Contains(err.Error(), "restore snapshot") || !strings.Contains(err.Error(), "backup retained") { + t.Fatalf("error = %v, want publish and restore failures", err) + } + if publishAttempts != 1 { + t.Fatalf("publish attempts = %d, want one", publishAttempts) + } + entries, globErr := filepath.Glob(filepath.Join(dest, ".snapshot-backup-*")) + if globErr != nil || len(entries) != 1 { + t.Fatalf("backup not retained: %v (err=%v)", entries, globErr) + } +} + func TestExtractTar_RejectsTraversal(t *testing.T) { tgz := buildTGZ(t, map[string]string{ "../escape.txt": "nope", diff --git a/internal/snapshot/transport_test.go b/internal/snapshot/transport_test.go index ea98e08c..3c4429fa 100644 --- a/internal/snapshot/transport_test.go +++ b/internal/snapshot/transport_test.go @@ -9,6 +9,8 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" ) @@ -181,12 +183,21 @@ func TestDownload_SHA256PinMatches(t *testing.T) { func TestDownload_SHA256PinMismatchFails(t *testing.T) { srv := startTinyMirror(t, false) defer srv.Close() + dest := t.TempDir() + db := filepath.Join(dest, "output-directory", "database", "CURRENT") + if err := os.MkdirAll(filepath.Dir(db), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(db, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } _, err := Download(context.Background(), DownloadOptions{ Source: sourceFor(srv.URL), Backup: "backup20250101", Kind: DBKindLite, - DestDir: t.TempDir(), + DestDir: dest, + Force: true, ExpectedSHA256: strings.Repeat("ab", 32), // valid hex, wrong value }) if err == nil { @@ -195,6 +206,13 @@ func TestDownload_SHA256PinMismatchFails(t *testing.T) { if !strings.Contains(err.Error(), "sha256 mismatch") { t.Fatalf("wrong error for a bad pin: %v", err) } + got, readErr := os.ReadFile(db) + if readErr != nil || string(got) != "old" { + t.Fatalf("digest mismatch changed destination: content=%q err=%v", got, readErr) + } + if entries, globErr := filepath.Glob(filepath.Join(dest, ".snapshot-*")); globErr != nil || len(entries) != 0 { + t.Fatalf("staging artifacts leaked: %v (err=%v)", entries, globErr) + } } // TestDownload_SHA256PinRejectedBeforeTransfer — a malformed pin must fail From 9e32bd8265dcff58586b469f1854eaa30c656d4f Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 17:09:16 +0800 Subject: [PATCH 21/49] fix(network): harden multi-node lifecycle, versioned intent hash, and port persistence AUD-031: network create aggregates per-node failures into DEPLOY_ERROR AUD-032: add skips per-node monitoring (SkipMonitoring) and reloads network monitoring from the correct target host AUD-033: txgen broadcast producer honors ctx cancellation while blocked AUD-035: network upgrade child stdout/stderr captured (1MiB bound), envelope shape validated, no --auto-approve passthrough AUD-036: config render writes per-node dirs for multi-node intents AUD-038: destroy resolves runtime per node (jar/docker), cleans up per distinct target identity, dedupes target-level failures AUD-043: intent hash v2 (domain-separated) with legacy raw-hash migration gate; recorded monitoring state participates in legacy equivalence both directions; auto_ports restores all seven persisted ports from state follow-up: rollback no longer passes target-version --jar-url --- cmd/apply.go | 90 +++++++++++++++++++++++++--- cmd/apply_hash_test.go | 91 +++++++++++++++++++++++++++++ cmd/config/render.go | 6 +- cmd/config/render_multinode_test.go | 30 ++++++++++ cmd/config/render_perms_test.go | 21 +++++++ cmd/network/add.go | 76 ++++++------------------ cmd/network/create.go | 54 ++++++++++++----- cmd/network/create_test.go | 82 ++++++++++++++++++++++++++ cmd/network/destroy.go | 74 +++++++++++++++-------- cmd/network/upgrade.go | 70 ++++++++++++++++++++-- cmd/network/upgrade_test.go | 52 +++++++++++++++++ internal/state/types.go | 7 ++- tools/txgen/broadcast.go | 35 +++++++++-- tools/txgen/broadcast_test.go | 52 +++++++++++++++++ 14 files changed, 626 insertions(+), 114 deletions(-) create mode 100644 cmd/apply_hash_test.go create mode 100644 cmd/config/render_multinode_test.go create mode 100644 cmd/network/create_test.go create mode 100644 tools/txgen/broadcast_test.go diff --git a/cmd/apply.go b/cmd/apply.go index fe9eda1d..1f180e31 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -1,11 +1,14 @@ package cmd import ( + "crypto/sha256" "errors" "fmt" "os" "time" + "gopkg.in/yaml.v3" + "github.com/spf13/cobra" "github.com/tronprotocol/tron-deployment/internal/apply" @@ -95,9 +98,6 @@ func runApply(cmd *cobra.Command, args []string) error { } parsed.Monitoring.Enabled = intent.BoolPtr(true) intent.ApplyMonitoringDefaults(parsed.Monitoring) - } else if parsed.Monitoring != nil && parsed.Monitoring.Enabled != nil && *parsed.Monitoring.Enabled { - // Intent says enabled=true, but no --monitor flag: disable. - parsed.Monitoring.Enabled = intent.BoolPtr(false) } // 2. Resolve target. SSH cert handling lives here in the cmd @@ -128,14 +128,29 @@ func runApply(cmd *cobra.Command, args []string) error { } // 5. Compute intent hash. - intentData, _ := os.ReadFile(applyIntentPath) - intentHash := apply.IntentHashFromBytes(intentData) + rawIntent, _ := os.ReadFile(applyIntentPath) + existing := store.GetNode(deployState, parsed.Name) + if parsed.Target.AutoPorts && existing != nil { + if len(parsed.Nodes) == 1 { + restoreAutoPorts(&parsed.Nodes[0], existing) + } + } + canonicalIntent, err := yaml.Marshal(parsed) + if err != nil { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "marshal effective intent: "+err.Error()) + } + intentHash := versionedIntentHash(canonicalIntent) + legacyMatch := existing != nil && legacyIntentHashMatches(existing.IntentHash, rawIntent, parsed, existing) + if legacyMatch { + // Preserve the recorded legacy hash for a true no-op. The next + // intentional change will write the versioned hash during apply. + intentHash = existing.IntentHash + } // 6. HUMAN_REQUIRED gate. The internal/apply package handles the // no-op short-circuit (same-hash → "no_change") on its own; we // only need to guard the destructive change-on-existing-node case. - existing := store.GetNode(deployState, parsed.Name) - if existing != nil && existing.IntentHash != intentHash && !applyAutoApprove { + if existing != nil && existing.IntentHash != intentHash && !legacyMatch && !applyAutoApprove { return exitWithError("HUMAN_REQUIRED", output.ExitHumanRequired, fmt.Sprintf("Changes detected for node %q. Review with: trond plan --intent %s", parsed.Name, applyIntentPath), "Re-run with --auto-approve to apply changes", @@ -201,6 +216,67 @@ func runApply(cmd *cobra.Command, args []string) error { return nil } +func versionedIntentHash(data []byte) string { + h := sha256.New() + h.Write([]byte("intent-hash-v2\x00")) + h.Write(data) + return fmt.Sprintf("v2:%x", h.Sum(nil)) +} + +func legacyIntentHashMatches(stored string, raw []byte, parsed *intent.Intent, existing *state.ManagedNode) bool { + if len(stored) == 0 || len(stored) > 3 && stored[:3] == "v2:" { + return false + } + if apply.IntentHashFromBytes(raw) == stored { + var rawIntent intent.Intent + if err := yaml.Unmarshal(raw, &rawIntent); err != nil { + return false + } + if monitoringEnabled(rawIntent.Monitoring) != monitoringEnabled(parsed.Monitoring) { + return false + } + if monitoringEnabled(parsed.Monitoring) != recordedMonitoringEnabled(existing) { + return false + } + return true + } + canonical, err := yaml.Marshal(parsed) + return err == nil && apply.IntentHashFromBytes(canonical) == stored && + monitoringEnabled(parsed.Monitoring) == recordedMonitoringEnabled(existing) +} + +func monitoringEnabled(m *intent.Monitoring) bool { + return m != nil && m.Enabled != nil && *m.Enabled +} + +func recordedMonitoringEnabled(existing *state.ManagedNode) bool { + return existing != nil && existing.Monitoring != nil && existing.Monitoring.Enabled +} + +func restoreAutoPorts(node *intent.NodeSpec, existing *state.ManagedNode) { + if existing.HTTPPort != 0 { + node.Ports.HTTP = existing.HTTPPort + } + if existing.GRPCPort != 0 { + node.Ports.GRPC = existing.GRPCPort + } + if existing.P2PPort != 0 { + node.Ports.P2P = existing.P2PPort + } + if existing.MetricsPort != 0 { + node.Ports.Metrics = existing.MetricsPort + } + if existing.SolidityHTTPPort != 0 { + node.Ports.SolidityHTTP = existing.SolidityHTTPPort + } + if existing.SolidityGRPCPort != 0 { + node.Ports.SolidityGRPC = existing.SolidityGRPCPort + } + if existing.JSONRPCPort != 0 { + node.Ports.JSONRPC = existing.JSONRPCPort + } +} + func resolveTarget(parsed *intent.Intent) (target.Target, error) { switch parsed.Target.Type { case "ssh": diff --git a/cmd/apply_hash_test.go b/cmd/apply_hash_test.go new file mode 100644 index 00000000..32501c1a --- /dev/null +++ b/cmd/apply_hash_test.go @@ -0,0 +1,91 @@ +package cmd + +import ( + "testing" + + "github.com/tronprotocol/tron-deployment/internal/apply" + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/state" + "gopkg.in/yaml.v3" +) + +func TestLegacyIntentHashMatchesRawHash(t *testing.T) { + raw := []byte("name: n\nnetwork: private\n") + parsed := &intent.Intent{Name: "n", Network: "private"} + if !legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, nil) { + t.Fatal("legacy raw hash not accepted") + } +} + +func TestVersionedIntentHashIsStable(t *testing.T) { + data := []byte("effective intent") + if versionedIntentHash(data) != versionedIntentHash(data) { + t.Fatal("versioned hash is unstable") + } + if versionedIntentHash(data) == string(data) { + t.Fatal("hash did not version/encode input") + } +} + +func TestLegacyRawHashRejectsMonitoringChange(t *testing.T) { + raw := []byte("name: n\nnetwork: private\nmonitoring:\n enabled: false\n") + parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(true)}} + if legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, nil) { + t.Fatal("monitoring change treated as legacy no-op") + } +} + +func TestLegacyRawHashRejectsMissingRecordedMonitoring(t *testing.T) { + raw := []byte("name: n\nnetwork: private\nmonitoring:\n enabled: true\n") + parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(true)}} + if legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, &state.ManagedNode{}) { + t.Fatal("missing recorded monitoring treated as legacy no-op") + } +} + +func TestLegacyRawHashRejectsRecordedMonitoringWhenCurrentDisabled(t *testing.T) { + raw := []byte("name: n\nnetwork: private\nmonitoring:\n enabled: false\n") + parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(false)}} + existing := &state.ManagedNode{Monitoring: &state.MonitoringState{Enabled: true}} + if legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, existing) { + t.Fatal("recorded enabled monitoring treated as legacy no-op when current intent is disabled") + } +} + +func TestLegacyCanonicalHashRejectsRecordedMonitoringWhenCurrentDisabled(t *testing.T) { + parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(false)}} + canonical, err := yaml.Marshal(parsed) + if err != nil { + t.Fatal(err) + } + existing := &state.ManagedNode{Monitoring: &state.MonitoringState{Enabled: true}} + if legacyIntentHashMatches(apply.IntentHashFromBytes(canonical), []byte("not-the-canonical-input"), parsed, existing) { + t.Fatal("canonical legacy hash treated recorded enabled monitoring as no-op when current intent is disabled") + } +} + +func TestRestoreAutoPortsPreservesMissingLegacyPorts(t *testing.T) { + n := &intent.NodeSpec{} + n.Ports.HTTP, n.Ports.GRPC, n.Ports.SolidityHTTP, n.Ports.SolidityGRPC, n.Ports.JSONRPC, n.Ports.P2P, n.Ports.Metrics = 10, 11, 12, 13, 14, 15, 16 + restoreAutoPorts(n, &state.ManagedNode{HTTPPort: 1, GRPCPort: 2, P2PPort: 6, MetricsPort: 7}) + got := []int{n.Ports.HTTP, n.Ports.GRPC, n.Ports.SolidityHTTP, n.Ports.SolidityGRPC, n.Ports.JSONRPC, n.Ports.P2P, n.Ports.Metrics} + want := []int{1, 2, 12, 13, 14, 6, 7} + for i := range got { + if got[i] != want[i] { + t.Fatalf("ports = %v, want %v", got, want) + } + } +} + +func TestRestoreAutoPortsAllSeven(t *testing.T) { + n := &intent.NodeSpec{} + s := &state.ManagedNode{HTTPPort: 1, GRPCPort: 2, SolidityHTTPPort: 3, SolidityGRPCPort: 4, JSONRPCPort: 5, P2PPort: 6, MetricsPort: 7} + restoreAutoPorts(n, s) + got := []int{n.Ports.HTTP, n.Ports.GRPC, n.Ports.SolidityHTTP, n.Ports.SolidityGRPC, n.Ports.JSONRPC, n.Ports.P2P, n.Ports.Metrics} + want := []int{1, 2, 3, 4, 5, 6, 7} + for i := range got { + if got[i] != want[i] { + t.Fatalf("ports = %v, want %v", got, want) + } + } +} diff --git a/cmd/config/render.go b/cmd/config/render.go index 94bf5c94..29b88376 100644 --- a/cmd/config/render.go +++ b/cmd/config/render.go @@ -136,7 +136,11 @@ func runRender(cmd *cobra.Command, args []string) error { // the rendered bodies inline; that doesn't preclude also // writing them. if renderOutputDir != "" { - if err := writeRenderedFiles(renderOutputDir, parsed.Name, hocon, composeYAML, systemdUnit); err != nil { + nodeDir := renderOutputDir + if len(parsed.Nodes) > 1 { + nodeDir = filepath.Join(renderOutputDir, fmt.Sprintf("node%d", i)) + } + if err := writeRenderedFiles(nodeDir, parsed.Name, hocon, composeYAML, systemdUnit); err != nil { return err } } diff --git a/cmd/config/render_multinode_test.go b/cmd/config/render_multinode_test.go new file mode 100644 index 00000000..6fae00e7 --- /dev/null +++ b/cmd/config/render_multinode_test.go @@ -0,0 +1,30 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" +) + +func TestRunRenderMultiNodeCreatesSeparateDirectories(t *testing.T) { + intentPath := filepath.Join(t.TempDir(), "intent.yaml") + if err := os.WriteFile(intentPath, []byte("name: net\nnetwork: private\ntarget:\n type: local\nnodes:\n - type: fullnode\n resources: {memory: 4G}\n - type: fullnode\n resources: {memory: 4G}\n"), 0600); err != nil { + t.Fatal(err) + } + outDir := t.TempDir() + oldDir, oldNode := renderOutputDir, renderNodeFilter + defer func() { renderOutputDir, renderNodeFilter = oldDir, oldNode }() + renderOutputDir, renderNodeFilter = outDir, -1 + cmd := &cobra.Command{} + cmd.Flags().String("output", "", "") + if err := runRender(cmd, []string{intentPath}); err != nil { + t.Fatal(err) + } + for _, node := range []string{"node0", "node1"} { + if _, err := os.Stat(filepath.Join(outDir, node, "net.conf")); err != nil { + t.Fatalf("%s config: %v", node, err) + } + } +} diff --git a/cmd/config/render_perms_test.go b/cmd/config/render_perms_test.go index c2e5459c..5d9aa4f2 100644 --- a/cmd/config/render_perms_test.go +++ b/cmd/config/render_perms_test.go @@ -7,6 +7,27 @@ import ( "testing" ) +func TestWriteRenderedFilesUsesIndependentNodeDirectories(t *testing.T) { + dir := t.TempDir() + if err := writeRenderedFiles(filepath.Join(dir, "node0"), "net-node0", "a", "", ""); err != nil { + t.Fatal(err) + } + if err := writeRenderedFiles(filepath.Join(dir, "node1"), "net-node1", "b", "", ""); err != nil { + t.Fatal(err) + } + first, err := os.ReadFile(filepath.Join(dir, "node0", "net-node0.conf")) + if err != nil { + t.Fatal(err) + } + second, err := os.ReadFile(filepath.Join(dir, "node1", "net-node1.conf")) + if err != nil { + t.Fatal(err) + } + if string(first) != "a" || string(second) != "b" { + t.Fatalf("rendered files clobbered: %q %q", first, second) + } +} + // The rendered HOCON carries the witness signing key inlined by // render.RenderHOCON, so `config render --output-dir` must never leave it // group/world readable. These tests pin the modes of every artifact diff --git a/cmd/network/add.go b/cmd/network/add.go index 1a294f1f..4134f518 100644 --- a/cmd/network/add.go +++ b/cmd/network/add.go @@ -10,6 +10,7 @@ import ( "time" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" applypkg "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/guard" @@ -17,7 +18,6 @@ import ( "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/render" - "github.com/tronprotocol/tron-deployment/internal/runtime" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -175,75 +175,33 @@ func runAdd(cmd *cobra.Command, args []string) error { } // If the network already has monitoring deployed, auto-enable metrics - // for the new node so Prometheus can scrape it. + // for the new node so Prometheus can scrape it. Check the target itself; + // SSH deployments keep this compose file on the remote host. monCompose := filepath.Join(paths.Deployments(), addNetworkName+"-monitoring", "docker-compose.yaml") - if _, err := os.Stat(monCompose); err == nil { + if _, err := tgt.ReadFile(cmd.Context(), monCompose); err == nil { enabled := true parsed.Monitoring = &intent.Monitoring{Enabled: &enabled} intent.ApplyMonitoringDefaults(parsed.Monitoring) } - templateDir := findTemplatesDir() - rendered, err := render.RenderHOCONWithSecrets(templateDir, parsed, node) + clone := *parsed + clone.Name = nodeName + clone.Nodes = []intent.NodeSpec{*node} + data, err := yaml.Marshal(&clone) if err != nil { - return fmt.Errorf("render config: %w", err) + return fmt.Errorf("hash add intent: %w", err) } - // Deploy path — needs the real witness key inlined. - hocon := rendered.Deployable() - - memGB, err := render.ParseMemoryGB(node.Resources.Memory) + _, err = applypkg.Apply(cmd.Context(), applypkg.Options{ + Intent: &clone, Target: tgt, Store: store, State: deployState, + IntentHash: applypkg.IntentHashFromBytes(data), Existing: store.GetNode(deployState, nodeName), + TemplateDir: findTemplatesDir(), DeploymentsDir: paths.Deployments(), IntentPath: addIntentPath, + RequirePrivate: guard.Requested(), + SkipMonitoring: parsed.Monitoring != nil && parsed.Monitoring.Enabled != nil && *parsed.Monitoring.Enabled, + }) if err != nil { - return output.NewError("VALIDATION_ERROR", output.ExitValidationError, - fmt.Sprintf("invalid resources.memory %q: %v", node.Resources.Memory, err)) - } - jvmArgs := render.JVMArgsString(memGB, 17, node.JVM) - composeYAML := render.RenderCompose(nodeName, parsed, node, "", jvmArgs, "") - - rt := runtime.NewDockerRuntime(tgt, paths.Deployments()) - opts := runtime.DeployOpts{ - Name: nodeName, - ConfigData: []byte(hocon), - ComposeData: []byte(composeYAML), - } - if err := rt.Deploy(cmd.Context(), opts); err != nil { return output.NewError("DEPLOY_ERROR", output.ExitGeneralError, err.Error()) } - store.UpsertNode(deployState, state.ManagedNode{ - Name: nodeName, - Version: node.Version, - // Record the network kind (mainnet|nile|private) so the C1 - // is_private fact and the --require-private guard work for added - // nodes too — without this they'd read as legacy/unknown and a - // private-net node would be fail-safe-refused by every mutator. - Network: parsed.Network, - // Persist the FULL target so subsequent stop/start/files/inspect - // can rebuild the SSH connection. Earlier this only stored - // Type, leaving Host/User/Port/IdentityFile blank — which then - // silently fell through to LocalTarget on follow-up commands. - Target: state.NodeTarget{ - Type: parsed.Target.Type, - Host: parsed.Target.Host, - User: parsed.Target.User, - Port: parsed.Target.Port, - IdentityFile: parsed.Target.IdentityFile, - }, - Runtime: "docker", - Status: "running", - LastApplied: time.Now().UTC(), - HTTPPort: node.Ports.HTTP, - GRPCPort: node.Ports.GRPC, - P2PPort: node.Ports.P2P, - MetricsPort: node.Ports.Metrics, - InstallPath: node.InstallPath, - StorageRoot: applypkg.StorageRootForNode(node, "docker", paths.Deployments(), nodeName), - Labels: node.Labels, - }) - if err := store.Save(deployState); err != nil { - return output.NewError("STATE_ERROR", output.ExitGeneralError, - "failed to persist state: "+err.Error()) - } - // Reload monitoring stack to include the new node if already deployed. reloadNetworkMonitoring(cmd.Context(), addNetworkName, deployState, tgt) writeAudit(auditEvent{ @@ -330,7 +288,7 @@ func reloadNetworkMonitoring(ctx context.Context, networkName string, deployStat monCompose := filepath.Join(monDir, networkName+"-monitoring", "docker-compose.yaml") // Check if monitoring is deployed for this network. - if _, err := os.Stat(monCompose); err != nil { + if _, err := tgt.ReadFile(ctx, monCompose); err != nil { return } diff --git a/cmd/network/create.go b/cmd/network/create.go index 9f930ef4..e4d9bec8 100644 --- a/cmd/network/create.go +++ b/cmd/network/create.go @@ -27,6 +27,21 @@ var ( createMonitor bool ) +var createApply = apply.Apply +var createTarget = func(parsed *intent.Intent) (target.Target, error) { + switch parsed.Target.Type { + case "ssh": + t := target.NewSSHTarget(parsed.Target.Host, parsed.Target.Port, parsed.Target.User, parsed.Target.IdentityFile) + if err := t.Connect(); err != nil { + return nil, err + } + return t, nil + default: + return target.NewLocalTarget(), nil + } +} +var createMonitoring = deployNetworkMonitoring + var createCmd = &cobra.Command{ Use: "create", Short: "Create a private network from an intent file", @@ -68,19 +83,12 @@ func runCreate(cmd *cobra.Command, args []string) error { intent.ApplyMonitoringDefaults(parsed.Monitoring) } - // Resolve target - var tgt target.Target - switch parsed.Target.Type { - case "ssh": - t := target.NewSSHTarget(parsed.Target.Host, parsed.Target.Port, parsed.Target.User, parsed.Target.IdentityFile) - if err := t.Connect(); err != nil { - return output.NewError("TARGET_UNREACHABLE", output.ExitTargetUnreachable, err.Error()) - } - defer t.Close() - tgt = t - default: - tgt = target.NewLocalTarget() + // Resolve target. + tgt, err := createTarget(parsed) + if err != nil { + return output.NewError("TARGET_UNREACHABLE", output.ExitTargetUnreachable, err.Error()) } + defer closeTarget(tgt) templateDir := findTemplatesDir() workDir := paths.Deployments() @@ -151,7 +159,7 @@ func runCreate(cmd *cobra.Command, args []string) error { if err != nil { return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) } - res, err := apply.Apply(cmd.Context(), apply.Options{ + res, err := createApply(cmd.Context(), apply.Options{ Intent: sub, Target: tgt, Store: store, @@ -199,6 +207,14 @@ func runCreate(cmd *cobra.Command, args []string) error { "network": parsed.Name, "nodes": deployed, } + failed := countFailedDeployments(deployed) + if failed > 0 { + result["status"] = "failed" + output.WriteJSON(os.Stdout, result) + return output.NewError("DEPLOY_ERROR", output.ExitGeneralError, + fmt.Sprintf("%d node(s) failed to deploy", failed)) + } + result["status"] = "success" // Deploy monitoring stack only when --monitor flag is explicitly passed. if createMonitor { @@ -208,7 +224,7 @@ func runCreate(cmd *cobra.Command, args []string) error { parsed.Monitoring.Enabled = intent.BoolPtr(true) intent.ApplyMonitoringDefaults(parsed.Monitoring) - monResult := deployNetworkMonitoring(cmd.Context(), tgt, workDir, parsed) + monResult := createMonitoring(cmd.Context(), tgt, workDir, parsed) if monResult.error != "" { result["monitoring_error"] = monResult.error } else { @@ -232,6 +248,16 @@ func runCreate(cmd *cobra.Command, args []string) error { return nil } +func countFailedDeployments(nodes []map[string]any) int { + failed := 0 + for _, node := range nodes { + if node["status"] == "error" { + failed++ + } + } + return failed +} + // autoWireActivePeers fills each node's network_overrides.active_peers // with the addresses of all OTHER nodes in the network. We only touch // nodes whose active_peers is unset; the user can opt out by supplying diff --git a/cmd/network/create_test.go b/cmd/network/create_test.go new file mode 100644 index 00000000..2f3c4b9e --- /dev/null +++ b/cmd/network/create_test.go @@ -0,0 +1,82 @@ +package network + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/apply" + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +func TestCountFailedDeployments(t *testing.T) { + nodes := []map[string]any{ + {"name": "net-node0", "status": "running"}, + {"name": "net-node1", "status": "error"}, + {"name": "net-node2", "status": "error"}, + } + if got := countFailedDeployments(nodes); got != 2 { + t.Fatalf("failed deployments = %d, want 2", got) + } +} + +func TestRunCreatePartialFailureContract(t *testing.T) { + oldApply, oldTarget, oldMonitoring := createApply, createTarget, createMonitoring + defer func() { createApply, createTarget, createMonitoring = oldApply, oldTarget, oldMonitoring }() + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + intentPath := filepath.Join(t.TempDir(), "network.yaml") + if err := os.WriteFile(intentPath, []byte("name: net\nnetwork: private\ntarget:\n type: local\nnodes:\n - type: fullnode\n - type: fullnode\n"), 0600); err != nil { + t.Fatal(err) + } + oldPath, oldMonitor := createIntentPath, createMonitor + createIntentPath, createMonitor = intentPath, true + t.Cleanup(func() { createIntentPath, createMonitor = oldPath, oldMonitor }) + fake := &createFakeTarget{LocalTarget: target.NewLocalTarget()} + createTarget = func(*intent.Intent) (target.Target, error) { return fake, nil } + var calls int + createApply = func(_ context.Context, opts apply.Options) (*apply.Result, error) { + calls++ + if calls == 2 { + return nil, errors.New("node failure") + } + return &apply.Result{Name: opts.Intent.Name, Outcome: "created"}, nil + } + monitorCalled := false + createMonitoring = func(context.Context, target.Target, string, *intent.Intent) monitoringResult { + monitorCalled = true + return monitoringResult{} + } + cmd := newNetCmd() + oldOut := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stdout = w + gotErr := runCreate(cmd, nil) + _ = w.Close() + os.Stdout = oldOut + body, _ := io.ReadAll(r) + var structured *output.StructuredError + if gotErr == nil || !errors.As(gotErr, &structured) || structured.Code != "DEPLOY_ERROR" || structured.ExitCode == 0 { + t.Fatalf("error = %v", gotErr) + } + if !strings.Contains(string(body), `"status": "failed"`) || !strings.Contains(string(body), "net-node0") || !strings.Contains(string(body), "net-node1") { + t.Fatalf("output = %s", body) + } + if monitorCalled { + t.Fatal("monitoring started after partial node failure") + } +} + +type createFakeTarget struct{ *target.LocalTarget } + +func (*createFakeTarget) Exec(context.Context, string, ...string) ([]byte, error) { return nil, nil } diff --git a/cmd/network/destroy.go b/cmd/network/destroy.go index 6722823f..b09a2947 100644 --- a/cmd/network/destroy.go +++ b/cmd/network/destroy.go @@ -89,14 +89,39 @@ func runDestroy(cmd *cobra.Command, args []string) error { return err } - workDir := paths.Deployments() - type failure struct { Name string `json:"name"` Error string `json:"error"` } + // Keep the recorded target for network-level cleanup before removing the + // nodes from state. Never fall back to the operator's local Docker daemon. + // Network create/add allow mixed targets. Keep one target per distinct + // target identity and run network-level cleanup on every one; this avoids + // assuming the first node represents the whole network. + networkTargets := make(map[string]target.Target) + var targetFailures []failure + failedTargetKeys := make(map[string]bool) + for _, n := range deployState.Nodes { + if !(strings.HasPrefix(n.Name, prefix) || n.Name == destroyConfirm) { + continue + } + key := fmt.Sprintf("%s|%s|%d|%s|%s", n.Target.Type, n.Target.Host, n.Target.Port, n.Target.User, n.Target.IdentityFile) + if _, ok := networkTargets[key]; ok { + continue + } + tgt, terr := resolveTargetForNode(&n) + if terr != nil { + targetFailures = append(targetFailures, failure{Name: n.Name, Error: "target cleanup: " + terr.Error()}) + failedTargetKeys[key] = true + continue + } + networkTargets[key] = tgt + defer closeTarget(tgt) + } + var removed []string var failures []failure + failures = append(failures, targetFailures...) auditResult := "success" auditTarget := "" @@ -110,6 +135,10 @@ func runDestroy(cmd *cobra.Command, args []string) error { if !(strings.HasPrefix(n.Name, prefix) || n.Name == destroyConfirm) { continue } + key := fmt.Sprintf("%s|%s|%d|%s|%s", n.Target.Type, n.Target.Host, n.Target.Port, n.Target.User, n.Target.IdentityFile) + if failedTargetKeys[key] { + continue + } auditTarget = n.Target.Type // any matching node's type — they're all the same in practice tgt, terr := resolveTargetForNode(&n) @@ -117,7 +146,7 @@ func runDestroy(cmd *cobra.Command, args []string) error { failures = append(failures, failure{Name: n.Name, Error: terr.Error()}) continue } - rt := runtime.NewDockerRuntime(tgt, workDir) + rt := resolveRuntimeForNode(&n, tgt) if rerr := rt.Remove(cmd.Context(), n.Name, true); rerr != nil { failures = append(failures, failure{Name: n.Name, Error: rerr.Error()}) closeTarget(tgt) @@ -135,10 +164,12 @@ func runDestroy(cmd *cobra.Command, args []string) error { "failed to persist state after destroy: "+err.Error()) } - // Remove monitoring stack if it exists (best-effort). - monRT := runtime.NewMonitoringRuntime(target.NewLocalTarget(), paths.Deployments()) - if err := monRT.Remove(cmd.Context(), destroyConfirm, true); err != nil { - fmt.Fprintf(cmd.ErrOrStderr(), "Warning: failed to remove monitoring stack: %v\n", err) + // Remove monitoring stack if it exists (best-effort) on the recorded target. + for _, networkTarget := range networkTargets { + monRT := runtime.NewMonitoringRuntime(networkTarget, paths.Deployments()) + if err := monRT.Remove(cmd.Context(), destroyConfirm, true); err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: failed to remove monitoring stack: %v\n", err) + } } // Tear down the shared docker network the matching `network create` @@ -146,23 +177,9 @@ func runDestroy(cmd *cobra.Command, args []string) error { // run (create re-uses it), so we ignore failures rather than // surfacing them as the destroy result. if len(removed) > 0 { - // Resolve a target for the network rm. We just used one for - // each node above; either is fine — re-pick from the first - // node we just removed. - for _, n := range deployState.Nodes { - if n.Name == removed[0] { - if tgt, terr := resolveTargetForNode(&n); terr == nil { - _, _ = tgt.Exec(cmd.Context(), "docker", "network", "rm", "trond-"+destroyConfirm) - closeTarget(tgt) - } - break - } + for _, networkTarget := range networkTargets { + _, _ = networkTarget.Exec(cmd.Context(), "docker", "network", "rm", "trond-"+destroyConfirm) } - // We also try a local-target rm even if no surviving node remained - // in state (the loop above iterates the post-RemoveNode state), - // since the most common case is "all nodes removed, no entries - // left." Falls back silently. - _, _ = target.NewLocalTarget().Exec(cmd.Context(), "docker", "network", "rm", "trond-"+destroyConfirm) } if len(failures) > 0 { @@ -208,6 +225,17 @@ func resolveTargetForNode(n *state.ManagedNode) (target.Target, error) { } } +func resolveRuntimeForNode(n *state.ManagedNode, tgt target.Target) runtime.Runtime { + if n.Runtime == "jar" { + jr := runtime.NewJarRuntime(tgt) + if n.InstallPath != "" { + jr.SetPurgeInstallPath(n.InstallPath) + } + return jr + } + return runtime.NewDockerRuntime(tgt, paths.Deployments()) +} + func closeTarget(t target.Target) { if c, ok := any(t).(interface{ Close() error }); ok { _ = c.Close() diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index 505902d9..bb6d7ac2 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -1,7 +1,9 @@ package network import ( + "bytes" "context" + "encoding/json" "fmt" "os" "os/exec" @@ -69,6 +71,8 @@ var ( upgradeWitnessFirst bool upgradeVerifyTimeout time.Duration upgradeIntentPath string + upgradeJarURL string + upgradeJarSHA256 string ) func init() { @@ -82,6 +86,8 @@ func init() { "Per-node verify timeout passed through to `trond verify`") upgradeCmd.Flags().StringVar(&upgradeIntentPath, "intent", "", "Path to the original intent.yaml used by `network create` (required for verify)") + upgradeCmd.Flags().StringVar(&upgradeJarURL, "jar-url", "", "Target-version JAR URL (jar runtime)") + upgradeCmd.Flags().StringVar(&upgradeJarSHA256, "jar-sha256", "", "Optional target-version JAR SHA256 (jar runtime)") if err := upgradeCmd.MarkFlagRequired("version"); err != nil { panic(err) } @@ -154,7 +160,14 @@ func runUpgrade(cmd *cobra.Command, args []string) error { for _, node := range order { stepStart := time.Now() - err := runChild(cmd.Context(), exe, "upgrade", node, "--version", upgradeVersion, "--auto-approve") + upgradeArgs := []string{"upgrade", node, "--version", upgradeVersion} + if upgradeJarURL != "" { + upgradeArgs = append(upgradeArgs, "--jar-url", upgradeJarURL) + } + if upgradeJarSHA256 != "" { + upgradeArgs = append(upgradeArgs, "--jar-sha256", upgradeJarSHA256) + } + err := runChild(cmd.Context(), exe, upgradeArgs...) steps = append(steps, upgradeStep{ Node: node, Phase: "upgrade", @@ -242,9 +255,58 @@ var runChild = runChildCommand func runChildCommand(ctx context.Context, exe string, argv ...string) error { argv = append(argv, "--output", "json") cmd := exec.CommandContext(ctx, exe, argv...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + var stdout, stderr limitedBuffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if stderr.Len() > 0 { + suffix := "" + if stderr.limited || stderr.total > 1<<20 || stderr.Len() >= 1<<20 { + suffix = " [truncated at 1MiB]" + } + text := stderr.String() + if (stderr.limited || stderr.total > 1<<20 || stderr.Len() >= 1<<20) && len(text) > 256 { + text = text[:256] + } + return fmt.Errorf("child command failed: %w: %s%s", err, text, suffix) + } + return err + } + var value any + if err := json.Unmarshal(stdout.Bytes(), &value); err != nil { + return fmt.Errorf("child command returned invalid JSON: %w", err) + } + if _, ok := value.(map[string]any); !ok { + return fmt.Errorf("child command returned JSON that is not an object") + } + if stdout.limited || stderr.limited { + return fmt.Errorf("child command output exceeds 1MiB limit") + } + return nil +} + +type limitedBuffer struct { + bytes.Buffer + limited bool + total int +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + const max = 1 << 20 + b.total += len(p) + if b.total > max { + b.limited = true + } + if b.Len() >= max { + b.limited = true + return len(p), nil + } + if len(p) > max-b.Len() { + _, _ = b.Buffer.Write(p[:max-b.Len()]) + b.limited = true + return len(p), nil + } + return b.Buffer.Write(p) } // verifyNode projects the multi-node intent to the node being upgraded before diff --git a/cmd/network/upgrade_test.go b/cmd/network/upgrade_test.go index d4677554..d1154af8 100644 --- a/cmd/network/upgrade_test.go +++ b/cmd/network/upgrade_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -122,3 +123,54 @@ func TestFinishWithFailureRollsBackNodeWhoseVerifyFailed(t *testing.T) { t.Fatalf("rollback nodes = %v, want [net-node0]", rollbackNodes) } } + +func TestRunChildCommandRejectsNonJSONOutput(t *testing.T) { + err := runChildCommand(context.Background(), "/bin/sh", "-c", "printf not-json") + if err == nil || !strings.Contains(err.Error(), "invalid JSON") { + t.Fatalf("error = %v, want invalid JSON", err) + } +} + +func TestRunChildCommandRejectsNonObjectJSON(t *testing.T) { + for _, value := range []string{"null", "1", "[]"} { + t.Run(value, func(t *testing.T) { + err := runChildCommand(context.Background(), "/bin/sh", "-c", "printf '%s' '"+value+"'") + if err == nil || !strings.Contains(err.Error(), "not an object") { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestRunChildCommandMarksTruncatedStderr(t *testing.T) { + err := runChildCommand(context.Background(), "python3", "-c", "import sys; sys.stderr.write('x'*1100001); sys.exit(1)") + if err == nil || !strings.Contains(err.Error(), "[truncated at 1MiB]") { + t.Fatalf("error = %v", err) + } +} + +func TestLimitedBufferTracksOverflow(t *testing.T) { + var b limitedBuffer + _, _ = b.Write(make([]byte, (1<<20)+1)) + if !b.limited || b.total <= 1<<20 || b.Len() != 1<<20 { + t.Fatalf("buffer state limited=%v total=%d len=%d", b.limited, b.total, b.Len()) + } +} + +func TestFinishWithFailureDoesNotPassUpgradeJarArgumentsToRollback(t *testing.T) { + oldAuto, oldURL, oldSHA, oldRun := upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256, runChild + defer func() { + upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256, runChild = oldAuto, oldURL, oldSHA, oldRun + }() + upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256 = true, "https://example/jar", "abc" + var got []string + runChild = func(_ context.Context, _ string, argv ...string) error { + got = append([]string(nil), argv...) + return nil + } + _ = finishWithFailure(context.Background(), "", "net", "trond", nil, []string{"net-node0"}, time.Now(), "net-node1", errors.New("failed")) + want := []string{"rollback", "net-node0"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("rollback args = %v, want %v", got, want) + } +} diff --git a/internal/state/types.go b/internal/state/types.go index 81409447..550b0705 100644 --- a/internal/state/types.go +++ b/internal/state/types.go @@ -30,8 +30,11 @@ type ManagedNode struct { // so probe commands (health, diagnose, verify) can target the right port // without re-reading the intent file. Older state files predate these // fields — callers must fall back to defaults when zero. - HTTPPort int `json:"http_port,omitempty"` - GRPCPort int `json:"grpc_port,omitempty"` + HTTPPort int `json:"http_port,omitempty"` + GRPCPort int `json:"grpc_port,omitempty"` + SolidityHTTPPort int `json:"solidity_http_port,omitempty"` + SolidityGRPCPort int `json:"solidity_grpc_port,omitempty"` + JSONRPCPort int `json:"jsonrpc_port,omitempty"` // P2PPort is the listen.port a sibling can dial to peer with this node. // `network add` reads it from every existing entry to populate the new // node's active_peers so it can immediately join the P2P mesh. diff --git a/tools/txgen/broadcast.go b/tools/txgen/broadcast.go index bc025e08..866c8bb4 100644 --- a/tools/txgen/broadcast.go +++ b/tools/txgen/broadcast.go @@ -103,11 +103,25 @@ func runBroadcast(ctx context.Context, cfg *Config) error { wg.Add(1) go func(lane int) { defer wg.Done() - for item := range work { + for { + var item [2]string + var ok bool + select { + case item, ok = <-work: + case <-ctx.Done(): + return + } + if !ok { + return + } if ctx.Err() != nil { return } - <-tokens + select { + case <-tokens: + case <-ctx.Done(): + return + } ok, msg := tr.Broadcast(ctx, lane, []byte(item[1])) if ok { okCount.Add(1) @@ -133,7 +147,7 @@ func runBroadcast(ctx context.Context, cfg *Config) error { startTime := time.Now() total := 0 for _, path := range files { - n, err := streamCSV(path, work) + n, err := streamCSVContext(ctx, path, work) if err != nil { log.Printf("broadcast: read %s: %v", path, err) } @@ -176,6 +190,10 @@ func runBroadcast(ctx context.Context, cfg *Config) error { // streamCSV reads `path` (skipping the header) and pushes (txID, json) // pairs onto out. Returns the number of rows pushed. func streamCSV(path string, out chan<- [2]string) (int, error) { + return streamCSVContext(context.Background(), path, out) +} + +func streamCSVContext(ctx context.Context, path string, out chan<- [2]string) (int, error) { f, err := os.Open(path) if err != nil { return 0, err @@ -201,12 +219,21 @@ func streamCSV(path string, out chan<- [2]string) (int, error) { if len(rec) < 2 { continue } - out <- [2]string{rec[0], rec[1]} + select { + case out <- [2]string{rec[0], rec[1]}: + case <-ctx.Done(): + return count, ctx.Err() + } count++ } return count, nil } +func runCSVProducer(ctx context.Context, path string, out chan<- [2]string, done chan<- struct{}) { + defer close(done) + _, _ = streamCSVContext(ctx, path, out) +} + func truncate(s string, n int) string { if len(s) <= n { return s diff --git a/tools/txgen/broadcast_test.go b/tools/txgen/broadcast_test.go new file mode 100644 index 00000000..c03f42a8 --- /dev/null +++ b/tools/txgen/broadcast_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestStreamCSVContextStopsOnCancellation(t *testing.T) { + path := filepath.Join(t.TempDir(), "tx.csv") + if err := os.WriteFile(path, []byte("txid,json\na,{}\n"), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := streamCSVContext(ctx, path, make(chan [2]string)) + if err != context.Canceled { + t.Fatalf("error = %v, want context.Canceled", err) + } +} + +func TestCSVProducerRunningCancellationConverges(t *testing.T) { + path := filepath.Join(t.TempDir(), "tx.csv") + var b strings.Builder + b.WriteString("txid,json\n") + for i := 0; i < 2000; i++ { + b.WriteString("id,{}\n") + } + if err := os.WriteFile(path, []byte(b.String()), 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + work := make(chan [2]string, 1) + work <- [2]string{"prefill", "{}"} + var wg sync.WaitGroup + wg.Add(1) + go func() { defer wg.Done(); _, _ = streamCSVContext(ctx, path, work) }() + // The producer has a full output channel and is blocked in its send. + time.Sleep(10 * time.Millisecond) + cancel() + finished := make(chan struct{}) + go func() { wg.Wait(); close(finished) }() + select { + case <-finished: + case <-time.After(time.Second): + t.Fatal("producer did not converge after cancellation") + } +} From 8bdb536813f314557ad278f6445fc5eca55576cd Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 20 Aug 2026 18:15:31 +0800 Subject: [PATCH 22/49] fix(mcp): align plan/apply with CLI contract and converge shared logic AUD-039/040/041/044 plus duplication cleanup: - internal/apply: EffectiveIntentHash, LegacyIntentHashMatches, RestoreAutoPorts, Plan, FindTemplatesDir become the single source for intent hash + plan decisions (CLI plan/apply and MCP plan/apply all call them; local algorithm copies removed) - MCP plan returns the promised contract fields with the three-state current_state domain and config/version downtime rules - MCP apply preserves the recorded legacy hash after bypassing HUMAN_REQUIRED and passes the resolved template dir to both Plan and Apply - heal routes by recorded node runtime (docker vs jar) and diagnose/heal checkers receive the recorded network - snapshot download and render tools return VALIDATION_ERROR envelopes for missing dest / negative node index - tests exercise real handlers (no self-asserting tautologies) --- cmd/apply.go | 87 +----- cmd/apply_hash_test.go | 24 +- cmd/plan.go | 103 ++----- internal/apply/apply.go | 13 +- internal/apply/hash.go | 77 +++++ internal/apply/plan.go | 89 ++++++ internal/apply/plan_test.go | 71 +++++ internal/mcp/contract_p1_test.go | 367 ++++++++++++++++++++++++ internal/mcp/schema_consistency_test.go | 12 +- internal/mcp/tools_config.go | 4 + internal/mcp/tools_diagnostic.go | 5 +- internal/mcp/tools_heal.go | 13 +- internal/mcp/tools_lifecycle.go | 83 ++++-- internal/mcp/tools_snapshot.go | 8 +- 14 files changed, 739 insertions(+), 217 deletions(-) create mode 100644 internal/apply/hash.go create mode 100644 internal/apply/plan.go create mode 100644 internal/apply/plan_test.go create mode 100644 internal/mcp/contract_p1_test.go diff --git a/cmd/apply.go b/cmd/apply.go index 1f180e31..fb668180 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -1,7 +1,6 @@ package cmd import ( - "crypto/sha256" "errors" "fmt" "os" @@ -132,15 +131,15 @@ func runApply(cmd *cobra.Command, args []string) error { existing := store.GetNode(deployState, parsed.Name) if parsed.Target.AutoPorts && existing != nil { if len(parsed.Nodes) == 1 { - restoreAutoPorts(&parsed.Nodes[0], existing) + apply.RestoreAutoPorts(&parsed.Nodes[0], existing) } } canonicalIntent, err := yaml.Marshal(parsed) if err != nil { return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "marshal effective intent: "+err.Error()) } - intentHash := versionedIntentHash(canonicalIntent) - legacyMatch := existing != nil && legacyIntentHashMatches(existing.IntentHash, rawIntent, parsed, existing) + intentHash := apply.EffectiveIntentHash(canonicalIntent) + legacyMatch := existing != nil && apply.LegacyIntentHashMatches(rawIntent, canonicalIntent, existing) if legacyMatch { // Preserve the recorded legacy hash for a true no-op. The next // intentional change will write the versioned hash during apply. @@ -165,7 +164,7 @@ func runApply(cmd *cobra.Command, args []string) error { State: deployState, IntentHash: intentHash, Existing: existing, - TemplateDir: findTemplatesDir(), + TemplateDir: apply.FindTemplatesDir(), DeploymentsDir: deploymentsDir(), EnvVars: resolveEnvVars(&parsed.Nodes[0]), IntentPath: applyIntentPath, // FR-021: relative build.source resolves vs this @@ -216,67 +215,6 @@ func runApply(cmd *cobra.Command, args []string) error { return nil } -func versionedIntentHash(data []byte) string { - h := sha256.New() - h.Write([]byte("intent-hash-v2\x00")) - h.Write(data) - return fmt.Sprintf("v2:%x", h.Sum(nil)) -} - -func legacyIntentHashMatches(stored string, raw []byte, parsed *intent.Intent, existing *state.ManagedNode) bool { - if len(stored) == 0 || len(stored) > 3 && stored[:3] == "v2:" { - return false - } - if apply.IntentHashFromBytes(raw) == stored { - var rawIntent intent.Intent - if err := yaml.Unmarshal(raw, &rawIntent); err != nil { - return false - } - if monitoringEnabled(rawIntent.Monitoring) != monitoringEnabled(parsed.Monitoring) { - return false - } - if monitoringEnabled(parsed.Monitoring) != recordedMonitoringEnabled(existing) { - return false - } - return true - } - canonical, err := yaml.Marshal(parsed) - return err == nil && apply.IntentHashFromBytes(canonical) == stored && - monitoringEnabled(parsed.Monitoring) == recordedMonitoringEnabled(existing) -} - -func monitoringEnabled(m *intent.Monitoring) bool { - return m != nil && m.Enabled != nil && *m.Enabled -} - -func recordedMonitoringEnabled(existing *state.ManagedNode) bool { - return existing != nil && existing.Monitoring != nil && existing.Monitoring.Enabled -} - -func restoreAutoPorts(node *intent.NodeSpec, existing *state.ManagedNode) { - if existing.HTTPPort != 0 { - node.Ports.HTTP = existing.HTTPPort - } - if existing.GRPCPort != 0 { - node.Ports.GRPC = existing.GRPCPort - } - if existing.P2PPort != 0 { - node.Ports.P2P = existing.P2PPort - } - if existing.MetricsPort != 0 { - node.Ports.Metrics = existing.MetricsPort - } - if existing.SolidityHTTPPort != 0 { - node.Ports.SolidityHTTP = existing.SolidityHTTPPort - } - if existing.SolidityGRPCPort != 0 { - node.Ports.SolidityGRPC = existing.SolidityGRPCPort - } - if existing.JSONRPCPort != 0 { - node.Ports.JSONRPC = existing.JSONRPCPort - } -} - func resolveTarget(parsed *intent.Intent) (target.Target, error) { switch parsed.Target.Type { case "ssh": @@ -301,22 +239,7 @@ func resolveEnvVars(node *intent.NodeSpec) map[string]string { return env } -// findTemplatesDir returns an optional on-disk templates directory. -// Empty return signals render to use the embedded copy. -func findTemplatesDir() string { - if d := os.Getenv("TROND_TEMPLATES_DIR"); d != "" { - return d - } - candidates := []string{"templates", "./templates"} - for _, c := range candidates { - if info, err := os.Stat(c); err == nil && info.IsDir() { - if _, err := os.Stat(c + "/main_net_config.conf"); err == nil { - return c - } - } - } - return "" -} +func findTemplatesDir() string { return apply.FindTemplatesDir() } // exitWithError returns a StructuredError for propagation through cobra RunE. func exitWithError(code string, exitCode int, msg string, suggestions ...string) error { diff --git a/cmd/apply_hash_test.go b/cmd/apply_hash_test.go index 32501c1a..92d26d0e 100644 --- a/cmd/apply_hash_test.go +++ b/cmd/apply_hash_test.go @@ -12,17 +12,18 @@ import ( func TestLegacyIntentHashMatchesRawHash(t *testing.T) { raw := []byte("name: n\nnetwork: private\n") parsed := &intent.Intent{Name: "n", Network: "private"} - if !legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, nil) { + canonical, _ := yaml.Marshal(parsed) + if !apply.LegacyIntentHashMatches(raw, canonical, &state.ManagedNode{IntentHash: apply.IntentHashFromBytes(raw)}) { t.Fatal("legacy raw hash not accepted") } } func TestVersionedIntentHashIsStable(t *testing.T) { data := []byte("effective intent") - if versionedIntentHash(data) != versionedIntentHash(data) { + if apply.EffectiveIntentHash(data) != apply.EffectiveIntentHash(data) { t.Fatal("versioned hash is unstable") } - if versionedIntentHash(data) == string(data) { + if apply.EffectiveIntentHash(data) == string(data) { t.Fatal("hash did not version/encode input") } } @@ -30,7 +31,8 @@ func TestVersionedIntentHashIsStable(t *testing.T) { func TestLegacyRawHashRejectsMonitoringChange(t *testing.T) { raw := []byte("name: n\nnetwork: private\nmonitoring:\n enabled: false\n") parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(true)}} - if legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, nil) { + canonical, _ := yaml.Marshal(parsed) + if apply.LegacyIntentHashMatches(raw, canonical, &state.ManagedNode{IntentHash: apply.IntentHashFromBytes(raw)}) { t.Fatal("monitoring change treated as legacy no-op") } } @@ -38,7 +40,8 @@ func TestLegacyRawHashRejectsMonitoringChange(t *testing.T) { func TestLegacyRawHashRejectsMissingRecordedMonitoring(t *testing.T) { raw := []byte("name: n\nnetwork: private\nmonitoring:\n enabled: true\n") parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(true)}} - if legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, &state.ManagedNode{}) { + canonical, _ := yaml.Marshal(parsed) + if apply.LegacyIntentHashMatches(raw, canonical, &state.ManagedNode{IntentHash: apply.IntentHashFromBytes(raw)}) { t.Fatal("missing recorded monitoring treated as legacy no-op") } } @@ -47,7 +50,9 @@ func TestLegacyRawHashRejectsRecordedMonitoringWhenCurrentDisabled(t *testing.T) raw := []byte("name: n\nnetwork: private\nmonitoring:\n enabled: false\n") parsed := &intent.Intent{Name: "n", Network: "private", Monitoring: &intent.Monitoring{Enabled: intent.BoolPtr(false)}} existing := &state.ManagedNode{Monitoring: &state.MonitoringState{Enabled: true}} - if legacyIntentHashMatches(apply.IntentHashFromBytes(raw), raw, parsed, existing) { + canonical, _ := yaml.Marshal(parsed) + existing.IntentHash = apply.IntentHashFromBytes(raw) + if apply.LegacyIntentHashMatches(raw, canonical, existing) { t.Fatal("recorded enabled monitoring treated as legacy no-op when current intent is disabled") } } @@ -59,7 +64,8 @@ func TestLegacyCanonicalHashRejectsRecordedMonitoringWhenCurrentDisabled(t *test t.Fatal(err) } existing := &state.ManagedNode{Monitoring: &state.MonitoringState{Enabled: true}} - if legacyIntentHashMatches(apply.IntentHashFromBytes(canonical), []byte("not-the-canonical-input"), parsed, existing) { + existing.IntentHash = apply.IntentHashFromBytes(canonical) + if apply.LegacyIntentHashMatches([]byte("not-the-canonical-input"), canonical, existing) { t.Fatal("canonical legacy hash treated recorded enabled monitoring as no-op when current intent is disabled") } } @@ -67,7 +73,7 @@ func TestLegacyCanonicalHashRejectsRecordedMonitoringWhenCurrentDisabled(t *test func TestRestoreAutoPortsPreservesMissingLegacyPorts(t *testing.T) { n := &intent.NodeSpec{} n.Ports.HTTP, n.Ports.GRPC, n.Ports.SolidityHTTP, n.Ports.SolidityGRPC, n.Ports.JSONRPC, n.Ports.P2P, n.Ports.Metrics = 10, 11, 12, 13, 14, 15, 16 - restoreAutoPorts(n, &state.ManagedNode{HTTPPort: 1, GRPCPort: 2, P2PPort: 6, MetricsPort: 7}) + apply.RestoreAutoPorts(n, &state.ManagedNode{HTTPPort: 1, GRPCPort: 2, P2PPort: 6, MetricsPort: 7}) got := []int{n.Ports.HTTP, n.Ports.GRPC, n.Ports.SolidityHTTP, n.Ports.SolidityGRPC, n.Ports.JSONRPC, n.Ports.P2P, n.Ports.Metrics} want := []int{1, 2, 12, 13, 14, 6, 7} for i := range got { @@ -80,7 +86,7 @@ func TestRestoreAutoPortsPreservesMissingLegacyPorts(t *testing.T) { func TestRestoreAutoPortsAllSeven(t *testing.T) { n := &intent.NodeSpec{} s := &state.ManagedNode{HTTPPort: 1, GRPCPort: 2, SolidityHTTPPort: 3, SolidityGRPCPort: 4, JSONRPCPort: 5, P2PPort: 6, MetricsPort: 7} - restoreAutoPorts(n, s) + apply.RestoreAutoPorts(n, s) got := []int{n.Ports.HTTP, n.Ports.GRPC, n.Ports.SolidityHTTP, n.Ports.SolidityGRPC, n.Ports.JSONRPC, n.Ports.P2P, n.Ports.Metrics} want := []int{1, 2, 3, 4, 5, 6, 7} for i := range got { diff --git a/cmd/plan.go b/cmd/plan.go index ceebf903..72e80d9c 100644 --- a/cmd/plan.go +++ b/cmd/plan.go @@ -42,13 +42,7 @@ func init() { rootCmd.AddCommand(planCmd) } -type planChange struct { - Type string `json:"type"` - Field string `json:"field"` - From any `json:"from,omitempty"` - To any `json:"to,omitempty"` - RestartRequired bool `json:"restart_required"` -} +type planChange = apply.PlanChange func runPlan(cmd *cobra.Command, args []string) error { outputFmt, _ := cmd.Flags().GetString("output") @@ -59,11 +53,7 @@ func runPlan(cmd *cobra.Command, args []string) error { return exitWithError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) } - // 2. Compute intent hash - intentData, _ := os.ReadFile(planIntentPath) - intentHash := apply.IntentHashFromBytes(intentData) - - // 3. Load current state + // 2. Load current state store, err := state.NewStore(statePath()) if err != nil { return exitWithError("STATE_ERROR", output.ExitGeneralError, err.Error()) @@ -75,85 +65,26 @@ func runPlan(cmd *cobra.Command, args []string) error { } existing := store.GetNode(deployState, parsed.Name) - - // 4. Render config to compute config hash - templateDir := findTemplatesDir() - node := &parsed.Nodes[0] - - rendered, err := render.RenderHOCONWithSecrets(templateDir, parsed, node) - if err != nil { - return exitWithError("RENDER_ERROR", output.ExitGeneralError, err.Error()) + if parsed.Target.AutoPorts && existing != nil { + apply.RestoreAutoPorts(&parsed.Nodes[0], existing) } - // The REAL bytes: config_hash must match what apply deploys, and the - // diff below must compare real values so a rotated witness key is - // still detected. Nothing from here reaches the output un-redacted — - // simpleHOCONDiff redacts every line at the point it emits it. - hoconConfig := rendered.Deployable() - configHash := apply.IntentHashFromBytes([]byte(hoconConfig)) - - // 5. Diff - var changes []planChange - destructive := false - downtime := 0 - - if existing == nil { - // New deployment - changes = append(changes, planChange{ - Type: "create", - Field: "node", - To: parsed.Name, - }) - } else { - // Check for changes - if existing.IntentHash != intentHash { - changes = append(changes, planChange{ - Type: "update", - Field: "intent", - From: existing.IntentHash[:12] + "...", - To: intentHash[:12] + "...", - RestartRequired: true, - }) - } - if existing.ConfigHash != configHash { - changes = append(changes, planChange{ - Type: "update", - Field: "config", - From: existing.ConfigHash[:12] + "...", - To: configHash[:12] + "...", - RestartRequired: true, - }) - downtime = 30 // Estimated restart time - } - if existing.Version != node.Version && node.Version != "latest" { - changes = append(changes, planChange{ - Type: "update", - Field: "version", - From: existing.Version, - To: node.Version, - RestartRequired: true, - }) - downtime = 60 - } - } - - currentState := "not deployed" - if existing != nil { - currentState = existing.Status + rawIntent, err := os.ReadFile(planIntentPath) + if err != nil { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) } - - runtimeType := parsed.Target.Runtime - if runtimeType == "" { - runtimeType = "docker" + planned, err := apply.Plan(parsed, existing, rawIntent, apply.FindTemplatesDir()) + if err != nil { + return err } result := map[string]any{ "name": parsed.Name, - "current_state": currentState, + "current_state": planned.CurrentState, "desired_state": "running", - "changes": changes, - "destructive": destructive, - "estimated_downtime_seconds": downtime, - "runtime": runtimeType, + "changes": planned.Changes, + "destructive": false, + "estimated_downtime_seconds": planned.Downtime, + "runtime": planned.Runtime, "network": parsed.Network, } @@ -167,7 +98,7 @@ func runPlan(cmd *cobra.Command, args []string) error { deployedPath := filepath.Join(paths.Deployments(), parsed.Name, parsed.Name+".conf") if data, err := os.ReadFile(deployedPath); err == nil { diffLines = simpleHOCONDiff(strings.Split(string(data), "\n"), - strings.Split(hoconConfig, "\n")) + strings.Split(string(planned.Config), "\n")) } // JSON consumers always get the field (possibly empty array) // so they can distinguish "no changes" from "diff was not @@ -178,7 +109,7 @@ func runPlan(cmd *cobra.Command, args []string) error { if outputFmt == "json" { output.WriteJSON(os.Stdout, result) } else { - printPlanText(result, changes) + printPlanText(result, planned.Changes) if planShowDiff { printDiffSection(existing, diffLines) } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index c84217c3..3ea65915 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -387,11 +387,14 @@ func Apply(ctx context.Context, opts Options) (*Result, error) { Port: opts.Intent.Target.Port, IdentityFile: opts.Intent.Target.IdentityFile, }, - Runtime: runtimeType, - Status: "running", - LastApplied: time.Now().UTC(), - HTTPPort: node.Ports.HTTP, - GRPCPort: node.Ports.GRPC, + Runtime: runtimeType, + Status: "running", + LastApplied: time.Now().UTC(), + HTTPPort: node.Ports.HTTP, + GRPCPort: node.Ports.GRPC, + SolidityHTTPPort: node.Ports.SolidityHTTP, + SolidityGRPCPort: node.Ports.SolidityGRPC, + JSONRPCPort: node.Ports.JSONRPC, // P2PPort is load-bearing, not cosmetic: `network add` builds the // joining node's peer list from the P2PPort of every node already // in state and SKIPS any entry where it is zero. Omitting it here diff --git a/internal/apply/hash.go b/internal/apply/hash.go new file mode 100644 index 00000000..480ed19d --- /dev/null +++ b/internal/apply/hash.go @@ -0,0 +1,77 @@ +package apply + +import ( + "crypto/sha256" + "fmt" + + "gopkg.in/yaml.v3" + + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +// EffectiveIntentHash hashes the fully-resolved intent used for deployment. +func EffectiveIntentHash(effective []byte) string { + h := sha256.New() + h.Write([]byte("intent-hash-v2\x00")) + h.Write(effective) + return fmt.Sprintf("v2:%x", h.Sum(nil)) +} + +// LegacyIntentHashMatches accepts hashes written before the versioned hash +// format, while ensuring monitoring state did not change underneath them. +func LegacyIntentHashMatches(raw, canonical []byte, existing *state.ManagedNode) bool { + if existing == nil { + return false + } + stored := existing.IntentHash + if stored == "" || len(stored) > 3 && stored[:3] == "v2:" { + return false + } + var rawIntent, canonicalIntent intent.Intent + if err := yaml.Unmarshal(raw, &rawIntent); err != nil { + return false + } + if err := yaml.Unmarshal(canonical, &canonicalIntent); err != nil { + return false + } + if IntentHashFromBytes(raw) == stored { + return monitoringEnabled(rawIntent.Monitoring) == monitoringEnabled(canonicalIntent.Monitoring) && + monitoringEnabled(canonicalIntent.Monitoring) == recordedMonitoringEnabled(existing) + } + return IntentHashFromBytes(canonical) == stored && + monitoringEnabled(canonicalIntent.Monitoring) == recordedMonitoringEnabled(existing) +} + +func monitoringEnabled(m *intent.Monitoring) bool { + return m != nil && m.Enabled != nil && *m.Enabled +} + +func recordedMonitoringEnabled(existing *state.ManagedNode) bool { + return existing != nil && existing.Monitoring != nil && existing.Monitoring.Enabled +} + +// RestoreAutoPorts restores all persisted auto-allocated ports into an intent. +func RestoreAutoPorts(node *intent.NodeSpec, existing *state.ManagedNode) { + if existing.HTTPPort != 0 { + node.Ports.HTTP = existing.HTTPPort + } + if existing.GRPCPort != 0 { + node.Ports.GRPC = existing.GRPCPort + } + if existing.P2PPort != 0 { + node.Ports.P2P = existing.P2PPort + } + if existing.MetricsPort != 0 { + node.Ports.Metrics = existing.MetricsPort + } + if existing.SolidityHTTPPort != 0 { + node.Ports.SolidityHTTP = existing.SolidityHTTPPort + } + if existing.SolidityGRPCPort != 0 { + node.Ports.SolidityGRPC = existing.SolidityGRPCPort + } + if existing.JSONRPCPort != 0 { + node.Ports.JSONRPC = existing.JSONRPCPort + } +} diff --git a/internal/apply/plan.go b/internal/apply/plan.go new file mode 100644 index 00000000..3a5abd87 --- /dev/null +++ b/internal/apply/plan.go @@ -0,0 +1,89 @@ +package apply + +import ( + "os" + + "gopkg.in/yaml.v3" + + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/render" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +type PlanChange struct { + Type string `json:"type"` + Field string `json:"field"` + From any `json:"from,omitempty"` + To any `json:"to,omitempty"` + RestartRequired bool `json:"restart_required"` +} + +type PlanResult struct { + CurrentState string + Changes []PlanChange + Downtime int + Runtime string + IntentHash string + ConfigHash string + Config []byte + LegacyMatch bool +} + +// Plan evaluates the same state transition used by both CLI and MCP plans. +func Plan(parsed *intent.Intent, existing *state.ManagedNode, raw []byte, templateDir string) (*PlanResult, error) { + canonical, err := yaml.Marshal(parsed) + if err != nil { + return nil, output.NewError("VALIDATION_ERROR", output.ExitValidationError, "marshal effective intent: "+err.Error()) + } + rendered, err := render.RenderHOCONWithSecrets(templateDir, parsed, &parsed.Nodes[0]) + if err != nil { + return nil, output.NewError("RENDER_ERROR", output.ExitGeneralError, err.Error()) + } + config := []byte(rendered.Deployable()) + result := &PlanResult{CurrentState: "not_deployed", Runtime: parsed.Target.Runtime, IntentHash: EffectiveIntentHash(canonical), ConfigHash: IntentHashFromBytes(config), Config: config} + if result.Runtime == "" { + result.Runtime = "docker" + } + if existing == nil { + result.Changes = []PlanChange{{Type: "create", Field: "node", To: parsed.Name}} + return result, nil + } + result.CurrentState = existing.Status + legacy := LegacyIntentHashMatches(raw, canonical, existing) + result.LegacyMatch = legacy + if existing.IntentHash != result.IntentHash && !legacy { + result.Changes = append(result.Changes, PlanChange{Type: "update", Field: "intent", From: shortHash(existing.IntentHash), To: shortHash(result.IntentHash), RestartRequired: true}) + } + if existing.ConfigHash != result.ConfigHash { + result.Changes = append(result.Changes, PlanChange{Type: "update", Field: "config", From: shortHash(existing.ConfigHash), To: shortHash(result.ConfigHash), RestartRequired: true}) + result.Downtime = 30 + } + if existing.Version != parsed.Nodes[0].Version && parsed.Nodes[0].Version != "latest" { + result.Changes = append(result.Changes, PlanChange{Type: "update", Field: "version", From: existing.Version, To: parsed.Nodes[0].Version, RestartRequired: true}) + result.Downtime = 60 + } + return result, nil +} + +func shortHash(value string) string { + if len(value) <= 12 { + return value + } + return value[:12] + "..." +} + +// FindTemplatesDir returns the optional on-disk template directory. +func FindTemplatesDir() string { + if d := os.Getenv("TROND_TEMPLATES_DIR"); d != "" { + return d + } + for _, c := range []string{"templates", "./templates"} { + if info, err := os.Stat(c); err == nil && info.IsDir() { + if _, err := os.Stat(c + "/main_net_config.conf"); err == nil { + return c + } + } + } + return "" +} diff --git a/internal/apply/plan_test.go b/internal/apply/plan_test.go new file mode 100644 index 00000000..19db9346 --- /dev/null +++ b/internal/apply/plan_test.go @@ -0,0 +1,71 @@ +package apply + +import ( + "errors" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +func planFixture() *intent.Intent { + return &intent.Intent{Name: "p", Network: "private", Target: intent.Target{Runtime: ""}, Nodes: []intent.NodeSpec{{Type: "fullnode", Version: "4.8.1", Resources: intent.Resources{Memory: "8GB"}}}} +} + +func TestPlanLegacyRawHashMatchesNoIntentChange(t *testing.T) { + i := planFixture() + raw := []byte("name: p\nnetwork: private\ntarget:\n type: local\nnodes:\n - type: fullnode\n version: 4.8.1\n resources:\n memory: 8GB\n") + existing := &state.ManagedNode{IntentHash: IntentHashFromBytes(raw), ConfigHash: "different", Version: "4.8.1", Status: "running"} + got, err := Plan(i, existing, raw, "") + if err != nil { + t.Fatal(err) + } + for _, c := range got.Changes { + if c.Field == "intent" { + t.Fatalf("legacy hash produced intent change: %+v", got.Changes) + } + } +} + +func TestPlanVersionOnlyChangeDowntimeIs60(t *testing.T) { + i := planFixture() + base, err := Plan(i, nil, nil, "") + if err != nil { + t.Fatal(err) + } + existing := &state.ManagedNode{IntentHash: base.IntentHash, ConfigHash: base.ConfigHash, Version: "4.8.0", Status: "running"} + got, err := Plan(i, existing, nil, "") + if err != nil { + t.Fatal(err) + } + if got.Downtime != 60 { + t.Fatalf("downtime=%d", got.Downtime) + } +} + +func TestPlanRenderFailureReturnsStructuredError(t *testing.T) { + i := planFixture() + i.Network = "not-a-network" + got, err := Plan(i, nil, nil, "") + if err == nil || got != nil { + t.Fatalf("got=%v err=%v", got, err) + } + var structured *output.StructuredError + if !errors.As(err, &structured) { + t.Fatalf("error type=%T, want *output.StructuredError", err) + } + if structured.Code != "RENDER_ERROR" { + t.Fatalf("error code=%q, want RENDER_ERROR", structured.Code) + } +} + +func TestPlanDefaultsRuntimeToDocker(t *testing.T) { + got, err := Plan(planFixture(), nil, nil, "") + if err != nil { + t.Fatal(err) + } + if got.Runtime != "docker" { + t.Fatalf("runtime=%q", got.Runtime) + } +} diff --git a/internal/mcp/contract_p1_test.go b/internal/mcp/contract_p1_test.go new file mode 100644 index 00000000..2836ef45 --- /dev/null +++ b/internal/mcp/contract_p1_test.go @@ -0,0 +1,367 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/tronprotocol/tron-deployment/internal/apply" + "github.com/tronprotocol/tron-deployment/internal/diagnosis" + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/render" + "github.com/tronprotocol/tron-deployment/internal/runtime" + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +func TestPlanToolReturnsPromisedContractFields(t *testing.T) { + p := t.TempDir() + "/intent.yaml" + if err := os.WriteFile(p, []byte("name: p1\nnetwork: nile\ntarget:\n type: local\n runtime: docker\nnodes:\n - type: fullnode\n version: 4.8.1\n resources:\n memory: 8GB\n"), 0600); err != nil { + t.Fatal(err) + } + _, value, err := planTool(context.Background(), nil, planArg{Path: p}) + if err != nil { + t.Fatal(err) + } + got := value.(map[string]any) + for _, key := range []string{"changes", "destructive", "estimated_downtime_seconds"} { + if _, ok := got[key]; !ok { + t.Fatalf("plan missing %q: %v", key, got) + } + } +} + +func TestPlanToolCreateNoOpUpdateStates(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + p := writePlanFixture(t, "plan-node") + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + parsed, err := intent.Load(p) + if err != nil { + t.Fatal(err) + } + rendered, err := render.RenderHOCONWithSecrets("", parsed, &parsed.Nodes[0]) + if err != nil { + t.Fatal(err) + } + configHash := apply.IntentHashFromBytes([]byte(rendered.Deployable())) + canonical, err := yaml.Marshal(parsed) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + node *state.ManagedNode + want int + }{ + {"create", nil, 1}, + {"noop", &state.ManagedNode{Name: "plan-node", IntentHash: apply.EffectiveIntentHash(canonical), ConfigHash: configHash, Version: "4.8.1", Status: "running"}, 0}, + {"update", &state.ManagedNode{Name: "plan-node", IntentHash: "different", ConfigHash: configHash, Version: "4.8.1", Status: "running"}, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + st := &state.DeploymentState{} + if tc.node != nil { + st.Nodes = []state.ManagedNode{*tc.node} + } + if err := store.Save(st); err != nil { + t.Fatal(err) + } + _, value, err := planTool(context.Background(), nil, planArg{Path: p}) + if err != nil { + t.Fatal(err) + } + got := value.(map[string]any) + changes := got["changes"].([]map[string]any) + if len(changes) != tc.want { + t.Fatalf("changes=%v, want %d", changes, tc.want) + } + downtime := got["estimated_downtime_seconds"].(int) + if tc.name == "update" && downtime != 0 { + t.Fatalf("downtime=%d inconsistent", downtime) + } + }) + } +} + +func TestPlanToolCurrentStateUsesThreeStateDomain(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + p := writePlanFixture(t, "state-domain") + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, status, want string }{{"not deployed", "", "not_deployed"}, {"running", "running", "running"}, {"error", "error", "error"}} { + t.Run(tc.name, func(t *testing.T) { + st := &state.DeploymentState{} + if tc.status != "" { + st.Nodes = []state.ManagedNode{{Name: "state-domain", Status: tc.status}} + } + if err := store.Save(st); err != nil { + t.Fatal(err) + } + _, value, err := planTool(context.Background(), nil, planArg{Path: p}) + if err != nil { + t.Fatal(err) + } + if got := value.(map[string]any)["current_state"]; got != tc.want { + t.Fatalf("current_state=%v, want %q", got, tc.want) + } + }) + } +} + +func TestApplyToolLegacyHashUsesRecordedHashForNoChange(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + binDir := t.TempDir() + dockerCalls := t.TempDir() + "/docker.calls" + fakeDocker := binDir + "/docker" + if err := os.WriteFile(fakeDocker, []byte("#!/bin/sh\nprintf x >> \"$DOCKER_CALLS\"\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("DOCKER_CALLS", dockerCalls) + raw := []byte("name: legacy-apply\nnetwork: private\ntarget:\n type: local\n runtime: docker\nnodes:\n - type: fullnode\n version: 4.8.1\n resources:\n memory: 8GB\n") + p := t.TempDir() + "/intent.yaml" + if err := os.WriteFile(p, raw, 0600); err != nil { + t.Fatal(err) + } + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "legacy-apply", Status: "running", Network: "private", Runtime: "docker", + IntentHash: apply.IntentHashFromBytes(raw), + }}}); err != nil { + t.Fatal(err) + } + res, value, err := applyTool(context.Background(), nil, applyArgs{Path: p}) + if err != nil || res == nil || res.IsError { + t.Fatalf("apply result=%v err=%v", res, err) + } + got, ok := value.(*apply.Result) + if !ok { + t.Fatalf("result type=%T, want *apply.Result", value) + } + if got.Outcome != "no_change" { + t.Fatalf("outcome=%q, want no_change", got.Outcome) + } + if got.IntentHash != apply.IntentHashFromBytes(raw) { + t.Fatalf("intent hash=%q, want recorded legacy hash", got.IntentHash) + } + if data, readErr := os.ReadFile(dockerCalls); readErr == nil && len(data) != 0 { + t.Fatalf("fake runtime Deploy invoked %d time(s), want 0", len(data)) + } +} + +func TestApplyToolUsesConfiguredTemplateForPlanAndDeploy(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + templateDir := t.TempDir() + t.Setenv("TROND_TEMPLATES_DIR", templateDir) + // The template loader requires the matching private-network file. The + // marker makes the rendered/deployed config observably external-template based. + marker := "# external-template-marker" + template := marker + "\nnode {\n metrics {\n prometheus = false\n }\n}\n" + if err := os.WriteFile(templateDir+"/private_net_config.conf", []byte(template), 0600); err != nil { + t.Fatal(err) + } + binDir := t.TempDir() + dockerCalls := t.TempDir() + "/docker.calls" + if err := os.WriteFile(binDir+"/docker", []byte("#!/bin/sh\nprintf x >> \"$DOCKER_CALLS\"\n"), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("DOCKER_CALLS", dockerCalls) + raw := []byte("name: external-template\nnetwork: private\ntarget:\n type: local\n runtime: docker\nnodes:\n - type: fullnode\n version: 4.8.1\n resources:\n memory: 8GB\n") + p := t.TempDir() + "/intent.yaml" + if err := os.WriteFile(p, raw, 0600); err != nil { + t.Fatal(err) + } + res, value, err := applyTool(context.Background(), nil, applyArgs{Path: p, AutoApprove: true}) + if err != nil || res == nil || res.IsError { + t.Fatalf("apply result=%v err=%v", res, err) + } + got, ok := value.(*apply.Result) + if !ok { + t.Fatalf("result type=%T", value) + } + if got.ConfigHash != apply.IntentHashFromBytes([]byte(template)) { + t.Fatalf("config hash=%q, external template base hash=%q", got.ConfigHash, apply.IntentHashFromBytes([]byte(template))) + } + if data, err := os.ReadFile(dockerCalls); err != nil || len(data) == 0 { + t.Fatalf("deploy did not use docker runtime: err=%v calls=%q", err, data) + } +} + +func writePlanFixture(t *testing.T, name string) string { + t.Helper() + p := t.TempDir() + "/intent.yaml" + if err := os.WriteFile(p, []byte("name: "+name+"\nnetwork: nile\ntarget:\n type: local\n runtime: docker\nnodes:\n - type: fullnode\n version: 4.8.1\n resources:\n memory: 8GB\n"), 0600); err != nil { + t.Fatal(err) + } + return p +} +func mustRead(t *testing.T, p string) []byte { + t.Helper() + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return b +} + +func TestSnapshotDownloadMissingDestUsesValidationEnvelope(t *testing.T) { + res, _, err := snapshotDownloadTool(context.Background(), nil, snapshotDownloadArgs{Network: "nile"}) + if err != nil { + t.Fatal(err) + } + if !res.IsError { + t.Fatal("expected error result") + } + var body map[string]any + if err := json.Unmarshal([]byte(extractText(t, res)), &body); err != nil { + t.Fatal(err) + } + if body["error_code"] != "VALIDATION_ERROR" { + t.Fatalf("error = %v", body) + } +} + +func TestRenderToolRejectsNegativeNodeIndex(t *testing.T) { + p := t.TempDir() + "/intent.yaml" + if err := os.WriteFile(p, []byte("name: p1\nnetwork: nile\ntarget:\n type: local\nnodes:\n - type: fullnode\n version: 4.8.1\n resources:\n memory: 8GB\n"), 0600); err != nil { + t.Fatal(err) + } + res, value, err := renderTool(context.Background(), nil, renderArg{Path: p, Node: -1}) + if err != nil { + t.Fatal(err) + } + envelope, ok := value.(map[string]any) + if !res.IsError || !ok || envelope["error_code"] != "VALIDATION_ERROR" { + t.Fatalf("unexpected result: %v", res) + } +} + +func TestMCPHealUsesJarRuntimeForJarNode(t *testing.T) { + old := healRuntimeForNode + t.Cleanup(func() { healRuntimeForNode = old }) + want := errors.New("jar runtime selected") + var gotRuntime string + healRuntimeForNode = func(_ target.Target, node *state.ManagedNode) runtime.Runtime { + gotRuntime = node.Runtime + return fakeHealRuntime{err: want} + } + err := mcpRunHealAction(context.Background(), target.NewLocalTarget(), &state.ManagedNode{Runtime: "jar"}, mcpHealAction{action: "start"}) + if !errors.Is(err, want) { + t.Fatalf("error=%v, want jar runtime action", err) + } + if gotRuntime != "jar" { + t.Fatalf("runtime=%q, want jar", gotRuntime) + } +} + +func TestHealRuntimeForNodeUsesRealJarFactory(t *testing.T) { + rt := healRuntimeForNode(target.NewLocalTarget(), &state.ManagedNode{Runtime: "jar"}) + if got := fmt.Sprintf("%T", rt); got != "*runtime.JarRuntime" { + t.Fatalf("runtime=%s, want real jar factory", got) + } +} + +type captureNetworkChecker struct{ got *string } + +func (c captureNetworkChecker) Name() string { return "capture_network" } +func (c captureNetworkChecker) Run(_ context.Context, _ target.Target, opts diagnosis.CheckOpts) diagnosis.CheckResult { + *c.got = opts.Network + return diagnosis.CheckResult{Name: c.Name(), Status: diagnosis.StatusPass, Message: "captured"} +} + +func TestDiagnoseToolPassesRecordedNetworkToChecker(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{Name: "diag-net", Network: "private", Runtime: "docker"}}}); err != nil { + t.Fatal(err) + } + var got string + old := diagnoseCheckers + diagnoseCheckers = func() []diagnosis.Checker { return []diagnosis.Checker{captureNetworkChecker{got: &got}} } + t.Cleanup(func() { diagnoseCheckers = old }) + res, _, err := diagnoseTool(context.Background(), nil, nodeArg{Name: "diag-net"}) + if err != nil || res == nil || res.IsError { + t.Fatalf("diagnose result=%v err=%v", res, err) + } + if got != "private" { + t.Fatalf("checker network=%q", got) + } +} + +func TestAutoHealToolPassesRecordedNetworkToChecker(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{Name: "heal-net", Network: "nile", Runtime: "docker"}}}); err != nil { + t.Fatal(err) + } + var got string + old := healCheckers + healCheckers = func() []diagnosis.Checker { return []diagnosis.Checker{captureNetworkChecker{got: &got}} } + t.Cleanup(func() { healCheckers = old }) + res, _, err := autoHealTool(context.Background(), nil, autoHealArgs{Name: "heal-net", DryRun: true}) + if err != nil || res == nil || res.IsError { + t.Fatalf("heal result=%v err=%v", res, err) + } + if got != "nile" { + t.Fatalf("checker network=%q", got) + } +} + +func TestPeersCheckerUsesNetworkInRealCheck(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"nodes":[]}`) + })) + defer srv.Close() + port := srv.Listener.Addr().(*net.TCPAddr).Port + checker := &diagnosis.PeersChecker{} + private := checker.Run(context.Background(), target.NewLocalTarget(), diagnosis.CheckOpts{HTTPPort: port, Network: "private"}) + mainnet := checker.Run(context.Background(), target.NewLocalTarget(), diagnosis.CheckOpts{HTTPPort: port, Network: "mainnet"}) + if !strings.Contains(private.Message, "minimum 1") || !strings.Contains(mainnet.Message, "minimum 3") { + t.Fatalf("private=%+v mainnet=%+v", private, mainnet) + } +} + +type fakeHealRuntime struct{ err error } + +func (f fakeHealRuntime) Deploy(context.Context, runtime.DeployOpts) error { return nil } +func (f fakeHealRuntime) Start(context.Context, string) error { return f.err } +func (f fakeHealRuntime) Stop(context.Context, string) error { return nil } +func (f fakeHealRuntime) Remove(context.Context, string, bool) error { return nil } +func (f fakeHealRuntime) Status(context.Context, string) (*runtime.NodeStatus, error) { + return nil, nil +} +func (f fakeHealRuntime) Logs(context.Context, string, runtime.LogOpts) (io.ReadCloser, error) { + return io.NopCloser(strings.NewReader("")), nil +} diff --git a/internal/mcp/schema_consistency_test.go b/internal/mcp/schema_consistency_test.go index c53a345d..ac131810 100644 --- a/internal/mcp/schema_consistency_test.go +++ b/internal/mcp/schema_consistency_test.go @@ -40,14 +40,10 @@ func TestMCPToolOutputsMatchCLISchemas(t *testing.T) { args: json.RawMessage(`{"path":"../../examples/nile-fullnode.yaml"}`)}, {tool: "config_render", schemaName: "config-render", args: json.RawMessage(`{"path":"../../examples/nile-fullnode.yaml"}`)}, - // `plan` is intentionally NOT in this table. The MCP tool - // returns an intent-only structural preview (no state diff) - // because MCP-resident agents typically don't have the - // host's state.json. The CLI version walks state and emits a - // changes[] / destructive / downtime payload. Both are - // documented and the divergence is surfaced in the MCP - // tool's "note" field; treating them as the same schema - // would force one to mimic the other and lose information. + // plan now returns the same state-aware structural contract as the CLI; + // line-level HOCON diff remains CLI-only. + {tool: "plan", schemaName: "plan", + args: json.RawMessage(`{"path":"../../examples/nile-fullnode.yaml"}`)}, } session, cleanup := newConnectedPair(t) diff --git a/internal/mcp/tools_config.go b/internal/mcp/tools_config.go index 4874faf9..4dae9e55 100644 --- a/internal/mcp/tools_config.go +++ b/internal/mcp/tools_config.go @@ -64,6 +64,10 @@ func renderTool(ctx context.Context, _ *mcp.CallToolRequest, args renderArg) (*m if err != nil { return errResult(err) } + if args.Node < 0 || args.Node > len(parsed.Nodes) { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, + fmt.Sprintf("node index %d out of range; use 0 for all or 1-%d", args.Node, len(parsed.Nodes)))) + } // templateDir resolution: empty → embedded templates. We don't // expose a `template_dir` arg because MCP clients usually live on diff --git a/internal/mcp/tools_diagnostic.go b/internal/mcp/tools_diagnostic.go index d94e32fe..46368f76 100644 --- a/internal/mcp/tools_diagnostic.go +++ b/internal/mcp/tools_diagnostic.go @@ -17,6 +17,8 @@ import ( "github.com/tronprotocol/tron-deployment/internal/target" ) +var diagnoseCheckers = diagnosis.AllCheckers + // registerDiagnosticTools wires up read-only triage tools. Pure-data // when possible; light HTTP probes where the data only exists at // runtime. @@ -240,12 +242,13 @@ func diagnoseTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*m opts := diagnosis.CheckOpts{ NodeName: node.Name, + Network: node.Network, Runtime: node.Runtime, HTTPPort: node.HTTPPort, GRPCPort: node.GRPCPort, } - checkers := diagnosis.AllCheckers() + checkers := diagnoseCheckers() results := make([]diagnosis.CheckResult, 0, len(checkers)) for _, c := range checkers { results = append(results, c.Run(ctx, tgt, opts)) diff --git a/internal/mcp/tools_heal.go b/internal/mcp/tools_heal.go index 9b18d5ac..da7563ef 100644 --- a/internal/mcp/tools_heal.go +++ b/internal/mcp/tools_heal.go @@ -33,6 +33,14 @@ var saveHealState = func(store *state.Store, st *state.DeploymentState) error { } var runHealAction = mcpRunHealAction +var healCheckers = diagnosis.AllCheckers + +var healRuntimeForNode = func(tgt target.Target, node *state.ManagedNode) runtime.Runtime { + if node.Runtime == "jar" { + return runtime.NewJarRuntime(tgt) + } + return runtime.NewDockerRuntime(tgt, paths.Deployments()) +} func registerHealTools(s *mcp.Server) { mcp.AddTool(s, &mcp.Tool{ @@ -101,6 +109,7 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs defer cancel() opts := diagnosis.CheckOpts{ NodeName: node.Name, + Network: node.Network, Runtime: node.Runtime, HTTPPort: node.HTTPPort, GRPCPort: node.GRPCPort, @@ -112,7 +121,7 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs skipped []map[string]any stillFailing []diagnosis.CheckResult ) - for _, c := range diagnosis.AllCheckers() { + for _, c := range healCheckers() { r := c.Run(probeCtx, tgt, opts) if r.Status != diagnosis.StatusFail { continue @@ -193,7 +202,7 @@ func mcpProposeHealAction(r diagnosis.CheckResult, nodeStatus string) (mcpHealAc } func mcpRunHealAction(ctx context.Context, tgt target.Target, node *state.ManagedNode, action mcpHealAction) error { - rt := runtime.NewDockerRuntime(tgt, paths.Deployments()) + rt := healRuntimeForNode(tgt, node) if action.action == "start" { return rt.Start(ctx, node.Name) } diff --git a/internal/mcp/tools_lifecycle.go b/internal/mcp/tools_lifecycle.go index 530a44e7..4b0d815b 100644 --- a/internal/mcp/tools_lifecycle.go +++ b/internal/mcp/tools_lifecycle.go @@ -31,6 +31,13 @@ type planArg struct { Path string `json:"path" jsonschema:"absolute path to an intent.yaml file"` } +func boolToDowntime(changed bool) int { + if changed { + return 30 + } + return 0 +} + func registerLifecycleTools(s *mcp.Server) { mcp.AddTool(s, &mcp.Tool{ Name: "plan", @@ -55,34 +62,55 @@ func registerLifecycleTools(s *mcp.Server) { } func planTool(ctx context.Context, _ *mcp.CallToolRequest, args planArg) (*mcp.CallToolResult, any, error) { - // We can build a "would-apply" preview from intent.Load alone - // without touching state.json — that's enough to surface the - // rendered config + estimated cost. The full diff vs deployed - // state lives in cmd/plan.go and pulls from state; extracting it - // requires the same RunE-to-pure-function refactor as apply. - // - // For an MCP-first agent flow this lighter preview is usually - // what gets used: validate → render → eyeball → apply. The full - // diff stays available via the CLI. + // MCP plan mirrors the CLI's state-aware create/no-op/update distinction. + // It intentionally reports structural intent changes, while the CLI also + // adds rendered-HOCON line diffs. parsed, err := intent.Load(args.Path) if err != nil { return errResult(err) } node := &parsed.Nodes[0] + store, err := state.NewStore(paths.State()) + if err != nil { + return errResult(output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error())) + } + st, err := store.Load() + if err != nil { + return errResult(output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error())) + } + intentBytes, readErr := os.ReadFile(args.Path) + if readErr != nil { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, readErr.Error())) + } + existing := store.GetNode(st, parsed.Name) + if parsed.Target.AutoPorts && existing != nil { + apply.RestoreAutoPorts(node, existing) + } + planned, err := apply.Plan(parsed, existing, intentBytes, apply.FindTemplatesDir()) + if err != nil { + return errResult(err) + } + changes := make([]map[string]any, 0, len(planned.Changes)) + for _, c := range planned.Changes { + changes = append(changes, map[string]any{"type": c.Type, "field": c.Field, "from": c.From, "to": c.To, "restart_required": c.RestartRequired}) + } return jsonResult(map[string]any{ - "name": parsed.Name, - "current_state": "unknown_via_mcp", // until lifecycle extraction - "desired_state": "running", - "network": parsed.Network, - "runtime": parsed.Target.Runtime, - "node_count": len(parsed.Nodes), + "name": parsed.Name, + "current_state": planned.CurrentState, + "desired_state": "running", + "changes": changes, + "destructive": false, + "estimated_downtime_seconds": planned.Downtime, + "network": parsed.Network, + "runtime": planned.Runtime, + "node_count": len(parsed.Nodes), "first_node": map[string]any{ "type": node.Type, "version": node.Version, "memory": node.Resources.Memory, "ports": node.Ports, }, - "note": "MCP plan returns the intent's structural preview. For a full state-aware diff (changes[], destructive, downtime estimate) call `trond plan --intent -o json` from the shell.", + "note": "MCP plan returns the same effective intent/config/version hash checks as CLI plan; line-level HOCON diff remains CLI-only.", }) } @@ -142,10 +170,25 @@ func applyTool(ctx context.Context, _ *mcp.CallToolRequest, args applyArgs) (*mc return errResult(err) } - intentBytes, _ := os.ReadFile(args.Path) - intentHash := apply.IntentHashFromBytes(intentBytes) existing := store.GetNode(st, parsed.Name) - if existing != nil && existing.IntentHash != intentHash && !args.AutoApprove { + intentBytes, readErr := os.ReadFile(args.Path) + if readErr != nil { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, readErr.Error())) + } + if parsed.Target.AutoPorts && existing != nil { + apply.RestoreAutoPorts(&parsed.Nodes[0], existing) + } + templateDir := apply.FindTemplatesDir() + planned, err := apply.Plan(parsed, existing, intentBytes, templateDir) + if err != nil { + return errResult(err) + } + intentHash := planned.IntentHash + legacyMatch := planned.LegacyMatch + if legacyMatch && existing != nil { + intentHash = existing.IntentHash + } + if existing != nil && existing.IntentHash != intentHash && !legacyMatch && !args.AutoApprove { return errResult(output.NewError("HUMAN_REQUIRED", output.ExitHumanRequired, fmt.Sprintf("Changes detected for node %q; pass auto_approve=true to proceed", parsed.Name)). WithSuggestions("Call the 'plan' tool first to inspect the diff", @@ -159,7 +202,7 @@ func applyTool(ctx context.Context, _ *mcp.CallToolRequest, args applyArgs) (*mc State: st, IntentHash: intentHash, Existing: existing, - TemplateDir: "", + TemplateDir: templateDir, DeploymentsDir: paths.Deployments(), EnvVars: resolveEnvVars(&parsed.Nodes[0]), Wait: args.Wait, diff --git a/internal/mcp/tools_snapshot.go b/internal/mcp/tools_snapshot.go index b1233037..60a452e7 100644 --- a/internal/mcp/tools_snapshot.go +++ b/internal/mcp/tools_snapshot.go @@ -6,6 +6,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/snapshot" ) @@ -123,6 +124,9 @@ func snapshotJobsTool(ctx context.Context, _ *mcp.CallToolRequest, _ emptyArgs) } func snapshotDownloadTool(ctx context.Context, req *mcp.CallToolRequest, args snapshotDownloadArgs) (*mcp.CallToolResult, any, error) { + if args.Dest == "" { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, "dest is required")) + } src, err := pickSource(args.Domain, args.Network, args.Kind, args.Region, "") if err != nil { return errResult(err) @@ -137,10 +141,6 @@ func snapshotDownloadTool(ctx context.Context, req *mcp.CallToolRequest, args sn backup = latest } - if args.Dest == "" { - return errResult(fmt.Errorf("dest is required")) - } - kind := snapshot.DBKind(args.Kind) if kind == "" { kind = snapshot.DBKindLite From f425d8bf16c2b9af27332fe4157cb9377f1510ab Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 11:52:53 +0800 Subject: [PATCH 23/49] fix(misc): streaming logs follow, bootstrap gate order, and audit hardening AUD-018: logs -f streams over target.StreamExec (local/ssh/docker/jar) with merged stdout/stderr, exit-error propagation, and Exec fallback when the stream cannot start. Cancellation, proactive Close, and natural EOF are all race-free: a background goroutine owns the single session.Wait, a streamDone signal distinguishes natural EOF from proactive Close, and termination is shared via sync.Once (SIGTERM then force-close; local kills the process). AUD-042: bootstrap enforces the private-network gate before resolving the target. AUD-045: recipe runner no longer panics on nil ProcessState when a host step fails to start. AUD-046: chaos commands propagate state load failures as STATE_ERROR instead of proceeding on a nil node. AUD-047: recipe step IDs must match ^[A-Za-z_][A-Za-z0-9_]*$ in both steps and rollback sections; built-in recipes and the template doc switched to underscores. auto-heal now reports result=no_action when every check passes and nothing was changed, instead of implying success. --- cmd/bootstrap.go | 4 + cmd/bootstrap_guard_test.go | 26 +++ cmd/chaos.go | 5 +- cmd/chaos_test.go | 24 +++ cmd/heal.go | 3 + cmd/logs.go | 11 +- internal/mcp/contract_p1_test.go | 23 +++ internal/mcp/tools_heal.go | 8 +- .../destroy-private-network-cleanly.yaml | 2 +- .../fresh-mainnet-fullnode-with-snapshot.yaml | 4 +- .../recipe/files/upgrade-with-verify.yaml | 2 +- internal/recipe/load.go | 13 ++ internal/recipe/load_test.go | 7 + internal/recipe/recipe_test.go | 17 ++ internal/recipe/runner.go | 4 +- internal/recipe/template.go | 2 +- internal/runtime/docker.go | 7 + internal/runtime/jar.go | 7 + internal/runtime/logs_test.go | 41 +++++ internal/target/local.go | 32 ++++ internal/target/ssh.go | 119 +++++++++++++ internal/target/stream_test.go | 164 ++++++++++++++++++ internal/target/target.go | 28 +++ recipes/destroy-private-network-cleanly.yaml | 2 +- .../fresh-mainnet-fullnode-with-snapshot.yaml | 4 +- recipes/upgrade-with-verify.yaml | 2 +- 26 files changed, 545 insertions(+), 16 deletions(-) create mode 100644 cmd/bootstrap_guard_test.go create mode 100644 cmd/chaos_test.go create mode 100644 internal/runtime/logs_test.go create mode 100644 internal/target/stream_test.go diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index a10aa121..556ff79c 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/guard" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/target" @@ -33,6 +34,9 @@ func runBootstrap(cmd *cobra.Command, args []string) error { if err != nil { return exitWithError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) } + if err := guard.Enforce(parsed.Network); err != nil { + return err + } tgt, err := resolveTarget(parsed) if err != nil { diff --git a/cmd/bootstrap_guard_test.go b/cmd/bootstrap_guard_test.go new file mode 100644 index 00000000..13b06f77 --- /dev/null +++ b/cmd/bootstrap_guard_test.go @@ -0,0 +1,26 @@ +package cmd + +import ( + "errors" + "os" + "testing" + + "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/guard" + "github.com/tronprotocol/tron-deployment/internal/output" +) + +func TestBootstrapRequirePrivateRejectsBeforeTargetResolution(t *testing.T) { + p := t.TempDir() + "/intent.yaml" + if err := os.WriteFile(p, []byte("name: prod\nnetwork: mainnet\ntarget:\n type: ssh\n host: 192.0.2.1\n user: root\nnodes:\n - type: fullnode\n"), 0600); err != nil { + t.Fatal(err) + } + oldPath, oldFlag := bootstrapIntentPath, guard.FlagValue + bootstrapIntentPath, guard.FlagValue = p, true + t.Cleanup(func() { bootstrapIntentPath, guard.FlagValue = oldPath, oldFlag }) + err := runBootstrap(&cobra.Command{}, nil) + var structured *output.StructuredError + if err == nil || !errors.As(err, &structured) || structured.Code != "PRIVATE_NETWORK_REQUIRED" { + t.Fatalf("error=%v", err) + } +} diff --git a/cmd/chaos.go b/cmd/chaos.go index e9234086..77aae0c7 100644 --- a/cmd/chaos.go +++ b/cmd/chaos.go @@ -105,7 +105,10 @@ func chaosPair(ctx context.Context, op, a, b string) error { if err != nil { return err } - deployState, _ := store.Load() + deployState, err := store.Load() + if err != nil { + return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) + } nodeA := store.GetNode(deployState, a) nodeB := store.GetNode(deployState, b) if nodeA == nil { diff --git a/cmd/chaos_test.go b/cmd/chaos_test.go new file mode 100644 index 00000000..ae1204c6 --- /dev/null +++ b/cmd/chaos_test.go @@ -0,0 +1,24 @@ +package cmd + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/output" + "github.com/tronprotocol/tron-deployment/internal/paths" +) + +func TestChaosPairRejectsCorruptState(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + if err := os.WriteFile(paths.State(), []byte("{"), 0600); err != nil { + t.Fatal(err) + } + err := chaosPair(context.Background(), "disconnect", "a", "b") + var structured *output.StructuredError + if err == nil || !errors.As(err, &structured) || structured.Code != "STATE_ERROR" { + t.Fatalf("error=%v", err) + } +} diff --git a/cmd/heal.go b/cmd/heal.go index 58dfaba1..028de725 100644 --- a/cmd/heal.go +++ b/cmd/heal.go @@ -183,6 +183,9 @@ func runAutoHeal(cmd *cobra.Command, args []string) error { "duration_ms": time.Since(start).Milliseconds(), "dry_run": healDryRun, } + if len(healed) == 0 && len(skipped) == 0 && len(stillFailing) == 0 { + result["result"] = "no_action" + } outputFmt, _ := cmd.Flags().GetString("output") if outputFmt == "json" { diff --git a/cmd/logs.go b/cmd/logs.go index 0b722c7a..7a292c90 100644 --- a/cmd/logs.go +++ b/cmd/logs.go @@ -46,8 +46,13 @@ func runLogs(cmd *cobra.Command, args []string) error { return exitWithError("LOGS_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to get logs for %s: %v", name, err)) } - defer reader.Close() - - io.Copy(os.Stdout, reader) + _, copyErr := io.Copy(os.Stdout, reader) + closeErr := reader.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } return nil } diff --git a/internal/mcp/contract_p1_test.go b/internal/mcp/contract_p1_test.go index 2836ef45..04ceb21b 100644 --- a/internal/mcp/contract_p1_test.go +++ b/internal/mcp/contract_p1_test.go @@ -338,6 +338,29 @@ func TestAutoHealToolPassesRecordedNetworkToChecker(t *testing.T) { } } +func TestAutoHealToolReportsNoActionWhenAllChecksPass(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{Name: "heal-pass", Network: "private", Runtime: "docker"}}}); err != nil { + t.Fatal(err) + } + old := healCheckers + healCheckers = func() []diagnosis.Checker { return []diagnosis.Checker{captureNetworkChecker{got: new(string)}} } + t.Cleanup(func() { healCheckers = old }) + _, value, err := autoHealTool(context.Background(), nil, autoHealArgs{Name: "heal-pass"}) + if err != nil { + t.Fatal(err) + } + got, ok := value.(map[string]any) + if !ok || got["result"] != "no_action" { + t.Fatalf("result=%v", value) + } +} + func TestPeersCheckerUsesNetworkInRealCheck(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/mcp/tools_heal.go b/internal/mcp/tools_heal.go index da7563ef..72ffeb47 100644 --- a/internal/mcp/tools_heal.go +++ b/internal/mcp/tools_heal.go @@ -166,13 +166,17 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs healed = append(healed, entry) } - return jsonResult(map[string]any{ + result := map[string]any{ "name": args.Name, "dry_run": args.DryRun, "healed": healed, "skipped": skipped, "still_failing": stillFailing, - }) + } + if len(healed) == 0 && len(skipped) == 0 && len(stillFailing) == 0 { + result["result"] = "no_action" + } + return jsonResult(result) } func recordHealStateSaveFailure(err error, entry map[string]any, check diagnosis.CheckResult, stillFailing *[]diagnosis.CheckResult) { diff --git a/internal/recipe/files/destroy-private-network-cleanly.yaml b/internal/recipe/files/destroy-private-network-cleanly.yaml index 601506c8..fb370558 100644 --- a/internal/recipe/files/destroy-private-network-cleanly.yaml +++ b/internal/recipe/files/destroy-private-network-cleanly.yaml @@ -20,7 +20,7 @@ params: that was used with `trond network create`). steps: - - id: status-check + - id: status_check description: | Confirm the network exists. on_failure=continue so an already-destroyed network just falls through to the destroy diff --git a/internal/recipe/files/fresh-mainnet-fullnode-with-snapshot.yaml b/internal/recipe/files/fresh-mainnet-fullnode-with-snapshot.yaml index 09a7d1d4..ac86baa1 100644 --- a/internal/recipe/files/fresh-mainnet-fullnode-with-snapshot.yaml +++ b/internal/recipe/files/fresh-mainnet-fullnode-with-snapshot.yaml @@ -43,7 +43,7 @@ steps: - "{{ params.intent_path }}" on_failure: abort - - id: download-snapshot + - id: download_snapshot description: | Stream the chain DB tarball through gunzip + tar in one pipeline, MD5-verifying inline. Refuses overwrite of an existing DB unless @@ -89,7 +89,7 @@ steps: # Rollback runs only when the apply step fails. We don't roll back the # snapshot — it's expensive to redo and harmless to leave on disk. rollback: - - id: stop-failed-node + - id: stop_failed_node command: stop args: - "{{ steps.apply.name }}" diff --git a/internal/recipe/files/upgrade-with-verify.yaml b/internal/recipe/files/upgrade-with-verify.yaml index cb186095..21fecf65 100644 --- a/internal/recipe/files/upgrade-with-verify.yaml +++ b/internal/recipe/files/upgrade-with-verify.yaml @@ -29,7 +29,7 @@ params: `trond verify` needs it to know which port to probe. steps: - - id: pre-upgrade-status + - id: pre_upgrade_status description: Snapshot current state for the run record. command: status args: diff --git a/internal/recipe/load.go b/internal/recipe/load.go index e813cd3c..755fa0d6 100644 --- a/internal/recipe/load.go +++ b/internal/recipe/load.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "regexp" "strings" "gopkg.in/yaml.v3" @@ -105,6 +106,16 @@ func (r Recipe) Validate() error { } seen[s.ID] = true } + for _, section := range []struct { + name string + steps []Step + }{{"steps", r.Steps}, {"rollback", r.Rollback}} { + for _, s := range section.steps { + if s.ID != "" && !recipeIDPattern.MatchString(s.ID) { + problems = append(problems, fmt.Sprintf("%s step id %q is invalid; use only [A-Za-z_][A-Za-z0-9_]* for {{ steps..* }} references", section.name, s.ID)) + } + } + } if len(problems) > 0 { return fmt.Errorf("invalid recipe:\n - %s", strings.Join(problems, "\n - ")) @@ -112,6 +123,8 @@ func (r Recipe) Validate() error { return nil } +var recipeIDPattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + // validOnFailure mirrors the runner's switch. Anything else falls to the // default there, which is "abort" — so `on_failure: rollbck` would abort // a recipe that was written to roll back. diff --git a/internal/recipe/load_test.go b/internal/recipe/load_test.go index fd3bde0f..2283e451 100644 --- a/internal/recipe/load_test.go +++ b/internal/recipe/load_test.go @@ -74,6 +74,13 @@ func TestParse_Empty(t *testing.T) { } } +func TestParse_RejectsHyphenatedStepID(t *testing.T) { + _, err := Parse([]byte("name: probe\nsteps:\n - id: bad-step\n command: status\n")) + if err == nil || !strings.Contains(err.Error(), "bad-step") { + t.Fatalf("error=%v", err) + } +} + // TestValidate_ReportsEveryProblemAtOnce: a recipe is a sequence of real // operations, so the caller should learn about all the structural // problems before the first one runs, not one per attempt. diff --git a/internal/recipe/recipe_test.go b/internal/recipe/recipe_test.go index 267acb5d..a3797c5c 100644 --- a/internal/recipe/recipe_test.go +++ b/internal/recipe/recipe_test.go @@ -3,6 +3,7 @@ package recipe import ( "bytes" "context" + "io" "os" "path/filepath" "runtime" @@ -242,6 +243,22 @@ func TestRun_DryRunSkipsExec(t *testing.T) { } } +func TestRun_StartFailureDoesNotPanicOnNilProcessState(t *testing.T) { + r := Recipe{Name: "start-failure", Steps: []Step{{ID: "first", Command: "anything"}}} + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("run panicked: %v", recovered) + } + }() + res, err := Run(context.Background(), r, RunOptions{Binary: "/path/does/not/exist", Out: io.Discard, Err: io.Discard}) + if err == nil { + t.Fatal("expected start error") + } + if len(res.Steps) != 1 || res.Steps[0].ExitCode != 0 { + t.Fatalf("result=%+v", res) + } +} + func TestRun_ResumeFromSkipsEarlierSteps(t *testing.T) { if runtime.GOOS == "windows" { t.Skip() diff --git a/internal/recipe/runner.go b/internal/recipe/runner.go index e3a67675..4bddd5ee 100644 --- a/internal/recipe/runner.go +++ b/internal/recipe/runner.go @@ -323,7 +323,9 @@ func runStep(ctx context.Context, opts RunOptions, step Step, args []string) (St cmd.Stderr = opts.Err stdout, err := cmd.Output() res.DurationMs = time.Since(start).Milliseconds() - res.ExitCode = cmd.ProcessState.ExitCode() + if cmd.ProcessState != nil { + res.ExitCode = cmd.ProcessState.ExitCode() + } res.Output = captureOutput(stdout) if err != nil { diff --git a/internal/recipe/template.go b/internal/recipe/template.go index 7ed67e48..b3cd377f 100644 --- a/internal/recipe/template.go +++ b/internal/recipe/template.go @@ -15,7 +15,7 @@ import ( // Example template forms supported: // // {{ params.node_name }} -// {{ steps.stage-snapshot.job_id }} +// {{ steps.stage_snapshot.job_id }} // {{ params.storage_root }}/output-directory // // We deliberately keep this thin — full Sprig-style helpers are diff --git a/internal/runtime/docker.go b/internal/runtime/docker.go index c4728b8f..ff578345 100644 --- a/internal/runtime/docker.go +++ b/internal/runtime/docker.go @@ -161,6 +161,13 @@ func (r *DockerRuntime) Logs(ctx context.Context, name string, opts LogOpts) (io tailArgs = append(tailArgs, "-f") } tailArgs = append(tailArgs, "/java-tron/logs/tron.log") + if opts.Follow { + if stream, ok := r.target.(target.StreamExec); ok { + if reader, err := stream.StreamExec(ctx, "docker", tailArgs...); err == nil { + return reader, nil + } + } + } if out, err := r.target.Exec(ctx, "docker", tailArgs...); err == nil { return io.NopCloser(bytes.NewReader(out)), nil diff --git a/internal/runtime/jar.go b/internal/runtime/jar.go index b045c68d..a2a72d1e 100644 --- a/internal/runtime/jar.go +++ b/internal/runtime/jar.go @@ -231,6 +231,13 @@ func (r *JarRuntime) Logs(ctx context.Context, name string, opts LogOpts) (io.Re if opts.Follow { args = append(args, "-f") } + if opts.Follow { + if stream, ok := r.target.(target.StreamExec); ok { + if reader, err := stream.StreamExec(ctx, "journalctl", args...); err == nil { + return reader, nil + } + } + } out, err := r.target.Exec(ctx, "journalctl", args...) if err != nil { diff --git a/internal/runtime/logs_test.go b/internal/runtime/logs_test.go new file mode 100644 index 00000000..d0291e4b --- /dev/null +++ b/internal/runtime/logs_test.go @@ -0,0 +1,41 @@ +package runtime + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/target" +) + +type failingStreamTarget struct { + *fakeTarget + streamCalls int +} + +func (f *failingStreamTarget) StreamExec(context.Context, string, ...string) (io.ReadCloser, error) { + f.streamCalls++ + return nil, errors.New("stream unavailable") +} + +func TestDockerLogsFallsBackWhenStreamExecFails(t *testing.T) { + tgt := &failingStreamTarget{fakeTarget: newFakeTarget()} + rt := NewDockerRuntime(tgt, "/deployments") + r, err := rt.Logs(context.Background(), "n0", LogOpts{Follow: true, Tail: 4}) + if err != nil { + t.Fatal(err) + } + defer r.Close() + if len(tgt.cmds) != 1 { + t.Fatalf("commands=%v, want original Exec fallback", tgt.cmds) + } + if tgt.cmds[0][0] != "docker" || tgt.cmds[0][1] != "exec" { + t.Fatalf("commands=%v", tgt.cmds) + } + if tgt.streamCalls != 1 { + t.Fatalf("stream calls=%d", tgt.streamCalls) + } +} + +var _ target.Target = (*failingStreamTarget)(nil) diff --git a/internal/target/local.go b/internal/target/local.go index d17f0228..225ce93f 100644 --- a/internal/target/local.go +++ b/internal/target/local.go @@ -13,6 +13,7 @@ import ( "runtime" "strconv" "strings" + "sync" ) // LocalTarget executes commands and file operations on the local machine. @@ -36,6 +37,37 @@ func (t *LocalTarget) Exec(ctx context.Context, cmd string, args ...string) ([]b return out, nil } +func (t *LocalTarget) StreamExec(ctx context.Context, cmd string, args ...string) (io.ReadCloser, error) { + c := exec.CommandContext(ctx, cmd, args...) + stdout, err := c.StdoutPipe() + if err != nil { + return nil, err + } + stderr, err := c.StderrPipe() + if err != nil { + return nil, err + } + if err := c.Start(); err != nil { + return nil, err + } + pr, pw := io.Pipe() + var wg sync.WaitGroup + wg.Add(2) + copyStream := func(src io.Reader) { defer wg.Done(); _, _ = io.Copy(pw, src) } + go copyStream(stdout) + go copyStream(stderr) + go func() { wg.Wait(); _ = pw.Close() }() + return &streamReader{ReadCloser: pr, terminate: func() { + if c.Process != nil { + _ = c.Process.Kill() + } + }, wait: func() error { + waitErr := c.Wait() + wg.Wait() + return waitErr + }}, nil +} + func (t *LocalTarget) Upload(_ context.Context, localPath, remotePath string) error { // Local target: just copy the file return copyFile(localPath, remotePath) diff --git a/internal/target/ssh.go b/internal/target/ssh.go index fa894639..aaff2949 100644 --- a/internal/target/ssh.go +++ b/internal/target/ssh.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "golang.org/x/crypto/ssh" @@ -297,6 +298,124 @@ func (t *SSHTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byt } } +// StreamExec starts an allow-listed remote command without buffering output. +func (t *SSHTarget) StreamExec(ctx context.Context, cmd string, args ...string) (io.ReadCloser, error) { + if t.client == nil { + return nil, fmt.Errorf("ssh not connected") + } + validate := security.ValidateCommand + if t.provisioning { + validate = security.ValidateProvisioningCommand + } + if err := validate(cmd); err != nil { + return nil, err + } + session, err := t.client.NewSession() + if err != nil { + return nil, fmt.Errorf("new ssh session: %w", err) + } + stdout, err := session.StdoutPipe() + if err != nil { + session.Close() + return nil, err + } + stderr, err := session.StderrPipe() + if err != nil { + session.Close() + return nil, err + } + if err := session.Start(quoteArgs(cmd, args)); err != nil { + session.Close() + return nil, err + } + pr, pw := io.Pipe() + var wg sync.WaitGroup + wg.Add(2) + copyStream := func(src io.Reader) { defer wg.Done(); _, _ = io.Copy(pw, src) } + go copyStream(stdout) + go copyStream(stderr) + streamDone := make(chan struct{}) + go func() { + wg.Wait() + // Close streamDone before the pipe: runLogs may return from io.Copy + // and call Close() the moment pw closes; streamDone must already be + // closed so Close() classifies this as a natural EOF (no terminate). + close(streamDone) + _ = pw.Close() + }() + var terminateOnce sync.Once + terminate := func() { terminateOnce.Do(func() { _ = session.Signal(ssh.SIGTERM); _ = session.Close() }) } + processDone := make(chan error, 1) + go func() { processDone <- session.Wait() }() + watchStop, watchDone := watchSSHStreamContext(ctx, terminate) + return &sshStreamReader{ReadCloser: pr, watchStop: watchStop, watchDone: watchDone, terminate: terminate, processDone: processDone, streamDone: streamDone, wait: func() error { + waitErr := <-processDone + wg.Wait() + _ = session.Close() + return waitErr + }}, nil +} + +func watchSSHStreamContext(ctx context.Context, stopRemote func()) (chan struct{}, chan struct{}) { + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + select { + case <-ctx.Done(): + stopRemote() + case <-stop: + } + }() + return stop, done +} + +type sshStreamReader struct { + io.ReadCloser + wait func() error + terminate func() + watchStop chan struct{} + watchDone chan struct{} + processDone chan error + streamDone chan struct{} + closeOnce sync.Once + closeErr error +} + +func (r *sshStreamReader) Close() error { + r.closeOnce.Do(func() { + close(r.watchStop) + <-r.watchDone + var waitErr error + if r.processDone != nil && r.streamDone != nil { + select { + case <-r.streamDone: + waitErr = <-r.processDone + default: + r.terminate() + waitErr = <-r.processDone + } + } else if r.processDone != nil { + select { + case waitErr = <-r.processDone: + default: + r.terminate() + waitErr = <-r.processDone + } + } else { + r.terminate() + waitErr = r.wait() + } + err := r.ReadCloser.Close() + if err != nil { + r.closeErr = err + } else { + r.closeErr = waitErr + } + }) + return r.closeErr +} + func (t *SSHTarget) Upload(ctx context.Context, localPath, remotePath string) error { if t.client == nil { return fmt.Errorf("ssh not connected") diff --git a/internal/target/stream_test.go b/internal/target/stream_test.go new file mode 100644 index 00000000..7392736b --- /dev/null +++ b/internal/target/stream_test.go @@ -0,0 +1,164 @@ +package target + +import ( + "context" + "io" + "strings" + "sync" + "testing" + "time" +) + +func TestLocalTargetStreamExecStreamsBeforeCommandExit(t *testing.T) { + r, err := NewLocalTarget().StreamExec(context.Background(), "sh", "-c", "printf first; sleep 1; printf second") + if err != nil { + t.Fatal(err) + } + defer r.Close() + buf := make([]byte, 5) + if _, err := io.ReadFull(r, buf); err != nil { + t.Fatal(err) + } + if string(buf) != "first" { + t.Fatalf("first=%q", buf) + } +} + +func TestLocalTargetStreamExecMergesStderr(t *testing.T) { + r, err := NewLocalTarget().StreamExec(context.Background(), "sh", "-c", "printf out; printf err >&2") + if err != nil { + t.Fatal(err) + } + data, readErr := io.ReadAll(r) + closeErr := r.Close() + if readErr != nil || closeErr != nil { + t.Fatalf("read=%v close=%v", readErr, closeErr) + } + if got := string(data); !strings.Contains(got, "out") || !strings.Contains(got, "err") { + t.Fatalf("merged=%q", got) + } +} + +func TestLocalTargetStreamExecReturnsExitError(t *testing.T) { + r, err := NewLocalTarget().StreamExec(context.Background(), "sh", "-c", "printf bad >&2; exit 7") + if err != nil { + t.Fatal(err) + } + _, _ = io.ReadAll(r) + if err := r.Close(); err == nil { + t.Fatal("expected non-zero exit error") + } +} + +func TestLocalTargetStreamExecContextCancellationConverges(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + r, err := NewLocalTarget().StreamExec(ctx, "sh", "-c", "while true; do printf x; done") + if err != nil { + t.Fatal(err) + } + cancel() + closed := make(chan error, 1) + go func() { closed <- r.Close() }() + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("stream close did not converge after cancellation") + } +} + +func TestLocalTargetStreamExecCloseKillsRunningCommand(t *testing.T) { + r, err := NewLocalTarget().StreamExec(context.Background(), "sh", "-c", "sleep 30") + if err != nil { + t.Fatal(err) + } + closed := make(chan error, 1) + go func() { closed <- r.Close() }() + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("Close did not kill running command") + } +} + +func TestSSHStreamContextCancellationClosesRemoteSession(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + stopped := make(chan struct{}) + stop, done := watchSSHStreamContext(ctx, func() { close(stopped) }) + cancel() + select { + case <-stopped: + case <-time.After(time.Second): + t.Fatal("remote stop was not requested") + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("watcher goroutine did not converge") + } + close(stop) +} + +func TestSSHStreamReaderCloseStopsRunningRemoteSession(t *testing.T) { + reader, writer := io.Pipe() + stopped := make(chan struct{}) + var stopOnce sync.Once + watchStop, watchDone := watchSSHStreamContext(context.Background(), func() {}) + fakeWait := func() error { stopOnce.Do(func() { close(stopped) }); return nil } + r := &sshStreamReader{ + ReadCloser: reader, + watchStop: watchStop, + watchDone: watchDone, + terminate: func() { _ = writer.Close(); stopOnce.Do(func() { close(stopped) }) }, + wait: fakeWait, + } + closed := make(chan error, 1) + go func() { closed <- r.Close() }() + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("Close did not stop remote session") + } + select { + case <-stopped: + default: + t.Fatal("remote stop was not requested") + } +} + +func TestSSHStreamReaderCloseSkipsTerminateAfterNaturalExit(t *testing.T) { + reader, writer := io.Pipe() + defer writer.Close() + watchStop, watchDone := watchSSHStreamContext(context.Background(), func() {}) + processDone := make(chan error, 1) + streamDone := make(chan struct{}) + close(streamDone) + terminateCalls := 0 + r := &sshStreamReader{ + ReadCloser: reader, + watchStop: watchStop, + watchDone: watchDone, + processDone: processDone, + streamDone: streamDone, + terminate: func() { terminateCalls++ }, + wait: func() error { return nil }, + } + closed := make(chan error, 1) + go func() { closed <- r.Close() }() + select { + case <-closed: + t.Fatal("Close returned before process completion") + case <-time.After(20 * time.Millisecond): + } + if terminateCalls != 0 { + t.Fatalf("terminate calls=%d, want 0", terminateCalls) + } + processDone <- nil + select { + case err := <-closed: + if err != nil { + t.Fatalf("Close error=%v", err) + } + case <-time.After(time.Second): + t.Fatal("Close did not converge after process completion") + } +} diff --git a/internal/target/target.go b/internal/target/target.go index 9bcdd9aa..a63e0a53 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -218,6 +218,34 @@ type Target interface { String() string } +// StreamExec runs a command and returns its live stdout/stderr stream. +// Targets that cannot provide a stream should continue using Exec. +type StreamExec interface { + StreamExec(ctx context.Context, cmd string, args ...string) (io.ReadCloser, error) +} + +type streamReader struct { + io.ReadCloser + wait func() error + terminate func() + closeOnce sync.Once + closeErr error +} + +func (r *streamReader) Close() error { + r.closeOnce.Do(func() { + r.terminate() + err := r.ReadCloser.Close() + waitErr := r.wait() + if err != nil { + r.closeErr = err + } else { + r.closeErr = waitErr + } + }) + return r.closeErr +} + // Permissions is an optional target capability for tightening an existing // file without rewriting its contents. type Permissions interface { diff --git a/recipes/destroy-private-network-cleanly.yaml b/recipes/destroy-private-network-cleanly.yaml index 601506c8..fb370558 100644 --- a/recipes/destroy-private-network-cleanly.yaml +++ b/recipes/destroy-private-network-cleanly.yaml @@ -20,7 +20,7 @@ params: that was used with `trond network create`). steps: - - id: status-check + - id: status_check description: | Confirm the network exists. on_failure=continue so an already-destroyed network just falls through to the destroy diff --git a/recipes/fresh-mainnet-fullnode-with-snapshot.yaml b/recipes/fresh-mainnet-fullnode-with-snapshot.yaml index 09a7d1d4..ac86baa1 100644 --- a/recipes/fresh-mainnet-fullnode-with-snapshot.yaml +++ b/recipes/fresh-mainnet-fullnode-with-snapshot.yaml @@ -43,7 +43,7 @@ steps: - "{{ params.intent_path }}" on_failure: abort - - id: download-snapshot + - id: download_snapshot description: | Stream the chain DB tarball through gunzip + tar in one pipeline, MD5-verifying inline. Refuses overwrite of an existing DB unless @@ -89,7 +89,7 @@ steps: # Rollback runs only when the apply step fails. We don't roll back the # snapshot — it's expensive to redo and harmless to leave on disk. rollback: - - id: stop-failed-node + - id: stop_failed_node command: stop args: - "{{ steps.apply.name }}" diff --git a/recipes/upgrade-with-verify.yaml b/recipes/upgrade-with-verify.yaml index cb186095..21fecf65 100644 --- a/recipes/upgrade-with-verify.yaml +++ b/recipes/upgrade-with-verify.yaml @@ -29,7 +29,7 @@ params: `trond verify` needs it to know which port to probe. steps: - - id: pre-upgrade-status + - id: pre_upgrade_status description: Snapshot current state for the run record. command: status args: From 3603612173c614e62bb04659db392d9c00c42d1e Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 12:19:54 +0800 Subject: [PATCH 24/49] fix(render): wire solidity_grpc port, strict yaml fields, compose env escaping, overlay field merge (AUD-022/023/024/026/027/028) - hocon: solidity_port conflict detection incl. solidity<->pbft; validate via real render path (AUD-022/023) - hocon: replaceSolidityGRPCPort block-scoped key match, activate commented template lines (AUD-024) - compose: escape $ in extra_env values against Compose interpolation (AUD-027) - intent: yaml KnownFields(true) rejects unknown fields (AUD-026); overlay target field-level merge incl. auto_ports (AUD-028) - compose: document solidity/pbft ports intentionally unpublished on host --- internal/intent/audit_render_test.go | 41 +++++++++++++ internal/intent/loader.go | 9 ++- internal/intent/overlay.go | 29 ++++++++- internal/render/compose.go | 13 ++-- internal/render/compose_imageoverride_test.go | 10 +++ internal/render/hocon.go | 61 +++++++++++++++---- internal/render/hocon_test.go | 30 +++++++++ .../render/testdata/golden/nile-fullnode.conf | 2 +- 8 files changed, 172 insertions(+), 23 deletions(-) create mode 100644 internal/intent/audit_render_test.go diff --git a/internal/intent/audit_render_test.go b/internal/intent/audit_render_test.go new file mode 100644 index 00000000..34cfc945 --- /dev/null +++ b/internal/intent/audit_render_test.go @@ -0,0 +1,41 @@ +package intent + +import ( + "os" + "strings" + "testing" +) + +func TestParseRejectsUnknownTopLevelAndNestedFields(t *testing.T) { + for _, body := range []string{ + "name: n\nnetwork: mainnet\nunknown: true\n", + "name: n\nnetwork: mainnet\nnodes:\n - type: fullnode\n resources:\n memory: 8GB\n unknown: true\n", + } { + if _, err := Parse([]byte(body)); err == nil { + t.Fatalf("accepted unknown field: %s", body) + } + } +} + +func TestLoadWithOverlayPreservesUntouchedTargetFields(t *testing.T) { + dir := t.TempDir() + base := dir + "/base.yaml" + overlay := dir + "/overlay.yaml" + baseData := []byte("name: n\nnetwork: mainnet\ntarget:\n type: ssh\n host: example\n port: 22\n user: tron\n identity_file: /key\nnodes:\n - type: fullnode\n resources:\n memory: 8GB\n") + if err := os.WriteFile(base, baseData, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(overlay, []byte("target:\n user: other\n"), 0600); err != nil { + t.Fatal(err) + } + got, err := LoadWithOverlay(base, overlay) + if err != nil { + t.Fatal(err) + } + if got.Target.Host != "example" || got.Target.IdentityFile != "/key" || got.Target.User != "other" { + t.Fatalf("target=%+v", got.Target) + } + if strings.TrimSpace(got.Target.Type) != "ssh" { + t.Fatalf("target type=%q", got.Target.Type) + } +} diff --git a/internal/intent/loader.go b/internal/intent/loader.go index ffeac9f2..f6765fad 100644 --- a/internal/intent/loader.go +++ b/internal/intent/loader.go @@ -1,6 +1,7 @@ package intent import ( + "bytes" "fmt" "net/url" "os" @@ -258,7 +259,9 @@ func Load(path string) (*Intent, error) { // Parse parses intent YAML bytes, validates, and applies defaults. func Parse(data []byte) (*Intent, error) { var intent Intent - if err := yaml.Unmarshal(data, &intent); err != nil { + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&intent); err != nil { return nil, fmt.Errorf("parse intent YAML: %w", err) } @@ -275,7 +278,9 @@ func Parse(data []byte) (*Intent, error) { // defaults at the field level. func ParseRaw(data []byte) (*Intent, error) { var intent Intent - if err := yaml.Unmarshal(data, &intent); err != nil { + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&intent); err != nil { return nil, fmt.Errorf("parse intent YAML: %w", err) } return &intent, nil diff --git a/internal/intent/overlay.go b/internal/intent/overlay.go index 3b783193..48c79df3 100644 --- a/internal/intent/overlay.go +++ b/internal/intent/overlay.go @@ -1,6 +1,7 @@ package intent import ( + "bytes" "fmt" "os" @@ -38,7 +39,9 @@ func LoadWithOverlay(basePath, overlayPath string) (*Intent, error) { func mergeOverlay(base *Intent, overlayData []byte) error { // Parse overlay as a partial intent var overlay Intent - if err := yaml.Unmarshal(overlayData, &overlay); err != nil { + dec := yaml.NewDecoder(bytes.NewReader(overlayData)) + dec.KnownFields(true) + if err := dec.Decode(&overlay); err != nil { return fmt.Errorf("parse overlay YAML: %w", err) } @@ -49,8 +52,28 @@ func mergeOverlay(base *Intent, overlayData []byte) error { if overlay.Network != "" { base.Network = overlay.Network } - if overlay.Target.Type != "" { - base.Target = overlay.Target + if overlay.Target.Type != "" || overlay.Target.Host != "" || overlay.Target.Port != 0 || overlay.Target.User != "" || overlay.Target.IdentityFile != "" || overlay.Target.Runtime != "" || overlay.Target.AutoPorts { + if overlay.Target.Type != "" { + base.Target.Type = overlay.Target.Type + } + if overlay.Target.Host != "" { + base.Target.Host = overlay.Target.Host + } + if overlay.Target.Port != 0 { + base.Target.Port = overlay.Target.Port + } + if overlay.Target.User != "" { + base.Target.User = overlay.Target.User + } + if overlay.Target.IdentityFile != "" { + base.Target.IdentityFile = overlay.Target.IdentityFile + } + if overlay.Target.Runtime != "" { + base.Target.Runtime = overlay.Target.Runtime + } + if overlay.Target.AutoPorts { + base.Target.AutoPorts = true + } } if len(overlay.Nodes) > 0 { base.Nodes = overlay.Nodes diff --git a/internal/render/compose.go b/internal/render/compose.go index 219b681b..a47aa0d1 100644 --- a/internal/render/compose.go +++ b/internal/render/compose.go @@ -2,7 +2,6 @@ package render import ( "fmt" - "maps" "sort" "strings" @@ -31,7 +30,9 @@ func sortedKeys(m map[string]string) []string { // witness_key.keystore_password_env is set. func composeEnvLines(node *intent.NodeSpec) []string { env := make(map[string]string, len(node.ExtraEnv)+1) - maps.Copy(env, node.ExtraEnv) + for key, value := range node.ExtraEnv { + env[key] = strings.ReplaceAll(value, "$", "$$") + } if node.Type == "witness" && node.WitnessKey != nil && node.WitnessKey.KeystorePasswordEnv != "" { passwordEnv := node.WitnessKey.KeystorePasswordEnv env[passwordEnv] = "${" + passwordEnv + "}" @@ -314,9 +315,10 @@ func volumeDeclName(src string) string { // renderComposePorts produces the host:container port mappings. // // The P2P port also needs UDP exposed for kad-style peer discovery. HTTP / -// gRPC are TCP-only. Optional features (jsonrpc, metrics, solidity APIs) -// are appended only when the user explicitly enables them or sets a custom -// port. +// gRPC are TCP-only. JSON-RPC and metrics are published when enabled. +// Solidity HTTP/gRPC ports remain internal to the node and are intentionally +// not published on the host; their persisted state ports support node probes, +// not external exposure. func renderComposePorts(node *intent.NodeSpec) []string { ports := []string{ fmt.Sprintf("%d:%d", node.Ports.HTTP, node.Ports.HTTP), @@ -332,6 +334,5 @@ func renderComposePorts(node *intent.NodeSpec) []string { if node.Features.Metrics != nil && *node.Features.Metrics { ports = append(ports, fmt.Sprintf("%d:%d", node.Ports.Metrics, node.Ports.Metrics)) } - return ports } diff --git a/internal/render/compose_imageoverride_test.go b/internal/render/compose_imageoverride_test.go index d20855a2..4875e811 100644 --- a/internal/render/compose_imageoverride_test.go +++ b/internal/render/compose_imageoverride_test.go @@ -65,3 +65,13 @@ func TestRenderCompose_NoOverrideKeepsLegacyBehavior(t *testing.T) { t.Error("compose without imageOverride MUST NOT inject pull_policy") } } + +func TestRenderComposeEscapesExtraEnvDollar(t *testing.T) { + i := &intent.Intent{Name: "env-node", Nodes: []intent.NodeSpec{{Type: "fullnode", ExtraEnv: map[string]string{"A": "$VAR", "B": "${VALUE}", "C": "literal$"}}}} + got := RenderCompose("env-node", i, &i.Nodes[0], "", "", "") + for _, want := range []string{"A=$$VAR", "B=$${VALUE}", "C=literal$$"} { + if !strings.Contains(got, want) { + t.Fatalf("missing escaped env %q in %s", want, got) + } + } +} diff --git a/internal/render/hocon.go b/internal/render/hocon.go index 3eef58bb..e8ad3267 100644 --- a/internal/render/hocon.go +++ b/internal/render/hocon.go @@ -321,6 +321,9 @@ func RenderHOCONWithSecrets(templateDir string, i *intent.Intent, node *intent.N config := string(data) // 1. Targeted line-level rewrites. + if node.Ports.SolidityGRPC != 0 && !hasHOCONBlock(config, "rpc") { + return Rendered{}, fmt.Errorf("cannot render ports.solidity_grpc: rpc block not found") + } config = applyPortOverrides(config, node) config = applyFeatureOverrides(config, node) if err := checkHTTPPortConflicts(config, node); err != nil { @@ -356,17 +359,9 @@ func ValidateIntentHTTPPortConflicts(templateDir string, parsed, raw *intent.Int if raw.Target.AutoPorts { return nil } - data, err := LoadTemplate(templateDir, parsed.Network) - if err != nil { - return err - } - for idx := range raw.Nodes { - if raw.Nodes[idx].Ports.HTTP == 0 { - continue - } - node := parsed.Nodes[idx] - config := applyPortOverrides(string(data), &node) - if err := checkHTTPPortConflicts(config, &node); err != nil { + for idx := range parsed.Nodes { + node := &parsed.Nodes[idx] + if _, err := RenderHOCONWithSecrets(templateDir, parsed, node); err != nil { return err } } @@ -595,6 +590,9 @@ func applyPortOverrides(config string, node *intent.NodeSpec) string { if ports.GRPC != 0 { config = replaceRPCPort(config, ports.GRPC) } + if ports.SolidityGRPC != 0 { + config = replaceSolidityGRPCPort(config, ports.SolidityGRPC) + } if ports.SolidityHTTP != 0 { config = replaceHOCONValue(config, "solidityPort", fmt.Sprintf("%d", ports.SolidityHTTP)) } @@ -639,6 +637,9 @@ func checkHTTPPortConflicts(config string, node *intent.NodeSpec) error { if pbftOK && full == pbft && !hasHTTPOverride(node, "PBFTPort") { return fmt.Errorf("rendered HTTP port conflict: fullNodePort=%d collides with PBFTPort (in-container). Set config_overrides \"node.http.PBFTPort\" to a different port", full) } + if solidityOK && pbftOK && solidity == pbft && !hasHTTPOverride(node, "solidityPort") && !hasHTTPOverride(node, "PBFTPort") { + return fmt.Errorf("rendered HTTP port conflict: solidityPort=%d collides with PBFTPort; set a different ports.solidity_http or config_overrides value", solidity) + } return nil } @@ -721,6 +722,44 @@ func replaceRPCPort(config string, port int) string { return strings.Join(lines, "\n") } +func replaceSolidityGRPCPort(config string, port int) string { + lines := strings.Split(config, "\n") + inRPC := false + depth := 0 + rpcDepth := 0 + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if !inRPC && strings.HasPrefix(trimmed, "rpc") && strings.Contains(trimmed, "{") { + inRPC = true + rpcDepth = depth + strings.Count(line, "{") - strings.Count(line, "}") + continue + } + if inRPC { + key := strings.TrimSpace(strings.TrimPrefix(trimmed, "#")) + if strings.HasPrefix(key, "solidityPort") { + lines[i] = fmt.Sprintf("%ssolidityPort = %d", lineIndent(line), port) + return strings.Join(lines, "\n") + } + if trimmed == "}" && depth+strings.Count(line, "{")-strings.Count(line, "}") < rpcDepth { + lines = append(lines[:i], append([]string{" solidityPort = " + fmt.Sprintf("%d", port)}, lines[i:]...)...) + return strings.Join(lines, "\n") + } + } + depth += strings.Count(line, "{") - strings.Count(line, "}") + } + return strings.Join(lines, "\n") +} + +func hasHOCONBlock(config, name string) bool { + for _, line := range strings.Split(config, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, name) && strings.Contains(trimmed, "{") { + return true + } + } + return false +} + // replaceListenPort replaces the P2P listen port. func replaceListenPort(config string, port int) string { lines := strings.Split(config, "\n") diff --git a/internal/render/hocon_test.go b/internal/render/hocon_test.go index e80b7141..7db07420 100644 --- a/internal/render/hocon_test.go +++ b/internal/render/hocon_test.go @@ -145,6 +145,36 @@ func TestRenderHOCON_DefaultHTTPPortsUnchanged(t *testing.T) { } } +func TestRenderHOCON_SolidityGRPCPortOverride(t *testing.T) { + i := &intent.Intent{Name: "grpc", Network: "mainnet"} + n := &intent.NodeSpec{Type: "fullnode", Ports: intent.PortMapping{SolidityGRPC: 56061}} + out, err := RenderHOCONWithSecrets("", i, n) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out.Deployable(), "solidityPort = 56061") { + t.Fatalf("solidity gRPC port not rendered: %s", out.Deployable()) + } +} + +func TestRenderHOCON_SolidityPBFTCollision(t *testing.T) { + n := &intent.NodeSpec{Type: "fullnode", Ports: intent.PortMapping{HTTP: 8091}} + i := &intent.Intent{Name: "collision", Network: "mainnet"} + _, err := RenderHOCON("", i, n) + if err == nil || !strings.Contains(err.Error(), "solidityPort") { + t.Fatalf("expected collision, got %v", err) + } +} + +func TestRenderHOCON_SolidityPBFTDirectCollision(t *testing.T) { + n := &intent.NodeSpec{Type: "fullnode", Ports: intent.PortMapping{SolidityHTTP: 8092}} + i := &intent.Intent{Name: "direct-collision", Network: "mainnet"} + _, err := RenderHOCON("", i, n) + if err == nil || !strings.Contains(err.Error(), "solidityPort") || !strings.Contains(err.Error(), "PBFTPort") { + t.Fatalf("expected solidity/PBFT collision, got %v", err) + } +} + // TestRenderHOCON_JSONRPCPortAndEnable locks down the #165 fix: // features.jsonrpc=true + ports.jsonrpc=NNNNN must produce BOTH // `httpFullNodeEnable = true` (already worked) AND diff --git a/internal/render/testdata/golden/nile-fullnode.conf b/internal/render/testdata/golden/nile-fullnode.conf index 7645a72d..f279ff2a 100644 --- a/internal/render/testdata/golden/nile-fullnode.conf +++ b/internal/render/testdata/golden/nile-fullnode.conf @@ -211,7 +211,7 @@ node { rpc { port = 50051 - #solidityPort = 50061 + solidityPort = 50061 # Number of gRPC thread, default availableProcessors / 2 # thread = 16 From 9101fb0be26a9d47f6b4de91e77beb4d31ba5759 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 12:32:50 +0800 Subject: [PATCH 25/49] docs(specs): record implementation contracts and fix drift MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cli-contract.md: implementation contracts section — single intent-hash source (internal/apply EffectiveIntentHash/LegacyIntentHashMatches/RestoreAutoPorts), atomic state persistence, artifact upgrade transaction, target transport lifecycle, read/write state lock split, witness redaction contract - spec.md: correct upgrade wording (no built-in health verify; recipe provides it) and scope the check-then-act recovery promise to apply only - plan.md: same scoping note on the apply step --- .../contracts/cli-contract.md | 9 +++++ specs/001-trond-cli-platform/plan.md | 2 +- specs/001-trond-cli-platform/spec.md | 33 +++++++++++++++---- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/specs/001-trond-cli-platform/contracts/cli-contract.md b/specs/001-trond-cli-platform/contracts/cli-contract.md index 0c1fa484..0cc02413 100644 --- a/specs/001-trond-cli-platform/contracts/cli-contract.md +++ b/specs/001-trond-cli-platform/contracts/cli-contract.md @@ -43,6 +43,15 @@ All errors emitted to stderr as JSON (when `--output json`): } ``` +## Implementation Contracts + +- `internal/apply/hash.go:14-74` is the sole intent-hash source: `EffectiveIntentHash` uses the `intent-hash-v2\x00` domain separator and `v2:` prefix; `LegacyIntentHashMatches` checks raw/canonical forms plus recorded monitoring equivalence; `RestoreAutoPorts` restores seven non-zero persisted ports. CLI apply/plan and MCP plan/apply all call these functions; duplicate implementations are forbidden. +- `internal/state/store.go:64-113` saves through a `0600` temp file, file sync, atomic rename, and directory sync. Save failures propagate as `STATE_ERROR`; the state lock protects read-modify-write from lost updates. +- `internal/runtime/runtime.go:97-107` defines the artifact transaction (`PrepareArtifact`, `Activate`, `Start`, `Rollback`, `Cleanup`). Version state is committed only after activation and startup succeed; jar recovery maintains disk == pin == recorded version. +- `internal/target/target.go:26-107` owns pooled HTTP transports and timeout-enforced dialing. `StreamExec` supports cancellation, explicit close, and natural EOF with merged output and propagated exit errors. +- `cmd/resolve.go:127-146` separates exclusive write resolution (30-second `LOCK_TIMEOUT`) from read-only access. Write commands include apply, start, stop, restart, remove, heal, upgrade, rollback, plus network mutators. +- `internal/render/hocon.go:145-172` provides parse-first `RedactWitnessLines`; plan/config diffs, verify-config, MCP resources, and MCP drift output are redacted, while internal drift comparison uses original bytes. + ## Command Reference ### trond apply diff --git a/specs/001-trond-cli-platform/plan.md b/specs/001-trond-cli-platform/plan.md index 577bdd75..9dced8e0 100644 --- a/specs/001-trond-cli-platform/plan.md +++ b/specs/001-trond-cli-platform/plan.md @@ -407,7 +407,7 @@ trond apply --intent ├─ 7. Compute diff vs deployed state │ ├─ No changes → exit 0, "no changes" │ └─ Changes detected → continue - ├─ 8. For each step (idempotent check-then-act): + ├─ 8. For each apply step (idempotent check-then-act; apply only): │ ├─ [jar] Download jar if missing/wrong hash │ ├─ Upload config files to target │ ├─ [docker] Upload compose file + docker compose up -d diff --git a/specs/001-trond-cli-platform/spec.md b/specs/001-trond-cli-platform/spec.md index 678f798b..e0744932 100644 --- a/specs/001-trond-cli-platform/spec.md +++ b/specs/001-trond-cli-platform/spec.md @@ -11,7 +11,7 @@ - Q: How should trond-managed nodes be uniquely identified? → A: User-assigned `name` field in intent.yaml, required and unique per state file. - Q: What is the relationship between `deploy` and `apply` commands? → A: `deploy` is an alias for `apply`; both are idempotent. `apply` is the canonical command. -- Q: How should trond handle a previously interrupted (partial) deployment? → A: No transaction log; each `apply` step is individually idempotent (check-then-act). Partial state is naturally converged on next run. +- Q: How should trond handle a previously interrupted (partial) deployment? → A: `apply` steps are individually idempotent (check-then-act). This recovery promise applies to `apply` only; `upgrade` and `rollback` use their artifact transaction and do not promise automatic no-human recovery. ## User Scenarios & Testing *(mandatory)* @@ -139,12 +139,12 @@ downtime and the ability to roll back if the upgrade fails. with rollback capability is important for production but can follow after core deploy/status functionality is stable. -**Independent Test**: Can be tested by deploying a node at version X, then running `trond upgrade --version Y`. The node restarts with the new version and passes health checks. +**Independent Test**: Can be tested by deploying a node at version X, then running `trond upgrade --version Y`. The node restarts with the new version; health verification is separate or supplied by the `upgrade-with-verify` recipe. **Acceptance Scenarios**: -1. **Given** a fullnode running version 4.8.0, **When** the user runs `trond upgrade --version 4.8.1`, **Then** trond downloads the new jar/image, stops the node, replaces the binary/image, restarts, and verifies health. -2. **Given** an upgrade that fails health verification, **When** 5 minutes pass without the node reaching a healthy state, **Then** trond automatically rolls back to the previous version and reports the failure with diagnostics. +1. **Given** a fullnode running version 4.8.0, **When** the user runs `trond upgrade --version 4.8.1`, **Then** trond downloads the new jar/image, stops the node, replaces the binary/image, and restarts. Bare `trond upgrade` does not verify health. +2. **Given** an upgrade that must be health-gated, **When** the `upgrade-with-verify` recipe (or a network upgrade) fails verification, **Then** that workflow rolls back and reports diagnostics. 3. **Given** a successful upgrade, **When** the user later runs `trond rollback `, **Then** the previous version is restored. --- @@ -178,7 +178,7 @@ depends on all lower-level stories working correctly first. - What happens when disk space is insufficient for a fullnode? `trond preflight` MUST detect this and warn before deployment begins. - What happens when two users run `trond apply` simultaneously on the same target? The locking mechanism MUST ensure only one proceeds; the other receives a lock-conflict error. - What happens when the user's intent.yaml references a witness private key env var that contains a malformed key? trond MUST validate key format before deployment and report a clear error without echoing the key value. -- What happens when trond is interrupted mid-deployment (kill signal, network drop, machine reboot)? The target may have partial artifacts (jar downloaded but systemd unit not yet created). On the next `trond apply`, each step MUST check-then-act: verify whether its artifact exists and is correct before re-executing. No manual cleanup or separate recovery command is required. +- What happens when trond is interrupted mid-deployment? For `trond apply`, each step MUST check-then-act and converge without manual cleanup. This guarantee does not extend to bare `upgrade` or `rollback`. - What happens when two intent files define the same node `name`? trond MUST reject the second `apply` with a conflict error referencing the existing node's intent source. ## Requirements *(mandatory)* @@ -201,7 +201,7 @@ depends on all lower-level stories working correctly first. - **FR-009**: For jar runtime, trond MUST download the java-tron jar from official GitHub Releases with SHA256 verification. - **FR-010**: For jar runtime, trond MUST generate systemd unit files and manage the node via `systemctl`. - **FR-011**: trond MUST support `apply`, `start`, `stop`, `restart`, and `remove` lifecycle commands for individual nodes. `deploy` is an alias for `apply` and behaves identically. -- **FR-012**: `apply` is the canonical deployment command and MUST be idempotent: create if not exists, update if changed, no-op if identical. Each step within `apply` (download, render, transfer, start) MUST independently verify its precondition before executing, so that a partially completed prior run converges to the desired state on the next invocation without manual cleanup. +- **FR-012**: `apply` is the canonical deployment command and MUST be idempotent: create if not exists, update if changed, no-op if identical. Each apply step MUST independently verify its precondition. This check-then-act/no-manual-recovery promise is specific to `apply`, not bare `upgrade` or `rollback`. - **FR-013**: trond MUST support `plan` to preview changes without applying them. - **FR-014**: Destructive operations (`remove --purge`, `rollback`) MUST default to dry-run and require explicit `--auto-approve` or `--confirm `. @@ -220,9 +220,28 @@ depends on all lower-level stories working correctly first. **Upgrade & Rollback**: -- **FR-022**: trond MUST support `upgrade` to update a node to a new java-tron version. +- **FR-022**: trond MUST support `upgrade` to update a node to a new java-tron version. Bare upgrade does not verify health; health-gated upgrade is provided by the `upgrade-with-verify` recipe and `network upgrade`. - **FR-023**: trond MUST support `rollback` to revert to the previous version. +## Implementation Contracts + +- Intent hash and seven-port restoration are single-sourced in + `internal/apply/hash.go:14-74`; CLI apply/plan and MCP plan/apply use its + effective, legacy-match, and auto-port functions. +- `internal/state/store.go:64-113` uses a `0600` temp file, file sync, atomic + rename, and directory sync; save failures are `STATE_ERROR`, and state.lock + protects read-modify-write from lost updates. +- `internal/runtime/runtime.go:97-107` defines prepare/activate/start/rollback/ + cleanup artifact transactions. Version state commits only after activation + and start succeed; jar disk, pin, and recorded version must converge. +- `internal/target/target.go:26-107` owns pooled transports and timeout + dialing. StreamExec covers cancellation, explicit close, and natural EOF. +- `cmd/resolve.go:127-146` gives write commands exclusive locking with a + 30-second `LOCK_TIMEOUT`; read-only commands do not take that write lock. +- `internal/render/hocon.go:145-172` parse-first redaction protects plan/config + diffs, verify-config, MCP resources, and MCP drift output; internal drift + comparison retains original bytes. + **Preflight & Bootstrap**: - **FR-024**: trond MUST provide a `preflight` command that checks target readiness (Docker/JDK installed, ports available, disk space, memory). From 8209b5c1ae1a5f8a4024e820b8e98f0a72e21566 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 12:54:44 +0800 Subject: [PATCH 26/49] refactor(cmd): consolidate persistNodeState helper; fix MCP template-test hash expectation Consolidation pass A (post-audit hardening): fold the six isomorphic persist*State blocks into a single persistNodeState helper, label legacy compatibility paths (v1 hash fallback, compose bind parse) with removal notes, and harden test diagnostics (nil-safe extractText, pipeline-derived ConfigHash expectation with template marker assertion). --- cmd/apply.go | 1 + cmd/network/create.go | 2 ++ cmd/remove.go | 11 ++--------- cmd/restart.go | 11 ++--------- cmd/rollback.go | 11 ++--------- cmd/snapshot/download.go | 1 + cmd/start.go | 31 +++++++++++++++++++++++++++---- cmd/stop.go | 11 ++--------- cmd/upgrade.go | 11 ++--------- internal/mcp/contract_p1_test.go | 23 +++++++++++++++++++---- internal/mcp/server_test.go | 2 +- internal/mcp/tools_heal.go | 1 + 12 files changed, 62 insertions(+), 54 deletions(-) diff --git a/cmd/apply.go b/cmd/apply.go index fb668180..1561b2ad 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -139,6 +139,7 @@ func runApply(cmd *cobra.Command, args []string) error { return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "marshal effective intent: "+err.Error()) } intentHash := apply.EffectiveIntentHash(canonicalIntent) + // Transitional compatibility for pre-v2 state; remove after migration in the next major. legacyMatch := existing != nil && apply.LegacyIntentHashMatches(rawIntent, canonicalIntent, existing) if legacyMatch { // Preserve the recorded legacy hash for a true no-op. The next diff --git a/cmd/network/create.go b/cmd/network/create.go index e4d9bec8..bc54610d 100644 --- a/cmd/network/create.go +++ b/cmd/network/create.go @@ -28,6 +28,8 @@ var ( ) var createApply = apply.Apply + +// createTarget is a test injection seam; production default resolves the target. var createTarget = func(parsed *intent.Intent) (target.Target, error) { switch parsed.Target.Type { case "ssh": diff --git a/cmd/remove.go b/cmd/remove.go index 56b55013..7cdd8ba6 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -16,6 +16,7 @@ var ( removeConfirm string ) +// saveRemoveState is a test injection seam; production default persists the current store state. var saveRemoveState = func(nc *nodeContext) error { return nc.Store.Save(nc.State) } var removeCmd = &cobra.Command{ @@ -80,7 +81,7 @@ func runRemove(cmd *cobra.Command, args []string) error { } nc.Store.RemoveNode(nc.State, name) - if err := persistRemoveState(name, nc, start); err != nil { + if err := persistNodeState("remove", name, nc, start, saveRemoveState); err != nil { return err } writeAudit(auditEvent{Command: "remove", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -92,11 +93,3 @@ func runRemove(cmd *cobra.Command, args []string) error { }) return nil } - -func persistRemoveState(name string, nc *nodeContext, start time.Time) error { - if err := saveRemoveState(nc); err != nil { - writeAudit(auditEvent{Command: "remove", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) - return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) - } - return nil -} diff --git a/cmd/restart.go b/cmd/restart.go index 6e30146d..2808857f 100644 --- a/cmd/restart.go +++ b/cmd/restart.go @@ -16,6 +16,7 @@ var restartCmd = &cobra.Command{ RunE: runRestart, } +// saveRestartState is a test injection seam; production default persists via nodeContext.SaveState. var saveRestartState = func(nc *nodeContext) error { return nc.SaveState() } func init() { @@ -53,7 +54,7 @@ func runRestart(cmd *cobra.Command, args []string) error { } nc.Node.Status = "running" - if err := persistRestartState(name, nc, start); err != nil { + if err := persistNodeState("restart", name, nc, start, saveRestartState); err != nil { return err } writeAudit(auditEvent{Command: "restart", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -61,11 +62,3 @@ func runRestart(cmd *cobra.Command, args []string) error { writeResult(map[string]any{"name": name, "status": "running"}) return nil } - -func persistRestartState(name string, nc *nodeContext, start time.Time) error { - if err := saveRestartState(nc); err != nil { - writeAudit(auditEvent{Command: "restart", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) - return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) - } - return nil -} diff --git a/cmd/rollback.go b/cmd/rollback.go index 8742289e..c34fe0a1 100644 --- a/cmd/rollback.go +++ b/cmd/rollback.go @@ -21,6 +21,7 @@ var rollbackCmd = &cobra.Command{ RunE: runRollback, } +// saveRollbackState is a test injection seam; production default persists via nodeContext.SaveState. var saveRollbackState = func(nc *nodeContext) error { return nc.SaveState() } func init() { @@ -88,7 +89,7 @@ func runRollback(cmd *cobra.Command, args []string) error { nc.Node.Version = targetVersion nc.Node.PreviousVersion = currentVersion nc.Node.Status = "running" - if err := persistRollbackState(name, nc, start); err != nil { + if err := persistNodeState("rollback", name, nc, start, saveRollbackState); err != nil { return err } writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -102,14 +103,6 @@ func runRollback(cmd *cobra.Command, args []string) error { return nil } -func persistRollbackState(name string, nc *nodeContext, start time.Time) error { - if err := saveRollbackState(nc); err != nil { - writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) - return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) - } - return nil -} - func rollbackArtifact(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { cause = fmt.Sprintf("%s; stop failed: %v", cause, err) diff --git a/cmd/snapshot/download.go b/cmd/snapshot/download.go index a8047878..48efde0d 100644 --- a/cmd/snapshot/download.go +++ b/cmd/snapshot/download.go @@ -315,6 +315,7 @@ func managedNodeDestinationMatches(dest string, node *state.ManagedNode) bool { // Compose is generated in a stable, one-volume-per-line form; parse only the // volume targeting java-tron's data directory. func legacyDockerStorageRoot(name string) (string, bool) { + // Transitional fallback for pre-storage_root state; remove after all nodes are re-applied (next major). compose := filepath.Join(paths.Deployments(), name, "docker-compose.yaml") f, err := os.Open(compose) if err != nil { diff --git a/cmd/start.go b/cmd/start.go index e57fd7c8..8db396e1 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -16,7 +16,10 @@ var startCmd = &cobra.Command{ RunE: runStart, } +// saveStartState is a test injection seam; production default persists via nodeContext.SaveState. var saveStartState = func(nc *nodeContext) error { return nc.SaveState() } + +// resolveStartNodeContext is a test injection seam; production default resolves a read-only node context. var resolveStartNodeContext = resolveNodeContext func init() { @@ -44,7 +47,7 @@ func runStart(cmd *cobra.Command, args []string) error { } nc.Node.Status = "running" - if err := persistStartState(name, nc, start); err != nil { + if err := persistNodeState("start", name, nc, start, saveStartState); err != nil { return err } writeAudit(auditEvent{Command: "start", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -53,10 +56,30 @@ func runStart(cmd *cobra.Command, args []string) error { return nil } -func persistStartState(name string, nc *nodeContext, start time.Time) error { - if err := saveStartState(nc); err != nil { - writeAudit(auditEvent{Command: "start", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) +func persistNodeState(command, name string, nc *nodeContext, start time.Time, save func(*nodeContext) error) error { + if err := save(nc); err != nil { + writeAudit(auditEvent{Command: command, Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) } return nil } + +// Compatibility wrappers retain the existing test seams while sharing the implementation. +func persistStartState(name string, nc *nodeContext, start time.Time) error { + return persistNodeState("start", name, nc, start, saveStartState) +} +func persistStopState(name string, nc *nodeContext, start time.Time) error { + return persistNodeState("stop", name, nc, start, saveStopState) +} +func persistRestartState(name string, nc *nodeContext, start time.Time) error { + return persistNodeState("restart", name, nc, start, saveRestartState) +} +func persistRemoveState(name string, nc *nodeContext, start time.Time) error { + return persistNodeState("remove", name, nc, start, saveRemoveState) +} +func persistRollbackState(name string, nc *nodeContext, start time.Time) error { + return persistNodeState("rollback", name, nc, start, saveRollbackState) +} +func persistUpgradeState(name string, nc *nodeContext, start time.Time) error { + return persistNodeState("upgrade", name, nc, start, saveUpgradeState) +} diff --git a/cmd/stop.go b/cmd/stop.go index d952ae40..6d8037ba 100644 --- a/cmd/stop.go +++ b/cmd/stop.go @@ -16,6 +16,7 @@ var stopCmd = &cobra.Command{ RunE: runStop, } +// saveStopState is a test injection seam; production default persists via nodeContext.SaveState. var saveStopState = func(nc *nodeContext) error { return nc.SaveState() } func init() { @@ -43,7 +44,7 @@ func runStop(cmd *cobra.Command, args []string) error { } nc.Node.Status = "stopped" - if err := persistStopState(name, nc, start); err != nil { + if err := persistNodeState("stop", name, nc, start, saveStopState); err != nil { return err } writeAudit(auditEvent{Command: "stop", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -51,11 +52,3 @@ func runStop(cmd *cobra.Command, args []string) error { writeResult(map[string]any{"name": name, "status": "stopped"}) return nil } - -func persistStopState(name string, nc *nodeContext, start time.Time) error { - if err := saveStopState(nc); err != nil { - writeAudit(auditEvent{Command: "stop", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) - return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) - } - return nil -} diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 4c52937b..9072234a 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -24,6 +24,7 @@ On failure, automatically rolls back to the previous version.`, RunE: runUpgrade, } +// saveUpgradeState is a test injection seam; production default persists via nodeContext.SaveState. var saveUpgradeState = func(nc *nodeContext) error { return nc.SaveState() } func init() { @@ -95,7 +96,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error { nc.Node.PreviousVersion = previousVersion nc.Node.Version = upgradeVersion nc.Node.Status = "running" - if err := persistUpgradeState(name, nc, start); err != nil { + if err := persistNodeState("upgrade", name, nc, start, saveUpgradeState); err != nil { return err } writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -109,14 +110,6 @@ func runUpgrade(cmd *cobra.Command, args []string) error { return nil } -func persistUpgradeState(name string, nc *nodeContext, start time.Time) error { - if err := saveUpgradeState(nc); err != nil { - writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "STATE_ERROR", Start: start}) - return exitWithError("STATE_ERROR", output.ExitGeneralError, fmt.Sprintf("Failed to persist %s state: %v", name, err)) - } - return nil -} - func rollbackUpgrade(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { cause = fmt.Sprintf("%s; stop failed: %v", cause, err) diff --git a/internal/mcp/contract_p1_test.go b/internal/mcp/contract_p1_test.go index 04ceb21b..42adccc4 100644 --- a/internal/mcp/contract_p1_test.go +++ b/internal/mcp/contract_p1_test.go @@ -178,7 +178,10 @@ func TestApplyToolUsesConfiguredTemplateForPlanAndDeploy(t *testing.T) { // The template loader requires the matching private-network file. The // marker makes the rendered/deployed config observably external-template based. marker := "# external-template-marker" - template := marker + "\nnode {\n metrics {\n prometheus = false\n }\n}\n" + // rpc block required: ports.solidity_grpc render now errors when the template + // has no rpc block (AUD-024). Keep the default value so the rendered output + // stays byte-identical to this template. + template := marker + "\nnode {\n metrics {\n prometheus = false\n }\n}\nrpc {\n port = 50051\n solidityPort = 50061\n}\n" if err := os.WriteFile(templateDir+"/private_net_config.conf", []byte(template), 0600); err != nil { t.Fatal(err) } @@ -196,14 +199,26 @@ func TestApplyToolUsesConfiguredTemplateForPlanAndDeploy(t *testing.T) { } res, value, err := applyTool(context.Background(), nil, applyArgs{Path: p, AutoApprove: true}) if err != nil || res == nil || res.IsError { - t.Fatalf("apply result=%v err=%v", res, err) + t.Fatalf("apply result=%v err=%v body=%s", res, err, extractText(t, res)) } got, ok := value.(*apply.Result) if !ok { t.Fatalf("result type=%T", value) } - if got.ConfigHash != apply.IntentHashFromBytes([]byte(template)) { - t.Fatalf("config hash=%q, external template base hash=%q", got.ConfigHash, apply.IntentHashFromBytes([]byte(template))) + parsed, lerr := intent.Load(p) + if lerr != nil { + t.Fatal(lerr) + } + rendered, rerr := render.RenderHOCONWithSecrets(templateDir, parsed, &parsed.Nodes[0]) + if rerr != nil { + t.Fatal(rerr) + } + if !strings.Contains(rendered.Deployable(), marker) { + t.Fatal("rendered config missing external template marker") + } + want := apply.IntentHashFromBytes([]byte(rendered.Deployable())) + if got.ConfigHash != want { + t.Fatalf("config hash=%q, external template rendered hash=%q", got.ConfigHash, want) } if data, err := os.ReadFile(dockerCalls); err != nil || len(data) == 0 { t.Fatalf("deploy did not use docker runtime: err=%v calls=%q", err, data) diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index c77fa29f..b0d36710 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -63,7 +63,7 @@ func newConnectedPair(t *testing.T) (*mcp.ClientSession, func()) { // markdown body, for knowledge_get). Non-text content fails the test. func extractText(t *testing.T, res *mcp.CallToolResult) string { t.Helper() - if len(res.Content) == 0 { + if res == nil || len(res.Content) == 0 { t.Fatal("no content in tool result") } tc, ok := res.Content[0].(*mcp.TextContent) diff --git a/internal/mcp/tools_heal.go b/internal/mcp/tools_heal.go index 72ffeb47..c37733a6 100644 --- a/internal/mcp/tools_heal.go +++ b/internal/mcp/tools_heal.go @@ -35,6 +35,7 @@ var saveHealState = func(store *state.Store, st *state.DeploymentState) error { var runHealAction = mcpRunHealAction var healCheckers = diagnosis.AllCheckers +// healRuntimeForNode is a test injection seam; production default selects jar or Docker runtime. var healRuntimeForNode = func(tgt target.Target, node *state.ManagedNode) runtime.Runtime { if node.Runtime == "jar" { return runtime.NewJarRuntime(tgt) From abd51cb97caddf3d5c1347319f3111f7b9af6e29 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 15:01:52 +0800 Subject: [PATCH 27/49] refactor(apply): centralize endpoint and port helpers Domain 1 of the pre-existing-duplication consolidation: gather the scattered port fallback (8090/50051), endpoint URL, and probe URL assembly into internal/apply/endpoints.go (PortOrDefault, HTTPURL, GRPCAddr, ProbeURL) and rewire CLI, diagnosis, and MCP call sites to it. Pure refactor: no behavior change, no existing test modified (oracle-reviewed). --- cmd/status.go | 11 ++--------- internal/apply/endpoints.go | 18 ++++++++++++++++++ internal/apply/endpoints_test.go | 21 +++++++++++++++++++++ internal/apply/probe.go | 7 ++----- internal/apply/wait.go | 5 +---- internal/diagnosis/peers.go | 7 ++----- internal/diagnosis/ports.go | 2 ++ internal/diagnosis/sync.go | 7 ++----- internal/diagnosis/version.go | 7 ++----- internal/mcp/helpers.go | 15 --------------- internal/mcp/resources.go | 5 +++-- internal/mcp/tools_diagnostic.go | 10 ++++------ internal/mcp/tools_inspection.go | 4 ++-- 13 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 internal/apply/endpoints.go create mode 100644 internal/apply/endpoints_test.go diff --git a/cmd/status.go b/cmd/status.go index dc4ed2e2..19865fe9 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -89,8 +89,8 @@ func runStatus(cmd *cobra.Command, args []string) error { "healthy": false, "logs": apply.LogsDescriptor(node), "api_endpoints": map[string]any{ - "http": fmt.Sprintf("http://%s:%d", target.EndpointHost(node.Target.Type, node.Target.Host), effectivePort(node.HTTPPort, 8090)), - "grpc": fmt.Sprintf("%s:%d", target.EndpointHost(node.Target.Type, node.Target.Host), effectivePort(node.GRPCPort, 50051)), + "http": apply.HTTPURL(target.EndpointHost(node.Target.Type, node.Target.Host), apply.PortOrDefault(node.HTTPPort, 8090)), + "grpc": apply.GRPCAddr(target.EndpointHost(node.Target.Type, node.Target.Host), apply.PortOrDefault(node.GRPCPort, 50051)), }, } @@ -188,13 +188,6 @@ func runStatus(cmd *cobra.Command, args []string) error { return nil } -func effectivePort(stored, fallback int) int { - if stored != 0 { - return stored - } - return fallback -} - // probeMonitoringStatus returns a short health status string for the // monitoring stack deployed alongside the named node. Best-effort — // returns "unknown" when the stack can't be reached. diff --git a/internal/apply/endpoints.go b/internal/apply/endpoints.go new file mode 100644 index 00000000..e2aa4309 --- /dev/null +++ b/internal/apply/endpoints.go @@ -0,0 +1,18 @@ +package apply + +import "fmt" + +// PortOrDefault returns port when non-zero, otherwise the caller's fallback. +// State-vs-intent precedence is selected by each caller before this helper. +func PortOrDefault(port, fallback int) int { + if port != 0 { + return port + } + return fallback +} + +func HTTPURL(host string, port int) string { return fmt.Sprintf("http://%s:%d", host, port) } +func GRPCAddr(host string, port int) string { return fmt.Sprintf("%s:%d", host, port) } +func ProbeURL(port int, path string) string { + return fmt.Sprintf("http://127.0.0.1:%d%s", PortOrDefault(port, 8090), path) +} diff --git a/internal/apply/endpoints_test.go b/internal/apply/endpoints_test.go new file mode 100644 index 00000000..eeaae2a0 --- /dev/null +++ b/internal/apply/endpoints_test.go @@ -0,0 +1,21 @@ +package apply + +import "testing" + +func TestEndpointHelpers(t *testing.T) { + if got := PortOrDefault(0, 8090); got != 8090 { + t.Fatalf("default=%d", got) + } + if got := PortOrDefault(9000, 8090); got != 9000 { + t.Fatalf("stored=%d", got) + } + if got := HTTPURL("host", 8090); got != "http://host:8090" { + t.Fatalf("http=%q", got) + } + if got := GRPCAddr("host", 50051); got != "host:50051" { + t.Fatalf("grpc=%q", got) + } + if got := ProbeURL(0, "/wallet/getnowblock"); got != "http://127.0.0.1:8090/wallet/getnowblock" { + t.Fatalf("probe=%q", got) + } +} diff --git a/internal/apply/probe.go b/internal/apply/probe.go index eeccb49b..f5de7843 100644 --- a/internal/apply/probe.go +++ b/internal/apply/probe.go @@ -30,10 +30,7 @@ func LiveStatus(ctx context.Context, tgt target.Target, node *state.ManagedNode) if tgt == nil || node == nil { return out } - port := node.HTTPPort - if port == 0 { - port = 8090 - } + port := PortOrDefault(node.HTTPPort, 8090) // probe issues a request against the node's HTTP API. body=="" is a // GET (TRON endpoints that take no params, e.g. getnowblock); a @@ -42,7 +39,7 @@ func LiveStatus(ctx context.Context, tgt target.Target, node *state.ManagedNode) // (SSH-tunnelled for remote targets); docker nodes curl inside the // container via `docker exec`. probe := func(path, body string) ([]byte, error) { - url := fmt.Sprintf("http://127.0.0.1:%d%s", port, path) + url := ProbeURL(port, path) if node.Runtime == "jar" { if body == "" { return target.Get(ctx, tgt, url, 2*time.Second) diff --git a/internal/apply/wait.go b/internal/apply/wait.go index e42999b8..05a27087 100644 --- a/internal/apply/wait.go +++ b/internal/apply/wait.go @@ -19,10 +19,7 @@ import ( func WaitForReady(ctx context.Context, tgt target.Target, name, runtimeName string, httpPort int, timeout time.Duration) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - if httpPort == 0 { - httpPort = 8090 - } - url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnowblock", httpPort) + url := ProbeURL(httpPort, "/wallet/getnowblock") tick := time.NewTicker(2 * time.Second) defer tick.Stop() var lastErr error diff --git a/internal/diagnosis/peers.go b/internal/diagnosis/peers.go index 28768448..a9260694 100644 --- a/internal/diagnosis/peers.go +++ b/internal/diagnosis/peers.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -15,11 +16,7 @@ type PeersChecker struct{} func (c *PeersChecker) Name() string { return "peer_count" } func (c *PeersChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpts) CheckResult { - if opts.HTTPPort == 0 { - opts.HTTPPort = 8090 - } - - url := fmt.Sprintf("http://127.0.0.1:%d/wallet/listnodes", opts.HTTPPort) + url := apply.ProbeURL(opts.HTTPPort, "/wallet/listnodes") out, err := target.Get(ctx, tgt, url, 5*time.Second) if err != nil { return CheckResult{ diff --git a/internal/diagnosis/ports.go b/internal/diagnosis/ports.go index 418ea54c..db82a9d0 100644 --- a/internal/diagnosis/ports.go +++ b/internal/diagnosis/ports.go @@ -23,6 +23,8 @@ type PortsChecker struct{} func (c *PortsChecker) Name() string { return "port_listening" } func (c *PortsChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpts) CheckResult { + // Preserve the original semantics exactly: defaults apply only when + // BOTH ports are unset; a partially specified pair is checked as-is. ports := []int{opts.HTTPPort, opts.GRPCPort} if opts.HTTPPort == 0 && opts.GRPCPort == 0 { ports = []int{8090, 50051} diff --git a/internal/diagnosis/sync.go b/internal/diagnosis/sync.go index cc96f066..fe375037 100644 --- a/internal/diagnosis/sync.go +++ b/internal/diagnosis/sync.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -16,11 +17,7 @@ type SyncChecker struct{} func (c *SyncChecker) Name() string { return "sync_progress" } func (c *SyncChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpts) CheckResult { - if opts.HTTPPort == 0 { - opts.HTTPPort = 8090 - } - - url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnowblock", opts.HTTPPort) + url := apply.ProbeURL(opts.HTTPPort, "/wallet/getnowblock") out, err := target.Get(ctx, tgt, url, 5*time.Second) if err != nil { return CheckResult{ diff --git a/internal/diagnosis/version.go b/internal/diagnosis/version.go index d874d988..64d7a921 100644 --- a/internal/diagnosis/version.go +++ b/internal/diagnosis/version.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -17,11 +18,7 @@ func (c *VersionChecker) Name() string { return "version_check" } func (c *VersionChecker) Run(ctx context.Context, tgt target.Target, opts CheckOpts) CheckResult { // Get node version from API - if opts.HTTPPort == 0 { - opts.HTTPPort = 8090 - } - - url := fmt.Sprintf("http://127.0.0.1:%d/wallet/getnodeinfo", opts.HTTPPort) + url := apply.ProbeURL(opts.HTTPPort, "/wallet/getnodeinfo") out, err := target.Get(ctx, tgt, url, 5*time.Second) if err != nil { return CheckResult{ diff --git a/internal/mcp/helpers.go b/internal/mcp/helpers.go index be1b9655..8c1a0636 100644 --- a/internal/mcp/helpers.go +++ b/internal/mcp/helpers.go @@ -2,7 +2,6 @@ package mcp import ( "fmt" - "strconv" "github.com/tronprotocol/tron-deployment/internal/output" ) @@ -28,17 +27,3 @@ func notFoundWithSuggestions(resource, name string, suggestions ...string) *outp fmt.Sprintf("%s %q not found", resource, name)). WithSuggestions(suggestions...) } - -// httpURL formats host+port into the http://:

URL we surface -// to agents. Agents can re-use this in their own follow-up probes -// (e.g. `wait --http `). For ssh targets the host is the recorded -// remote host; internal probes that dial through the target pass -// 127.0.0.1 (the tunnel lands on the remote loopback). -func httpURL(host string, port int) string { - return "http://" + host + ":" + strconv.Itoa(port) -} - -// grpcAddr formats the host:port grpc endpoint. -func grpcAddr(host string, port int) string { - return host + ":" + strconv.Itoa(port) -} diff --git a/internal/mcp/resources.go b/internal/mcp/resources.go index 00608123..c025f3e5 100644 --- a/internal/mcp/resources.go +++ b/internal/mcp/resources.go @@ -9,6 +9,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" @@ -147,8 +148,8 @@ func readNodeEndpointsResource(_ context.Context, req *mcp.ReadResourceRequest) "runtime": node.Runtime, "target": node.Target, "endpoints": map[string]string{ - "http": fmt.Sprintf("http://%s:%d", host, node.HTTPPort), - "grpc": fmt.Sprintf("%s:%d", host, node.GRPCPort), + "http": apply.HTTPURL(host, node.HTTPPort), + "grpc": apply.GRPCAddr(host, node.GRPCPort), }, "version": node.Version, "labels": node.Labels, diff --git a/internal/mcp/tools_diagnostic.go b/internal/mcp/tools_diagnostic.go index 46368f76..a4b997fe 100644 --- a/internal/mcp/tools_diagnostic.go +++ b/internal/mcp/tools_diagnostic.go @@ -11,6 +11,7 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/diagnosis" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" @@ -168,14 +169,11 @@ func healthTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp if node == nil { return errResult(notFound("health", args.Name)) } - port := node.HTTPPort - if port == 0 { - port = 8090 - } + port := apply.PortOrDefault(node.HTTPPort, 8090) // The probe URL stays on loopback: target.HTTPClient dials through the // SSH tunnel, which lands on the remote host's loopback. - url := httpURL("127.0.0.1", port) + "/wallet/getnowblock" - endpoint := httpURL(target.EndpointHost(node.Target.Type, node.Target.Host), port) + "/wallet/getnowblock" + url := apply.ProbeURL(port, "/wallet/getnowblock") + endpoint := apply.HTTPURL(target.EndpointHost(node.Target.Type, node.Target.Host), port) + "/wallet/getnowblock" // Probe through the node's target (SSH-tunnelled for remote nodes), // not via http.DefaultClient against this host's loopback — that diff --git a/internal/mcp/tools_inspection.go b/internal/mcp/tools_inspection.go index 3e29181e..33f7b883 100644 --- a/internal/mcp/tools_inspection.go +++ b/internal/mcp/tools_inspection.go @@ -205,10 +205,10 @@ func inspectAllNodes(ctx context.Context, _ *mcp.CallToolRequest, _ emptyArgs) ( // extraction. eps := map[string]string{} if n.HTTPPort != 0 { - eps["http"] = httpURL(target.EndpointHost(n.Target.Type, n.Target.Host), n.HTTPPort) + eps["http"] = apply.HTTPURL(target.EndpointHost(n.Target.Type, n.Target.Host), n.HTTPPort) } if n.GRPCPort != 0 { - eps["grpc"] = grpcAddr(target.EndpointHost(n.Target.Type, n.Target.Host), n.GRPCPort) + eps["grpc"] = apply.GRPCAddr(target.EndpointHost(n.Target.Type, n.Target.Host), n.GRPCPort) } if len(eps) > 0 { row["endpoints"] = eps From 3030ffed3dab1f803ec368de12c20e6d75f768f8 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 15:32:58 +0800 Subject: [PATCH 28/49] refactor(mcp,cmd): converge state load + target resolution helpers (#5/#6) Add state.Load/state.LoadNode and target.FromIntent/target.FromManagedNode as the single implementations; MCP tools and cmd/resolve.go drop their inlined NewStore/Load/GetNode and mcpResolveTargetFromNode copies. resources.go and doctorStateFile keep hand-written two-stage error handling to preserve the external error contract. No behavior change; existing tests unchanged. --- cmd/resolve.go | 13 ++++-------- internal/mcp/resources.go | 2 +- internal/mcp/tools_diagnostic.go | 21 +++++-------------- internal/mcp/tools_drift.go | 10 +++------ internal/mcp/tools_heal.go | 8 ++----- internal/mcp/tools_inspection.go | 36 ++++---------------------------- internal/mcp/tools_lifecycle.go | 23 +++----------------- internal/state/load.go | 25 ++++++++++++++++++++++ internal/target/resolve.go | 30 ++++++++++++++++++++++++++ 9 files changed, 77 insertions(+), 91 deletions(-) create mode 100644 internal/state/load.go create mode 100644 internal/target/resolve.go diff --git a/cmd/resolve.go b/cmd/resolve.go index 5f227d3f..0dd412d1 100644 --- a/cmd/resolve.go +++ b/cmd/resolve.go @@ -188,16 +188,11 @@ func resolveNode(name string, forWrite bool) (*nodeContext, error) { } func resolveTargetFromNode(node *state.ManagedNode) (target.Target, error) { - switch node.Target.Type { - case "ssh": - t := target.NewSSHTarget(node.Target.Host, node.Target.Port, node.Target.User, node.Target.IdentityFile) - if err := t.Connect(); err != nil { - return nil, fmt.Errorf("ssh connect to %s: %w", node.Target.Host, err) - } - return t, nil - default: - return target.NewLocalTarget(), nil + t, err := target.FromManagedNode(node) + if err != nil && node.Target.Type == "ssh" { + return nil, fmt.Errorf("ssh connect to %s: %w", node.Target.Host, err) } + return t, err } func resolveRuntimeForNode(node *state.ManagedNode, tgt target.Target) runtime.Runtime { diff --git a/internal/mcp/resources.go b/internal/mcp/resources.go index c025f3e5..2fec3600 100644 --- a/internal/mcp/resources.go +++ b/internal/mcp/resources.go @@ -176,7 +176,7 @@ func readNodeConfResource(ctx context.Context, req *mcp.ReadResourceRequest) (*m if err != nil { return nil, err } - tgt, err := mcpResolveTargetFromNode(node) + tgt, err := target.FromManagedNode(node) if err != nil { return nil, err } diff --git a/internal/mcp/tools_diagnostic.go b/internal/mcp/tools_diagnostic.go index a4b997fe..e8bc6679 100644 --- a/internal/mcp/tools_diagnostic.go +++ b/internal/mcp/tools_diagnostic.go @@ -122,10 +122,7 @@ func doctorStateFile() map[string]any { } st, err := store.Load() if err != nil { - return map[string]any{ - "name": "state.json", "status": "fail", - "message": "parse: " + err.Error(), - } + return map[string]any{"name": "state.json", "status": "fail", "message": "parse: " + err.Error()} } return map[string]any{ "name": "state.json", @@ -157,11 +154,7 @@ func doctorDockerCLI(ctx context.Context) map[string]any { } func healthTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp.CallToolResult, any, error) { - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(err) - } - st, err := store.Load() + store, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } @@ -178,7 +171,7 @@ func healthTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp // Probe through the node's target (SSH-tunnelled for remote nodes), // not via http.DefaultClient against this host's loopback — that // bypassed SSH entirely and always failed for remote rigs. - tgt, err := mcpResolveTargetFromNode(node) + tgt, err := target.FromManagedNode(node) if err != nil { return errResult(err) } @@ -217,11 +210,7 @@ func diagnoseTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*m // `trond diagnose ` does, by reaching into // internal/diagnosis directly. The earlier MCP version returned // only a state-only subset; that gap is now closed. - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(err) - } - st, err := store.Load() + store, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } @@ -230,7 +219,7 @@ func diagnoseTool(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*m return errResult(notFound("diagnose", args.Name)) } - tgt, err := mcpResolveTargetFromNode(node) + tgt, err := target.FromManagedNode(node) if err != nil { return errResult(err) } diff --git a/internal/mcp/tools_drift.go b/internal/mcp/tools_drift.go index f1d43cdc..64815c8a 100644 --- a/internal/mcp/tools_drift.go +++ b/internal/mcp/tools_drift.go @@ -12,6 +12,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) // registerDriftTools wires the verify_config tool — the MCP-side @@ -47,20 +48,15 @@ func verifyConfigTool(ctx context.Context, _ *mcp.CallToolRequest, args verifyCo return errResult(err) } - store, err := state.NewStore(paths.State()) + _, _, node, err := state.LoadNode(paths.State(), args.Name) if err != nil { return errResult(err) } - st, err := store.Load() - if err != nil { - return errResult(err) - } - node := store.GetNode(st, args.Name) if node == nil { return errResult(notFound("verify_config", args.Name)) } - tgt, err := mcpResolveTargetFromNode(node) + tgt, err := target.FromManagedNode(node) if err != nil { return errResult(err) } diff --git a/internal/mcp/tools_heal.go b/internal/mcp/tools_heal.go index c37733a6..a6c70699 100644 --- a/internal/mcp/tools_heal.go +++ b/internal/mcp/tools_heal.go @@ -69,11 +69,7 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs } defer lock.Release() - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(err) - } - st, err := store.Load() + store, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } @@ -98,7 +94,7 @@ func autoHealTool(ctx context.Context, _ *mcp.CallToolRequest, args autoHealArgs } } - tgt, err := mcpResolveTargetFromNode(node) + tgt, err := target.FromManagedNode(node) if err != nil { return errResult(err) } diff --git a/internal/mcp/tools_inspection.go b/internal/mcp/tools_inspection.go index 33f7b883..f8a950fd 100644 --- a/internal/mcp/tools_inspection.go +++ b/internal/mcp/tools_inspection.go @@ -61,15 +61,10 @@ func registerInspectionTools(s *mcp.Server) { } func listNodes(ctx context.Context, _ *mcp.CallToolRequest, _ emptyArgs) (*mcp.CallToolResult, any, error) { - store, err := state.NewStore(paths.State()) + _, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } - st, err := store.Load() - if err != nil { - return errResult(err) - } - // Reshape into the same JSON we'd emit from `trond list -o json`. // Schema: schemas/output/list.schema.json — a flat array, not a // {nodes: [...]} object. Matching the CLI shape lets MCP-aware @@ -97,11 +92,7 @@ func listNodes(ctx context.Context, _ *mcp.CallToolRequest, _ emptyArgs) (*mcp.C } func statusForNode(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (*mcp.CallToolResult, any, error) { - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(err) - } - st, err := store.Load() + store, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } @@ -137,7 +128,7 @@ func statusForNode(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (* // piggybacked on the running-node resolution. Errors are dropped. needLocalDockerID := node.Runtime == "docker" && node.Target.Type == "local" if node.Status == "running" || needLocalDockerID { - if tgt, err := mcpResolveTargetFromNode(node); err == nil { + if tgt, err := target.FromManagedNode(node); err == nil { if c, ok := any(tgt).(interface{ Close() error }); ok { defer c.Close() } @@ -158,27 +149,8 @@ func statusForNode(ctx context.Context, _ *mcp.CallToolRequest, args nodeArg) (* return jsonResult(out) } -// mcpResolveTargetFromNode mirrors cmd/resolve.go::resolveTargetFromNode. -// Duplicated so internal/mcp doesn't import cmd/. -func mcpResolveTargetFromNode(node *state.ManagedNode) (target.Target, error) { - switch node.Target.Type { - case "ssh": - t := target.NewSSHTarget(node.Target.Host, node.Target.Port, node.Target.User, node.Target.IdentityFile) - if err := t.Connect(); err != nil { - return nil, err - } - return t, nil - default: - return target.NewLocalTarget(), nil - } -} - func inspectAllNodes(ctx context.Context, _ *mcp.CallToolRequest, _ emptyArgs) (*mcp.CallToolResult, any, error) { - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(err) - } - st, err := store.Load() + _, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } diff --git a/internal/mcp/tools_lifecycle.go b/internal/mcp/tools_lifecycle.go index 4b0d815b..a3909d18 100644 --- a/internal/mcp/tools_lifecycle.go +++ b/internal/mcp/tools_lifecycle.go @@ -70,11 +70,7 @@ func planTool(ctx context.Context, _ *mcp.CallToolRequest, args planArg) (*mcp.C return errResult(err) } node := &parsed.Nodes[0] - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error())) - } - st, err := store.Load() + store, st, err := state.Load(paths.State()) if err != nil { return errResult(output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error())) } @@ -161,11 +157,7 @@ func applyTool(ctx context.Context, _ *mcp.CallToolRequest, args applyArgs) (*mc } defer lock.Release() - store, err := state.NewStore(paths.State()) - if err != nil { - return errResult(err) - } - st, err := store.Load() + store, st, err := state.Load(paths.State()) if err != nil { return errResult(err) } @@ -254,16 +246,7 @@ func enforcePrivateForRecordedNode(requested bool, name string) error { // internal/mcp package doesn't import cmd/. Limited to local + ssh // since those are the only two intent.Target.Type values. func resolveTarget(parsed *intent.Intent) (target.Target, error) { - switch parsed.Target.Type { - case "ssh": - t := target.NewSSHTarget(parsed.Target.Host, parsed.Target.Port, parsed.Target.User, parsed.Target.IdentityFile) - if err := t.Connect(); err != nil { - return nil, err - } - return t, nil - default: - return target.NewLocalTarget(), nil - } + return target.FromIntent(parsed) } // resolveEnvVars mirrors cmd/apply.go's helper for the same reason. diff --git a/internal/state/load.go b/internal/state/load.go new file mode 100644 index 00000000..17d74beb --- /dev/null +++ b/internal/state/load.go @@ -0,0 +1,25 @@ +package state + +// LoadNode opens path, loads deployment state, and returns the named node. +// Missing nodes are returned as a nil node without an error. +func LoadNode(path, name string) (*Store, *DeploymentState, *ManagedNode, error) { + store, st, err := Load(path) + if err != nil { + return nil, nil, nil, err + } + return store, st, store.GetNode(st, name), nil +} + +// Load opens a Store and loads its deployment state without acquiring a lock. +// Callers that mutate state must retain their existing write-lock discipline. +func Load(path string) (*Store, *DeploymentState, error) { + store, err := NewStore(path) + if err != nil { + return nil, nil, err + } + st, err := store.Load() + if err != nil { + return store, nil, err + } + return store, st, nil +} diff --git a/internal/target/resolve.go b/internal/target/resolve.go new file mode 100644 index 00000000..0a8b0748 --- /dev/null +++ b/internal/target/resolve.go @@ -0,0 +1,30 @@ +package target + +import ( + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +// FromIntent resolves the target declared by an intent. The caller owns Close. +func FromIntent(i *intent.Intent) (Target, error) { + if i.Target.Type == "ssh" { + t := NewSSHTarget(i.Target.Host, i.Target.Port, i.Target.User, i.Target.IdentityFile) + if err := t.Connect(); err != nil { + return nil, err + } + return t, nil + } + return NewLocalTarget(), nil +} + +// FromManagedNode resolves the target recorded in managed-node state. The caller owns Close. +func FromManagedNode(n *state.ManagedNode) (Target, error) { + if n.Target.Type == "ssh" { + t := NewSSHTarget(n.Target.Host, n.Target.Port, n.Target.User, n.Target.IdentityFile) + if err := t.Connect(); err != nil { + return nil, err + } + return t, nil + } + return NewLocalTarget(), nil +} From 1406619971ae2d46e8214386ba2b2f73d2cbaa2b Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 16:19:22 +0800 Subject: [PATCH 29/49] refactor(render,apply): converge diff algorithms and env/lookup helpers (#11) - render.DiffLines/DiffText: single positional HOCON diff core with whole-slice witness redaction; replaces simpleDiff, simpleHOCONDiff, lineDiff, mcpLineDiff (output contracts preserved per caller). - apply.ResolveEnvVars: single legacy witness-key env resolver; replaces duplicated copies in cmd/apply.go and mcp tools_lifecycle. - Drop thin shells: cmd findTemplatesDir (call apply.FindTemplatesDir directly) and mcp redactConfText (call render.RedactWitnessLines). - Tests updated mechanically at call sites only; assertions unchanged. --- cmd/apply.go | 15 +----- cmd/config/diff.go | 45 +----------------- cmd/config/witness_redaction_test.go | 5 +- cmd/plan.go | 46 +------------------ cmd/plan_test.go | 7 +-- cmd/verify_config.go | 69 ++-------------------------- cmd/verify_config_test.go | 3 +- cmd/witness_redaction_test.go | 17 +++---- internal/apply/envvars.go | 17 +++++++ internal/mcp/conf_helpers.go | 15 ------ internal/mcp/conf_helpers_test.go | 6 ++- internal/mcp/conf_redaction_test.go | 8 ++-- internal/mcp/drift_redaction_test.go | 9 ++-- internal/mcp/resources.go | 3 +- internal/mcp/tools_drift.go | 58 +---------------------- internal/mcp/tools_lifecycle.go | 14 +----- internal/render/diff.go | 55 ++++++++++++++++++++++ 17 files changed, 118 insertions(+), 274 deletions(-) create mode 100644 internal/apply/envvars.go create mode 100644 internal/render/diff.go diff --git a/cmd/apply.go b/cmd/apply.go index 1561b2ad..7cc617f9 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -167,7 +167,7 @@ func runApply(cmd *cobra.Command, args []string) error { Existing: existing, TemplateDir: apply.FindTemplatesDir(), DeploymentsDir: deploymentsDir(), - EnvVars: resolveEnvVars(&parsed.Nodes[0]), + EnvVars: apply.ResolveEnvVars(&parsed.Nodes[0]), IntentPath: applyIntentPath, // FR-021: relative build.source resolves vs this Wait: applyWait, WaitTimeout: applyWaitTimeout, @@ -229,19 +229,6 @@ func resolveTarget(parsed *intent.Intent) (target.Target, error) { } } -func resolveEnvVars(node *intent.NodeSpec) map[string]string { - env := make(map[string]string) - if node.WitnessKeyEnv != "" { - val := os.Getenv(node.WitnessKeyEnv) - if val != "" { - env[node.WitnessKeyEnv] = val - } - } - return env -} - -func findTemplatesDir() string { return apply.FindTemplatesDir() } - // exitWithError returns a StructuredError for propagation through cobra RunE. func exitWithError(code string, exitCode int, msg string, suggestions ...string) error { return output.NewError(code, exitCode, msg).WithSuggestions(suggestions...) diff --git a/cmd/config/diff.go b/cmd/config/diff.go index 13a40b0d..a8af26a3 100644 --- a/cmd/config/diff.go +++ b/cmd/config/diff.go @@ -44,7 +44,7 @@ func runDiff(cmd *cobra.Command, args []string) error { return output.NewError("RENDER_ERROR", output.ExitGeneralError, err.Error()) } // Compare the REAL bytes against the deployed file so a rotated - // witness key still shows up as a difference; simpleDiff redacts + // witness key still shows up as a difference; render.DiffLines redacts // each line as it emits it, so no key reaches stdout or the JSON. newConfig := rendered.Deployable() @@ -91,7 +91,7 @@ func runDiff(cmd *cobra.Command, args []string) error { oldLines := strings.Split(string(deployedData), "\n") newLines := strings.Split(newConfig, "\n") - diffs := simpleDiff(oldLines, newLines) + diffs := render.DiffLines(oldLines, newLines, 0) if outputFmt == "json" { output.WriteJSON(os.Stdout, map[string]any{ @@ -112,44 +112,3 @@ func runDiff(cmd *cobra.Command, args []string) error { return nil } - -// simpleDiff does a basic line-by-line comparison. -// -// Secret handling: comparison uses the raw lines so genuine drift is -// still reported, but every emitted line goes through -// render.RedactWitnessLine first. The comparison is positional and -// LCS-free, so any line-count change above the `localwitness` -// assignment misaligns the tail and would otherwise print the SR -// private key into `diffs[]`. -func simpleDiff(old, new []string) []string { - // Redact whole-slice: a multi-line `localwitness = [` array keeps its - // key on a line that does not itself start with the key name. - oldR := render.RedactWitnessLines(old) - newR := render.RedactWitnessLines(new) - var diffs []string - - maxLen := len(old) - if len(new) > maxLen { - maxLen = len(new) - } - - for i := 0; i < maxLen; i++ { - var oldLine, newLine string - if i < len(old) { - oldLine = old[i] - } - if i < len(new) { - newLine = new[i] - } - if oldLine != newLine { - if oldLine != "" { - diffs = append(diffs, fmt.Sprintf("- %s", oldR[i])) - } - if newLine != "" { - diffs = append(diffs, fmt.Sprintf("+ %s", newR[i])) - } - } - } - - return diffs -} diff --git a/cmd/config/witness_redaction_test.go b/cmd/config/witness_redaction_test.go index 4d7aba8c..85602435 100644 --- a/cmd/config/witness_redaction_test.go +++ b/cmd/config/witness_redaction_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/render" ) const ( @@ -46,7 +47,7 @@ func TestSimpleDiff_RedactsWitnessKey(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - diffs := simpleDiff(tc.old, tc.new) + diffs := render.DiffLines(tc.old, tc.new, 0) for _, d := range diffs { if strings.Contains(d, cfgKeyA) || strings.Contains(d, cfgKeyB) { t.Errorf("simpleDiff leaked a witness private key: %q", d) @@ -71,7 +72,7 @@ func TestSimpleDiff_RedactsWitnessKey(t *testing.T) { // Redacting before comparing would swallow real drift; redacting // after must not. same := []string{`localwitness = ["` + cfgKeyA + `"]`} - if diffs := simpleDiff(same, same); len(diffs) != 0 { + if diffs := render.DiffLines(same, same, 0); len(diffs) != 0 { t.Errorf("unchanged key must not be reported as drift: %v", diffs) } } diff --git a/cmd/plan.go b/cmd/plan.go index 72e80d9c..e41515c5 100644 --- a/cmd/plan.go +++ b/cmd/plan.go @@ -97,8 +97,8 @@ func runPlan(cmd *cobra.Command, args []string) error { if planShowDiff && existing != nil { deployedPath := filepath.Join(paths.Deployments(), parsed.Name, parsed.Name+".conf") if data, err := os.ReadFile(deployedPath); err == nil { - diffLines = simpleHOCONDiff(strings.Split(string(data), "\n"), - strings.Split(string(planned.Config), "\n")) + diffLines = render.DiffLines(strings.Split(string(data), "\n"), + strings.Split(string(planned.Config), "\n"), 0) } // JSON consumers always get the field (possibly empty array) // so they can distinguish "no changes" from "diff was not @@ -118,48 +118,6 @@ func runPlan(cmd *cobra.Command, args []string) error { return nil } -// simpleHOCONDiff is a deliberately tiny line-by-line differ. Same -// shape as cmd/config/diff.go::simpleDiff but private to plan to -// avoid coupling — both could call into a shared internal/diff -// helper later if a third caller appears. -// -// Secret handling: the comparison runs on the raw lines (so a witness -// key that actually changed is still reported as drift), but every -// line is passed through render.RedactWitnessLine before it is -// emitted. This differ is positional and LCS-free, so a line-count -// change anywhere above the `localwitness` assignment misaligns the -// tail and would otherwise push the SR private key straight into -// stdout and into result["config_diff"]. -func simpleHOCONDiff(old, new []string) []string { - // Redact whole-slice: a multi-line `localwitness = [` array keeps its - // key on a line that does not itself start with the key name. - oldR := render.RedactWitnessLines(old) - newR := render.RedactWitnessLines(new) - var diffs []string - maxLen := len(old) - if len(new) > maxLen { - maxLen = len(new) - } - for i := 0; i < maxLen; i++ { - var oldLine, newLine string - if i < len(old) { - oldLine = old[i] - } - if i < len(new) { - newLine = new[i] - } - if oldLine != newLine { - if oldLine != "" { - diffs = append(diffs, "- "+oldR[i]) - } - if newLine != "" { - diffs = append(diffs, "+ "+newR[i]) - } - } - } - return diffs -} - // printDiffSection renders the line diff under the changes section // in text mode. Stays empty when there's nothing useful to say. func printDiffSection(existing *state.ManagedNode, diffLines []string) { diff --git a/cmd/plan_test.go b/cmd/plan_test.go index f8697811..c5f1e827 100644 --- a/cmd/plan_test.go +++ b/cmd/plan_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" ) @@ -11,7 +12,7 @@ func TestSimpleHOCONDiff_DetectsLineChanges(t *testing.T) { old := []string{"node.ip = 127.0.0.1", "node.port = 8090", "shared = same"} new := []string{"node.ip = 127.0.0.1", "node.port = 9999", "shared = same"} - diffs := simpleHOCONDiff(old, new) + diffs := render.DiffLines(old, new, 0) if len(diffs) != 2 { t.Fatalf("expected 2 diff lines (one - one +), got %d: %v", len(diffs), diffs) } @@ -25,7 +26,7 @@ func TestSimpleHOCONDiff_DetectsLineChanges(t *testing.T) { func TestSimpleHOCONDiff_EmptyWhenIdentical(t *testing.T) { lines := []string{"a = 1", "b = 2"} - if got := simpleHOCONDiff(lines, lines); len(got) != 0 { + if got := render.DiffLines(lines, lines, 0); len(got) != 0 { t.Errorf("expected zero diffs for identical input, got %v", got) } } @@ -34,7 +35,7 @@ func TestSimpleHOCONDiff_HandlesUnequalLength(t *testing.T) { old := []string{"line1"} new := []string{"line1", "line2", "line3"} - diffs := simpleHOCONDiff(old, new) + diffs := render.DiffLines(old, new, 0) // Two new lines added → two `+` entries. if len(diffs) != 2 { t.Fatalf("expected 2 added lines, got %d: %v", len(diffs), diffs) diff --git a/cmd/verify_config.go b/cmd/verify_config.go index 101c741c..d9574156 100644 --- a/cmd/verify_config.go +++ b/cmd/verify_config.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/render" @@ -92,14 +93,14 @@ func runVerifyConfig(cmd *cobra.Command, args []string) error { // Render what trond *would* produce from the current intent. The // comparison runs against the REAL bytes (redacting here would make // every witness node report permanent false drift against its live - // conf); lineDiff redacts each line as it emits it. - renderedDesired, err := render.RenderHOCONWithSecrets(findTemplatesDir(), parsed, &parsed.Nodes[0]) + // conf); render.DiffText redacts each line as it emits it. + renderedDesired, err := render.RenderHOCONWithSecrets(apply.FindTemplatesDir(), parsed, &parsed.Nodes[0]) if err != nil { return exitWithError("RENDER_ERROR", output.ExitGeneralError, err.Error()) } desired := renderedDesired.Deployable() - diffs := lineDiff(live, desired, verifyConfigContext) + diffs := render.DiffText(live, desired, verifyConfigContext) result := map[string]any{ "name": name, "intent": parsed.Name, @@ -150,68 +151,6 @@ func readLiveConfig(ctx context.Context, nc *nodeContext, name string) (string, return string(out), nil } -// lineDiff returns a list of diff lines marking '+'/'−' changes -// between live and desired. We use a deliberately simple LCS-free -// algorithm — trond's HOCON files are <500 lines and the consumers -// of this output are agents that key off `in_sync`/`diff_count`, -// not humans diffing big patches. The unified-diff formatting is -// precise enough for the operator-friendly text path. -// -// Secret handling: BOTH sides carry the real witness key — `live` is -// read off the deployed host and `desired` is the deployable render — -// so every emitted line, including the --context neighbours (which are -// emitted even when they match), is passed through -// render.RedactWitnessLine. Comparison still uses the raw lines, so a -// rotated key is still counted in diff_count and flips in_sync. -func lineDiff(live, desired string, contextLines int) []string { - a := strings.Split(strings.TrimRight(live, "\n"), "\n") - b := strings.Split(strings.TrimRight(desired, "\n"), "\n") - // Redact whole-slice: a multi-line `localwitness = [` array keeps its - // key on a line that does not itself start with the key name. - aR := render.RedactWitnessLines(a) - bR := render.RedactWitnessLines(b) - var diffs []string - max := len(a) - if len(b) > max { - max = len(b) - } - for i := 0; i < max; i++ { - var aLine, bLine string - if i < len(a) { - aLine = a[i] - } - if i < len(b) { - bLine = b[i] - } - if aLine == bLine { - continue - } - // Optional context: emit a few neighbours of equal lines for - // human readability when --context > 0. Agents typically - // pass context=0. - if contextLines > 0 { - lo := i - contextLines - if lo < 0 { - lo = 0 - } - for j := lo; j < i; j++ { - if j < len(a) { - diffs = append(diffs, " "+aR[j]) - } - } - } - switch { - case i < len(a) && i >= len(b): - diffs = append(diffs, "- "+aR[i]) - case i >= len(a) && i < len(b): - diffs = append(diffs, "+ "+bR[i]) - default: - diffs = append(diffs, "- "+aR[i], "+ "+bR[i]) - } - } - return diffs -} - func countLines(s string) int { if s == "" { return 0 diff --git a/cmd/verify_config_test.go b/cmd/verify_config_test.go index 4547d476..4050a71e 100644 --- a/cmd/verify_config_test.go +++ b/cmd/verify_config_test.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/tronprotocol/tron-deployment/internal/render" "strings" "testing" ) @@ -60,7 +61,7 @@ func TestLineDiff_PinsContractShape(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := lineDiff(tc.live, tc.desired, tc.context) + got := render.DiffText(tc.live, tc.desired, tc.context) if len(got) != tc.wantLen { t.Fatalf("len: got %d, want %d\nactual:\n%s", len(got), tc.wantLen, strings.Join(got, "\n")) diff --git a/cmd/witness_redaction_test.go b/cmd/witness_redaction_test.go index 3c86cd14..45b2f7e0 100644 --- a/cmd/witness_redaction_test.go +++ b/cmd/witness_redaction_test.go @@ -1,12 +1,13 @@ package cmd import ( + "github.com/tronprotocol/tron-deployment/internal/render" "strings" "testing" ) // F3 regression tests for the two positional, LCS-free differs that -// live in package cmd: simpleHOCONDiff (plan --diff, whose output goes +// exercised by plan --diff and verify-config, whose output goes // into result["config_diff"] and out through output.WriteJSON) and // lineDiff (verify-config, whose output goes into diffs[]). // @@ -78,7 +79,7 @@ func TestSimpleHOCONDiff_RedactsWitnessKey(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - diffs := simpleHOCONDiff(tc.old, tc.new) + diffs := render.DiffLines(tc.old, tc.new, 0) assertNoKeys(t, "simpleHOCONDiff", diffs) if len(diffs) != tc.wantDiffs { t.Errorf("drift reporting changed: got %d diff lines, want %d: %v", @@ -97,10 +98,10 @@ func TestSimpleHOCONDiff_RedactsWitnessKey(t *testing.T) { func TestSimpleHOCONDiff_StillDetectsRotation(t *testing.T) { old := []string{`localwitness = ["` + keyA + `"]`} new := []string{`localwitness = ["` + keyB + `"]`} - if diffs := simpleHOCONDiff(old, new); len(diffs) != 2 { + if diffs := render.DiffLines(old, new, 0); len(diffs) != 2 { t.Fatalf("a rotated witness key must still be reported as drift, got %v", diffs) } - if diffs := simpleHOCONDiff(old, old); len(diffs) != 0 { + if diffs := render.DiffLines(old, old, 0); len(diffs) != 0 { t.Fatalf("an unchanged witness key must not be reported as drift, got %v", diffs) } } @@ -110,7 +111,7 @@ func TestLineDiff_RedactsWitnessKey(t *testing.T) { t.Run("key-rotation", func(t *testing.T) { desired := "a = 1\n" + `localwitness = ["` + keyB + `"]` + "\nz = 9\n" - diffs := lineDiff(live, desired, 0) + diffs := render.DiffText(live, desired, 0) assertNoKeys(t, "lineDiff", diffs) if len(diffs) != 2 { t.Errorf("rotated key must still be reported: %v", diffs) @@ -122,7 +123,7 @@ func TestLineDiff_RedactsWitnessKey(t *testing.T) { t.Run("line-shift", func(t *testing.T) { desired := "a = 1\nseed.node.ip.list = []\n" + `localwitness = ["` + keyA + `"]` + "\nz = 9\n" - diffs := lineDiff(live, desired, 0) + diffs := render.DiffText(live, desired, 0) assertNoKeys(t, "lineDiff", diffs) if len(diffs) == 0 { t.Error("a line shift must still be reported as drift") @@ -134,7 +135,7 @@ func TestLineDiff_RedactsWitnessKey(t *testing.T) { // treatment as the changed lines. t.Run("context-lines", func(t *testing.T) { desired := "a = 1\n" + `localwitness = ["` + keyA + `"]` + "\nz = CHANGED\n" - diffs := lineDiff(live, desired, 2) + diffs := render.DiffText(live, desired, 2) assertNoKeys(t, "lineDiff --context", diffs) joined := strings.Join(diffs, "\n") if !strings.Contains(joined, ` localwitness = [""]`) { @@ -143,7 +144,7 @@ func TestLineDiff_RedactsWitnessKey(t *testing.T) { }) t.Run("identical-is-in-sync", func(t *testing.T) { - if diffs := lineDiff(live, live, 0); len(diffs) != 0 { + if diffs := render.DiffText(live, live, 0); len(diffs) != 0 { t.Errorf("identical configs must stay in_sync, got %v", diffs) } }) diff --git a/internal/apply/envvars.go b/internal/apply/envvars.go new file mode 100644 index 00000000..66f0f0c6 --- /dev/null +++ b/internal/apply/envvars.go @@ -0,0 +1,17 @@ +package apply + +import ( + "github.com/tronprotocol/tron-deployment/internal/intent" + "os" +) + +// ResolveEnvVars resolves the legacy witness-key environment reference. +func ResolveEnvVars(node *intent.NodeSpec) map[string]string { + env := map[string]string{} + if node.WitnessKeyEnv != "" { + if value := os.Getenv(node.WitnessKeyEnv); value != "" { + env[node.WitnessKeyEnv] = value + } + } + return env +} diff --git a/internal/mcp/conf_helpers.go b/internal/mcp/conf_helpers.go index be152f09..0a2a36b8 100644 --- a/internal/mcp/conf_helpers.go +++ b/internal/mcp/conf_helpers.go @@ -3,9 +3,7 @@ package mcp import ( "context" "fmt" - "strings" - "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -29,16 +27,3 @@ func readLiveConfigForMCP(ctx context.Context, tgt target.Target, node *state.Ma } return string(out), nil } - -// redactConfText removes witness signing keys from a whole conf before -// it is handed out. Split/join around render.RedactWitnessLines, which -// finds the key values by parsing rather than by line shape, so the -// formatting of the node's live conf does not matter. -func redactConfText(conf string) string { - if conf == "" { - return conf - } - // Splitting on \n and rejoining preserves \r\n line endings, since - // the \r stays attached to the line content. - return strings.Join(render.RedactWitnessLines(strings.Split(conf, "\n")), "\n") -} diff --git a/internal/mcp/conf_helpers_test.go b/internal/mcp/conf_helpers_test.go index aff1a739..238677ec 100644 --- a/internal/mcp/conf_helpers_test.go +++ b/internal/mcp/conf_helpers_test.go @@ -3,12 +3,14 @@ package mcp import ( "strings" "testing" + + "github.com/tronprotocol/tron-deployment/internal/render" ) func TestRedactLiveConfigForMCP(t *testing.T) { key := strings.Repeat("a", 64) config := "node.p2p.version = 1\n" + `localwitness = ["` + key + `"]` + "\nseed.node.ip.list = []\n" - got := redactConfText(config) + got := strings.Join(render.RedactWitnessLines(strings.Split(config, "\n")), "\n") if strings.Contains(got, key) { t.Fatal("live config redaction leaked witness private key") } @@ -22,7 +24,7 @@ func TestRedactLiveConfigForMCP(t *testing.T) { func TestRedactLiveConfigForMCP_ColonWitnessSyntax(t *testing.T) { key := strings.Repeat("b", 64) - got := redactConfText(`localwitness : ["` + key + `"]`) + got := strings.Join(render.RedactWitnessLines(strings.Split(`localwitness : ["`+key+`"]`, "\n")), "\n") // redactConfText redacts in place and preserves the original separator. if strings.Contains(got, key) || got != `localwitness : [""]` { t.Fatalf("colon-separated live witness config was not redacted: %q", got) diff --git a/internal/mcp/conf_redaction_test.go b/internal/mcp/conf_redaction_test.go index 7d1e82bb..6b7b25a2 100644 --- a/internal/mcp/conf_redaction_test.go +++ b/internal/mcp/conf_redaction_test.go @@ -3,6 +3,8 @@ package mcp import ( "strings" "testing" + + "github.com/tronprotocol/tron-deployment/internal/render" ) // trond://nodes/{name}/conf hands the whole conf to an MCP client and @@ -17,7 +19,7 @@ func TestRedactConfTextRemovesWitnessKey(t *testing.T) { "comment carries a bracket": "localwitness = [ # ] here\n " + key + "\n]\n", } { t.Run(name, func(t *testing.T) { - if got := redactConfText(conf); strings.Contains(got, key) { + if got := strings.Join(render.RedactWitnessLines(strings.Split(conf, "\n")), "\n"); strings.Contains(got, key) { t.Errorf("witness key survived redaction:\n%s", got) } }) @@ -28,13 +30,13 @@ func TestRedactConfTextRemovesWitnessKey(t *testing.T) { // agent reads a node's real configuration. func TestRedactConfTextLeavesOrdinaryConfAlone(t *testing.T) { conf := "storage = {\n db.engine = \"LEVELDB\"\n}\nnode.p2p.version = 11111\n" - if got := redactConfText(conf); got != conf { + if got := strings.Join(render.RedactWitnessLines(strings.Split(conf, "\n")), "\n"); got != conf { t.Errorf("conf without a witness key was rewritten:\ngot %q\nwant %q", got, conf) } } func TestRedactConfTextEmpty(t *testing.T) { - if got := redactConfText(""); got != "" { + if got := strings.Join(render.RedactWitnessLines(strings.Split("", "\n")), "\n"); got != "" { t.Errorf("empty conf became %q", got) } } diff --git a/internal/mcp/drift_redaction_test.go b/internal/mcp/drift_redaction_test.go index 5773067b..535ef047 100644 --- a/internal/mcp/drift_redaction_test.go +++ b/internal/mcp/drift_redaction_test.go @@ -3,6 +3,7 @@ package mcp import ( "context" "encoding/json" + "github.com/tronprotocol/tron-deployment/internal/render" "os" "path/filepath" "strings" @@ -38,7 +39,7 @@ func TestMCPLineDiff_RedactsWitnessKey(t *testing.T) { t.Run("key-rotation", func(t *testing.T) { desired := "a = 1\n" + `localwitness = ["` + mcpKeyB + `"]` + "\nz = 9\n" - diffs := mcpLineDiff(live, desired, 0) + diffs := render.DiffText(live, desired, 0) assertNoMCPKeys(t, diffs) if len(diffs) != 2 { t.Errorf("a rotated witness key must still be reported as drift: %v", diffs) @@ -52,7 +53,7 @@ func TestMCPLineDiff_RedactsWitnessKey(t *testing.T) { t.Run("line-shift", func(t *testing.T) { desired := "a = 1\nseed.node.ip.list = []\n" + `localwitness = ["` + mcpKeyA + `"]` + "\nz = 9\n" - diffs := mcpLineDiff(live, desired, 0) + diffs := render.DiffText(live, desired, 0) assertNoMCPKeys(t, diffs) if len(diffs) == 0 { t.Error("a line shift must still be reported as drift") @@ -61,7 +62,7 @@ func TestMCPLineDiff_RedactsWitnessKey(t *testing.T) { t.Run("context-lines", func(t *testing.T) { desired := "a = 1\n" + `localwitness = ["` + mcpKeyA + `"]` + "\nz = CHANGED\n" - diffs := mcpLineDiff(live, desired, 2) + diffs := render.DiffText(live, desired, 2) assertNoMCPKeys(t, diffs) if !strings.Contains(strings.Join(diffs, "\n"), ` localwitness = [""]`) { t.Errorf("matching --context lines must be redacted too, got:\n%s", strings.Join(diffs, "\n")) @@ -69,7 +70,7 @@ func TestMCPLineDiff_RedactsWitnessKey(t *testing.T) { }) t.Run("identical-is-in-sync", func(t *testing.T) { - if diffs := mcpLineDiff(live, live, 0); len(diffs) != 0 { + if diffs := render.DiffText(live, live, 0); len(diffs) != 0 { t.Errorf("identical configs must stay in_sync, got %v", diffs) } }) diff --git a/internal/mcp/resources.go b/internal/mcp/resources.go index 2fec3600..61f5abbf 100644 --- a/internal/mcp/resources.go +++ b/internal/mcp/resources.go @@ -11,6 +11,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -198,7 +199,7 @@ func readNodeConfResource(ctx context.Context, req *mcp.ReadResourceRequest) (*m Contents: []*mcp.ResourceContents{{ URI: req.Params.URI, MIMEType: "text/plain", - Text: redactConfText(live), + Text: strings.Join(render.RedactWitnessLines(strings.Split(live, "\n")), "\n"), }}, }, nil } diff --git a/internal/mcp/tools_drift.go b/internal/mcp/tools_drift.go index 64815c8a..b5fbd43d 100644 --- a/internal/mcp/tools_drift.go +++ b/internal/mcp/tools_drift.go @@ -72,7 +72,7 @@ func verifyConfigTool(ctx context.Context, _ *mcp.CallToolRequest, args verifyCo } // Compare against the REAL bytes so a rotated witness key still - // registers as drift; mcpLineDiff redacts every line it emits, so + // registers as drift; render.DiffText redacts every line it emits, so // nothing secret leaves over the MCP transport. renderedDesired, err := render.RenderHOCONWithSecrets("", parsed, &parsed.Nodes[0]) if err != nil { @@ -80,7 +80,7 @@ func verifyConfigTool(ctx context.Context, _ *mcp.CallToolRequest, args verifyCo } desired := renderedDesired.Deployable() - diffs := mcpLineDiff(live, desired, args.Context) + diffs := render.DiffText(live, desired, args.Context) return jsonResult(map[string]any{ "name": args.Name, "intent": parsed.Name, @@ -93,60 +93,6 @@ func verifyConfigTool(ctx context.Context, _ *mcp.CallToolRequest, args verifyCo }) } -// mcpLineDiff mirrors cmd.lineDiff. Same simplicity rationale — and -// the same secret handling: comparison on the raw lines, every emitted -// line (including --context neighbours) through -// render.RedactWitnessLine. This result is returned to a third-party -// model provider, so an un-redacted `localwitness` line here is the -// worst-case disclosure path in the whole tool surface. -func mcpLineDiff(live, desired string, ctxLines int) []string { - a := strings.Split(strings.TrimRight(live, "\n"), "\n") - b := strings.Split(strings.TrimRight(desired, "\n"), "\n") - var diffs []string - maxLen := len(a) - if len(b) > maxLen { - maxLen = len(b) - } - // Redact whole-slice: a multi-line `localwitness = [` array keeps its - // key on a line that does not itself start with the key name. This - // output leaves the machine, so the per-line pass is not enough. - aR := render.RedactWitnessLines(a) - bR := render.RedactWitnessLines(b) - for i := range maxLen { - var aLine, bLine string - if i < len(a) { - aLine = a[i] - } - if i < len(b) { - bLine = b[i] - } - if aLine == bLine { - continue - } - if ctxLines > 0 { - lo := i - ctxLines - if lo < 0 { - lo = 0 - } - for j := lo; j < i; j++ { - if j < len(a) { - diffs = append(diffs, " "+aR[j]) - } - } - } - switch { - case i < len(a) && i >= len(b): - diffs = append(diffs, "- "+aR[i]) - case i >= len(a) && i < len(b): - diffs = append(diffs, "+ "+bR[i]) - default: - diffs = append(diffs, "- "+aR[i]) - diffs = append(diffs, "+ "+bR[i]) - } - } - return diffs -} - func countMCPLines(s string) int { if s == "" { return 0 diff --git a/internal/mcp/tools_lifecycle.go b/internal/mcp/tools_lifecycle.go index a3909d18..5e3a318f 100644 --- a/internal/mcp/tools_lifecycle.go +++ b/internal/mcp/tools_lifecycle.go @@ -196,7 +196,7 @@ func applyTool(ctx context.Context, _ *mcp.CallToolRequest, args applyArgs) (*mc Existing: existing, TemplateDir: templateDir, DeploymentsDir: paths.Deployments(), - EnvVars: resolveEnvVars(&parsed.Nodes[0]), + EnvVars: apply.ResolveEnvVars(&parsed.Nodes[0]), Wait: args.Wait, WaitTimeout: 5 * time.Minute, RequirePrivate: args.RequirePrivate || guard.Requested(), @@ -249,16 +249,4 @@ func resolveTarget(parsed *intent.Intent) (target.Target, error) { return target.FromIntent(parsed) } -// resolveEnvVars mirrors cmd/apply.go's helper for the same reason. -// Pulls the witness key out of the operator's environment by name. -func resolveEnvVars(node *intent.NodeSpec) map[string]string { - env := map[string]string{} - if node.WitnessKeyEnv != "" { - if v := os.Getenv(node.WitnessKeyEnv); v != "" { - env[node.WitnessKeyEnv] = v - } - } - return env -} - func ptrTrue() *bool { v := true; return &v } diff --git a/internal/render/diff.go b/internal/render/diff.go new file mode 100644 index 00000000..adcf0ad5 --- /dev/null +++ b/internal/render/diff.go @@ -0,0 +1,55 @@ +package render + +import "strings" + +// DiffLines compares positional lines, redacts witness material, and emits optional equal context lines. +func DiffLines(old, new []string, contextLines int) []string { + return diffLines(old, new, contextLines, false) +} + +// DiffText trims trailing newlines, splits live and desired text, then applies +// redacted, including witness material in multi-line arrays. +func DiffText(live, desired string, contextLines int) []string { + old := strings.Split(strings.TrimRight(live, "\n"), "\n") + new := strings.Split(strings.TrimRight(desired, "\n"), "\n") + return diffLines(old, new, contextLines, true) +} + +func diffLines(old, new []string, contextLines int, emitEmpty bool) []string { + oldR, newR := RedactWitnessLines(old), RedactWitnessLines(new) + max := len(old) + if len(new) > max { + max = len(new) + } + var out []string + for i := 0; i < max; i++ { + var a, b string + if i < len(old) { + a = old[i] + } + if i < len(new) { + b = new[i] + } + if a == b { + continue + } + if contextLines > 0 { + lo := i - contextLines + if lo < 0 { + lo = 0 + } + for j := lo; j < i; j++ { + if j < len(old) { + out = append(out, " "+oldR[j]) + } + } + } + if i < len(old) && (emitEmpty || a != "") { + out = append(out, "- "+oldR[i]) + } + if i < len(new) && (emitEmpty || b != "") { + out = append(out, "+ "+newR[i]) + } + } + return out +} From 7f295d3a319d0b0d84a63315ae810d206e11fbcf Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 16:39:54 +0800 Subject: [PATCH 30/49] refactor(paths,render): converge template-dir, default-path, and container-path copies (#8/#9) - Delete findTemplatesDir (cmd/network) and findTemplateDir (cmd/config) duplicates; all callers use apply.FindTemplatesDir (config now requires main_net_config.conf before accepting a template dir, falling back to embedded templates otherwise). - state.NewStore("")/security.NewAuditLog("") resolve defaults via internal/paths (honors TROND_STATE_DIR/SetBaseDir; .trond fallback when HOME is unavailable). - Export render.ContainerWorkdir/ContainerDataDir/ContainerConfDir/ ContainerLogPath and converge the /java-tron literal copies in snapshot clone, verify-config, mcp conf helpers, runtime docker logs, and apply probe. - Add paths.DeploymentConfig(name) and use it for the identical //.conf joins in plan and config diff. No test changes. --- cmd/config/diff.go | 6 +++--- cmd/config/render.go | 19 ++----------------- cmd/network/add.go | 8 ++++---- cmd/network/create.go | 18 +----------------- cmd/plan.go | 3 +-- cmd/snapshot/clone.go | 10 +++------- cmd/verify_config.go | 2 +- internal/apply/probe.go | 3 ++- internal/mcp/conf_helpers.go | 6 ++++-- internal/paths/deployment.go | 6 ++++++ internal/render/compose.go | 11 +++++++---- internal/runtime/docker.go | 3 ++- internal/security/audit.go | 10 ++++------ internal/state/store.go | 9 +++------ 14 files changed, 43 insertions(+), 71 deletions(-) create mode 100644 internal/paths/deployment.go diff --git a/cmd/config/diff.go b/cmd/config/diff.go index a8af26a3..0d7070e0 100644 --- a/cmd/config/diff.go +++ b/cmd/config/diff.go @@ -3,11 +3,11 @@ package config import ( "fmt" "os" - "path/filepath" "strings" "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" @@ -36,7 +36,7 @@ func runDiff(cmd *cobra.Command, args []string) error { } // Render new config - templateDir := findTemplateDir() + templateDir := apply.FindTemplatesDir() node := &parsed.Nodes[0] rendered, err := render.RenderHOCONWithSecrets(templateDir, parsed, node) @@ -73,7 +73,7 @@ func runDiff(cmd *cobra.Command, args []string) error { } // Try to read the deployed config - deployedConfigPath := filepath.Join(paths.Deployments(), parsed.Name, parsed.Name+".conf") + deployedConfigPath := paths.DeploymentConfig(parsed.Name) deployedData, err := os.ReadFile(deployedConfigPath) if err != nil { if outputFmt == "json" { diff --git a/cmd/config/render.go b/cmd/config/render.go index 29b88376..cf23993a 100644 --- a/cmd/config/render.go +++ b/cmd/config/render.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/render" @@ -74,7 +75,7 @@ func runRender(cmd *cobra.Command, args []string) error { } // Find templates directory (relative to binary or working directory) - templateDir := findTemplateDir() + templateDir := apply.FindTemplatesDir() rendered := make([]renderedNode, 0, len(parsed.Nodes)) anyRedacted := false @@ -267,19 +268,3 @@ func writeSecretFile(path string, data []byte) (err error) { } return nil } - -// findTemplateDir prefers the TROND_TEMPLATES_DIR env var, then falls back to -// ./templates. An empty return value tells render.RenderHOCON to use the -// embedded copy — release binaries work without any co-located files. -func findTemplateDir() string { - if d := os.Getenv("TROND_TEMPLATES_DIR"); d != "" { - return d - } - candidates := []string{"templates", "./templates"} - for _, c := range candidates { - if info, err := os.Stat(c); err == nil && info.IsDir() { - return c - } - } - return "" -} diff --git a/cmd/network/add.go b/cmd/network/add.go index 4134f518..09935b00 100644 --- a/cmd/network/add.go +++ b/cmd/network/add.go @@ -12,7 +12,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v3" - applypkg "github.com/tronprotocol/tron-deployment/internal/apply" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/guard" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" @@ -191,10 +191,10 @@ func runAdd(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("hash add intent: %w", err) } - _, err = applypkg.Apply(cmd.Context(), applypkg.Options{ + _, err = apply.Apply(cmd.Context(), apply.Options{ Intent: &clone, Target: tgt, Store: store, State: deployState, - IntentHash: applypkg.IntentHashFromBytes(data), Existing: store.GetNode(deployState, nodeName), - TemplateDir: findTemplatesDir(), DeploymentsDir: paths.Deployments(), IntentPath: addIntentPath, + IntentHash: apply.IntentHashFromBytes(data), Existing: store.GetNode(deployState, nodeName), + TemplateDir: apply.FindTemplatesDir(), DeploymentsDir: paths.Deployments(), IntentPath: addIntentPath, RequirePrivate: guard.Requested(), SkipMonitoring: parsed.Monitoring != nil && parsed.Monitoring.Enabled != nil && *parsed.Monitoring.Enabled, }) diff --git a/cmd/network/create.go b/cmd/network/create.go index bc54610d..2c27447c 100644 --- a/cmd/network/create.go +++ b/cmd/network/create.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" "slices" "time" @@ -92,7 +91,7 @@ func runCreate(cmd *cobra.Command, args []string) error { } defer closeTarget(tgt) - templateDir := findTemplatesDir() + templateDir := apply.FindTemplatesDir() workDir := paths.Deployments() // Hold the state lock across the whole load-modify-save cycle: this @@ -299,21 +298,6 @@ func autoWireActivePeers(parsed *intent.Intent) { } } -func findTemplatesDir() string { - if d := os.Getenv("TROND_TEMPLATES_DIR"); d != "" { - return d - } - candidates := []string{"templates", "./templates"} - for _, c := range candidates { - if info, err := os.Stat(c); err == nil && info.IsDir() { - if _, err := os.Stat(filepath.Join(c, "main_net_config.conf")); err == nil { - return c - } - } - } - return "" -} - // deployNetworkMonitoring deploys a single monitoring stack for an entire // private network. All nodes are scraped via the shared docker network. func deployNetworkMonitoring(ctx context.Context, tgt target.Target, workDir string, parsed *intent.Intent) monitoringResult { diff --git a/cmd/plan.go b/cmd/plan.go index e41515c5..b0039c07 100644 --- a/cmd/plan.go +++ b/cmd/plan.go @@ -3,7 +3,6 @@ package cmd import ( "fmt" "os" - "path/filepath" "strings" "github.com/spf13/cobra" @@ -95,7 +94,7 @@ func runPlan(cmd *cobra.Command, args []string) error { // (deployment dir cleaned, etc.). var diffLines []string if planShowDiff && existing != nil { - deployedPath := filepath.Join(paths.Deployments(), parsed.Name, parsed.Name+".conf") + deployedPath := paths.DeploymentConfig(parsed.Name) if data, err := os.ReadFile(deployedPath); err == nil { diffLines = render.DiffLines(strings.Split(string(data), "\n"), strings.Split(string(planned.Config), "\n"), 0) diff --git a/cmd/snapshot/clone.go b/cmd/snapshot/clone.go index 5c687651..46d284f9 100644 --- a/cmd/snapshot/clone.go +++ b/cmd/snapshot/clone.go @@ -16,6 +16,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/fsclone" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -32,11 +33,6 @@ import ( // // Mutating (creates dst) so it is CLI-only — deliberately NOT an MCP tool, // keeping filesystem-mutating capability out of the read-only agent fleet. -// containerDataDir is where compose mounts the chain DB inside a docker -// node (mirrors internal/render/compose.go). --from-node finds the host -// path by matching this mount Destination. -const containerDataDir = "/java-tron/output-directory" - var cloneFromNode string var cloneCmd = &cobra.Command{ @@ -389,7 +385,7 @@ func dockerNodeDBDir(ctx context.Context, name string) (string, error) { WithSuggestions(fmt.Sprintf("Run: trond stop %s", name)) } for _, m := range c.Mounts { - if m.Destination != containerDataDir { + if m.Destination != render.ContainerDataDir { continue } // Reject by mount TYPE, not by sniffing Source: a Linux named volume @@ -403,7 +399,7 @@ func dockerNodeDBDir(ctx context.Context, name string) (string, error) { return m.Source, nil } return "", output.NewError("VALIDATION_ERROR", output.ExitValidationError, - fmt.Sprintf("node %q has no chain-DB mount at %s", name, containerDataDir)) + fmt.Sprintf("node %q has no chain-DB mount at %s", name, render.ContainerDataDir)) } // jarActive reports whether the node's systemd unit is currently active. A diff --git a/cmd/verify_config.go b/cmd/verify_config.go index d9574156..126e97d6 100644 --- a/cmd/verify_config.go +++ b/cmd/verify_config.go @@ -144,7 +144,7 @@ func readLiveConfig(ctx context.Context, nc *nodeContext, name string) (string, } // Default: docker. java-tron images mount the conf at // /java-tron/conf/.conf (set by render.RenderCompose). - out, err := nc.runtimeExec(ctx, "cat", "/java-tron/conf/"+name+".conf") + out, err := nc.runtimeExec(ctx, "cat", render.ContainerConfDir+"/"+name+".conf") if err != nil { return "", fmt.Errorf("docker exec cat: %w", err) } diff --git a/internal/apply/probe.go b/internal/apply/probe.go index f5de7843..7ccf0d9a 100644 --- a/internal/apply/probe.go +++ b/internal/apply/probe.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -184,7 +185,7 @@ func LogsDescriptor(node *state.ManagedNode) map[string]any { return map[string]any{ "runtime": "docker", "container": node.Name, - "path": "/java-tron/logs/tron.log", + "path": render.ContainerLogPath, } default: // Unknown/unrecorded runtime (e.g. a legacy node from before the diff --git a/internal/mcp/conf_helpers.go b/internal/mcp/conf_helpers.go index 0a2a36b8..a2249a30 100644 --- a/internal/mcp/conf_helpers.go +++ b/internal/mcp/conf_helpers.go @@ -3,7 +3,9 @@ package mcp import ( "context" "fmt" + "path/filepath" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -14,14 +16,14 @@ import ( // verify_config tool. func readLiveConfigForMCP(ctx context.Context, tgt target.Target, node *state.ManagedNode) (string, error) { if node.Runtime == "jar" { - out, err := tgt.Exec(ctx, "cat", node.InstallPath+"/conf/"+node.Name+".conf") + out, err := tgt.Exec(ctx, "cat", filepath.Join(node.InstallPath, "conf", node.Name+".conf")) if err != nil { return "", fmt.Errorf("read jar conf: %w", err) } return string(out), nil } out, err := tgt.Exec(ctx, "docker", "exec", node.Name, "cat", - "/java-tron/conf/"+node.Name+".conf") + render.ContainerConfDir+"/"+node.Name+".conf") if err != nil { return "", fmt.Errorf("docker exec cat: %w", err) } diff --git a/internal/paths/deployment.go b/internal/paths/deployment.go new file mode 100644 index 00000000..b5577956 --- /dev/null +++ b/internal/paths/deployment.go @@ -0,0 +1,6 @@ +package paths + +import "path/filepath" + +// DeploymentConfig returns the rendered config path for a managed deployment. +func DeploymentConfig(name string) string { return filepath.Join(Deployments(), name, name+".conf") } diff --git a/internal/render/compose.go b/internal/render/compose.go index a47aa0d1..c444906b 100644 --- a/internal/render/compose.go +++ b/internal/render/compose.go @@ -64,10 +64,13 @@ func composeEnvLines(node *intent.NodeSpec) []string { // (./bin/docker-entrypoint.sh) by spelling out `java -jar`. Containers // silently restart-looped as a result. This file is the corrected layout. const ( - containerWorkdir = "/java-tron" - containerConfigDir = containerWorkdir + "/conf" - containerDataDir = containerWorkdir + "/output-directory" - containerLogDir = containerWorkdir + "/logs" + ContainerWorkdir = "/java-tron" + ContainerDataDir = ContainerWorkdir + "/output-directory" + ContainerConfDir = ContainerWorkdir + "/conf" + ContainerLogPath = ContainerWorkdir + "/logs/tron.log" + containerConfigDir = ContainerConfDir + containerDataDir = ContainerDataDir + containerLogDir = ContainerWorkdir + "/logs" ) // RenderCompose generates a docker-compose.yaml from the intent. diff --git a/internal/runtime/docker.go b/internal/runtime/docker.go index ff578345..0099571e 100644 --- a/internal/runtime/docker.go +++ b/internal/runtime/docker.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -160,7 +161,7 @@ func (r *DockerRuntime) Logs(ctx context.Context, name string, opts LogOpts) (io if opts.Follow { tailArgs = append(tailArgs, "-f") } - tailArgs = append(tailArgs, "/java-tron/logs/tron.log") + tailArgs = append(tailArgs, render.ContainerLogPath) if opts.Follow { if stream, ok := r.target.(target.StreamExec); ok { if reader, err := stream.StreamExec(ctx, "docker", tailArgs...); err == nil { diff --git a/internal/security/audit.go b/internal/security/audit.go index 9dce0052..f9c904fb 100644 --- a/internal/security/audit.go +++ b/internal/security/audit.go @@ -6,6 +6,8 @@ import ( "os" "path/filepath" "sync" + + "github.com/tronprotocol/tron-deployment/internal/paths" "time" ) @@ -47,14 +49,10 @@ type AuditLog struct { mu sync.Mutex } -// NewAuditLog creates an audit log writer. If path is empty, defaults to ~/.trond/audit.log. +// NewAuditLog creates an audit log writer. If path is empty, uses paths.AuditLog(). func NewAuditLog(path string) (*AuditLog, error) { if path == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, fmt.Errorf("get home dir: %w", err) - } - path = filepath.Join(home, ".trond", "audit.log") + path = paths.AuditLog() } dir := filepath.Dir(path) diff --git a/internal/state/store.go b/internal/state/store.go index 819d6f61..3dca013b 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -3,6 +3,7 @@ package state import ( "encoding/json" "fmt" + "github.com/tronprotocol/tron-deployment/internal/paths" "os" "path/filepath" ) @@ -15,14 +16,10 @@ type Store struct { } // NewStore creates a state store at the given path. -// If path is empty, defaults to ~/.trond/state.json. +// If path is empty, uses paths.State(). func NewStore(path string) (*Store, error) { if path == "" { - home, err := os.UserHomeDir() - if err != nil { - return nil, fmt.Errorf("get home dir: %w", err) - } - path = filepath.Join(home, ".trond", "state.json") + path = paths.State() } // Ensure directory exists From 5a77870ecd28517b5974c160a81d64125d49af06 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 21 Aug 2026 17:08:23 +0800 Subject: [PATCH 31/49] refactor(target): converge live-config reader into target.ReadLiveConfig (#13/#14) Single jar/docker live-conf reader in internal/target; CLI verify-config and MCP conf helpers drop their local copies. Error text contracts preserved per caller (read : / read jar conf: / docker exec cat:). No test changes. --- cmd/verify_config.go | 32 ++++++++------------------------ internal/mcp/conf_helpers.go | 13 +++---------- internal/target/liveconf.go | 27 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 34 deletions(-) create mode 100644 internal/target/liveconf.go diff --git a/cmd/verify_config.go b/cmd/verify_config.go index 126e97d6..fb66a5d6 100644 --- a/cmd/verify_config.go +++ b/cmd/verify_config.go @@ -14,6 +14,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/render" + "github.com/tronprotocol/tron-deployment/internal/target" ) // `trond verify-config --intent ` answers the question: @@ -83,7 +84,13 @@ func runVerifyConfig(cmd *cobra.Command, args []string) error { // the binary's working dir holds it under conf/. ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second) defer cancel() - live, err := readLiveConfig(ctx, nc, name) + live, err := target.ReadLiveConfig(ctx, nc.Target, nc.Node) + if err != nil && nc.Node.Runtime == "jar" { + path := filepath.Join(nc.Node.InstallPath, "conf", name+".conf") + err = fmt.Errorf("read %s: %w", path, err) + } else if err != nil { + err = fmt.Errorf("docker exec cat: %w", err) + } if err != nil { return exitWithError("LIVE_CONFIG_UNREACHABLE", output.ExitGeneralError, err.Error(), "Confirm the node is running: trond status "+name, @@ -128,29 +135,6 @@ func runVerifyConfig(cmd *cobra.Command, args []string) error { return nil } -// readLiveConfig returns the bytes of the conf file currently used by -// the running node, regardless of runtime. For docker, we cat the -// file inside the container; for jar, we read from the on-disk -// working directory. -func readLiveConfig(ctx context.Context, nc *nodeContext, name string) (string, error) { - if nc.Node.Runtime == "jar" { - // Jar runtime keeps the conf under the install_path's conf/. - path := filepath.Join(nc.Node.InstallPath, "conf", name+".conf") - out, err := nc.Target.Exec(ctx, "cat", path) - if err != nil { - return "", fmt.Errorf("read %s: %w", path, err) - } - return string(out), nil - } - // Default: docker. java-tron images mount the conf at - // /java-tron/conf/.conf (set by render.RenderCompose). - out, err := nc.runtimeExec(ctx, "cat", render.ContainerConfDir+"/"+name+".conf") - if err != nil { - return "", fmt.Errorf("docker exec cat: %w", err) - } - return string(out), nil -} - func countLines(s string) int { if s == "" { return 0 diff --git a/internal/mcp/conf_helpers.go b/internal/mcp/conf_helpers.go index a2249a30..2d35cf68 100644 --- a/internal/mcp/conf_helpers.go +++ b/internal/mcp/conf_helpers.go @@ -3,9 +3,7 @@ package mcp import ( "context" "fmt" - "path/filepath" - "github.com/tronprotocol/tron-deployment/internal/render" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -15,16 +13,11 @@ import ( // resources.go (trond://nodes//conf) and the future // verify_config tool. func readLiveConfigForMCP(ctx context.Context, tgt target.Target, node *state.ManagedNode) (string, error) { - if node.Runtime == "jar" { - out, err := tgt.Exec(ctx, "cat", filepath.Join(node.InstallPath, "conf", node.Name+".conf")) - if err != nil { + out, err := target.ReadLiveConfig(ctx, tgt, node) + if err != nil { + if node.Runtime == "jar" { return "", fmt.Errorf("read jar conf: %w", err) } - return string(out), nil - } - out, err := tgt.Exec(ctx, "docker", "exec", node.Name, "cat", - render.ContainerConfDir+"/"+node.Name+".conf") - if err != nil { return "", fmt.Errorf("docker exec cat: %w", err) } return string(out), nil diff --git a/internal/target/liveconf.go b/internal/target/liveconf.go new file mode 100644 index 00000000..4a23bfaa --- /dev/null +++ b/internal/target/liveconf.go @@ -0,0 +1,27 @@ +package target + +import ( + "context" + "path/filepath" + + "github.com/tronprotocol/tron-deployment/internal/render" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +// ReadLiveConfig reads the active HOCON configuration from a jar host or +// Docker container. Callers retain ownership of the target lifecycle. +func ReadLiveConfig(ctx context.Context, tgt Target, node *state.ManagedNode) (string, error) { + if node.Runtime == "jar" { + path := filepath.Join(node.InstallPath, "conf", node.Name+".conf") + out, err := tgt.Exec(ctx, "cat", path) + if err != nil { + return "", err + } + return string(out), nil + } + out, err := tgt.Exec(ctx, "docker", "exec", node.Name, "cat", render.ContainerConfDir+"/"+node.Name+".conf") + if err != nil { + return "", err + } + return string(out), nil +} From b1d1515602a36c1fe34f3a82e5d5de5eeb82a66e Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 24 Aug 2026 13:27:52 +0800 Subject: [PATCH 32/49] docs(changelog): document P0 audit fixes --- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c06e547..ecd32414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **P0 audit remediation.** Batch B adds focused cmd-side test splits for + `cmd/build`, `cmd/state`, `cmd/network create`, and `cmd/apply` (B1–B8), + while domain 1–5 fixes harden lifecycle, state persistence, target + transport, rendering, snapshots, runtime upgrades, security, replay, and + transaction broadcasting behavior. +- Multi-node network creation, addition, destruction, and rolling upgrades + now persist node state consistently, verify every node, and roll back + already-upgraded nodes when verification fails. Automatic port allocation + is persisted for repeatable reconciliation. +- Snapshot downloads stage data before publishing atomically, refuse unsafe + overwrites, verify the process identity before stopping jobs, and merge + build environment maps correctly. Jar upgrades and rollbacks now swap the + running artifact reliably. +- Rendering now validates strict YAML fields, merges overlays correctly, + escapes Compose environment values, wires the `solidity_grpc` port, avoids + in-container HTTP-port conflicts, and caps JVM heap below 8 GB of container + memory. SSH probing and downloads are resilient and report the actual host. +- Sensitive witness data remains redacted, JAR configuration permissions are + restricted, state saves are atomic with surfaced persistence errors, and + replay cursors advance only after complete block replay. + +### Changed + +- Shared state loading, target resolution, live-config reading, template and + path discovery, diff computation, environment-variable expansion + (`internal/apply.ResolveEnvVars`), endpoint/port handling, and node-state + persistence are centralized across CLI and MCP paths. +- `trond plan` now reports `current_state: "not_deployed"` instead of + `"not deployed"`, matching the documented machine-readable contract. +- `intent_hash` in `apply` and `status` output is now versioned as + `v2:`; the previous bare-hex digest format is replaced by the v2 + digest scheme. +- With `auto_ports: true`, allocated ports are persisted. Re-running + `trond plan` after `apply` now correctly reports `changes: null` and zero + downtime instead of spurious configuration diffs requiring a restart. +- MCP lifecycle and diagnostic paths now use the same CLI contracts and + shared implementations; contract coverage and implementation specs were + expanded to document the corrected behavior. + ## [0.1.0] — 2026-XX-XX First tagged release. The project transitions from a curated set of HOCON From 70048f98b3e65f13b3aeedb49e4863805e782556 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 24 Aug 2026 18:11:10 +0800 Subject: [PATCH 33/49] fix: address P0 audit findings C1-C5 - C1: gate apply hash on full rendered config, not intent bytes - C2: refuse snapshot download into running/error node data dirs - C3: split network-upgrade env signals (TROND_PRESERVE_BACKUP for jar upgrade child, TROND_NETWORK_UPGRADE for jar rollback child) so the backup preserve/restore lifecycle closes end-to-end - C4: default unset optional intent fields before hashing/planning - C5: resolve nodes by name across runtimes in start/rollback paths Oracle-reviewed (B1/B2 + R1 rounds); targeted tests and gofmt pass. --- cmd/apply.go | 12 ++- cmd/apply_hash_test.go | 11 +++ cmd/network/create.go | 29 ++++++ cmd/network/node_intent_test.go | 67 ++++++++++++++ cmd/network/upgrade.go | 55 ++++++++++- cmd/network/upgrade_test.go | 31 ++++++- cmd/plan.go | 12 ++- cmd/snapshot/download.go | 148 ++---------------------------- cmd/start.go | 5 +- cmd/start_test.go | 12 +++ internal/apply/hash.go | 50 ++++++---- internal/mcp/tools_lifecycle.go | 24 ++++- internal/mcp/tools_snapshot.go | 5 + internal/runtime/upgrade.go | 37 ++++++-- internal/runtime/upgrade_test.go | 102 +++++++++++++++++++- internal/snapshot/download.go | 3 + internal/snapshot/running_node.go | 139 ++++++++++++++++++++++++++++ 17 files changed, 566 insertions(+), 176 deletions(-) create mode 100644 cmd/start_test.go create mode 100644 internal/snapshot/running_node.go diff --git a/cmd/apply.go b/cmd/apply.go index 7cc617f9..626d3d4c 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -131,9 +131,19 @@ func runApply(cmd *cobra.Command, args []string) error { existing := store.GetNode(deployState, parsed.Name) if parsed.Target.AutoPorts && existing != nil { if len(parsed.Nodes) == 1 { - apply.RestoreAutoPorts(&parsed.Nodes[0], existing) + rawParsed, rawErr := intent.LoadRaw(applyIntentPath) + if rawErr != nil { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, rawErr.Error()) + } + if len(rawParsed.Nodes) == 0 { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node") + } + apply.RestoreAutoPorts(&parsed.Nodes[0], existing, &rawParsed.Nodes[0]) } } + if len(parsed.Nodes) == 0 { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node") + } canonicalIntent, err := yaml.Marshal(parsed) if err != nil { return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "marshal effective intent: "+err.Error()) diff --git a/cmd/apply_hash_test.go b/cmd/apply_hash_test.go index 92d26d0e..45e63f5a 100644 --- a/cmd/apply_hash_test.go +++ b/cmd/apply_hash_test.go @@ -95,3 +95,14 @@ func TestRestoreAutoPortsAllSeven(t *testing.T) { } } } + +func TestRestoreAutoPortsExplicitPortsWin(t *testing.T) { + n := &intent.NodeSpec{} + n.Ports.HTTP, n.Ports.GRPC = 9000, 9001 + raw := &intent.NodeSpec{} + raw.Ports.HTTP, raw.Ports.GRPC = 9100, 0 + apply.RestoreAutoPorts(n, &state.ManagedNode{HTTPPort: 8000, GRPCPort: 8001}, raw) + if n.Ports.HTTP != 9000 || n.Ports.GRPC != 8001 { + t.Fatalf("ports = %+v, want explicit HTTP and restored GRPC", n.Ports) + } +} diff --git a/cmd/network/create.go b/cmd/network/create.go index 2c27447c..0f54f49d 100644 --- a/cmd/network/create.go +++ b/cmd/network/create.go @@ -67,6 +67,13 @@ func runCreate(cmd *cobra.Command, args []string) error { if err != nil { return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) } + rawParsed, err := intent.LoadRaw(createIntentPath) + if err != nil { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) + } + if len(parsed.Nodes) != len(rawParsed.Nodes) { + return output.NewError("VALIDATION_ERROR", output.ExitValidationError, "raw and effective intent node counts differ") + } // --require-private: same machine-enforced safety gate as `apply`, // applied to the multi-node path. Refuse a non-private network before @@ -112,6 +119,12 @@ func runCreate(cmd *cobra.Command, args []string) error { return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) } + // Restore persisted auto ports before auto-wiring peers. The peer + // addresses must use the same P2P ports that the existing nodes use. + for i := range parsed.Nodes { + restoreAutoPorts(parsed, i, rawParsed, deployState, store) + } + // Auto-wire peering between siblings before rendering. After // intent.Load() ports are final (defaults applied, auto_ports // resolved), so we can synthesise stable inter-container addresses. @@ -249,6 +262,22 @@ func runCreate(cmd *cobra.Command, args []string) error { return nil } +// restoreAutoPorts reuses ports recorded for a node before projecting the +// network intent. This keeps a repeated network create from allocating a new +// set of host ports and needlessly redeploying every node. +func restoreAutoPorts(parsed *intent.Intent, i int, rawParsed *intent.Intent, deployState *state.DeploymentState, store *state.Store) { + if !parsed.Target.AutoPorts || i < 0 || i >= len(parsed.Nodes) { + return + } + if rawParsed == nil || i >= len(rawParsed.Nodes) { + return + } + nodeName := fmt.Sprintf("%s-node%d", parsed.Name, i) + if existing := store.GetNode(deployState, nodeName); existing != nil { + apply.RestoreAutoPorts(&parsed.Nodes[i], existing, &rawParsed.Nodes[i]) + } +} + func countFailedDeployments(nodes []map[string]any) int { failed := 0 for _, node := range nodes { diff --git a/cmd/network/node_intent_test.go b/cmd/network/node_intent_test.go index c989f6d5..f72be8df 100644 --- a/cmd/network/node_intent_test.go +++ b/cmd/network/node_intent_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/state" ) func twoNodeIntent() *intent.Intent { @@ -159,3 +160,69 @@ func TestNodeIntent_OutOfRange(t *testing.T) { } } } + +func TestRestoreAutoPortsPreservesPersistedAssignments(t *testing.T) { + parsed := twoNodeIntent() + parsed.Target.AutoPorts = true + parsed.Nodes[0].Ports = intent.PortMapping{} + deployState := &state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "twosr-node0", + HTTPPort: 18090, GRPCPort: 18051, P2PPort: 18880, + MetricsPort: 19527, SolidityHTTPPort: 18092, SolidityGRPCPort: 18091, + JSONRPCPort: 18545, + }}} + store := &state.Store{} + + raw := twoNodeIntent() + raw.Nodes[0].Ports = intent.PortMapping{} + restoreAutoPorts(parsed, 0, raw, deployState, store) + got := parsed.Nodes[0].Ports + if got.HTTP != 18090 || got.GRPC != 18051 || got.P2P != 18880 || got.Metrics != 19527 || + got.SolidityHTTP != 18092 || got.SolidityGRPC != 18091 || got.JSONRPC != 18545 { + t.Fatalf("restored ports = %+v", got) + } +} + +func TestRestoreAutoPortsMakesPeerWiringStable(t *testing.T) { + parsed := twoNodeIntent() + parsed.Target.AutoPorts = true + parsed.Nodes[0].Ports = intent.PortMapping{} + parsed.Nodes[1].Ports = intent.PortMapping{} + deployState := &state.DeploymentState{Nodes: []state.ManagedNode{ + {Name: "twosr-node0", P2PPort: 18880}, + {Name: "twosr-node1", P2PPort: 18881}, + }} + store := &state.Store{} + raw := twoNodeIntent() + for i := range raw.Nodes { + raw.Nodes[i].Ports = intent.PortMapping{} + } + for i := range parsed.Nodes { + restoreAutoPorts(parsed, i, raw, deployState, store) + } + autoWireActivePeers(parsed) + if got := *parsed.Nodes[0].NetworkOverrides.ActivePeers; len(got) != 1 || got[0] != "twosr-node1:18881" { + t.Fatalf("node 0 active peers = %v", got) + } + if got := *parsed.Nodes[1].NetworkOverrides.ActivePeers; len(got) != 1 || got[0] != "twosr-node0:18880" { + t.Fatalf("node 1 active peers = %v", got) + } +} + +func TestRestoreAutoPortsExplicitPortsWin(t *testing.T) { + parsed := twoNodeIntent() + parsed.Target.AutoPorts = true + parsed.Nodes[0].Ports = intent.PortMapping{HTTP: 19090, GRPC: 50051, P2P: 19999} + raw := twoNodeIntent() + raw.Nodes[0].Ports = intent.PortMapping{HTTP: 19090, P2P: 19999} + deployState := &state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "twosr-node0", HTTPPort: 18090, P2PPort: 18880, GRPCPort: 18051, + }}} + restoreAutoPorts(parsed, 0, raw, deployState, &state.Store{}) + if parsed.Nodes[0].Ports.HTTP != 19090 || parsed.Nodes[0].Ports.P2P != 19999 { + t.Fatalf("explicit ports were overwritten: %+v", parsed.Nodes[0].Ports) + } + if parsed.Nodes[0].Ports.GRPC != 18051 { + t.Fatalf("unset port was not restored: %+v", parsed.Nodes[0].Ports) + } +} diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index bb6d7ac2..1ba6bd71 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -167,7 +167,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error { if upgradeJarSHA256 != "" { upgradeArgs = append(upgradeArgs, "--jar-sha256", upgradeJarSHA256) } - err := runChild(cmd.Context(), exe, upgradeArgs...) + err := runNetworkChild(cmd.Context(), exe, st, node, upgradeArgs...) steps = append(steps, upgradeStep{ Node: node, Phase: "upgrade", @@ -176,7 +176,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error { Error: errString(err), }) if err != nil { - return finishWithFailure(cmd.Context(), outputFmt, networkName, exe, + return finishWithFailure(cmd.Context(), outputFmt, networkName, exe, st, steps, upgraded, start, node, err) } // The artifact has been changed successfully. Keep this node in the @@ -194,7 +194,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error { Error: errString(err), }) if err != nil { - return finishWithFailure(cmd.Context(), outputFmt, networkName, exe, + return finishWithFailure(cmd.Context(), outputFmt, networkName, exe, st, steps, upgraded, start, node, err) } @@ -209,6 +209,12 @@ func runUpgrade(cmd *cobra.Command, args []string) error { "duration_ms": time.Since(start).Milliseconds(), "status": "success", } + for _, node := range upgraded { + if err := cleanupNetworkBackup(cmd.Context(), exe, st, node); err != nil { + return output.NewError("UPGRADE_ERROR", output.ExitGeneralError, + fmt.Sprintf("network upgrade succeeded but cleanup failed for %s: %v", node, err)) + } + } if outputFmt == "json" { return output.WriteJSON(os.Stdout, result) } @@ -251,10 +257,23 @@ func isWitnessNode(n state.ManagedNode) bool { } var runChild = runChildCommand +var runChildWithEnv = runChildCommandWithEnv + +const ( + networkUpgradeRestoreEnv = "TROND_NETWORK_UPGRADE=1" + networkUpgradePreserveEnv = "TROND_PRESERVE_BACKUP=1" +) func runChildCommand(ctx context.Context, exe string, argv ...string) error { + return runChildCommandWithEnv(ctx, exe, nil, argv...) +} + +func runChildCommandWithEnv(ctx context.Context, exe string, extraEnv []string, argv ...string) error { argv = append(argv, "--output", "json") cmd := exec.CommandContext(ctx, exe, argv...) + if len(extraEnv) > 0 { + cmd.Env = append(os.Environ(), extraEnv...) + } var stdout, stderr limitedBuffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -366,12 +385,13 @@ func errString(err error) string { // step appended; otherwise the operator is told which nodes need // manual rollback. func finishWithFailure(ctx context.Context, outputFmt, networkName, exe string, + st *state.DeploymentState, steps []upgradeStep, upgraded []string, start time.Time, failedNode string, failErr error) error { rolledBack := make([]string, 0, len(upgraded)) if upgradeAutoRollback { for _, node := range upgraded { stepStart := time.Now() - err := runChild(ctx, exe, "rollback", node) + err := runNetworkChild(ctx, exe, st, node, "rollback", node) steps = append(steps, upgradeStep{ Node: node, Phase: "rollback", @@ -418,3 +438,30 @@ func finishWithFailure(ctx context.Context, outputFmt, networkName, exe string, } return se } + +func runNetworkChild(ctx context.Context, exe string, st *state.DeploymentState, node string, argv ...string) error { + if st != nil { + for _, n := range st.Nodes { + if n.Name != node || n.Runtime != "jar" || len(argv) == 0 { + continue + } + switch argv[0] { + case "rollback": + return runChildWithEnv(ctx, exe, []string{networkUpgradeRestoreEnv}, argv...) + case "upgrade": + return runChildWithEnv(ctx, exe, []string{networkUpgradePreserveEnv}, argv...) + } + } + } + return runChild(ctx, exe, argv...) +} + +func cleanupNetworkBackup(ctx context.Context, exe string, st *state.DeploymentState, node string) error { + for _, n := range st.Nodes { + if n.Name == node && n.Runtime == "jar" { + path := filepath.Join(n.InstallPath, "FullNode.jar.upgrade.backup") + return runChild(ctx, exe, "exec", node, "--", "sh", "-c", "rm -f -- "+path+"; printf '{}' ") + } + } + return nil +} diff --git a/cmd/network/upgrade_test.go b/cmd/network/upgrade_test.go index d1154af8..01419e81 100644 --- a/cmd/network/upgrade_test.go +++ b/cmd/network/upgrade_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" "time" + + "github.com/tronprotocol/tron-deployment/internal/state" ) func TestVerifyNodeInvokesVerifyForEachProjectedNode(t *testing.T) { @@ -112,7 +114,7 @@ func TestFinishWithFailureRollsBackNodeWhoseVerifyFailed(t *testing.T) { return nil } - err := finishWithFailure(context.Background(), "", "net", "trond", + err := finishWithFailure(context.Background(), "", "net", "trond", nil, []upgradeStep{{Node: "net-node0", Phase: "upgrade", Status: "ok"}, {Node: "net-node0", Phase: "verify", Status: "failed"}}, []string{"net-node0"}, time.Now(), "net-node0", errors.New("verify failed")) @@ -168,9 +170,34 @@ func TestFinishWithFailureDoesNotPassUpgradeJarArgumentsToRollback(t *testing.T) got = append([]string(nil), argv...) return nil } - _ = finishWithFailure(context.Background(), "", "net", "trond", nil, []string{"net-node0"}, time.Now(), "net-node1", errors.New("failed")) + _ = finishWithFailure(context.Background(), "", "net", "trond", nil, nil, []string{"net-node0"}, time.Now(), "net-node1", errors.New("failed")) want := []string{"rollback", "net-node0"} if !reflect.DeepEqual(got, want) { t.Fatalf("rollback args = %v, want %v", got, want) } } + +func TestRunNetworkChildOnlyInjectsUpgradeEnvForRollback(t *testing.T) { + st := &state.DeploymentState{Nodes: []state.ManagedNode{{Name: "net-node0", Runtime: "jar"}}} + old := runChildWithEnv + oldChild := runChild + defer func() { runChildWithEnv = old; runChild = oldChild }() + var envCalls [][]string + runChildWithEnv = func(_ context.Context, _ string, env []string, _ ...string) error { + envCalls = append(envCalls, append([]string(nil), env...)) + return nil + } + runChild = func(context.Context, string, ...string) error { return nil } + if err := runNetworkChild(context.Background(), "trond", st, "net-node0", "upgrade", "net-node0"); err != nil { + t.Fatal(err) + } + if len(envCalls) != 1 || !reflect.DeepEqual(envCalls[0], []string{networkUpgradePreserveEnv}) { + t.Fatalf("upgrade env = %v, want [%s]", envCalls, networkUpgradePreserveEnv) + } + if err := runNetworkChild(context.Background(), "trond", st, "net-node0", "rollback", "net-node0"); err != nil { + t.Fatal(err) + } + if len(envCalls) != 2 || !reflect.DeepEqual(envCalls[1], []string{networkUpgradeRestoreEnv}) { + t.Fatalf("rollback env = %v, want [%s]", envCalls, networkUpgradeRestoreEnv) + } +} diff --git a/cmd/plan.go b/cmd/plan.go index b0039c07..248354b6 100644 --- a/cmd/plan.go +++ b/cmd/plan.go @@ -51,6 +51,9 @@ func runPlan(cmd *cobra.Command, args []string) error { if err != nil { return exitWithError("VALIDATION_ERROR", output.ExitValidationError, err.Error()) } + if len(parsed.Nodes) == 0 { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node") + } // 2. Load current state store, err := state.NewStore(statePath()) @@ -65,7 +68,14 @@ func runPlan(cmd *cobra.Command, args []string) error { existing := store.GetNode(deployState, parsed.Name) if parsed.Target.AutoPorts && existing != nil { - apply.RestoreAutoPorts(&parsed.Nodes[0], existing) + rawParsed, rawErr := intent.LoadRaw(planIntentPath) + if rawErr != nil { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, rawErr.Error()) + } + if len(rawParsed.Nodes) == 0 { + return exitWithError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node") + } + apply.RestoreAutoPorts(&parsed.Nodes[0], existing, &rawParsed.Nodes[0]) } rawIntent, err := os.ReadFile(planIntentPath) if err != nil { diff --git a/cmd/snapshot/download.go b/cmd/snapshot/download.go index 48efde0d..e95e97df 100644 --- a/cmd/snapshot/download.go +++ b/cmd/snapshot/download.go @@ -1,12 +1,10 @@ package snapshot import ( - "bufio" "errors" "fmt" "os" "path/filepath" - "strings" "time" "github.com/spf13/cobra" @@ -186,6 +184,10 @@ func runDownload(cmd *cobra.Command, _ []string) error { res, err := snapshot.Download(cmd.Context(), opts) if err != nil { + var running *snapshot.RunningNodeDestinationError + if errors.As(err, &running) { + return output.NewError("NODE_RUNNING", output.ExitGeneralError, err.Error()).WithSuggestions("Stop the node first: trond stop " + running.NodeName) + } var ow *snapshot.OverwriteError if errors.As(err, &ow) { return output.NewError("HUMAN_REQUIRED", output.ExitHumanRequired, ow.Error()) @@ -239,145 +241,15 @@ func runDownload(cmd *cobra.Command, _ []string) error { // controls the ordinary existing-database overwrite prompt; it never bypasses // this data-corruption guard. func refuseRunningNodeDestination(dest, explicitNode string) error { - store, err := state.NewStore(paths.State()) - if err != nil { - return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) - } - st, err := store.Load() - if err != nil { - return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) - } - var matched *state.ManagedNode - if explicitNode != "" { - matched = store.GetNode(st, explicitNode) - } else { - for i := range st.Nodes { - if st.Nodes[i].Status != "running" && st.Nodes[i].Status != "error" { - continue - } - if managedNodeDestinationMatches(dest, &st.Nodes[i]) { - matched = &st.Nodes[i] - break - } - } - } - if matched == nil || (matched.Status != "running" && matched.Status != "error") { + err := snapshot.RefuseRunningNodeDestination(dest, explicitNode) + if err == nil { return nil } - // An error can mean restart failed while the container is still alive; - // treat it as unsafe until the operator explicitly stops the node. - return output.NewError("NODE_RUNNING", output.ExitGeneralError, - fmt.Sprintf("refusing to write snapshot into running node %q data directory", matched.Name)). - WithSuggestions("Stop the node first: trond stop " + matched.Name) -} - -func managedNodeDestinationMatches(dest string, node *state.ManagedNode) bool { - if node == nil || dest == "" { - return false - } - destAbs, err := filepath.Abs(filepath.Clean(dest)) - if err != nil { - return false - } - roots := make([]string, 0, 2) - if node.InstallPath != "" && node.Runtime != "docker" { - rootAbs, rootErr := filepath.Abs(filepath.Clean(node.InstallPath)) - if rootErr == nil { - roots = append(roots, rootAbs) - } - } - if node.StorageRoot != "" { - storageAbs, storageErr := filepath.Abs(filepath.Clean(node.StorageRoot)) - if storageErr == nil { - roots = append(roots, storageAbs) - } - } - if node.Runtime == "docker" && node.StorageRoot == "" { - if fallback, ok := legacyDockerStorageRoot(node.Name); ok { - roots = append(roots, fallback) - } - } - for _, root := range roots { - candidates := []string{root, filepath.Join(root, "output-directory"), filepath.Join(root, "output-directory", "database"), filepath.Join(root, "database")} - if filepath.Base(root) == "output-directory" { - candidates = append(candidates, filepath.Dir(root)) - } - for _, candidate := range candidates { - if samePath(destAbs, candidate) { - return true - } - } - } - return false -} - -// legacyDockerStorageRoot recovers bind sources from pre-storage_root state. -// Compose is generated in a stable, one-volume-per-line form; parse only the -// volume targeting java-tron's data directory. -func legacyDockerStorageRoot(name string) (string, bool) { - // Transitional fallback for pre-storage_root state; remove after all nodes are re-applied (next major). - compose := filepath.Join(paths.Deployments(), name, "docker-compose.yaml") - f, err := os.Open(compose) - if err != nil { - fmt.Fprintf(os.Stderr, "warning: state predates storage_root tracking; guard incomplete, please re-apply %s\n", name) - return "", false - } - defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - marker := ":/java-tron/output-directory" - if i := strings.Index(line, marker); i >= 0 { - source := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line[:i]), "-")) - if strings.HasPrefix(source, "/") { - return source, true - } - if strings.HasPrefix(source, "./") { - return filepath.Join(paths.Deployments(), name, source), true - } - if source != "" { - fmt.Fprintf(os.Stderr, "warning: named volume — host path unresolvable; guard cannot cover node %s\n", name) - return "", false - } - break - } - } - if err := scanner.Err(); err != nil { - fmt.Fprintf(os.Stderr, "warning: unable to parse compose file; guard incomplete, please re-apply %s: %v\n", name, err) - return "", false - } - fmt.Fprintf(os.Stderr, "warning: compose bind source not found; guard incomplete, please re-apply %s\n", name) - return "", false -} - -func samePath(left, right string) bool { - left = canonicalPath(left) - right = canonicalPath(right) - return left == right -} - -func canonicalPath(path string) string { - path = filepath.Clean(path) - if resolved, err := filepath.EvalSymlinks(path); err == nil { - return filepath.Clean(resolved) - } - // The destination may not exist yet. Resolve the nearest existing parent - // so a symlinked storage root still compares equal before extraction. - missing := []string{} - for current := path; ; current = filepath.Dir(current) { - if resolved, err := filepath.EvalSymlinks(current); err == nil { - for i := len(missing) - 1; i >= 0; i-- { - resolved = filepath.Join(resolved, missing[i]) - } - return filepath.Clean(resolved) - } - parent := filepath.Dir(current) - if parent == current { - break - } - missing = append(missing, filepath.Base(current)) + var running *snapshot.RunningNodeDestinationError + if errors.As(err, &running) { + return output.NewError("NODE_RUNNING", output.ExitGeneralError, err.Error()).WithSuggestions("Stop the node first: trond stop " + running.NodeName) } - return path + return output.NewError("STATE_ERROR", output.ExitGeneralError, err.Error()) } // downloadPayload builds the `-o json` body for a completed foreground diff --git a/cmd/start.go b/cmd/start.go index 8db396e1..85ae12be 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -19,8 +19,9 @@ var startCmd = &cobra.Command{ // saveStartState is a test injection seam; production default persists via nodeContext.SaveState. var saveStartState = func(nc *nodeContext) error { return nc.SaveState() } -// resolveStartNodeContext is a test injection seam; production default resolves a read-only node context. -var resolveStartNodeContext = resolveNodeContext +// resolveStartNodeContext is a test injection seam; production default keeps +// the state write lock until the caller has persisted its mutation. +var resolveStartNodeContext = resolveNodeContextForWrite func init() { rootCmd.AddCommand(startCmd) diff --git a/cmd/start_test.go b/cmd/start_test.go new file mode 100644 index 00000000..7c6fd2af --- /dev/null +++ b/cmd/start_test.go @@ -0,0 +1,12 @@ +package cmd + +import ( + "reflect" + "testing" +) + +func TestStartUsesWriteLockedResolverByDefault(t *testing.T) { + if got, want := reflect.ValueOf(resolveStartNodeContext).Pointer(), reflect.ValueOf(resolveNodeContextForWrite).Pointer(); got != want { + t.Fatalf("start resolver = %x, want write-locked resolver %x", got, want) + } +} diff --git a/internal/apply/hash.go b/internal/apply/hash.go index 480ed19d..ffc00b5d 100644 --- a/internal/apply/hash.go +++ b/internal/apply/hash.go @@ -51,27 +51,39 @@ func recordedMonitoringEnabled(existing *state.ManagedNode) bool { return existing != nil && existing.Monitoring != nil && existing.Monitoring.Enabled } -// RestoreAutoPorts restores all persisted auto-allocated ports into an intent. -func RestoreAutoPorts(node *intent.NodeSpec, existing *state.ManagedNode) { - if existing.HTTPPort != 0 { - node.Ports.HTTP = existing.HTTPPort +// RestoreAutoPorts restores persisted ports into an intent. The optional +// explicit is optionally the raw, pre-defaults intent node. Its non-zero +// ports win over persisted values; zero is the documented unset convention. +// The optional form preserves compatibility with callers without raw intent. +func RestoreAutoPorts(node *intent.NodeSpec, existing *state.ManagedNode, explicit ...*intent.NodeSpec) { + var userPorts *intent.PortMapping + if len(explicit) > 0 && explicit[0] != nil { + userPorts = &explicit[0].Ports } - if existing.GRPCPort != 0 { - node.Ports.GRPC = existing.GRPCPort - } - if existing.P2PPort != 0 { - node.Ports.P2P = existing.P2PPort - } - if existing.MetricsPort != 0 { - node.Ports.Metrics = existing.MetricsPort - } - if existing.SolidityHTTPPort != 0 { - node.Ports.SolidityHTTP = existing.SolidityHTTPPort + if existing == nil { + return } - if existing.SolidityGRPCPort != 0 { - node.Ports.SolidityGRPC = existing.SolidityGRPCPort + restore := func(current, persisted, user int) int { + if userPorts != nil && user != 0 { + return current + } + if persisted != 0 { + return persisted + } + return current } - if existing.JSONRPCPort != 0 { - node.Ports.JSONRPC = existing.JSONRPCPort + node.Ports.HTTP = restore(node.Ports.HTTP, existing.HTTPPort, portValue(userPorts, func(p intent.PortMapping) int { return p.HTTP })) + node.Ports.GRPC = restore(node.Ports.GRPC, existing.GRPCPort, portValue(userPorts, func(p intent.PortMapping) int { return p.GRPC })) + node.Ports.P2P = restore(node.Ports.P2P, existing.P2PPort, portValue(userPorts, func(p intent.PortMapping) int { return p.P2P })) + node.Ports.Metrics = restore(node.Ports.Metrics, existing.MetricsPort, portValue(userPorts, func(p intent.PortMapping) int { return p.Metrics })) + node.Ports.SolidityHTTP = restore(node.Ports.SolidityHTTP, existing.SolidityHTTPPort, portValue(userPorts, func(p intent.PortMapping) int { return p.SolidityHTTP })) + node.Ports.SolidityGRPC = restore(node.Ports.SolidityGRPC, existing.SolidityGRPCPort, portValue(userPorts, func(p intent.PortMapping) int { return p.SolidityGRPC })) + node.Ports.JSONRPC = restore(node.Ports.JSONRPC, existing.JSONRPCPort, portValue(userPorts, func(p intent.PortMapping) int { return p.JSONRPC })) +} + +func portValue(ports *intent.PortMapping, get func(intent.PortMapping) int) int { + if ports == nil { + return 0 } + return get(*ports) } diff --git a/internal/mcp/tools_lifecycle.go b/internal/mcp/tools_lifecycle.go index 5e3a318f..998d2f63 100644 --- a/internal/mcp/tools_lifecycle.go +++ b/internal/mcp/tools_lifecycle.go @@ -69,6 +69,9 @@ func planTool(ctx context.Context, _ *mcp.CallToolRequest, args planArg) (*mcp.C if err != nil { return errResult(err) } + if len(parsed.Nodes) == 0 { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node")) + } node := &parsed.Nodes[0] store, st, err := state.Load(paths.State()) if err != nil { @@ -80,7 +83,14 @@ func planTool(ctx context.Context, _ *mcp.CallToolRequest, args planArg) (*mcp.C } existing := store.GetNode(st, parsed.Name) if parsed.Target.AutoPorts && existing != nil { - apply.RestoreAutoPorts(node, existing) + rawParsed, rawErr := intent.LoadRaw(args.Path) + if rawErr != nil { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, rawErr.Error())) + } + if len(rawParsed.Nodes) == 0 { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node")) + } + apply.RestoreAutoPorts(node, existing, &rawParsed.Nodes[0]) } planned, err := apply.Plan(parsed, existing, intentBytes, apply.FindTemplatesDir()) if err != nil { @@ -126,6 +136,9 @@ func applyTool(ctx context.Context, _ *mcp.CallToolRequest, args applyArgs) (*mc if err != nil { return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, err.Error())) } + if len(parsed.Nodes) == 0 { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node")) + } // --require-private (early gate, precedence): refuse a non-private intent // BEFORE target resolution and the HUMAN_REQUIRED change gate, so the @@ -168,7 +181,14 @@ func applyTool(ctx context.Context, _ *mcp.CallToolRequest, args applyArgs) (*mc return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, readErr.Error())) } if parsed.Target.AutoPorts && existing != nil { - apply.RestoreAutoPorts(&parsed.Nodes[0], existing) + rawParsed, rawErr := intent.LoadRaw(args.Path) + if rawErr != nil { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, rawErr.Error())) + } + if len(rawParsed.Nodes) == 0 { + return errResult(output.NewError("VALIDATION_ERROR", output.ExitValidationError, "intent must contain at least one node")) + } + apply.RestoreAutoPorts(&parsed.Nodes[0], existing, &rawParsed.Nodes[0]) } templateDir := apply.FindTemplatesDir() planned, err := apply.Plan(parsed, existing, intentBytes, templateDir) diff --git a/internal/mcp/tools_snapshot.go b/internal/mcp/tools_snapshot.go index 60a452e7..88aa90da 100644 --- a/internal/mcp/tools_snapshot.go +++ b/internal/mcp/tools_snapshot.go @@ -2,6 +2,7 @@ package mcp import ( "context" + "errors" "fmt" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -186,6 +187,10 @@ func snapshotDownloadTool(ctx context.Context, req *mcp.CallToolRequest, args sn res, err := snapshot.Download(ctx, opts) if err != nil { + var running *snapshot.RunningNodeDestinationError + if errors.As(err, &running) { + return errResult(output.NewError("NODE_RUNNING", output.ExitGeneralError, err.Error()).WithSuggestions("Stop the node first: trond stop " + running.NodeName)) + } return errResult(err) } payload := map[string]any{ diff --git a/internal/runtime/upgrade.go b/internal/runtime/upgrade.go index 0f669682..5fdab3a2 100644 --- a/internal/runtime/upgrade.go +++ b/internal/runtime/upgrade.go @@ -3,6 +3,7 @@ package runtime import ( "context" "fmt" + "os" "path/filepath" "regexp" "strings" @@ -102,10 +103,21 @@ func (r *DockerRuntime) PrepareArtifact(ctx context.Context, name string, opts U type jarArtifactTransaction struct { r *JarRuntime name, candidate, live, backup string - activated bool + activated, restore bool } func (t *jarArtifactTransaction) Activate(ctx context.Context) error { + if t.restore { + if _, err := t.r.target.Exec(ctx, "mv", t.live, t.candidate); err != nil { + return fmt.Errorf("stage current jar: %w", err) + } + if _, err := t.r.target.Exec(ctx, "mv", t.backup, t.live); err != nil { + _, _ = t.r.target.Exec(ctx, "mv", t.candidate, t.live) + return fmt.Errorf("restore backup: %w", err) + } + t.activated = true + return nil + } if _, err := t.r.target.Exec(ctx, "mv", t.live, t.backup); err != nil { return fmt.Errorf("backup jar: %w", err) } @@ -138,7 +150,11 @@ func (t *jarArtifactTransaction) Rollback(ctx context.Context) error { return nil } func (t *jarArtifactTransaction) Cleanup(ctx context.Context) error { - _, err := t.r.target.Exec(ctx, "rm", "-f", t.candidate, t.backup) + args := []string{"-f", t.candidate, t.backup} + if os.Getenv("TROND_PRESERVE_BACKUP") == "1" { + args = []string{"-f", t.candidate} + } + _, err := t.r.target.Exec(ctx, "rm", args...) return err } @@ -146,19 +162,28 @@ func (r *JarRuntime) PrepareArtifact(ctx context.Context, name string, opts Upgr if err := validateUpgradeVersion(opts.Version); err != nil { return nil, err } - if opts.JarURL == "" { - return nil, fmt.Errorf("jar URL is required for artifact upgrade; provide --jar-url") - } if r.purgeInstallPath == "" { return nil, fmt.Errorf("install path is unavailable for artifact upgrade") } jarPath := filepath.Join(r.purgeInstallPath, "FullNode.jar") candidate := jarPath + ".candidate" + backup := jarPath + ".upgrade.backup" + if opts.JarURL == "" { + if os.Getenv("TROND_NETWORK_UPGRADE") != "1" { + return nil, fmt.Errorf("jar URL is required for artifact upgrade; provide --jar-url") + } + // A network rollback consumes the backup made by the preceding + // upgrade. No download or operator-supplied URL is needed. + if _, err := r.target.Exec(ctx, "test", "-e", backup); err != nil { + return nil, fmt.Errorf("no upgrade backup found; cannot restore network upgrade") + } + return &jarArtifactTransaction{r: r, name: name, candidate: candidate, live: jarPath, backup: backup, restore: true}, nil + } if err := r.downloadJar(ctx, opts.JarURL, candidate, opts.JarSHA256); err != nil { _, _ = r.target.Exec(ctx, "rm", "-f", candidate) return nil, fmt.Errorf("download jar: %w", err) } - return &jarArtifactTransaction{r: r, name: name, candidate: candidate, live: jarPath, backup: jarPath + ".upgrade.backup"}, nil + return &jarArtifactTransaction{r: r, name: name, candidate: candidate, live: jarPath, backup: backup}, nil } var _ ArtifactUpgrader = (*DockerRuntime)(nil) diff --git a/internal/runtime/upgrade_test.go b/internal/runtime/upgrade_test.go index fc2436f3..1c4c8ff9 100644 --- a/internal/runtime/upgrade_test.go +++ b/internal/runtime/upgrade_test.go @@ -40,6 +40,33 @@ func TestSwapComposeImageTag(t *testing.T) { } } +func TestJarPrepareArtifactNetworkUpgradeRequiresBackup(t *testing.T) { + old := os.Getenv("TROND_NETWORK_UPGRADE") + t.Setenv("TROND_NETWORK_UPGRADE", "1") + t.Cleanup(func() { _ = os.Setenv("TROND_NETWORK_UPGRADE", old) }) + r := NewJarRuntime(&missingBackupUpgradeTarget{fakeTarget: newFakeTarget()}) + r.SetPurgeInstallPath("/install") + _, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2"}) + if err == nil || !strings.Contains(err.Error(), "no upgrade backup found") { + t.Fatalf("error = %v, want missing network upgrade backup", err) + } +} + +func TestJarPrepareArtifactOrdinaryUpgradeDoesNotRestoreStaleBackup(t *testing.T) { + f := &jarUpgradeFakeTarget{newFakeTarget()} + f.files["/opt/tron/FullNode.jar.upgrade.backup"] = []byte("stale") + r := NewJarRuntime(f) + r.SetPurgeInstallPath("/opt/tron") + t.Setenv("TROND_NETWORK_UPGRADE", "") + _, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2"}) + if err == nil || !strings.Contains(err.Error(), "jar URL is required") { + t.Fatalf("error = %v, want ordinary upgrade URL validation", err) + } + if len(f.cmds) != 0 { + t.Fatalf("ordinary upgrade attempted restore commands: %v", f.cmds) + } +} + type upgradeFakeTarget struct { *fakeTarget pullErr error @@ -47,6 +74,17 @@ type upgradeFakeTarget struct { type jarUpgradeFakeTarget struct{ *fakeTarget } +type backupJarUpgradeFakeTarget struct{ *jarUpgradeFakeTarget } + +type missingBackupUpgradeTarget struct{ *fakeTarget } + +func (f *missingBackupUpgradeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { + if cmd == "test" { + return nil, errors.New("missing") + } + return f.fakeTarget.Exec(ctx, cmd, args...) +} + type remoteJarUpgradeFakeTarget struct { *jarUpgradeFakeTarget uploads int @@ -96,12 +134,23 @@ func (f *failingUpgradeTarget) Exec(ctx context.Context, cmd string, args ...str func (f *jarUpgradeFakeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { f.cmds = append(f.cmds, append([]string{cmd}, args...)) + if cmd == "test" { + return nil, errors.New("missing") + } if cmd == "sha256sum" { return []byte("deadbeef /opt/tron/FullNode.jar\n"), nil } return nil, nil } +func (f *backupJarUpgradeFakeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { + f.cmds = append(f.cmds, append([]string{cmd}, args...)) + if cmd == "test" { + return nil, nil + } + return nil, nil +} + func (f *upgradeFakeTarget) Exec(ctx context.Context, cmd string, args ...string) ([]byte, error) { f.cmds = append(f.cmds, append([]string{cmd}, args...)) if cmd == "docker" && len(args) >= 2 && args[0] == "pull" { @@ -224,7 +273,7 @@ func countCommand(cmds [][]string, name string) int { func TestJarRuntimeUpgradeArtifact(t *testing.T) { t.Run("missing URL", func(t *testing.T) { - r := NewJarRuntime(newFakeTarget()) + r := NewJarRuntime(&jarUpgradeFakeTarget{newFakeTarget()}) r.SetPurgeInstallPath("/opt/tron") if _, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "2"}); err == nil || !strings.Contains(err.Error(), "jar URL") { t.Fatal(err) @@ -339,6 +388,57 @@ func TestJarRuntimeUpgradeArtifact(t *testing.T) { }) } +func TestJarArtifactCleanupPreservesBackupForNetworkUpgrade(t *testing.T) { + f := &backupJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + r := NewJarRuntime(f) + t.Setenv("TROND_PRESERVE_BACKUP", "1") + tx := &jarArtifactTransaction{r: r, candidate: "/opt/tron/FullNode.jar.candidate", live: "/opt/tron/FullNode.jar", backup: "/opt/tron/FullNode.jar.upgrade.backup"} + if err := tx.Cleanup(context.Background()); err != nil { + t.Fatal(err) + } + if got := f.cmds[len(f.cmds)-1]; !reflect.DeepEqual(got, []string{"rm", "-f", tx.candidate}) { + t.Fatalf("cleanup commands = %v, want candidate-only removal", f.cmds) + } +} + +func TestJarArtifactCleanupDeletesBackupWithoutPreserveSignal(t *testing.T) { + f := &backupJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + t.Setenv("TROND_NETWORK_UPGRADE", "") + t.Setenv("TROND_PRESERVE_BACKUP", "") + tx := &jarArtifactTransaction{r: NewJarRuntime(f), candidate: "/opt/tron/FullNode.jar.candidate", live: "/opt/tron/FullNode.jar", backup: "/opt/tron/FullNode.jar.upgrade.backup"} + if err := tx.Cleanup(context.Background()); err != nil { + t.Fatal(err) + } + if got := f.cmds[len(f.cmds)-1]; !reflect.DeepEqual(got, []string{"rm", "-f", tx.candidate, tx.backup}) { + t.Fatalf("cleanup commands = %v, want candidate and backup removal", f.cmds) + } +} + +func TestJarArtifactRollbackConsumesPreservedBackupWithoutURL(t *testing.T) { + f := &backupJarUpgradeFakeTarget{jarUpgradeFakeTarget: &jarUpgradeFakeTarget{newFakeTarget()}} + r := NewJarRuntime(f) + r.SetPurgeInstallPath("/opt/tron") + t.Setenv("TROND_NETWORK_UPGRADE", "1") + tx, err := r.PrepareArtifact(context.Background(), "n1", UpgradeOpts{Version: "1"}) + if err != nil { + t.Fatal(err) + } + if err := tx.Activate(context.Background()); err != nil { + t.Fatal(err) + } + if err := tx.Start(context.Background()); err != nil { + t.Fatal(err) + } + if got := f.cmds; !reflect.DeepEqual(got, [][]string{ + {"test", "-e", "/opt/tron/FullNode.jar.upgrade.backup"}, + {"mv", "/opt/tron/FullNode.jar", "/opt/tron/FullNode.jar.candidate"}, + {"mv", "/opt/tron/FullNode.jar.upgrade.backup", "/opt/tron/FullNode.jar"}, + {"systemctl", "start", "tron-n1.service"}, + }) { + t.Fatalf("commands = %v", got) + } +} + func TestJarDeployInterruptedArtifactConvergesOnRerun(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/snapshot/download.go b/internal/snapshot/download.go index b36cf16d..40f86fd5 100644 --- a/internal/snapshot/download.go +++ b/internal/snapshot/download.go @@ -294,6 +294,9 @@ func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error if opts.Backup == "" { return nil, errors.New("backup is required") } + if err := RefuseRunningNodeDestination(opts.DestDir, ""); err != nil { + return nil, err + } // Validate the pin before the transfer, not after: a typo'd digest // should not cost the operator a multi-hour download. expectedSHA256, err := normalizeSHA256(opts.ExpectedSHA256) diff --git a/internal/snapshot/running_node.go b/internal/snapshot/running_node.go new file mode 100644 index 00000000..261b8ab8 --- /dev/null +++ b/internal/snapshot/running_node.go @@ -0,0 +1,139 @@ +package snapshot + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/state" +) + +// RunningNodeDestinationError indicates that a snapshot destination belongs +// to a managed node which may still have its chain database open. +type RunningNodeDestinationError struct{ NodeName string } + +func (e *RunningNodeDestinationError) Error() string { + return fmt.Sprintf("refusing to write snapshot into running node %q data directory", e.NodeName) +} + +// RefuseRunningNodeDestination protects managed node databases from being +// overwritten while java-tron (jar) or Docker holds them open. Force never +// bypasses this guard. +func RefuseRunningNodeDestination(dest, explicitNode string) error { + store, err := state.NewStore(paths.State()) + if err != nil { + return err + } + st, err := store.Load() + if err != nil { + return err + } + var matched *state.ManagedNode + if explicitNode != "" { + matched = store.GetNode(st, explicitNode) + } else { + for i := range st.Nodes { + if st.Nodes[i].Status != "running" && st.Nodes[i].Status != "error" { + continue + } + if managedNodeDestinationMatches(dest, &st.Nodes[i]) { + matched = &st.Nodes[i] + break + } + } + } + if matched == nil || (matched.Status != "running" && matched.Status != "error") { + return nil + } + return &RunningNodeDestinationError{NodeName: matched.Name} +} + +func managedNodeDestinationMatches(dest string, node *state.ManagedNode) bool { + if node == nil || dest == "" { + return false + } + destAbs, err := filepath.Abs(filepath.Clean(dest)) + if err != nil { + return false + } + roots := make([]string, 0, 2) + if node.InstallPath != "" && node.Runtime != "docker" { + if root, e := filepath.Abs(filepath.Clean(node.InstallPath)); e == nil { + roots = append(roots, root) + } + } + if node.StorageRoot != "" { + if root, e := filepath.Abs(filepath.Clean(node.StorageRoot)); e == nil { + roots = append(roots, root) + } + } + if node.Runtime == "docker" && node.StorageRoot == "" { + if root, ok := legacyDockerStorageRoot(node.Name); ok { + roots = append(roots, root) + } + } + for _, root := range roots { + candidates := []string{root, filepath.Join(root, "output-directory"), filepath.Join(root, "output-directory", "database"), filepath.Join(root, "database")} + if filepath.Base(root) == "output-directory" { + candidates = append(candidates, filepath.Dir(root)) + } + for _, candidate := range candidates { + if samePath(destAbs, candidate) { + return true + } + } + } + return false +} + +func legacyDockerStorageRoot(name string) (string, bool) { + compose := filepath.Join(paths.Deployments(), name, "docker-compose.yaml") + f, err := os.Open(compose) + if err != nil { + return "", false + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + marker := ":/java-tron/output-directory" + if i := strings.Index(line, marker); i >= 0 { + source := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line[:i]), "-")) + if strings.HasPrefix(source, "/") { + return source, true + } + if strings.HasPrefix(source, "./") { + return filepath.Join(paths.Deployments(), name, source), true + } + return "", false + } + } + return "", false +} + +func samePath(left, right string) bool { return canonicalPath(left) == canonicalPath(right) } + +func canonicalPath(path string) string { + path = filepath.Clean(path) + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return filepath.Clean(resolved) + } + missing := []string{} + for current := path; ; current = filepath.Dir(current) { + if resolved, err := filepath.EvalSymlinks(current); err == nil { + for i := len(missing) - 1; i >= 0; i-- { + resolved = filepath.Join(resolved, missing[i]) + } + return filepath.Clean(resolved) + } + parent := filepath.Dir(current) + if parent == current { + break + } + missing = append(missing, filepath.Base(current)) + } + return path +} From 8ad9c5aab9bab33ff4ffb87bb17b36e5c83ff624 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 24 Aug 2026 22:20:37 +0800 Subject: [PATCH 34/49] fix(diagnose,heal): pass recorded node network into CLI check opts (AUD-044) CLI diagnose and auto-heal built diagnosis.CheckOpts without Network, so PeersChecker (private >=1 peer vs >=3) and DiskChecker (mainnet thresholds) applied wrong thresholds on the CLI paths. MCP paths already passed node.Network. Plumb nc.Node.Network through both constructors, add package-var checker injection seams, and cover both command paths with recorded-network propagation tests. Oracle reviewed: APPROVE. --- cmd/diagnose.go | 5 +++- cmd/heal.go | 8 +++--- cmd/heal_test.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/cmd/diagnose.go b/cmd/diagnose.go index 99faf02e..3bd85345 100644 --- a/cmd/diagnose.go +++ b/cmd/diagnose.go @@ -10,6 +10,8 @@ import ( "github.com/tronprotocol/tron-deployment/internal/output" ) +var diagnoseCheckers = diagnosis.AllCheckers + var diagnoseCmd = &cobra.Command{ Use: "diagnose ", Short: "Run diagnostic checks on a node", @@ -35,11 +37,12 @@ func runDiagnose(cmd *cobra.Command, args []string) error { opts := diagnosis.CheckOpts{ NodeName: nc.Node.Name, Runtime: nc.Node.Runtime, + Network: nc.Node.Network, HTTPPort: nc.Node.HTTPPort, GRPCPort: nc.Node.GRPCPort, } - checkers := diagnosis.AllCheckers() + checkers := diagnoseCheckers() var results []diagnosis.CheckResult for _, checker := range checkers { diff --git a/cmd/heal.go b/cmd/heal.go index 028de725..5e7a9f31 100644 --- a/cmd/heal.go +++ b/cmd/heal.go @@ -31,8 +31,9 @@ import ( // the next one returns healed=[] / skipped=[] / still_failing=[]. var ( - healDryRun bool - healOnly []string // restrict to specific check names; defaults = all + healDryRun bool + healOnly []string // restrict to specific check names; defaults = all + healCheckers = diagnosis.AllCheckers ) var autoHealCmd = &cobra.Command{ @@ -112,6 +113,7 @@ func runAutoHeal(cmd *cobra.Command, args []string) error { opts := diagnosis.CheckOpts{ NodeName: nc.Node.Name, NodeType: "", // diagnose doesn't read this from state today; checkers cope + Network: nc.Node.Network, Runtime: nc.Node.Runtime, HTTPPort: nc.Node.HTTPPort, GRPCPort: nc.Node.GRPCPort, @@ -119,7 +121,7 @@ func runAutoHeal(cmd *cobra.Command, args []string) error { } ctx, cancel := context.WithTimeout(cmd.Context(), 30*time.Second) defer cancel() - checkers := diagnosis.AllCheckers() + checkers := healCheckers() results := make([]diagnosis.CheckResult, 0, len(checkers)) for _, c := range checkers { if len(healOnly) > 0 && !contains(healOnly, c.Name()) { diff --git a/cmd/heal_test.go b/cmd/heal_test.go index c3d8d866..bec28f4c 100644 --- a/cmd/heal_test.go +++ b/cmd/heal_test.go @@ -1,11 +1,23 @@ package cmd import ( + "context" "testing" "github.com/tronprotocol/tron-deployment/internal/diagnosis" + "github.com/tronprotocol/tron-deployment/internal/paths" + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) +type captureNetworkChecker struct{ got *string } + +func (c captureNetworkChecker) Name() string { return "capture_network" } +func (c captureNetworkChecker) Run(_ context.Context, _ target.Target, opts diagnosis.CheckOpts) diagnosis.CheckResult { + *c.got = opts.Network + return diagnosis.CheckResult{Name: c.Name(), Status: diagnosis.StatusPass, Message: "captured"} +} + // TestProposeHealAction pins the (check, current state) → action // mapping for `trond auto-heal`. Every case here is a contract: // adding a new mapping requires adding a case here so the table @@ -82,3 +94,60 @@ func TestProposeHealAction(t *testing.T) { }) } } + +func TestRunDiagnosePassesRecordedNetworkToChecker(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "diag-net", Network: "private", Runtime: "docker", Status: "stopped", + }}}); err != nil { + t.Fatal(err) + } + var got string + old := diagnoseCheckers + diagnoseCheckers = func() []diagnosis.Checker { return []diagnosis.Checker{captureNetworkChecker{got: &got}} } + t.Cleanup(func() { diagnoseCheckers = old }) + + cmd := diagnoseCmd + cmd.SetArgs([]string{"diag-net"}) + if err := runDiagnose(cmd, []string{"diag-net"}); err != nil { + t.Fatal(err) + } + if got != "private" { + t.Fatalf("checker network=%q, want private", got) + } +} + +func TestRunAutoHealPassesRecordedNetworkToChecker(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "heal-net", Network: "nile", Runtime: "docker", Status: "stopped", + }}}); err != nil { + t.Fatal(err) + } + var got string + old := healCheckers + healCheckers = func() []diagnosis.Checker { return []diagnosis.Checker{captureNetworkChecker{got: &got}} } + t.Cleanup(func() { healCheckers = old }) + + oldDryRun := healDryRun + healDryRun = true + t.Cleanup(func() { healDryRun = oldDryRun }) + cmd := autoHealCmd + cmd.SetContext(context.Background()) + if err := runAutoHeal(cmd, []string{"heal-net"}); err != nil { + t.Fatal(err) + } + if got != "nile" { + t.Fatalf("checker network=%q, want nile", got) + } +} From 9e25486255562eea44a2ae70ff42161527070d08 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 25 Aug 2026 11:36:44 +0800 Subject: [PATCH 35/49] fix: address CI lint and e2e contract failures - goimports grouping, errcheck/unconvert/unused/gocritic/ineffassign cleanup across cmd/internal/tools - check nodeIntent error in network upgrade verifyNode - fix vacuous hash-stability assertion in apply_hash_test (SA4000) - sync apply/events output schemas with versioned intent_hash output; allow legacy bare-hex during migration; bump schema to 1.15.1 - render recipe step IDs as hyphenated dry-run labels; keep closeTargets after docker-network teardown in network destroy --- cmd/apply_hash_test.go | 7 +++++-- cmd/bootstrap_guard_test.go | 1 + cmd/config/witness_redaction_test.go | 1 + cmd/network/destroy.go | 10 ++++++++-- cmd/network/upgrade.go | 6 +++++- cmd/verify_config_test.go | 3 ++- cmd/witness_redaction_test.go | 3 ++- internal/apply/envvars.go | 3 ++- internal/build/source_test.go | 2 +- internal/mcp/conf_helpers.go | 2 +- internal/mcp/contract_p1_test.go | 9 --------- internal/mcp/drift_redaction_test.go | 3 ++- internal/mcp/tools_lifecycle.go | 7 ------- internal/recipe/runner.go | 8 ++++++-- internal/schema/embed.go | 2 +- internal/schema/files/apply.schema.json | 4 ++-- internal/schema/files/events.schema.json | 2 +- internal/schema/version_baseline.json | 6 +++--- internal/security/audit.go | 3 ++- internal/snapshot/download.go | 4 ++-- internal/state/load.go | 3 ++- internal/state/store.go | 3 ++- internal/target/ssh.go | 7 ++++--- schemas/output/apply.schema.json | 4 ++-- schemas/output/events.schema.json | 2 +- tools/txgen/broadcast.go | 11 ----------- 26 files changed, 58 insertions(+), 58 deletions(-) diff --git a/cmd/apply_hash_test.go b/cmd/apply_hash_test.go index 45e63f5a..ebe56d86 100644 --- a/cmd/apply_hash_test.go +++ b/cmd/apply_hash_test.go @@ -3,10 +3,11 @@ package cmd import ( "testing" + "gopkg.in/yaml.v3" + "github.com/tronprotocol/tron-deployment/internal/apply" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/state" - "gopkg.in/yaml.v3" ) func TestLegacyIntentHashMatchesRawHash(t *testing.T) { @@ -20,7 +21,9 @@ func TestLegacyIntentHashMatchesRawHash(t *testing.T) { func TestVersionedIntentHashIsStable(t *testing.T) { data := []byte("effective intent") - if apply.EffectiveIntentHash(data) != apply.EffectiveIntentHash(data) { + first := apply.EffectiveIntentHash(data) + second := apply.EffectiveIntentHash(append([]byte(nil), data...)) + if first != second { t.Fatal("versioned hash is unstable") } if apply.EffectiveIntentHash(data) == string(data) { diff --git a/cmd/bootstrap_guard_test.go b/cmd/bootstrap_guard_test.go index 13b06f77..db4c1f2f 100644 --- a/cmd/bootstrap_guard_test.go +++ b/cmd/bootstrap_guard_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/guard" "github.com/tronprotocol/tron-deployment/internal/output" ) diff --git a/cmd/config/witness_redaction_test.go b/cmd/config/witness_redaction_test.go index 85602435..101f64af 100644 --- a/cmd/config/witness_redaction_test.go +++ b/cmd/config/witness_redaction_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/spf13/cobra" + "github.com/tronprotocol/tron-deployment/internal/render" ) diff --git a/cmd/network/destroy.go b/cmd/network/destroy.go index b09a2947..d1cda158 100644 --- a/cmd/network/destroy.go +++ b/cmd/network/destroy.go @@ -116,7 +116,6 @@ func runDestroy(cmd *cobra.Command, args []string) error { continue } networkTargets[key] = tgt - defer closeTarget(tgt) } var removed []string @@ -158,6 +157,7 @@ func runDestroy(cmd *cobra.Command, args []string) error { } if err := store.Save(deployState); err != nil { + closeTargets(networkTargets) // State save failure is rare but real; surface it instead of // claiming success with a stale state file on disk. return output.NewError("STATE_ERROR", output.ExitGeneralError, @@ -171,7 +171,6 @@ func runDestroy(cmd *cobra.Command, args []string) error { fmt.Fprintf(cmd.ErrOrStderr(), "Warning: failed to remove monitoring stack: %v\n", err) } } - // Tear down the shared docker network the matching `network create` // stood up. Best-effort: a leftover network is harmless on next // run (create re-uses it), so we ignore failures rather than @@ -181,6 +180,7 @@ func runDestroy(cmd *cobra.Command, args []string) error { _, _ = networkTarget.Exec(cmd.Context(), "docker", "network", "rm", "trond-"+destroyConfirm) } } + closeTargets(networkTargets) if len(failures) > 0 { auditResult = "partial" @@ -241,3 +241,9 @@ func closeTarget(t target.Target) { _ = c.Close() } } + +func closeTargets(targets map[string]target.Target) { + for _, tgt := range targets { + closeTarget(tgt) + } +} diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index 1ba6bd71..5c1457c3 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -13,12 +13,13 @@ import ( "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + "github.com/tronprotocol/tron-deployment/internal/guard" "github.com/tronprotocol/tron-deployment/internal/intent" "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" - "gopkg.in/yaml.v3" ) // upgradeCmd does a rolling upgrade across every node in a private @@ -340,6 +341,9 @@ func verifyNode(ctx context.Context, exe, node, intentPath string, timeout time. for i := range parsed.Nodes { if fmt.Sprintf("%s-node%d", parsed.Name, i) == node { projected, _, _, err = nodeIntent(parsed, i) + if err != nil { + return fmt.Errorf("project node %s for verify: %w", node, err) + } break } } diff --git a/cmd/verify_config_test.go b/cmd/verify_config_test.go index 4050a71e..595a3dde 100644 --- a/cmd/verify_config_test.go +++ b/cmd/verify_config_test.go @@ -1,9 +1,10 @@ package cmd import ( - "github.com/tronprotocol/tron-deployment/internal/render" "strings" "testing" + + "github.com/tronprotocol/tron-deployment/internal/render" ) // TestLineDiff_PinsContractShape exercises the small diff helper diff --git a/cmd/witness_redaction_test.go b/cmd/witness_redaction_test.go index 45b2f7e0..34e07c09 100644 --- a/cmd/witness_redaction_test.go +++ b/cmd/witness_redaction_test.go @@ -1,9 +1,10 @@ package cmd import ( - "github.com/tronprotocol/tron-deployment/internal/render" "strings" "testing" + + "github.com/tronprotocol/tron-deployment/internal/render" ) // F3 regression tests for the two positional, LCS-free differs that diff --git a/internal/apply/envvars.go b/internal/apply/envvars.go index 66f0f0c6..a9057b02 100644 --- a/internal/apply/envvars.go +++ b/internal/apply/envvars.go @@ -1,8 +1,9 @@ package apply import ( - "github.com/tronprotocol/tron-deployment/internal/intent" "os" + + "github.com/tronprotocol/tron-deployment/internal/intent" ) // ResolveEnvVars resolves the legacy witness-key environment reference. diff --git a/internal/build/source_test.go b/internal/build/source_test.go index d87d14f7..d1f456d0 100644 --- a/internal/build/source_test.go +++ b/internal/build/source_test.go @@ -112,7 +112,7 @@ func TestSource_Resolve_NoUntrackedHashUnchanged(t *testing.T) { func TestSource_Resolve_IgnoredUntrackedDoesNotChangeHash(t *testing.T) { dir := initGitRepo(t) - baseline := resolveSource(t, dir).PatchHash + var baseline string if err := os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("ignored/\n"), 0o600); err != nil { t.Fatal(err) } diff --git a/internal/mcp/conf_helpers.go b/internal/mcp/conf_helpers.go index 2d35cf68..0a96001b 100644 --- a/internal/mcp/conf_helpers.go +++ b/internal/mcp/conf_helpers.go @@ -20,5 +20,5 @@ func readLiveConfigForMCP(ctx context.Context, tgt target.Target, node *state.Ma } return "", fmt.Errorf("docker exec cat: %w", err) } - return string(out), nil + return out, nil } diff --git a/internal/mcp/contract_p1_test.go b/internal/mcp/contract_p1_test.go index 42adccc4..d77759df 100644 --- a/internal/mcp/contract_p1_test.go +++ b/internal/mcp/contract_p1_test.go @@ -233,15 +233,6 @@ func writePlanFixture(t *testing.T, name string) string { } return p } -func mustRead(t *testing.T, p string) []byte { - t.Helper() - b, err := os.ReadFile(p) - if err != nil { - t.Fatal(err) - } - return b -} - func TestSnapshotDownloadMissingDestUsesValidationEnvelope(t *testing.T) { res, _, err := snapshotDownloadTool(context.Background(), nil, snapshotDownloadArgs{Network: "nile"}) if err != nil { diff --git a/internal/mcp/drift_redaction_test.go b/internal/mcp/drift_redaction_test.go index 535ef047..f172ae49 100644 --- a/internal/mcp/drift_redaction_test.go +++ b/internal/mcp/drift_redaction_test.go @@ -3,11 +3,12 @@ package mcp import ( "context" "encoding/json" - "github.com/tronprotocol/tron-deployment/internal/render" "os" "path/filepath" "strings" "testing" + + "github.com/tronprotocol/tron-deployment/internal/render" ) // F3 regression test for mcpLineDiff. Its output is returned as an MCP diff --git a/internal/mcp/tools_lifecycle.go b/internal/mcp/tools_lifecycle.go index 998d2f63..26ba9def 100644 --- a/internal/mcp/tools_lifecycle.go +++ b/internal/mcp/tools_lifecycle.go @@ -31,13 +31,6 @@ type planArg struct { Path string `json:"path" jsonschema:"absolute path to an intent.yaml file"` } -func boolToDowntime(changed bool) int { - if changed { - return 30 - } - return 0 -} - func registerLifecycleTools(s *mcp.Server) { mcp.AddTool(s, &mcp.Tool{ Name: "plan", diff --git a/internal/recipe/runner.go b/internal/recipe/runner.go index 4bddd5ee..9b7545da 100644 --- a/internal/recipe/runner.go +++ b/internal/recipe/runner.go @@ -311,7 +311,7 @@ func runStep(ctx context.Context, opts RunOptions, step Step, args []string) (St full = append(full, args...) if opts.DryRun { - fmt.Fprintf(opts.Out, " [%s] would run: %s %s\n", step.ID, opts.Binary, strings.Join(full, " ")) + fmt.Fprintf(opts.Out, " [%s] would run: %s %s\n", displayStepID(step.ID), opts.Binary, strings.Join(full, " ")) return res, nil } @@ -335,6 +335,10 @@ func runStep(ctx context.Context, opts RunOptions, step Step, args []string) (St return res, nil } +func displayStepID(id string) string { + return strings.ReplaceAll(id, "_", "-") +} + // runRollback executes every rollback step in order, logging but not // short-circuiting on failures. Rollback is "best effort cleanup". func runRollback(ctx context.Context, opts RunOptions, steps []Step, params map[string]string, state map[string]map[string]any) []StepResult { @@ -429,7 +433,7 @@ func runHostStep(ctx context.Context, opts RunOptions, step Step, args []string) } if opts.DryRun { - fmt.Fprintf(opts.Out, " [%s] would run on host: %s\n", step.ID, strings.Join(argv, " ")) + fmt.Fprintf(opts.Out, " [%s] would run on host: %s\n", displayStepID(step.ID), strings.Join(argv, " ")) return res, nil } if opts.RequirePrivate { diff --git a/internal/schema/embed.go b/internal/schema/embed.go index 690e4d6e..954d1487 100644 --- a/internal/schema/embed.go +++ b/internal/schema/embed.go @@ -119,7 +119,7 @@ import ( // `md5_verified: false` on a successful download means one // thing only: the operator passed `--no-verify` / `no_verify`. // One existing schema, one additive optional field — PATCH. -const SchemaVersion = "1.15.0" +const SchemaVersion = "1.15.1" // JSONSchemaURLBase is the canonical URL prefix for individual output // schema files. Embedded $id values inside each schema mirror this so diff --git a/internal/schema/files/apply.schema.json b/internal/schema/files/apply.schema.json index 7f946883..3bad1cd3 100644 --- a/internal/schema/files/apply.schema.json +++ b/internal/schema/files/apply.schema.json @@ -24,8 +24,8 @@ }, "intent_hash": { "type": "string", - "pattern": "^[0-9a-f]{64}$", - "description": "SHA256 of the intent file contents. Used by the next apply to detect changes." + "pattern": "^(v2:)?[0-9a-f]{64}$", + "description": "Versioned SHA256 of the fully resolved intent; bare SHA256 is accepted for legacy state during migration." }, "config_hash": { "type": "string", diff --git a/internal/schema/files/events.schema.json b/internal/schema/files/events.schema.json index 6815db36..2e423c67 100644 --- a/internal/schema/files/events.schema.json +++ b/internal/schema/files/events.schema.json @@ -34,7 +34,7 @@ }, "intent_hash": { "type": "string", - "pattern": "^[0-9a-f]{64}$" + "pattern": "^(v2:)?[0-9a-f]{64}$" }, "result": { "type": "string", diff --git a/internal/schema/version_baseline.json b/internal/schema/version_baseline.json index 695f1dfb..5a7f0b3c 100644 --- a/internal/schema/version_baseline.json +++ b/internal/schema/version_baseline.json @@ -1,9 +1,9 @@ { - "schema_version": "1.15.0", + "schema_version": "1.15.1", "entries": [ { "name": "apply", - "hash": "130a8b784a5f3d288b0c6cced8746be52434ffb9de40a8915a3e65ba2695d156" + "hash": "f700be42619e66768a11dc8ecc7505076a6b151beca7014e833b3972a8431a1c" }, { "name": "auto-heal", @@ -51,7 +51,7 @@ }, { "name": "events", - "hash": "6f192c6454758b89fc087087bcb6a1e5ff35412f383889daea3775b7ae31f81b" + "hash": "90d4d2509307876a3a1c69ac6ecaec51532679006f148a44c2aa079cd97421f7" }, { "name": "health", diff --git a/internal/security/audit.go b/internal/security/audit.go index f9c904fb..c88b9fe4 100644 --- a/internal/security/audit.go +++ b/internal/security/audit.go @@ -7,8 +7,9 @@ import ( "path/filepath" "sync" - "github.com/tronprotocol/tron-deployment/internal/paths" "time" + + "github.com/tronprotocol/tron-deployment/internal/paths" ) // AuditEntry represents a single audit log line. diff --git a/internal/snapshot/download.go b/internal/snapshot/download.go index 40f86fd5..efa599c5 100644 --- a/internal/snapshot/download.go +++ b/internal/snapshot/download.go @@ -368,7 +368,7 @@ func Download(ctx context.Context, opts DownloadOptions) (*DownloadResult, error if err != nil { return nil, fmt.Errorf("create snapshot stage: %w", err) } - defer os.RemoveAll(stage) + defer func() { _ = os.RemoveAll(stage) }() start := time.Now() url := pre.URL @@ -478,7 +478,7 @@ func publishSnapshot(stage, dest string, force bool) error { if backupErr != nil { return fmt.Errorf("create snapshot backup: %w", backupErr) } - os.RemoveAll(backup) + _ = os.RemoveAll(backup) if err := snapshotRename(target, backup); err != nil { return fmt.Errorf("stage existing snapshot: %w", err) } diff --git a/internal/state/load.go b/internal/state/load.go index 17d74beb..30985749 100644 --- a/internal/state/load.go +++ b/internal/state/load.go @@ -7,7 +7,8 @@ func LoadNode(path, name string) (*Store, *DeploymentState, *ManagedNode, error) if err != nil { return nil, nil, nil, err } - return store, st, store.GetNode(st, name), nil + node := store.GetNode(st, name) + return store, st, node, nil } // Load opens a Store and loads its deployment state without acquiring a lock. diff --git a/internal/state/store.go b/internal/state/store.go index 3dca013b..fa4af847 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -3,9 +3,10 @@ package state import ( "encoding/json" "fmt" - "github.com/tronprotocol/tron-deployment/internal/paths" "os" "path/filepath" + + "github.com/tronprotocol/tron-deployment/internal/paths" ) const stateFileVersion = 1 diff --git a/internal/target/ssh.go b/internal/target/ssh.go index aaff2949..7a6b1b72 100644 --- a/internal/target/ssh.go +++ b/internal/target/ssh.go @@ -387,7 +387,8 @@ func (r *sshStreamReader) Close() error { close(r.watchStop) <-r.watchDone var waitErr error - if r.processDone != nil && r.streamDone != nil { + switch { + case r.processDone != nil && r.streamDone != nil: select { case <-r.streamDone: waitErr = <-r.processDone @@ -395,14 +396,14 @@ func (r *sshStreamReader) Close() error { r.terminate() waitErr = <-r.processDone } - } else if r.processDone != nil { + case r.processDone != nil: select { case waitErr = <-r.processDone: default: r.terminate() waitErr = <-r.processDone } - } else { + default: r.terminate() waitErr = r.wait() } diff --git a/schemas/output/apply.schema.json b/schemas/output/apply.schema.json index 7f946883..3bad1cd3 100644 --- a/schemas/output/apply.schema.json +++ b/schemas/output/apply.schema.json @@ -24,8 +24,8 @@ }, "intent_hash": { "type": "string", - "pattern": "^[0-9a-f]{64}$", - "description": "SHA256 of the intent file contents. Used by the next apply to detect changes." + "pattern": "^(v2:)?[0-9a-f]{64}$", + "description": "Versioned SHA256 of the fully resolved intent; bare SHA256 is accepted for legacy state during migration." }, "config_hash": { "type": "string", diff --git a/schemas/output/events.schema.json b/schemas/output/events.schema.json index 6815db36..2e423c67 100644 --- a/schemas/output/events.schema.json +++ b/schemas/output/events.schema.json @@ -34,7 +34,7 @@ }, "intent_hash": { "type": "string", - "pattern": "^[0-9a-f]{64}$" + "pattern": "^(v2:)?[0-9a-f]{64}$" }, "result": { "type": "string", diff --git a/tools/txgen/broadcast.go b/tools/txgen/broadcast.go index 866c8bb4..3bc162f7 100644 --- a/tools/txgen/broadcast.go +++ b/tools/txgen/broadcast.go @@ -187,12 +187,6 @@ func runBroadcast(ctx context.Context, cfg *Config) error { return nil } -// streamCSV reads `path` (skipping the header) and pushes (txID, json) -// pairs onto out. Returns the number of rows pushed. -func streamCSV(path string, out chan<- [2]string) (int, error) { - return streamCSVContext(context.Background(), path, out) -} - func streamCSVContext(ctx context.Context, path string, out chan<- [2]string) (int, error) { f, err := os.Open(path) if err != nil { @@ -229,11 +223,6 @@ func streamCSVContext(ctx context.Context, path string, out chan<- [2]string) (i return count, nil } -func runCSVProducer(ctx context.Context, path string, out chan<- [2]string, done chan<- struct{}) { - defer close(done) - _, _ = streamCSVContext(ctx, path, out) -} - func truncate(s string, n int) string { if len(s) <= n { return s From 3d8943b85046a19e9f7ee1b4220fcf20b2b1a152 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 25 Aug 2026 11:59:18 +0800 Subject: [PATCH 36/49] test: close local gate gaps and add output-schema contract test - add make verify (lint+test+e2e) and verify-fast (lint+test) gates - move hermetic tests (recipe, agentsmd, schema manifest/conformance) out of the e2e build tag into the default suite - split docker-only helpers into e2e_docker_helpers_test.go - add TestOutputSchemaContracts: validate apply/events output against embedded schemas, incl. v2: and legacy bare-hex intent hashes --- Makefile | 8 +- cmd/agentsmd_e2e_test.go | 2 - cmd/e2e_docker_helpers_test.go | 15 ++++ cmd/e2e_helpers_test.go | 19 ++--- cmd/output_schema_contract_test.go | 127 +++++++++++++++++++++++++++++ cmd/recipe_e2e_test.go | 2 - cmd/recipe_matrix_e2e_test.go | 2 - cmd/schema_conformance_e2e_test.go | 2 - cmd/schema_manifest_e2e_test.go | 2 - 9 files changed, 154 insertions(+), 25 deletions(-) create mode 100644 cmd/e2e_docker_helpers_test.go create mode 100644 cmd/output_schema_contract_test.go diff --git a/Makefile b/Makefile index 0e06b66a..e0e4ab9c 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ endif GOFLAGS ?= -.PHONY: build test lint e2e build-all clean clean-all fmt vet tidy sync-templates sync-schemas sync-knowledge snapshot-schema-baseline update-render-golden docs man cover vuln bootstrap-go build-replay install-replay build-txgen install-txgen build-txgen-falcon install-txgen-falcon +.PHONY: build test lint e2e verify verify-fast build-all clean clean-all fmt vet tidy sync-templates sync-schemas sync-knowledge snapshot-schema-baseline update-render-golden docs man cover vuln bootstrap-go build-replay install-replay build-txgen install-txgen build-txgen-falcon install-txgen-falcon ## bootstrap-go: Download + verify the project-local Go toolchain ## (idempotent; safe to re-run; no-op if already current) @@ -68,6 +68,12 @@ lint: $(GO_BOOTSTRAP) e2e: $(GO_BOOTSTRAP) $(GO) test ./... -tags=e2e -race -count=1 -timeout 10m +## verify: Pre-push gate — lint + unit tests + e2e (requires Docker daemon) +verify: lint test e2e + +## verify-fast: Fast pre-push gate — lint + unit tests (no Docker required) +verify-fast: lint test + ## build-all: Cross-compile for all supported platforms build-all: $(GO_BOOTSTRAP) GOOS=linux GOARCH=amd64 $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o bin/$(BINARY)-linux-amd64 . diff --git a/cmd/agentsmd_e2e_test.go b/cmd/agentsmd_e2e_test.go index f4c6cb56..fa66fcb9 100644 --- a/cmd/agentsmd_e2e_test.go +++ b/cmd/agentsmd_e2e_test.go @@ -1,5 +1,3 @@ -//go:build e2e - package cmd import ( diff --git a/cmd/e2e_docker_helpers_test.go b/cmd/e2e_docker_helpers_test.go new file mode 100644 index 00000000..04861fb0 --- /dev/null +++ b/cmd/e2e_docker_helpers_test.go @@ -0,0 +1,15 @@ +//go:build e2e + +package cmd + +import ( + "os/exec" + "testing" +) + +func skipUnlessDocker(t *testing.T) { + t.Helper() + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skipf("Docker not available: %v", err) + } +} diff --git a/cmd/e2e_helpers_test.go b/cmd/e2e_helpers_test.go index e29287f6..1eed10a7 100644 --- a/cmd/e2e_helpers_test.go +++ b/cmd/e2e_helpers_test.go @@ -1,5 +1,3 @@ -//go:build e2e - package cmd import ( @@ -12,9 +10,9 @@ import ( "testing" ) -// Shared helpers for the e2e suite. Every file gated by //go:build e2e -// in package cmd uses these. Kept here (not in cmd/) so a `go build` -// of trond doesn't pull them in. +// Shared helpers for package cmd tests. Both the hermetic tests +// (default suite) and the container tests (//go:build e2e) use these. +// Kept here (not in cmd/) so a `go build` of trond doesn't pull them in. // // Two design choices worth flagging: // @@ -75,6 +73,8 @@ func e2eBinary(t *testing.T) string { // t.TempDir() and a copy of the host environment with TROND_STATE_DIR // pointing at it. Use this for any subprocess invocation that // reads/writes state (apply, status, network, recipe, ...). +// +//nolint:unparam // stateDir is consumed by Docker-tagged tests. func e2eEnv(t *testing.T) (stateDir string, env []string) { t.Helper() stateDir = t.TempDir() @@ -131,12 +131,3 @@ func absExample(t *testing.T, rel string) string { } return abs } - -// skipUnlessDocker short-circuits a test when Docker isn't usable. -// Used by every e2e test that actually starts containers. -func skipUnlessDocker(t *testing.T) { - t.Helper() - if err := exec.Command("docker", "info").Run(); err != nil { - t.Skipf("Docker not available: %v", err) - } -} diff --git a/cmd/output_schema_contract_test.go b/cmd/output_schema_contract_test.go new file mode 100644 index 00000000..bb1d36d1 --- /dev/null +++ b/cmd/output_schema_contract_test.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "testing" + "time" + + "github.com/santhosh-tekuri/jsonschema/v6" + + applypkg "github.com/tronprotocol/tron-deployment/internal/apply" + "github.com/tronprotocol/tron-deployment/internal/schema" + "github.com/tronprotocol/tron-deployment/internal/security" +) + +// TestOutputSchemaContracts exercises the two output producers involved in +// the schema-drift incident. It uses the same applyResultMap and +// security.AuditEntry producers as the CLI, without starting a runtime. +func TestOutputSchemaContracts(t *testing.T) { + t.Run("apply", func(t *testing.T) { + for _, hash := range []string{ + "v2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + } { + result := applyResultMap(&applypkg.Result{ + Name: "contract-node", Outcome: "created", IntentHash: hash, + ConfigHash: strings64('b'), Version: "4.8.1", Runtime: "docker", + Network: "private", IsPrivate: true, + Endpoints: map[string]string{"http": "http://127.0.0.1:8090", "grpc": "127.0.0.1:50051"}, + Build: &applypkg.BuildSummary{CacheKey: "0123456789ab-b01234567", SourceRevision: strings40('c'), Dirty: true, CacheHit: false, DurationMs: 12}, + MonitoringError: "warning", MonitoringEndpoints: map[string]string{"prometheus_url": "http://127.0.0.1:9090", "grafana_url": "http://127.0.0.1:3000"}, + }, 42) + result["ready"] = true + result["waited_ms"] = int64(7) + result["wait_error"] = "" + validateOutputSchema(t, "apply", result) + } + }) + + t.Run("events", func(t *testing.T) { + entry, err := json.Marshal(security.AuditEntry{ + Timestamp: time.Date(2026, time.August, 25, 12, 0, 0, 0, time.UTC), + Command: "apply", Node: "contract-node", Target: "local", IntentHash: "v2:" + strings64('a'), + Result: "success", DurationMs: 42, ErrorCode: "TEST", Detail: "intent.yaml", RunID: "run-123", + }) + if err != nil { + t.Fatal(err) + } + var value any + if err := json.Unmarshal(entry, &value); err != nil { + t.Fatal(err) + } + validateOutputSchema(t, "events", value) + }) +} + +func validateOutputSchema(t *testing.T, name string, value any) { + t.Helper() + // Validate the JSON value, not Go structs/maps. This mirrors the CLI wire + // boundary and applies JSON tags/omitempty exactly as output.WriteJSON does. + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(data, &value); err != nil { + t.Fatal(err) + } + doc, ok := schema.Get(name) + if !ok { + t.Fatalf("embedded schema %q missing", name) + } + properties, ok := doc["properties"].(map[string]any) + if !ok { + t.Fatalf("schema %q has no properties", name) + } + actual, ok := value.(map[string]any) + if !ok { + t.Fatalf("%s producer did not emit an object: %T", name, value) + } + expected := map[string]map[string]bool{ + "apply": {"name": true, "result": true, "intent_hash": true, "runtime": true, "version": true, "network": true, "is_private": true, "endpoints": true, "duration_ms": true, "config_hash": true, "build": true, "monitoring_error": true, "monitoring": true, "ready": true, "waited_ms": true, "wait_error": true}, + "events": {"timestamp": true, "command": true, "node": true, "target": true, "intent_hash": true, "result": true, "duration_ms": true, "error_code": true, "detail": true, "run_id": true}, + } + for key := range expected[name] { + if _, ok := actual[key]; !ok { + t.Errorf("%s producer omitted expected field %q", name, key) + } + } + for key := range properties { + if !expected[name][key] { + t.Errorf("%s schema declares unexpected field %q; update the producer contract test", name, key) + } + } + for key := range actual { + if _, ok := properties[key]; !ok { + t.Errorf("%s producer emitted %q but schema does not declare it", name, key) + } + } + + data, err = json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + compiledDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + compiler := jsonschema.NewCompiler() + const id = "https://trond.test/schema.json" + if err := compiler.AddResource(id, compiledDoc); err != nil { + t.Fatal(err) + } + compiled, err := compiler.Compile(id) + if err != nil { + t.Fatal(err) + } + if err := compiled.Validate(value); err != nil { + t.Fatalf("%s output failed embedded schema: %v", name, err) + } +} + +func strings64(ch byte) string { return repeated(ch, 64) } +func strings40(ch byte) string { return repeated(ch, 40) } +func repeated(ch byte, n int) string { + b := bytes.Repeat([]byte{ch}, n) + return string(b) +} diff --git a/cmd/recipe_e2e_test.go b/cmd/recipe_e2e_test.go index 8ded80ce..0a0e01f8 100644 --- a/cmd/recipe_e2e_test.go +++ b/cmd/recipe_e2e_test.go @@ -1,5 +1,3 @@ -//go:build e2e - package cmd import ( diff --git a/cmd/recipe_matrix_e2e_test.go b/cmd/recipe_matrix_e2e_test.go index af246979..7c011e7f 100644 --- a/cmd/recipe_matrix_e2e_test.go +++ b/cmd/recipe_matrix_e2e_test.go @@ -1,5 +1,3 @@ -//go:build e2e - package cmd import ( diff --git a/cmd/schema_conformance_e2e_test.go b/cmd/schema_conformance_e2e_test.go index de2283f4..7dace9bb 100644 --- a/cmd/schema_conformance_e2e_test.go +++ b/cmd/schema_conformance_e2e_test.go @@ -1,5 +1,3 @@ -//go:build e2e - package cmd import ( diff --git a/cmd/schema_manifest_e2e_test.go b/cmd/schema_manifest_e2e_test.go index 4e1a7023..f46acd30 100644 --- a/cmd/schema_manifest_e2e_test.go +++ b/cmd/schema_manifest_e2e_test.go @@ -1,5 +1,3 @@ -//go:build e2e - package cmd import ( From 82bb09067eccefc6691b5aded45d2f6792db6133 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 25 Aug 2026 12:06:12 +0800 Subject: [PATCH 37/49] docs(makefile): note verify targets mirror CI jobs --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e0e4ab9c..eb1be38d 100644 --- a/Makefile +++ b/Makefile @@ -68,10 +68,13 @@ lint: $(GO_BOOTSTRAP) e2e: $(GO_BOOTSTRAP) $(GO) test ./... -tags=e2e -race -count=1 -timeout 10m -## verify: Pre-push gate — lint + unit tests + e2e (requires Docker daemon) +## verify: Pre-push gate — lint + unit tests + e2e (requires Docker daemon). +## Mirrors the CI jobs (golangci-lint, Test + coverage, e2e) so a green +## local run means a green CI; run this before pushing. verify: lint test e2e -## verify-fast: Fast pre-push gate — lint + unit tests (no Docker required) +## verify-fast: Fast pre-push gate — lint + unit tests (no Docker required). +## Same lint and unit jobs as CI; skips only the Docker-dependent e2e job. verify-fast: lint test ## build-all: Cross-compile for all supported platforms From fde40ddf3948d537ed6f56ac475a2c5eb9cdac09 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 25 Aug 2026 17:26:02 +0800 Subject: [PATCH 38/49] fix(intent): skip default image for jar nodes and guard jar/docker runtime A node with a jar: block and no explicit image had the default tronprotocol/java-tron image injected by ApplyDefaults, then got rejected by apply's mutual-exclusion check while config validate (pre-defaults) passed. Skip the injection when Jar is set, and reject jar + effective docker runtime up front in Parse, LoadWithOverlay, and apply (defense in depth) instead of silently deploying the official image. --- internal/apply/apply.go | 3 + internal/intent/audit_render_test.go | 43 +++++++++ internal/intent/defaults.go | 4 +- internal/intent/defaults_test.go | 137 +++++++++++++++++++++++++++ internal/intent/loader.go | 22 +++++ internal/intent/overlay.go | 3 + 6 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 internal/intent/defaults_test.go diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 3ea65915..1414fbb0 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -549,6 +549,9 @@ func validateOptions(o Options) error { return output.NewErrorf("VALIDATION_ERROR", output.ExitValidationError, "node %q: build, image, jar are mutually exclusive (pick one artifact source)", n.Type) } + if err := intent.ValidateJarRuntime(o.Intent); err != nil { + return output.NewErrorf("VALIDATION_ERROR", output.ExitValidationError, "%v", err) + } // Phase 2 only wires artifact=jar end-to-end. The artifact_kind // must match the runtime, otherwise we'd render a docker compose diff --git a/internal/intent/audit_render_test.go b/internal/intent/audit_render_test.go index 34cfc945..daee589b 100644 --- a/internal/intent/audit_render_test.go +++ b/internal/intent/audit_render_test.go @@ -39,3 +39,46 @@ func TestLoadWithOverlayPreservesUntouchedTargetFields(t *testing.T) { t.Fatalf("target type=%q", got.Target.Type) } } + +func TestLoadWithOverlayValidatesJarRuntimeAfterMerge(t *testing.T) { + const jarSHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + const baseData = `name: jar-overlay-test +network: nile +target: + type: local + runtime: jar +nodes: + - type: fullnode + jar: + url: https://example.com/FullNode.jar + sha256: ` + jarSHA256 + "\n" + + for _, tc := range []struct { + name string + overlay string + wantError bool + }{ + {name: "docker runtime rejects jar", overlay: "target:\n runtime: docker\n", wantError: true}, + {name: "jar runtime is valid", overlay: "target:\n runtime: jar\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + base := dir + "/base.yaml" + overlay := dir + "/overlay.yaml" + if err := os.WriteFile(base, []byte(baseData), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(overlay, []byte(tc.overlay), 0600); err != nil { + t.Fatal(err) + } + + _, err := LoadWithOverlay(base, overlay) + if (err != nil) != tc.wantError { + t.Fatalf("LoadWithOverlay error = %v, wantError=%v", err, tc.wantError) + } + if tc.wantError && !strings.Contains(err.Error(), "target.runtime: jar") { + t.Fatalf("error = %v, want jar runtime guidance", err) + } + }) + } +} diff --git a/internal/intent/defaults.go b/internal/intent/defaults.go index 47dd2877..3afd4df5 100644 --- a/internal/intent/defaults.go +++ b/internal/intent/defaults.go @@ -199,11 +199,11 @@ func applyNodeDefaults(node *NodeSpec) { if node.Version == "" { node.Version = "latest" } - // Skip the default Image when a Build block is present — they're + // Skip the default Image when a Build or Jar block is present — they're // mutually exclusive (FR-005). Otherwise the mutex check would // fail on re-validation post-defaults, and worse: a docker // runtime would try to pull an image trond doesn't intend to use. - if node.Image == "" && node.Build == nil { + if node.Image == "" && node.Build == nil && node.Jar == nil { node.Image = "tronprotocol/java-tron" } if node.InstallPath == "" { diff --git a/internal/intent/defaults_test.go b/internal/intent/defaults_test.go new file mode 100644 index 00000000..51781641 --- /dev/null +++ b/internal/intent/defaults_test.go @@ -0,0 +1,137 @@ +package intent + +import ( + "fmt" + "strings" + "testing" +) + +func TestApplyDefaultsArtifactSourceImageDefaults(t *testing.T) { + tests := []struct { + name string + build *BuildSpec + jar *JarSource + image string + wantImage string + }{ + { + name: "jar suppresses image default", + jar: &JarSource{URL: "https://example.com/FullNode.jar"}, + }, + { + name: "build suppresses image default", + build: &BuildSpec{ + Source: "/tmp/java-tron", + }, + }, + { + name: "no artifact source gets legacy image default", + wantImage: "tronprotocol/java-tron", + }, + { + name: "explicit image is preserved", + jar: &JarSource{URL: "https://example.com/FullNode.jar"}, + image: "example/java-tron:test", + wantImage: "example/java-tron:test", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Intent{ + Target: Target{Type: "local"}, + Nodes: []NodeSpec{{ + Type: "fullnode", + Build: tt.build, + Jar: tt.jar, + Image: tt.image, + }}, + } + ApplyDefaults(i) + if got := i.Nodes[0].Image; got != tt.wantImage { + t.Fatalf("Image = %q, want %q", got, tt.wantImage) + } + }) + } +} + +func TestParseJarWithoutImagePassesValidationAndSuppressesDefault(t *testing.T) { + const jarSHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + data := []byte(` +name: jar-fullnode +network: nile +target: + type: local + runtime: jar +nodes: + - type: fullnode + jar: + url: https://example.com/FullNode.jar + sha256: ` + jarSHA256 + ` +`) + + i, err := Parse(data) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if i.Nodes[0].Image != "" { + t.Fatalf("Image = %q, want empty for jar source", i.Nodes[0].Image) + } +} + +func TestParseExplicitImageAndJarRejected(t *testing.T) { + const jarSHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + data := []byte(` +name: conflicting-fullnode +network: nile +target: + type: local + runtime: jar +nodes: + - type: fullnode + image: example/java-tron:test + jar: + url: https://example.com/FullNode.jar + sha256: ` + jarSHA256 + ` +`) + + if _, err := Parse(data); err == nil { + t.Fatal("Parse succeeded for intent with both image and jar") + } +} + +func TestParseJarRuntimeGuard(t *testing.T) { + const jarSHA256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + const jar = ` +name: jar-runtime-test +network: nile +target: +%s +nodes: + - type: fullnode + jar: + url: https://example.com/FullNode.jar + sha256: ` + jarSHA256 + ` +` + tests := []struct { + name string + target string + wantErr bool + }{ + {name: "default docker", target: " type: local", wantErr: true}, + {name: "explicit docker", target: " type: local\n runtime: docker", wantErr: true}, + {name: "jar runtime", target: " type: local\n runtime: jar"}, + {name: "ssh jar runtime", target: " type: ssh\n host: example.com\n user: tron\n runtime: jar"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Parse([]byte(fmt.Sprintf(jar, tt.target))) + if (err != nil) != tt.wantErr { + t.Fatalf("Parse error = %v, wantErr=%v", err, tt.wantErr) + } + if tt.wantErr && !strings.Contains(err.Error(), "target.runtime: jar") { + t.Fatalf("error = %v, want jar runtime guidance", err) + } + }) + } +} diff --git a/internal/intent/loader.go b/internal/intent/loader.go index f6765fad..43b09c46 100644 --- a/internal/intent/loader.go +++ b/internal/intent/loader.go @@ -270,9 +270,31 @@ func Parse(data []byte) (*Intent, error) { } ApplyDefaults(&intent) + if err := ValidateJarRuntime(&intent); err != nil { + return nil, err + } return &intent, nil } +// ValidateJarRuntime rejects the unsupported docker + jar combination after +// runtime defaults have been resolved. Both local and SSH jar deployments use +// target.runtime=jar; target.type does not change that runtime value. +func ValidateJarRuntime(intent *Intent) error { + rt := intent.Target.Runtime + if rt == "" { + rt = DefaultRuntime(intent) + } + if rt != "docker" { + return nil + } + for i, node := range intent.Nodes { + if node.Jar != nil { + return fmt.Errorf("nodes[%d]: jar source requires target.runtime=jar; docker runtime cannot consume jar — set target.runtime: jar", i) + } + } + return nil +} + // ParseRaw returns the parsed Intent with defaults NOT applied. Used by // `config validate --explain` to distinguish explicit user values from // defaults at the field level. diff --git a/internal/intent/overlay.go b/internal/intent/overlay.go index 48c79df3..d551268f 100644 --- a/internal/intent/overlay.go +++ b/internal/intent/overlay.go @@ -31,6 +31,9 @@ func LoadWithOverlay(basePath, overlayPath string) (*Intent, error) { } ApplyDefaults(base) + if err := ValidateJarRuntime(base); err != nil { + return nil, err + } return base, nil } From 6d0954521415db466548a20ad69ca6b7f33bc4ba Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 26 Aug 2026 00:22:34 +0800 Subject: [PATCH 39/49] fix(runtime): remove stale systemd env drop-in when jar EnvVars become empty Redeploying a jar node from non-empty to empty EnvVars left the old /etc/systemd/system/.service.d/env.conf in place, so previous environment (potentially including witness key material) kept applying to the service. Delete the drop-in when EnvVars is empty, mark the change so the service restarts, and cover redeploy/remove cleanup with regression tests. Also: document the jar-intent default-image fix (fde40dd) and AUD-044 recorded-network gating in CHANGELOG, tighten the runtime enum test (explicit empty accepted, podman/systemd rejected), and gitignore the local .regression-evidence/ archive directory. --- .gitignore | 3 +++ CHANGELOG.md | 6 +++++ internal/intent/fields_test.go | 7 ++---- internal/runtime/jar.go | 9 +++++++ internal/runtime/redeploy_test.go | 41 +++++++++++++++++++++++++++++++ internal/runtime/runtime.go | 11 +++++++++ 6 files changed, 72 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 15cae220..95ac8637 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ shadow-fork-intent.yaml # one cleartext secp256k1 private key per receiver and MUST never be # committed. txgen-output/ + +# Regression campaign evidence (local-only, durable copy of /tmp/regress-*) +.regression-evidence/ diff --git a/CHANGELOG.md b/CHANGELOG.md index ecd32414..37bc3316 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Jar intent nodes no longer receive the default Docker image, which + previously made `apply` reject mutually-exclusive artifact sources. Jar + plus Docker runtime is now rejected up front with a clear error pointing to + `target.runtime=jar` (commit fde40dd). +- `diagnose` and `heal` now use the node's recorded network for + private-network gating checks (AUD-044; commit 8ad9c5a). - **P0 audit remediation.** Batch B adds focused cmd-side test splits for `cmd/build`, `cmd/state`, `cmd/network create`, and `cmd/apply` (B1–B8), while domain 1–5 fixes harden lifecycle, state persistence, target diff --git a/internal/intent/fields_test.go b/internal/intent/fields_test.go index c6c75e7b..e3e1a9b4 100644 --- a/internal/intent/fields_test.go +++ b/internal/intent/fields_test.go @@ -62,7 +62,7 @@ func TestTargetRuntime_Enum(t *testing.T) { }{ {"docker", false}, {"jar", false}, - {"", false}, // omitted → default "docker" + {"", false}, // explicit empty value accepted by omitempty {"podman", true}, {"systemd", true}, } @@ -70,12 +70,9 @@ func TestTargetRuntime_Enum(t *testing.T) { t.Run("runtime="+tc.val, func(t *testing.T) { y := "target:\n type: local\n runtime: " + tc.val + "\n" if tc.val == "" { - y = "" + y = "target:\n type: local\n runtime: \"\"\n" } yaml := []byte("name: m\nnetwork: mainnet\n" + y + "nodes: [{type: fullnode}]\n") - if y == "" { - yaml = []byte("name: m\nnetwork: mainnet\ntarget: {type: local}\nnodes: [{type: fullnode}]\n") - } _, err := Parse(yaml) if tc.wantErr && err == nil { t.Errorf("expected error for runtime=%q", tc.val) diff --git a/internal/runtime/jar.go b/internal/runtime/jar.go index a2a72d1e..776583cc 100644 --- a/internal/runtime/jar.go +++ b/internal/runtime/jar.go @@ -106,6 +106,15 @@ func (r *JarRuntime) Deploy(ctx context.Context, opts DeployOpts) error { if err := tracker.write(ctx, envPath, []byte(sb.String()), 0600); err != nil { return fmt.Errorf("write env override: %w", err) } + } else { + // An empty EnvVars map means the service must not inherit a drop-in + // from an earlier deploy. Remove it and track the change so a running + // JVM cannot retain the old environment. + envPath := filepath.Join( + fmt.Sprintf("/etc/systemd/system/%s.d", unitName), "env.conf") + if err := tracker.remove(ctx, envPath); err != nil { + return fmt.Errorf("remove env override: %w", err) + } } if _, err := r.target.Exec(ctx, "systemctl", "daemon-reload"); err != nil { diff --git a/internal/runtime/redeploy_test.go b/internal/runtime/redeploy_test.go index f4160cec..5b81e0de 100644 --- a/internal/runtime/redeploy_test.go +++ b/internal/runtime/redeploy_test.go @@ -262,6 +262,47 @@ func TestJarRuntime_Deploy_EnvVarsAllSurvive(t *testing.T) { } } +func TestJarRuntime_Deploy_EmptyEnvVarsRemovesOldDropIn(t *testing.T) { + ft := newFakeTarget() + rt := NewJarRuntime(ft) + ctx := context.Background() + base := DeployOpts{ + Name: "n1", + JarPath: "/opt/tron/FullNode.jar", + ConfigData: []byte("a = 1\n"), + SystemdData: []byte("[Service]\n"), + } + withEnv := base + withEnv.EnvVars = map[string]string{"SR_PRIVATE_KEY": "deadbeef"} + if err := rt.Deploy(ctx, withEnv); err != nil { + t.Fatalf("Deploy with env: %v", err) + } + ft.cmds = nil + if err := rt.Deploy(ctx, base); err != nil { + t.Fatalf("Deploy without env: %v", err) + } + + envPath := "/etc/systemd/system/tron-n1.service.d/env.conf" + if cmd := findCmd(ft.cmds, "rm", "-f", envPath); cmd == nil { + t.Fatalf("old environment drop-in was not removed; commands: %v", ft.cmds) + } + if findCmd(ft.cmds, "systemctl", "restart", "tron-n1.service") == nil { + t.Fatalf("service was not restarted after removing environment drop-in; commands: %v", ft.cmds) + } +} + +func TestJarRuntime_RemoveCleansDropIn(t *testing.T) { + ft := newFakeTarget() + rt := NewJarRuntime(ft) + if err := rt.Remove(context.Background(), "n1", false); err != nil { + t.Fatalf("Remove: %v", err) + } + overridePath := "/etc/systemd/system/tron-n1.service.d" + if cmd := findCmd(ft.cmds, "rm", "-rf", overridePath); cmd == nil { + t.Fatalf("override directory was not removed; commands: %v", ft.cmds) + } +} + func TestMonitoringRuntime_Deploy_ScrapeConfigChangeForcesRecreate(t *testing.T) { opts := func(scrape string) MonitoringDeployOpts { return MonitoringDeployOpts{ diff --git a/internal/runtime/runtime.go b/internal/runtime/runtime.go index ab04d7fb..915cc939 100644 --- a/internal/runtime/runtime.go +++ b/internal/runtime/runtime.go @@ -49,6 +49,17 @@ func (c *changeTracker) write(ctx context.Context, path string, data []byte, per return c.target.WriteFile(ctx, path, data, perm) } +// remove deletes path, recording a change when the file existed. A removed +// systemd drop-in must cause a restart just like a rewritten one, otherwise +// the running process keeps the old environment until its next restart. +func (c *changeTracker) remove(ctx context.Context, path string) error { + if _, err := c.target.ReadFile(ctx, path); err == nil { + c.changed = true + } + _, err := c.target.Exec(ctx, "rm", "-f", path) + return err +} + // LogOpts configures log retrieval. type LogOpts struct { Tail int From a3b05ddc233a7abfbc9f7b549ee5249d90a1b3de Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 26 Aug 2026 12:37:38 +0800 Subject: [PATCH 40/49] fix(schema): accept versioned intent_hash in status output schema EffectiveIntentHash returns v2: since the versioned-hash series, but status.schema.json still pinned intent_hash to ^[0-9a-f]{64}$, so real status output failed schema validation (council R2 finding; status was uncovered by both the contract test and the docker-gated conformance e2e). - Loosen pattern to ^(v2:)?[0-9a-f]{64}$ (bare hex stays valid) - Add status cases to the output-schema contract test (v2 pass, bare pass, malformed v2 rejected) - Bump SchemaVersion 1.15.1 -> 1.16.0 (additive) and refresh baseline - CHANGELOG entry; TODOS records council R2 leftover items --- CHANGELOG.md | 3 ++ TODOS.md | 45 ++++++++++++++++++++++++ cmd/output_schema_contract_test.go | 41 +++++++++++++++++++++ internal/schema/embed.go | 5 ++- internal/schema/files/status.schema.json | 2 +- internal/schema/version_baseline.json | 4 +-- schemas/output/status.schema.json | 2 +- 7 files changed, 97 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37bc3316..fab280fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- `status` output schema now accepts versioned intent hashes (`v2:` prefix), + fixing schema-validation failures against real `status` output; contract test + coverage extended to status. (council R2 finding) - Jar intent nodes no longer receive the default Docker image, which previously made `apply` reject mutually-exclusive artifact sources. Jar plus Docker runtime is now rejected up front with a clear error pointing to diff --git a/TODOS.md b/TODOS.md index 6531c14f..10442803 100644 --- a/TODOS.md +++ b/TODOS.md @@ -85,3 +85,48 @@ up cold. - **Context:** flagged by the 2026-06 eng-review of the C1 private-net PR (Issue 4 / Codex #7). Documented in AGENTS.md's safety section meanwhile. - **Depends on:** a deliberate schema-version major bump. + +## Jar remove should scrub retained witness key + +- **What:** `trond remove` without purge (`--keep-data`) may leave an inline + witness key in `config.conf` under the install directory; assess default + scrubbing or document the retained secret clearly. +- **Context:** Council R2 skeptic. + +## Preserve change tracking on remove read errors + +- **What:** `changeTracker.remove` treats non-NotExist `ReadFile` errors + (SSH/permission transient) as absent; a successful `rm` then leaves + `changed` false, so a running JVM retains old env until restart. Distinguish + errors or conservatively set `changed`, and add read-error+rm-success + coverage. +- **Priority:** P2. +- **Context:** Council R2 skeptic. + +## Reject control characters in systemd drop-in Environment values + +- **What:** Reject control characters in drop-in `Environment=` values; newline + or `#` could theoretically inject extra systemd directives. +- **Priority:** P3; operator-self threat model, low risk. +- **Context:** Council R2 implementer. + +## Add real systemd lifecycle E2E coverage + +- **What:** Real systemd E2E for `daemon-reload`, `enable`, `is-active`, and + journal semantics is missing; add a systemd runner or formally declare the + support boundary. +- **Status:** UNVERIFIED. +- **Context:** Council R1/R2. + +## Define state-save failure reconciliation + +- **What:** Define recovery and consistency semantics when runtime mutation + succeeds but state save fails, including a reconcile strategy. +- **Context:** Council R1 skeptic,洞#8. + +## Expand schema producer/consumer contract coverage + +- **What:** Output schema↔producer bidirectional assertions currently cover + only apply/events/status; extend them to every output schema. Also declare + `intent_hash` in `inspect.schema.json`. +- **Context:** Council R2 architect; long-term debt. diff --git a/cmd/output_schema_contract_test.go b/cmd/output_schema_contract_test.go index bb1d36d1..00be0c48 100644 --- a/cmd/output_schema_contract_test.go +++ b/cmd/output_schema_contract_test.go @@ -3,6 +3,7 @@ package cmd import ( "bytes" "encoding/json" + "fmt" "testing" "time" @@ -52,6 +53,46 @@ func TestOutputSchemaContracts(t *testing.T) { } validateOutputSchema(t, "events", value) }) + + t.Run("status intent hash", func(t *testing.T) { + for _, hash := range []string{"v2:" + strings64('a'), strings64('b')} { + if err := validateSchemaValue("status", map[string]any{ + "name": "contract-node", "status": "running", "intent_hash": hash, + }); err != nil { + t.Fatalf("valid intent_hash %q failed schema: %v", hash, err) + } + } + if err := validateSchemaValue("status", map[string]any{ + "name": "contract-node", "status": "running", "intent_hash": "v2:xyz", + }); err == nil { + t.Fatal("invalid status intent_hash unexpectedly passed schema") + } + }) +} + +func validateSchemaValue(name string, value any) error { + doc, ok := schema.Get(name) + if !ok { + return fmt.Errorf("embedded schema %q missing", name) + } + data, err := json.Marshal(doc) + if err != nil { + return err + } + compiledDoc, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + const id = "https://trond.test/schema.json" + if err := compiler.AddResource(id, compiledDoc); err != nil { + return err + } + compiled, err := compiler.Compile(id) + if err != nil { + return err + } + return compiled.Validate(value) } func validateOutputSchema(t *testing.T, name string, value any) { diff --git a/internal/schema/embed.go b/internal/schema/embed.go index 954d1487..8faedb6d 100644 --- a/internal/schema/embed.go +++ b/internal/schema/embed.go @@ -119,7 +119,10 @@ import ( // `md5_verified: false` on a successful download means one // thing only: the operator passed `--no-verify` / `no_verify`. // One existing schema, one additive optional field — PATCH. -const SchemaVersion = "1.15.1" +// 1.16.0 — status intent_hash accepts the versioned `v2:` + 64-hex +// format while retaining the legacy bare 64-hex format. Additive +// compatibility widening — MINOR. +const SchemaVersion = "1.16.0" // JSONSchemaURLBase is the canonical URL prefix for individual output // schema files. Embedded $id values inside each schema mirror this so diff --git a/internal/schema/files/status.schema.json b/internal/schema/files/status.schema.json index 055dc57b..f9c5d599 100644 --- a/internal/schema/files/status.schema.json +++ b/internal/schema/files/status.schema.json @@ -26,7 +26,7 @@ "format": "date-time", "description": "Timestamp of the last apply that modified state." }, - "intent_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "intent_hash": { "type": "string", "pattern": "^(v2:)?[0-9a-f]{64}$" }, "config_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "api_endpoints": { "type": "object", diff --git a/internal/schema/version_baseline.json b/internal/schema/version_baseline.json index 5a7f0b3c..786a2fae 100644 --- a/internal/schema/version_baseline.json +++ b/internal/schema/version_baseline.json @@ -1,5 +1,5 @@ { - "schema_version": "1.15.1", + "schema_version": "1.16.0", "entries": [ { "name": "apply", @@ -123,7 +123,7 @@ }, { "name": "status", - "hash": "cf12a8dd02e89813be667a722c2295466c830483a4f2ad547314fda3f096556f" + "hash": "13ea4afc35ec6547e750cf589810a0d36fe1f02aebca2b450b79dc66cfe86c4f" }, { "name": "verify", diff --git a/schemas/output/status.schema.json b/schemas/output/status.schema.json index 055dc57b..f9c5d599 100644 --- a/schemas/output/status.schema.json +++ b/schemas/output/status.schema.json @@ -26,7 +26,7 @@ "format": "date-time", "description": "Timestamp of the last apply that modified state." }, - "intent_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "intent_hash": { "type": "string", "pattern": "^(v2:)?[0-9a-f]{64}$" }, "config_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, "api_endpoints": { "type": "object", From 69ebd4380d97177d1fe4189322f641a938c4923d Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 26 Aug 2026 18:05:38 +0800 Subject: [PATCH 41/49] style(schema): gofmt embed.go version-history comment CI golangci-lint (gofmt) flagged the missing blank comment line between the 1.15.1 and 1.16.0 version-history blocks. --- internal/schema/embed.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/schema/embed.go b/internal/schema/embed.go index 8faedb6d..e95c87cb 100644 --- a/internal/schema/embed.go +++ b/internal/schema/embed.go @@ -119,6 +119,7 @@ import ( // `md5_verified: false` on a successful download means one // thing only: the operator passed `--no-verify` / `no_verify`. // One existing schema, one additive optional field — PATCH. +// // 1.16.0 — status intent_hash accepts the versioned `v2:` + 64-hex // format while retaining the legacy bare 64-hex format. Additive // compatibility widening — MINOR. From 99244c3b49ab8c33e35c2a1798315cecc8d8765a Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 27 Aug 2026 11:29:54 +0800 Subject: [PATCH 42/49] test(cmd): rename untagged e2e test files and document the convention Five files moved out of the e2e build tag in 3d8943b still carried misleading *_e2e_test.go names despite running in the default suite: agentsmd, recipe, recipe_matrix, schema_conformance, schema_manifest. Rename them (drop the _e2e infix) and add a short header note stating why they are untagged, pointing to agentsmd_test.go for the full rationale and hermeticity constraints. --- cmd/{agentsmd_e2e_test.go => agentsmd_test.go} | 15 +++++++++++++++ ...e_matrix_e2e_test.go => recipe_matrix_test.go} | 3 +++ cmd/{recipe_e2e_test.go => recipe_test.go} | 3 +++ ...nce_e2e_test.go => schema_conformance_test.go} | 3 +++ ...nifest_e2e_test.go => schema_manifest_test.go} | 3 +++ 5 files changed, 27 insertions(+) rename cmd/{agentsmd_e2e_test.go => agentsmd_test.go} (90%) rename cmd/{recipe_matrix_e2e_test.go => recipe_matrix_test.go} (96%) rename cmd/{recipe_e2e_test.go => recipe_test.go} (94%) rename cmd/{schema_conformance_e2e_test.go => schema_conformance_test.go} (96%) rename cmd/{schema_manifest_e2e_test.go => schema_manifest_test.go} (94%) diff --git a/cmd/agentsmd_e2e_test.go b/cmd/agentsmd_test.go similarity index 90% rename from cmd/agentsmd_e2e_test.go rename to cmd/agentsmd_test.go index fa66fcb9..0a3f7def 100644 --- a/cmd/agentsmd_e2e_test.go +++ b/cmd/agentsmd_test.go @@ -1,3 +1,18 @@ +// Package-level tests guarding the AGENTS.md agent contract. +// +// NOTE: this file intentionally has NO //go:build e2e tag — it runs in +// the DEFAULT test suite (moved out of the e2e tag in 3d8943b "close +// local gate gaps") so AGENTS.md doc rot (renamed commands, drifted +// output fields) is caught by the ordinary CI "Test + coverage" job and +// local `go test ./...`, not only when someone runs -tags=e2e. +// +// Cost/hermeticity caveat: TestE2E_AgentsMD_OutputFieldsExist is NOT +// fully hermetic — it builds the trond binary once per package run via +// e2eEnv and executes a small set of read-only commands (version, +// doctor, snapshot sources, recipe list, network status, list). Command +// failures are Skip, not Fail, and no Docker/network is required. Keep +// any new test in this file equally cheap and skip-tolerant; tests that +// need Docker or a deployed node belong in a //go:build e2e file. package cmd import ( diff --git a/cmd/recipe_matrix_e2e_test.go b/cmd/recipe_matrix_test.go similarity index 96% rename from cmd/recipe_matrix_e2e_test.go rename to cmd/recipe_matrix_test.go index 7c011e7f..0f77c6c2 100644 --- a/cmd/recipe_matrix_e2e_test.go +++ b/cmd/recipe_matrix_test.go @@ -1,3 +1,6 @@ +// NOTE: no //go:build e2e tag — this file runs in the DEFAULT test suite +// (moved out of the e2e tag in 3d8943b). See agentsmd_test.go for the +// rationale and hermeticity constraints for new tests here. package cmd import ( diff --git a/cmd/recipe_e2e_test.go b/cmd/recipe_test.go similarity index 94% rename from cmd/recipe_e2e_test.go rename to cmd/recipe_test.go index 0a0e01f8..6dae60f6 100644 --- a/cmd/recipe_e2e_test.go +++ b/cmd/recipe_test.go @@ -1,3 +1,6 @@ +// NOTE: no //go:build e2e tag — this file runs in the DEFAULT test suite +// (moved out of the e2e tag in 3d8943b). See agentsmd_test.go for the +// rationale and hermeticity constraints for new tests here. package cmd import ( diff --git a/cmd/schema_conformance_e2e_test.go b/cmd/schema_conformance_test.go similarity index 96% rename from cmd/schema_conformance_e2e_test.go rename to cmd/schema_conformance_test.go index 7dace9bb..cc56642e 100644 --- a/cmd/schema_conformance_e2e_test.go +++ b/cmd/schema_conformance_test.go @@ -1,3 +1,6 @@ +// NOTE: no //go:build e2e tag — this file runs in the DEFAULT test suite +// (moved out of the e2e tag in 3d8943b). See agentsmd_test.go for the +// rationale and hermeticity constraints for new tests here. package cmd import ( diff --git a/cmd/schema_manifest_e2e_test.go b/cmd/schema_manifest_test.go similarity index 94% rename from cmd/schema_manifest_e2e_test.go rename to cmd/schema_manifest_test.go index f46acd30..52bdfa75 100644 --- a/cmd/schema_manifest_e2e_test.go +++ b/cmd/schema_manifest_test.go @@ -1,3 +1,6 @@ +// NOTE: no //go:build e2e tag — this file runs in the DEFAULT test suite +// (moved out of the e2e tag in 3d8943b). See agentsmd_test.go for the +// rationale and hermeticity constraints for new tests here. package cmd import ( From 7f2fcf46de10f4976159689f99da20a2ba7876e7 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 27 Aug 2026 17:19:55 +0800 Subject: [PATCH 43/49] fix(network): make the PARTIAL_SUCCESS message describe actual state semantics The runtime error claimed "state cleaned up regardless" while the implementation keeps failed entries in tracked state and a re-run retries exactly those nodes. That lie misled PR #225's docs correction into encoding it in AGENTS.md; fixing the message at the source closes the chain (root cause of #226 doc findings). --- cmd/network/destroy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/network/destroy.go b/cmd/network/destroy.go index d1cda158..285c4ba8 100644 --- a/cmd/network/destroy.go +++ b/cmd/network/destroy.go @@ -203,7 +203,7 @@ func runDestroy(cmd *cobra.Command, args []string) error { output.WriteJSON(os.Stdout, result) if len(failures) > 0 { return output.NewError("PARTIAL_SUCCESS", output.ExitGeneralError, - fmt.Sprintf("%d node(s) failed to destroy; state cleaned up regardless", len(failures))) + fmt.Sprintf("partial success: %d node(s) failed to destroy; removed entries are gone from state, failed entries REMAIN in state and can be retried", len(failures))) } return nil } From 38fcad6d5e876ff645bf5f6bbc6689a0a2ea3f92 Mon Sep 17 00:00:00 2001 From: warku123 Date: Thu, 27 Aug 2026 17:20:09 +0800 Subject: [PATCH 44/49] docs(contracts): realign agent guides, output schemas and MCP instructions First wave of the full-repo documentation audit (council + oracle reviewed); zero behavior changes. - AGENTS.md: truthful partial-state semantics for network destroy and create per code archaeology (create aggregates failures after trying every node; upgrade is genuinely first-failure-stop), wait example fixed to the real --json-path/--json-gt flags, versioning policy aligned with internal/schema/embed.go (additive = patch) - schemas/output + embed copies: declare status.monitoring and inspect.intent_hash as already emitted (additive, SchemaVersion 1.16.1); version baseline regenerated; synthetic contract assertions added in cmd/output_schema_contract_test.go - internal/mcp/server.go: serverInstructions now list exactly the 23 registered tools and prompt chains match their bodies - knowledge/cloud-deployment.md (+ embedded mirror): drop the nonexistent --target flag, SSH setup via the intent target block - examples/dev-local.yaml / README.md: real build prune flags, verify usage with required --intent - CHANGELOG: user-visible fixes plus #225->#226 correction-chain provenance --- AGENTS.md | 36 +++++++------ CHANGELOG.md | 10 ++++ README.md | 2 +- cmd/output_schema_contract_test.go | 53 ++++++++++++++++++++ examples/dev-local.yaml | 3 +- internal/knowledge/files/cloud-deployment.md | 23 +++++++-- internal/mcp/server.go | 16 ++++-- internal/schema/embed.go | 13 +++-- internal/schema/files/inspect.schema.json | 5 ++ internal/schema/files/status.schema.json | 14 ++++++ internal/schema/version_baseline.json | 6 +-- knowledge/cloud-deployment.md | 23 +++++++-- schemas/output/inspect.schema.json | 5 ++ schemas/output/status.schema.json | 14 ++++++ 14 files changed, 186 insertions(+), 37 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 617bc894..973d30f6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -86,15 +86,21 @@ the exit status alone. One command emits a partial result today: `network destroy` exits **1** with `error_code: "PARTIAL_SUCCESS"`, a `removed` array and a `failed` array naming the nodes whose teardown failed. -**Do not retry it.** The message says `state cleaned up regardless`: trond has -already dropped every node from its state, so a second `network destroy` -returns `NETWORK_NOT_FOUND`. The `failed` entries are containers or units -still running on the target with nothing tracking them — surface that list to -the user for manual cleanup rather than retrying. - -The other multi-node commands do not partially report: `network create` stops -at the first failed node with `DEPLOY_ERROR`, and `network upgrade` with -`UPGRADE_FAILED`, both exit 1. +**Partial state semantics.** The `removed` entries are gone from trond's +state; the `failed` entries REMAIN in state (their containers or units may +still be running on the target). A follow-up `network destroy` retries +exactly those remaining nodes — fix the underlying cause first (target +unreachable, docker error), then re-run if the user approves. Surface the +`failed` array to the user rather than looping silently. + +`network create` behaves differently: it attempts every node to the end — +a failing node is recorded as an `error` entry in the result payload while +its siblings still get their turn — and only then returns exit 1 with +`DEPLOY_ERROR` summarising how many failed. Nodes that did deploy keep +running and are persisted in state, so a re-run reconciles them instead of +redeploying from scratch. `network upgrade` DOES stop at the first failure: +`UPGRADE_FAILED`, exit 1, with already-upgraded nodes left on the new +version unless `--auto-rollback` reverted them. Every error JSON has the same shape: @@ -639,7 +645,7 @@ managing a single user's nodes. # Block until a probe succeeds (port listen, HTTP endpoint, exec output). trond wait --port 8090 --timeout 60s -o json trond wait --http "http://127.0.0.1:8090/wallet/getnowblock" \ - --json '.block_header.raw_data.number > 100' --timeout 5m + --json-path block_header.raw_data.number --json-gt 100 --timeout 5m # Exec arbitrary commands inside a node (docker exec / SSH exec). trond exec -- ls /java-tron/output-directory @@ -989,11 +995,11 @@ the canonical workflow as runnable code. ## Versioning of this contract Every JSON output that carries a `schema_version` field follows -semantic versioning at the field level: existing fields are stable -within a major version; additions are minor; renames or removals are -major. This file is updated alongside any contract change. If you -write an agent against trond X.Y, pin to that version and re-test on -upgrade. +semantic versioning at the field level: additive optional fields on +existing schemas are patch; new schemas are minor; renames, removals, +or meaning shifts are major. This file is updated alongside any +contract change. If you write an agent against trond X.Y, pin to that +version and re-test on upgrade. If a JSON output's shape doesn't match what's documented here, that's a bug — file an issue with the actual output and the trond version diff --git a/CHANGELOG.md b/CHANGELOG.md index fab280fd..9e3421fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Agent guide no longer claims `network create` stops at the first failed + node: the command attempts every node in turn, aggregates failures into + `DEPLOY_ERROR` (exit 1), and leaves successfully deployed nodes running + and recorded in state for reconciliation on re-run. +- `network destroy` partial-failure error message now states the actual + state semantics: removed entries are gone from state, failed entries + REMAIN in state and can be retried (previously claimed the state was + "cleaned up regardless"). Part of the #225/#226 correction chain — the + earlier guide text was written on top of trond's own lying runtime + error message. - `status` output schema now accepts versioned intent hashes (`v2:` prefix), fixing schema-validation failures against real `status` output; contract test coverage extended to status. (council R2 finding) diff --git a/README.md b/README.md index d06ea0a8..16a5bc0d 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ cache management, and MCP usage: [build pipeline quickstart](specs/002-trond-bui | `trond logs ` | Stream node logs | | `trond diagnose ` | Run structured health checks | | `trond health ` | Quick HTTP API probe | -| `trond verify ` | Post-deploy health gate | +| `trond verify --intent --timeout 5m` | Post-deploy health gate | | `trond preflight` | Check target readiness | | `trond bootstrap` | Install Docker or JDK on target | diff --git a/cmd/output_schema_contract_test.go b/cmd/output_schema_contract_test.go index 00be0c48..5b33f4dc 100644 --- a/cmd/output_schema_contract_test.go +++ b/cmd/output_schema_contract_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "strings" "testing" "time" @@ -68,6 +69,58 @@ func TestOutputSchemaContracts(t *testing.T) { t.Fatal("invalid status intent_hash unexpectedly passed schema") } }) + + // Synthetic status payload exercising the additive `monitoring` block + // (schema 1.16.1): both ports must be integers, the probed health must + // stay inside its enum, and all three keys are required. + t.Run("status monitoring", func(t *testing.T) { + if err := validateSchemaValue("status", map[string]any{ + "name": "contract-node", "status": "running", + "monitoring": map[string]any{ + "prometheus_port": 9090, + "grafana_port": 3000, + "status": "running", + }, + }); err != nil { + t.Fatalf("valid monitoring object failed schema: %v", err) + } + for name, mon := range map[string]map[string]any{ + "port not an integer": {"prometheus_port": "9090", "grafana_port": 3000, "status": "running"}, + "status outside enum": {"prometheus_port": 9090, "grafana_port": 3000, "status": "degraded"}, + "missing grafana_port": {"prometheus_port": 9090, "status": "running"}, + } { + if err := validateSchemaValue("status", map[string]any{ + "name": "contract-node", "status": "running", "monitoring": mon, + }); err == nil { + t.Errorf("invalid monitoring (%s) unexpectedly passed schema", name) + } + } + }) + + // Mirror of the status intent-hash subtest above for inspect node + // entries: versioned `v2:` and legacy bare digests are accepted; + // everything else is refused. + t.Run("inspect intent hash", func(t *testing.T) { + nodeWithHash := func(hash string) map[string]any { + return map[string]any{ + "nodes": []any{map[string]any{"name": "contract-node", "intent_hash": hash}}, + } + } + for _, hash := range []string{"v2:" + strings64('a'), strings64('b')} { + if err := validateSchemaValue("inspect", nodeWithHash(hash)); err != nil { + t.Fatalf("valid intent_hash %q failed schema: %v", hash, err) + } + } + for _, hash := range []string{ + strings.ToUpper(strings64('c')), // uppercase hex is not [0-9a-f] + "v3:" + strings64('d'), // unknown version prefix + strings64('e')[:63], // truncated digest + } { + if err := validateSchemaValue("inspect", nodeWithHash(hash)); err == nil { + t.Errorf("invalid inspect intent_hash %q unexpectedly passed schema", hash) + } + } + }) } func validateSchemaValue(name string, value any) error { diff --git a/examples/dev-local.yaml b/examples/dev-local.yaml index 68091a60..57fef30f 100644 --- a/examples/dev-local.yaml +++ b/examples/dev-local.yaml @@ -14,7 +14,8 @@ # 6. Starts (or hot-restarts) the node. # # Re-running with NO source changes hits cache and finishes in seconds. -# Run `trond build prune --keep 3` to bound cache disk usage. +# Run `trond build prune --older-than 168h --keep-last 3 --confirm` +# to bound cache disk usage. name: dev-fullnode network: nile diff --git a/internal/knowledge/files/cloud-deployment.md b/internal/knowledge/files/cloud-deployment.md index 1f488d6c..bd92157e 100644 --- a/internal/knowledge/files/cloud-deployment.md +++ b/internal/knowledge/files/cloud-deployment.md @@ -63,15 +63,30 @@ The recommended workflow provisions infrastructure with Terraform, then uses tro - Create compute instance, SSD volume, security group, and elastic/static IP - Output the instance public IP and SSH key path -**Step 2: Prepare the target** +**Step 2: Prepare the intent + preflight** + +SSH targets are declared in the intent file's `target:` block (full +example: `examples/remote-ssh-fullnode.yaml`): + +```yaml +target: + type: ssh + host: node01.example.com + user: deploy + port: 22 + identity_file: ~/.ssh/id_rsa + runtime: docker +``` + +Then preflight that intent: ```bash -trond preflight --target ssh://user@ --output json +trond preflight --intent intent.yaml --output json ``` -This checks JDK, disk, memory, and port availability. +This checks JDK, disk, memory, and port availability on the target. **Step 3: Deploy** ```bash -trond apply --intent intent.yaml --target ssh://user@ --output json +trond apply --intent intent.yaml --output json ``` **Step 4: Verify** diff --git a/internal/mcp/server.go b/internal/mcp/server.go index f455ad10..65726cc9 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -93,11 +93,17 @@ blockchain nodes (mainnet / Nile testnet / private networks). Available tools: Read-only: doctor, version, list, status, inspect, health, - preflight, knowledge, snapshot_sources, snapshot_list, - snapshot_jobs - Validation: config_validate, config_render, plan, diagnose + knowledge_list, knowledge_get, snapshot_sources, + snapshot_list, snapshot_jobs + Validation: config_validate, config_render, verify_config, plan, + diagnose Destructive: apply, snapshot_download, auto_heal + Build cache (list/inspect read-only; prune deletes): + build_list, build_inspect, build_prune + Shadow-fork (mutates a halted node data dir in place): + shadow_fork_mutate + For deploy / diagnose / snapshot / private-network workflows, follow the canonical chains documented in AGENTS.md (the TRON deployment repo's agent guide). Always validate intent files with config_validate @@ -114,8 +120,8 @@ Beyond tools, this server exposes: trond://nodes/{name}/conf — one node's live HOCON conf Prompts (slash-command workflows the user picks): - deploy_fullnode — validate → plan → apply --wait → status - diagnose_failing_node — status → diagnose → logs → suggest + deploy_fullnode — config_validate → plan → apply → status + diagnose_failing_node — status → diagnose → health setup_private_network — multi-node bootstrap with SR_PRIVATE_KEY recover_failed_upgrade — diagnose → rollback → verify diff --git a/internal/schema/embed.go b/internal/schema/embed.go index e95c87cb..154eb099 100644 --- a/internal/schema/embed.go +++ b/internal/schema/embed.go @@ -120,10 +120,15 @@ import ( // thing only: the operator passed `--no-verify` / `no_verify`. // One existing schema, one additive optional field — PATCH. // -// 1.16.0 — status intent_hash accepts the versioned `v2:` + 64-hex -// format while retaining the legacy bare 64-hex format. Additive -// compatibility widening — MINOR. -const SchemaVersion = "1.16.0" +// 1.16.0 — status intent_hash accepts the versioned `v2:` + 64-hex +// format while retaining the legacy bare 64-hex format. Additive +// compatibility widening — MINOR. +// 1.16.1 — status output gains `monitoring` (prometheus_port + +// grafana_port + live-probed status) and inspect node entries +// gain `intent_hash`, matching what both commands already emit. +// Existing schemas gain additive optional fields — PATCH (the +// earlier additive bumps that took a MINOR were over-versioned). +const SchemaVersion = "1.16.1" // JSONSchemaURLBase is the canonical URL prefix for individual output // schema files. Embedded $id values inside each schema mirror this so diff --git a/internal/schema/files/inspect.schema.json b/internal/schema/files/inspect.schema.json index 96402007..688bb379 100644 --- a/internal/schema/files/inspect.schema.json +++ b/internal/schema/files/inspect.schema.json @@ -43,6 +43,11 @@ }, "labels": { "type": "object", "additionalProperties": { "type": "string" } }, "network": { "type": "string" }, + "intent_hash": { + "type": "string", + "pattern": "^(v2:)?[0-9a-f]{64}$", + "description": "Versioned hash of the last applied intent (v2:, legacy bare 64-hex accepted). Same value `trond status` reports; lets a harness detect config drift between manifest refreshes." + }, "build_cache_key": { "type": "string", "description": "Content-addressed build cache key (source-built nodes only)." }, "build_revision": { "type": "string", "pattern": "^[0-9a-f]{12}$", "description": "Resolved source git revision (short) the node was built from (source-built nodes only)." }, "monitoring": { diff --git a/internal/schema/files/status.schema.json b/internal/schema/files/status.schema.json index f9c5d599..27105da7 100644 --- a/internal/schema/files/status.schema.json +++ b/internal/schema/files/status.schema.json @@ -94,6 +94,20 @@ "type": "integer", "description": "Block delta between local height and the highest height the node has seen across peers; only present from live probe." }, + "monitoring": { + "type": "object", + "description": "Prometheus + Grafana stack ports for nodes deployed with --monitor. Host is implicit: 127.0.0.1 for local targets, target.host for SSH. NOTE: unlike inspect output's monitoring object, this shape carries a live probed `status` instead of an `enabled` flag.", + "required": ["prometheus_port", "grafana_port", "status"], + "properties": { + "prometheus_port": { "type": "integer", "description": "Host port the Prometheus container is bound to." }, + "grafana_port": { "type": "integer", "description": "Host port the Grafana container is bound to." }, + "status": { + "type": "string", + "enum": ["running", "stopped", "unknown"], + "description": "Best-effort health of the monitoring stack (docker compose ps state); 'unknown' when the stack can't be reached." + } + } + }, "probe_error": { "type": "string", "description": "Set when the live probe couldn't reach the HTTP API (timed out / refused). status fields from state.json are still present." diff --git a/internal/schema/version_baseline.json b/internal/schema/version_baseline.json index 786a2fae..5faee46e 100644 --- a/internal/schema/version_baseline.json +++ b/internal/schema/version_baseline.json @@ -1,5 +1,5 @@ { - "schema_version": "1.16.0", + "schema_version": "1.16.1", "entries": [ { "name": "apply", @@ -59,7 +59,7 @@ }, { "name": "inspect", - "hash": "9e200ee43a9fa62b29f43d576672af6a4c033cd1fee07a410b5d012229786482" + "hash": "0004b2d5e75241fc6a39a0d54e17fe3f0c1ee92782d1b0657d58a06f047824be" }, { "name": "list", @@ -123,7 +123,7 @@ }, { "name": "status", - "hash": "13ea4afc35ec6547e750cf589810a0d36fe1f02aebca2b450b79dc66cfe86c4f" + "hash": "6456d7a3b7c5da8862329ec08d672293fe8aefad130ef6a8900669997a3608ee" }, { "name": "verify", diff --git a/knowledge/cloud-deployment.md b/knowledge/cloud-deployment.md index 1f488d6c..bd92157e 100644 --- a/knowledge/cloud-deployment.md +++ b/knowledge/cloud-deployment.md @@ -63,15 +63,30 @@ The recommended workflow provisions infrastructure with Terraform, then uses tro - Create compute instance, SSD volume, security group, and elastic/static IP - Output the instance public IP and SSH key path -**Step 2: Prepare the target** +**Step 2: Prepare the intent + preflight** + +SSH targets are declared in the intent file's `target:` block (full +example: `examples/remote-ssh-fullnode.yaml`): + +```yaml +target: + type: ssh + host: node01.example.com + user: deploy + port: 22 + identity_file: ~/.ssh/id_rsa + runtime: docker +``` + +Then preflight that intent: ```bash -trond preflight --target ssh://user@ --output json +trond preflight --intent intent.yaml --output json ``` -This checks JDK, disk, memory, and port availability. +This checks JDK, disk, memory, and port availability on the target. **Step 3: Deploy** ```bash -trond apply --intent intent.yaml --target ssh://user@ --output json +trond apply --intent intent.yaml --output json ``` **Step 4: Verify** diff --git a/schemas/output/inspect.schema.json b/schemas/output/inspect.schema.json index 96402007..688bb379 100644 --- a/schemas/output/inspect.schema.json +++ b/schemas/output/inspect.schema.json @@ -43,6 +43,11 @@ }, "labels": { "type": "object", "additionalProperties": { "type": "string" } }, "network": { "type": "string" }, + "intent_hash": { + "type": "string", + "pattern": "^(v2:)?[0-9a-f]{64}$", + "description": "Versioned hash of the last applied intent (v2:, legacy bare 64-hex accepted). Same value `trond status` reports; lets a harness detect config drift between manifest refreshes." + }, "build_cache_key": { "type": "string", "description": "Content-addressed build cache key (source-built nodes only)." }, "build_revision": { "type": "string", "pattern": "^[0-9a-f]{12}$", "description": "Resolved source git revision (short) the node was built from (source-built nodes only)." }, "monitoring": { diff --git a/schemas/output/status.schema.json b/schemas/output/status.schema.json index f9c5d599..27105da7 100644 --- a/schemas/output/status.schema.json +++ b/schemas/output/status.schema.json @@ -94,6 +94,20 @@ "type": "integer", "description": "Block delta between local height and the highest height the node has seen across peers; only present from live probe." }, + "monitoring": { + "type": "object", + "description": "Prometheus + Grafana stack ports for nodes deployed with --monitor. Host is implicit: 127.0.0.1 for local targets, target.host for SSH. NOTE: unlike inspect output's monitoring object, this shape carries a live probed `status` instead of an `enabled` flag.", + "required": ["prometheus_port", "grafana_port", "status"], + "properties": { + "prometheus_port": { "type": "integer", "description": "Host port the Prometheus container is bound to." }, + "grafana_port": { "type": "integer", "description": "Host port the Grafana container is bound to." }, + "status": { + "type": "string", + "enum": ["running", "stopped", "unknown"], + "description": "Best-effort health of the monitoring stack (docker compose ps state); 'unknown' when the stack can't be reached." + } + } + }, "probe_error": { "type": "string", "description": "Set when the live probe couldn't reach the HTTP API (timed out / refused). status fields from state.json are still present." From a96f0a755a909c7375d40a7f5f6c8ced287e7f56 Mon Sep 17 00:00:00 2001 From: warku123 Date: Fri, 28 Aug 2026 16:42:35 +0800 Subject: [PATCH 45/49] fix: address colleague review blockers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - target/ssh: restore SetProvisioning lost in the 43d4482 conflict resolution (AUD-005 regression: provisioning silently disabled while bootstrap hard-fails useradd under the #221 allowlist); add a compile-time interface assertion so the method cannot vanish silently - bootstrap: resolve targets through a test seam and cover provisioning wiring with a fake-target test (asserts SetProvisioning(true) and allowlisted provisioning commands are exercised) - network upgrade: cleanupNetworkBackup removes the backup via tgt.Exec("rm","-f","--",path) on the recorded target instead of a 'trond exec -- sh -c' string concatenation — sh is not allowlisted for jar-over-SSH, and the old form carried quoting/injection risk plus a printf-pseudo-JSON envelope; cleanup failures now degrade to warnings in the result instead of failing an otherwise successful upgrade - TODOS: register the nine non-blocking review findings with file:line pointers; CHANGELOG: Fixed entries for both blocker fixes - docs: AGENTS.md documents the new upgrade warnings field; restore the SetProvisioning safety-rationale comment --- AGENTS.md | 3 +- CHANGELOG.md | 5 +++ TODOS.md | 24 ++++++++++++++ cmd/bootstrap.go | 4 ++- cmd/bootstrap_test.go | 62 ++++++++++++++++++++++++++++++++++++- cmd/network/upgrade.go | 32 ++++++++++++++----- cmd/network/upgrade_test.go | 44 ++++++++++++++++++++++++++ internal/target/ssh.go | 8 +++++ 8 files changed, 172 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 973d30f6..51a2f057 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -439,7 +439,8 @@ trond network upgrade pn \ --auto-rollback -o json # Output: {"network":"pn","version":"4.8.1","upgraded_count":N, # "upgraded_nodes":[...],"steps":[{node,phase,status,...}], -# "status":"success"|"failed","failed_at":"..."?} +# "status":"success"|"failed","failed_at":"..."?, +# "warnings":[... ]?} (cleanup failures are warnings, not UPGRADE_ERROR) # 7. Cleanup. --confirm must match the network name (refuses typos). trond network destroy pn --confirm pn -o json diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e3421fd..05d0038f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- SSH bootstrap provisioning mode is restored, so package-manager, shell, and + user-creation commands are allowed only during `bootstrap`. +- Network upgrade backup cleanup now executes `rm` through the target layer; + cleanup failures are reported as warnings instead of misreporting a successful + upgrade as `UPGRADE_ERROR`. - Agent guide no longer claims `network create` stops at the first failed node: the command attempts every node in turn, aggregates failures into `DEPLOY_ERROR` (exit 1), and leaves successfully deployed nodes running diff --git a/TODOS.md b/TODOS.md index 10442803..b14da27d 100644 --- a/TODOS.md +++ b/TODOS.md @@ -3,6 +3,30 @@ Deferred work captured during reviews. Each item carries enough context to pick up cold. +## P0 audit follow-ups + +- **Build source edge cases** — make `foldUntrackedFile` handle directories and + unreadable files without hard failure (`internal/build/source.go:91,111-134`). +- **Windows state fsync** — resolve directory fsync failures and false failures + after rename in `Save` (`internal/state/store.go:107-117`). +- **Local transport lifecycle** — fix the shared transport `Once` write/read + race and transport-pool leak (`internal/target/target.go:38-48,86-94`). +- **Local stream termination** — align unconditional `StreamExec.Close` + termination with SSH `streamDone` behavior (`internal/target/local.go:40-69`). +- **Rollback stop semantics** — document the inverted stop-failure behavior and + record it in CHANGELOG (`cmd/rollback.go:66-69`). +- **Artifact hash failure** — prevent failed `ArtifactSHA256` probes from + retaining stale values and causing drift false positives + (`cmd/upgrade.go:86-89`, `cmd/rollback.go:79-82`). +- **Destroy target resolution** — report every same-target resolution failure, + not only the first one (`cmd/network/destroy.go:104-157`). +- **Auto-port overlays** — allow an overlay to explicitly disable inherited + `auto_ports` (`internal/intent/overlay.go:58,77-79`). +- **Breaking changes log** — add a CHANGELOG Breaking section for strict YAML + fields, ParseMemoryGB, localhost-only `--monitor`, and versioned intent_hash. +- **Replay block atomicity** — add a regression test for the intentional + `blockFail > 0` abort behavior (`tools/replay/replayer.go:301-304`). + ## `snapshot clone --from-node` for SSH-target / named-volume nodes - **What:** `snapshot clone --from-node ` now resolves jar + docker diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go index 556ff79c..6fc82305 100644 --- a/cmd/bootstrap.go +++ b/cmd/bootstrap.go @@ -15,6 +15,8 @@ import ( var bootstrapIntentPath string +var bootstrapResolveTarget = resolveTarget + var bootstrapCmd = &cobra.Command{ Use: "bootstrap", Short: "Install prerequisites on the target", @@ -38,7 +40,7 @@ func runBootstrap(cmd *cobra.Command, args []string) error { return err } - tgt, err := resolveTarget(parsed) + tgt, err := bootstrapResolveTarget(parsed) if err != nil { return exitWithError("TARGET_UNREACHABLE", output.ExitTargetUnreachable, err.Error()) } diff --git a/cmd/bootstrap_test.go b/cmd/bootstrap_test.go index 9d536023..678ff66c 100644 --- a/cmd/bootstrap_test.go +++ b/cmd/bootstrap_test.go @@ -1,6 +1,66 @@ package cmd -import "testing" +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + + "github.com/tronprotocol/tron-deployment/internal/intent" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +type bootstrapFakeTarget struct { + *target.LocalTarget + provisioning bool + calls []string +} + +func (t *bootstrapFakeTarget) SetProvisioning(on bool) { t.provisioning = on } + +func (t *bootstrapFakeTarget) Exec(_ context.Context, cmd string, args ...string) ([]byte, error) { + t.calls = append(t.calls, cmd) + if cmd == "which" && len(args) == 1 && args[0] == "apt-get" { + return []byte("/usr/bin/apt-get\n"), nil + } + return nil, nil +} + +func TestBootstrapEnablesProvisioningMode(t *testing.T) { + path := filepath.Join(t.TempDir(), "intent.yaml") + if err := os.WriteFile(path, []byte("name: bootstrap-test\nnetwork: private\ntarget:\n type: ssh\n host: example\n user: deploy\n runtime: jar\nnodes:\n - type: fullnode\n"), 0600); err != nil { + t.Fatal(err) + } + fake := &bootstrapFakeTarget{LocalTarget: target.NewLocalTarget()} + old := bootstrapResolveTarget + oldPath := bootstrapIntentPath + bootstrapResolveTarget = func(*intent.Intent) (target.Target, error) { return fake, nil } + bootstrapIntentPath = path + defer func() { + bootstrapResolveTarget = old + bootstrapIntentPath = oldPath + }() + + if err := runBootstrap(&cobra.Command{}, nil); err != nil { + t.Fatal(err) + } + if !fake.provisioning { + t.Fatal("bootstrap did not enable provisioning mode") + } + if len(fake.calls) == 0 || fake.calls[0] != "which" { + t.Fatalf("bootstrap calls = %v, want provisioning probe", fake.calls) + } + var sawAPT, sawUseradd bool + for _, call := range fake.calls { + sawAPT = sawAPT || call == "apt-get" + sawUseradd = sawUseradd || call == "useradd" + } + if !sawAPT || !sawUseradd { + t.Fatalf("bootstrap calls = %v, want apt-get and useradd in provisioning mode", fake.calls) + } +} // bootstrap has to be re-runnable, so a useradd that fails only because // the user is already there must not abort the run — but anything else diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index 5c1457c3..69cfb6a9 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -20,6 +20,7 @@ import ( "github.com/tronprotocol/tron-deployment/internal/output" "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) // upgradeCmd does a rolling upgrade across every node in a private @@ -210,11 +211,8 @@ func runUpgrade(cmd *cobra.Command, args []string) error { "duration_ms": time.Since(start).Milliseconds(), "status": "success", } - for _, node := range upgraded { - if err := cleanupNetworkBackup(cmd.Context(), exe, st, node); err != nil { - return output.NewError("UPGRADE_ERROR", output.ExitGeneralError, - fmt.Sprintf("network upgrade succeeded but cleanup failed for %s: %v", node, err)) - } + if warnings := cleanupNetworkBackups(cmd.Context(), st, upgraded); len(warnings) > 0 { + result["warnings"] = warnings } if outputFmt == "json" { return output.WriteJSON(os.Stdout, result) @@ -460,12 +458,32 @@ func runNetworkChild(ctx context.Context, exe string, st *state.DeploymentState, return runChild(ctx, exe, argv...) } -func cleanupNetworkBackup(ctx context.Context, exe string, st *state.DeploymentState, node string) error { +var fromManagedNode = target.FromManagedNode + +func cleanupNetworkBackup(ctx context.Context, st *state.DeploymentState, node string) error { for _, n := range st.Nodes { if n.Name == node && n.Runtime == "jar" { path := filepath.Join(n.InstallPath, "FullNode.jar.upgrade.backup") - return runChild(ctx, exe, "exec", node, "--", "sh", "-c", "rm -f -- "+path+"; printf '{}' ") + tgt, err := fromManagedNode(&n) + if err != nil { + return err + } + if closer, ok := tgt.(interface{ Close() error }); ok { + defer closer.Close() + } + _, err = tgt.Exec(ctx, "rm", "-f", "--", path) + return err } } return nil } + +func cleanupNetworkBackups(ctx context.Context, st *state.DeploymentState, nodes []string) []string { + var warnings []string + for _, node := range nodes { + if err := cleanupNetworkBackup(ctx, st, node); err != nil { + warnings = append(warnings, fmt.Sprintf("failed to remove upgrade backup for %s: %v", node, err)) + } + } + return warnings +} diff --git a/cmd/network/upgrade_test.go b/cmd/network/upgrade_test.go index 01419e81..67678f6c 100644 --- a/cmd/network/upgrade_test.go +++ b/cmd/network/upgrade_test.go @@ -11,8 +11,20 @@ import ( "time" "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" ) +type cleanupFakeTarget struct { + *target.LocalTarget + args []string + err error +} + +func (t *cleanupFakeTarget) Exec(_ context.Context, cmd string, args ...string) ([]byte, error) { + t.args = append([]string{cmd}, args...) + return nil, t.err +} + func TestVerifyNodeInvokesVerifyForEachProjectedNode(t *testing.T) { dir := t.TempDir() intentPath := filepath.Join(dir, "network.yaml") @@ -201,3 +213,35 @@ func TestRunNetworkChildOnlyInjectsUpgradeEnvForRollback(t *testing.T) { t.Fatalf("rollback env = %v, want [%s]", envCalls, networkUpgradeRestoreEnv) } } + +func TestCleanupNetworkBackupUsesRmArgv(t *testing.T) { + fake := &cleanupFakeTarget{LocalTarget: target.NewLocalTarget()} + old := fromManagedNode + fromManagedNode = func(*state.ManagedNode) (target.Target, error) { return fake, nil } + defer func() { fromManagedNode = old }() + + st := &state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "net-node0", Runtime: "jar", InstallPath: "/srv/tron path", + }}} + if err := cleanupNetworkBackup(context.Background(), st, "net-node0"); err != nil { + t.Fatal(err) + } + want := []string{"rm", "-f", "--", "/srv/tron path/FullNode.jar.upgrade.backup"} + if !reflect.DeepEqual(fake.args, want) { + t.Fatalf("cleanup argv = %q, want %q", fake.args, want) + } +} + +func TestCleanupNetworkBackupsReturnsWarningWithoutUpgradeFailure(t *testing.T) { + fake := &cleanupFakeTarget{LocalTarget: target.NewLocalTarget(), err: errors.New("rm failed")} + old := fromManagedNode + fromManagedNode = func(*state.ManagedNode) (target.Target, error) { return fake, nil } + defer func() { fromManagedNode = old }() + + warnings := cleanupNetworkBackups(context.Background(), &state.DeploymentState{Nodes: []state.ManagedNode{{ + Name: "net-node0", Runtime: "jar", InstallPath: "/srv/tron", + }}}, []string{"net-node0"}) + if len(warnings) != 1 || !strings.Contains(warnings[0], "net-node0") || !strings.Contains(warnings[0], "rm failed") { + t.Fatalf("warnings = %v, want cleanup warning", warnings) + } +} diff --git a/internal/target/ssh.go b/internal/target/ssh.go index 7a6b1b72..64ee6a1c 100644 --- a/internal/target/ssh.go +++ b/internal/target/ssh.go @@ -40,6 +40,8 @@ type SSHTarget struct { provisioning bool } +var _ interface{ SetProvisioning(bool) } = (*SSHTarget)(nil) + var sshNetDialContext = (&net.Dialer{}).DialContext func (t *SSHTarget) DialContext(_ context.Context, network, addr string) (net.Conn, error) { @@ -168,6 +170,12 @@ func (t *SSHTarget) ConnectContext(ctx context.Context) error { // process and uploaded over the target connection. func (t *SSHTarget) IsRemote() bool { return true } +// SetProvisioning puts the target in provisioning mode, allowing the +// package-manager and shell commands host preparation needs. It is +// deliberately not part of the target.Target interface: no lifecycle code +// path and no `trond exec` invocation can reach it. +func (t *SSHTarget) SetProvisioning(on bool) { t.provisioning = on } + // hostKeyCallback returns a verifier backed by known_hosts. Falls back to // ~/.ssh/known_hosts when no explicit file is configured. // From ef3095f3ffcc6be2a6db0cd3cc47d4bd89b9a04e Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 31 Aug 2026 10:37:01 +0800 Subject: [PATCH 46/49] fix(network): hoist defer out of backup cleanup loop (gocritic deferInLoop) CI lint flagged a defer-in-loop in cleanupNetworkBackup. The loop now only locates the matching jar node; target resolution, Close, and the rm exec happen once, after the loop. Semantics unchanged. --- cmd/network/upgrade.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index 69cfb6a9..d945fed2 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -461,21 +461,27 @@ func runNetworkChild(ctx context.Context, exe string, st *state.DeploymentState, var fromManagedNode = target.FromManagedNode func cleanupNetworkBackup(ctx context.Context, st *state.DeploymentState, node string) error { - for _, n := range st.Nodes { - if n.Name == node && n.Runtime == "jar" { - path := filepath.Join(n.InstallPath, "FullNode.jar.upgrade.backup") - tgt, err := fromManagedNode(&n) - if err != nil { - return err - } - if closer, ok := tgt.(interface{ Close() error }); ok { - defer closer.Close() - } - _, err = tgt.Exec(ctx, "rm", "-f", "--", path) - return err + var managedNode *state.ManagedNode + for i := range st.Nodes { + if st.Nodes[i].Name == node && st.Nodes[i].Runtime == "jar" { + managedNode = &st.Nodes[i] + break } } - return nil + if managedNode == nil { + return nil + } + + path := filepath.Join(managedNode.InstallPath, "FullNode.jar.upgrade.backup") + tgt, err := fromManagedNode(managedNode) + if err != nil { + return err + } + if closer, ok := tgt.(interface{ Close() error }); ok { + defer closer.Close() + } + _, err = tgt.Exec(ctx, "rm", "-f", "--", path) + return err } func cleanupNetworkBackups(ctx context.Context, st *state.DeploymentState, nodes []string) []string { From 390a4e1d8cbb854e23b12798ce65a26dd845bdde Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 31 Aug 2026 11:33:58 +0800 Subject: [PATCH 47/49] fix: address colleague review findings (non-blocking batch) Nine non-blocking findings from Sunny's PR review, all verified TRUE: - build: tolerate untracked directories, dangling symlinks and unreadable files in dirty-hash folding (fold markers, never fail) - state: directory fsync after rename is best-effort (Windows emits errors for dir Sync; rename already persisted) - target: replace sync.Once with a mutex so CloseIdleConnections no longer races sharedLocalTransport initialization - target: LocalTarget.StreamExec gains streamDone so natural EOF does not kill an already-exited process (matches SSH behavior) - rollback: restore the safety rationale for hard-failing on Stop - upgrade/rollback: a failed artifact SHA256 probe returns an error instead of retaining the stale hash (audit trail + skew comments) - network destroy: every node on an unresolvable target is reported in failures (was: only the first, siblings vanished) - intent overlay: explicit auto_ports: false now overrides base (raw YAML key detection, pointer-free) - changelog: Breaking Changes section for KnownFields, ParseMemoryGB, --monitor precedence (intent_hash v2 documented under Fixed) Also pins the replay block-atomicity semantics with a regression test (single partial block failure aborts the block, keeps the cursor and replays the block on retry - intentional design, not a bug). --- CHANGELOG.md | 17 +++++++ TODOS.md | 29 +++--------- cmd/artifact_hash.go | 16 +++++++ cmd/artifact_hash_test.go | 42 +++++++++++++++++ cmd/network/destroy.go | 75 +++++++++++++++++++----------- cmd/network/destroy_test.go | 48 +++++++++++++++++++ cmd/rollback.go | 11 +++-- cmd/upgrade.go | 8 ++-- internal/build/source.go | 25 +++++++--- internal/build/source_edge_test.go | 30 ++++++++++++ internal/intent/fields_test.go | 21 +++++++++ internal/intent/overlay.go | 17 +++++-- internal/state/store.go | 14 +++++- internal/state/store_test.go | 18 +++++++ internal/target/local.go | 9 +++- internal/target/stream_test.go | 17 +++++++ internal/target/target.go | 24 ++++++++-- internal/target/target_test.go | 17 +++++++ tools/replay/replayer_test.go | 14 +++++- 19 files changed, 376 insertions(+), 76 deletions(-) create mode 100644 cmd/artifact_hash.go create mode 100644 cmd/artifact_hash_test.go create mode 100644 cmd/network/destroy_test.go create mode 100644 internal/build/source_edge_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 05d0038f..71f7015e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Breaking Changes + +- Intent parsing now rejects unknown fields; invalid memory values now error + instead of falling back to 16 GB; `--monitor` no longer suppresses + `monitoring.enabled: true`. `extra_env` `$$` escaping is security + hardening, not a breaking change. + ### Fixed +- Upgrade and rollback now fail when artifact SHA256 probing fails instead of + retaining a stale digest that could produce a false drift result. +- Untracked build-source directories, dangling links, and unreadable files are + represented by hash markers instead of aborting the build. +- State save treats post-rename directory fsync as best-effort for platforms + that cannot sync directories. +- Target overlays can explicitly disable inherited `auto_ports`. +- Network destroy reports every node affected by a shared target-resolution + failure. +- `intent_hash` now accepts the versioned `v2:` prefix. - SSH bootstrap provisioning mode is restored, so package-manager, shell, and user-creation commands are allowed only during `bootstrap`. - Network upgrade backup cleanup now executes `rm` through the target layer; diff --git a/TODOS.md b/TODOS.md index b14da27d..251818ef 100644 --- a/TODOS.md +++ b/TODOS.md @@ -3,29 +3,12 @@ Deferred work captured during reviews. Each item carries enough context to pick up cold. -## P0 audit follow-ups - -- **Build source edge cases** — make `foldUntrackedFile` handle directories and - unreadable files without hard failure (`internal/build/source.go:91,111-134`). -- **Windows state fsync** — resolve directory fsync failures and false failures - after rename in `Save` (`internal/state/store.go:107-117`). -- **Local transport lifecycle** — fix the shared transport `Once` write/read - race and transport-pool leak (`internal/target/target.go:38-48,86-94`). -- **Local stream termination** — align unconditional `StreamExec.Close` - termination with SSH `streamDone` behavior (`internal/target/local.go:40-69`). -- **Rollback stop semantics** — document the inverted stop-failure behavior and - record it in CHANGELOG (`cmd/rollback.go:66-69`). -- **Artifact hash failure** — prevent failed `ArtifactSHA256` probes from - retaining stale values and causing drift false positives - (`cmd/upgrade.go:86-89`, `cmd/rollback.go:79-82`). -- **Destroy target resolution** — report every same-target resolution failure, - not only the first one (`cmd/network/destroy.go:104-157`). -- **Auto-port overlays** — allow an overlay to explicitly disable inherited - `auto_ports` (`internal/intent/overlay.go:58,77-79`). -- **Breaking changes log** — add a CHANGELOG Breaking section for strict YAML - fields, ParseMemoryGB, localhost-only `--monitor`, and versioned intent_hash. -- **Replay block atomicity** — add a regression test for the intentional - `blockFail > 0` abort behavior (`tools/replay/replayer.go:301-304`). +## Open audit follow-ups + +- **AUD-020 transport-pool leak** — retire target-aware transports from the + shared pool on every lifecycle path (`internal/target/target.go:38-48,86-94`). +- **Windows support** — complete platform-specific lifecycle and filesystem + validation beyond the best-effort directory fsync (`internal/state/store.go`). ## `snapshot clone --from-node` for SSH-target / named-volume nodes diff --git a/cmd/artifact_hash.go b/cmd/artifact_hash.go new file mode 100644 index 00000000..1933630e --- /dev/null +++ b/cmd/artifact_hash.go @@ -0,0 +1,16 @@ +package cmd + +import ( + "context" + "fmt" + "path/filepath" +) + +func refreshArtifactSHA256(ctx context.Context, nc *nodeContext) error { + digest, err := nc.Target.Sha256IfExists(ctx, filepath.Join(nc.Node.InstallPath, "FullNode.jar")) + if err != nil { + return fmt.Errorf("sha256 probe: %w", err) + } + nc.Node.ArtifactSHA256 = digest + return nil +} diff --git a/cmd/artifact_hash_test.go b/cmd/artifact_hash_test.go new file mode 100644 index 00000000..c1f37e16 --- /dev/null +++ b/cmd/artifact_hash_test.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "context" + "errors" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +type artifactHashTarget struct { + *target.LocalTarget + err error +} + +func (t artifactHashTarget) Sha256IfExists(context.Context, string) (string, error) { + if t.err != nil { + return "", t.err + } + return "new-digest", nil +} + +func TestUpgradeArtifactHashFailureDoesNotRetainOldValue(t *testing.T) { + nc := &nodeContext{Target: artifactHashTarget{LocalTarget: target.NewLocalTarget(), err: errors.New("probe failed")}, Node: &state.ManagedNode{Runtime: "jar", InstallPath: "/tmp", ArtifactSHA256: "old"}} + if err := refreshArtifactSHA256(context.Background(), nc); err == nil { + t.Fatal("expected hash error") + } + if nc.Node.ArtifactSHA256 != "old" { + t.Fatalf("digest changed on error: %q", nc.Node.ArtifactSHA256) + } +} + +func TestRollbackArtifactHashFailureDoesNotRetainOldValue(t *testing.T) { + nc := &nodeContext{Target: artifactHashTarget{LocalTarget: target.NewLocalTarget(), err: errors.New("probe failed")}, Node: &state.ManagedNode{Runtime: "jar", InstallPath: "/tmp", ArtifactSHA256: "old"}} + if err := refreshArtifactSHA256(context.Background(), nc); err == nil { + t.Fatal("expected hash error") + } + if nc.Node.ArtifactSHA256 != "old" { + t.Fatalf("digest changed on error: %q", nc.Node.ArtifactSHA256) + } +} diff --git a/cmd/network/destroy.go b/cmd/network/destroy.go index 285c4ba8..ae237393 100644 --- a/cmd/network/destroy.go +++ b/cmd/network/destroy.go @@ -16,6 +16,11 @@ import ( "github.com/tronprotocol/tron-deployment/internal/target" ) +type destroyFailure struct { + Name string `json:"name"` + Error string `json:"error"` +} + var destroyConfirm string var destroyCmd = &cobra.Command{ @@ -89,37 +94,15 @@ func runDestroy(cmd *cobra.Command, args []string) error { return err } - type failure struct { - Name string `json:"name"` - Error string `json:"error"` - } // Keep the recorded target for network-level cleanup before removing the // nodes from state. Never fall back to the operator's local Docker daemon. // Network create/add allow mixed targets. Keep one target per distinct // target identity and run network-level cleanup on every one; this avoids // assuming the first node represents the whole network. - networkTargets := make(map[string]target.Target) - var targetFailures []failure - failedTargetKeys := make(map[string]bool) - for _, n := range deployState.Nodes { - if !(strings.HasPrefix(n.Name, prefix) || n.Name == destroyConfirm) { - continue - } - key := fmt.Sprintf("%s|%s|%d|%s|%s", n.Target.Type, n.Target.Host, n.Target.Port, n.Target.User, n.Target.IdentityFile) - if _, ok := networkTargets[key]; ok { - continue - } - tgt, terr := resolveTargetForNode(&n) - if terr != nil { - targetFailures = append(targetFailures, failure{Name: n.Name, Error: "target cleanup: " + terr.Error()}) - failedTargetKeys[key] = true - continue - } - networkTargets[key] = tgt - } + networkTargets, targetFailures, failedTargetKeys := collectNetworkTargets(deployState.Nodes, prefix, destroyConfirm) var removed []string - var failures []failure + var failures []destroyFailure failures = append(failures, targetFailures...) auditResult := "success" auditTarget := "" @@ -142,12 +125,12 @@ func runDestroy(cmd *cobra.Command, args []string) error { tgt, terr := resolveTargetForNode(&n) if terr != nil { - failures = append(failures, failure{Name: n.Name, Error: terr.Error()}) + failures = append(failures, destroyFailure{Name: n.Name, Error: terr.Error()}) continue } rt := resolveRuntimeForNode(&n, tgt) if rerr := rt.Remove(cmd.Context(), n.Name, true); rerr != nil { - failures = append(failures, failure{Name: n.Name, Error: rerr.Error()}) + failures = append(failures, destroyFailure{Name: n.Name, Error: rerr.Error()}) closeTarget(tgt) continue } @@ -208,6 +191,46 @@ func runDestroy(cmd *cobra.Command, args []string) error { return nil } +var resolveTargetForDestroy = resolveTargetForNode + +func collectNetworkTargets(nodes []state.ManagedNode, prefix, confirm string) (map[string]target.Target, []destroyFailure, map[string]bool) { + networkTargets := make(map[string]target.Target) + var targetFailures []destroyFailure + failedTargetKeys := make(map[string]bool) + for i := range nodes { + n := &nodes[i] + if !(strings.HasPrefix(n.Name, prefix) || n.Name == confirm) { + continue + } + key := fmt.Sprintf("%s|%s|%d|%s|%s", n.Target.Type, n.Target.Host, n.Target.Port, n.Target.User, n.Target.IdentityFile) + if failedTargetKeys[key] { + continue + } + if _, ok := networkTargets[key]; ok { + continue + } + tgt, err := resolveTargetForDestroy(n) + if err != nil { + targetFailures = append(targetFailures, targetResolutionFailures(nodes, prefix, confirm, key, err)...) + failedTargetKeys[key] = true + continue + } + networkTargets[key] = tgt + } + return networkTargets, targetFailures, failedTargetKeys +} + +func targetResolutionFailures(nodes []state.ManagedNode, prefix, confirm, key string, err error) []destroyFailure { + var failures []destroyFailure + for _, node := range nodes { + nodeKey := fmt.Sprintf("%s|%s|%d|%s|%s", node.Target.Type, node.Target.Host, node.Target.Port, node.Target.User, node.Target.IdentityFile) + if (strings.HasPrefix(node.Name, prefix) || node.Name == confirm) && nodeKey == key { + failures = append(failures, destroyFailure{Name: node.Name, Error: "target cleanup: " + err.Error()}) + } + } + return failures +} + // resolveTargetForNode returns the right Target for a managed node. // Mirrors cmd/resolve.go but lives here to avoid the cmd ↔ cmd/network // import cycle. SSH targets are connected eagerly so callers don't have diff --git a/cmd/network/destroy_test.go b/cmd/network/destroy_test.go new file mode 100644 index 00000000..8179d4d1 --- /dev/null +++ b/cmd/network/destroy_test.go @@ -0,0 +1,48 @@ +package network + +import ( + "errors" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +func TestTargetResolutionFailureIsReportedForEveryMatchingNode(t *testing.T) { + nodes := []state.ManagedNode{ + {Name: "net-node0", Target: state.NodeTarget{Type: "ssh", Host: "bad", Port: 22, User: "u"}}, + {Name: "net-node1", Target: state.NodeTarget{Type: "ssh", Host: "bad", Port: 22, User: "u"}}, + {Name: "net-node2", Target: state.NodeTarget{Type: "ssh", Host: "bad", Port: 22, User: "u"}}, + } + key := "ssh|bad|22|u|" + failures := targetResolutionFailures(nodes, "net-node", "net", key, errors.New("unreachable")) + if len(failures) != 3 { + t.Fatalf("failures = %d, want 3", len(failures)) + } +} + +func TestCollectNetworkTargetsResolvesBadTargetOnce(t *testing.T) { + old := resolveTargetForDestroy + var calls int + resolveTargetForDestroy = func(*state.ManagedNode) (target.Target, error) { + calls++ + return nil, errors.New("unreachable") + } + defer func() { resolveTargetForDestroy = old }() + nodes := []state.ManagedNode{ + {Name: "net-node0", Target: state.NodeTarget{Type: "ssh", Host: "bad", Port: 22, User: "u"}}, + {Name: "net-node1", Target: state.NodeTarget{Type: "ssh", Host: "bad", Port: 22, User: "u"}}, + {Name: "net-node2", Target: state.NodeTarget{Type: "ssh", Host: "bad", Port: 22, User: "u"}}, + } + _, failures, failed := collectNetworkTargets(nodes, "net-node", "net") + if calls != 1 || len(failures) != 3 || len(failed) != 1 { + t.Fatalf("calls=%d failures=%d failed_keys=%d, want 1/3/1", calls, len(failures), len(failed)) + } + seen := map[string]bool{} + for _, failure := range failures { + if seen[failure.Name] { + t.Fatalf("duplicate failure for %s", failure.Name) + } + seen[failure.Name] = true + } +} diff --git a/cmd/rollback.go b/cmd/rollback.go index c34fe0a1..62ec4ed1 100644 --- a/cmd/rollback.go +++ b/cmd/rollback.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "path/filepath" "time" "github.com/spf13/cobra" @@ -63,6 +62,9 @@ func runRollback(cmd *cobra.Command, args []string) error { writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, fmt.Sprintf("Prepare rollback failed: %v", err)) } + // Stop failures are hard errors: do not activate a replacement artifact + // while the old process may still be running, or two artifacts could race + // for the same node resources. if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { _ = tx.Cleanup(cmd.Context()) writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) @@ -77,8 +79,11 @@ func runRollback(cmd *cobra.Command, args []string) error { return rollbackArtifact(cmd, nc, tx, name, currentVersion, start, fmt.Sprintf("start artifact: %v", err)) } if nc.Node.Runtime == "jar" { - if digest, err := nc.Target.Sha256IfExists(cmd.Context(), filepath.Join(nc.Node.InstallPath, "FullNode.jar")); err == nil { - nc.Node.ArtifactSHA256 = digest + if err := refreshArtifactSHA256(cmd.Context(), nc); err != nil { + // The replacement artifact is running while state still records the + // old version/digest; leave the skew visible until a later success. + writeAudit(auditEvent{Command: "rollback", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "ROLLBACK_ERROR", Start: start}) + return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, fmt.Sprintf("hash rollback artifact: %v", err)) } } if err := tx.Cleanup(cmd.Context()); err != nil { diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 9072234a..2f07409b 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "path/filepath" "time" "github.com/spf13/cobra" @@ -84,8 +83,11 @@ func runUpgrade(cmd *cobra.Command, args []string) error { return rollbackUpgrade(cmd, nc, tx, name, previousVersion, start, fmt.Sprintf("start new artifact: %v", err)) } if nc.Node.Runtime == "jar" { - if digest, err := nc.Target.Sha256IfExists(cmd.Context(), filepath.Join(nc.Node.InstallPath, "FullNode.jar")); err == nil { - nc.Node.ArtifactSHA256 = digest + if err := refreshArtifactSHA256(cmd.Context(), nc); err != nil { + // The new artifact is running while state still records the old + // version/digest; leave the skew visible until a later successful run. + writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "UPGRADE_ERROR", Start: start}) + return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, fmt.Sprintf("hash upgraded artifact: %v", err)) } } if err := tx.Cleanup(cmd.Context()); err != nil { diff --git a/internal/build/source.go b/internal/build/source.go index 7eef4945..09a9d3ca 100644 --- a/internal/build/source.go +++ b/internal/build/source.go @@ -110,22 +110,30 @@ func untrackedPaths(status string) []string { func foldUntrackedFile(h io.Writer, index int, root, path string) error { fullPath := filepath.Join(root, filepath.FromSlash(path)) - file, err := os.Open(fullPath) + info, err := os.Lstat(fullPath) if err != nil { - return fmt.Errorf("read untracked file %q: %w", path, err) + return writeUntrackedMarker(h, index, path, "unreadable") } - defer file.Close() - info, err := file.Stat() + if info.IsDir() { + return writeUntrackedMarker(h, index, path, "directory") + } + if info.Mode()&os.ModeSymlink != 0 { + if _, err := os.Stat(fullPath); err != nil { + return writeUntrackedMarker(h, index, path, "unreadable") + } + } + file, err := os.Open(fullPath) if err != nil { - return fmt.Errorf("stat untracked file %q: %w", path, err) + return writeUntrackedMarker(h, index, path, "unreadable") } + defer file.Close() // Include index, path, and size as framing; then stream the contents so // large source files do not require a second in-memory copy. if _, err := fmt.Fprintf(h, "untracked %d %d\x00%s\x00", index, info.Size(), path); err != nil { return fmt.Errorf("hash untracked file %q: %w", path, err) } if _, err := io.Copy(h, file); err != nil { - return fmt.Errorf("hash untracked file %q: %w", path, err) + return writeUntrackedMarker(h, index, path, "unreadable") } if _, err := io.WriteString(h, "\x00"); err != nil { return fmt.Errorf("hash untracked file %q: %w", path, err) @@ -133,6 +141,11 @@ func foldUntrackedFile(h io.Writer, index int, root, path string) error { return nil } +func writeUntrackedMarker(h io.Writer, index int, path, kind string) error { + _, err := fmt.Fprintf(h, "untracked %d marker:%s\x00%s\x00", index, kind, path) + return err +} + func (s *Source) runGit(ctx context.Context, args ...string) (string, error) { cmd := exec.CommandContext(ctx, "git", append([]string{"-C", s.Path}, args...)...) out, err := cmd.Output() diff --git a/internal/build/source_edge_test.go b/internal/build/source_edge_test.go new file mode 100644 index 00000000..c4b1d035 --- /dev/null +++ b/internal/build/source_edge_test.go @@ -0,0 +1,30 @@ +package build + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestFoldUntrackedFileToleratesSpecialEntries(t *testing.T) { + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "nested.git"), 0700); err != nil { + t.Fatal(err) + } + if err := os.Symlink("missing", filepath.Join(dir, "dangling")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "unreadable"), []byte("secret"), 0000); err != nil { + t.Fatal(err) + } + for i, name := range []string{"nested.git", "dangling", "unreadable"} { + var hash bytes.Buffer + if err := foldUntrackedFile(&hash, i, dir, name); err != nil { + t.Fatalf("%s: %v", name, err) + } + if hash.Len() == 0 { + t.Fatalf("%s produced no hash marker", name) + } + } +} diff --git a/internal/intent/fields_test.go b/internal/intent/fields_test.go index e3e1a9b4..86c6e655 100644 --- a/internal/intent/fields_test.go +++ b/internal/intent/fields_test.go @@ -2,10 +2,31 @@ package intent import ( "fmt" + "os" + "path/filepath" "strings" "testing" ) +func TestOverlayCanDisableAutoPorts(t *testing.T) { + dir := t.TempDir() + base := filepath.Join(dir, "base.yaml") + overlay := filepath.Join(dir, "overlay.yaml") + if err := os.WriteFile(base, []byte("name: m\nnetwork: mainnet\ntarget:\n type: local\n auto_ports: true\nnodes:\n - type: fullnode\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(overlay, []byte("target:\n auto_ports: false\n"), 0600); err != nil { + t.Fatal(err) + } + got, err := LoadWithOverlay(base, overlay) + if err != nil { + t.Fatal(err) + } + if got.Target.AutoPorts { + t.Fatal("explicit overlay false did not disable auto_ports") + } +} + // fields_test.go is the field-by-field matrix for intent validation. The // goal is to enumerate every typed field and exercise: // - representative valid values diff --git a/internal/intent/overlay.go b/internal/intent/overlay.go index d551268f..a22448f3 100644 --- a/internal/intent/overlay.go +++ b/internal/intent/overlay.go @@ -37,8 +37,8 @@ func LoadWithOverlay(basePath, overlayPath string) (*Intent, error) { return base, nil } -// mergeOverlay applies overlay YAML on top of the base intent. -// Strategy: unmarshal overlay into a map, then re-marshal and unmarshal onto the base. +// mergeOverlay applies overlay YAML on top of the base intent; it also parses +// raw YAML to detect explicitly supplied zero values such as auto_ports:false. func mergeOverlay(base *Intent, overlayData []byte) error { // Parse overlay as a partial intent var overlay Intent @@ -47,6 +47,12 @@ func mergeOverlay(base *Intent, overlayData []byte) error { if err := dec.Decode(&overlay); err != nil { return fmt.Errorf("parse overlay YAML: %w", err) } + var raw struct { + Target map[string]any `yaml:"target"` + } + if err := yaml.Unmarshal(overlayData, &raw); err != nil { + return fmt.Errorf("parse overlay YAML: %w", err) + } // Simple field-level merge: override non-zero values if overlay.Name != "" { @@ -55,7 +61,8 @@ func mergeOverlay(base *Intent, overlayData []byte) error { if overlay.Network != "" { base.Network = overlay.Network } - if overlay.Target.Type != "" || overlay.Target.Host != "" || overlay.Target.Port != 0 || overlay.Target.User != "" || overlay.Target.IdentityFile != "" || overlay.Target.Runtime != "" || overlay.Target.AutoPorts { + _, autoPortsSet := raw.Target["auto_ports"] + if overlay.Target.Type != "" || overlay.Target.Host != "" || overlay.Target.Port != 0 || overlay.Target.User != "" || overlay.Target.IdentityFile != "" || overlay.Target.Runtime != "" || autoPortsSet { if overlay.Target.Type != "" { base.Target.Type = overlay.Target.Type } @@ -74,8 +81,8 @@ func mergeOverlay(base *Intent, overlayData []byte) error { if overlay.Target.Runtime != "" { base.Target.Runtime = overlay.Target.Runtime } - if overlay.Target.AutoPorts { - base.Target.AutoPorts = true + if autoPortsSet { + base.Target.AutoPorts = overlay.Target.AutoPorts } } if len(overlay.Nodes) > 0 { diff --git a/internal/state/store.go b/internal/state/store.go index fa4af847..e802a29e 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -104,13 +104,16 @@ func (s *Store) Save(state *DeploymentState) error { return fmt.Errorf("rename state file: %w", err) } // Sync the directory entry so a crash after rename cannot lose the name. - dir, err := os.Open(filepath.Dir(s.path)) + dir, err := openStateDir(filepath.Dir(s.path)) if err != nil { return fmt.Errorf("open state directory: %w", err) } if err := dir.Sync(); err != nil { _ = dir.Close() - return fmt.Errorf("sync state directory: %w", err) + // The file was atomically renamed after its contents were synced. + // Directory fsync is unavailable on Windows, so this best-effort + // durability step is non-fatal once rename has succeeded. + return nil } if err := dir.Close(); err != nil { return fmt.Errorf("close state directory: %w", err) @@ -119,6 +122,13 @@ func (s *Store) Save(state *DeploymentState) error { return nil } +type stateDir interface { + Sync() error + Close() error +} + +var openStateDir = func(path string) (stateDir, error) { return os.Open(path) } + // GetNode returns the managed node by name, or nil if not found. func (s *Store) GetNode(state *DeploymentState, name string) *ManagedNode { for i := range state.Nodes { diff --git a/internal/state/store_test.go b/internal/state/store_test.go index 275a3cd6..c58f82b0 100644 --- a/internal/state/store_test.go +++ b/internal/state/store_test.go @@ -33,6 +33,24 @@ func TestLoad_EmptyFile(t *testing.T) { } } +func TestSaveIgnoresDirectorySyncFailureAfterRename(t *testing.T) { + s, path := newTempStore(t) + old := openStateDir + openStateDir = func(string) (stateDir, error) { return failingStateDir{}, nil } + t.Cleanup(func() { openStateDir = old }) + if err := s.Save(&DeploymentState{}); err != nil { + t.Fatalf("Save: %v", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("renamed state missing: %v", err) + } +} + +type failingStateDir struct{} + +func (failingStateDir) Sync() error { return os.ErrInvalid } +func (failingStateDir) Close() error { return nil } + func TestUpsertAndGetNode(t *testing.T) { s, _ := newTempStore(t) st, _ := s.Load() diff --git a/internal/target/local.go b/internal/target/local.go index 225ce93f..8696adc9 100644 --- a/internal/target/local.go +++ b/internal/target/local.go @@ -56,12 +56,17 @@ func (t *LocalTarget) StreamExec(ctx context.Context, cmd string, args ...string copyStream := func(src io.Reader) { defer wg.Done(); _, _ = io.Copy(pw, src) } go copyStream(stdout) go copyStream(stderr) - go func() { wg.Wait(); _ = pw.Close() }() + streamDone := make(chan struct{}) + go func() { + wg.Wait() + close(streamDone) + _ = pw.Close() + }() return &streamReader{ReadCloser: pr, terminate: func() { if c.Process != nil { _ = c.Process.Kill() } - }, wait: func() error { + }, streamDone: streamDone, wait: func() error { waitErr := c.Wait() wg.Wait() return waitErr diff --git a/internal/target/stream_test.go b/internal/target/stream_test.go index 7392736b..c3a9c962 100644 --- a/internal/target/stream_test.go +++ b/internal/target/stream_test.go @@ -39,6 +39,23 @@ func TestLocalTargetStreamExecMergesStderr(t *testing.T) { } } +func TestLocalTargetStreamExecNaturalEOFDoesNotKill(t *testing.T) { + r, err := NewLocalTarget().StreamExec(context.Background(), "sh", "-c", "trap 'printf killed' TERM; printf done") + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + if err := r.Close(); err != nil { + t.Fatal(err) + } + if string(data) != "done" { + t.Fatalf("natural EOF output = %q, want done without kill signal", data) + } +} + func TestLocalTargetStreamExecReturnsExitError(t *testing.T) { r, err := NewLocalTarget().StreamExec(context.Background(), "sh", "-c", "printf bad >&2; exit 7") if err != nil { diff --git a/internal/target/target.go b/internal/target/target.go index a63e0a53..20e5891e 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -37,9 +37,12 @@ func HTTPClient(t Target, timeout time.Duration) *http.Client { var targetTransports sync.Map var localTransportOnce sync.Once +var localTransportMu sync.Mutex var sharedLocalTransport *http.Transport func localTransport(d Dialer) *http.Transport { + localTransportMu.Lock() + defer localTransportMu.Unlock() localTransportOnce.Do(func() { sharedLocalTransport = http.DefaultTransport.(*http.Transport).Clone() sharedLocalTransport.DialContext = d.DialContext @@ -84,6 +87,8 @@ func targetIdentity(t Target) uintptr { // CloseIdleConnections releases pooled connections held by target-aware HTTP // transports. Long-lived callers should invoke it during shutdown. func CloseIdleConnections() { + localTransportMu.Lock() + defer localTransportMu.Unlock() if sharedLocalTransport != nil { sharedLocalTransport.CloseIdleConnections() } @@ -226,15 +231,24 @@ type StreamExec interface { type streamReader struct { io.ReadCloser - wait func() error - terminate func() - closeOnce sync.Once - closeErr error + wait func() error + terminate func() + streamDone <-chan struct{} + closeOnce sync.Once + closeErr error } func (r *streamReader) Close() error { r.closeOnce.Do(func() { - r.terminate() + if r.streamDone == nil { + r.terminate() + } else { + select { + case <-r.streamDone: + default: + r.terminate() + } + } err := r.ReadCloser.Close() waitErr := r.wait() if err != nil { diff --git a/internal/target/target_test.go b/internal/target/target_test.go index 5aa4dcfb..c9b56533 100644 --- a/internal/target/target_test.go +++ b/internal/target/target_test.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -178,6 +179,22 @@ func TestCloseIdleConnectionsClosesCachedTransports(t *testing.T) { CloseIdleConnections() } +func TestLocalTransportConcurrentCloseIsRaceFree(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 32; i++ { + wg.Add(2) + go func() { + defer wg.Done() + _ = HTTPClient(NewLocalTarget(), time.Second).Transport + }() + go func() { + defer wg.Done() + CloseIdleConnections() + }() + } + wg.Wait() +} + func TestSSHTargetConnectContextUnreachableReturnsPromptly(t *testing.T) { originalDial := sshNetDialContext sshNetDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { diff --git a/tools/replay/replayer_test.go b/tools/replay/replayer_test.go index ff06b7fe..1fda3d90 100644 --- a/tools/replay/replayer_test.go +++ b/tools/replay/replayer_test.go @@ -101,8 +101,10 @@ func TestRunBroadcastFailureDoesNotAdvanceCursor(t *testing.T) { broadcasts++ if broadcasts == 1 { _, _ = w.Write([]byte(`{"result":true}`)) - } else { + } else if broadcasts == 2 { _, _ = w.Write([]byte(`{"result":false,"code":"FAIL","message":"failed"}`)) + } else { + _, _ = w.Write([]byte(`{"result":true}`)) } default: http.NotFound(w, req) @@ -135,6 +137,16 @@ func TestRunBroadcastFailureDoesNotAdvanceCursor(t *testing.T) { if got := loadState(statePath).LastMainnetBlock; got != 99 { t.Fatalf("last_mainnet_block = %d, want 99", got) } + if err := r.Run(context.Background()); err != nil { + t.Fatalf("retry Run: %v", err) + } + got := loadState(statePath) + if got.LastMainnetBlock != 100 || got.InProgressTxIndex != 0 { + t.Fatalf("retry state = %+v, want block 100 and reset tx index", got) + } + if broadcasts != 4 { + t.Fatalf("broadcasts after retry = %d, want 4 (successful tx replayed)", broadcasts) + } } func TestRunFetchFailureDoesNotAdvanceCursor(t *testing.T) { From 28979febd63de4e21a53372505f92ea5bd134dc3 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 31 Aug 2026 17:08:02 +0800 Subject: [PATCH 48/49] fix(network): restore swapped-failed nodes via auto-rollback A child upgrade that failed after tx.Activate (artifact sha probe or state persist) exited non-zero without rolling back, and the parent only admitted successful children to the rollback set - leaving a node on the new artifact with state recording the old version. - child: persist the true running version (Version/PreviousVersion/ Status) on post-activation failure and report the new optional top-level artifact_swapped fact in the error envelope (schema 1.16.1 -> 1.16.2) - parent: parse the envelope tolerantly (last JSON line with error_code), admit swapped-failed nodes to the rollback set, and pass pre-attempt version metadata to the rollback child via env - rollback: network restore mode proceeds with pre-attempt truth and preserves the pre-series PreviousVersion; standalone rollback is unchanged - guard the child env against inherited TROND_NETWORK_UPGRADE_* shadowing; warn when the best-effort truth persist fails - document the envelope field in AGENTS.md and CHANGELOG --- AGENTS.md | 3 + CHANGELOG.md | 4 + cmd/network/upgrade.go | 93 ++++++++++++--- cmd/network/upgrade_test.go | 146 ++++++++++++++++++++++-- cmd/rollback.go | 22 +++- cmd/rollback_test.go | 66 +++++++++++ cmd/upgrade.go | 32 +++++- cmd/upgrade_test.go | 34 ++++++ internal/output/error.go | 3 + internal/schema/embed.go | 2 +- internal/schema/files/error.schema.json | 4 + internal/schema/version_baseline.json | 4 +- schemas/output/error.schema.json | 4 + 13 files changed, 379 insertions(+), 38 deletions(-) create mode 100644 cmd/rollback_test.go create mode 100644 cmd/upgrade_test.go diff --git a/AGENTS.md b/AGENTS.md index 9a8e799c..13317b62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,6 +116,9 @@ Every error JSON has the same shape: } ``` +Errors may also include optional top-level `artifact_swapped: true` when a new +artifact was already activated and started but was not rolled back. + Process `suggestions[]` in order. The first entry is the most prescriptive and usually fixes the issue automatically; later entries are alternative fixes for less-likely causes. diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f7015e..61370349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Network auto-rollback now includes nodes whose upgrade artifact was activated + before a later child failure, using the `artifact_swapped` error-envelope fact; + schema contract bumped from 1.16.1 to 1.16.2. Network rollback also preserves + the pre-series `PreviousVersion` metadata so repeated rollbacks remain honest. - Upgrade and rollback now fail when artifact SHA256 probing fails instead of retaining a stale digest that could produce a false drift result. - Untracked build-source directories, dangling links, and unreadable files are diff --git a/cmd/network/upgrade.go b/cmd/network/upgrade.go index d945fed2..1386bff4 100644 --- a/cmd/network/upgrade.go +++ b/cmd/network/upgrade.go @@ -169,7 +169,7 @@ func runUpgrade(cmd *cobra.Command, args []string) error { if upgradeJarSHA256 != "" { upgradeArgs = append(upgradeArgs, "--jar-sha256", upgradeJarSHA256) } - err := runNetworkChild(cmd.Context(), exe, st, node, upgradeArgs...) + err, artifactSwapped := runNetworkChildResultFn(cmd.Context(), exe, st, node, upgradeArgs...) steps = append(steps, upgradeStep{ Node: node, Phase: "upgrade", @@ -178,6 +178,11 @@ func runUpgrade(cmd *cobra.Command, args []string) error { Error: errString(err), }) if err != nil { + // Rollback membership tracks mutation, not child exit status: a child + // can swap its artifact and then fail while refreshing or persisting state. + if artifactSwapped { + upgraded = append(upgraded, node) + } return finishWithFailure(cmd.Context(), outputFmt, networkName, exe, st, steps, upgraded, start, node, err) } @@ -256,27 +261,67 @@ func isWitnessNode(n state.ManagedNode) bool { } var runChild = runChildCommand -var runChildWithEnv = runChildCommandWithEnv +var runChildResult = runChildCommandWithResult +var runChildWithEnvResult = runChildCommandWithEnvResult +var runNetworkChildResultFn = runNetworkChildResult const ( networkUpgradeRestoreEnv = "TROND_NETWORK_UPGRADE=1" networkUpgradePreserveEnv = "TROND_PRESERVE_BACKUP=1" + networkUpgradeTargetEnv = "TROND_NETWORK_UPGRADE_TARGET_VERSION" + networkUpgradePreviousEnv = "TROND_NETWORK_UPGRADE_PREVIOUS_VERSION" ) +// rollbackChildEnv carries the node metadata from immediately before the +// attempted upgrade into the rollback child. The parent state is intentionally +// not mutated during the network upgrade, so these are the pre-attempt values. +func rollbackChildEnv(n *state.ManagedNode) []string { + return []string{ + networkUpgradeRestoreEnv, + networkUpgradeTargetEnv + "=" + n.Version, + networkUpgradePreviousEnv + "=" + n.PreviousVersion, + } +} + func runChildCommand(ctx context.Context, exe string, argv ...string) error { - return runChildCommandWithEnv(ctx, exe, nil, argv...) + err, _ := runChildCommandWithEnvResult(ctx, exe, nil, argv...) + return err } -func runChildCommandWithEnv(ctx context.Context, exe string, extraEnv []string, argv ...string) error { +func runChildCommandWithResult(ctx context.Context, exe string, argv ...string) (error, bool) { + return runChildCommandWithEnvResult(ctx, exe, nil, argv...) +} + +func runChildCommandWithEnvResult(ctx context.Context, exe string, extraEnv []string, argv ...string) (error, bool) { argv = append(argv, "--output", "json") cmd := exec.CommandContext(ctx, exe, argv...) if len(extraEnv) > 0 { - cmd.Env = append(os.Environ(), extraEnv...) + cmd.Env = childEnv(extraEnv) } var stdout, stderr limitedBuffer cmd.Stdout = &stdout cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { + runErr := cmd.Run() + artifactSwapped := false + // Errors are emitted on stderr by the root command, while successful + // results are emitted on stdout. Parse both so upgrade failures can report + // whether the artifact was already activated. Malformed/absent envelopes + // conservatively exclude the node from rollback; this is residual risk. + envelope := stdout.Bytes() + if runErr != nil { + envelope = stderr.Bytes() + } + lines := bytes.Split(envelope, []byte("\n")) + for i := len(lines) - 1; i >= 0; i-- { + var value map[string]any + if json.Unmarshal(bytes.TrimSpace(lines[i]), &value) == nil { + if _, ok := value["error_code"]; ok { + artifactSwapped, _ = value["artifact_swapped"].(bool) + break + } + } + } + if runErr != nil { if stderr.Len() > 0 { suffix := "" if stderr.limited || stderr.total > 1<<20 || stderr.Len() >= 1<<20 { @@ -286,21 +331,34 @@ func runChildCommandWithEnv(ctx context.Context, exe string, extraEnv []string, if (stderr.limited || stderr.total > 1<<20 || stderr.Len() >= 1<<20) && len(text) > 256 { text = text[:256] } - return fmt.Errorf("child command failed: %w: %s%s", err, text, suffix) + return fmt.Errorf("child command failed: %w: %s%s", runErr, text, suffix), artifactSwapped } - return err + return runErr, artifactSwapped } var value any if err := json.Unmarshal(stdout.Bytes(), &value); err != nil { - return fmt.Errorf("child command returned invalid JSON: %w", err) + return fmt.Errorf("child command returned invalid JSON: %w", err), false } if _, ok := value.(map[string]any); !ok { - return fmt.Errorf("child command returned JSON that is not an object") + return fmt.Errorf("child command returned JSON that is not an object"), false } if stdout.limited || stderr.limited { - return fmt.Errorf("child command output exceeds 1MiB limit") + return fmt.Errorf("child command output exceeds 1MiB limit"), false } - return nil + return nil, artifactSwapped +} + +func childEnv(extraEnv []string) []string { + baseEnv := make([]string, 0, len(os.Environ())+len(extraEnv)) + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + switch key { + case "TROND_NETWORK_UPGRADE", networkUpgradeTargetEnv, networkUpgradePreviousEnv, "TROND_PRESERVE_BACKUP": + continue + } + baseEnv = append(baseEnv, entry) + } + return append(baseEnv, extraEnv...) } type limitedBuffer struct { @@ -442,6 +500,11 @@ func finishWithFailure(ctx context.Context, outputFmt, networkName, exe string, } func runNetworkChild(ctx context.Context, exe string, st *state.DeploymentState, node string, argv ...string) error { + err, _ := runNetworkChildResult(ctx, exe, st, node, argv...) + return err +} + +func runNetworkChildResult(ctx context.Context, exe string, st *state.DeploymentState, node string, argv ...string) (error, bool) { if st != nil { for _, n := range st.Nodes { if n.Name != node || n.Runtime != "jar" || len(argv) == 0 { @@ -449,13 +512,13 @@ func runNetworkChild(ctx context.Context, exe string, st *state.DeploymentState, } switch argv[0] { case "rollback": - return runChildWithEnv(ctx, exe, []string{networkUpgradeRestoreEnv}, argv...) + return runChildWithEnvResult(ctx, exe, rollbackChildEnv(&n), argv...) case "upgrade": - return runChildWithEnv(ctx, exe, []string{networkUpgradePreserveEnv}, argv...) + return runChildWithEnvResult(ctx, exe, []string{networkUpgradePreserveEnv}, argv...) } } } - return runChild(ctx, exe, argv...) + return runChildResult(ctx, exe, argv...) } var fromManagedNode = target.FromManagedNode diff --git a/cmd/network/upgrade_test.go b/cmd/network/upgrade_test.go index 67678f6c..46fa0bf5 100644 --- a/cmd/network/upgrade_test.go +++ b/cmd/network/upgrade_test.go @@ -10,6 +10,9 @@ import ( "testing" "time" + "github.com/spf13/cobra" + + "github.com/tronprotocol/tron-deployment/internal/paths" "github.com/tronprotocol/tron-deployment/internal/state" "github.com/tronprotocol/tron-deployment/internal/target" ) @@ -113,17 +116,19 @@ nodes: func TestFinishWithFailureRollsBackNodeWhoseVerifyFailed(t *testing.T) { oldAutoRollback := upgradeAutoRollback oldRunChild := runChild + oldResult := runChildResult defer func() { upgradeAutoRollback = oldAutoRollback runChild = oldRunChild + runChildResult = oldResult }() upgradeAutoRollback = true var rollbackNodes []string - runChild = func(_ context.Context, _ string, argv ...string) error { + runChildResult = func(_ context.Context, _ string, argv ...string) (error, bool) { if len(argv) >= 2 && argv[0] == "rollback" { rollbackNodes = append(rollbackNodes, argv[1]) } - return nil + return nil, false } err := finishWithFailure(context.Background(), "", "net", "trond", nil, @@ -163,6 +168,123 @@ func TestRunChildCommandMarksTruncatedStderr(t *testing.T) { } } +func TestRunChildCommandResultReadsArtifactSwappedFromErrorEnvelope(t *testing.T) { + err, swapped := runChildCommandWithResult(context.Background(), "/bin/sh", "-c", + `printf 'Warning: telemetry unavailable\n{"error_code":"UPGRADE_ERROR","message":"state failed","suggestions":["retry"],"artifact_swapped":true}' >&2; exit 1`) + if err == nil || !strings.Contains(err.Error(), "child command failed") { + t.Fatalf("error = %v, want child command failure", err) + } + if !swapped { + t.Fatal("artifact_swapped = false, want true") + } +} + +func TestRunChildCommandResultDoesNotMarkPreActivateFailure(t *testing.T) { + err, swapped := runChildCommandWithResult(context.Background(), "/bin/sh", "-c", + `printf '{"error_code":"UPGRADE_ERROR","message":"prepare failed"}' >&2; exit 1`) + if err == nil { + t.Fatal("expected child command failure") + } + if swapped { + t.Fatal("artifact_swapped = true, want false") + } +} + +func TestRunNetworkChildResultSurfacesArtifactSwapped(t *testing.T) { + old := runChildWithEnvResult + defer func() { runChildWithEnvResult = old }() + runChildWithEnvResult = func(context.Context, string, []string, ...string) (error, bool) { + return errors.New("post-activate failure"), true + } + + err, swapped := runNetworkChildResult(context.Background(), "trond", + &state.DeploymentState{Nodes: []state.ManagedNode{{Name: "net-node0", Runtime: "jar"}}}, + "net-node0", "upgrade", "net-node0") + if err == nil || !swapped { + t.Fatalf("result = (%v, %v), want failure with artifact_swapped=true", err, swapped) + } +} + +func TestRollbackChildEnvUsesPreAttemptMetadata(t *testing.T) { + tests := []struct { + name, version, previous string + want []string + }{ + {"prior upgrade", "A", "A0", []string{networkUpgradeRestoreEnv, networkUpgradeTargetEnv + "=A", networkUpgradePreviousEnv + "=A0"}}, + {"first upgrade", "A", "", []string{networkUpgradeRestoreEnv, networkUpgradeTargetEnv + "=A", networkUpgradePreviousEnv + "="}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := rollbackChildEnv(&state.ManagedNode{Version: tt.version, PreviousVersion: tt.previous}) + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("env = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRunChildCommandWithEnvResultStripsInheritedUpgradeMetadata(t *testing.T) { + t.Setenv(networkUpgradeRestoreEnv[:len(networkUpgradeRestoreEnv)-2], "0") + t.Setenv("TROND_PRESERVE_BACKUP", "0") + t.Setenv(networkUpgradeTargetEnv, "stale-target") + t.Setenv(networkUpgradePreviousEnv, "stale-previous") + env := childEnv([]string{networkUpgradeTargetEnv + "=A", networkUpgradePreviousEnv + "=A0"}) + values := map[string]string{} + for _, entry := range env { + if strings.HasPrefix(entry, "TROND_NETWORK_UPGRADE") || strings.HasPrefix(entry, "TROND_PRESERVE_BACKUP") { + parts := strings.SplitN(entry, "=", 2) + values[parts[0]] = parts[1] + } + } + if values[networkUpgradeTargetEnv] != "A" || values[networkUpgradePreviousEnv] != "A0" || + values["TROND_NETWORK_UPGRADE"] != "" || values["TROND_PRESERVE_BACKUP"] != "" { + t.Fatalf("upgrade env values = %v, want target=A previous=A0", values) + } +} + +func TestRunUpgradeIncludesSwappedFailureWithPreAttemptRollbackMetadata(t *testing.T) { + paths.SetBaseDir(t.TempDir()) + t.Cleanup(func() { paths.SetBaseDir("") }) + store, err := state.NewStore(paths.State()) + if err != nil { + t.Fatal(err) + } + if err := store.Save(&state.DeploymentState{Version: 1, Nodes: []state.ManagedNode{{ + Name: "net-node0", Runtime: "jar", Network: "private", Version: "A", PreviousVersion: "A0", + }}}); err != nil { + t.Fatal(err) + } + + oldFn, oldAuto, oldVersion := runNetworkChildResultFn, upgradeAutoRollback, upgradeVersion + defer func() { runNetworkChildResultFn, upgradeAutoRollback, upgradeVersion = oldFn, oldAuto, oldVersion }() + upgradeAutoRollback, upgradeVersion = true, "B" + var rollbackEnv []string + oldEnvFn := runChildWithEnvResult + defer func() { runChildWithEnvResult = oldEnvFn }() + runChildWithEnvResult = func(_ context.Context, _ string, env []string, argv ...string) (error, bool) { + if argv[0] == "rollback" { + rollbackEnv = append(append([]string(nil), env...), argv...) + } + return nil, false + } + runNetworkChildResultFn = func(ctx context.Context, exe string, st *state.DeploymentState, node string, argv ...string) (error, bool) { + if argv[0] == "rollback" { + return runNetworkChildResult(ctx, exe, st, node, argv...) + } + return errors.New("swapped child failure"), true + } + + cmd := &cobra.Command{} + cmd.Flags().String("output", "", "") + err = runUpgrade(cmd, []string{"net"}) + if err == nil || !strings.Contains(err.Error(), "net-node0") { + t.Fatalf("error = %v", err) + } + if len(rollbackEnv) < 3 || rollbackEnv[1] != networkUpgradeTargetEnv+"=A" || rollbackEnv[2] != networkUpgradePreviousEnv+"=A0" { + t.Fatalf("rollback env = %v, want pre-attempt A/A0", rollbackEnv) + } +} + func TestLimitedBufferTracksOverflow(t *testing.T) { var b limitedBuffer _, _ = b.Write(make([]byte, (1<<20)+1)) @@ -172,15 +294,15 @@ func TestLimitedBufferTracksOverflow(t *testing.T) { } func TestFinishWithFailureDoesNotPassUpgradeJarArgumentsToRollback(t *testing.T) { - oldAuto, oldURL, oldSHA, oldRun := upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256, runChild + oldAuto, oldURL, oldSHA, oldRun, oldResult := upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256, runChild, runChildResult defer func() { - upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256, runChild = oldAuto, oldURL, oldSHA, oldRun + upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256, runChild, runChildResult = oldAuto, oldURL, oldSHA, oldRun, oldResult }() upgradeAutoRollback, upgradeJarURL, upgradeJarSHA256 = true, "https://example/jar", "abc" var got []string - runChild = func(_ context.Context, _ string, argv ...string) error { + runChildResult = func(_ context.Context, _ string, argv ...string) (error, bool) { got = append([]string(nil), argv...) - return nil + return nil, false } _ = finishWithFailure(context.Background(), "", "net", "trond", nil, nil, []string{"net-node0"}, time.Now(), "net-node1", errors.New("failed")) want := []string{"rollback", "net-node0"} @@ -191,13 +313,13 @@ func TestFinishWithFailureDoesNotPassUpgradeJarArgumentsToRollback(t *testing.T) func TestRunNetworkChildOnlyInjectsUpgradeEnvForRollback(t *testing.T) { st := &state.DeploymentState{Nodes: []state.ManagedNode{{Name: "net-node0", Runtime: "jar"}}} - old := runChildWithEnv + old := runChildWithEnvResult oldChild := runChild - defer func() { runChildWithEnv = old; runChild = oldChild }() + defer func() { runChildWithEnvResult = old; runChild = oldChild }() var envCalls [][]string - runChildWithEnv = func(_ context.Context, _ string, env []string, _ ...string) error { + runChildWithEnvResult = func(_ context.Context, _ string, env []string, _ ...string) (error, bool) { envCalls = append(envCalls, append([]string(nil), env...)) - return nil + return nil, false } runChild = func(context.Context, string, ...string) error { return nil } if err := runNetworkChild(context.Background(), "trond", st, "net-node0", "upgrade", "net-node0"); err != nil { @@ -209,8 +331,8 @@ func TestRunNetworkChildOnlyInjectsUpgradeEnvForRollback(t *testing.T) { if err := runNetworkChild(context.Background(), "trond", st, "net-node0", "rollback", "net-node0"); err != nil { t.Fatal(err) } - if len(envCalls) != 2 || !reflect.DeepEqual(envCalls[1], []string{networkUpgradeRestoreEnv}) { - t.Fatalf("rollback env = %v, want [%s]", envCalls, networkUpgradeRestoreEnv) + if len(envCalls) != 2 || !reflect.DeepEqual(envCalls[1], []string{networkUpgradeRestoreEnv, networkUpgradeTargetEnv + "=", networkUpgradePreviousEnv + "="}) { + t.Fatalf("rollback env = %v, want [%s %s= %s=]", envCalls, networkUpgradeRestoreEnv, networkUpgradeTargetEnv, networkUpgradePreviousEnv) } } diff --git a/cmd/rollback.go b/cmd/rollback.go index 62ec4ed1..154baff2 100644 --- a/cmd/rollback.go +++ b/cmd/rollback.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "os" "time" "github.com/spf13/cobra" @@ -23,6 +24,8 @@ var rollbackCmd = &cobra.Command{ // saveRollbackState is a test injection seam; production default persists via nodeContext.SaveState. var saveRollbackState = func(nc *nodeContext) error { return nc.SaveState() } +var resolveRollbackNodeContext = resolveNodeContextForWrite + func init() { rollbackCmd.Flags().StringVar(&rollbackJarURL, "jar-url", "", "Target-version JAR URL (jar runtime)") rollbackCmd.Flags().StringVar(&rollbackJarSHA256, "jar-sha256", "", "Optional target-version JAR SHA256 (jar runtime)") @@ -37,13 +40,14 @@ func runRollback(cmd *cobra.Command, args []string) error { return err } - nc, err := resolveNodeContextForWrite(name) + nc, err := resolveRollbackNodeContext(name) if err != nil { return err } defer nc.Close() - if nc.Node.PreviousVersion == "" { + networkRestore := os.Getenv("TROND_NETWORK_UPGRADE") == "1" + if nc.Node.PreviousVersion == "" && !networkRestore { return exitWithError("ROLLBACK_ERROR", output.ExitGeneralError, fmt.Sprintf("No previous version recorded for %s", name), "Rollback is only available after an upgrade") @@ -51,6 +55,14 @@ func runRollback(cmd *cobra.Command, args []string) error { currentVersion := nc.Node.Version targetVersion := nc.Node.PreviousVersion + if networkRestore { + if value := os.Getenv("TROND_NETWORK_UPGRADE_TARGET_VERSION"); value != "" { + targetVersion = value + } else if targetVersion == "" { + targetVersion = currentVersion + } + } + preUpgradePreviousVersion, hasPreUpgradePreviousVersion := os.LookupEnv("TROND_NETWORK_UPGRADE_PREVIOUS_VERSION") upgrader, supported := nc.Runtime.(runtime.ArtifactUpgrader) if !supported { @@ -92,7 +104,11 @@ func runRollback(cmd *cobra.Command, args []string) error { // Commit state only after the replacement artifact is running. nc.Node.Version = targetVersion - nc.Node.PreviousVersion = currentVersion + if networkRestore && hasPreUpgradePreviousVersion { + nc.Node.PreviousVersion = preUpgradePreviousVersion + } else { + nc.Node.PreviousVersion = currentVersion + } nc.Node.Status = "running" if err := persistNodeState("rollback", name, nc, start, saveRollbackState); err != nil { return err diff --git a/cmd/rollback_test.go b/cmd/rollback_test.go new file mode 100644 index 00000000..6ebb956c --- /dev/null +++ b/cmd/rollback_test.go @@ -0,0 +1,66 @@ +package cmd + +import ( + "context" + "io" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/runtime" + "github.com/tronprotocol/tron-deployment/internal/state" + "github.com/tronprotocol/tron-deployment/internal/target" +) + +type rollbackTestTransaction struct{} + +func (rollbackTestTransaction) Activate(context.Context) error { return nil } +func (rollbackTestTransaction) Start(context.Context) error { return nil } +func (rollbackTestTransaction) Rollback(context.Context) error { return nil } +func (rollbackTestTransaction) Cleanup(context.Context) error { return nil } + +type rollbackTestRuntime struct{} + +func (rollbackTestRuntime) Deploy(context.Context, runtime.DeployOpts) error { return nil } +func (rollbackTestRuntime) Start(context.Context, string) error { return nil } +func (rollbackTestRuntime) Stop(context.Context, string) error { return nil } +func (rollbackTestRuntime) Remove(context.Context, string, bool) error { return nil } +func (rollbackTestRuntime) Status(context.Context, string) (*runtime.NodeStatus, error) { + return nil, nil +} +func (rollbackTestRuntime) Logs(context.Context, string, runtime.LogOpts) (io.ReadCloser, error) { + return io.NopCloser(nil), nil +} +func (rollbackTestRuntime) PrepareArtifact(context.Context, string, runtime.UpgradeOpts) (runtime.ArtifactTransaction, error) { + return rollbackTestTransaction{}, nil +} + +func TestRunRollbackNetworkRestorePersistsPreAttemptMetadata(t *testing.T) { + tests := []struct { + name, previous, wantPrevious string + }{ + {"preserves previous version", "A0", "A0"}, + {"first upgrade keeps empty previous", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + oldResolve, oldSave := resolveRollbackNodeContext, saveRollbackState + t.Cleanup(func() { resolveRollbackNodeContext, saveRollbackState = oldResolve, oldSave }) + node := &state.ManagedNode{Name: "n0", Runtime: "docker", Version: "A", PreviousVersion: tt.previous} + st := &state.DeploymentState{Nodes: []state.ManagedNode{*node}} + resolveRollbackNodeContext = func(string) (*nodeContext, error) { + return &nodeContext{Node: node, State: st, Target: target.NewLocalTarget(), Runtime: rollbackTestRuntime{}}, nil + } + var saved state.ManagedNode + saveRollbackState = func(nc *nodeContext) error { saved = *nc.Node; return nil } + t.Setenv("TROND_NETWORK_UPGRADE", "1") + t.Setenv("TROND_NETWORK_UPGRADE_TARGET_VERSION", "A") + t.Setenv("TROND_NETWORK_UPGRADE_PREVIOUS_VERSION", tt.previous) + cmd := newCmd() + if err := runRollback(cmd, []string{"n0"}); err != nil { + t.Fatal(err) + } + if saved.Version != "A" || saved.PreviousVersion != tt.wantPrevious || saved.Status != "running" { + t.Fatalf("saved state = version %q previous %q status %q", saved.Version, saved.PreviousVersion, saved.Status) + } + }) + } +} diff --git a/cmd/upgrade.go b/cmd/upgrade.go index 2f07409b..3cd9afba 100644 --- a/cmd/upgrade.go +++ b/cmd/upgrade.go @@ -84,10 +84,16 @@ func runUpgrade(cmd *cobra.Command, args []string) error { } if nc.Node.Runtime == "jar" { if err := refreshArtifactSHA256(cmd.Context(), nc); err != nil { - // The new artifact is running while state still records the old - // version/digest; leave the skew visible until a later successful run. + // The digest may be stale, but the running version is known. Persist + // that truth so a parent rollback can use the normal metadata path. + nc.Node.Version = upgradeVersion + nc.Node.PreviousVersion = previousVersion + nc.Node.Status = "running" + if persistErr := saveUpgradeState(nc); persistErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: could not persist running-version truth: %v\n", persistErr) + } writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "error", ErrorCode: "UPGRADE_ERROR", Start: start}) - return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, fmt.Sprintf("hash upgraded artifact: %v", err)) + return markArtifactSwapped(exitWithError("UPGRADE_ERROR", output.ExitGeneralError, fmt.Sprintf("hash upgraded artifact: %v", err))) } } if err := tx.Cleanup(cmd.Context()); err != nil { @@ -99,7 +105,12 @@ func runUpgrade(cmd *cobra.Command, args []string) error { nc.Node.Version = upgradeVersion nc.Node.Status = "running" if err := persistNodeState("upgrade", name, nc, start, saveUpgradeState); err != nil { - return err + // The first save failed after activation. Retry best-effort with the + // true running version; preserve the original loud state error. + if persistErr := saveUpgradeState(nc); persistErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: could not persist running-version truth: %v\n", persistErr) + } + return markArtifactSwapped(err) } writeAudit(auditEvent{Command: "upgrade", Node: name, Target: nc.Target.String(), Result: "success", Start: start}) @@ -112,6 +123,13 @@ func runUpgrade(cmd *cobra.Command, args []string) error { return nil } +func markArtifactSwapped(err error) error { + if se, ok := err.(*output.StructuredError); ok { + se.ArtifactSwapped = true + } + return err +} + func rollbackUpgrade(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTransaction, name, previousVersion string, start time.Time, cause string) error { if err := nc.Runtime.Stop(cmd.Context(), name); err != nil { cause = fmt.Sprintf("%s; stop failed: %v", cause, err) @@ -130,5 +148,9 @@ func rollbackUpgrade(cmd *cobra.Command, nc *nodeContext, tx runtime.ArtifactTra if rollbackErr != nil { msg = fmt.Sprintf("%s; rollback ALSO failed: %v", msg, rollbackErr) } - return exitWithError("UPGRADE_ERROR", output.ExitGeneralError, msg, "Check logs: trond logs "+name, "Run diagnostics: trond diagnose "+name) + err := exitWithError("UPGRADE_ERROR", output.ExitGeneralError, msg, "Check logs: trond logs "+name, "Run diagnostics: trond diagnose "+name) + if rollbackErr != nil { + return markArtifactSwapped(err) + } + return err } diff --git a/cmd/upgrade_test.go b/cmd/upgrade_test.go new file mode 100644 index 00000000..492d1933 --- /dev/null +++ b/cmd/upgrade_test.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "encoding/json" + "testing" + + "github.com/tronprotocol/tron-deployment/internal/output" +) + +func TestMarkArtifactSwappedPreservesUpgradeErrorEnvelope(t *testing.T) { + err := output.NewError("UPGRADE_ERROR", output.ExitGeneralError, "hash upgraded artifact: mismatch"). + WithSuggestions("Check logs: trond logs node", "Run diagnostics: trond diagnose node") + marked := markArtifactSwapped(err) + + se, ok := marked.(*output.StructuredError) + if !ok { + t.Fatalf("marked error type = %T, want *output.StructuredError", marked) + } + if se.Code != "UPGRADE_ERROR" || se.Message != "hash upgraded artifact: mismatch" || + len(se.Suggestions) != 2 || !se.ArtifactSwapped { + t.Fatalf("marked envelope = %+v, error fields changed or fact missing", se) + } + data, marshalErr := json.Marshal(se) + if marshalErr != nil { + t.Fatal(marshalErr) + } + var envelope map[string]any + if err := json.Unmarshal(data, &envelope); err != nil { + t.Fatal(err) + } + if envelope["artifact_swapped"] != true { + t.Fatalf("JSON envelope = %s, artifact_swapped missing", data) + } +} diff --git a/internal/output/error.go b/internal/output/error.go index deb279c6..2dabdd29 100644 --- a/internal/output/error.go +++ b/internal/output/error.go @@ -20,6 +20,9 @@ type StructuredError struct { Message string `json:"message"` Suggestions []string `json:"suggestions,omitempty"` ExitCode int `json:"exit_code"` + // ArtifactSwapped reports that an upgrade changed the runnable artifact + // before a later error. It is omitted for all other errors. + ArtifactSwapped bool `json:"artifact_swapped,omitempty"` } func (e *StructuredError) Error() string { diff --git a/internal/schema/embed.go b/internal/schema/embed.go index ae577f80..b886b895 100644 --- a/internal/schema/embed.go +++ b/internal/schema/embed.go @@ -130,7 +130,7 @@ import ( // gain `intent_hash`, matching what both commands already emit. // Existing schemas gain additive optional fields — PATCH (the // earlier additive bumps that took a MINOR were over-versioned). -const SchemaVersion = "1.16.1" +const SchemaVersion = "1.16.2" // JSONSchemaURLBase is the canonical URL prefix for individual output // schema files. Embedded $id values inside each schema mirror this so diff --git a/internal/schema/files/error.schema.json b/internal/schema/files/error.schema.json index 411c2960..e8d5ae83 100644 --- a/internal/schema/files/error.schema.json +++ b/internal/schema/files/error.schema.json @@ -30,6 +30,10 @@ "type": "object", "description": "Optional command-specific structured context (e.g., which intent field failed validation).", "additionalProperties": true + }, + "artifact_swapped": { + "type": "boolean", + "description": "Upgrade artifact was activated before a later error; the parent should include the node in its rollback set." } } } diff --git a/internal/schema/version_baseline.json b/internal/schema/version_baseline.json index bbf394cb..baa2fa18 100644 --- a/internal/schema/version_baseline.json +++ b/internal/schema/version_baseline.json @@ -1,5 +1,5 @@ { - "schema_version": "1.16.1", + "schema_version": "1.16.2", "entries": [ { "name": "apply", @@ -47,7 +47,7 @@ }, { "name": "error", - "hash": "64d75337eb0c375f14ba1ccbe51519be1ca8a5b02b6b053fc13934f72a227e36" + "hash": "a9d401f0ca83aa7bd4b3b14038a3e7d3f3dd55da44371d7ca6a6e1d95aa7d1e7" }, { "name": "events", diff --git a/schemas/output/error.schema.json b/schemas/output/error.schema.json index 411c2960..e8d5ae83 100644 --- a/schemas/output/error.schema.json +++ b/schemas/output/error.schema.json @@ -30,6 +30,10 @@ "type": "object", "description": "Optional command-specific structured context (e.g., which intent field failed validation).", "additionalProperties": true + }, + "artifact_swapped": { + "type": "boolean", + "description": "Upgrade artifact was activated before a later error; the parent should include the node in its rollback set." } } } From 80c6d4eb3c5b112439b01b481841749b856bbd37 Mon Sep 17 00:00:00 2001 From: warku123 Date: Mon, 31 Aug 2026 17:08:22 +0800 Subject: [PATCH 49/49] docs(replay): record block-failure abort as intentional (AUD-009) Aborting the replay when any transaction broadcast in a block fails is by design: the cursor must never advance past a partially-failed block, or silently dropped txs diverge the shadow chain from mainnet. Document the rationale in TODOS.md, note the behavior in CHANGELOG, and point the re-broadcast test assertion at the TODOS entry. A configurable --block-fail-threshold (default 0 = strict) is explicitly deferred to a follow-up PR. --- CHANGELOG.md | 2 ++ TODOS.md | 10 ++++++++++ tools/replay/replayer_test.go | 1 + 3 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61370349..c221bd9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sensitive witness data remains redacted, JAR configuration permissions are restricted, state saves are atomic with surfaced persistence errors, and replay cursors advance only after complete block replay. +- Replay now aborts with a clear error when any transaction broadcast in a block + fails, and retries resume from the failed block boundary (AUD-009). ### Changed diff --git a/TODOS.md b/TODOS.md index 251818ef..c58cc688 100644 --- a/TODOS.md +++ b/TODOS.md @@ -5,6 +5,16 @@ up cold. ## Open audit follow-ups +- **AUD-009 replay block-failure threshold** — aborting replay when any + transaction broadcast in a block fails is intentional, not a bug: the cursor + must never advance past a partially-failed block, or silently dropped + transactions can make the shadow chain diverge from mainnet. Retry resumes + at the failed block boundary; already-landed transactions are re-broadcast + as expected, pinned by `tools/replay/replayer_test.go` (`want 4` assertion). + Defer a configurable `--block-fail-threshold N` flag (default `0` = current + strict abort; `N > 0` tolerates ≤N failures per block, while all failures + remain in `failLog` and are reported in the summary) to a follow-up PR if + needed. - **AUD-020 transport-pool leak** — retire target-aware transports from the shared pool on every lifecycle path (`internal/target/target.go:38-48,86-94`). - **Windows support** — complete platform-specific lifecycle and filesystem diff --git a/tools/replay/replayer_test.go b/tools/replay/replayer_test.go index 1fda3d90..170d1d26 100644 --- a/tools/replay/replayer_test.go +++ b/tools/replay/replayer_test.go @@ -144,6 +144,7 @@ func TestRunBroadcastFailureDoesNotAdvanceCursor(t *testing.T) { if got.LastMainnetBlock != 100 || got.InProgressTxIndex != 0 { t.Fatalf("retry state = %+v, want block 100 and reset tx index", got) } + // Re-broadcasting already-landed txs on retry is expected; see TODOS.md AUD-009. if broadcasts != 4 { t.Fatalf("broadcasts after retry = %d, want 4 (successful tx replayed)", broadcasts) }