test: cover scp, tunnels and the Xdebug tunnel with mock SSH forwarding

CLAUDE.md

@@ -155,7 +155,7 @@ make download-php
 
 Tests use github.com/stretchr/testify for assertions. Table-driven tests are preferred with a "cases" slice containing simple test case structs.
 
-Integration tests (`integration-tests/`) call `t.Parallel()`, so they must not share state: no `t.Setenv` or unsynchronized package variables, and use `cmdFactory.useConfig` to change the CLI config file. Write output with `t.Log` (e.g. `mockapi.RequestLogger`), not to stdout or stderr: it is then attributed to its test, and shown only on failure or with `-v`. Tests of environment operations use `setUpProject`, and tests of Git commands (push, clone, fetch) use `setUpGitProject`, which also serves a bare repository over the mock SSH server.
+Integration tests (`integration-tests/`) call `t.Parallel()`, so they must not share state: no `t.Setenv` or unsynchronized package variables, and use `cmdFactory.useConfig` to change the CLI config file. Write output with `t.Log` (e.g. `mockapi.RequestLogger`), not to stdout or stderr: it is then attributed to its test, and shown only on failure or with `-v`. Tests of environment operations use `setUpProject`, and tests of Git commands (push, clone, fetch) use `setUpGitProject`, which also serves a bare repository over the mock SSH server. Tests of other SSH commands use `serveSSH` with an app from `sshEnv`, and fake remote executables from `remoteEnv`. The mock SSH server can also serve SFTP (`SFTPDir`) and forward connections (`DialTCP`, `ListenUnix`); tests of commands that start background processes (tunnels) must make sure they exit.
 
 ### Configuration
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/godbus/dbus/v5 v5.2.3
 	github.com/gofrs/flock v0.13.1
 	github.com/oklog/ulid/v2 v2.1.2
+	github.com/pkg/sftp v1.13.11
 	github.com/platformsh/platformify v0.5.0
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
@@ -91,6 +92,7 @@ require (
 	github.com/kevinburke/ssh_config v1.6.0 // indirect
 	github.com/klauspost/compress v1.18.7 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
+	github.com/kr/fs v0.1.0 // indirect
 	github.com/leodido/go-urn v1.5.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
 	github.com/mailru/easyjson v0.9.2 // indirect

go.sum

@@ -173,6 +173,8 @@ github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
 github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -226,6 +228,8 @@ github.com/pjbgf/sha1cd v0.7.0 h1:ZRNPKHj+gfkLBf0KJv/p1Hmz+7bqCT5o311YX0nq+DA=
 github.com/pjbgf/sha1cd v0.7.0/go.mod h1:pKR5Li+qTCo+ebqITZfwPlJGoCFCvSO00qxjbNtTUFQ=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.11 h1:0N92SLTB8JqASJB14ZLHHzFnBV8mG9zw4K7jghEFWuE=
+github.com/pkg/sftp v1.13.11/go.mod h1:uNkH9roSXglNJqM+glJJi+TQXQUm0fXFWqCFmT8hsN0=
 github.com/platformsh/platformify v0.5.0 h1:i9cRbM0K1f14Yr0dJ6dhYq9BcmZb8fsx/mpYSnw8ssA=
 github.com/platformsh/platformify v0.5.0/go.mod h1:fgmCcfQfHbhe1oXsIdIhpnniyZu8IdIMOlcBAa/ygic=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=

integration-tests/db_ssh_test.go

@@ -0,0 +1,152 @@
+package tests
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/upsun/cli/pkg/mockapi"
+	"github.com/upsun/cli/pkg/mockssh"
+)
+
+// fakeDBClients are remote scripts which print their name and arguments, and any input.
+var fakeDBClients = map[string]string{
+	"mariadb":      `echo "mariadb $*"; cat`,
+	"mariadb-dump": `echo "mariadb-dump $*"`,
+	"psql":         `echo "psql $*"; cat`,
+	"pg_dump":      `echo "pg_dump $*"`,
+}
+
+func dbRelationships() map[string]any {
+	return map[string]any{
+		"database": []any{map[string]any{
+			"service": "db", "rel": "mysql", "scheme": "mysql", "type": "mariadb:11.4",
+			"host": "database.internal", "port": 3306, "username": "user", "password": "", "path": "main",
+		}},
+		"pg": []any{map[string]any{
+			"service": "pg", "rel": "postgresql", "scheme": "pgsql", "type": "postgresql:16",
+			"host": "pg.internal", "port": 5432, "username": "main", "password": "secret", "path": "main",
+		}},
+	}
+}
+
+// setUpDBProject serves an app with database relationships over SSH, with fake database clients.
+func setUpDBProject(t *testing.T, clients map[string]string) *testProject {
+	t.Helper()
+	p := setUpProject(t, sshEnv(&mockapi.Deployment{
+		WebApps: map[string]mockapi.App{
+			"app": {Name: "app", Type: "golang:1.23", Size: "M", Mounts: map[string]mockapi.Mount{}},
+		},
+		Services: map[string]mockapi.App{
+			"db": {Name: "db", Type: "mariadb:11.4", Size: "M", Disk: 1024},
+			"pg": {Name: "pg", Type: "postgresql:16", Size: "M", Disk: 2048},
+		},
+	}))
+	p.serveSSH(t, mockssh.ExecHandler(t.TempDir(), remoteEnv(t, clients, dbRelationships())))
+	return p
+}
+
+func TestDbDump(t *testing.T) {
+	t.Parallel()
+	p := setUpDBProject(t, fakeDBClients)
+	f := p.f
+	args := []string{"-p", layoutProjectID, "-e", "main"}
+	mariadbArgs := "--user=user --password= --host=database.internal --port=3306 main"
+
+	_, stdErr, err := f.RunCombinedOutput(append([]string{"db:dump", "-r", "database"}, args...)...)
+	require.NoError(t, err)
+	dumpFile := filepath.Join(f.dir, layoutProjectID+"--main-xyz--db--main--dump.sql")
+	assert.Contains(t, stdErr, "Creating SQL dump file: "+dumpFile)
+	assert.Equal(t, "mariadb-dump --single-transaction "+mariadbArgs+"\n", readFile(t, dumpFile))
+
+	// An existing file is overwritten in non-interactive mode.
+	_, stdErr, err = f.RunCombinedOutput(append([]string{"db:dump", "-r", "database"}, args...)...)
+	require.NoError(t, err)
+	assert.Contains(t, stdErr, "File exists: "+dumpFile+". Overwrite? [Y/n] y")
+
+	_, stdErr, err = f.RunCombinedOutput(append([]string{"db:dump", "-r", "database", "--gzip", "-f", "dump.sql.gz",
+		"--table", "users", "--table", "posts", "--exclude-table", "cache"}, args...)...)
+	require.NoError(t, err)
+	assert.Contains(t, stdErr, "Creating gzipped SQL dump file: "+filepath.Join(f.dir, "dump.sql.gz"))
+	assert.Equal(t, "mariadb-dump --single-transaction "+mariadbArgs+
+		" --ignore-table=main.cache --tables users posts\n", gunzipFile(t, filepath.Join(f.dir, "dump.sql.gz")))
+
+	tableDir := t.TempDir()
+	f.Run(append([]string{"db:dump", "-r", "database", "--table", "users", "--schema-only", "-d", tableDir}, args...)...)
+	assert.Equal(t, "mariadb-dump --single-transaction "+mariadbArgs+" --no-data --tables users\n",
+		readFile(t, filepath.Join(tableDir, layoutProjectID+"--main-xyz--db--main--users--schema--dump.sql")))
+
+	output := f.Run(append([]string{"db:dump", "-r", "pg", "--stdout", "--schema-only",
+		"--table", "users", "--exclude-table", "cache", "--charset", "UTF8"}, args...)...)
+	assert.Equal(t, "pg_dump --no-owner --if-exists --clean --blobs postgresql://main:secret@pg.internal:5432/main"+
+		" --schema-only --table=users --exclude-table=cache --encoding=UTF8\n", output)
+
+	// The remote exit code is returned.
+	failing := setUpDBProject(t, map[string]string{"mariadb-dump": "echo 'dump failed' >&2; exit 3"})
+	_, stdErr, err = failing.f.RunCombinedOutput(append([]string{"db:dump", "-r", "database", "--stdout"}, args...)...)
+	assertExitCode(t, 3, err)
+	assert.Contains(t, stdErr, "dump failed")
+}
+
+func TestDbSQL(t *testing.T) {
+	t.Parallel()
+	p := setUpDBProject(t, fakeDBClients)
+	f := p.f
+	args := []string{"-p", layoutProjectID, "-e", "main"}
+
+	assert.Equal(t, "mariadb --no-auto-rehash --user=user --password= --host=database.internal --port=3306 main"+
+		" --execute SELECT 1\n", f.Run(append([]string{"db:sql", "-r", "database", "SELECT 1"}, args...)...))
+
+	assert.Equal(t, "mariadb --no-auto-rehash --user=user --password= --host=database.internal --port=3306 other"+
+		" --batch --raw --execute SELECT 2\n",
+		f.Run(append([]string{"db:sql", "-r", "database", "--schema", "other", "--raw", "SELECT 2"}, args...)...))
+
+	assert.Equal(t, "psql postgresql://main:secret@pg.internal:5432/main -t -c SELECT 3\n",
+		f.Run(append([]string{"sql", "-r", "pg", "--raw", "SELECT 3"}, args...)...))
+
+	// A query can be piped in.
+	f.stdin = strings.NewReader("SELECT 4;\n")
+	assert.Equal(t, "psql postgresql://main:secret@pg.internal:5432/main\nSELECT 4;\n",
+		f.Run(append([]string{"db:sql", "-r", "pg"}, args...)...))
+}
+
+func TestDbSize(t *testing.T) {
+	t.Parallel()
+	// Each query returns 50 MiB.
+	p := setUpDBProject(t, map[string]string{
+		"mariadb": "cat > /dev/null; echo 52428800",
+		"psql":    "cat > /dev/null; echo 52428800",
+	})
+	f := p.f
+	args := []string{"-p", layoutProjectID, "-e", "main"}
+
+	output, stdErr, err := f.RunCombinedOutput(append([]string{"db:size", "-r", "pg", "--format", "csv"}, args...)...)
+	require.NoError(t, err)
+	assert.Contains(t, stdErr, "Checking database service pg...")
+	assert.Equal(t, "Allocated disk,Estimated usage,% used\n2147483648,52428800,2\n", output)
+
+	output = f.Run(append([]string{"db:size", "-r", "pg", "--no-header"}, args...)...)
+	assert.Contains(t, output, "| 2.0 GiB | 50.0 MiB | ~ 2% |")
+
+	// The MySQL usage includes InnoDB and other tables separately.
+	output = f.Run(append([]string{"db:size", "-r", "database", "--format", "csv", "--no-header"}, args...)...)
+	assert.Equal(t, "1073741824,104857600,10\n", output)
+}
+
+func gunzipFile(t *testing.T, filename string) string {
+	t.Helper()
+	b, err := os.ReadFile(filename)
+	require.NoError(t, err)
+	r, err := gzip.NewReader(bytes.NewReader(b))
+	require.NoError(t, err)
+	out, err := io.ReadAll(r)
+	require.NoError(t, err)
+	return string(out)
+}

integration-tests/environment_scp_test.go

@@ -0,0 +1,53 @@
+package tests
+
+import (
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/upsun/cli/pkg/mockapi"
+	"github.com/upsun/cli/pkg/mockssh"
+)
+
+func TestEnvironmentScp(t *testing.T) {
+	t.Parallel()
+	if _, err := exec.LookPath("scp"); err != nil {
+		t.Skip("scp is not installed")
+	}
+	remoteDir := t.TempDir()
+	writeFiles(t, remoteDir, map[string]string{"remote.txt": "remote", "logs/a.log": "a", "logs/b.log": "b"})
+
+	p := setUpProject(t, sshEnv(&mockapi.Deployment{
+		WebApps: map[string]mockapi.App{
+			"app": {Name: "app", Type: "golang:1.23", Size: "M", Mounts: map[string]mockapi.Mount{}},
+		},
+	}))
+	// Older scp clients run "scp -t" or "scp -f" instead of using SFTP.
+	sshServer := p.serveSSH(t, mockssh.ExecHandler(remoteDir, nil))
+	sshServer.SFTPDir = remoteDir
+	f := p.f
+	f.dir = t.TempDir()
+	writeFiles(t, f.dir, map[string]string{"local.txt": "local"})
+	args := []string{"scp", "-p", layoutProjectID, "-e", "main"}
+
+	f.Run(append(args, "--", "local.txt", "remote:uploaded.txt")...)
+	assert.Equal(t, "local", readFile(t, filepath.Join(remoteDir, "uploaded.txt")))
+
+	f.Run(append(args, "--", "remote:remote.txt", "downloaded.txt")...)
+	assert.Equal(t, "remote", readFile(t, filepath.Join(f.dir, "downloaded.txt")))
+
+	f.Run(append(args, "-r", "--", "remote:logs", "logs")...)
+	assert.Equal(t, "a", readFile(t, filepath.Join(f.dir, "logs", "a.log")))
+	assert.Equal(t, "b", readFile(t, filepath.Join(f.dir, "logs", "b.log")))
+
+	_, stdErr, err := f.RunCombinedOutput(append(args, "--", "remote:missing.txt", ".")...)
+	require.Error(t, err)
+	assert.Contains(t, stdErr, "missing.txt")
+
+	_, stdErr, err = f.RunCombinedOutput(append(args, "--", "local.txt", "other.txt")...)
+	require.Error(t, err)
+	assert.Contains(t, stdErr, `At least one argument needs to contain the "remote:" prefix`)
+}

integration-tests/environment_xdebug_test.go

@@ -1,10 +1,17 @@
 package tests
 
 import (
+	"io"
+	"net"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -121,3 +128,91 @@ func TestEnvironmentXdebug(t *testing.T) {
 		mu.Unlock()
 	}
 }
+
+func TestEnvironmentXdebugTunnel(t *testing.T) {
+	t.Parallel()
+	p := setUpProject(t, sshEnv(&mockapi.Deployment{
+		WebApps: map[string]mockapi.App{
+			"app": {Name: "app", Type: "php:8.4", Size: "M", Mounts: map[string]mockapi.Mount{},
+				Runtime: map[string]any{"xdebug": map[string]any{"idekey": "secret_key"}}},
+		},
+	}))
+	sshServer := p.serveSSH(t, func(_ ssh.ConnMetadata, _ string, _ mockssh.CommandIO) int { return 0 })
+
+	// The server listens at a local socket instead of /run/xdebug-tunnel.sock. It is
+	// in /tmp, as socket paths are limited to about 100 characters.
+	socketDir, err := os.MkdirTemp("/tmp", "xdebug")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = os.RemoveAll(socketDir) })
+	socket := filepath.Join(socketDir, "xdebug.sock")
+	var requestedPath atomic.Value
+	sshServer.ListenUnix = func(path string) (net.Listener, error) {
+		requestedPath.Store(path)
+		return net.Listen("unix", socket)
+	}
+
+	// The IDE listens locally.
+	ide, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ide.Close() })
+	port := tcpPort(t, ide)
+	received := make(chan string, 1)
+	go func() {
+		c, err := ide.Accept()
+		if err != nil {
+			return
+		}
+		defer c.Close()
+		b, _ := io.ReadAll(c)
+		received <- string(b)
+	}()
+
+	cmd := p.f.buildCommand("xdebug", "-p", layoutProjectID, "-e", "main", "--port", strconv.Itoa(port))
+	var stdErr syncBuffer
+	cmd.Stderr = &stdErr
+	require.NoError(t, cmd.Start())
+	var waitErr error
+	exited := make(chan struct{})
+	go func() {
+		waitErr = cmd.Wait()
+		close(exited)
+	}()
+	t.Cleanup(func() {
+		select {
+		case <-exited:
+		case <-time.After(10 * time.Second):
+			_ = cmd.Process.Kill()
+			t.Error("xdebug did not exit")
+		}
+		p.f.logStderr(stdErr.String())
+	})
+
+	// A connection from the app (Xdebug) to the socket reaches the IDE.
+	var conn net.Conn
+	require.Eventually(t, func() bool {
+		conn, err = net.Dial("unix", socket)
+		return err == nil
+	}, 10*time.Second, 50*time.Millisecond)
+	_, err = io.WriteString(conn, "<init idekey=\"secret_key\"/>")
+	require.NoError(t, err)
+	require.NoError(t, conn.Close())
+	select {
+	case got := <-received:
+		assert.Equal(t, "<init idekey=\"secret_key\"/>", got)
+	case <-time.After(10 * time.Second):
+		t.Fatal("the IDE did not receive a connection")
+	}
+	assert.Equal(t, "/run/xdebug-tunnel.sock", requestedPath.Load())
+	require.Eventually(t, func() bool {
+		return strings.Contains(stdErr.String(), "Xdebug tunnel opened at: 127.0.0.1:"+strconv.Itoa(port))
+	}, 10*time.Second, 50*time.Millisecond)
+
+	// The tunnel closes with the SSH connection.
+	require.NoError(t, sshServer.Stop())
+	select {
+	case <-exited:
+	case <-time.After(10 * time.Second):
+		t.Fatal("xdebug did not exit after the SSH connection closed")
+	}
+	assertExitCode(t, 255, waitErr)
+}