From 04a63533a95f69c09aa19004355bc6cd1856aa0c Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 10:58:02 +0100 Subject: [PATCH 1/9] tests: make tests work with --ssh-host localhost --- cmdeploy/src/cmdeploy/cmdeploy.py | 6 +++++- cmdeploy/src/cmdeploy/tests/online/test_1_basic.py | 12 ++++++------ .../src/cmdeploy/tests/online/test_2_deltachat.py | 4 ++-- cmdeploy/src/cmdeploy/tests/plugin.py | 7 ++++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cmdeploy/src/cmdeploy/cmdeploy.py b/cmdeploy/src/cmdeploy/cmdeploy.py index 9fd9763fe..260ef5612 100644 --- a/cmdeploy/src/cmdeploy/cmdeploy.py +++ b/cmdeploy/src/cmdeploy/cmdeploy.py @@ -207,6 +207,7 @@ def test_cmd_options(parser): action="store_true", help="also run slow tests", ) + add_ssh_host_option(parser) def test_cmd(args, out): @@ -218,6 +219,9 @@ def test_cmd(args, out): x = importlib.util.find_spec("deltachat") if x is None: out.check_call(f"{sys.executable} -m pip install deltachat") + env = os.environ.copy() + if args.ssh_host: + env["CHATMAIL_SSH"] = args.ssh_host pytest_path = shutil.which("pytest") pytest_args = [ @@ -231,7 +235,7 @@ def test_cmd(args, out): ] if args.slow: pytest_args.append("--slow") - ret = out.run_ret(pytest_args) + ret = out.run_ret(pytest_args, env=env) return ret diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index c8525125d..631d9a1bc 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -7,21 +7,21 @@ import pytest from cmdeploy import remote -from cmdeploy.sshexec import SSHExec +from cmdeploy.cmdeploy import get_sshexec class TestSSHExecutor: @pytest.fixture(scope="class") def sshexec(self, sshdomain): - return SSHExec(sshdomain) + return get_sshexec(sshdomain) def test_ls(self, sshexec): - out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls")) - out2 = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls")) + out = sshexec.logged(call=remote.rdns.shell, kwargs=dict(command="ls")) + out2 = sshexec.logged(call=remote.rdns.shell, kwargs=dict(command="ls")) assert out == out2 def test_perform_initial(self, sshexec, maildomain): - res = sshexec( + res = sshexec.logged( remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) assert res["A"] or res["AAAA"] @@ -61,7 +61,7 @@ def test_exception(self, sshexec, capsys): def test_opendkim_restarted(self, sshexec): """check that opendkim is not running for longer than a day.""" cmd = "systemctl show opendkim --timestamp=utc --property=ActiveEnterTimestamp" - out = sshexec(call=remote.rshell.shell, kwargs=dict(command=cmd)) + out = sshexec.logged(call=remote.rshell.shell, kwargs=dict(command=cmd)) datestring = out.split("=")[1] since_date = datetime.datetime.strptime(datestring, "%a %Y-%m-%d %H:%M:%S %Z") now = datetime.datetime.now(since_date.tzinfo) diff --git a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py index a87081fd0..b16447948 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_2_deltachat.py @@ -7,7 +7,7 @@ import requests from cmdeploy.remote import rshell -from cmdeploy.sshexec import SSHExec +from cmdeploy.cmdeploy import get_sshexec @pytest.fixture @@ -91,7 +91,7 @@ def parse_size_limit(limit: str) -> int: lp.sec(f"filling remote inbox for {user}") fn = f"7743102289.M843172P2484002.c20,S={quota},W=2398:2," path = chatmail_config.mailboxes_dir.joinpath(user, "cur", fn) - sshexec = SSHExec(sshdomain) + sshexec = get_sshexec(sshdomain) sshexec(call=rshell.write_numbytes, kwargs=dict(path=str(path), num=120)) res = sshexec(call=rshell.dovecot_recalc_quota, kwargs=dict(user=user)) assert res["percent"] >= 100 diff --git a/cmdeploy/src/cmdeploy/tests/plugin.py b/cmdeploy/src/cmdeploy/tests/plugin.py index 6bef2e7e0..d813e5ca4 100644 --- a/cmdeploy/src/cmdeploy/tests/plugin.py +++ b/cmdeploy/src/cmdeploy/tests/plugin.py @@ -361,8 +361,13 @@ def __init__(self, sshdomain): def iter_output(self, logcmd=""): getjournal = "journalctl -f" if not logcmd else logcmd + print(self.sshdomain) + match self.sshdomain: + case "@local": command = [getjournal] + case "localhost": command = [getjournal] + case _: command = ["ssh", f"root@{self.sshdomain}", getjournal] self.popen = subprocess.Popen( - ["ssh", f"root@{self.sshdomain}", getjournal], + command, stdout=subprocess.PIPE, ) while 1: From 723b7f48cb5844d4acccee42d92c31f7bb1f3d28 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 15:38:38 +0100 Subject: [PATCH 2/9] tests: fix test_remote[imap] --- cmdeploy/src/cmdeploy/tests/plugin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmdeploy/src/cmdeploy/tests/plugin.py b/cmdeploy/src/cmdeploy/tests/plugin.py index d813e5ca4..60d27efca 100644 --- a/cmdeploy/src/cmdeploy/tests/plugin.py +++ b/cmdeploy/src/cmdeploy/tests/plugin.py @@ -363,9 +363,10 @@ def iter_output(self, logcmd=""): getjournal = "journalctl -f" if not logcmd else logcmd print(self.sshdomain) match self.sshdomain: - case "@local": command = [getjournal] - case "localhost": command = [getjournal] - case _: command = ["ssh", f"root@{self.sshdomain}", getjournal] + case "@local": command = [] + case "localhost": command = [] + case _: command = ["ssh", f"root@{self.sshdomain}"] + [command.append(arg) for arg in getjournal.split()] self.popen = subprocess.Popen( command, stdout=subprocess.PIPE, From d795a84ad6b24b2f9e9c476c9d5d94ff049d9e5f Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 16:04:53 +0100 Subject: [PATCH 3/9] cmdeploy: call LocalExec directly, not .logged() --- cmdeploy/src/cmdeploy/sshexec.py | 5 +++++ cmdeploy/src/cmdeploy/tests/online/test_1_basic.py | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmdeploy/src/cmdeploy/sshexec.py b/cmdeploy/src/cmdeploy/sshexec.py index c8dd2c798..5658de41b 100644 --- a/cmdeploy/src/cmdeploy/sshexec.py +++ b/cmdeploy/src/cmdeploy/sshexec.py @@ -89,6 +89,11 @@ def __init__(self, verbose=False, docker=False): self.verbose = verbose self.docker = docker + def __call__(self, call, kwargs=None, log_callback=None): + if kwargs is None: + kwargs = {} + return call(**kwargs) + def logged(self, call, kwargs: dict): where = "locally" if self.docker: diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 631d9a1bc..1512634d6 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -16,12 +16,12 @@ def sshexec(self, sshdomain): return get_sshexec(sshdomain) def test_ls(self, sshexec): - out = sshexec.logged(call=remote.rdns.shell, kwargs=dict(command="ls")) - out2 = sshexec.logged(call=remote.rdns.shell, kwargs=dict(command="ls")) + out = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls")) + out2 = sshexec(call=remote.rdns.shell, kwargs=dict(command="ls")) assert out == out2 def test_perform_initial(self, sshexec, maildomain): - res = sshexec.logged( + res = sshexec( remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) assert res["A"] or res["AAAA"] @@ -61,7 +61,7 @@ def test_exception(self, sshexec, capsys): def test_opendkim_restarted(self, sshexec): """check that opendkim is not running for longer than a day.""" cmd = "systemctl show opendkim --timestamp=utc --property=ActiveEnterTimestamp" - out = sshexec.logged(call=remote.rshell.shell, kwargs=dict(command=cmd)) + out = sshexec(call=remote.rshell.shell, kwargs=dict(command=cmd)) datestring = out.split("=")[1] since_date = datetime.datetime.strptime(datestring, "%a %Y-%m-%d %H:%M:%S %Z") now = datetime.datetime.now(since_date.tzinfo) From 3b971b998ee9cad0a9fa9af3d71063355071237a Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 18:21:18 +0100 Subject: [PATCH 4/9] tests: fix TestSSHExecutor.test_logged --- cmdeploy/src/cmdeploy/sshexec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/sshexec.py b/cmdeploy/src/cmdeploy/sshexec.py index 5658de41b..8e96f212b 100644 --- a/cmdeploy/src/cmdeploy/sshexec.py +++ b/cmdeploy/src/cmdeploy/sshexec.py @@ -95,6 +95,7 @@ def __call__(self, call, kwargs=None, log_callback=None): return call(**kwargs) def logged(self, call, kwargs: dict): + title = call.__doc__ where = "locally" if self.docker: if call == remote.rdns.perform_initial_checks: @@ -102,4 +103,9 @@ def logged(self, call, kwargs: dict): where = "in docker" if self.verbose: print(f"Running {where}: {call.__name__}(**{kwargs})") - return call(**kwargs) + return self(call, kwargs, log_callback=print_stderr) + else: + print_stderr(title, end="") + res = self(call, kwargs, log_callback=remote.rshell.log_progress) + print_stderr() + return res From c5bd1c32364433c674ca901e7ea0b777a6b58bd7 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 19:04:19 +0100 Subject: [PATCH 5/9] tests: fix test_status_cmd with --ssh-host @local --- cmdeploy/src/cmdeploy/tests/online/test_3_status.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/tests/online/test_3_status.py b/cmdeploy/src/cmdeploy/tests/online/test_3_status.py index 1783c32e4..d505faf7d 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_3_status.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_3_status.py @@ -5,7 +5,11 @@ def test_status_cmd(chatmail_config, capsys, request): os.chdir(request.config.invocation_params.dir) - assert main(["status"]) == 0 + command = ["status"] + if os.getenv("CHATMAIL_SSH"): + command.append("--ssh-host") + command.append(os.getenv("CHATMAIL_SSH")) + assert main(command) == 0 status_out = capsys.readouterr() print(status_out.out) From 99421a6235eed9c46e34cebe3e9aecb6ebd75dc9 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 19:08:17 +0100 Subject: [PATCH 6/9] tests: fix test_logged with --ssh-host localhost --- cmdeploy/src/cmdeploy/sshexec.py | 4 +++- cmdeploy/src/cmdeploy/tests/online/test_1_basic.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/sshexec.py b/cmdeploy/src/cmdeploy/sshexec.py index 8e96f212b..348276994 100644 --- a/cmdeploy/src/cmdeploy/sshexec.py +++ b/cmdeploy/src/cmdeploy/sshexec.py @@ -96,13 +96,15 @@ def __call__(self, call, kwargs=None, log_callback=None): def logged(self, call, kwargs: dict): title = call.__doc__ + if not title: + title = call.__name__ where = "locally" if self.docker: if call == remote.rdns.perform_initial_checks: kwargs["pre_command"] = "docker exec chatmail " where = "in docker" if self.verbose: - print(f"Running {where}: {call.__name__}(**{kwargs})") + print_stderr(f"Running {where}: {title}(**{kwargs})") return self(call, kwargs, log_callback=print_stderr) else: print_stderr(title, end="") diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index 1512634d6..fbbe7a9c7 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -27,6 +27,7 @@ def test_perform_initial(self, sshexec, maildomain): assert res["A"] or res["AAAA"] def test_logged(self, sshexec, maildomain, capsys): + sshexec.verbose = False sshexec.logged( remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=maildomain) ) From 4f7a25a356b140735042e7941f29b3c8565bfd21 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 17 Feb 2026 19:35:41 +0100 Subject: [PATCH 7/9] tests: fix TestSSHExecutor::test_exception with --ssh-host localhost --- cmdeploy/src/cmdeploy/sshexec.py | 2 ++ cmdeploy/src/cmdeploy/tests/online/test_1_basic.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmdeploy/src/cmdeploy/sshexec.py b/cmdeploy/src/cmdeploy/sshexec.py index 348276994..7470d9277 100644 --- a/cmdeploy/src/cmdeploy/sshexec.py +++ b/cmdeploy/src/cmdeploy/sshexec.py @@ -85,6 +85,8 @@ def logged(self, call, kwargs): class LocalExec: + FuncError = FuncError + def __init__(self, verbose=False, docker=False): self.verbose = verbose self.docker = docker diff --git a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py index fbbe7a9c7..61f34381e 100644 --- a/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py +++ b/cmdeploy/src/cmdeploy/tests/online/test_1_basic.py @@ -53,6 +53,8 @@ def test_exception(self, sshexec, capsys): remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=None), ) + except AssertionError: + pass except sshexec.FuncError as e: assert "rdns.py" in str(e) assert "AssertionError" in str(e) From 8a93fedd56acd8f316d7e6116c84fe499e59b3d3 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 18 Feb 2026 08:27:01 +0100 Subject: [PATCH 8/9] ci: deploy with --ssh-host localhost on staging-ipv4 --- .../workflows/test-and-deploy-ipv4only.yaml | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-and-deploy-ipv4only.yaml b/.github/workflows/test-and-deploy-ipv4only.yaml index ecc79def7..158bab98d 100644 --- a/.github/workflows/test-and-deploy-ipv4only.yaml +++ b/.github/workflows/test-and-deploy-ipv4only.yaml @@ -71,25 +71,35 @@ jobs: - name: run deploy-chatmail offline tests run: pytest --pyargs cmdeploy - - run: | - cmdeploy init staging-ipv4.testrun.org - sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' chatmail.ini - sed -i 's/#\s*mtail_address/mtail_address/' chatmail.ini - cmdeploy run --verbose --skip-dns-check + - name: setup dependencies + run: | + ssh root@staging-ipv4.testrun.org apt update + ssh root@staging-ipv4.testrun.org apt install -y git python3.11-venv python3-dev gcc + ssh root@staging-ipv4.testrun.org git clone https://github.com/chatmail/relay + ssh root@staging-ipv4.testrun.org "cd relay && git checkout " ${{ github.head_ref }} + ssh root@staging-ipv4.testrun.org "cd relay && scripts/initenv.sh" + + - name: initialize config + run: | + ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy init staging-ipv4.testrun.org" + ssh root@staging-ipv4.testrun.org "sed -i 's#disable_ipv6 = False#disable_ipv6 = True#' relay/chatmail.ini" + ssh root@staging-ipv4.testrun.org "sed -i 's/#\s*mtail_address/mtail_address/' relay/chatmail.ini" + + - run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy run --verbose --skip-dns-check --ssh-host localhost" - name: set DNS entries run: | - ssh -o StrictHostKeyChecking=accept-new -v root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys - cmdeploy dns --zonefile staging-generated.zone - cat staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone + ssh root@staging-ipv4.testrun.org chown opendkim:opendkim -R /etc/dkimkeys + ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns --zonefile staging-generated.zone --ssh-host localhost" + ssh root@staging-ipv4.testrun.org cat relay/staging-generated.zone >> .github/workflows/staging-ipv4.testrun.org-default.zone cat .github/workflows/staging-ipv4.testrun.org-default.zone scp .github/workflows/staging-ipv4.testrun.org-default.zone root@ns.testrun.org:/etc/nsd/staging-ipv4.testrun.org.zone ssh root@ns.testrun.org nsd-checkzone staging-ipv4.testrun.org /etc/nsd/staging-ipv4.testrun.org.zone ssh root@ns.testrun.org systemctl reload nsd - name: cmdeploy test - run: CHATMAIL_DOMAIN2=ci-chatmail.testrun.org cmdeploy test --slow + run: ssh root@staging-ipv4.testrun.org "cd relay && CHATMAIL_DOMAIN2=ci-chatmail.testrun.org scripts/cmdeploy test --slow --ssh-host localhost" - name: cmdeploy dns - run: cmdeploy dns -v + run: ssh root@staging-ipv4.testrun.org "cd relay && scripts/cmdeploy dns -v --ssh-host localhost" From c410f7ab49b1d2bc64272a2f964ed1fd5a0bc1e4 Mon Sep 17 00:00:00 2001 From: missytake Date: Wed, 18 Feb 2026 11:17:22 +0100 Subject: [PATCH 9/9] metadata: lower RestartSec --- cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f b/cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f index 968b48858..d9bdebe01 100644 --- a/cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f +++ b/cmdeploy/src/cmdeploy/service/chatmail-metadata.service.f @@ -4,7 +4,7 @@ [Service] ExecStart={execpath} /run/chatmail-metadata/metadata.socket {config_path} Restart=always -RestartSec=30 +RestartSec=5 User=vmail RuntimeDirectory=chatmail-metadata UMask=0077