Skip to content

Commit eb9d112

Browse files
committed
test: configure ssh once and bound the calls
Keep the identity and the bounds in one ssh config, and retry the migrated-file check, so a transient stall does not fail the migration. A bounded wait can expire on a frozen guest, whose journal is then unreachable. Keep the hypervisor serial consoles beside the runner logs, because the final kernel output of that guest exists nowhere else. Signed-off-by: Fabian Wiesel <fabian.wiesel@sap.com>
1 parent 01548c8 commit eb9d112

5 files changed

Lines changed: 97 additions & 126 deletions

File tree

‎.github/actions/test/integration/collect-logs/action.yml‎

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,45 +6,46 @@ runs:
66
- name: Obtain logs from test runner
77
shell: bash
88
run: |
9+
# Required from the job environment; the shared key is the identity
10+
# that the ssh config names.
11+
[ -s /opt/ssh_host_ed25519_key ] || { echo "the shared SSH key /opt/ssh_host_ed25519_key is missing" >&2; exit 1; }
912
sudo lscpu > lscpu-RUNNER.log || true
1013
sudo lshw > lshw-runner.log || true
1114
sudo journalctl --no-pager > journal-RUNNER.log || true
1215
1316
sudo cp /var/log/libvirt/qemu/HV1.log "libvirt-HV1.log" || true
1417
sudo cp /var/log/libvirt/qemu/HV2.log "libvirt-HV2.log" || true
1518
19+
# The HV serial consoles: the only record of a frozen guest's
20+
# final kernel output (the journal goes down with the guest).
21+
sudo cp /var/log/HV1.log "serial-HV1.log" || true
22+
sudo cp /var/log/HV2.log "serial-HV2.log" || true
23+
1624
- name: Obtain logs from hypervisors
1725
shell: bash
1826
run: |
19-
KEY=/opt/ssh_host_ed25519_key
20-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
21-
USER="root"
2227
HV1_IP="192.168.122.2"
2328
HV2_IP="192.168.122.3"
2429
25-
ssh $SSH_OPTS $USER@$HV1_IP "journalctl --no-pager" > "journal-HV1.log" || true
26-
scp $SSH_OPTS $USER@$HV1_IP:/var/log/libvirt/qemu/VM-QEMU.log "libvirt-VM-QEMU-HV1.log" || true
27-
scp $SSH_OPTS $USER@$HV1_IP:/var/log/libvirt/ch/VM-CHV.log "libvirt-VM-CHV-HV1.log" || true
28-
scp $SSH_OPTS $USER@$HV1_IP:/var/log/QEMU.log "serial-VM-QEMU-HV1.log" || true
29-
scp $SSH_OPTS $USER@$HV1_IP:/var/log/CHV.log "serial-VM-CHV-HV1.log" || true
30+
ssh "$HV1_IP" "journalctl --no-pager" > "journal-HV1.log" || true
31+
scp "$HV1_IP:/var/log/libvirt/qemu/VM-QEMU.log" "libvirt-VM-QEMU-HV1.log" || true
32+
scp "$HV1_IP:/var/log/libvirt/ch/VM-CHV.log" "libvirt-VM-CHV-HV1.log" || true
33+
scp "$HV1_IP:/var/log/QEMU.log" "serial-VM-QEMU-HV1.log" || true
34+
scp "$HV1_IP:/var/log/CHV.log" "serial-VM-CHV-HV1.log" || true
3035
31-
ssh $SSH_OPTS $USER@$HV2_IP "journalctl --no-pager" > "journal-HV2.log" || true
32-
scp $SSH_OPTS $USER@$HV2_IP:/var/log/libvirt/qemu/VM-QEMU.log "libvirt-VM-QEMU-HV2.log" || true
33-
scp $SSH_OPTS $USER@$HV2_IP:/var/log/libvirt/ch/VM-CHV.log "libvirt-VM-CHV-HV2.log" || true
34-
scp $SSH_OPTS $USER@$HV2_IP:/var/log/QEMU.log "serial-VM-QEMU-HV2.log" || true
35-
scp $SSH_OPTS $USER@$HV2_IP:/var/log/CHV.log "serial-VM-CHV-HV2.log" || true
36+
ssh "$HV2_IP" "journalctl --no-pager" > "journal-HV2.log" || true
37+
scp "$HV2_IP:/var/log/libvirt/qemu/VM-QEMU.log" "libvirt-VM-QEMU-HV2.log" || true
38+
scp "$HV2_IP:/var/log/libvirt/ch/VM-CHV.log" "libvirt-VM-CHV-HV2.log" || true
39+
scp "$HV2_IP:/var/log/QEMU.log" "serial-VM-QEMU-HV2.log" || true
40+
scp "$HV2_IP:/var/log/CHV.log" "serial-VM-CHV-HV2.log" || true
3641
3742
- name: Obtain network diagnostics from hypervisors
3843
shell: bash
3944
run: |
40-
KEY=/opt/ssh_host_ed25519_key
41-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
42-
USER="root"
43-
4445
for HV_NAME in HV1 HV2; do
4546
IP="192.168.122.$([[ $HV_NAME == HV1 ]] && echo 2 || echo 3)"
4647
47-
ssh $SSH_OPTS $USER@$IP "
48+
ssh "$IP" "
4849
echo '=== ip addr ==='
4950
ip addr
5051
echo '=== ip route ==='
@@ -75,19 +76,15 @@ runs:
7576
- name: Set permissions
7677
shell: bash
7778
run: |
78-
sudo chmod 644 *.log
79+
sudo chmod 644 *.log || true
7980
8081
- name: Collect coredumps from hypervisors
8182
shell: bash
8283
run: |
83-
KEY=/opt/ssh_host_ed25519_key
84-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
85-
USER="root"
86-
8784
for HV_NAME in HV1 HV2; do
8885
IP="192.168.122.$([[ $HV_NAME == HV1 ]] && echo 2 || echo 3)"
8986
90-
ssh $SSH_OPTS $USER@$IP "
87+
ssh "$IP" "
9188
# Only proceed if coredumps exist
9289
if coredumpctl list --no-legend 2>/dev/null | grep -q .; then
9390
mkdir -p /tmp/coredumps-$HV_NAME
@@ -101,8 +98,8 @@ runs:
10198
fi
10299
" || true
103100
104-
if ssh $SSH_OPTS $USER@$IP "[ -d /tmp/coredumps-$HV_NAME ] && [ -n \"\$(ls /tmp/coredumps-$HV_NAME 2>/dev/null)\" ]"; then
105-
scp -r $SSH_OPTS $USER@$IP:/tmp/coredumps-$HV_NAME ./ || true
101+
if ssh "$IP" "[ -d /tmp/coredumps-$HV_NAME ] && [ -n \"\$(ls /tmp/coredumps-$HV_NAME 2>/dev/null)\" ]"; then
102+
scp -r "$IP:/tmp/coredumps-$HV_NAME" ./ || true
106103
fi
107104
done
108105

‎.github/actions/test/integration/setup/action.yml‎

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,26 @@ description: Set up Hypervisor and VM Environment for Integration Tests
33
runs:
44
using: "composite"
55
steps:
6+
- name: Write the shared SSH configuration
7+
shell: bash
8+
run: |
9+
# The boot step below copies the key to this path; the later steps
10+
# rely on the matching Host block instead of repeating the options.
11+
mkdir -p ~/.ssh
12+
chmod 700 ~/.ssh
13+
cat > ~/.ssh/config <<'EOF'
14+
Host 192.168.122.* 192.168.222.*
15+
IdentityFile /opt/ssh_host_ed25519_key
16+
IdentitiesOnly yes
17+
StrictHostKeyChecking no
18+
UserKnownHostsFile /dev/null
19+
ConnectTimeout 10
20+
ServerAliveInterval 15
21+
ServerAliveCountMax 4
22+
User root
23+
EOF
24+
chmod 600 ~/.ssh/config
25+
626
- name: Configure the default libvirt network
727
shell: bash
828
run: |
@@ -49,32 +69,23 @@ runs:
4969
- name: Configure SSH and libvirt on HyperVisors
5070
shell: bash
5171
run: |
52-
KEY=/opt/ssh_host_ed25519_key
53-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
54-
USER="root"
5572
MAX_ITER=30
5673
5774
setup_hv() {
5875
local IP="$1"
5976
6077
# Configure SSH between HyperVisors
61-
scp $SSH_OPTS /opt/ssh_host_ed25519_key "$USER@$IP:/opt/ssh_host_ed25519_key"
62-
ssh $SSH_OPTS "$USER@$IP" << EOF
78+
scp /opt/ssh_host_ed25519_key "$IP:/opt/ssh_host_ed25519_key"
79+
scp "$HOME/.ssh/config" "$IP:/root/.ssh/config"
80+
ssh "$IP" << EOF
6381
chmod 700 /root/.ssh
64-
echo "" > /root/.ssh/config
65-
echo "Host 192.168.122.*" >> /root/.ssh/config
66-
echo " StrictHostKeyChecking no" >> /root/.ssh/config
67-
echo " UserKnownHostsFile /dev/null" >> /root/.ssh/config
68-
echo " IdentityFile /opt/ssh_host_ed25519_key" >> /root/.ssh/config
69-
echo " IdentitiesOnly yes" >> /root/.ssh/config
70-
echo " User root" >> /root/.ssh/config
7182
chmod 600 /root/.ssh/config
7283
chmod 600 /opt/ssh_host_ed25519_key
7384
EOF
7485
7586
# Set up libvirt storage pool and network
76-
scp $SSH_OPTS ./.github/actions/test/integration/setup/virbr-vm.xml "$USER@$IP:/opt/virbr-vm.xml"
77-
ssh $SSH_OPTS "$USER@$IP" << EOF
87+
scp ./.github/actions/test/integration/setup/virbr-vm.xml "$IP:/opt/virbr-vm.xml"
88+
ssh "$IP" << EOF
7889
systemctl start libvirtd
7990
8091
virsh pool-destroy default || true
@@ -99,7 +110,7 @@ runs:
99110
local IP=$(sudo virsh domifaddr "$VM_NAME" | awk '/ipv4/ {print $4}' | cut -d'/' -f1)
100111
if [ -z "$IP" ]; then
101112
echo "No IP address found for $VM_NAME. Retrying..."
102-
elif ssh $SSH_OPTS "$USER@$IP" 'exit' 2>/dev/null; then
113+
elif ssh "$IP" 'exit' 2>/dev/null; then
103114
echo "$VM_NAME is up at IP: $IP"
104115
setup_hv "$IP"
105116
break

‎.github/actions/test/integration/test/cloudhypervisor/action.yml‎

Lines changed: 22 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -6,63 +6,53 @@ runs:
66
- name: Copy VM defintions and artifacts to the HyperVisors
77
shell: bash
88
run: |
9-
KEY=/opt/ssh_host_ed25519_key
10-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
11-
USER="root"
9+
# Required from the job environment; the shared key is the identity
10+
# that the ssh config names.
11+
[ -s /opt/ssh_host_ed25519_key ] || { echo "the shared SSH key /opt/ssh_host_ed25519_key is missing" >&2; exit 1; }
1212
IP_HV1="192.168.122.2"
1313
IP_HV2="192.168.122.3"
1414
15-
scp $SSH_OPTS ./.github/actions/test/integration/test/cloudhypervisor/vm.xml "$USER@$IP_HV1:/opt/vm-chv.xml"
16-
scp $SSH_OPTS /opt/ubuntu.raw "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu.raw"
17-
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
15+
scp ./.github/actions/test/integration/test/cloudhypervisor/vm.xml "$IP_HV1:/opt/vm-chv.xml"
16+
scp /opt/ubuntu.raw "$IP_HV1:/var/lib/libvirt/images/ubuntu.raw"
17+
scp /opt/ubuntu-cloud-init-ds.iso "$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
1818
1919
# libvirt migration between cloudhypervisor instances does not support storage migration yet
20-
scp $SSH_OPTS /opt/ubuntu.raw "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu.raw"
20+
scp /opt/ubuntu.raw "$IP_HV2:/var/lib/libvirt/images/ubuntu.raw"
2121
# the cloud-init data is not migrated from source to destination, but needed for the VM to run at the destination
22-
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
22+
scp /opt/ubuntu-cloud-init-ds.iso "$IP_HV2:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
2323
2424
- name: Start VM on HyperVisor 1
2525
shell: bash
2626
run: |
27-
KEY=/opt/ssh_host_ed25519_key
28-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
29-
USER="root"
3027
IP_HV1="192.168.122.2"
3128
32-
ssh $SSH_OPTS "$USER@$IP_HV1" << EOF
29+
ssh "$IP_HV1" << EOF
3330
virsh -c ch:///session define /opt/vm-chv.xml
3431
virsh -c ch:///session start VM-CHV
3532
EOF
3633
3734
- name: Wait for VM to be ready
3835
shell: bash
3936
run: |
40-
KEY=/opt/ssh_host_ed25519_key
41-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
42-
USER="root"
4337
IP_HV1="192.168.122.2"
4438
45-
ssh $SSH_OPTS "$USER@$IP_HV1" << EOF
46-
KEY=/opt/ssh_host_ed25519_key
47-
SSH_OPTS="-i \$KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
48-
USER="root"
49-
39+
ssh "$IP_HV1" << EOF
5040
MAX_ITER="${BOOT_MAX_ITER:-60}"
5141
for ((i=1; i<=MAX_ITER; i++)); do
5242
IP=\$(virsh -c ch:///session domifaddr VM-CHV | awk '/ipv4/ {print \$4}' | cut -d'/' -f1)
5343
if [ -z "\$IP" ]; then
5444
echo "No IP address found for VM. Retrying..."
5545
elif (( i >= MAX_ITER - 2 )); then
5646
# Show SSH errors on last 3 attempts for debugging
57-
if ssh \$SSH_OPTS "\$USER@\$IP" 'exit'; then
47+
if ssh "\$IP" 'exit'; then
5848
echo "VM is up at IP: \$IP"
59-
ssh \$SSH_OPTS "\$USER@\$IP" 'echo "Hello, World!" > /opt/hello.txt'
49+
ssh "\$IP" 'echo "Hello, World!" > /opt/hello.txt'
6050
break
6151
fi
62-
elif ssh \$SSH_OPTS "\$USER@\$IP" 'exit' 2>/dev/null; then
52+
elif ssh "\$IP" 'exit' 2>/dev/null; then
6353
echo "VM is up at IP: \$IP"
6454
# create a dummy file for verification later
65-
ssh \$SSH_OPTS "\$USER@\$IP" 'echo "Hello, World!" > /opt/hello.txt'
55+
ssh "\$IP" 'echo "Hello, World!" > /opt/hello.txt'
6656
break
6757
fi
6858
if (( i == MAX_ITER )); then
@@ -77,7 +67,7 @@ runs:
7767
echo "--- arp -n ---"
7868
arp -n 2>/dev/null || ip neigh || true
7969
echo "--- ssh -v attempt ---"
80-
ssh -v \$SSH_OPTS "\$USER@\$IP" 'exit' 2>&1 || true
70+
ssh -v "\$IP" 'exit' 2>&1 || true
8171
echo "=== END DIAGNOSTICS ==="
8272
exit 1
8373
fi
@@ -86,19 +76,15 @@ runs:
8676
done
8777
EOF
8878
89-
- name: Mirgrate the VM to HyperVisor 2
79+
- name: Migrate the VM to HyperVisor 2
9080
shell: bash
9181
run: |
92-
KEY=/opt/ssh_host_ed25519_key
93-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
94-
USER="root"
9582
IP_HV1="192.168.122.2"
9683
97-
ssh $SSH_OPTS "$USER@$IP_HV1" << EOF
98-
USER="root"
84+
ssh "$IP_HV1" << EOF
9985
IP_HV2="192.168.122.3"
10086
101-
virsh -c ch:///session migrate --p2p --persistent --undefinesource --live VM-CHV ch+ssh://\$USER@\$IP_HV2/system
87+
virsh -c ch:///session migrate --p2p --persistent --undefinesource --live VM-CHV ch+ssh://root@\$IP_HV2/system
10288
if virsh -c ch:///session dominfo VM-CHV &>/dev/null; then
10389
echo "VM is still present on HV1 after migration!"
10490
exit 1
@@ -108,37 +94,30 @@ runs:
10894
- name: Verify VM status on HyperVisor 2
10995
shell: bash
11096
run: |
111-
KEY=/opt/ssh_host_ed25519_key
112-
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
113-
USER="root"
11497
IP_HV2="192.168.122.3"
11598
116-
ssh $SSH_OPTS "$USER@$IP_HV2" << EOF
99+
ssh "$IP_HV2" << EOF
117100
if virsh -c ch:///session dominfo VM-CHV | awk '/State:/ {print \$2}' | grep -q "running"; then
118101
echo "VM is running on HV2."
119102
else
120103
echo "VM is not running on HV2."
121104
exit 1
122105
fi
123106
124-
KEY=/opt/ssh_host_ed25519_key
125-
SSH_OPTS="-i \$KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
126-
USER="root"
127107
# waiting for the IP to be visible to the new HyperVisor via DHCP takes WAYYYYYYY too long
128108
# the IP is static, so we can just use it directly
129109
IP="192.168.222.3"
130110
131111
MAX_ITER="${VERIFY_MAX_ITER:-15}"
132112
for ((i=1; i<=MAX_ITER; i++)); do
133-
if ssh \$SSH_OPTS "\$USER@\$IP" 'exit' 2>/dev/null; then
113+
if ssh "\$IP" 'exit' 2>/dev/null; then
134114
echo "VM is up at IP: \$IP"
135115
# verify the migration by checking the dummy file
136-
if ssh \$SSH_OPTS "\$USER@\$IP" 'test -f /opt/hello.txt'; then
116+
if ssh "\$IP" 'test -f /opt/hello.txt'; then
137117
echo "Dummy file exists on VM, migration successful."
138118
break
139119
else
140120
echo "Dummy file does not exist on VM, migration failed."
141-
exit 1
142121
fi
143122
fi
144123
if (( i == MAX_ITER )); then
@@ -149,7 +128,7 @@ runs:
149128
echo "--- ip route ---"
150129
ip route || true
151130
echo "--- ssh -v attempt ---"
152-
ssh -v \$SSH_OPTS "\$USER@\$IP" 'exit' 2>&1 || true
131+
ssh -v "\$IP" 'exit' 2>&1 || true
153132
echo "=== END DIAGNOSTICS ==="
154133
exit 1
155134
fi

0 commit comments

Comments
 (0)