Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 23 additions & 26 deletions .github/actions/test/integration/collect-logs/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,45 +6,46 @@ runs:
- name: Obtain logs from test runner
shell: bash
run: |
# Required from the job environment; the shared key is the identity
# that the ssh config names.
[ -s /opt/ssh_host_ed25519_key ] || { echo "the shared SSH key /opt/ssh_host_ed25519_key is missing" >&2; exit 1; }
sudo lscpu > lscpu-RUNNER.log || true
sudo lshw > lshw-runner.log || true
sudo journalctl --no-pager > journal-RUNNER.log || true

sudo cp /var/log/libvirt/qemu/HV1.log "libvirt-HV1.log" || true
sudo cp /var/log/libvirt/qemu/HV2.log "libvirt-HV2.log" || true

# The HV serial consoles: the only record of a frozen guest's
# final kernel output (the journal goes down with the guest).
sudo cp /var/log/HV1.log "serial-HV1.log" || true
sudo cp /var/log/HV2.log "serial-HV2.log" || true

- name: Obtain logs from hypervisors
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
HV1_IP="192.168.122.2"
HV2_IP="192.168.122.3"

ssh $SSH_OPTS $USER@$HV1_IP "journalctl --no-pager" > "journal-HV1.log" || true
scp $SSH_OPTS $USER@$HV1_IP:/var/log/libvirt/qemu/VM-QEMU.log "libvirt-VM-QEMU-HV1.log" || true
scp $SSH_OPTS $USER@$HV1_IP:/var/log/libvirt/ch/VM-CHV.log "libvirt-VM-CHV-HV1.log" || true
scp $SSH_OPTS $USER@$HV1_IP:/var/log/QEMU.log "serial-VM-QEMU-HV1.log" || true
scp $SSH_OPTS $USER@$HV1_IP:/var/log/CHV.log "serial-VM-CHV-HV1.log" || true
ssh "$HV1_IP" "journalctl --no-pager" > "journal-HV1.log" || true
scp "$HV1_IP:/var/log/libvirt/qemu/VM-QEMU.log" "libvirt-VM-QEMU-HV1.log" || true
scp "$HV1_IP:/var/log/libvirt/ch/VM-CHV.log" "libvirt-VM-CHV-HV1.log" || true
scp "$HV1_IP:/var/log/QEMU.log" "serial-VM-QEMU-HV1.log" || true
scp "$HV1_IP:/var/log/CHV.log" "serial-VM-CHV-HV1.log" || true

ssh $SSH_OPTS $USER@$HV2_IP "journalctl --no-pager" > "journal-HV2.log" || true
scp $SSH_OPTS $USER@$HV2_IP:/var/log/libvirt/qemu/VM-QEMU.log "libvirt-VM-QEMU-HV2.log" || true
scp $SSH_OPTS $USER@$HV2_IP:/var/log/libvirt/ch/VM-CHV.log "libvirt-VM-CHV-HV2.log" || true
scp $SSH_OPTS $USER@$HV2_IP:/var/log/QEMU.log "serial-VM-QEMU-HV2.log" || true
scp $SSH_OPTS $USER@$HV2_IP:/var/log/CHV.log "serial-VM-CHV-HV2.log" || true
ssh "$HV2_IP" "journalctl --no-pager" > "journal-HV2.log" || true
scp "$HV2_IP:/var/log/libvirt/qemu/VM-QEMU.log" "libvirt-VM-QEMU-HV2.log" || true
scp "$HV2_IP:/var/log/libvirt/ch/VM-CHV.log" "libvirt-VM-CHV-HV2.log" || true
scp "$HV2_IP:/var/log/QEMU.log" "serial-VM-QEMU-HV2.log" || true
scp "$HV2_IP:/var/log/CHV.log" "serial-VM-CHV-HV2.log" || true

- name: Obtain network diagnostics from hypervisors
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"

for HV_NAME in HV1 HV2; do
IP="192.168.122.$([[ $HV_NAME == HV1 ]] && echo 2 || echo 3)"

ssh $SSH_OPTS $USER@$IP "
ssh "$IP" "
echo '=== ip addr ==='
ip addr
echo '=== ip route ==='
Expand Down Expand Up @@ -75,19 +76,15 @@ runs:
- name: Set permissions
shell: bash
run: |
sudo chmod 644 *.log
sudo chmod 644 *.log || true

- name: Collect coredumps from hypervisors
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"

for HV_NAME in HV1 HV2; do
IP="192.168.122.$([[ $HV_NAME == HV1 ]] && echo 2 || echo 3)"

ssh $SSH_OPTS $USER@$IP "
ssh "$IP" "
# Only proceed if coredumps exist
if coredumpctl list --no-legend 2>/dev/null | grep -q .; then
mkdir -p /tmp/coredumps-$HV_NAME
Expand All @@ -101,8 +98,8 @@ runs:
fi
" || true

if ssh $SSH_OPTS $USER@$IP "[ -d /tmp/coredumps-$HV_NAME ] && [ -n \"\$(ls /tmp/coredumps-$HV_NAME 2>/dev/null)\" ]"; then
scp -r $SSH_OPTS $USER@$IP:/tmp/coredumps-$HV_NAME ./ || true
if ssh "$IP" "[ -d /tmp/coredumps-$HV_NAME ] && [ -n \"\$(ls /tmp/coredumps-$HV_NAME 2>/dev/null)\" ]"; then
scp -r "$IP:/tmp/coredumps-$HV_NAME" ./ || true
fi
done

Expand Down
41 changes: 26 additions & 15 deletions .github/actions/test/integration/setup/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,26 @@ description: Set up Hypervisor and VM Environment for Integration Tests
runs:
using: "composite"
steps:
- name: Write the shared SSH configuration
shell: bash
run: |
# The boot step below copies the key to this path; the later steps
# rely on the matching Host block instead of repeating the options.
mkdir -p ~/.ssh
chmod 700 ~/.ssh
cat > ~/.ssh/config <<'EOF'
Host 192.168.122.* 192.168.222.*
IdentityFile /opt/ssh_host_ed25519_key
IdentitiesOnly yes
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
ConnectTimeout 10
ServerAliveInterval 15
ServerAliveCountMax 4
User root
EOF
chmod 600 ~/.ssh/config

- name: Configure the default libvirt network
shell: bash
run: |
Expand Down Expand Up @@ -49,32 +69,23 @@ runs:
- name: Configure SSH and libvirt on HyperVisors
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
MAX_ITER=30

setup_hv() {
local IP="$1"

# Configure SSH between HyperVisors
scp $SSH_OPTS /opt/ssh_host_ed25519_key "$USER@$IP:/opt/ssh_host_ed25519_key"
ssh $SSH_OPTS "$USER@$IP" << EOF
scp /opt/ssh_host_ed25519_key "$IP:/opt/ssh_host_ed25519_key"
scp "$HOME/.ssh/config" "$IP:/root/.ssh/config"
ssh "$IP" << EOF
chmod 700 /root/.ssh
echo "" > /root/.ssh/config
echo "Host 192.168.122.*" >> /root/.ssh/config
echo " StrictHostKeyChecking no" >> /root/.ssh/config
echo " UserKnownHostsFile /dev/null" >> /root/.ssh/config
echo " IdentityFile /opt/ssh_host_ed25519_key" >> /root/.ssh/config
echo " IdentitiesOnly yes" >> /root/.ssh/config
echo " User root" >> /root/.ssh/config
chmod 600 /root/.ssh/config
chmod 600 /opt/ssh_host_ed25519_key
EOF

# Set up libvirt storage pool and network
scp $SSH_OPTS ./.github/actions/test/integration/setup/virbr-vm.xml "$USER@$IP:/opt/virbr-vm.xml"
ssh $SSH_OPTS "$USER@$IP" << EOF
scp ./.github/actions/test/integration/setup/virbr-vm.xml "$IP:/opt/virbr-vm.xml"
ssh "$IP" << EOF
systemctl start libvirtd

virsh pool-destroy default || true
Expand All @@ -99,7 +110,7 @@ runs:
local IP=$(sudo virsh domifaddr "$VM_NAME" | awk '/ipv4/ {print $4}' | cut -d'/' -f1)
if [ -z "$IP" ]; then
echo "No IP address found for $VM_NAME. Retrying..."
elif ssh $SSH_OPTS "$USER@$IP" 'exit' 2>/dev/null; then
elif ssh "$IP" 'exit' 2>/dev/null; then
echo "$VM_NAME is up at IP: $IP"
setup_hv "$IP"
break
Expand Down
65 changes: 22 additions & 43 deletions .github/actions/test/integration/test/cloudhypervisor/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,63 +6,53 @@ runs:
- name: Copy VM defintions and artifacts to the HyperVisors
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
# Required from the job environment; the shared key is the identity
# that the ssh config names.
[ -s /opt/ssh_host_ed25519_key ] || { echo "the shared SSH key /opt/ssh_host_ed25519_key is missing" >&2; exit 1; }
IP_HV1="192.168.122.2"
IP_HV2="192.168.122.3"

scp $SSH_OPTS ./.github/actions/test/integration/test/cloudhypervisor/vm.xml "$USER@$IP_HV1:/opt/vm-chv.xml"
scp $SSH_OPTS /opt/ubuntu.raw "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu.raw"
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
scp ./.github/actions/test/integration/test/cloudhypervisor/vm.xml "$IP_HV1:/opt/vm-chv.xml"
scp /opt/ubuntu.raw "$IP_HV1:/var/lib/libvirt/images/ubuntu.raw"
scp /opt/ubuntu-cloud-init-ds.iso "$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"

# libvirt migration between cloudhypervisor instances does not support storage migration yet
scp $SSH_OPTS /opt/ubuntu.raw "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu.raw"
scp /opt/ubuntu.raw "$IP_HV2:/var/lib/libvirt/images/ubuntu.raw"
# the cloud-init data is not migrated from source to destination, but needed for the VM to run at the destination
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
scp /opt/ubuntu-cloud-init-ds.iso "$IP_HV2:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"

- name: Start VM on HyperVisor 1
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
IP_HV1="192.168.122.2"

ssh $SSH_OPTS "$USER@$IP_HV1" << EOF
ssh "$IP_HV1" << EOF
virsh -c ch:///session define /opt/vm-chv.xml
virsh -c ch:///session start VM-CHV
EOF

- name: Wait for VM to be ready
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
IP_HV1="192.168.122.2"

ssh $SSH_OPTS "$USER@$IP_HV1" << EOF
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i \$KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"

ssh "$IP_HV1" << EOF
MAX_ITER="${BOOT_MAX_ITER:-60}"
for ((i=1; i<=MAX_ITER; i++)); do
IP=\$(virsh -c ch:///session domifaddr VM-CHV | awk '/ipv4/ {print \$4}' | cut -d'/' -f1)
if [ -z "\$IP" ]; then
echo "No IP address found for VM. Retrying..."
elif (( i >= MAX_ITER - 2 )); then
# Show SSH errors on last 3 attempts for debugging
if ssh \$SSH_OPTS "\$USER@\$IP" 'exit'; then
if ssh "\$IP" 'exit'; then
echo "VM is up at IP: \$IP"
ssh \$SSH_OPTS "\$USER@\$IP" 'echo "Hello, World!" > /opt/hello.txt'
ssh "\$IP" 'echo "Hello, World!" > /opt/hello.txt'
break
fi
elif ssh \$SSH_OPTS "\$USER@\$IP" 'exit' 2>/dev/null; then
elif ssh "\$IP" 'exit' 2>/dev/null; then
echo "VM is up at IP: \$IP"
# create a dummy file for verification later
ssh \$SSH_OPTS "\$USER@\$IP" 'echo "Hello, World!" > /opt/hello.txt'
ssh "\$IP" 'echo "Hello, World!" > /opt/hello.txt'
break
fi
if (( i == MAX_ITER )); then
Expand All @@ -77,7 +67,7 @@ runs:
echo "--- arp -n ---"
arp -n 2>/dev/null || ip neigh || true
echo "--- ssh -v attempt ---"
ssh -v \$SSH_OPTS "\$USER@\$IP" 'exit' 2>&1 || true
ssh -v "\$IP" 'exit' 2>&1 || true
echo "=== END DIAGNOSTICS ==="
exit 1
fi
Expand All @@ -86,19 +76,15 @@ runs:
done
EOF

- name: Mirgrate the VM to HyperVisor 2
- name: Migrate the VM to HyperVisor 2
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
IP_HV1="192.168.122.2"

ssh $SSH_OPTS "$USER@$IP_HV1" << EOF
USER="root"
ssh "$IP_HV1" << EOF
IP_HV2="192.168.122.3"

virsh -c ch:///session migrate --p2p --persistent --undefinesource --live VM-CHV ch+ssh://\$USER@\$IP_HV2/system
virsh -c ch:///session migrate --p2p --persistent --undefinesource --live VM-CHV ch+ssh://root@\$IP_HV2/system
if virsh -c ch:///session dominfo VM-CHV &>/dev/null; then
echo "VM is still present on HV1 after migration!"
exit 1
Expand All @@ -108,37 +94,30 @@ runs:
- name: Verify VM status on HyperVisor 2
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
IP_HV2="192.168.122.3"

ssh $SSH_OPTS "$USER@$IP_HV2" << EOF
ssh "$IP_HV2" << EOF
if virsh -c ch:///session dominfo VM-CHV | awk '/State:/ {print \$2}' | grep -q "running"; then
echo "VM is running on HV2."
else
echo "VM is not running on HV2."
exit 1
fi

KEY=/opt/ssh_host_ed25519_key
SSH_OPTS="-i \$KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
USER="root"
# waiting for the IP to be visible to the new HyperVisor via DHCP takes WAYYYYYYY too long
# the IP is static, so we can just use it directly
IP="192.168.222.3"

MAX_ITER="${VERIFY_MAX_ITER:-15}"
for ((i=1; i<=MAX_ITER; i++)); do
if ssh \$SSH_OPTS "\$USER@\$IP" 'exit' 2>/dev/null; then
if ssh "\$IP" 'exit' 2>/dev/null; then
echo "VM is up at IP: \$IP"
# verify the migration by checking the dummy file
if ssh \$SSH_OPTS "\$USER@\$IP" 'test -f /opt/hello.txt'; then
if ssh "\$IP" 'test -f /opt/hello.txt'; then
echo "Dummy file exists on VM, migration successful."
break
else
echo "Dummy file does not exist on VM, migration failed."
exit 1
fi
fi
if (( i == MAX_ITER )); then
Expand All @@ -149,7 +128,7 @@ runs:
echo "--- ip route ---"
ip route || true
echo "--- ssh -v attempt ---"
ssh -v \$SSH_OPTS "\$USER@\$IP" 'exit' 2>&1 || true
ssh -v "\$IP" 'exit' 2>&1 || true
echo "=== END DIAGNOSTICS ==="
exit 1
fi
Expand Down
Loading