bug: a panel moved to Tailscale kept its LAN InternalIP, so exec, logs and metrics for it failed on the kubelet certificate (#652)

game-node-server-setup.sh restarts the panel's k3s onto its Tailscale IP, which
moves the kubelet and its certificate, but the cloud-node controller only takes
an address matching alpha.kubernetes.io/provided-node-ip, which the kubelet
wrote once at registration. update.sh now moves that annotation to the
configured node-ip once k3s reports it, so the InternalIP follows.

Author: lukepolo

.github/workflows/panel-scripts.yml

@@ -7,6 +7,7 @@ on:
       - debug.sh
       - dev.sh
       - plugin.sh
+      - update.sh
       - "utils/**"
       - "overlays/**/*.env.example"
       - base/api/deployment.yaml
@@ -18,6 +19,7 @@ on:
       - debug.sh
       - dev.sh
       - plugin.sh
+      - update.sh
       - "utils/**"
       - "overlays/**/*.env.example"
       - base/api/deployment.yaml
@@ -30,8 +32,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: shellcheck -x utils/ensure_auth_cookie_domain.sh tests/scripts/*.sh tests/scripts/stub/kubectl
+      - run: shellcheck -x utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh tests/scripts/*.sh tests/scripts/stub/kubectl
       - run: bash tests/scripts/setup_env_test.sh
       - run: bash tests/scripts/debug_test.sh
       - run: bash tests/scripts/startup_probe_test.sh
       - run: bash tests/scripts/plugin_test.sh
+      - run: bash tests/scripts/node_ip_test.sh

tests/scripts/node_ip_test.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# shellcheck source=tests/scripts/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+STUB_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/stub" && pwd)"
+
+nodes() {
+    printf '%s\n' "$@" >"$WORK_DIR/nodes"
+}
+
+run_ensure_node_ip() {
+    if [ -n "$1" ]; then
+        printf 'node-ip: %s\n' "$1" >"$WORK_DIR/config.yaml"
+    fi
+    (
+        export PATH="$STUB_BIN:$PATH" K3S_CONFIG="$WORK_DIR/config.yaml" KUBECONFIG=/dev/null NODE_IP_ATTEMPTS=1
+        # shellcheck source=utils/colors.sh
+        source "$TEST_ROOT/utils/colors.sh"
+        # shellcheck source=utils/ensure_node_ip.sh
+        source "$TEST_ROOT/utils/ensure_node_ip.sh"
+        ensure_node_ip
+    ) >"$WORK_DIR/output" 2>&1 || fail "ensure_node_ip exited $?: $(cat "$WORK_DIR/output")"
+}
+
+annotations() {
+    grep ' annotate ' "$WORK_DIR/calls" 2>/dev/null || true
+}
+
+# The panel registered on its LAN IP, then game-node-server-setup.sh moved k3s
+# to Tailscale: the kubelet followed, the Node's InternalIP never did.
+test_a_panel_moved_to_tailscale_has_its_provided_ip_moved_too() {
+    nodes "$(printf 'dietpi\t100.117.134.62,fd7a:115c:a1e0::c02b:863f\t192.168.2.199')" \
+        "$(printf 'game-node\t100.64.0.9\t100.64.0.9')"
+    run_ensure_node_ip 100.117.134.62
+    assert_eq "$(annotations)" "--kubeconfig=/dev/null annotate node dietpi alpha.kubernetes.io/provided-node-ip=100.117.134.62 --overwrite"
+}
+
+test_a_node_already_on_its_ip_is_left_alone() {
+    nodes "$(printf 'dietpi\t100.117.134.62\t100.117.134.62')"
+    run_ensure_node_ip 100.117.134.62
+    assert_eq "$(annotations)" ""
+}
+
+test_a_panel_without_node_ip_never_asks_the_cluster() {
+    nodes "$(printf 'dietpi\t192.168.2.199\t192.168.2.199')"
+    run_ensure_node_ip ""
+    assert_eq "$(cat "$WORK_DIR/calls" 2>/dev/null)" ""
+}
+
+# Annotating before k3s has restarted onto the new IP would name an address
+# the node does not have yet.
+test_a_node_k3s_has_not_moved_yet_is_left_alone() {
+    nodes "$(printf 'dietpi\t192.168.2.199\t192.168.2.199')"
+    run_ensure_node_ip 100.117.134.62
+    assert_eq "$(annotations)" ""
+    assert_contains "$(cat "$WORK_DIR/output")" "re-run ./update.sh"
+}
+
+run_tests

tests/scripts/stub/kubectl

@@ -19,6 +19,7 @@ while [ $# -gt 0 ]; do
 done
 
 case "$args" in
+    *" get nodes "*) if [ -f "$WORK_DIR/nodes" ]; then cat "$WORK_DIR/nodes"; fi ;;
     *" get pods "*) cat "$WORK_DIR/pods" ;;
     *" get pod "*) cat "$WORK_DIR/containers-$pod" ;;
     *" logs "*)

update.sh

@@ -9,6 +9,10 @@ if ! plan_timescaledb_major_upgrade; then
     exit 1
 fi
 
+# Also how a panel that game-node-server-setup.sh already moved to Tailscale gets
+# repaired: see utils/ensure_node_ip.sh.
+ensure_node_ip
+
 # Nothing terminates TLS in the cluster in this mode, so any Certificate left
 # over from a previous https install keeps retrying HTTP-01 against a server
 # that no longer answers the challenge -- silently accumulating failed orders

utils/ensure_node_ip.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# k3s's node-ip moves the kubelet (its serving certificate and the
+# k3s.io/internal-ip annotation) but not the Node's InternalIP. The cloud-node
+# controller only takes an address that matches alpha.kubernetes.io/provided-node-ip,
+# and the kubelet writes that once, when the node first registers. A panel
+# installed on its LAN IP and then moved to Tailscale by
+# game-node-server-setup.sh kept its LAN InternalIP, which its kubelet's
+# certificate no longer covers, so every exec, log stream and metrics scrape of
+# that node failed with x509 errors.
+ensure_node_ip() {
+    local config="${K3S_CONFIG:-/etc/rancher/k3s/config.yaml}"
+    local attempts="${NODE_IP_ATTEMPTS:-30}"
+    local want attempt name internal provided
+
+    want=$(sed -n 's/^node-ip:[[:space:]]*//p' "$config" 2>/dev/null | tail -n 1 | tr -d '\r')
+    if [ -z "$want" ]; then
+        return 0
+    fi
+
+    # Right after a restart k3s has not rewritten k3s.io/internal-ip yet, and
+    # annotating before it has would name an address the node does not have.
+    for ((attempt = 1; attempt <= attempts; attempt++)); do
+        while IFS=$'\t' read -r name internal provided; do
+            if [ "${internal%%,*}" != "${want%%,*}" ]; then
+                continue
+            fi
+            if [ "$provided" = "$want" ]; then
+                return 0
+            fi
+            if ! kubectl --kubeconfig="$KUBECONFIG" annotate node "$name" "alpha.kubernetes.io/provided-node-ip=$want" --overwrite >/dev/null; then
+                warn "Could not move node $name to $want; kubectl exec, logs and metrics for it stay broken."
+                return 0
+            fi
+            echo "Node $name moved to $want (was ${provided:-unset}); its InternalIP follows within 5 minutes."
+            return 0
+        done < <(kubectl --kubeconfig="$KUBECONFIG" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.annotations.k3s\.io/internal-ip}{"\t"}{.metadata.annotations.alpha\.kubernetes\.io/provided-node-ip}{"\n"}{end}' 2>/dev/null)
+        if [ "$attempt" -lt "$attempts" ]; then
+            sleep 2
+        fi
+    done
+    warn "No node reports the node-ip $want from $config yet; re-run ./update.sh once k3s is up."
+}

utils/utils.sh

@@ -18,6 +18,7 @@ source "$SCRIPT_DIR/replace_rand32_in_env_files.sh"
 source "$SCRIPT_DIR/ensure_turn_secret.sh"
 source "$SCRIPT_DIR/ensure_s3_config_keys.sh"
 source "$SCRIPT_DIR/ensure_auth_cookie_domain.sh"
+source "$SCRIPT_DIR/ensure_node_ip.sh"
 source "$SCRIPT_DIR/resync_vault_secrets.sh"
 source "$SCRIPT_DIR/update_timescaledb_extension.sh"
 source "$SCRIPT_DIR/upgrade_timescaledb_major.sh"