✨ Headless load tests

packages/step-cli/scripts/setup-telephone-load-test.sh

@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2026 Sequent Tech Inc <legal@sequentech.io>
+#
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Stage 1 of the telephone (IVR) load test: provisions an election event for
+# a tenant, bulk-creates DTMF-safe voters, runs the keys ceremony, publishes,
+# and opens the TELEPHONE voting channel. Writes a summary.json + voters CSV
+# that Stage 2 (driving `ivr-cli` calls) consumes. See IVR_LOAD_TEST_DESIGN.md
+# at the repo root for the full design.
+#
+# Requires the `step-cli` binary on PATH (cd packages/step-cli && cargo build
+# --release, or run inside `devenv shell` which already does this for you).
+
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage: setup-telephone-load-test.sh --election-event-json <path> [options]
+
+Required:
+  --election-event-json <path>   Election event JSON to import (see
+                                  packages/headless-load-test/data/local/
+                                  export_election_event-*.json for an example)
+
+Options (default to this repo's devcontainer dev tenant/Keycloak):
+  --tenant-id <id>                Default: $SUPER_ADMIN_TENANT_ID
+  --num-voters <n>                Default: 20
+  --voter-pin-digits <n>          Numeric PIN length, max 8 (DTMF limit). Default: 6
+  --threshold <n>                 Key ceremony trustee threshold. Default: 2
+  --endpoint-url <url>            Hasura GraphQL endpoint. Default: $HASURA_ENDPOINT
+  --keycloak-url <url>            Default: $KEYCLOAK_URL
+  --keycloak-admin-user <user>    Default: $KEYCLOAK_ADMIN
+  --keycloak-admin-password <pw>  Default: $KEYCLOAK_ADMIN
+  --keycloak-client-id <id>       Default: api-key-client (needs "gold" acr
+                                  for publish/voting-status; the devcontainer's
+                                  $KEYCLOAK_CLI_CLIENT_ID is a lower tier and
+                                  will 403 on those two steps)
+  --keycloak-client-secret <s>    Default: this repo's devcontainer secret for
+                                  api-key-client
+  --trustee1-user <user>          Default: trustee1
+  --trustee1-password <pw>        Default: trustee1
+  --trustee2-user <user>          Default: trustee2
+  --trustee2-password <pw>        Default: trustee2
+  --out-dir <dir>                 Default: a fresh temp dir
+  -h, --help                      Show this help
+USAGE
+}
+
+TENANT_ID="${SUPER_ADMIN_TENANT_ID:-}"
+ELECTION_EVENT_JSON=""
+NUM_VOTERS=20
+VOTER_PIN_DIGITS=6
+THRESHOLD=2
+ENDPOINT_URL="${HASURA_ENDPOINT:-}"
+KEYCLOAK_URL="${KEYCLOAK_URL:-}"
+KEYCLOAK_ADMIN_USER="${KEYCLOAK_ADMIN:-}"
+KEYCLOAK_ADMIN_PASSWORD="${KEYCLOAK_ADMIN:-}"
+# NOT $KEYCLOAK_CLI_CLIENT_ID: that client (admin-portal in this devcontainer)
+# gets Keycloak's default "silver" acr on direct-grant login, and `publish` /
+# `update-event-voting-status` require "gold" (sequent-core's
+# has_gold_permission checks claims.acr == "gold"). api-key-client is the one
+# client configured with `default.acr.values: gold`, matching what every CLI
+# tutorial in docs/docusaurus hardcodes for this same reason.
+KEYCLOAK_CLIENT_ID="api-key-client"
+KEYCLOAK_CLIENT_SECRET="4lzmxNgZHjfzS5BwDVlyrRUDqwvFLUvL"
+TRUSTEE1_USER="trustee1"
+TRUSTEE1_PASSWORD="trustee1"
+TRUSTEE2_USER="trustee2"
+TRUSTEE2_PASSWORD="trustee2"
+OUT_DIR=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --tenant-id) TENANT_ID="$2"; shift 2 ;;
+    --election-event-json) ELECTION_EVENT_JSON="$2"; shift 2 ;;
+    --num-voters) NUM_VOTERS="$2"; shift 2 ;;
+    --voter-pin-digits) VOTER_PIN_DIGITS="$2"; shift 2 ;;
+    --threshold) THRESHOLD="$2"; shift 2 ;;
+    --endpoint-url) ENDPOINT_URL="$2"; shift 2 ;;
+    --keycloak-url) KEYCLOAK_URL="$2"; shift 2 ;;
+    --keycloak-admin-user) KEYCLOAK_ADMIN_USER="$2"; shift 2 ;;
+    --keycloak-admin-password) KEYCLOAK_ADMIN_PASSWORD="$2"; shift 2 ;;
+    --keycloak-client-id) KEYCLOAK_CLIENT_ID="$2"; shift 2 ;;
+    --keycloak-client-secret) KEYCLOAK_CLIENT_SECRET="$2"; shift 2 ;;
+    --trustee1-user) TRUSTEE1_USER="$2"; shift 2 ;;
+    --trustee1-password) TRUSTEE1_PASSWORD="$2"; shift 2 ;;
+    --trustee2-user) TRUSTEE2_USER="$2"; shift 2 ;;
+    --trustee2-password) TRUSTEE2_PASSWORD="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "Unknown argument: $1" >&2; usage; exit 1 ;;
+  esac
+done
+
+[[ -n "$ELECTION_EVENT_JSON" ]] || { echo "Error: --election-event-json is required" >&2; usage; exit 1; }
+[[ -f "$ELECTION_EVENT_JSON" ]] || { echo "Error: no such file: $ELECTION_EVENT_JSON" >&2; exit 1; }
+[[ -n "$TENANT_ID" ]] || { echo "Error: --tenant-id is required (or set \$SUPER_ADMIN_TENANT_ID)" >&2; exit 1; }
+[[ -n "$ENDPOINT_URL" ]] || { echo "Error: --endpoint-url is required (or set \$HASURA_ENDPOINT)" >&2; exit 1; }
+[[ -n "$KEYCLOAK_URL" ]] || { echo "Error: --keycloak-url is required (or set \$KEYCLOAK_URL)" >&2; exit 1; }
+[[ -n "$KEYCLOAK_ADMIN_USER" ]] || { echo "Error: --keycloak-admin-user is required (or set \$KEYCLOAK_ADMIN)" >&2; exit 1; }
+[[ -n "$KEYCLOAK_CLIENT_ID" ]] || { echo "Error: --keycloak-client-id is required (or set \$KEYCLOAK_CLI_CLIENT_ID)" >&2; exit 1; }
+[[ -n "$KEYCLOAK_CLIENT_SECRET" ]] || { echo "Error: --keycloak-client-secret is required (or set \$KEYCLOAK_CLI_CLIENT_SECRET)" >&2; exit 1; }
+(( VOTER_PIN_DIGITS >= 1 && VOTER_PIN_DIGITS <= 8 )) || { echo "Error: --voter-pin-digits must be between 1 and 8 (DTMF voter auth limit)" >&2; exit 1; }
+command -v step-cli >/dev/null 2>&1 || { echo "Error: step-cli not found on PATH. Build it: (cd packages/step-cli && cargo build --release)" >&2; exit 1; }
+
+export NO_COLOR=1
+
+log() { echo "==> $*" >&2; }
+
+# step-cli always exits 0, even on failure (commands eprintln "Error! ..."
+# and return); detect failure by scanning the captured output instead of $?.
+run_step() {
+  local out
+  out="$(step-cli step "$@" 2>&1 | sed -E 's/\x1b\[[0-9;]*[a-zA-Z]//g')"
+  echo "$out" >&2
+  if grep -q '^Error!' <<<"$out"; then
+    echo "==> step-cli step $* failed" >&2
+    return 1
+  fi
+  printf '%s' "$out"
+}
+
+# The trustee containers complete the key ceremony asynchronously (polling
+# the bulletin board on their own schedule, running an actual DKG protocol
+# round), so `complete-key-ceremony` can legitimately 500 if called before a
+# trustee has caught up to a just-started ceremony. Retry with backoff
+# instead of treating the first failure as fatal.
+retry_step() {
+  local attempts="$1" delay="$2"
+  shift 2
+  local i=1
+  while true; do
+    if run_step "$@" >/dev/null; then
+      return 0
+    fi
+    if (( i >= attempts )); then
+      echo "==> step-cli step $* did not succeed after $attempts attempts" >&2
+      return 1
+    fi
+    log "Retrying in ${delay}s (attempt $((i + 1))/$attempts)..."
+    sleep "$delay"
+    i=$((i + 1))
+  done
+}
+
+# step-cli prints "Success! ... ID: <uuid>" (or, inconsistently, "ID <uuid>"
+# with no colon) on the last line of a successful run; take the last ID-like
+# token on the last such line.
+extract_id() {
+  grep -oE 'ID:? +[A-Za-z0-9._-]+' <<<"$1" | tail -1 | awk '{print $NF}'
+}
+
+configure_as() {
+  local user="$1" password="$2"
+  run_step config \
+    --tenant-id "$TENANT_ID" \
+    --endpoint-url "$ENDPOINT_URL" \
+    --keycloak-url "$KEYCLOAK_URL" \
+    --keycloak-user "$user" \
+    --keycloak-password "$password" \
+    --keycloak-client-id "$KEYCLOAK_CLIENT_ID" \
+    --keycloak-client-secret "$KEYCLOAK_CLIENT_SECRET" >/dev/null
+}
+
+if [[ -z "$OUT_DIR" ]]; then
+  OUT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/telephone-load-test-XXXXXX")"
+fi
+mkdir -p "$OUT_DIR"
+log "Writing outputs to $OUT_DIR"
+
+log "[1/7] Authenticating as admin ($KEYCLOAK_ADMIN_USER)"
+configure_as "$KEYCLOAK_ADMIN_USER" "$KEYCLOAK_ADMIN_PASSWORD"
+
+log "[2/7] Importing election event from $ELECTION_EVENT_JSON"
+out="$(run_step import-election --file-path "$ELECTION_EVENT_JSON" --is-local)"
+ELECTION_EVENT_ID="$(extract_id "$out")"
+[[ -n "$ELECTION_EVENT_ID" ]] || { echo "Error: could not parse election_event_id from import-election output" >&2; exit 1; }
+log "    election_event_id=$ELECTION_EVENT_ID"
+
+log "[3/7] Generating $NUM_VOTERS voters with numeric, ${VOTER_PIN_DIGITS}-digit DTMF-safe credentials"
+cp "$ELECTION_EVENT_JSON" "$OUT_DIR/election-event.json"
+cat >"$OUT_DIR/external_config.json" <<EXTCFG
+{
+  "election_event_json_file": "election-event.json",
+  "realm_name": "tenant-${TENANT_ID}-event-${ELECTION_EVENT_ID}",
+  "tenant_id": "${TENANT_ID}",
+  "election_event_id": "${ELECTION_EVENT_ID}",
+  "area_id": "",
+  "election_id": "",
+  "generate_voters": {
+    "csv_file_name": "voters",
+    "fields": ["username", "area_name", "password", "email", "email_verified"],
+    "excluded_columns": [],
+    "email_prefix": "telephone-load-test",
+    "domain": "example.invalid",
+    "sequence_email_number": true,
+    "sequence_start_number": 0,
+    "voter_password": "",
+    "voter_password_policy": {"type": "random-numeric", "digits": ${VOTER_PIN_DIGITS}},
+    "password_salt": "",
+    "hashed_password": "",
+    "overseas_reference": "",
+    "min_age": 18,
+    "max_age": 90,
+    "authorized_elections_count": 0,
+    "email_verified": true
+  },
+  "duplicate_votes": {"row_id_to_clone": ""},
+  "generate_applications": {"applicant_data": {}, "annotations": {}}
+}
+EXTCFG
+run_step generate-voters --working-directory "$OUT_DIR" --num-users "$NUM_VOTERS" >/dev/null
+VOTERS_CSV="$OUT_DIR/voters_${NUM_VOTERS}.csv"
+[[ -f "$VOTERS_CSV" ]] || { echo "Error: expected voters CSV at $VOTERS_CSV, not found" >&2; exit 1; }
+log "    voters_csv=$VOTERS_CSV"
+
+log "[4/7] Bulk-importing voters into the election event"
+run_step import-voters --election-event-id "$ELECTION_EVENT_ID" --file-path "$VOTERS_CSV" --is-local >/dev/null
+
+log "[5/7] Starting the keys ceremony (threshold=$THRESHOLD)"
+out="$(run_step start-key-ceremony --election-event-id "$ELECTION_EVENT_ID" --threshold "$THRESHOLD")"
+KEY_CEREMONY_ID="$(extract_id "$out")"
+[[ -n "$KEY_CEREMONY_ID" ]] || { echo "Error: could not parse key_ceremony_id from start-key-ceremony output" >&2; exit 1; }
+log "    key_ceremony_id=$KEY_CEREMONY_ID"
+
+log "[6/7] Completing the keys ceremony as $TRUSTEE1_USER, then $TRUSTEE2_USER"
+configure_as "$TRUSTEE1_USER" "$TRUSTEE1_PASSWORD"
+retry_step 30 5 complete-key-ceremony --election-event-id "$ELECTION_EVENT_ID" --key-ceremony-id "$KEY_CEREMONY_ID"
+configure_as "$TRUSTEE2_USER" "$TRUSTEE2_PASSWORD"
+retry_step 30 5 complete-key-ceremony --election-event-id "$ELECTION_EVENT_ID" --key-ceremony-id "$KEY_CEREMONY_ID"
+
+log "[7/7] Publishing and opening the TELEPHONE voting channel"
+configure_as "$KEYCLOAK_ADMIN_USER" "$KEYCLOAK_ADMIN_PASSWORD"
+run_step publish --election-event-id "$ELECTION_EVENT_ID" >/dev/null
+run_step update-event-voting-status --election-event-id "$ELECTION_EVENT_ID" --voting-status OPEN --voting-channel TELEPHONE >/dev/null
+
+REALM_NAME="tenant-${TENANT_ID}-event-${ELECTION_EVENT_ID}"
+cat >"$OUT_DIR/summary.json" <<SUMMARY
+{
+  "tenant_id": "${TENANT_ID}",
+  "election_event_id": "${ELECTION_EVENT_ID}",
+  "keycloak_realm": "${REALM_NAME}",
+  "keycloak_url": "${KEYCLOAK_URL}",
+  "hasura_url": "${ENDPOINT_URL}",
+  "voters_csv": "${VOTERS_CSV}",
+  "num_voters": ${NUM_VOTERS}
+}
+SUMMARY
+
+log "Done. Election event $ELECTION_EVENT_ID is open for TELEPHONE voting."
+log "Summary: $OUT_DIR/summary.json"
+log "Voters (username,password are the DTMF voter-id/PIN): $VOTERS_CSV"

packages/step-cli/src/commands/import_election_event.rs

@@ -9,6 +9,10 @@ use crate::{
 use clap::Args;
 use colored::Colorize;
 use graphql_client::{GraphQLQuery, Response};
+use std::{
+    thread::sleep,
+    time::{Duration, Instant},
+};
 
 #[derive(Args)]
 #[command(about = "Import Election Event", long_about = None)]
@@ -46,6 +50,29 @@ impl ImportElectionEventFile {
     }
 }
 
+fn wait_for_task(task_execution_id: &str) -> Result<(), Box<dyn std::error::Error>> {
+    let start_time = Instant::now();
+    let timeout = Duration::from_secs(300);
+    let polling_interval = Duration::from_secs(3);
+
+    loop {
+        match crate::utils::tasks::get_task_status(task_execution_id) {
+            Ok(status) if status == "SUCCESS" => return Ok(()),
+            Ok(status) if status == "FAILED" => {
+                return Err("Import election event task failed".into())
+            }
+            Ok(_) => {
+                if Instant::now().duration_since(start_time) >= timeout {
+                    return Err("Timeout while waiting for import election event task to complete"
+                        .into());
+                }
+                sleep(polling_interval);
+            }
+            Err(e) => return Err(format!("Error checking task status: {}", e).into()),
+        }
+    }
+}
+
 pub fn import(file_path: &str, is_local: bool) -> Result<String, Box<dyn std::error::Error>> {
     let config = read_config()?;
     let client = reqwest::blocking::Client::new();
@@ -71,6 +98,14 @@ pub fn import(file_path: &str, is_local: bool) -> Result<String, Box<dyn std::er
                 if let Some(err) = e.error {
                     Err(Box::from(err))
                 } else if let Some(id) = e.id {
+                    // The mutation only enqueues the import; the realm, areas,
+                    // and bulletin board are created asynchronously by the
+                    // matching celery task. Wait for it so callers (like
+                    // `import-voters` right after) see a fully-provisioned
+                    // election event rather than racing it.
+                    if let Some(task_execution) = e.task_execution {
+                        wait_for_task(&task_execution.id)?;
+                    }
                     Ok(id)
                 } else {
                     Err(Box::from("failed generating id"))