diff --git a/Makefile b/Makefile index c44b029..b739918 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,35 @@ help: ## Display this help. ##@ Development +.PHONY: dev-vm-up dev-vm-provider dev-vm-down + +dev-vm-up: ## Create the dev VM if needed and start it. + @if $(LIMA) list --quiet $(DEV_VM_NAME) 2>/dev/null | grep -Fxq "$(DEV_VM_NAME)"; then \ + echo "$(DEV_VM_NAME) already exists"; \ + else \ + $(LIMA) create --yes --name=$(DEV_VM_NAME) \ + --set='.portForwards[0].hostSocket = "$(DEV_VM_SOCKET)"' \ + $(DEV_VM_TEMPLATE); \ + fi + $(LIMA) start --yes $(DEV_VM_NAME) + @echo "Provider address: unix://$(DEV_VM_SOCKET)" + +dev-vm-provider: dev-vm-up ## Start the provider inside the dev VM. + $(LIMA) shell $(DEV_VM_NAME) -- sudo -n -iu chp-dev bash -lc 'cd "$(CURDIR)" && exec go run ./cmd/cloud-hypervisor-provider \ + --address /run/chp/iri-machinebroker.sock \ + --provider-root-dir /var/lib/chp \ + --cloud-hypervisor-firmware-path /usr/local/bin/hypervisor-fw \ + --network-interface-plugin-name isolated \ + --machine-class t1,2,1073741824' + +dev-vm-down: ## Delete the dev VM. + -$(LIMA) delete --yes --force $(DEV_VM_NAME) + +LIMA ?= limactl +DEV_VM_NAME ?= chp-dev +DEV_VM_TEMPLATE ?= dev-vm/chp-dev.yaml +DEV_VM_SOCKET ?= $(CURDIR)/iri-machinebroker.sock + .PHONY: manifests FILE="config/machinepoollet-broker/broker-rbac/role.yaml" manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. diff --git a/config/samples/machine.yaml b/config/samples/machine.yaml index d883056..8630801 100644 --- a/config/samples/machine.yaml +++ b/config/samples/machine.yaml @@ -1,9 +1,13 @@ metadata: - annotations: - foo: bar labels: - bar: baz + app: local-test spec: - class: small - image: - image: ghcr.io/ironcore-dev/ironcore-image/gardenlinux:latest + power: 0 + class: t1 + volumes: + - name: root + device: oda + local_disk: + size_bytes: 5368709120 # 5 GiB + image: + image: ghcr.io/ironcore-dev/gardenlinux/gardener:2150.12.0-kvm diff --git a/dev-vm/chp-dev.yaml b/dev-vm/chp-dev.yaml index f23b7f9..4526381 100644 --- a/dev-vm/chp-dev.yaml +++ b/dev-vm/chp-dev.yaml @@ -275,6 +275,7 @@ provision: set -Eeuo pipefail GO_VERSION="1.26.2" + IRONCORE_VERSION="v0.6.1" case "$(uname -m)" in x86_64) GO_ARCH="amd64" ;; @@ -303,7 +304,7 @@ provision: # it is available on PATH for local testing. GOBIN=/usr/local/bin GOCACHE=/tmp/gocache GOPATH=/tmp/gopath \ /usr/local/go/bin/go install \ - github.com/ironcore-dev/ironcore/irictl-machine/cmd/irictl-machine@main + github.com/ironcore-dev/ironcore/irictl-machine/cmd/irictl-machine@${IRONCORE_VERSION} rm -rf /tmp/gocache /tmp/gopath - mode: data path: /opt/cloud-hypervisor-prepare.sh @@ -589,7 +590,9 @@ networks: # Port forwarding rules. Forwarding between ports 22 and ssh.localPort cannot be overridden. # Rules are checked sequentially until the first one matches. -# portForwards: +portForwards: +- guestSocket: /run/chp/iri-machinebroker.sock + hostSocket: iri-machinebroker.sock # - guestPort: 443 # hostIP: "0.0.0.0" # overrides the default value "127.0.0.1"; allows privileged port forwarding # # default: hostPort: 443 (same as guestPort) diff --git a/dev-vm/install-chp.sh b/dev-vm/install-chp.sh deleted file mode 100644 index 441ba09..0000000 --- a/dev-vm/install-chp.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -VERSION="${VERSION:-51.1}" -ARCH="${ARCH:-x86_64}" # x86_64 or aarch64 -BASE_DIR="/usr/local/bin/cloud-hypervisor" - - -normalize_arch() { - case "$1" in - amd64|x86_64) echo "x86_64" ;; - arm64|aarch64) echo "aarch64" ;; - *) - echo "ERROR: Unsupported architecture '$1'" >&2 - exit 1 - ;; - esac -} - -ARCH="$(normalize_arch "$ARCH")" -HOST_ARCH="$(normalize_arch "$(uname -m)")" - -echo "Install settings:" -echo " VERSION = ${VERSION}" -echo " ARCH = ${ARCH}" -echo " HOST_ARCH = ${HOST_ARCH}" - -if [[ "$ARCH" != "$HOST_ARCH" ]]; then - echo "ERROR: ARCH=${ARCH} does not match host architecture (${HOST_ARCH})." >&2 - echo "Refusing to install wrong binary (would cause 'Exec format error')." >&2 - exit 2 -fi - -case "$ARCH" in - x86_64) - CHP_ASSET="cloud-hypervisor-static" - CHR_ASSET="ch-remote" - ;; - aarch64) - CHP_ASSET="cloud-hypervisor-static-aarch64" - CHR_ASSET="ch-remote-static-aarch64" - ;; -esac - -BASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v${VERSION}" -# DEST_DIR="${BASE_DIR}/${VERSION}/bin" -DEST_DIR="${BASE_DIR}" - -install_bin() { - local name="$1" - local asset="$2" - - local url="${BASE_URL}/${asset}" - local dest="${DEST_DIR}/${name}" - - echo "Installing ${name}" - echo " url: ${url}" - echo " dest: ${dest}" - - local tmp - tmp="$(mktemp)" - trap 'rm -f "$tmp"' RETURN - - curl -fL "$url" -o "$tmp" - chmod 0755 "$tmp" - sudo mkdir -p "$DEST_DIR" - sudo mv "$tmp" "$dest" -} - -install_bin "cloud-hypervisor" "$CHP_ASSET" -install_bin "ch-remote" "$CHR_ASSET" - -# Optional sanity checks -"${DEST_DIR}/cloud-hypervisor" --version || true -"${DEST_DIR}/ch-remote" --help >/dev/null 2>&1 || true - - -echo "All done." diff --git a/docs/development/dev_setup.md b/docs/development/dev_setup.md index cf39772..5312d8c 100644 --- a/docs/development/dev_setup.md +++ b/docs/development/dev_setup.md @@ -7,79 +7,70 @@ with nested virtualization enabled. - [Lima](https://lima-vm.io/) v2.0.0+ - macOS with Apple Silicon (M3 or later for nested virtualization) -- Go (version matching `go.mod`) -## 1. Create the Dev VM +## 1. Start the Dev VM The dev VM definition is located at `dev-vm/chp-dev.yaml`. It provisions an Ubuntu VM with: -- A `chp` system user/group (UID/GID 65532) that runs cloud-hypervisor -- A `chp-dev` user (your dev user inside the VM) -- Two cloud-hypervisor instances (`vm1`, `vm2`) managed via systemd -- Nested virtualization enabled for KVM access +- A `chp` system user/group (UID/GID 65532) for the QMP monitor and shared storage +- A `chp-dev` user (your dev user inside the VM) with access to `chp` and `kvm` +- Cloud Hypervisor instances started on demand by the provider ```bash -limactl create --name chp-dev dev-vm/chp-dev.yaml -limactl start chp-dev +make dev-vm-up ``` -## 2. Shell into the VM +Use `make dev-vm-down` to delete the instance. After changing the Lima +template, run `make dev-vm-down && make dev-vm-up` so the instance picks up the +new configuration. + +## 2. Run the Provider ```bash -limactl shell chp-dev +make dev-vm-provider ``` -Your host home directory is mounted read-only at `~` inside the VM. +The provider uses `/var/lib/chp` as its root directory, shared between +`chp-dev` and `chp`. The target starts it inside the VM using the repository +mounted from the host and refreshes the `chp`/`kvm` group membership for the +process. It remains attached to the terminal; stop it with `Ctrl-C`. -## 3. Clone / Access the Repository +### Access the provider socket from macOS -Since the host home directory is mounted, you can access the repository directly. -For a writable copy: +Lima forwards the provider's Unix socket to the host automatically. Once the +provider is running, use the `unix://` URI form required by gRPC: ```bash -cd ~ -git clone https://github.com/ironcore-dev/cloud-hypervisor-provider.git -cd cloud-hypervisor-provider +irictl-machine --address "unix://$(pwd)/iri-machinebroker.sock" get machine ``` -## 4. Verify Cloud-Hypervisor is Running - -The VM provisioning automatically starts two cloud-hypervisor instances via systemd: +The `irictl-machine` client must be compatible with the provider. Alternatively, +use the client installed inside the VM: ```bash -# Check service status -systemctl status cloud-hypervisor@vm1 -systemctl status cloud-hypervisor@vm2 - -# Verify sockets exist -ls -la /run/chp/ch/ +limactl shell chp-dev -- irictl-machine \ + --address unix:///run/chp/iri-machinebroker.sock get machine ``` -You should see `vm1.sock` and `vm2.sock` owned by `chp:chp`. +`make dev-vm-up` prints the resolved `unix://` address after starting the VM. +If the provider listens on a different address, update the `guestSocket` entry +in `dev-vm/chp-dev.yaml` and recreate the VM. -## 5. Verify KVM Access +## 3. Shell into the VM (optional) ```bash -# Check KVM is available -ls -la /dev/kvm - -# Your user should be in the chp group -id -# Expected: groups=... chp ... +limactl shell chp-dev ``` -## 6. Run the Provider - -The provider needs to use `/var/lib/chp` as its root directory (shared between `chp-dev` and `chp`): +The host home directory is mounted read-only at the same absolute path inside +the VM, while the guest user's `~` is a separate writable home directory. The +repository is therefore available without cloning it: ```bash -go run ./cmd/cloud-hypervisor-provider \ - --provider-root-dir /var/lib/chp \ - --cloud-hypervisor-sockets-path /run/chp/ch \ - --cloud-hypervisor-firmware-path /usr/local/bin/hypervisor-fw +cd /Users//path/to/cloud-hypervisor-provider ``` -## 7. Run Tests +## 4. Run Tests ### Unit Tests @@ -91,48 +82,37 @@ make test ### Integration Tests -Integration tests require the cloud-hypervisor instances to be running and accessible: +From inside the VM, integration tests start Cloud Hypervisor on demand and +require KVM plus the prepared firmware: ```bash -export CH_SOCKET_DIR=/run/chp/ch/ export CH_FIRMWARE_PATH=/usr/local/bin/hypervisor-fw make integration-tests ``` -## Architecture Overview - -``` -Host (macOS) - └── Lima VM (chp-dev) - ├── chp-dev user ── runs the provider process - │ └── writes disk images to /var/lib/chp (0775, chp:chp) - ├── chp user ────── runs cloud-hypervisor via systemd - │ ├── /run/chp/ch/vm1.sock - │ ├── /run/chp/ch/vm2.sock - │ └── reads disk images from /var/lib/chp - └── /dev/kvm ────── chp user has access via kvm group -``` - ## Troubleshooting ### "Failed to ping cloud-hypervisor socket" -The provider can't connect to the sockets. Check permissions: +The provider cannot connect to a per-machine socket. Check the provider root +and the effective groups used by the provider: ```bash -ls -la /run/chp/ch/*.sock +sudo -iu chp-dev id +ls -la /var/lib/chp ``` -Sockets need group read/write for `chp-dev`. Fix with `sudo chmod g+rw /run/chp/ch/*.sock`. +Per-machine Cloud Hypervisor sockets are created by the provider below +`/var/lib/chp/machines` and are not pre-created by systemd. ### "Cannot open disk path" / "Permission denied (os error 13)" -Cloud-hypervisor (`chp` user) can't read disk images created by the provider (`chp-dev` user). -Ensure: +The QMP service (`chp` user) and provider (`chp-dev` user) must be able to +access the provider data directory. Ensure: 1. The provider uses `--provider-root-dir /var/lib/chp` (not the home directory) 2. `/var/lib/chp` has permissions `0775` and is owned by `chp:chp` -3. `chp-dev` is in the `chp` group (`id` should show `chp` in groups) +3. `chp-dev` is in the `chp` group (`sudo -iu chp-dev id` should show `chp`) ### "Error manipulating firmware file" / "No such file or directory" @@ -154,7 +134,7 @@ sudo systemctl start cloud-hypervisor-prepare.service Ensure nested virtualization is enabled (requires Apple M3+): ```bash -grep -c vmx /proc/cpuinfo 2>/dev/null || grep -c svm /proc/cpuinfo 2>/dev/null +ls -l /dev/kvm ``` -If 0, nested virtualization is not available on your hardware. +If `/dev/kvm` is missing, nested virtualization is not available to the VM.