From fa80089fd80eab3e8beedc6baf148902a34b4721 Mon Sep 17 00:00:00 2001 From: Hardik Dodiya Date: Thu, 3 Sep 2026 17:21:28 +0200 Subject: [PATCH 1/4] Implement a ztp-mode to generate the ztp script from the Switch CR Signed-off-by: Hardik Dodiya --- api/v1alpha1/switch_types.go | 97 +++++- api/v1alpha1/zz_generated.deepcopy.go | 90 +++++- cmd/main.go | 13 +- ...etworking.metal.ironcore.dev_switches.yaml | 93 +++++- config/manager/manager.yaml | 16 +- .../samples/networking_v1alpha1_switch.yaml | 22 ++ docs/usage/provisioning.md | 99 +++++- internal/controller/switch_controller.go | 26 +- internal/controller/switch_controller_test.go | 10 +- internal/ztp/ztp.go | 295 ++++++++++++++++-- internal/ztp/ztp_test.go | 144 ++++++++- 11 files changed, 844 insertions(+), 61 deletions(-) diff --git a/api/v1alpha1/switch_types.go b/api/v1alpha1/switch_types.go index c74d4a0..ff7e9db 100644 --- a/api/v1alpha1/switch_types.go +++ b/api/v1alpha1/switch_types.go @@ -36,18 +36,108 @@ type ZTPConfigMapReference struct { // ZTP script. SourceAddress is the address observed by the provisioning server, // which can differ from the management address used after provisioning. type ZTP struct { - SourceAddress string `json:"sourceAddress"` - ScriptRef ZTPConfigMapReference `json:"scriptRef"` + SourceAddress string `json:"sourceAddress"` + + // ScriptRef identifies the complete script served when --ztp-mode=configmap. + // It is not used by --ztp-mode=generated. + // +optional + ScriptRef *ZTPConfigMapReference `json:"scriptRef,omitempty"` +} + +// BootstrapContainer declares a Docker container that the provisioning server +// starts on the switch during generated ZTP provisioning. +type BootstrapContainer struct { + // Name is both the Docker container name and its stable identity on the switch. + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // Image is the container image pulled by Docker on the switch. + // +kubebuilder:validation:MinLength=1 + Image string `json:"image"` + + // Command overrides the image entrypoint. + // +optional + Command []string `json:"command,omitempty"` + + // Args are appended after Command, or after the image entrypoint when Command is empty. + // +optional + Args []string `json:"args,omitempty"` + + // SecurityContext configures the Unix identity used to run the container. + // +optional + SecurityContext *BootstrapContainerSecurityContext `json:"securityContext,omitempty"` + + // InjectControlKubeconfig mounts the operator's configured control kubeconfig + // into this container and sets KUBECONFIG to its in-container path. A + // securityContext with runAsUser is required so the generated script can + // grant access to a private, per-container credential file. + // +optional + InjectControlKubeconfig bool `json:"injectControlKubeconfig,omitempty"` +} + +// BootstrapContainerSecurityContext is the supported subset of Kubernetes +// container securityContext for generated Docker containers. +type BootstrapContainerSecurityContext struct { + // RunAsUser is the numeric Unix user ID used by the container. + // +optional + // +kubebuilder:validation:Minimum=0 + RunAsUser *int64 `json:"runAsUser,omitempty"` + + // RunAsGroup is the numeric Unix group ID used by the container. It requires + // runAsUser to be set as well. + // +optional + // +kubebuilder:validation:Minimum=0 + RunAsGroup *int64 `json:"runAsGroup,omitempty"` } +// Bootstrap declares generic containers installed by ZTP. It intentionally +// contains no workload-specific configuration or Secret references. +type Bootstrap struct { + // Containers are started with host networking and restart automatically on + // subsequent SONiC boots. + // +optional + // +listType=map + // +listMapKey=name + Containers []BootstrapContainer `json:"containers,omitempty"` +} + +// NextBootMode describes the desired behavior of the switch's next boot. +// Providers translate this high-level intent to their platform-specific boot +// mechanism. For SONiC, InstallOS enters ONIE install discovery. +type NextBootMode string + +const ( + // NextBootModeNone leaves the normal installed network OS boot path intact. + NextBootModeNone NextBootMode = "None" + // NextBootModeInstallOS enters the platform's OS installation/discovery flow. + NextBootModeInstallOS NextBootMode = "InstallOS" +) + // SwitchSpec defines the desired state of Switch type SwitchSpec struct { + // Hostname is configured on the switch by --ztp-mode=generated. If omitted, + // the generated script uses the Switch object name. + // +optional + Hostname string `json:"hostname,omitempty"` + Management Management `json:"management,omitempty"` - // ZTP selects a custom script when the operator runs with --ztp-mode=configmap. + // ZTP identifies the switch while it requests its initial provisioning script. // +optional ZTP *ZTP `json:"ztp,omitempty"` + // Bootstrap declares containers that are started on the switch by + // --ztp-mode=generated. + // +optional + Bootstrap *Bootstrap `json:"bootstrap,omitempty"` + + // NextBootMode declares the desired behavior of the next boot. The default + // is None. It is configured by --ztp-mode=generated. + // +optional + // +kubebuilder:validation:Enum=None;InstallOS + NextBootMode NextBootMode `json:"nextBootMode,omitempty"` + // MacAddress is the MAC address assigned to this interface. MacAddress string `json:"macAddress"` @@ -105,7 +195,6 @@ type SwitchStatus struct { // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // +kubebuilder:ac:generate=false - // Switch is the Schema for the switch API type Switch struct { metav1.TypeMeta `json:",inline"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2d22add..53377b9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,83 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Bootstrap) DeepCopyInto(out *Bootstrap) { + *out = *in + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]BootstrapContainer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Bootstrap. +func (in *Bootstrap) DeepCopy() *Bootstrap { + if in == nil { + return nil + } + out := new(Bootstrap) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapContainer) DeepCopyInto(out *BootstrapContainer) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(BootstrapContainerSecurityContext) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapContainer. +func (in *BootstrapContainer) DeepCopy() *BootstrapContainer { + if in == nil { + return nil + } + out := new(BootstrapContainer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BootstrapContainerSecurityContext) DeepCopyInto(out *BootstrapContainerSecurityContext) { + *out = *in + if in.RunAsUser != nil { + in, out := &in.RunAsUser, &out.RunAsUser + *out = new(int64) + **out = **in + } + if in.RunAsGroup != nil { + in, out := &in.RunAsGroup, &out.RunAsGroup + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapContainerSecurityContext. +func (in *BootstrapContainerSecurityContext) DeepCopy() *BootstrapContainerSecurityContext { + if in == nil { + return nil + } + out := new(BootstrapContainerSecurityContext) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Management) DeepCopyInto(out *Management) { *out = *in @@ -332,7 +409,12 @@ func (in *SwitchSpec) DeepCopyInto(out *SwitchSpec) { if in.ZTP != nil { in, out := &in.ZTP, &out.ZTP *out = new(ZTP) - **out = **in + (*in).DeepCopyInto(*out) + } + if in.Bootstrap != nil { + in, out := &in.Bootstrap, &out.Bootstrap + *out = new(Bootstrap) + (*in).DeepCopyInto(*out) } if in.Ports != nil { in, out := &in.Ports, &out.Ports @@ -383,7 +465,11 @@ func (in *SwitchStatus) DeepCopy() *SwitchStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ZTP) DeepCopyInto(out *ZTP) { *out = *in - out.ScriptRef = in.ScriptRef + if in.ScriptRef != nil { + in, out := &in.ScriptRef, &out.ScriptRef + *out = new(ZTPConfigMapReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ZTP. diff --git a/cmd/main.go b/cmd/main.go index e3785bf..5db0fe6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -55,7 +55,7 @@ func main() { var secureMetrics bool var enableHTTP2 bool var disableProvisionsingServer bool - var httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode string + var httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode, bootstrapControlKubeconfigFile string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -76,7 +76,8 @@ func main() { "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.StringVar(&httpServerAddr, "http-server-address", "0", "The address the HTTP server for ZTP and ONIE binds to.") flag.StringVar(&ztpConfigFile, "ztp-config-file", "/etc/ztp.json", "Config file containing the parameters to render ZTP scripts.") - flag.StringVar(&ztpMode, "ztp-mode", "templates", "ZTP source: templates or configmap. Configmap mode never falls back to templates.") + flag.StringVar(&ztpMode, "ztp-mode", "templates", "ZTP source: templates, configmap, or generated. Configmap mode serves the referenced script verbatim; generated mode renders from Switch objects.") + flag.StringVar(&bootstrapControlKubeconfigFile, "bootstrap-control-kubeconfig-file", "", "Optional kubeconfig file injected into bootstrap containers that set injectControlKubeconfig=true.") flag.StringVar(&onieImagesDir, "onie-images-dir", "/var/lib/sonic-operator/onie", "The directory which contains the ONIE and SONiC installer image files.") flag.StringVar(&onieConfigFile, "onie-config-file", "/etc/onie.json", "Config file containing machine-to-image mappings for ONIE provisioning.") flag.BoolVar(&disableProvisionsingServer, "disable-static-config", false, "If set, the HTTP server for ZTP and ONIE will not be started.") @@ -212,7 +213,7 @@ func main() { } if !disableProvisionsingServer { setupLog.Info("starting HTTP server") - provServer, err := setupProvisioningServer(httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode, mgr.GetAPIReader()) + provServer, err := setupProvisioningServer(httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode, bootstrapControlKubeconfigFile, mgr.GetAPIReader()) if err != nil { setupLog.Error(err, "unable to setup HTTP server") os.Exit(1) @@ -232,7 +233,7 @@ func main() { } } -func setupProvisioningServer(addr string, onieImagesDir string, onieConfigPath string, ztpConfigPath string, ztpMode string, reader client.Reader) (*http.Server, error) { +func setupProvisioningServer(addr string, onieImagesDir string, onieConfigPath string, ztpConfigPath string, ztpMode string, bootstrapControlKubeconfigFile string, reader client.Reader) (*http.Server, error) { of, err := os.Open(onieConfigPath) if err != nil { return nil, fmt.Errorf("unable to open onie config file: %w", err) @@ -268,8 +269,10 @@ func setupProvisioningServer(addr string, onieImagesDir string, onieConfigPath s ztp.Register(mux, ztpConf) case "configmap": ztp.RegisterConfigMap(mux, reader) + case "generated": + ztp.RegisterGenerated(mux, reader, ztp.GeneratedOptions{ControlKubeconfigFile: bootstrapControlKubeconfigFile}) default: - return nil, fmt.Errorf("unsupported ztp mode %q (supported: templates, configmap)", ztpMode) + return nil, fmt.Errorf("unsupported ztp mode %q (supported: templates, configmap, generated)", ztpMode) } onie.Register(mux, onieImagesDir, onieConf) diff --git a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml index f191481..3312e7b 100644 --- a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml +++ b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml @@ -49,6 +49,81 @@ spec: spec: description: spec defines the desired state of Switch properties: + bootstrap: + description: |- + Bootstrap declares containers that are started on the switch by + --ztp-mode=generated. + properties: + containers: + description: |- + Containers are started with host networking and restart automatically on + subsequent SONiC boots. + items: + description: |- + BootstrapContainer declares a Docker container that the provisioning server + starts on the switch during generated ZTP provisioning. + properties: + args: + description: Args are appended after Command, or after the + image entrypoint when Command is empty. + items: + type: string + type: array + command: + description: Command overrides the image entrypoint. + items: + type: string + type: array + image: + description: Image is the container image pulled by Docker + on the switch. + minLength: 1 + type: string + injectControlKubeconfig: + description: |- + InjectControlKubeconfig mounts the operator's configured control kubeconfig + into this container and sets KUBECONFIG to its in-container path. A + securityContext with runAsUser is required so the generated script can + grant access to a private, per-container credential file. + type: boolean + name: + description: Name is both the Docker container name and + its stable identity on the switch. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + securityContext: + description: SecurityContext configures the Unix identity + used to run the container. + properties: + runAsGroup: + description: |- + RunAsGroup is the numeric Unix group ID used by the container. It requires + runAsUser to be set as well. + format: int64 + minimum: 0 + type: integer + runAsUser: + description: RunAsUser is the numeric Unix user ID used + by the container. + format: int64 + minimum: 0 + type: integer + type: object + required: + - image + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + hostname: + description: |- + Hostname is configured on the switch by --ztp-mode=generated. If omitted, + the generated script uses the Switch object name. + type: string macAddress: description: MacAddress is the MAC address assigned to this interface. type: string @@ -107,6 +182,14 @@ spec: - host - port type: object + nextBootMode: + description: |- + NextBootMode declares the desired behavior of the next boot. The default + is None. It is configured by --ztp-mode=generated. + enum: + - None + - InstallOS + type: string ports: description: Ports the physical ports available on the Switch. items: @@ -118,14 +201,13 @@ spec: type: object type: array ztp: - description: ZTP selects a custom script when the operator runs with - --ztp-mode=configmap. + description: ZTP identifies the switch while it requests its initial + provisioning script. properties: scriptRef: description: |- - ZTPConfigMapReference identifies the ConfigMap entry containing a switch's - complete ZTP script. The referenced ConfigMap may be in a provisioning - namespace chosen by the cluster administrator. + ScriptRef identifies the complete script served when --ztp-mode=configmap. + It is not used by --ztp-mode=generated. properties: key: type: string @@ -141,7 +223,6 @@ spec: sourceAddress: type: string required: - - scriptRef - sourceAddress type: object required: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ddff94e..e3adf2c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -63,6 +63,8 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 + - --ztp-mode=generated + - --bootstrap-control-kubeconfig-file=/etc/sonic-operator/control-kubeconfig/kubeconfig image: controller:latest name: manager ports: [] @@ -93,7 +95,17 @@ spec: requests: cpu: 10m memory: 64Mi - volumeMounts: [] - volumes: [] + volumeMounts: + - name: control-kubeconfig + mountPath: /etc/sonic-operator/control-kubeconfig + readOnly: true + volumes: + - name: control-kubeconfig + secret: + secretName: control-kubeconfig + optional: true + items: + - key: kubeconfig + path: kubeconfig serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/samples/networking_v1alpha1_switch.yaml b/config/samples/networking_v1alpha1_switch.yaml index e2463e5..4b834e3 100644 --- a/config/samples/networking_v1alpha1_switch.yaml +++ b/config/samples/networking_v1alpha1_switch.yaml @@ -6,6 +6,28 @@ metadata: app.kubernetes.io/managed-by: kustomize name: switch-sample spec: + hostname: switch-sample + + ztp: + sourceAddress: "192.0.2.11" + + nextBootMode: InstallOS + + # Bootstrap containers are rendered by --ztp-mode=generated. + # The control kubeconfig is provided globally by the operator, not by this + # Switch object. + bootstrap: + containers: + - name: wirelet + image: ghcr.io/hardikdr/wirelet:fixed-1 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + args: + - --name=switch-sample + - --interface=Ethernet0 + injectControlKubeconfig: true + management: host: 192.0.2.10 port: "50051" diff --git a/docs/usage/provisioning.md b/docs/usage/provisioning.md index 0d89e59..d7f0804 100644 --- a/docs/usage/provisioning.md +++ b/docs/usage/provisioning.md @@ -5,17 +5,19 @@ The provisioning server serves ZTP scripts and ONIE installer artifacts over HTT ## Manager flags - `--http-server-address`: bind address for the provisioning server. - `--ztp-config-file`: JSON file with ZTP parameters (default `/etc/ztp.json`). -- `--ztp-mode`: ZTP source, either `templates` (default) or `configmap`. Modes are strict: ConfigMap mode never falls back to templates. +- `--ztp-mode`: ZTP source: `templates` (default), `configmap`, or `generated`. +- `--bootstrap-control-kubeconfig-file`: optional control kubeconfig mounted into the operator and made available to opted-in generated bootstrap containers. - `--onie-installer-dir`: directory containing ONIE installer files (default `/var/lib/sonic-operator/onie`). ## ZTP -- Scripts are rendered from templates in `internal/ztp/templates`. -- The source IP of the requesting switch is used to select parameters from the ZTP config file. +- `templates` renders the static scripts in `internal/ztp/templates` from the JSON ZTP configuration. +- `configmap` serves the selected ConfigMap key verbatim. +- `generated` renders one complete script from the matching `Switch` object. - The ZTP script is served at `GET /ztp`. ## ConfigMap ZTP mode -Set `--ztp-mode=configmap` to select a full ZTP script from a ConfigMap. The provisioning server matches the source address of `GET /ztp` to `spec.ztp.sourceAddress` on a `Switch`, then returns the configured ConfigMap key verbatim. A missing switch, ConfigMap, or key is an error; the leaf/spine templates are never used in this mode. +Set `--ztp-mode=configmap` to select a full ZTP script from a ConfigMap. The provisioning server matches the source address of `GET /ztp` to `spec.ztp.sourceAddress` on a `Switch`, then returns the configured ConfigMap key verbatim. A missing switch, ConfigMap, or key is an error; no generated commands are appended and the leaf/spine templates are never used in this mode. ```yaml apiVersion: sonic.networking.metal.ironcore.dev/v1alpha1 @@ -44,6 +46,95 @@ data: hostnamectl set-hostname leaf-01 ``` +## Generated ZTP mode + +Set `--ztp-mode=generated` to render the entire ZTP script from the matching +`Switch` object. No ConfigMap script is read in this mode. The generated script +configures the hostname, saves SONiC configuration, writes any requested global +credential, and starts declared Docker containers. + +```yaml +apiVersion: sonic.networking.metal.ironcore.dev/v1alpha1 +kind: Switch +metadata: + name: switch-1 +spec: + hostname: switch-1 + ztp: + sourceAddress: "2001:db8:100::11" + bootstrap: + containers: + - name: wirelet + image: ghcr.io/ironcore-dev/wirelet:fixed-1 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + args: + - --name=switch-1 + - --interface=Ethernet0 + injectControlKubeconfig: true +``` + +Each container is pulled and started with host networking and Docker's +`unless-stopped` restart policy. `command` overrides the image entrypoint; +`args` are appended after `command` or the image entrypoint. `hostname` +defaults to the `Switch` object name when omitted. Bootstrap containers are +best effort: a failure to prepare or start one is logged and does not prevent +other containers from being attempted. + +### ONIE boot discovery + +Generated ZTP can persistently configure the next SONiC reboot to enter ONIE +install discovery: + +```yaml +spec: + nextBootMode: InstallOS +``` + +The generated script creates an `onieboot.mount` unit for the `ONIE-BOOT` +partition and a `sonic-operator-onie-install.service` unit. The latter runs on +every SONiC boot and sets `next_entry=ONIE` plus `onie_mode=install`. It does +not reboot the switch immediately; the next reboot enters ONIE install mode. +`None` (or omitting `nextBootMode`) leaves the normal boot path unchanged. +This boot-lifecycle setup is rendered before hostname and bootstrap-container +steps, and remains strict: a failure to configure it stops the script. + +### Global control kubeconfig + +`injectControlKubeconfig: true` is an explicit opt-in to the operator's global +control kubeconfig. The default manager manifest mounts a Secret named +`control-kubeconfig` in the operator namespace at +`/etc/sonic-operator/control-kubeconfig/kubeconfig` and passes that path with +`--bootstrap-control-kubeconfig-file`. + +Create the Secret before provisioning a switch that requests injection: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: control-kubeconfig + namespace: sonic-operator-system +type: Opaque +stringData: + kubeconfig: | + # a deliberately restricted control-cluster kubeconfig +``` + +Containers use `securityContext.runAsUser` and `securityContext.runAsGroup`, +which map to Docker's `--user` option. The generated ZTP script writes a +private credential file per opted-in container, owned by that container's +configured user/group with mode `0600`, bind-mounts it read-only at +`/var/run/sonic-operator/control-kubeconfig`, and sets `KUBECONFIG` to that +path in the opted-in container. `injectControlKubeconfig: true` requires +`securityContext.runAsUser`. + +This is a shared bootstrap credential. Use a restricted identity only; do not +mount an administrator kubeconfig. The current ZTP endpoint is HTTP, so this +mechanism is appropriate for controlled lab networks only until credential +delivery is protected by HTTPS and a short-lived bootstrap flow. + ## ONIE - Files are served from the installer directory at HTTP root (`/`). - This supports ONIE discovery workflows for delivering SONiC or other OS installers. diff --git a/internal/controller/switch_controller.go b/internal/controller/switch_controller.go index bb968fe..4069d91 100644 --- a/internal/controller/switch_controller.go +++ b/internal/controller/switch_controller.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/go-logr/logr" "github.com/ironcore-dev/controller-utils/clientutils" @@ -27,7 +28,8 @@ import ( ) var ( - fieldOwner = client.FieldOwner("switch-controller") + fieldOwner = client.FieldOwner("switch-controller") + agentRetryAfter = time.Minute ) // SwitchReconciler reconciles a Switch object @@ -88,20 +90,24 @@ func (r *SwitchReconciler) reconcile(ctx context.Context, log logr.Logger, s *ne } }() - if s.Status.State == "" { - s.Status.State = networkingv1alpha1.SwitchStatePending + // A Switch is declaratively ready as soon as it is accepted by the + // controller. The SONiC agent enriches status when reachable, but agent + // availability must not prevent ZTP/bootstrap workflows. + if s.Status.State != networkingv1alpha1.SwitchStateReady { + s.Status.State = networkingv1alpha1.SwitchStateReady return ctrl.Result{}, nil } switchAgentClient, err := switchUtil.NewAgentClientForSwitch(ctx, s) if err != nil { - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } switchDevice, err := switchAgentClient.GetDeviceInfo(ctx) if err != nil { - s.Status.State = networkingv1alpha1.SwitchStateFailed - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } s.Status.MACAddress = switchDevice.LocalMacAddress @@ -110,8 +116,8 @@ func (r *SwitchReconciler) reconcile(ctx context.Context, log logr.Logger, s *ne interfaceList, err := switchAgentClient.ListInterfaces(ctx) if err != nil { - s.Status.State = networkingv1alpha1.SwitchStateFailed - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } for _, iface := range interfaceList.Items { @@ -122,8 +128,8 @@ func (r *SwitchReconciler) reconcile(ctx context.Context, log logr.Logger, s *ne portList, err := switchAgentClient.ListPorts(ctx) if err != nil { - s.Status.State = networkingv1alpha1.SwitchStateFailed - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } if len(portList.Items) > 0 { diff --git a/internal/controller/switch_controller_test.go b/internal/controller/switch_controller_test.go index fc2c738..137d917 100644 --- a/internal/controller/switch_controller_test.go +++ b/internal/controller/switch_controller_test.go @@ -64,8 +64,14 @@ var _ = Describe("Switch Controller", func() { NamespacedName: typeNamespacedName, }) Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + updated := &networkingv1alpha1.Switch{} + Expect(k8sClient.Get(ctx, typeNamespacedName, updated)).To(Succeed()) + Expect(updated.Status.State).To(Equal(networkingv1alpha1.SwitchStateReady)) }) }) }) diff --git a/internal/ztp/ztp.go b/internal/ztp/ztp.go index f13a19e..9672fd1 100644 --- a/internal/ztp/ztp.go +++ b/internal/ztp/ztp.go @@ -5,10 +5,14 @@ package ztp import ( "embed" + "encoding/base64" "fmt" "log/slog" "net/http" "net/netip" + "os" + "strconv" + "strings" "text/template" corev1 "k8s.io/api/core/v1" @@ -79,6 +83,19 @@ type configMapHandler struct { reader client.Reader } +type generatedHandler struct { + reader client.Reader + controlKubeconfigFilePath string +} + +// GeneratedOptions configures declarative, Switch-backed ZTP rendering. +type GeneratedOptions struct { + // ControlKubeconfigFile is an optional kubeconfig mounted into the operator + // pod. It is injected only into generated bootstrap containers that + // explicitly opt in. + ControlKubeconfigFile string +} + func Register(mux *http.ServeMux, c Config) { t := template.New("ztp-scripts") t = t.Funcs(template.FuncMap{ @@ -99,6 +116,16 @@ func RegisterConfigMap(mux *http.ServeMux, reader client.Reader) { mux.Handle("GET /ztp", &configMapHandler{reader: reader}) } +// RegisterGenerated renders a complete ZTP script from the matching Switch +// object. Unlike configmap mode, no user-provided ZTP script is read or +// modified. +func RegisterGenerated(mux *http.ServeMux, reader client.Reader, options GeneratedOptions) { + mux.Handle("GET /ztp", &generatedHandler{ + reader: reader, + controlKubeconfigFilePath: options.ControlKubeconfigFile, + }) +} + // interfacePrefix takes the interface ID and the /64 prefix of the switch and // reserves a unique /112 prefix for each interface based on the interface ID. func interfacePrefix(prefix netip.Prefix, interfaceID int) (string, error) { @@ -141,18 +168,64 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (h *configMapHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ap, err := netip.ParseAddrPort(r.RemoteAddr) + matched, err := matchingSwitch(r, h.reader) if err != nil { handleErr(w, err) return } + if matched.Spec.ZTP.ScriptRef == nil { + handleErr(w, fmt.Errorf("switch %q has no ztp.scriptRef for configmap mode", matched.Name)) + return + } - switches := &networkingv1alpha1.SwitchList{} - if err := h.reader.List(r.Context(), switches); err != nil { - handleErr(w, fmt.Errorf("listing switches for ZTP: %w", err)) + ref := matched.Spec.ZTP.ScriptRef + configMap := &corev1.ConfigMap{} + if err := h.reader.Get(r.Context(), types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}, configMap); err != nil { + handleErr(w, fmt.Errorf("getting ZTP ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err)) + return + } + script, ok := configMap.Data[ref.Key] + if !ok { + handleErr(w, fmt.Errorf("ZTP ConfigMap %s/%s has no key %q", ref.Namespace, ref.Name, ref.Key)) return } + w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") + if _, err := fmt.Fprint(w, script); err != nil { + slog.Error("failed to write ZTP ConfigMap script", "switch", matched.Name, "err", err) + } +} + +func (h *generatedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + matched, err := matchingSwitch(r, h.reader) + if err != nil { + handleErr(w, err) + return + } + + script, err := renderGeneratedScript(matched, h.controlKubeconfigFilePath) + if err != nil { + handleErr(w, fmt.Errorf("rendering generated ZTP for switch %q: %w", matched.Name, err)) + return + } + + w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") + if _, err := fmt.Fprint(w, script); err != nil { + slog.Error("failed to write generated ZTP script", "switch", matched.Name, "err", err) + } +} + +func matchingSwitch(r *http.Request, reader client.Reader) (*networkingv1alpha1.Switch, error) { + ap, err := netip.ParseAddrPort(r.RemoteAddr) + if err != nil { + return nil, err + } + + switches := &networkingv1alpha1.SwitchList{} + if err := reader.List(r.Context(), switches); err != nil { + return nil, fmt.Errorf("listing switches for ZTP: %w", err) + } + var matched *networkingv1alpha1.Switch for i := range switches.Items { switchConfig := switches.Items[i].Spec.ZTP @@ -161,39 +234,217 @@ func (h *configMapHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } configuredAddr, err := netip.ParseAddr(switchConfig.SourceAddress) if err != nil { - handleErr(w, fmt.Errorf("switch %q has invalid ZTP source address %q: %w", switches.Items[i].Name, switchConfig.SourceAddress, err)) - return + return nil, fmt.Errorf("switch %q has invalid ZTP source address %q: %w", switches.Items[i].Name, switchConfig.SourceAddress, err) } if configuredAddr != ap.Addr() { continue } if matched != nil { - handleErr(w, fmt.Errorf("multiple switches use ZTP source address %q", ap.Addr())) - return + return nil, fmt.Errorf("multiple switches use ZTP source address %q", ap.Addr()) } matched = &switches.Items[i] } if matched == nil { - handleErr(w, fmt.Errorf("no switch configured for ZTP source address %q", ap.Addr())) - return + return nil, fmt.Errorf("no switch configured for ZTP source address %q", ap.Addr()) } + return matched, nil +} - ref := matched.Spec.ZTP.ScriptRef - configMap := &corev1.ConfigMap{} - if err := h.reader.Get(r.Context(), types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}, configMap); err != nil { - handleErr(w, fmt.Errorf("getting ZTP ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err)) - return +func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubeconfigFilePath string) (string, error) { + hostname := switchObject.Spec.Hostname + if hostname == "" { + hostname = switchObject.Name } - script, ok := configMap.Data[ref.Key] - if !ok { - handleErr(w, fmt.Errorf("ZTP ConfigMap %s/%s has no key %q", ref.Namespace, ref.Name, ref.Key)) - return + if hostname == "" { + return "", fmt.Errorf("hostname is empty") } - w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") - if _, err := fmt.Fprint(w, script); err != nil { - slog.Error("failed to write ZTP ConfigMap script", "switch", matched.Name, "err", err) + var b strings.Builder + b.WriteString("#!/bin/bash\n") + b.WriteString("set -euo pipefail\n\n") + b.WriteString("# Generated by sonic-operator ") + b.WriteString(shellQuote(switchObject.Name)) + b.WriteString("\n") + + // Apply boot lifecycle intent before any fallible switch configuration or + // bootstrap workload operation. This is intentionally strict: a failure to + // persist the requested next boot mode must not be hidden. + switch switchObject.Spec.NextBootMode { + case "", networkingv1alpha1.NextBootModeNone: + case networkingv1alpha1.NextBootModeInstallOS: + appendONIEInstallBootConfig(&b) + default: + return "", fmt.Errorf("unsupported next boot mode %q", switchObject.Spec.NextBootMode) } + + b.WriteString("config hostname ") + b.WriteString(shellQuote(hostname)) + b.WriteString("\n") + b.WriteString("config save -y\n") + + if switchObject.Spec.Bootstrap != nil && len(switchObject.Spec.Bootstrap.Containers) > 0 { + containers := switchObject.Spec.Bootstrap.Containers + needsControlKubeconfig := false + for _, container := range containers { + if container.InjectControlKubeconfig { + needsControlKubeconfig = true + break + } + } + + b.WriteString("\n# sonic-operator bootstrap containers\n") + + var encodedControlKubeconfig string + if needsControlKubeconfig { + if controlKubeconfigFilePath == "" { + return "", fmt.Errorf("injectControlKubeconfig requires --bootstrap-control-kubeconfig-file") + } + kubeconfig, err := os.ReadFile(controlKubeconfigFilePath) + if err != nil { + return "", fmt.Errorf("reading control kubeconfig file %q: %w", controlKubeconfigFilePath, err) + } + + encodedControlKubeconfig = base64.StdEncoding.EncodeToString(kubeconfig) + } + + for _, container := range containers { + dockerUser, err := bootstrapContainerDockerUser(container) + if err != nil { + return "", err + } + + controlKubeconfigPath := "" + if container.InjectControlKubeconfig { + if container.SecurityContext == nil || container.SecurityContext.RunAsUser == nil { + return "", fmt.Errorf("bootstrap container %q injects the control kubeconfig but has no securityContext.runAsUser", container.Name) + } + controlKubeconfigPath = "/etc/sonic-operator/credentials/" + container.Name + "/control-kubeconfig" + } + + // A workload is best effort. Its failure must not prevent later + // containers from being attempted or obscure the already-applied boot + // lifecycle configuration above. + b.WriteString("if ! (\n") + if container.InjectControlKubeconfig { + b.WriteString("install -d -m 0700 /etc/sonic-operator/credentials &&\n") + b.WriteString("install -d -m 0700 ") + b.WriteString(shellQuote("/etc/sonic-operator/credentials/" + container.Name)) + b.WriteString(" &&\n") + b.WriteString("printf '%s' ") + b.WriteString(shellQuote(encodedControlKubeconfig)) + b.WriteString(" | base64 -d >") + b.WriteString(shellQuote(controlKubeconfigPath)) + b.WriteString(" &&\n") + b.WriteString("chown ") + b.WriteString(dockerUser) + b.WriteByte(' ') + b.WriteString(shellQuote(controlKubeconfigPath)) + b.WriteString(" &&\n") + b.WriteString("chmod 0600 ") + b.WriteString(shellQuote(controlKubeconfigPath)) + b.WriteString(" &&\n") + } + + b.WriteString("docker pull ") + b.WriteString(shellQuote(container.Image)) + b.WriteString(" &&\n") + b.WriteString("(docker rm -f ") + b.WriteString(shellQuote(container.Name)) + b.WriteString(" >/dev/null 2>&1 || true) &&\n") + b.WriteString("docker run -d --name ") + b.WriteString(shellQuote(container.Name)) + b.WriteString(" --network host --restart unless-stopped") + if dockerUser != "" { + b.WriteString(" --user ") + b.WriteString(shellQuote(dockerUser)) + } + if container.InjectControlKubeconfig { + b.WriteString(" -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig") + b.WriteString(" -v ") + b.WriteString(shellQuote(controlKubeconfigPath + ":/var/run/sonic-operator/control-kubeconfig:ro")) + } + b.WriteByte(' ') + b.WriteString(shellQuote(container.Image)) + for _, command := range container.Command { + b.WriteByte(' ') + b.WriteString(shellQuote(command)) + } + for _, arg := range container.Args { + b.WriteByte(' ') + b.WriteString(shellQuote(arg)) + } + b.WriteString("\n); then\n") + b.WriteString(" echo ") + b.WriteString(shellQuote("sonic-operator: bootstrap container " + container.Name + " failed; continuing")) + b.WriteString(" >&2\nfi\n") + } + } + + b.WriteString("sync\n") + return b.String(), nil +} + +func bootstrapContainerDockerUser(container networkingv1alpha1.BootstrapContainer) (string, error) { + if container.SecurityContext == nil { + return "", nil + } + + securityContext := container.SecurityContext + if securityContext.RunAsGroup != nil && securityContext.RunAsUser == nil { + return "", fmt.Errorf("bootstrap container %q sets securityContext.runAsGroup without securityContext.runAsUser", container.Name) + } + if securityContext.RunAsUser == nil { + return "", nil + } + + user := strconv.FormatInt(*securityContext.RunAsUser, 10) + if securityContext.RunAsGroup == nil { + return user, nil + } + return user + ":" + strconv.FormatInt(*securityContext.RunAsGroup, 10), nil +} + +func appendONIEInstallBootConfig(b *strings.Builder) { + b.WriteString(` +# Configure ONIE install discovery for the next and every subsequent SONiC reboot. +mkdir -p /onieboot +cat >/etc/systemd/system/onieboot.mount <<'EOF' +[Unit] +Description=ONIE boot partition + +[Mount] +What=LABEL=ONIE-BOOT +Where=/onieboot +Type=auto +Options=defaults + +[Install] +WantedBy=multi-user.target +EOF + +cat >/etc/systemd/system/sonic-operator-onie-install.service <<'EOF' +[Unit] +Description=Configure next boot for ONIE install discovery +Requires=onieboot.mount +After=onieboot.mount + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'grub-editenv /host/grub/grubenv set next_entry=ONIE' +ExecStart=/bin/sh -c 'grub-editenv /onieboot/grub/grubenv set onie_mode=install' + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable --now onieboot.mount +systemctl enable --now sonic-operator-onie-install.service +`) +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\\\"'\\\"'") + "'" } func handleErr(w http.ResponseWriter, e error) { diff --git a/internal/ztp/ztp_test.go b/internal/ztp/ztp_test.go index 864ac47..dc83b06 100644 --- a/internal/ztp/ztp_test.go +++ b/internal/ztp/ztp_test.go @@ -6,6 +6,10 @@ package ztp import ( "net/http" "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" "testing" corev1 "k8s.io/api/core/v1" @@ -28,10 +32,16 @@ func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( &networkingv1alpha1.Switch{ ObjectMeta: metav1.ObjectMeta{Name: "leaf-01"}, - Spec: networkingv1alpha1.SwitchSpec{ZTP: &networkingv1alpha1.ZTP{ - SourceAddress: "192.0.2.10", - ScriptRef: networkingv1alpha1.ZTPConfigMapReference{Namespace: "provisioning", Name: "leaf-01-ztp", Key: "ztp.sh"}, - }}, + Spec: networkingv1alpha1.SwitchSpec{ + ZTP: &networkingv1alpha1.ZTP{ + SourceAddress: "192.0.2.10", + ScriptRef: &networkingv1alpha1.ZTPConfigMapReference{Namespace: "provisioning", Name: "leaf-01-ztp", Key: "ztp.sh"}, + }, + Bootstrap: &networkingv1alpha1.Bootstrap{Containers: []networkingv1alpha1.BootstrapContainer{{ + Name: "ignored-in-configmap-mode", + Image: "example.invalid/ignored:latest", + }}}, + }, }, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: "provisioning", Name: "leaf-01-ztp"}, @@ -79,3 +89,129 @@ func TestConfigMapHandlerDoesNotFallBackForUnknownSwitch(t *testing.T) { t.Error("expected an error body for an unknown switch") } } + +func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := networkingv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + kubeconfigPath := filepath.Join(t.TempDir(), "kubeconfig") + if err := os.WriteFile(kubeconfigPath, []byte("apiVersion: v1\nkind: Config\n"), 0o600); err != nil { + t.Fatal(err) + } + wireletUID := int64(65532) + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "switch-1"}, + Spec: networkingv1alpha1.SwitchSpec{ + Hostname: "switch-1.lab.example", + ZTP: &networkingv1alpha1.ZTP{ + SourceAddress: "192.0.2.10", + }, + Bootstrap: &networkingv1alpha1.Bootstrap{Containers: []networkingv1alpha1.BootstrapContainer{{ + Name: "wirelet", + Image: "ghcr.io/hardikdr/wirelet:fixed-1", + SecurityContext: &networkingv1alpha1.BootstrapContainerSecurityContext{RunAsUser: &wireletUID, RunAsGroup: &wireletUID}, + Args: []string{"--name=switch-1", "--interface=Ethernet0"}, + InjectControlKubeconfig: true, + }}}, + NextBootMode: networkingv1alpha1.NextBootModeInstallOS, + }, + }, + ).Build() + + mux := http.NewServeMux() + RegisterGenerated(mux, c, GeneratedOptions{ControlKubeconfigFile: kubeconfigPath}) + req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) + req.RemoteAddr = "192.0.2.10:1234" + response := httptest.NewRecorder() + + mux.ServeHTTP(response, req) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) + } + body := response.Body.String() + for _, want := range []string{ + "#!/bin/bash", + "set -euo pipefail", + "config hostname 'switch-1.lab.example'", + "config save -y", + "printf '%s' 'YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCg==' | base64 -d >'/etc/sonic-operator/credentials/wirelet/control-kubeconfig'", + "chown 65532:65532 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig'", + "chmod 0600 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig'", + "docker pull 'ghcr.io/hardikdr/wirelet:fixed-1'", + "docker run -d --name 'wirelet' --network host --restart unless-stopped --user '65532:65532'", + "-e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig", + "-v '/etc/sonic-operator/credentials/wirelet/control-kubeconfig:/var/run/sonic-operator/control-kubeconfig:ro'", + "'--name=switch-1' '--interface=Ethernet0'", + "What=LABEL=ONIE-BOOT", + "sonic-operator-onie-install.service", + "set next_entry=ONIE", + "set onie_mode=install", + } { + if !strings.Contains(body, want) { + t.Errorf("response does not contain %q:\n%s", want, body) + } + } + if strings.Index(body, "# Configure ONIE install discovery") > strings.Index(body, "config hostname") { + t.Error("ONIE boot lifecycle configuration must be rendered before hostname configuration") + } + if !strings.Contains(body, "sonic-operator: bootstrap container wirelet failed; continuing") { + t.Errorf("response does not isolate bootstrap-container failures:\n%s", body) + } + + scriptPath := filepath.Join(t.TempDir(), "generated-ztp.sh") + if err := os.WriteFile(scriptPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if output, err := exec.Command("bash", "-n", scriptPath).CombinedOutput(); err != nil { + t.Fatalf("generated script is not valid Bash: %v: %s", err, output) + } +} + +func TestGeneratedHandlerRejectsMissingControlKubeconfig(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := networkingv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "switch-1"}, + Spec: networkingv1alpha1.SwitchSpec{ + ZTP: &networkingv1alpha1.ZTP{ + SourceAddress: "192.0.2.10", + }, + Bootstrap: &networkingv1alpha1.Bootstrap{Containers: []networkingv1alpha1.BootstrapContainer{{ + Name: "wirelet", + Image: "example.invalid/wirelet:latest", + InjectControlKubeconfig: true, + }}}, + }, + }, + ).Build() + + mux := http.NewServeMux() + RegisterGenerated(mux, c, GeneratedOptions{}) + req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) + req.RemoteAddr = "192.0.2.10:1234" + response := httptest.NewRecorder() + + mux.ServeHTTP(response, req) + + if response.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusInternalServerError, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "bootstrap-control-kubeconfig-file") { + t.Errorf("response = %q, want missing kubeconfig error", response.Body.String()) + } +} From f7d0a578a29e05ffe041324ae2e3d68fa3ea665c Mon Sep 17 00:00:00 2001 From: Hardik Dodiya Date: Mon, 7 Sep 2026 15:04:40 +0200 Subject: [PATCH 2/4] Support mounting volumes inside the boostrap containers of the Switch Signed-off-by: Hardik Dodiya --- api/v1alpha1/switch_types.go | 71 +++++-- api/v1alpha1/zz_generated.deepcopy.go | 117 +++++++---- cmd/main.go | 2 +- ...etworking.metal.ironcore.dev_switches.yaml | 182 +++++++++++------- .../samples/networking_v1alpha1_switch.yaml | 23 ++- docs/api-reference/api.md | 124 +++++++++++- docs/usage/provisioning.md | 54 ++++-- internal/ztp/ztp.go | 86 +++++++-- internal/ztp/ztp_test.go | 59 ++++-- 9 files changed, 549 insertions(+), 169 deletions(-) diff --git a/api/v1alpha1/switch_types.go b/api/v1alpha1/switch_types.go index ff7e9db..3a484eb 100644 --- a/api/v1alpha1/switch_types.go +++ b/api/v1alpha1/switch_types.go @@ -44,9 +44,9 @@ type ZTP struct { ScriptRef *ZTPConfigMapReference `json:"scriptRef,omitempty"` } -// BootstrapContainer declares a Docker container that the provisioning server -// starts on the switch during generated ZTP provisioning. -type BootstrapContainer struct { +// Container declares a Docker container that the provisioning server starts on +// the switch during generated ZTP provisioning. +type Container struct { // Name is both the Docker container name and its stable identity on the switch. // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` // +kubebuilder:validation:MaxLength=63 @@ -64,9 +64,14 @@ type BootstrapContainer struct { // +optional Args []string `json:"args,omitempty"` + // VolumeMounts describes the volumes mounted into the container. Each mount + // name must refer to an entry in SwitchSpec.Volumes. + // +optional + VolumeMounts []VolumeMount `json:"volumeMounts,omitempty"` + // SecurityContext configures the Unix identity used to run the container. // +optional - SecurityContext *BootstrapContainerSecurityContext `json:"securityContext,omitempty"` + SecurityContext *ContainerSecurityContext `json:"securityContext,omitempty"` // InjectControlKubeconfig mounts the operator's configured control kubeconfig // into this container and sets KUBECONFIG to its in-container path. A @@ -76,9 +81,9 @@ type BootstrapContainer struct { InjectControlKubeconfig bool `json:"injectControlKubeconfig,omitempty"` } -// BootstrapContainerSecurityContext is the supported subset of Kubernetes +// ContainerSecurityContext is the supported subset of Kubernetes // container securityContext for generated Docker containers. -type BootstrapContainerSecurityContext struct { +type ContainerSecurityContext struct { // RunAsUser is the numeric Unix user ID used by the container. // +optional // +kubebuilder:validation:Minimum=0 @@ -91,15 +96,38 @@ type BootstrapContainerSecurityContext struct { RunAsGroup *int64 `json:"runAsGroup,omitempty"` } -// Bootstrap declares generic containers installed by ZTP. It intentionally -// contains no workload-specific configuration or Secret references. -type Bootstrap struct { - // Containers are started with host networking and restart automatically on - // subsequent SONiC boots. +// Volume represents a named storage volume made available to Switch containers. +// It follows the Kubernetes Pod volume model. Generated ZTP currently supports +// hostPath volumes only. +type Volume struct { + // Name is the stable volume identity referenced by Container.VolumeMounts. + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // HostPath represents a pre-existing file or directory on the SONiC host. // +optional - // +listType=map - // +listMapKey=name - Containers []BootstrapContainer `json:"containers,omitempty"` + HostPath *HostPathVolumeSource `json:"hostPath,omitempty"` +} + +// HostPathVolumeSource represents a host directory or file mounted into a +// Switch container. +type HostPathVolumeSource struct { + // Path is the absolute path on the SONiC host. + Path string `json:"path"` + + // Type describes the expected host-path type, following the Kubernetes Pod + // hostPath API. Generated ZTP does not create missing paths. + // +optional + Type *v1.HostPathType `json:"type,omitempty"` +} + +// VolumeMount describes a volume mounted into a Switch container. It follows +// the Kubernetes Pod volumeMount API. +type VolumeMount struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly,omitempty"` } // NextBootMode describes the desired behavior of the switch's next boot. @@ -127,10 +155,19 @@ type SwitchSpec struct { // +optional ZTP *ZTP `json:"ztp,omitempty"` - // Bootstrap declares containers that are started on the switch by - // --ztp-mode=generated. + // Containers are started with host networking and Docker's unless-stopped + // restart policy by --ztp-mode=generated. + // +optional + // +listType=map + // +listMapKey=name + Containers []Container `json:"containers,omitempty"` + + // Volumes are named storage sources available to containers. Generated ZTP + // currently supports hostPath volumes only. // +optional - Bootstrap *Bootstrap `json:"bootstrap,omitempty"` + // +listType=map + // +listMapKey=name + Volumes []Volume `json:"volumes,omitempty"` // NextBootMode declares the desired behavior of the next boot. The default // is None. It is configured by --ztp-mode=generated. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 53377b9..0a5643c 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -14,29 +14,7 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Bootstrap) DeepCopyInto(out *Bootstrap) { - *out = *in - if in.Containers != nil { - in, out := &in.Containers, &out.Containers - *out = make([]BootstrapContainer, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Bootstrap. -func (in *Bootstrap) DeepCopy() *Bootstrap { - if in == nil { - return nil - } - out := new(Bootstrap) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapContainer) DeepCopyInto(out *BootstrapContainer) { +func (in *Container) DeepCopyInto(out *Container) { *out = *in if in.Command != nil { in, out := &in.Command, &out.Command @@ -48,25 +26,30 @@ func (in *BootstrapContainer) DeepCopyInto(out *BootstrapContainer) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]VolumeMount, len(*in)) + copy(*out, *in) + } if in.SecurityContext != nil { in, out := &in.SecurityContext, &out.SecurityContext - *out = new(BootstrapContainerSecurityContext) + *out = new(ContainerSecurityContext) (*in).DeepCopyInto(*out) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapContainer. -func (in *BootstrapContainer) DeepCopy() *BootstrapContainer { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container. +func (in *Container) DeepCopy() *Container { if in == nil { return nil } - out := new(BootstrapContainer) + out := new(Container) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *BootstrapContainerSecurityContext) DeepCopyInto(out *BootstrapContainerSecurityContext) { +func (in *ContainerSecurityContext) DeepCopyInto(out *ContainerSecurityContext) { *out = *in if in.RunAsUser != nil { in, out := &in.RunAsUser, &out.RunAsUser @@ -80,12 +63,32 @@ func (in *BootstrapContainerSecurityContext) DeepCopyInto(out *BootstrapContaine } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BootstrapContainerSecurityContext. -func (in *BootstrapContainerSecurityContext) DeepCopy() *BootstrapContainerSecurityContext { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerSecurityContext. +func (in *ContainerSecurityContext) DeepCopy() *ContainerSecurityContext { + if in == nil { + return nil + } + out := new(ContainerSecurityContext) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostPathVolumeSource) DeepCopyInto(out *HostPathVolumeSource) { + *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + *out = new(v1.HostPathType) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathVolumeSource. +func (in *HostPathVolumeSource) DeepCopy() *HostPathVolumeSource { if in == nil { return nil } - out := new(BootstrapContainerSecurityContext) + out := new(HostPathVolumeSource) in.DeepCopyInto(out) return out } @@ -411,10 +414,19 @@ func (in *SwitchSpec) DeepCopyInto(out *SwitchSpec) { *out = new(ZTP) (*in).DeepCopyInto(*out) } - if in.Bootstrap != nil { - in, out := &in.Bootstrap, &out.Bootstrap - *out = new(Bootstrap) - (*in).DeepCopyInto(*out) + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Ports != nil { in, out := &in.Ports, &out.Ports @@ -462,6 +474,41 @@ func (in *SwitchStatus) DeepCopy() *SwitchStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Volume) DeepCopyInto(out *Volume) { + *out = *in + if in.HostPath != nil { + in, out := &in.HostPath, &out.HostPath + *out = new(HostPathVolumeSource) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Volume. +func (in *Volume) DeepCopy() *Volume { + if in == nil { + return nil + } + out := new(Volume) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeMount) DeepCopyInto(out *VolumeMount) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMount. +func (in *VolumeMount) DeepCopy() *VolumeMount { + if in == nil { + return nil + } + out := new(VolumeMount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ZTP) DeepCopyInto(out *ZTP) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index 5db0fe6..20d54e6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -77,7 +77,7 @@ func main() { flag.StringVar(&httpServerAddr, "http-server-address", "0", "The address the HTTP server for ZTP and ONIE binds to.") flag.StringVar(&ztpConfigFile, "ztp-config-file", "/etc/ztp.json", "Config file containing the parameters to render ZTP scripts.") flag.StringVar(&ztpMode, "ztp-mode", "templates", "ZTP source: templates, configmap, or generated. Configmap mode serves the referenced script verbatim; generated mode renders from Switch objects.") - flag.StringVar(&bootstrapControlKubeconfigFile, "bootstrap-control-kubeconfig-file", "", "Optional kubeconfig file injected into bootstrap containers that set injectControlKubeconfig=true.") + flag.StringVar(&bootstrapControlKubeconfigFile, "bootstrap-control-kubeconfig-file", "", "Optional kubeconfig file injected into generated containers that set injectControlKubeconfig=true.") flag.StringVar(&onieImagesDir, "onie-images-dir", "/var/lib/sonic-operator/onie", "The directory which contains the ONIE and SONiC installer image files.") flag.StringVar(&onieConfigFile, "onie-config-file", "/etc/onie.json", "Config file containing machine-to-image mappings for ONIE provisioning.") flag.BoolVar(&disableProvisionsingServer, "disable-static-config", false, "If set, the HTTP server for ZTP and ONIE will not be started.") diff --git a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml index 3312e7b..8ef6518 100644 --- a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml +++ b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml @@ -49,76 +49,90 @@ spec: spec: description: spec defines the desired state of Switch properties: - bootstrap: + containers: description: |- - Bootstrap declares containers that are started on the switch by - --ztp-mode=generated. - properties: - containers: - description: |- - Containers are started with host networking and restart automatically on - subsequent SONiC boots. - items: + Containers are started with host networking and Docker's unless-stopped + restart policy by --ztp-mode=generated. + items: + description: |- + Container declares a Docker container that the provisioning server starts on + the switch during generated ZTP provisioning. + properties: + args: + description: Args are appended after Command, or after the image + entrypoint when Command is empty. + items: + type: string + type: array + command: + description: Command overrides the image entrypoint. + items: + type: string + type: array + image: + description: Image is the container image pulled by Docker on + the switch. + minLength: 1 + type: string + injectControlKubeconfig: description: |- - BootstrapContainer declares a Docker container that the provisioning server - starts on the switch during generated ZTP provisioning. + InjectControlKubeconfig mounts the operator's configured control kubeconfig + into this container and sets KUBECONFIG to its in-container path. A + securityContext with runAsUser is required so the generated script can + grant access to a private, per-container credential file. + type: boolean + name: + description: Name is both the Docker container name and its + stable identity on the switch. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + securityContext: + description: SecurityContext configures the Unix identity used + to run the container. properties: - args: - description: Args are appended after Command, or after the - image entrypoint when Command is empty. - items: - type: string - type: array - command: - description: Command overrides the image entrypoint. - items: - type: string - type: array - image: - description: Image is the container image pulled by Docker - on the switch. - minLength: 1 - type: string - injectControlKubeconfig: + runAsGroup: description: |- - InjectControlKubeconfig mounts the operator's configured control kubeconfig - into this container and sets KUBECONFIG to its in-container path. A - securityContext with runAsUser is required so the generated script can - grant access to a private, per-container credential file. - type: boolean - name: - description: Name is both the Docker container name and - its stable identity on the switch. - maxLength: 63 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - securityContext: - description: SecurityContext configures the Unix identity - used to run the container. - properties: - runAsGroup: - description: |- - RunAsGroup is the numeric Unix group ID used by the container. It requires - runAsUser to be set as well. - format: int64 - minimum: 0 - type: integer - runAsUser: - description: RunAsUser is the numeric Unix user ID used - by the container. - format: int64 - minimum: 0 - type: integer - type: object - required: - - image - - name + RunAsGroup is the numeric Unix group ID used by the container. It requires + runAsUser to be set as well. + format: int64 + minimum: 0 + type: integer + runAsUser: + description: RunAsUser is the numeric Unix user ID used + by the container. + format: int64 + minimum: 0 + type: integer type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - type: object + volumeMounts: + description: |- + VolumeMounts describes the volumes mounted into the container. Each mount + name must refer to an entry in SwitchSpec.Volumes. + items: + description: |- + VolumeMount describes a volume mounted into a Switch container. It follows + the Kubernetes Pod volumeMount API. + properties: + mountPath: + type: string + name: + type: string + readOnly: + type: boolean + required: + - mountPath + - name + type: object + type: array + required: + - image + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map hostname: description: |- Hostname is configured on the switch by --ztp-mode=generated. If omitted, @@ -200,6 +214,44 @@ spec: - name type: object type: array + volumes: + description: |- + Volumes are named storage sources available to containers. Generated ZTP + currently supports hostPath volumes only. + items: + description: |- + Volume represents a named storage volume made available to Switch containers. + It follows the Kubernetes Pod volume model. Generated ZTP currently supports + hostPath volumes only. + properties: + hostPath: + description: HostPath represents a pre-existing file or directory + on the SONiC host. + properties: + path: + description: Path is the absolute path on the SONiC host. + type: string + type: + description: |- + Type describes the expected host-path type, following the Kubernetes Pod + hostPath API. Generated ZTP does not create missing paths. + type: string + required: + - path + type: object + name: + description: Name is the stable volume identity referenced by + Container.VolumeMounts. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map ztp: description: ZTP identifies the switch while it requests its initial provisioning script. diff --git a/config/samples/networking_v1alpha1_switch.yaml b/config/samples/networking_v1alpha1_switch.yaml index 4b834e3..68283a3 100644 --- a/config/samples/networking_v1alpha1_switch.yaml +++ b/config/samples/networking_v1alpha1_switch.yaml @@ -13,20 +13,19 @@ spec: nextBootMode: InstallOS - # Bootstrap containers are rendered by --ztp-mode=generated. + # Containers are rendered by --ztp-mode=generated. # The control kubeconfig is provided globally by the operator, not by this # Switch object. - bootstrap: - containers: - - name: wirelet - image: ghcr.io/hardikdr/wirelet:fixed-1 - securityContext: - runAsUser: 65532 - runAsGroup: 65532 - args: - - --name=switch-sample - - --interface=Ethernet0 - injectControlKubeconfig: true + containers: + - name: wirelet + image: ghcr.io/hardikdr/wirelet:fixed-1 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + args: + - --name=switch-sample + - --interface=Ethernet0 + injectControlKubeconfig: true management: host: 192.0.2.10 diff --git a/docs/api-reference/api.md b/docs/api-reference/api.md index e47f177..293912b 100644 --- a/docs/api-reference/api.md +++ b/docs/api-reference/api.md @@ -36,6 +36,65 @@ _Appears in:_ | `Down` | | +#### Container + + + +Container declares a Docker container that the provisioning server starts on +the switch during generated ZTP provisioning. + + + +_Appears in:_ +- [SwitchSpec](#switchspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is both the Docker container name and its stable identity on the switch. | | MaxLength: 63
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
| +| `image` _string_ | Image is the container image pulled by Docker on the switch. | | MinLength: 1
| +| `command` _string array_ | Command overrides the image entrypoint. | | | +| `args` _string array_ | Args are appended after Command, or after the image entrypoint when Command is empty. | | | +| `volumeMounts` _[VolumeMount](#volumemount) array_ | VolumeMounts describes the volumes mounted into the container. Each mount
name must refer to an entry in SwitchSpec.Volumes. | | | +| `securityContext` _[ContainerSecurityContext](#containersecuritycontext)_ | SecurityContext configures the Unix identity used to run the container. | | | +| `injectControlKubeconfig` _boolean_ | InjectControlKubeconfig mounts the operator's configured control kubeconfig
into this container and sets KUBECONFIG to its in-container path. A
securityContext with runAsUser is required so the generated script can
grant access to a private, per-container credential file. | | | + + +#### ContainerSecurityContext + + + +ContainerSecurityContext is the supported subset of Kubernetes +container securityContext for generated Docker containers. + + + +_Appears in:_ +- [Container](#container) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `runAsUser` _integer_ | RunAsUser is the numeric Unix user ID used by the container. | | Minimum: 0
| +| `runAsGroup` _integer_ | RunAsGroup is the numeric Unix group ID used by the container. It requires
runAsUser to be set as well. | | Minimum: 0
| + + +#### HostPathVolumeSource + + + +HostPathVolumeSource represents a host directory or file mounted into a +Switch container. + + + +_Appears in:_ +- [Volume](#volume) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `path` _string_ | Path is the absolute path on the SONiC host. | | | +| `type` _[HostPathType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#hostpathtype-v1-core)_ | Type describes the expected host-path type, following the Kubernetes Pod
hostPath API. Generated ZTP does not create missing paths. | | | + + #### Management @@ -72,6 +131,25 @@ _Appears in:_ | `interfaceHandle` _string_ | InterfaceHandle is the name of the remote switch interface. | | | +#### NextBootMode + +_Underlying type:_ _string_ + +NextBootMode describes the desired behavior of the switch's next boot. +Providers translate this high-level intent to their platform-specific boot +mechanism. For SONiC, InstallOS enters ONIE install discovery. + + + +_Appears in:_ +- [SwitchSpec](#switchspec) + +| Field | Description | +| --- | --- | +| `None` | NextBootModeNone leaves the normal installed network OS boot path intact.
| +| `InstallOS` | NextBootModeInstallOS enters the platform's OS installation/discovery flow.
| + + #### OperationState _Underlying type:_ _string_ @@ -255,8 +333,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | +| `hostname` _string_ | Hostname is configured on the switch by --ztp-mode=generated. If omitted,
the generated script uses the Switch object name. | | | | `management` _[Management](#management)_ | | | | -| `ztp` _[ZTP](#ztp)_ | ZTP selects a custom script when the operator runs with --ztp-mode=configmap. | | | +| `ztp` _[ZTP](#ztp)_ | ZTP identifies the switch while it requests its initial provisioning script. | | | +| `containers` _[Container](#container) array_ | Containers are started with host networking and Docker's unless-stopped
restart policy by --ztp-mode=generated. | | | +| `volumes` _[Volume](#volume) array_ | Volumes are named storage sources available to containers. Generated ZTP
currently supports hostPath volumes only. | | | +| `nextBootMode` _[NextBootMode](#nextbootmode)_ | NextBootMode declares the desired behavior of the next boot. The default
is None. It is configured by --ztp-mode=generated. | | Enum: [None InstallOS]
| | `macAddress` _string_ | MacAddress is the MAC address assigned to this interface. | | | | `ports` _[PortSpec](#portspec) array_ | Ports the physical ports available on the Switch. | | | @@ -300,6 +382,44 @@ _Appears in:_ | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array_ | The status of each condition is one of True, False, or Unknown. | | | +#### Volume + + + +Volume represents a named storage volume made available to Switch containers. +It follows the Kubernetes Pod volume model. Generated ZTP currently supports +hostPath volumes only. + + + +_Appears in:_ +- [SwitchSpec](#switchspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the stable volume identity referenced by Container.VolumeMounts. | | MaxLength: 63
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
| +| `hostPath` _[HostPathVolumeSource](#hostpathvolumesource)_ | HostPath represents a pre-existing file or directory on the SONiC host. | | | + + +#### VolumeMount + + + +VolumeMount describes a volume mounted into a Switch container. It follows +the Kubernetes Pod volumeMount API. + + + +_Appears in:_ +- [Container](#container) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | | | | +| `mountPath` _string_ | | | | +| `readOnly` _boolean_ | | | | + + #### ZTP @@ -316,7 +436,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `sourceAddress` _string_ | | | | -| `scriptRef` _[ZTPConfigMapReference](#ztpconfigmapreference)_ | | | | +| `scriptRef` _[ZTPConfigMapReference](#ztpconfigmapreference)_ | ScriptRef identifies the complete script served when --ztp-mode=configmap.
It is not used by --ztp-mode=generated. | | | #### ZTPConfigMapReference diff --git a/docs/usage/provisioning.md b/docs/usage/provisioning.md index d7f0804..8587066 100644 --- a/docs/usage/provisioning.md +++ b/docs/usage/provisioning.md @@ -6,7 +6,7 @@ The provisioning server serves ZTP scripts and ONIE installer artifacts over HTT - `--http-server-address`: bind address for the provisioning server. - `--ztp-config-file`: JSON file with ZTP parameters (default `/etc/ztp.json`). - `--ztp-mode`: ZTP source: `templates` (default), `configmap`, or `generated`. -- `--bootstrap-control-kubeconfig-file`: optional control kubeconfig mounted into the operator and made available to opted-in generated bootstrap containers. +- `--bootstrap-control-kubeconfig-file`: optional control kubeconfig mounted into the operator and made available to opted-in generated containers. - `--onie-installer-dir`: directory containing ONIE installer files (default `/var/lib/sonic-operator/onie`). ## ZTP @@ -62,26 +62,54 @@ spec: hostname: switch-1 ztp: sourceAddress: "2001:db8:100::11" - bootstrap: - containers: - - name: wirelet - image: ghcr.io/ironcore-dev/wirelet:fixed-1 - securityContext: - runAsUser: 65532 - runAsGroup: 65532 - args: - - --name=switch-1 - - --interface=Ethernet0 - injectControlKubeconfig: true + containers: + - name: wirelet + image: ghcr.io/ironcore-dev/wirelet:fixed-1 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + args: + - --name=switch-1 + - --interface=Ethernet0 + injectControlKubeconfig: true ``` Each container is pulled and started with host networking and Docker's `unless-stopped` restart policy. `command` overrides the image entrypoint; `args` are appended after `command` or the image entrypoint. `hostname` -defaults to the `Switch` object name when omitted. Bootstrap containers are +defaults to the `Switch` object name when omitted. Containers are best effort: a failure to prepare or start one is logged and does not prevent other containers from being attempted. +### Volumes + +Generated ZTP follows the Kubernetes Pod volume model: define named volumes in +`spec.volumes`, then refer to them from a container's `volumeMounts`. The +current generated Docker runtime supports `hostPath` volumes only: + +```yaml +spec: + volumes: + - name: dbus + hostPath: + path: /var/run/dbus + type: Directory + containers: + - name: sonic-agent + image: ghcr.io/giluerre/sonic-agent:latest + command: ["/switch-agent-server"] + args: ["-port", "57400"] + securityContext: + runAsUser: 0 + volumeMounts: + - name: dbus + mountPath: /var/run/dbus +``` + +This renders a Docker mount before the image, for example +`-v /var/run/dbus:/var/run/dbus:rw`. The volume and mount names must match; +host paths and mount paths must be absolute. + ### ONIE boot discovery Generated ZTP can persistently configure the next SONiC reboot to enter ONIE diff --git a/internal/ztp/ztp.go b/internal/ztp/ztp.go index 9672fd1..2d4ab3d 100644 --- a/internal/ztp/ztp.go +++ b/internal/ztp/ztp.go @@ -91,7 +91,7 @@ type generatedHandler struct { // GeneratedOptions configures declarative, Switch-backed ZTP rendering. type GeneratedOptions struct { // ControlKubeconfigFile is an optional kubeconfig mounted into the operator - // pod. It is injected only into generated bootstrap containers that + // pod. It is injected only into generated containers that // explicitly opt in. ControlKubeconfigFile string } @@ -282,8 +282,12 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec b.WriteString("\n") b.WriteString("config save -y\n") - if switchObject.Spec.Bootstrap != nil && len(switchObject.Spec.Bootstrap.Containers) > 0 { - containers := switchObject.Spec.Bootstrap.Containers + if len(switchObject.Spec.Containers) > 0 { + containers := switchObject.Spec.Containers + volumes, err := hostPathVolumes(switchObject.Spec.Volumes) + if err != nil { + return "", err + } needsControlKubeconfig := false for _, container := range containers { if container.InjectControlKubeconfig { @@ -292,7 +296,7 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec } } - b.WriteString("\n# sonic-operator bootstrap containers\n") + b.WriteString("\n# sonic-operator containers\n") var encodedControlKubeconfig string if needsControlKubeconfig { @@ -308,7 +312,11 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec } for _, container := range containers { - dockerUser, err := bootstrapContainerDockerUser(container) + dockerUser, err := containerDockerUser(container) + if err != nil { + return "", err + } + volumeMounts, err := containerVolumeMounts(container, volumes) if err != nil { return "", err } @@ -316,7 +324,7 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec controlKubeconfigPath := "" if container.InjectControlKubeconfig { if container.SecurityContext == nil || container.SecurityContext.RunAsUser == nil { - return "", fmt.Errorf("bootstrap container %q injects the control kubeconfig but has no securityContext.runAsUser", container.Name) + return "", fmt.Errorf("container %q injects the control kubeconfig but has no securityContext.runAsUser", container.Name) } controlKubeconfigPath = "/etc/sonic-operator/credentials/" + container.Name + "/control-kubeconfig" } @@ -358,16 +366,26 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec b.WriteString(" --user ") b.WriteString(shellQuote(dockerUser)) } + for _, volumeMount := range volumeMounts { + b.WriteString(" -v ") + b.WriteString(shellQuote(volumeMount)) + } if container.InjectControlKubeconfig { b.WriteString(" -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig") b.WriteString(" -v ") b.WriteString(shellQuote(controlKubeconfigPath + ":/var/run/sonic-operator/control-kubeconfig:ro")) } + if len(container.Command) > 0 { + b.WriteString(" --entrypoint ") + b.WriteString(shellQuote(container.Command[0])) + } b.WriteByte(' ') b.WriteString(shellQuote(container.Image)) - for _, command := range container.Command { - b.WriteByte(' ') - b.WriteString(shellQuote(command)) + if len(container.Command) > 1 { + for _, command := range container.Command[1:] { + b.WriteByte(' ') + b.WriteString(shellQuote(command)) + } } for _, arg := range container.Args { b.WriteByte(' ') @@ -375,7 +393,7 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec } b.WriteString("\n); then\n") b.WriteString(" echo ") - b.WriteString(shellQuote("sonic-operator: bootstrap container " + container.Name + " failed; continuing")) + b.WriteString(shellQuote("sonic-operator: container " + container.Name + " failed; continuing")) b.WriteString(" >&2\nfi\n") } } @@ -384,14 +402,14 @@ func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubec return b.String(), nil } -func bootstrapContainerDockerUser(container networkingv1alpha1.BootstrapContainer) (string, error) { +func containerDockerUser(container networkingv1alpha1.Container) (string, error) { if container.SecurityContext == nil { return "", nil } securityContext := container.SecurityContext if securityContext.RunAsGroup != nil && securityContext.RunAsUser == nil { - return "", fmt.Errorf("bootstrap container %q sets securityContext.runAsGroup without securityContext.runAsUser", container.Name) + return "", fmt.Errorf("container %q sets securityContext.runAsGroup without securityContext.runAsUser", container.Name) } if securityContext.RunAsUser == nil { return "", nil @@ -404,6 +422,50 @@ func bootstrapContainerDockerUser(container networkingv1alpha1.BootstrapContaine return user + ":" + strconv.FormatInt(*securityContext.RunAsGroup, 10), nil } +func hostPathVolumes(volumes []networkingv1alpha1.Volume) (map[string]string, error) { + result := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if volume.Name == "" { + return nil, fmt.Errorf("volume name is empty") + } + if _, exists := result[volume.Name]; exists { + return nil, fmt.Errorf("duplicate volume %q", volume.Name) + } + if volume.HostPath == nil { + return nil, fmt.Errorf("volume %q must specify hostPath", volume.Name) + } + if !strings.HasPrefix(volume.HostPath.Path, "/") { + return nil, fmt.Errorf("volume %q hostPath %q must be absolute", volume.Name, volume.HostPath.Path) + } + result[volume.Name] = volume.HostPath.Path + } + return result, nil +} + +func containerVolumeMounts(container networkingv1alpha1.Container, volumes map[string]string) ([]string, error) { + result := make([]string, 0, len(container.VolumeMounts)) + mountPaths := make(map[string]struct{}, len(container.VolumeMounts)) + for _, mount := range container.VolumeMounts { + hostPath, exists := volumes[mount.Name] + if !exists { + return nil, fmt.Errorf("container %q mounts unknown volume %q", container.Name, mount.Name) + } + if !strings.HasPrefix(mount.MountPath, "/") { + return nil, fmt.Errorf("container %q mountPath %q must be absolute", container.Name, mount.MountPath) + } + if _, exists := mountPaths[mount.MountPath]; exists { + return nil, fmt.Errorf("container %q mounts multiple volumes at %q", container.Name, mount.MountPath) + } + mountPaths[mount.MountPath] = struct{}{} + mode := "rw" + if mount.ReadOnly { + mode = "ro" + } + result = append(result, hostPath+":"+mount.MountPath+":"+mode) + } + return result, nil +} + func appendONIEInstallBootConfig(b *strings.Builder) { b.WriteString(` # Configure ONIE install discovery for the next and every subsequent SONiC reboot. diff --git a/internal/ztp/ztp_test.go b/internal/ztp/ztp_test.go index dc83b06..b6be6e3 100644 --- a/internal/ztp/ztp_test.go +++ b/internal/ztp/ztp_test.go @@ -37,10 +37,10 @@ func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { SourceAddress: "192.0.2.10", ScriptRef: &networkingv1alpha1.ZTPConfigMapReference{Namespace: "provisioning", Name: "leaf-01-ztp", Key: "ztp.sh"}, }, - Bootstrap: &networkingv1alpha1.Bootstrap{Containers: []networkingv1alpha1.BootstrapContainer{{ + Containers: []networkingv1alpha1.Container{{ Name: "ignored-in-configmap-mode", Image: "example.invalid/ignored:latest", - }}}, + }}, }, }, &corev1.ConfigMap{ @@ -104,6 +104,7 @@ func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { t.Fatal(err) } wireletUID := int64(65532) + agentUID := int64(0) c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( &networkingv1alpha1.Switch{ @@ -113,13 +114,32 @@ func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { ZTP: &networkingv1alpha1.ZTP{ SourceAddress: "192.0.2.10", }, - Bootstrap: &networkingv1alpha1.Bootstrap{Containers: []networkingv1alpha1.BootstrapContainer{{ - Name: "wirelet", - Image: "ghcr.io/hardikdr/wirelet:fixed-1", - SecurityContext: &networkingv1alpha1.BootstrapContainerSecurityContext{RunAsUser: &wireletUID, RunAsGroup: &wireletUID}, - Args: []string{"--name=switch-1", "--interface=Ethernet0"}, - InjectControlKubeconfig: true, - }}}, + Volumes: []networkingv1alpha1.Volume{ + {Name: "dbus", HostPath: &networkingv1alpha1.HostPathVolumeSource{Path: "/var/run/dbus"}}, + {Name: "sonic-version", HostPath: &networkingv1alpha1.HostPathVolumeSource{Path: "/etc/sonic/sonic_version.yml"}}, + }, + Containers: []networkingv1alpha1.Container{ + { + Name: "wirelet", + Image: "ghcr.io/hardikdr/wirelet:fixed-1", + SecurityContext: &networkingv1alpha1.ContainerSecurityContext{RunAsUser: &wireletUID, RunAsGroup: &wireletUID}, + Args: []string{"--name=switch-1", "--interface=Ethernet0"}, + InjectControlKubeconfig: true, + }, + { + Name: "sonic-agent", + Image: "ghcr.io/giluerre/sonic-agent:latest", + Command: []string{"/switch-agent-server"}, + Args: []string{"-port", "57400"}, + SecurityContext: &networkingv1alpha1.ContainerSecurityContext{ + RunAsUser: &agentUID, + }, + VolumeMounts: []networkingv1alpha1.VolumeMount{ + {Name: "dbus", MountPath: "/var/run/dbus"}, + {Name: "sonic-version", MountPath: "/etc/sonic/sonic_version.yml", ReadOnly: true}, + }, + }, + }, NextBootMode: networkingv1alpha1.NextBootModeInstallOS, }, }, @@ -150,6 +170,7 @@ func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { "-e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig", "-v '/etc/sonic-operator/credentials/wirelet/control-kubeconfig:/var/run/sonic-operator/control-kubeconfig:ro'", "'--name=switch-1' '--interface=Ethernet0'", + "docker run -d --name 'sonic-agent' --network host --restart unless-stopped --user '0' -v '/var/run/dbus:/var/run/dbus:rw' -v '/etc/sonic/sonic_version.yml:/etc/sonic/sonic_version.yml:ro' --entrypoint '/switch-agent-server' 'ghcr.io/giluerre/sonic-agent:latest' '-port' '57400'", "What=LABEL=ONIE-BOOT", "sonic-operator-onie-install.service", "set next_entry=ONIE", @@ -162,7 +183,7 @@ func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { if strings.Index(body, "# Configure ONIE install discovery") > strings.Index(body, "config hostname") { t.Error("ONIE boot lifecycle configuration must be rendered before hostname configuration") } - if !strings.Contains(body, "sonic-operator: bootstrap container wirelet failed; continuing") { + if !strings.Contains(body, "sonic-operator: container wirelet failed; continuing") { t.Errorf("response does not isolate bootstrap-container failures:\n%s", body) } @@ -191,11 +212,11 @@ func TestGeneratedHandlerRejectsMissingControlKubeconfig(t *testing.T) { ZTP: &networkingv1alpha1.ZTP{ SourceAddress: "192.0.2.10", }, - Bootstrap: &networkingv1alpha1.Bootstrap{Containers: []networkingv1alpha1.BootstrapContainer{{ + Containers: []networkingv1alpha1.Container{{ Name: "wirelet", Image: "example.invalid/wirelet:latest", InjectControlKubeconfig: true, - }}}, + }}, }, }, ).Build() @@ -215,3 +236,17 @@ func TestGeneratedHandlerRejectsMissingControlKubeconfig(t *testing.T) { t.Errorf("response = %q, want missing kubeconfig error", response.Body.String()) } } + +func TestRenderGeneratedScriptDefaultsHostnameToSwitchName(t *testing.T) { + switchObject := &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "leaf-1"}, + } + + script, err := renderGeneratedScript(switchObject, "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(script, "config hostname 'leaf-1'") { + t.Errorf("generated script does not default hostname to switch name:\n%s", script) + } +} From 1e0b1d1b65dc3b50326dc1d3374e51282b4023fa Mon Sep 17 00:00:00 2001 From: Hardik Dodiya Date: Tue, 15 Sep 2026 14:18:02 +0200 Subject: [PATCH 3/4] Use go-templates for better handling of the script Signed-off-by: Hardik Dodiya --- REUSE.toml | 1 + internal/ztp/generated_mode.go | 212 ++++++++++++++ .../ztp/templates/generated-mode.sh.gotmpl | 57 ++++ .../ztp/testdata/generated-mode.golden.sh | 64 +++++ internal/ztp/ztp.go | 269 +----------------- internal/ztp/ztp_test.go | 15 +- 6 files changed, 351 insertions(+), 267 deletions(-) create mode 100644 internal/ztp/generated_mode.go create mode 100644 internal/ztp/templates/generated-mode.sh.gotmpl create mode 100644 internal/ztp/testdata/generated-mode.golden.sh diff --git a/REUSE.toml b/REUSE.toml index 9a67a48..eee36b6 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -19,6 +19,7 @@ SPDX-PackageDownloadLocation = "https://github.com/ironcore-dev/sonic-operator" "package-lock.json", "package.json", "internal/ztp/templates/**", + "internal/ztp/testdata/**", "REUSE.toml", "hack/**", ".devcontainer/**", diff --git a/internal/ztp/generated_mode.go b/internal/ztp/generated_mode.go new file mode 100644 index 0000000..2dd6e80 --- /dev/null +++ b/internal/ztp/generated_mode.go @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package ztp + +import ( + "bytes" + "encoding/base64" + "fmt" + "os" + "strconv" + "strings" + "text/template" + + networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" +) + +var generatedModeTemplate = template.Must( + template.New("generated-mode.sh.gotmpl"). + Funcs(template.FuncMap{"shellQuote": shellQuote}). + ParseFS(templateFS, "templates/generated-mode.sh.gotmpl"), +) + +type generatedModeTemplateData struct { + SwitchName string + Hostname string + ConfigureONIE bool + Containers []generatedModeContainerData +} + +type generatedModeContainerData struct { + Name string + Image string + DockerUser string + VolumeMounts []string + InjectControlKubeconfig bool + EncodedControlKubeconfig string + ControlKubeconfigPath string + Entrypoint string + Command []string + Args []string + FailureMessage string +} + +func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubeconfigFilePath string) (string, error) { + data, err := buildGeneratedModeTemplateData(switchObject, controlKubeconfigFilePath) + if err != nil { + return "", err + } + + var output bytes.Buffer + if err := generatedModeTemplate.ExecuteTemplate(&output, "generated-mode.sh.gotmpl", data); err != nil { + return "", fmt.Errorf("executing generated mode template: %w", err) + } + return output.String(), nil +} + +func buildGeneratedModeTemplateData( + switchObject *networkingv1alpha1.Switch, + controlKubeconfigFilePath string, +) (*generatedModeTemplateData, error) { + hostname := switchObject.Spec.Hostname + if hostname == "" { + hostname = switchObject.Name + } + if hostname == "" { + return nil, fmt.Errorf("hostname is empty") + } + + data := &generatedModeTemplateData{ + SwitchName: switchObject.Name, + Hostname: hostname, + Containers: make([]generatedModeContainerData, 0, len(switchObject.Spec.Containers)), + } + + // Boot lifecycle intent is strict and appears before fallible switch or + // workload configuration in the rendered script. + switch switchObject.Spec.NextBootMode { + case "", networkingv1alpha1.NextBootModeNone: + case networkingv1alpha1.NextBootModeInstallOS: + data.ConfigureONIE = true + default: + return nil, fmt.Errorf("unsupported next boot mode %q", switchObject.Spec.NextBootMode) + } + + var volumes map[string]string + if len(switchObject.Spec.Containers) > 0 { + var err error + volumes, err = hostPathVolumes(switchObject.Spec.Volumes) + if err != nil { + return nil, err + } + } + + var encodedControlKubeconfig string + for _, container := range switchObject.Spec.Containers { + if !container.InjectControlKubeconfig || encodedControlKubeconfig != "" { + continue + } + if controlKubeconfigFilePath == "" { + return nil, fmt.Errorf("injectControlKubeconfig requires --bootstrap-control-kubeconfig-file") + } + kubeconfig, err := os.ReadFile(controlKubeconfigFilePath) + if err != nil { + return nil, fmt.Errorf("reading control kubeconfig file %q: %w", controlKubeconfigFilePath, err) + } + encodedControlKubeconfig = base64.StdEncoding.EncodeToString(kubeconfig) + } + + for _, container := range switchObject.Spec.Containers { + dockerUser, err := containerDockerUser(container) + if err != nil { + return nil, err + } + volumeMounts, err := containerVolumeMounts(container, volumes) + if err != nil { + return nil, err + } + + containerData := generatedModeContainerData{ + Name: container.Name, + Image: container.Image, + DockerUser: dockerUser, + VolumeMounts: volumeMounts, + InjectControlKubeconfig: container.InjectControlKubeconfig, + EncodedControlKubeconfig: encodedControlKubeconfig, + Args: container.Args, + FailureMessage: "sonic-operator: container " + container.Name + " failed; continuing", + } + if len(container.Command) > 0 { + containerData.Entrypoint = container.Command[0] + containerData.Command = container.Command[1:] + } + if container.InjectControlKubeconfig { + if container.SecurityContext == nil || container.SecurityContext.RunAsUser == nil { + return nil, fmt.Errorf("container %q injects the control kubeconfig but has no securityContext.runAsUser", container.Name) + } + containerData.ControlKubeconfigPath = "/etc/sonic-operator/credentials/" + container.Name + "/control-kubeconfig" + } + data.Containers = append(data.Containers, containerData) + } + + return data, nil +} + +func containerDockerUser(container networkingv1alpha1.Container) (string, error) { + if container.SecurityContext == nil { + return "", nil + } + + securityContext := container.SecurityContext + if securityContext.RunAsGroup != nil && securityContext.RunAsUser == nil { + return "", fmt.Errorf("container %q sets securityContext.runAsGroup without securityContext.runAsUser", container.Name) + } + if securityContext.RunAsUser == nil { + return "", nil + } + + user := strconv.FormatInt(*securityContext.RunAsUser, 10) + if securityContext.RunAsGroup == nil { + return user, nil + } + return user + ":" + strconv.FormatInt(*securityContext.RunAsGroup, 10), nil +} + +func hostPathVolumes(volumes []networkingv1alpha1.Volume) (map[string]string, error) { + result := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if volume.Name == "" { + return nil, fmt.Errorf("volume name is empty") + } + if _, exists := result[volume.Name]; exists { + return nil, fmt.Errorf("duplicate volume %q", volume.Name) + } + if volume.HostPath == nil { + return nil, fmt.Errorf("volume %q must specify hostPath", volume.Name) + } + if !strings.HasPrefix(volume.HostPath.Path, "/") { + return nil, fmt.Errorf("volume %q hostPath %q must be absolute", volume.Name, volume.HostPath.Path) + } + result[volume.Name] = volume.HostPath.Path + } + return result, nil +} + +func containerVolumeMounts(container networkingv1alpha1.Container, volumes map[string]string) ([]string, error) { + result := make([]string, 0, len(container.VolumeMounts)) + mountPaths := make(map[string]struct{}, len(container.VolumeMounts)) + for _, mount := range container.VolumeMounts { + hostPath, exists := volumes[mount.Name] + if !exists { + return nil, fmt.Errorf("container %q mounts unknown volume %q", container.Name, mount.Name) + } + if !strings.HasPrefix(mount.MountPath, "/") { + return nil, fmt.Errorf("container %q mountPath %q must be absolute", container.Name, mount.MountPath) + } + if _, exists := mountPaths[mount.MountPath]; exists { + return nil, fmt.Errorf("container %q mounts multiple volumes at %q", container.Name, mount.MountPath) + } + mountPaths[mount.MountPath] = struct{}{} + mode := "rw" + if mount.ReadOnly { + mode = "ro" + } + result = append(result, hostPath+":"+mount.MountPath+":"+mode) + } + return result, nil +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\\\"'\\\"'") + "'" +} diff --git a/internal/ztp/templates/generated-mode.sh.gotmpl b/internal/ztp/templates/generated-mode.sh.gotmpl new file mode 100644 index 0000000..f3cd63b --- /dev/null +++ b/internal/ztp/templates/generated-mode.sh.gotmpl @@ -0,0 +1,57 @@ +#!/bin/bash +set -euo pipefail + +# Generated by sonic-operator {{ shellQuote .SwitchName }} +{{ if .ConfigureONIE }} +# Configure ONIE install discovery for the next and every subsequent SONiC reboot. +mkdir -p /onieboot +cat >/etc/systemd/system/onieboot.mount <<'EOF' +[Unit] +Description=ONIE boot partition + +[Mount] +What=LABEL=ONIE-BOOT +Where=/onieboot +Type=auto +Options=defaults + +[Install] +WantedBy=multi-user.target +EOF + +cat >/etc/systemd/system/sonic-operator-onie-install.service <<'EOF' +[Unit] +Description=Configure next boot for ONIE install discovery +Requires=onieboot.mount +After=onieboot.mount + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'grub-editenv /host/grub/grubenv set next_entry=ONIE' +ExecStart=/bin/sh -c 'grub-editenv /onieboot/grub/grubenv set onie_mode=install' + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable --now onieboot.mount +systemctl enable --now sonic-operator-onie-install.service +{{ end }} +config hostname {{ shellQuote .Hostname }} +config save -y +{{ if .Containers }} +# containers +{{ range .Containers }}if ! ( +{{ if .InjectControlKubeconfig }}install -d -m 0700 /etc/sonic-operator/credentials && +install -d -m 0700 {{ shellQuote (printf "/etc/sonic-operator/credentials/%s" .Name) }} && +printf '%s' {{ shellQuote .EncodedControlKubeconfig }} | base64 -d >{{ shellQuote .ControlKubeconfigPath }} && +chown {{ .DockerUser }} {{ shellQuote .ControlKubeconfigPath }} && +chmod 0600 {{ shellQuote .ControlKubeconfigPath }} && +{{ end }}docker pull {{ shellQuote .Image }} && +(docker rm -f {{ shellQuote .Name }} >/dev/null 2>&1 || true) && +docker run -d --name {{ shellQuote .Name }} --network host --restart unless-stopped{{ if .DockerUser }} --user {{ shellQuote .DockerUser }}{{ end }}{{ range .VolumeMounts }} -v {{ shellQuote . }}{{ end }}{{ if .InjectControlKubeconfig }} -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig -v {{ shellQuote (printf "%s:/var/run/sonic-operator/control-kubeconfig:ro" .ControlKubeconfigPath) }}{{ end }}{{ if .Entrypoint }} --entrypoint {{ shellQuote .Entrypoint }}{{ end }} {{ shellQuote .Image }}{{ range .Command }} {{ shellQuote . }}{{ end }}{{ range .Args }} {{ shellQuote . }}{{ end }} +); then + echo {{ shellQuote .FailureMessage }} >&2 +fi +{{ end }}{{ end }}sync diff --git a/internal/ztp/testdata/generated-mode.golden.sh b/internal/ztp/testdata/generated-mode.golden.sh new file mode 100644 index 0000000..4b18433 --- /dev/null +++ b/internal/ztp/testdata/generated-mode.golden.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -euo pipefail + +# Generated by sonic-operator 'switch-1' + +# Configure ONIE install discovery for the next and every subsequent SONiC reboot. +mkdir -p /onieboot +cat >/etc/systemd/system/onieboot.mount <<'EOF' +[Unit] +Description=ONIE boot partition + +[Mount] +What=LABEL=ONIE-BOOT +Where=/onieboot +Type=auto +Options=defaults + +[Install] +WantedBy=multi-user.target +EOF + +cat >/etc/systemd/system/sonic-operator-onie-install.service <<'EOF' +[Unit] +Description=Configure next boot for ONIE install discovery +Requires=onieboot.mount +After=onieboot.mount + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'grub-editenv /host/grub/grubenv set next_entry=ONIE' +ExecStart=/bin/sh -c 'grub-editenv /onieboot/grub/grubenv set onie_mode=install' + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable --now onieboot.mount +systemctl enable --now sonic-operator-onie-install.service + +config hostname 'switch-1.lab.example' +config save -y + +# containers +if ! ( +install -d -m 0700 /etc/sonic-operator/credentials && +install -d -m 0700 '/etc/sonic-operator/credentials/wirelet' && +printf '%s' 'YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCg==' | base64 -d >'/etc/sonic-operator/credentials/wirelet/control-kubeconfig' && +chown 65532:65532 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig' && +chmod 0600 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig' && +docker pull 'ghcr.io/hardikdr/wirelet:fixed-1' && +(docker rm -f 'wirelet' >/dev/null 2>&1 || true) && +docker run -d --name 'wirelet' --network host --restart unless-stopped --user '65532:65532' -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig -v '/etc/sonic-operator/credentials/wirelet/control-kubeconfig:/var/run/sonic-operator/control-kubeconfig:ro' 'ghcr.io/hardikdr/wirelet:fixed-1' '--name=switch-1' '--interface=Ethernet0' +); then + echo 'sonic-operator: container wirelet failed; continuing' >&2 +fi +if ! ( +docker pull 'ghcr.io/giluerre/sonic-agent:latest' && +(docker rm -f 'sonic-agent' >/dev/null 2>&1 || true) && +docker run -d --name 'sonic-agent' --network host --restart unless-stopped --user '0' -v '/var/run/dbus:/var/run/dbus:rw' -v '/etc/sonic/sonic_version.yml:/etc/sonic/sonic_version.yml:ro' --entrypoint '/switch-agent-server' 'ghcr.io/giluerre/sonic-agent:latest' '-port' '57400' +); then + echo 'sonic-operator: container sonic-agent failed; continuing' >&2 +fi +sync diff --git a/internal/ztp/ztp.go b/internal/ztp/ztp.go index 2d4ab3d..5531a22 100644 --- a/internal/ztp/ztp.go +++ b/internal/ztp/ztp.go @@ -5,14 +5,10 @@ package ztp import ( "embed" - "encoding/base64" "fmt" "log/slog" "net/http" "net/netip" - "os" - "strconv" - "strings" "text/template" corev1 "k8s.io/api/core/v1" @@ -104,7 +100,11 @@ func Register(mux *http.ServeMux, c Config) { "searchDomain": func() string { return c.SearchDomain }, "interfacePrefix": interfacePrefix, }) - t = template.Must(t.ParseFS(templateFS, "templates/*.gotmpl")) + t = template.Must(t.ParseFS( + templateFS, + "templates/leaf.sh.gotmpl", + "templates/spine.sh.gotmpl", + )) mux.Handle("GET /ztp", &handler{t: t, m: c.SwitchParams}) } @@ -250,265 +250,6 @@ func matchingSwitch(r *http.Request, reader client.Reader) (*networkingv1alpha1. return matched, nil } -func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubeconfigFilePath string) (string, error) { - hostname := switchObject.Spec.Hostname - if hostname == "" { - hostname = switchObject.Name - } - if hostname == "" { - return "", fmt.Errorf("hostname is empty") - } - - var b strings.Builder - b.WriteString("#!/bin/bash\n") - b.WriteString("set -euo pipefail\n\n") - b.WriteString("# Generated by sonic-operator ") - b.WriteString(shellQuote(switchObject.Name)) - b.WriteString("\n") - - // Apply boot lifecycle intent before any fallible switch configuration or - // bootstrap workload operation. This is intentionally strict: a failure to - // persist the requested next boot mode must not be hidden. - switch switchObject.Spec.NextBootMode { - case "", networkingv1alpha1.NextBootModeNone: - case networkingv1alpha1.NextBootModeInstallOS: - appendONIEInstallBootConfig(&b) - default: - return "", fmt.Errorf("unsupported next boot mode %q", switchObject.Spec.NextBootMode) - } - - b.WriteString("config hostname ") - b.WriteString(shellQuote(hostname)) - b.WriteString("\n") - b.WriteString("config save -y\n") - - if len(switchObject.Spec.Containers) > 0 { - containers := switchObject.Spec.Containers - volumes, err := hostPathVolumes(switchObject.Spec.Volumes) - if err != nil { - return "", err - } - needsControlKubeconfig := false - for _, container := range containers { - if container.InjectControlKubeconfig { - needsControlKubeconfig = true - break - } - } - - b.WriteString("\n# sonic-operator containers\n") - - var encodedControlKubeconfig string - if needsControlKubeconfig { - if controlKubeconfigFilePath == "" { - return "", fmt.Errorf("injectControlKubeconfig requires --bootstrap-control-kubeconfig-file") - } - kubeconfig, err := os.ReadFile(controlKubeconfigFilePath) - if err != nil { - return "", fmt.Errorf("reading control kubeconfig file %q: %w", controlKubeconfigFilePath, err) - } - - encodedControlKubeconfig = base64.StdEncoding.EncodeToString(kubeconfig) - } - - for _, container := range containers { - dockerUser, err := containerDockerUser(container) - if err != nil { - return "", err - } - volumeMounts, err := containerVolumeMounts(container, volumes) - if err != nil { - return "", err - } - - controlKubeconfigPath := "" - if container.InjectControlKubeconfig { - if container.SecurityContext == nil || container.SecurityContext.RunAsUser == nil { - return "", fmt.Errorf("container %q injects the control kubeconfig but has no securityContext.runAsUser", container.Name) - } - controlKubeconfigPath = "/etc/sonic-operator/credentials/" + container.Name + "/control-kubeconfig" - } - - // A workload is best effort. Its failure must not prevent later - // containers from being attempted or obscure the already-applied boot - // lifecycle configuration above. - b.WriteString("if ! (\n") - if container.InjectControlKubeconfig { - b.WriteString("install -d -m 0700 /etc/sonic-operator/credentials &&\n") - b.WriteString("install -d -m 0700 ") - b.WriteString(shellQuote("/etc/sonic-operator/credentials/" + container.Name)) - b.WriteString(" &&\n") - b.WriteString("printf '%s' ") - b.WriteString(shellQuote(encodedControlKubeconfig)) - b.WriteString(" | base64 -d >") - b.WriteString(shellQuote(controlKubeconfigPath)) - b.WriteString(" &&\n") - b.WriteString("chown ") - b.WriteString(dockerUser) - b.WriteByte(' ') - b.WriteString(shellQuote(controlKubeconfigPath)) - b.WriteString(" &&\n") - b.WriteString("chmod 0600 ") - b.WriteString(shellQuote(controlKubeconfigPath)) - b.WriteString(" &&\n") - } - - b.WriteString("docker pull ") - b.WriteString(shellQuote(container.Image)) - b.WriteString(" &&\n") - b.WriteString("(docker rm -f ") - b.WriteString(shellQuote(container.Name)) - b.WriteString(" >/dev/null 2>&1 || true) &&\n") - b.WriteString("docker run -d --name ") - b.WriteString(shellQuote(container.Name)) - b.WriteString(" --network host --restart unless-stopped") - if dockerUser != "" { - b.WriteString(" --user ") - b.WriteString(shellQuote(dockerUser)) - } - for _, volumeMount := range volumeMounts { - b.WriteString(" -v ") - b.WriteString(shellQuote(volumeMount)) - } - if container.InjectControlKubeconfig { - b.WriteString(" -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig") - b.WriteString(" -v ") - b.WriteString(shellQuote(controlKubeconfigPath + ":/var/run/sonic-operator/control-kubeconfig:ro")) - } - if len(container.Command) > 0 { - b.WriteString(" --entrypoint ") - b.WriteString(shellQuote(container.Command[0])) - } - b.WriteByte(' ') - b.WriteString(shellQuote(container.Image)) - if len(container.Command) > 1 { - for _, command := range container.Command[1:] { - b.WriteByte(' ') - b.WriteString(shellQuote(command)) - } - } - for _, arg := range container.Args { - b.WriteByte(' ') - b.WriteString(shellQuote(arg)) - } - b.WriteString("\n); then\n") - b.WriteString(" echo ") - b.WriteString(shellQuote("sonic-operator: container " + container.Name + " failed; continuing")) - b.WriteString(" >&2\nfi\n") - } - } - - b.WriteString("sync\n") - return b.String(), nil -} - -func containerDockerUser(container networkingv1alpha1.Container) (string, error) { - if container.SecurityContext == nil { - return "", nil - } - - securityContext := container.SecurityContext - if securityContext.RunAsGroup != nil && securityContext.RunAsUser == nil { - return "", fmt.Errorf("container %q sets securityContext.runAsGroup without securityContext.runAsUser", container.Name) - } - if securityContext.RunAsUser == nil { - return "", nil - } - - user := strconv.FormatInt(*securityContext.RunAsUser, 10) - if securityContext.RunAsGroup == nil { - return user, nil - } - return user + ":" + strconv.FormatInt(*securityContext.RunAsGroup, 10), nil -} - -func hostPathVolumes(volumes []networkingv1alpha1.Volume) (map[string]string, error) { - result := make(map[string]string, len(volumes)) - for _, volume := range volumes { - if volume.Name == "" { - return nil, fmt.Errorf("volume name is empty") - } - if _, exists := result[volume.Name]; exists { - return nil, fmt.Errorf("duplicate volume %q", volume.Name) - } - if volume.HostPath == nil { - return nil, fmt.Errorf("volume %q must specify hostPath", volume.Name) - } - if !strings.HasPrefix(volume.HostPath.Path, "/") { - return nil, fmt.Errorf("volume %q hostPath %q must be absolute", volume.Name, volume.HostPath.Path) - } - result[volume.Name] = volume.HostPath.Path - } - return result, nil -} - -func containerVolumeMounts(container networkingv1alpha1.Container, volumes map[string]string) ([]string, error) { - result := make([]string, 0, len(container.VolumeMounts)) - mountPaths := make(map[string]struct{}, len(container.VolumeMounts)) - for _, mount := range container.VolumeMounts { - hostPath, exists := volumes[mount.Name] - if !exists { - return nil, fmt.Errorf("container %q mounts unknown volume %q", container.Name, mount.Name) - } - if !strings.HasPrefix(mount.MountPath, "/") { - return nil, fmt.Errorf("container %q mountPath %q must be absolute", container.Name, mount.MountPath) - } - if _, exists := mountPaths[mount.MountPath]; exists { - return nil, fmt.Errorf("container %q mounts multiple volumes at %q", container.Name, mount.MountPath) - } - mountPaths[mount.MountPath] = struct{}{} - mode := "rw" - if mount.ReadOnly { - mode = "ro" - } - result = append(result, hostPath+":"+mount.MountPath+":"+mode) - } - return result, nil -} - -func appendONIEInstallBootConfig(b *strings.Builder) { - b.WriteString(` -# Configure ONIE install discovery for the next and every subsequent SONiC reboot. -mkdir -p /onieboot -cat >/etc/systemd/system/onieboot.mount <<'EOF' -[Unit] -Description=ONIE boot partition - -[Mount] -What=LABEL=ONIE-BOOT -Where=/onieboot -Type=auto -Options=defaults - -[Install] -WantedBy=multi-user.target -EOF - -cat >/etc/systemd/system/sonic-operator-onie-install.service <<'EOF' -[Unit] -Description=Configure next boot for ONIE install discovery -Requires=onieboot.mount -After=onieboot.mount - -[Service] -Type=oneshot -ExecStart=/bin/sh -c 'grub-editenv /host/grub/grubenv set next_entry=ONIE' -ExecStart=/bin/sh -c 'grub-editenv /onieboot/grub/grubenv set onie_mode=install' - -[Install] -WantedBy=multi-user.target -EOF - -systemctl daemon-reload -systemctl enable --now onieboot.mount -systemctl enable --now sonic-operator-onie-install.service -`) -} - -func shellQuote(value string) string { - return "'" + strings.ReplaceAll(value, "'", "'\\\"'\\\"'") + "'" -} - func handleErr(w http.ResponseWriter, e error) { w.WriteHeader(http.StatusInternalServerError) _, err := fmt.Fprint(w, e.Error()) diff --git a/internal/ztp/ztp_test.go b/internal/ztp/ztp_test.go index b6be6e3..c7b3158 100644 --- a/internal/ztp/ztp_test.go +++ b/internal/ztp/ztp_test.go @@ -20,6 +20,8 @@ import ( networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" ) +const matchingSwitchRemoteAddr = "192.0.2.10:1234" + func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { @@ -52,7 +54,7 @@ func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { mux := http.NewServeMux() RegisterConfigMap(mux, c) req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) - req.RemoteAddr = "192.0.2.10:1234" + req.RemoteAddr = matchingSwitchRemoteAddr response := httptest.NewRecorder() mux.ServeHTTP(response, req) @@ -148,7 +150,7 @@ func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { mux := http.NewServeMux() RegisterGenerated(mux, c, GeneratedOptions{ControlKubeconfigFile: kubeconfigPath}) req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) - req.RemoteAddr = "192.0.2.10:1234" + req.RemoteAddr = matchingSwitchRemoteAddr response := httptest.NewRecorder() mux.ServeHTTP(response, req) @@ -157,6 +159,13 @@ func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) } body := response.Body.String() + wantBody, err := os.ReadFile("testdata/generated-mode.golden.sh") + if err != nil { + t.Fatal(err) + } + if body != string(wantBody) { + t.Errorf("rendered script differs from testdata/generated-mode.golden.sh\ngot:\n%s\nwant:\n%s", body, wantBody) + } for _, want := range []string{ "#!/bin/bash", "set -euo pipefail", @@ -224,7 +233,7 @@ func TestGeneratedHandlerRejectsMissingControlKubeconfig(t *testing.T) { mux := http.NewServeMux() RegisterGenerated(mux, c, GeneratedOptions{}) req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) - req.RemoteAddr = "192.0.2.10:1234" + req.RemoteAddr = matchingSwitchRemoteAddr response := httptest.NewRecorder() mux.ServeHTTP(response, req) From 861af71b25dcd7ea787a4efcaedf1c3996ea6498 Mon Sep 17 00:00:00 2001 From: Hardik Dodiya Date: Thu, 24 Sep 2026 13:19:17 +0200 Subject: [PATCH 4/4] Address generated ZTP review feedback Signed-off-by: Hardik Dodiya --- api/v1alpha1/switch_types.go | 2 ++ ...etworking.metal.ironcore.dev_switches.yaml | 2 ++ docs/api-reference/api.md | 2 +- internal/ztp/generated_mode.go | 2 +- internal/ztp/ztp_test.go | 22 +++++++++++++++++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/api/v1alpha1/switch_types.go b/api/v1alpha1/switch_types.go index 3a484eb..66864c8 100644 --- a/api/v1alpha1/switch_types.go +++ b/api/v1alpha1/switch_types.go @@ -147,6 +147,8 @@ type SwitchSpec struct { // Hostname is configured on the switch by --ztp-mode=generated. If omitted, // the generated script uses the Switch object name. // +optional + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +kubebuilder:validation:MaxLength=253 Hostname string `json:"hostname,omitempty"` Management Management `json:"management,omitempty"` diff --git a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml index 8ef6518..af8f11e 100644 --- a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml +++ b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml @@ -137,6 +137,8 @@ spec: description: |- Hostname is configured on the switch by --ztp-mode=generated. If omitted, the generated script uses the Switch object name. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string macAddress: description: MacAddress is the MAC address assigned to this interface. diff --git a/docs/api-reference/api.md b/docs/api-reference/api.md index 293912b..5581fa1 100644 --- a/docs/api-reference/api.md +++ b/docs/api-reference/api.md @@ -333,7 +333,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `hostname` _string_ | Hostname is configured on the switch by --ztp-mode=generated. If omitted,
the generated script uses the Switch object name. | | | +| `hostname` _string_ | Hostname is configured on the switch by --ztp-mode=generated. If omitted,
the generated script uses the Switch object name. | | MaxLength: 253
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
| | `management` _[Management](#management)_ | | | | | `ztp` _[ZTP](#ztp)_ | ZTP identifies the switch while it requests its initial provisioning script. | | | | `containers` _[Container](#container) array_ | Containers are started with host networking and Docker's unless-stopped
restart policy by --ztp-mode=generated. | | | diff --git a/internal/ztp/generated_mode.go b/internal/ztp/generated_mode.go index 2dd6e80..e1ce4fd 100644 --- a/internal/ztp/generated_mode.go +++ b/internal/ztp/generated_mode.go @@ -208,5 +208,5 @@ func containerVolumeMounts(container networkingv1alpha1.Container, volumes map[s } func shellQuote(value string) string { - return "'" + strings.ReplaceAll(value, "'", "'\\\"'\\\"'") + "'" + return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'" } diff --git a/internal/ztp/ztp_test.go b/internal/ztp/ztp_test.go index c7b3158..31bc354 100644 --- a/internal/ztp/ztp_test.go +++ b/internal/ztp/ztp_test.go @@ -259,3 +259,25 @@ func TestRenderGeneratedScriptDefaultsHostnameToSwitchName(t *testing.T) { t.Errorf("generated script does not default hostname to switch name:\n%s", script) } } + +func TestShellQuoteRoundTripsThroughBash(t *testing.T) { + tests := map[string]string{ + "empty": "", + "simple": "simple", + "spaces": "contains spaces", + "single quote": "a'b", + "command expansion": `$(echo unsafe)`, + } + + for name, value := range tests { + t.Run(name, func(t *testing.T) { + output, err := exec.Command("bash", "-c", "printf '%s' "+shellQuote(value)).CombinedOutput() + if err != nil { + t.Fatalf("executing quoted value: %v: %s", err, output) + } + if got := string(output); got != value { + t.Errorf("round-trip = %q, want %q", got, value) + } + }) + } +}