diff --git a/REUSE.toml b/REUSE.toml index 9a67a48..eee36b6 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -19,6 +19,7 @@ SPDX-PackageDownloadLocation = "https://github.com/ironcore-dev/sonic-operator" "package-lock.json", "package.json", "internal/ztp/templates/**", + "internal/ztp/testdata/**", "REUSE.toml", "hack/**", ".devcontainer/**", diff --git a/api/v1alpha1/switch_types.go b/api/v1alpha1/switch_types.go index c74d4a0..66864c8 100644 --- a/api/v1alpha1/switch_types.go +++ b/api/v1alpha1/switch_types.go @@ -36,18 +36,147 @@ type ZTPConfigMapReference struct { // ZTP script. SourceAddress is the address observed by the provisioning server, // which can differ from the management address used after provisioning. type ZTP struct { - SourceAddress string `json:"sourceAddress"` - ScriptRef ZTPConfigMapReference `json:"scriptRef"` + SourceAddress string `json:"sourceAddress"` + + // ScriptRef identifies the complete script served when --ztp-mode=configmap. + // It is not used by --ztp-mode=generated. + // +optional + ScriptRef *ZTPConfigMapReference `json:"scriptRef,omitempty"` +} + +// Container declares a Docker container that the provisioning server starts on +// the switch during generated ZTP provisioning. +type Container struct { + // Name is both the Docker container name and its stable identity on the switch. + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // Image is the container image pulled by Docker on the switch. + // +kubebuilder:validation:MinLength=1 + Image string `json:"image"` + + // Command overrides the image entrypoint. + // +optional + Command []string `json:"command,omitempty"` + + // Args are appended after Command, or after the image entrypoint when Command is empty. + // +optional + Args []string `json:"args,omitempty"` + + // VolumeMounts describes the volumes mounted into the container. Each mount + // name must refer to an entry in SwitchSpec.Volumes. + // +optional + VolumeMounts []VolumeMount `json:"volumeMounts,omitempty"` + + // SecurityContext configures the Unix identity used to run the container. + // +optional + SecurityContext *ContainerSecurityContext `json:"securityContext,omitempty"` + + // InjectControlKubeconfig mounts the operator's configured control kubeconfig + // into this container and sets KUBECONFIG to its in-container path. A + // securityContext with runAsUser is required so the generated script can + // grant access to a private, per-container credential file. + // +optional + InjectControlKubeconfig bool `json:"injectControlKubeconfig,omitempty"` +} + +// ContainerSecurityContext is the supported subset of Kubernetes +// container securityContext for generated Docker containers. +type ContainerSecurityContext struct { + // RunAsUser is the numeric Unix user ID used by the container. + // +optional + // +kubebuilder:validation:Minimum=0 + RunAsUser *int64 `json:"runAsUser,omitempty"` + + // RunAsGroup is the numeric Unix group ID used by the container. It requires + // runAsUser to be set as well. + // +optional + // +kubebuilder:validation:Minimum=0 + RunAsGroup *int64 `json:"runAsGroup,omitempty"` +} + +// Volume represents a named storage volume made available to Switch containers. +// It follows the Kubernetes Pod volume model. Generated ZTP currently supports +// hostPath volumes only. +type Volume struct { + // Name is the stable volume identity referenced by Container.VolumeMounts. + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // HostPath represents a pre-existing file or directory on the SONiC host. + // +optional + HostPath *HostPathVolumeSource `json:"hostPath,omitempty"` +} + +// HostPathVolumeSource represents a host directory or file mounted into a +// Switch container. +type HostPathVolumeSource struct { + // Path is the absolute path on the SONiC host. + Path string `json:"path"` + + // Type describes the expected host-path type, following the Kubernetes Pod + // hostPath API. Generated ZTP does not create missing paths. + // +optional + Type *v1.HostPathType `json:"type,omitempty"` } +// VolumeMount describes a volume mounted into a Switch container. It follows +// the Kubernetes Pod volumeMount API. +type VolumeMount struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` + ReadOnly bool `json:"readOnly,omitempty"` +} + +// NextBootMode describes the desired behavior of the switch's next boot. +// Providers translate this high-level intent to their platform-specific boot +// mechanism. For SONiC, InstallOS enters ONIE install discovery. +type NextBootMode string + +const ( + // NextBootModeNone leaves the normal installed network OS boot path intact. + NextBootModeNone NextBootMode = "None" + // NextBootModeInstallOS enters the platform's OS installation/discovery flow. + NextBootModeInstallOS NextBootMode = "InstallOS" +) + // SwitchSpec defines the desired state of Switch type SwitchSpec struct { + // Hostname is configured on the switch by --ztp-mode=generated. If omitted, + // the generated script uses the Switch object name. + // +optional + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +kubebuilder:validation:MaxLength=253 + Hostname string `json:"hostname,omitempty"` + Management Management `json:"management,omitempty"` - // ZTP selects a custom script when the operator runs with --ztp-mode=configmap. + // ZTP identifies the switch while it requests its initial provisioning script. // +optional ZTP *ZTP `json:"ztp,omitempty"` + // Containers are started with host networking and Docker's unless-stopped + // restart policy by --ztp-mode=generated. + // +optional + // +listType=map + // +listMapKey=name + Containers []Container `json:"containers,omitempty"` + + // Volumes are named storage sources available to containers. Generated ZTP + // currently supports hostPath volumes only. + // +optional + // +listType=map + // +listMapKey=name + Volumes []Volume `json:"volumes,omitempty"` + + // NextBootMode declares the desired behavior of the next boot. The default + // is None. It is configured by --ztp-mode=generated. + // +optional + // +kubebuilder:validation:Enum=None;InstallOS + NextBootMode NextBootMode `json:"nextBootMode,omitempty"` + // MacAddress is the MAC address assigned to this interface. MacAddress string `json:"macAddress"` @@ -105,7 +234,6 @@ type SwitchStatus struct { // +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // +kubebuilder:ac:generate=false - // Switch is the Schema for the switch API type Switch struct { metav1.TypeMeta `json:",inline"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2d22add..0a5643c 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -13,6 +13,86 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Container) DeepCopyInto(out *Container) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]VolumeMount, len(*in)) + copy(*out, *in) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(ContainerSecurityContext) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Container. +func (in *Container) DeepCopy() *Container { + if in == nil { + return nil + } + out := new(Container) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ContainerSecurityContext) DeepCopyInto(out *ContainerSecurityContext) { + *out = *in + if in.RunAsUser != nil { + in, out := &in.RunAsUser, &out.RunAsUser + *out = new(int64) + **out = **in + } + if in.RunAsGroup != nil { + in, out := &in.RunAsGroup, &out.RunAsGroup + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerSecurityContext. +func (in *ContainerSecurityContext) DeepCopy() *ContainerSecurityContext { + if in == nil { + return nil + } + out := new(ContainerSecurityContext) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostPathVolumeSource) DeepCopyInto(out *HostPathVolumeSource) { + *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + *out = new(v1.HostPathType) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostPathVolumeSource. +func (in *HostPathVolumeSource) DeepCopy() *HostPathVolumeSource { + if in == nil { + return nil + } + out := new(HostPathVolumeSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Management) DeepCopyInto(out *Management) { *out = *in @@ -332,7 +412,21 @@ func (in *SwitchSpec) DeepCopyInto(out *SwitchSpec) { if in.ZTP != nil { in, out := &in.ZTP, &out.ZTP *out = new(ZTP) - **out = **in + (*in).DeepCopyInto(*out) + } + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]Container, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Ports != nil { in, out := &in.Ports, &out.Ports @@ -380,10 +474,49 @@ func (in *SwitchStatus) DeepCopy() *SwitchStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Volume) DeepCopyInto(out *Volume) { + *out = *in + if in.HostPath != nil { + in, out := &in.HostPath, &out.HostPath + *out = new(HostPathVolumeSource) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Volume. +func (in *Volume) DeepCopy() *Volume { + if in == nil { + return nil + } + out := new(Volume) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeMount) DeepCopyInto(out *VolumeMount) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeMount. +func (in *VolumeMount) DeepCopy() *VolumeMount { + if in == nil { + return nil + } + out := new(VolumeMount) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ZTP) DeepCopyInto(out *ZTP) { *out = *in - out.ScriptRef = in.ScriptRef + if in.ScriptRef != nil { + in, out := &in.ScriptRef, &out.ScriptRef + *out = new(ZTPConfigMapReference) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ZTP. diff --git a/cmd/main.go b/cmd/main.go index e3785bf..20d54e6 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -55,7 +55,7 @@ func main() { var secureMetrics bool var enableHTTP2 bool var disableProvisionsingServer bool - var httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode string + var httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode, bootstrapControlKubeconfigFile string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -76,7 +76,8 @@ func main() { "If set, HTTP/2 will be enabled for the metrics and webhook servers") flag.StringVar(&httpServerAddr, "http-server-address", "0", "The address the HTTP server for ZTP and ONIE binds to.") flag.StringVar(&ztpConfigFile, "ztp-config-file", "/etc/ztp.json", "Config file containing the parameters to render ZTP scripts.") - flag.StringVar(&ztpMode, "ztp-mode", "templates", "ZTP source: templates or configmap. Configmap mode never falls back to templates.") + flag.StringVar(&ztpMode, "ztp-mode", "templates", "ZTP source: templates, configmap, or generated. Configmap mode serves the referenced script verbatim; generated mode renders from Switch objects.") + flag.StringVar(&bootstrapControlKubeconfigFile, "bootstrap-control-kubeconfig-file", "", "Optional kubeconfig file injected into generated containers that set injectControlKubeconfig=true.") flag.StringVar(&onieImagesDir, "onie-images-dir", "/var/lib/sonic-operator/onie", "The directory which contains the ONIE and SONiC installer image files.") flag.StringVar(&onieConfigFile, "onie-config-file", "/etc/onie.json", "Config file containing machine-to-image mappings for ONIE provisioning.") flag.BoolVar(&disableProvisionsingServer, "disable-static-config", false, "If set, the HTTP server for ZTP and ONIE will not be started.") @@ -212,7 +213,7 @@ func main() { } if !disableProvisionsingServer { setupLog.Info("starting HTTP server") - provServer, err := setupProvisioningServer(httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode, mgr.GetAPIReader()) + provServer, err := setupProvisioningServer(httpServerAddr, onieImagesDir, onieConfigFile, ztpConfigFile, ztpMode, bootstrapControlKubeconfigFile, mgr.GetAPIReader()) if err != nil { setupLog.Error(err, "unable to setup HTTP server") os.Exit(1) @@ -232,7 +233,7 @@ func main() { } } -func setupProvisioningServer(addr string, onieImagesDir string, onieConfigPath string, ztpConfigPath string, ztpMode string, reader client.Reader) (*http.Server, error) { +func setupProvisioningServer(addr string, onieImagesDir string, onieConfigPath string, ztpConfigPath string, ztpMode string, bootstrapControlKubeconfigFile string, reader client.Reader) (*http.Server, error) { of, err := os.Open(onieConfigPath) if err != nil { return nil, fmt.Errorf("unable to open onie config file: %w", err) @@ -268,8 +269,10 @@ func setupProvisioningServer(addr string, onieImagesDir string, onieConfigPath s ztp.Register(mux, ztpConf) case "configmap": ztp.RegisterConfigMap(mux, reader) + case "generated": + ztp.RegisterGenerated(mux, reader, ztp.GeneratedOptions{ControlKubeconfigFile: bootstrapControlKubeconfigFile}) default: - return nil, fmt.Errorf("unsupported ztp mode %q (supported: templates, configmap)", ztpMode) + return nil, fmt.Errorf("unsupported ztp mode %q (supported: templates, configmap, generated)", ztpMode) } onie.Register(mux, onieImagesDir, onieConf) diff --git a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml index f191481..af8f11e 100644 --- a/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml +++ b/config/crd/bases/sonic.networking.metal.ironcore.dev_switches.yaml @@ -49,6 +49,97 @@ spec: spec: description: spec defines the desired state of Switch properties: + containers: + description: |- + Containers are started with host networking and Docker's unless-stopped + restart policy by --ztp-mode=generated. + items: + description: |- + Container declares a Docker container that the provisioning server starts on + the switch during generated ZTP provisioning. + properties: + args: + description: Args are appended after Command, or after the image + entrypoint when Command is empty. + items: + type: string + type: array + command: + description: Command overrides the image entrypoint. + items: + type: string + type: array + image: + description: Image is the container image pulled by Docker on + the switch. + minLength: 1 + type: string + injectControlKubeconfig: + description: |- + InjectControlKubeconfig mounts the operator's configured control kubeconfig + into this container and sets KUBECONFIG to its in-container path. A + securityContext with runAsUser is required so the generated script can + grant access to a private, per-container credential file. + type: boolean + name: + description: Name is both the Docker container name and its + stable identity on the switch. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + securityContext: + description: SecurityContext configures the Unix identity used + to run the container. + properties: + runAsGroup: + description: |- + RunAsGroup is the numeric Unix group ID used by the container. It requires + runAsUser to be set as well. + format: int64 + minimum: 0 + type: integer + runAsUser: + description: RunAsUser is the numeric Unix user ID used + by the container. + format: int64 + minimum: 0 + type: integer + type: object + volumeMounts: + description: |- + VolumeMounts describes the volumes mounted into the container. Each mount + name must refer to an entry in SwitchSpec.Volumes. + items: + description: |- + VolumeMount describes a volume mounted into a Switch container. It follows + the Kubernetes Pod volumeMount API. + properties: + mountPath: + type: string + name: + type: string + readOnly: + type: boolean + required: + - mountPath + - name + type: object + type: array + required: + - image + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostname: + description: |- + Hostname is configured on the switch by --ztp-mode=generated. If omitted, + the generated script uses the Switch object name. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string macAddress: description: MacAddress is the MAC address assigned to this interface. type: string @@ -107,6 +198,14 @@ spec: - host - port type: object + nextBootMode: + description: |- + NextBootMode declares the desired behavior of the next boot. The default + is None. It is configured by --ztp-mode=generated. + enum: + - None + - InstallOS + type: string ports: description: Ports the physical ports available on the Switch. items: @@ -117,15 +216,52 @@ spec: - name type: object type: array + volumes: + description: |- + Volumes are named storage sources available to containers. Generated ZTP + currently supports hostPath volumes only. + items: + description: |- + Volume represents a named storage volume made available to Switch containers. + It follows the Kubernetes Pod volume model. Generated ZTP currently supports + hostPath volumes only. + properties: + hostPath: + description: HostPath represents a pre-existing file or directory + on the SONiC host. + properties: + path: + description: Path is the absolute path on the SONiC host. + type: string + type: + description: |- + Type describes the expected host-path type, following the Kubernetes Pod + hostPath API. Generated ZTP does not create missing paths. + type: string + required: + - path + type: object + name: + description: Name is the stable volume identity referenced by + Container.VolumeMounts. + maxLength: 63 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map ztp: - description: ZTP selects a custom script when the operator runs with - --ztp-mode=configmap. + description: ZTP identifies the switch while it requests its initial + provisioning script. properties: scriptRef: description: |- - ZTPConfigMapReference identifies the ConfigMap entry containing a switch's - complete ZTP script. The referenced ConfigMap may be in a provisioning - namespace chosen by the cluster administrator. + ScriptRef identifies the complete script served when --ztp-mode=configmap. + It is not used by --ztp-mode=generated. properties: key: type: string @@ -141,7 +277,6 @@ spec: sourceAddress: type: string required: - - scriptRef - sourceAddress type: object required: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ddff94e..e3adf2c 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -63,6 +63,8 @@ spec: args: - --leader-elect - --health-probe-bind-address=:8081 + - --ztp-mode=generated + - --bootstrap-control-kubeconfig-file=/etc/sonic-operator/control-kubeconfig/kubeconfig image: controller:latest name: manager ports: [] @@ -93,7 +95,17 @@ spec: requests: cpu: 10m memory: 64Mi - volumeMounts: [] - volumes: [] + volumeMounts: + - name: control-kubeconfig + mountPath: /etc/sonic-operator/control-kubeconfig + readOnly: true + volumes: + - name: control-kubeconfig + secret: + secretName: control-kubeconfig + optional: true + items: + - key: kubeconfig + path: kubeconfig serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/samples/networking_v1alpha1_switch.yaml b/config/samples/networking_v1alpha1_switch.yaml index e2463e5..68283a3 100644 --- a/config/samples/networking_v1alpha1_switch.yaml +++ b/config/samples/networking_v1alpha1_switch.yaml @@ -6,6 +6,27 @@ metadata: app.kubernetes.io/managed-by: kustomize name: switch-sample spec: + hostname: switch-sample + + ztp: + sourceAddress: "192.0.2.11" + + nextBootMode: InstallOS + + # Containers are rendered by --ztp-mode=generated. + # The control kubeconfig is provided globally by the operator, not by this + # Switch object. + containers: + - name: wirelet + image: ghcr.io/hardikdr/wirelet:fixed-1 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + args: + - --name=switch-sample + - --interface=Ethernet0 + injectControlKubeconfig: true + management: host: 192.0.2.10 port: "50051" diff --git a/docs/api-reference/api.md b/docs/api-reference/api.md index e47f177..5581fa1 100644 --- a/docs/api-reference/api.md +++ b/docs/api-reference/api.md @@ -36,6 +36,65 @@ _Appears in:_ | `Down` | | +#### Container + + + +Container declares a Docker container that the provisioning server starts on +the switch during generated ZTP provisioning. + + + +_Appears in:_ +- [SwitchSpec](#switchspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is both the Docker container name and its stable identity on the switch. | | MaxLength: 63
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
| +| `image` _string_ | Image is the container image pulled by Docker on the switch. | | MinLength: 1
| +| `command` _string array_ | Command overrides the image entrypoint. | | | +| `args` _string array_ | Args are appended after Command, or after the image entrypoint when Command is empty. | | | +| `volumeMounts` _[VolumeMount](#volumemount) array_ | VolumeMounts describes the volumes mounted into the container. Each mount
name must refer to an entry in SwitchSpec.Volumes. | | | +| `securityContext` _[ContainerSecurityContext](#containersecuritycontext)_ | SecurityContext configures the Unix identity used to run the container. | | | +| `injectControlKubeconfig` _boolean_ | InjectControlKubeconfig mounts the operator's configured control kubeconfig
into this container and sets KUBECONFIG to its in-container path. A
securityContext with runAsUser is required so the generated script can
grant access to a private, per-container credential file. | | | + + +#### ContainerSecurityContext + + + +ContainerSecurityContext is the supported subset of Kubernetes +container securityContext for generated Docker containers. + + + +_Appears in:_ +- [Container](#container) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `runAsUser` _integer_ | RunAsUser is the numeric Unix user ID used by the container. | | Minimum: 0
| +| `runAsGroup` _integer_ | RunAsGroup is the numeric Unix group ID used by the container. It requires
runAsUser to be set as well. | | Minimum: 0
| + + +#### HostPathVolumeSource + + + +HostPathVolumeSource represents a host directory or file mounted into a +Switch container. + + + +_Appears in:_ +- [Volume](#volume) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `path` _string_ | Path is the absolute path on the SONiC host. | | | +| `type` _[HostPathType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#hostpathtype-v1-core)_ | Type describes the expected host-path type, following the Kubernetes Pod
hostPath API. Generated ZTP does not create missing paths. | | | + + #### Management @@ -72,6 +131,25 @@ _Appears in:_ | `interfaceHandle` _string_ | InterfaceHandle is the name of the remote switch interface. | | | +#### NextBootMode + +_Underlying type:_ _string_ + +NextBootMode describes the desired behavior of the switch's next boot. +Providers translate this high-level intent to their platform-specific boot +mechanism. For SONiC, InstallOS enters ONIE install discovery. + + + +_Appears in:_ +- [SwitchSpec](#switchspec) + +| Field | Description | +| --- | --- | +| `None` | NextBootModeNone leaves the normal installed network OS boot path intact.
| +| `InstallOS` | NextBootModeInstallOS enters the platform's OS installation/discovery flow.
| + + #### OperationState _Underlying type:_ _string_ @@ -255,8 +333,12 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | +| `hostname` _string_ | Hostname is configured on the switch by --ztp-mode=generated. If omitted,
the generated script uses the Switch object name. | | MaxLength: 253
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
| | `management` _[Management](#management)_ | | | | -| `ztp` _[ZTP](#ztp)_ | ZTP selects a custom script when the operator runs with --ztp-mode=configmap. | | | +| `ztp` _[ZTP](#ztp)_ | ZTP identifies the switch while it requests its initial provisioning script. | | | +| `containers` _[Container](#container) array_ | Containers are started with host networking and Docker's unless-stopped
restart policy by --ztp-mode=generated. | | | +| `volumes` _[Volume](#volume) array_ | Volumes are named storage sources available to containers. Generated ZTP
currently supports hostPath volumes only. | | | +| `nextBootMode` _[NextBootMode](#nextbootmode)_ | NextBootMode declares the desired behavior of the next boot. The default
is None. It is configured by --ztp-mode=generated. | | Enum: [None InstallOS]
| | `macAddress` _string_ | MacAddress is the MAC address assigned to this interface. | | | | `ports` _[PortSpec](#portspec) array_ | Ports the physical ports available on the Switch. | | | @@ -300,6 +382,44 @@ _Appears in:_ | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#condition-v1-meta) array_ | The status of each condition is one of True, False, or Unknown. | | | +#### Volume + + + +Volume represents a named storage volume made available to Switch containers. +It follows the Kubernetes Pod volume model. Generated ZTP currently supports +hostPath volumes only. + + + +_Appears in:_ +- [SwitchSpec](#switchspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | Name is the stable volume identity referenced by Container.VolumeMounts. | | MaxLength: 63
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
| +| `hostPath` _[HostPathVolumeSource](#hostpathvolumesource)_ | HostPath represents a pre-existing file or directory on the SONiC host. | | | + + +#### VolumeMount + + + +VolumeMount describes a volume mounted into a Switch container. It follows +the Kubernetes Pod volumeMount API. + + + +_Appears in:_ +- [Container](#container) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `name` _string_ | | | | +| `mountPath` _string_ | | | | +| `readOnly` _boolean_ | | | | + + #### ZTP @@ -316,7 +436,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `sourceAddress` _string_ | | | | -| `scriptRef` _[ZTPConfigMapReference](#ztpconfigmapreference)_ | | | | +| `scriptRef` _[ZTPConfigMapReference](#ztpconfigmapreference)_ | ScriptRef identifies the complete script served when --ztp-mode=configmap.
It is not used by --ztp-mode=generated. | | | #### ZTPConfigMapReference diff --git a/docs/usage/provisioning.md b/docs/usage/provisioning.md index 0d89e59..8587066 100644 --- a/docs/usage/provisioning.md +++ b/docs/usage/provisioning.md @@ -5,17 +5,19 @@ The provisioning server serves ZTP scripts and ONIE installer artifacts over HTT ## Manager flags - `--http-server-address`: bind address for the provisioning server. - `--ztp-config-file`: JSON file with ZTP parameters (default `/etc/ztp.json`). -- `--ztp-mode`: ZTP source, either `templates` (default) or `configmap`. Modes are strict: ConfigMap mode never falls back to templates. +- `--ztp-mode`: ZTP source: `templates` (default), `configmap`, or `generated`. +- `--bootstrap-control-kubeconfig-file`: optional control kubeconfig mounted into the operator and made available to opted-in generated containers. - `--onie-installer-dir`: directory containing ONIE installer files (default `/var/lib/sonic-operator/onie`). ## ZTP -- Scripts are rendered from templates in `internal/ztp/templates`. -- The source IP of the requesting switch is used to select parameters from the ZTP config file. +- `templates` renders the static scripts in `internal/ztp/templates` from the JSON ZTP configuration. +- `configmap` serves the selected ConfigMap key verbatim. +- `generated` renders one complete script from the matching `Switch` object. - The ZTP script is served at `GET /ztp`. ## ConfigMap ZTP mode -Set `--ztp-mode=configmap` to select a full ZTP script from a ConfigMap. The provisioning server matches the source address of `GET /ztp` to `spec.ztp.sourceAddress` on a `Switch`, then returns the configured ConfigMap key verbatim. A missing switch, ConfigMap, or key is an error; the leaf/spine templates are never used in this mode. +Set `--ztp-mode=configmap` to select a full ZTP script from a ConfigMap. The provisioning server matches the source address of `GET /ztp` to `spec.ztp.sourceAddress` on a `Switch`, then returns the configured ConfigMap key verbatim. A missing switch, ConfigMap, or key is an error; no generated commands are appended and the leaf/spine templates are never used in this mode. ```yaml apiVersion: sonic.networking.metal.ironcore.dev/v1alpha1 @@ -44,6 +46,123 @@ data: hostnamectl set-hostname leaf-01 ``` +## Generated ZTP mode + +Set `--ztp-mode=generated` to render the entire ZTP script from the matching +`Switch` object. No ConfigMap script is read in this mode. The generated script +configures the hostname, saves SONiC configuration, writes any requested global +credential, and starts declared Docker containers. + +```yaml +apiVersion: sonic.networking.metal.ironcore.dev/v1alpha1 +kind: Switch +metadata: + name: switch-1 +spec: + hostname: switch-1 + ztp: + sourceAddress: "2001:db8:100::11" + containers: + - name: wirelet + image: ghcr.io/ironcore-dev/wirelet:fixed-1 + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + args: + - --name=switch-1 + - --interface=Ethernet0 + injectControlKubeconfig: true +``` + +Each container is pulled and started with host networking and Docker's +`unless-stopped` restart policy. `command` overrides the image entrypoint; +`args` are appended after `command` or the image entrypoint. `hostname` +defaults to the `Switch` object name when omitted. Containers are +best effort: a failure to prepare or start one is logged and does not prevent +other containers from being attempted. + +### Volumes + +Generated ZTP follows the Kubernetes Pod volume model: define named volumes in +`spec.volumes`, then refer to them from a container's `volumeMounts`. The +current generated Docker runtime supports `hostPath` volumes only: + +```yaml +spec: + volumes: + - name: dbus + hostPath: + path: /var/run/dbus + type: Directory + containers: + - name: sonic-agent + image: ghcr.io/giluerre/sonic-agent:latest + command: ["/switch-agent-server"] + args: ["-port", "57400"] + securityContext: + runAsUser: 0 + volumeMounts: + - name: dbus + mountPath: /var/run/dbus +``` + +This renders a Docker mount before the image, for example +`-v /var/run/dbus:/var/run/dbus:rw`. The volume and mount names must match; +host paths and mount paths must be absolute. + +### ONIE boot discovery + +Generated ZTP can persistently configure the next SONiC reboot to enter ONIE +install discovery: + +```yaml +spec: + nextBootMode: InstallOS +``` + +The generated script creates an `onieboot.mount` unit for the `ONIE-BOOT` +partition and a `sonic-operator-onie-install.service` unit. The latter runs on +every SONiC boot and sets `next_entry=ONIE` plus `onie_mode=install`. It does +not reboot the switch immediately; the next reboot enters ONIE install mode. +`None` (or omitting `nextBootMode`) leaves the normal boot path unchanged. +This boot-lifecycle setup is rendered before hostname and bootstrap-container +steps, and remains strict: a failure to configure it stops the script. + +### Global control kubeconfig + +`injectControlKubeconfig: true` is an explicit opt-in to the operator's global +control kubeconfig. The default manager manifest mounts a Secret named +`control-kubeconfig` in the operator namespace at +`/etc/sonic-operator/control-kubeconfig/kubeconfig` and passes that path with +`--bootstrap-control-kubeconfig-file`. + +Create the Secret before provisioning a switch that requests injection: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: control-kubeconfig + namespace: sonic-operator-system +type: Opaque +stringData: + kubeconfig: | + # a deliberately restricted control-cluster kubeconfig +``` + +Containers use `securityContext.runAsUser` and `securityContext.runAsGroup`, +which map to Docker's `--user` option. The generated ZTP script writes a +private credential file per opted-in container, owned by that container's +configured user/group with mode `0600`, bind-mounts it read-only at +`/var/run/sonic-operator/control-kubeconfig`, and sets `KUBECONFIG` to that +path in the opted-in container. `injectControlKubeconfig: true` requires +`securityContext.runAsUser`. + +This is a shared bootstrap credential. Use a restricted identity only; do not +mount an administrator kubeconfig. The current ZTP endpoint is HTTP, so this +mechanism is appropriate for controlled lab networks only until credential +delivery is protected by HTTPS and a short-lived bootstrap flow. + ## ONIE - Files are served from the installer directory at HTTP root (`/`). - This supports ONIE discovery workflows for delivering SONiC or other OS installers. diff --git a/internal/controller/switch_controller.go b/internal/controller/switch_controller.go index bb968fe..4069d91 100644 --- a/internal/controller/switch_controller.go +++ b/internal/controller/switch_controller.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/go-logr/logr" "github.com/ironcore-dev/controller-utils/clientutils" @@ -27,7 +28,8 @@ import ( ) var ( - fieldOwner = client.FieldOwner("switch-controller") + fieldOwner = client.FieldOwner("switch-controller") + agentRetryAfter = time.Minute ) // SwitchReconciler reconciles a Switch object @@ -88,20 +90,24 @@ func (r *SwitchReconciler) reconcile(ctx context.Context, log logr.Logger, s *ne } }() - if s.Status.State == "" { - s.Status.State = networkingv1alpha1.SwitchStatePending + // A Switch is declaratively ready as soon as it is accepted by the + // controller. The SONiC agent enriches status when reachable, but agent + // availability must not prevent ZTP/bootstrap workflows. + if s.Status.State != networkingv1alpha1.SwitchStateReady { + s.Status.State = networkingv1alpha1.SwitchStateReady return ctrl.Result{}, nil } switchAgentClient, err := switchUtil.NewAgentClientForSwitch(ctx, s) if err != nil { - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } switchDevice, err := switchAgentClient.GetDeviceInfo(ctx) if err != nil { - s.Status.State = networkingv1alpha1.SwitchStateFailed - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } s.Status.MACAddress = switchDevice.LocalMacAddress @@ -110,8 +116,8 @@ func (r *SwitchReconciler) reconcile(ctx context.Context, log logr.Logger, s *ne interfaceList, err := switchAgentClient.ListInterfaces(ctx) if err != nil { - s.Status.State = networkingv1alpha1.SwitchStateFailed - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } for _, iface := range interfaceList.Items { @@ -122,8 +128,8 @@ func (r *SwitchReconciler) reconcile(ctx context.Context, log logr.Logger, s *ne portList, err := switchAgentClient.ListPorts(ctx) if err != nil { - s.Status.State = networkingv1alpha1.SwitchStateFailed - return ctrl.Result{}, err + log.Info("Switch agent is unavailable; keeping Switch ready", "err", err) + return ctrl.Result{RequeueAfter: agentRetryAfter}, nil } if len(portList.Items) > 0 { diff --git a/internal/controller/switch_controller_test.go b/internal/controller/switch_controller_test.go index fc2c738..137d917 100644 --- a/internal/controller/switch_controller_test.go +++ b/internal/controller/switch_controller_test.go @@ -64,8 +64,14 @@ var _ = Describe("Switch Controller", func() { NamespacedName: typeNamespacedName, }) Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + + updated := &networkingv1alpha1.Switch{} + Expect(k8sClient.Get(ctx, typeNamespacedName, updated)).To(Succeed()) + Expect(updated.Status.State).To(Equal(networkingv1alpha1.SwitchStateReady)) }) }) }) diff --git a/internal/ztp/generated_mode.go b/internal/ztp/generated_mode.go new file mode 100644 index 0000000..e1ce4fd --- /dev/null +++ b/internal/ztp/generated_mode.go @@ -0,0 +1,212 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package ztp + +import ( + "bytes" + "encoding/base64" + "fmt" + "os" + "strconv" + "strings" + "text/template" + + networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" +) + +var generatedModeTemplate = template.Must( + template.New("generated-mode.sh.gotmpl"). + Funcs(template.FuncMap{"shellQuote": shellQuote}). + ParseFS(templateFS, "templates/generated-mode.sh.gotmpl"), +) + +type generatedModeTemplateData struct { + SwitchName string + Hostname string + ConfigureONIE bool + Containers []generatedModeContainerData +} + +type generatedModeContainerData struct { + Name string + Image string + DockerUser string + VolumeMounts []string + InjectControlKubeconfig bool + EncodedControlKubeconfig string + ControlKubeconfigPath string + Entrypoint string + Command []string + Args []string + FailureMessage string +} + +func renderGeneratedScript(switchObject *networkingv1alpha1.Switch, controlKubeconfigFilePath string) (string, error) { + data, err := buildGeneratedModeTemplateData(switchObject, controlKubeconfigFilePath) + if err != nil { + return "", err + } + + var output bytes.Buffer + if err := generatedModeTemplate.ExecuteTemplate(&output, "generated-mode.sh.gotmpl", data); err != nil { + return "", fmt.Errorf("executing generated mode template: %w", err) + } + return output.String(), nil +} + +func buildGeneratedModeTemplateData( + switchObject *networkingv1alpha1.Switch, + controlKubeconfigFilePath string, +) (*generatedModeTemplateData, error) { + hostname := switchObject.Spec.Hostname + if hostname == "" { + hostname = switchObject.Name + } + if hostname == "" { + return nil, fmt.Errorf("hostname is empty") + } + + data := &generatedModeTemplateData{ + SwitchName: switchObject.Name, + Hostname: hostname, + Containers: make([]generatedModeContainerData, 0, len(switchObject.Spec.Containers)), + } + + // Boot lifecycle intent is strict and appears before fallible switch or + // workload configuration in the rendered script. + switch switchObject.Spec.NextBootMode { + case "", networkingv1alpha1.NextBootModeNone: + case networkingv1alpha1.NextBootModeInstallOS: + data.ConfigureONIE = true + default: + return nil, fmt.Errorf("unsupported next boot mode %q", switchObject.Spec.NextBootMode) + } + + var volumes map[string]string + if len(switchObject.Spec.Containers) > 0 { + var err error + volumes, err = hostPathVolumes(switchObject.Spec.Volumes) + if err != nil { + return nil, err + } + } + + var encodedControlKubeconfig string + for _, container := range switchObject.Spec.Containers { + if !container.InjectControlKubeconfig || encodedControlKubeconfig != "" { + continue + } + if controlKubeconfigFilePath == "" { + return nil, fmt.Errorf("injectControlKubeconfig requires --bootstrap-control-kubeconfig-file") + } + kubeconfig, err := os.ReadFile(controlKubeconfigFilePath) + if err != nil { + return nil, fmt.Errorf("reading control kubeconfig file %q: %w", controlKubeconfigFilePath, err) + } + encodedControlKubeconfig = base64.StdEncoding.EncodeToString(kubeconfig) + } + + for _, container := range switchObject.Spec.Containers { + dockerUser, err := containerDockerUser(container) + if err != nil { + return nil, err + } + volumeMounts, err := containerVolumeMounts(container, volumes) + if err != nil { + return nil, err + } + + containerData := generatedModeContainerData{ + Name: container.Name, + Image: container.Image, + DockerUser: dockerUser, + VolumeMounts: volumeMounts, + InjectControlKubeconfig: container.InjectControlKubeconfig, + EncodedControlKubeconfig: encodedControlKubeconfig, + Args: container.Args, + FailureMessage: "sonic-operator: container " + container.Name + " failed; continuing", + } + if len(container.Command) > 0 { + containerData.Entrypoint = container.Command[0] + containerData.Command = container.Command[1:] + } + if container.InjectControlKubeconfig { + if container.SecurityContext == nil || container.SecurityContext.RunAsUser == nil { + return nil, fmt.Errorf("container %q injects the control kubeconfig but has no securityContext.runAsUser", container.Name) + } + containerData.ControlKubeconfigPath = "/etc/sonic-operator/credentials/" + container.Name + "/control-kubeconfig" + } + data.Containers = append(data.Containers, containerData) + } + + return data, nil +} + +func containerDockerUser(container networkingv1alpha1.Container) (string, error) { + if container.SecurityContext == nil { + return "", nil + } + + securityContext := container.SecurityContext + if securityContext.RunAsGroup != nil && securityContext.RunAsUser == nil { + return "", fmt.Errorf("container %q sets securityContext.runAsGroup without securityContext.runAsUser", container.Name) + } + if securityContext.RunAsUser == nil { + return "", nil + } + + user := strconv.FormatInt(*securityContext.RunAsUser, 10) + if securityContext.RunAsGroup == nil { + return user, nil + } + return user + ":" + strconv.FormatInt(*securityContext.RunAsGroup, 10), nil +} + +func hostPathVolumes(volumes []networkingv1alpha1.Volume) (map[string]string, error) { + result := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if volume.Name == "" { + return nil, fmt.Errorf("volume name is empty") + } + if _, exists := result[volume.Name]; exists { + return nil, fmt.Errorf("duplicate volume %q", volume.Name) + } + if volume.HostPath == nil { + return nil, fmt.Errorf("volume %q must specify hostPath", volume.Name) + } + if !strings.HasPrefix(volume.HostPath.Path, "/") { + return nil, fmt.Errorf("volume %q hostPath %q must be absolute", volume.Name, volume.HostPath.Path) + } + result[volume.Name] = volume.HostPath.Path + } + return result, nil +} + +func containerVolumeMounts(container networkingv1alpha1.Container, volumes map[string]string) ([]string, error) { + result := make([]string, 0, len(container.VolumeMounts)) + mountPaths := make(map[string]struct{}, len(container.VolumeMounts)) + for _, mount := range container.VolumeMounts { + hostPath, exists := volumes[mount.Name] + if !exists { + return nil, fmt.Errorf("container %q mounts unknown volume %q", container.Name, mount.Name) + } + if !strings.HasPrefix(mount.MountPath, "/") { + return nil, fmt.Errorf("container %q mountPath %q must be absolute", container.Name, mount.MountPath) + } + if _, exists := mountPaths[mount.MountPath]; exists { + return nil, fmt.Errorf("container %q mounts multiple volumes at %q", container.Name, mount.MountPath) + } + mountPaths[mount.MountPath] = struct{}{} + mode := "rw" + if mount.ReadOnly { + mode = "ro" + } + result = append(result, hostPath+":"+mount.MountPath+":"+mode) + } + return result, nil +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'" +} diff --git a/internal/ztp/templates/generated-mode.sh.gotmpl b/internal/ztp/templates/generated-mode.sh.gotmpl new file mode 100644 index 0000000..f3cd63b --- /dev/null +++ b/internal/ztp/templates/generated-mode.sh.gotmpl @@ -0,0 +1,57 @@ +#!/bin/bash +set -euo pipefail + +# Generated by sonic-operator {{ shellQuote .SwitchName }} +{{ if .ConfigureONIE }} +# Configure ONIE install discovery for the next and every subsequent SONiC reboot. +mkdir -p /onieboot +cat >/etc/systemd/system/onieboot.mount <<'EOF' +[Unit] +Description=ONIE boot partition + +[Mount] +What=LABEL=ONIE-BOOT +Where=/onieboot +Type=auto +Options=defaults + +[Install] +WantedBy=multi-user.target +EOF + +cat >/etc/systemd/system/sonic-operator-onie-install.service <<'EOF' +[Unit] +Description=Configure next boot for ONIE install discovery +Requires=onieboot.mount +After=onieboot.mount + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'grub-editenv /host/grub/grubenv set next_entry=ONIE' +ExecStart=/bin/sh -c 'grub-editenv /onieboot/grub/grubenv set onie_mode=install' + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable --now onieboot.mount +systemctl enable --now sonic-operator-onie-install.service +{{ end }} +config hostname {{ shellQuote .Hostname }} +config save -y +{{ if .Containers }} +# containers +{{ range .Containers }}if ! ( +{{ if .InjectControlKubeconfig }}install -d -m 0700 /etc/sonic-operator/credentials && +install -d -m 0700 {{ shellQuote (printf "/etc/sonic-operator/credentials/%s" .Name) }} && +printf '%s' {{ shellQuote .EncodedControlKubeconfig }} | base64 -d >{{ shellQuote .ControlKubeconfigPath }} && +chown {{ .DockerUser }} {{ shellQuote .ControlKubeconfigPath }} && +chmod 0600 {{ shellQuote .ControlKubeconfigPath }} && +{{ end }}docker pull {{ shellQuote .Image }} && +(docker rm -f {{ shellQuote .Name }} >/dev/null 2>&1 || true) && +docker run -d --name {{ shellQuote .Name }} --network host --restart unless-stopped{{ if .DockerUser }} --user {{ shellQuote .DockerUser }}{{ end }}{{ range .VolumeMounts }} -v {{ shellQuote . }}{{ end }}{{ if .InjectControlKubeconfig }} -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig -v {{ shellQuote (printf "%s:/var/run/sonic-operator/control-kubeconfig:ro" .ControlKubeconfigPath) }}{{ end }}{{ if .Entrypoint }} --entrypoint {{ shellQuote .Entrypoint }}{{ end }} {{ shellQuote .Image }}{{ range .Command }} {{ shellQuote . }}{{ end }}{{ range .Args }} {{ shellQuote . }}{{ end }} +); then + echo {{ shellQuote .FailureMessage }} >&2 +fi +{{ end }}{{ end }}sync diff --git a/internal/ztp/testdata/generated-mode.golden.sh b/internal/ztp/testdata/generated-mode.golden.sh new file mode 100644 index 0000000..4b18433 --- /dev/null +++ b/internal/ztp/testdata/generated-mode.golden.sh @@ -0,0 +1,64 @@ +#!/bin/bash +set -euo pipefail + +# Generated by sonic-operator 'switch-1' + +# Configure ONIE install discovery for the next and every subsequent SONiC reboot. +mkdir -p /onieboot +cat >/etc/systemd/system/onieboot.mount <<'EOF' +[Unit] +Description=ONIE boot partition + +[Mount] +What=LABEL=ONIE-BOOT +Where=/onieboot +Type=auto +Options=defaults + +[Install] +WantedBy=multi-user.target +EOF + +cat >/etc/systemd/system/sonic-operator-onie-install.service <<'EOF' +[Unit] +Description=Configure next boot for ONIE install discovery +Requires=onieboot.mount +After=onieboot.mount + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'grub-editenv /host/grub/grubenv set next_entry=ONIE' +ExecStart=/bin/sh -c 'grub-editenv /onieboot/grub/grubenv set onie_mode=install' + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable --now onieboot.mount +systemctl enable --now sonic-operator-onie-install.service + +config hostname 'switch-1.lab.example' +config save -y + +# containers +if ! ( +install -d -m 0700 /etc/sonic-operator/credentials && +install -d -m 0700 '/etc/sonic-operator/credentials/wirelet' && +printf '%s' 'YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCg==' | base64 -d >'/etc/sonic-operator/credentials/wirelet/control-kubeconfig' && +chown 65532:65532 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig' && +chmod 0600 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig' && +docker pull 'ghcr.io/hardikdr/wirelet:fixed-1' && +(docker rm -f 'wirelet' >/dev/null 2>&1 || true) && +docker run -d --name 'wirelet' --network host --restart unless-stopped --user '65532:65532' -e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig -v '/etc/sonic-operator/credentials/wirelet/control-kubeconfig:/var/run/sonic-operator/control-kubeconfig:ro' 'ghcr.io/hardikdr/wirelet:fixed-1' '--name=switch-1' '--interface=Ethernet0' +); then + echo 'sonic-operator: container wirelet failed; continuing' >&2 +fi +if ! ( +docker pull 'ghcr.io/giluerre/sonic-agent:latest' && +(docker rm -f 'sonic-agent' >/dev/null 2>&1 || true) && +docker run -d --name 'sonic-agent' --network host --restart unless-stopped --user '0' -v '/var/run/dbus:/var/run/dbus:rw' -v '/etc/sonic/sonic_version.yml:/etc/sonic/sonic_version.yml:ro' --entrypoint '/switch-agent-server' 'ghcr.io/giluerre/sonic-agent:latest' '-port' '57400' +); then + echo 'sonic-operator: container sonic-agent failed; continuing' >&2 +fi +sync diff --git a/internal/ztp/ztp.go b/internal/ztp/ztp.go index f13a19e..5531a22 100644 --- a/internal/ztp/ztp.go +++ b/internal/ztp/ztp.go @@ -79,6 +79,19 @@ type configMapHandler struct { reader client.Reader } +type generatedHandler struct { + reader client.Reader + controlKubeconfigFilePath string +} + +// GeneratedOptions configures declarative, Switch-backed ZTP rendering. +type GeneratedOptions struct { + // ControlKubeconfigFile is an optional kubeconfig mounted into the operator + // pod. It is injected only into generated containers that + // explicitly opt in. + ControlKubeconfigFile string +} + func Register(mux *http.ServeMux, c Config) { t := template.New("ztp-scripts") t = t.Funcs(template.FuncMap{ @@ -87,7 +100,11 @@ func Register(mux *http.ServeMux, c Config) { "searchDomain": func() string { return c.SearchDomain }, "interfacePrefix": interfacePrefix, }) - t = template.Must(t.ParseFS(templateFS, "templates/*.gotmpl")) + t = template.Must(t.ParseFS( + templateFS, + "templates/leaf.sh.gotmpl", + "templates/spine.sh.gotmpl", + )) mux.Handle("GET /ztp", &handler{t: t, m: c.SwitchParams}) } @@ -99,6 +116,16 @@ func RegisterConfigMap(mux *http.ServeMux, reader client.Reader) { mux.Handle("GET /ztp", &configMapHandler{reader: reader}) } +// RegisterGenerated renders a complete ZTP script from the matching Switch +// object. Unlike configmap mode, no user-provided ZTP script is read or +// modified. +func RegisterGenerated(mux *http.ServeMux, reader client.Reader, options GeneratedOptions) { + mux.Handle("GET /ztp", &generatedHandler{ + reader: reader, + controlKubeconfigFilePath: options.ControlKubeconfigFile, + }) +} + // interfacePrefix takes the interface ID and the /64 prefix of the switch and // reserves a unique /112 prefix for each interface based on the interface ID. func interfacePrefix(prefix netip.Prefix, interfaceID int) (string, error) { @@ -141,18 +168,64 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (h *configMapHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - ap, err := netip.ParseAddrPort(r.RemoteAddr) + matched, err := matchingSwitch(r, h.reader) if err != nil { handleErr(w, err) return } + if matched.Spec.ZTP.ScriptRef == nil { + handleErr(w, fmt.Errorf("switch %q has no ztp.scriptRef for configmap mode", matched.Name)) + return + } - switches := &networkingv1alpha1.SwitchList{} - if err := h.reader.List(r.Context(), switches); err != nil { - handleErr(w, fmt.Errorf("listing switches for ZTP: %w", err)) + ref := matched.Spec.ZTP.ScriptRef + configMap := &corev1.ConfigMap{} + if err := h.reader.Get(r.Context(), types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}, configMap); err != nil { + handleErr(w, fmt.Errorf("getting ZTP ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err)) + return + } + script, ok := configMap.Data[ref.Key] + if !ok { + handleErr(w, fmt.Errorf("ZTP ConfigMap %s/%s has no key %q", ref.Namespace, ref.Name, ref.Key)) + return + } + + w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") + if _, err := fmt.Fprint(w, script); err != nil { + slog.Error("failed to write ZTP ConfigMap script", "switch", matched.Name, "err", err) + } +} + +func (h *generatedHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + matched, err := matchingSwitch(r, h.reader) + if err != nil { + handleErr(w, err) return } + script, err := renderGeneratedScript(matched, h.controlKubeconfigFilePath) + if err != nil { + handleErr(w, fmt.Errorf("rendering generated ZTP for switch %q: %w", matched.Name, err)) + return + } + + w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") + if _, err := fmt.Fprint(w, script); err != nil { + slog.Error("failed to write generated ZTP script", "switch", matched.Name, "err", err) + } +} + +func matchingSwitch(r *http.Request, reader client.Reader) (*networkingv1alpha1.Switch, error) { + ap, err := netip.ParseAddrPort(r.RemoteAddr) + if err != nil { + return nil, err + } + + switches := &networkingv1alpha1.SwitchList{} + if err := reader.List(r.Context(), switches); err != nil { + return nil, fmt.Errorf("listing switches for ZTP: %w", err) + } + var matched *networkingv1alpha1.Switch for i := range switches.Items { switchConfig := switches.Items[i].Spec.ZTP @@ -161,39 +234,20 @@ func (h *configMapHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } configuredAddr, err := netip.ParseAddr(switchConfig.SourceAddress) if err != nil { - handleErr(w, fmt.Errorf("switch %q has invalid ZTP source address %q: %w", switches.Items[i].Name, switchConfig.SourceAddress, err)) - return + return nil, fmt.Errorf("switch %q has invalid ZTP source address %q: %w", switches.Items[i].Name, switchConfig.SourceAddress, err) } if configuredAddr != ap.Addr() { continue } if matched != nil { - handleErr(w, fmt.Errorf("multiple switches use ZTP source address %q", ap.Addr())) - return + return nil, fmt.Errorf("multiple switches use ZTP source address %q", ap.Addr()) } matched = &switches.Items[i] } if matched == nil { - handleErr(w, fmt.Errorf("no switch configured for ZTP source address %q", ap.Addr())) - return - } - - ref := matched.Spec.ZTP.ScriptRef - configMap := &corev1.ConfigMap{} - if err := h.reader.Get(r.Context(), types.NamespacedName{Namespace: ref.Namespace, Name: ref.Name}, configMap); err != nil { - handleErr(w, fmt.Errorf("getting ZTP ConfigMap %s/%s: %w", ref.Namespace, ref.Name, err)) - return - } - script, ok := configMap.Data[ref.Key] - if !ok { - handleErr(w, fmt.Errorf("ZTP ConfigMap %s/%s has no key %q", ref.Namespace, ref.Name, ref.Key)) - return - } - - w.Header().Set("Content-Type", "text/x-shellscript; charset=utf-8") - if _, err := fmt.Fprint(w, script); err != nil { - slog.Error("failed to write ZTP ConfigMap script", "switch", matched.Name, "err", err) + return nil, fmt.Errorf("no switch configured for ZTP source address %q", ap.Addr()) } + return matched, nil } func handleErr(w http.ResponseWriter, e error) { diff --git a/internal/ztp/ztp_test.go b/internal/ztp/ztp_test.go index 864ac47..31bc354 100644 --- a/internal/ztp/ztp_test.go +++ b/internal/ztp/ztp_test.go @@ -6,6 +6,10 @@ package ztp import ( "net/http" "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" "testing" corev1 "k8s.io/api/core/v1" @@ -16,6 +20,8 @@ import ( networkingv1alpha1 "github.com/ironcore-dev/sonic-operator/api/v1alpha1" ) +const matchingSwitchRemoteAddr = "192.0.2.10:1234" + func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { @@ -28,10 +34,16 @@ func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( &networkingv1alpha1.Switch{ ObjectMeta: metav1.ObjectMeta{Name: "leaf-01"}, - Spec: networkingv1alpha1.SwitchSpec{ZTP: &networkingv1alpha1.ZTP{ - SourceAddress: "192.0.2.10", - ScriptRef: networkingv1alpha1.ZTPConfigMapReference{Namespace: "provisioning", Name: "leaf-01-ztp", Key: "ztp.sh"}, - }}, + Spec: networkingv1alpha1.SwitchSpec{ + ZTP: &networkingv1alpha1.ZTP{ + SourceAddress: "192.0.2.10", + ScriptRef: &networkingv1alpha1.ZTPConfigMapReference{Namespace: "provisioning", Name: "leaf-01-ztp", Key: "ztp.sh"}, + }, + Containers: []networkingv1alpha1.Container{{ + Name: "ignored-in-configmap-mode", + Image: "example.invalid/ignored:latest", + }}, + }, }, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Namespace: "provisioning", Name: "leaf-01-ztp"}, @@ -42,7 +54,7 @@ func TestConfigMapHandlerServesMatchingSwitchScript(t *testing.T) { mux := http.NewServeMux() RegisterConfigMap(mux, c) req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) - req.RemoteAddr = "192.0.2.10:1234" + req.RemoteAddr = matchingSwitchRemoteAddr response := httptest.NewRecorder() mux.ServeHTTP(response, req) @@ -79,3 +91,193 @@ func TestConfigMapHandlerDoesNotFallBackForUnknownSwitch(t *testing.T) { t.Error("expected an error body for an unknown switch") } } + +func TestGeneratedHandlerRendersCompleteSwitchScript(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := networkingv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + kubeconfigPath := filepath.Join(t.TempDir(), "kubeconfig") + if err := os.WriteFile(kubeconfigPath, []byte("apiVersion: v1\nkind: Config\n"), 0o600); err != nil { + t.Fatal(err) + } + wireletUID := int64(65532) + agentUID := int64(0) + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "switch-1"}, + Spec: networkingv1alpha1.SwitchSpec{ + Hostname: "switch-1.lab.example", + ZTP: &networkingv1alpha1.ZTP{ + SourceAddress: "192.0.2.10", + }, + Volumes: []networkingv1alpha1.Volume{ + {Name: "dbus", HostPath: &networkingv1alpha1.HostPathVolumeSource{Path: "/var/run/dbus"}}, + {Name: "sonic-version", HostPath: &networkingv1alpha1.HostPathVolumeSource{Path: "/etc/sonic/sonic_version.yml"}}, + }, + Containers: []networkingv1alpha1.Container{ + { + Name: "wirelet", + Image: "ghcr.io/hardikdr/wirelet:fixed-1", + SecurityContext: &networkingv1alpha1.ContainerSecurityContext{RunAsUser: &wireletUID, RunAsGroup: &wireletUID}, + Args: []string{"--name=switch-1", "--interface=Ethernet0"}, + InjectControlKubeconfig: true, + }, + { + Name: "sonic-agent", + Image: "ghcr.io/giluerre/sonic-agent:latest", + Command: []string{"/switch-agent-server"}, + Args: []string{"-port", "57400"}, + SecurityContext: &networkingv1alpha1.ContainerSecurityContext{ + RunAsUser: &agentUID, + }, + VolumeMounts: []networkingv1alpha1.VolumeMount{ + {Name: "dbus", MountPath: "/var/run/dbus"}, + {Name: "sonic-version", MountPath: "/etc/sonic/sonic_version.yml", ReadOnly: true}, + }, + }, + }, + NextBootMode: networkingv1alpha1.NextBootModeInstallOS, + }, + }, + ).Build() + + mux := http.NewServeMux() + RegisterGenerated(mux, c, GeneratedOptions{ControlKubeconfigFile: kubeconfigPath}) + req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) + req.RemoteAddr = matchingSwitchRemoteAddr + response := httptest.NewRecorder() + + mux.ServeHTTP(response, req) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusOK, response.Body.String()) + } + body := response.Body.String() + wantBody, err := os.ReadFile("testdata/generated-mode.golden.sh") + if err != nil { + t.Fatal(err) + } + if body != string(wantBody) { + t.Errorf("rendered script differs from testdata/generated-mode.golden.sh\ngot:\n%s\nwant:\n%s", body, wantBody) + } + for _, want := range []string{ + "#!/bin/bash", + "set -euo pipefail", + "config hostname 'switch-1.lab.example'", + "config save -y", + "printf '%s' 'YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCg==' | base64 -d >'/etc/sonic-operator/credentials/wirelet/control-kubeconfig'", + "chown 65532:65532 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig'", + "chmod 0600 '/etc/sonic-operator/credentials/wirelet/control-kubeconfig'", + "docker pull 'ghcr.io/hardikdr/wirelet:fixed-1'", + "docker run -d --name 'wirelet' --network host --restart unless-stopped --user '65532:65532'", + "-e KUBECONFIG=/var/run/sonic-operator/control-kubeconfig", + "-v '/etc/sonic-operator/credentials/wirelet/control-kubeconfig:/var/run/sonic-operator/control-kubeconfig:ro'", + "'--name=switch-1' '--interface=Ethernet0'", + "docker run -d --name 'sonic-agent' --network host --restart unless-stopped --user '0' -v '/var/run/dbus:/var/run/dbus:rw' -v '/etc/sonic/sonic_version.yml:/etc/sonic/sonic_version.yml:ro' --entrypoint '/switch-agent-server' 'ghcr.io/giluerre/sonic-agent:latest' '-port' '57400'", + "What=LABEL=ONIE-BOOT", + "sonic-operator-onie-install.service", + "set next_entry=ONIE", + "set onie_mode=install", + } { + if !strings.Contains(body, want) { + t.Errorf("response does not contain %q:\n%s", want, body) + } + } + if strings.Index(body, "# Configure ONIE install discovery") > strings.Index(body, "config hostname") { + t.Error("ONIE boot lifecycle configuration must be rendered before hostname configuration") + } + if !strings.Contains(body, "sonic-operator: container wirelet failed; continuing") { + t.Errorf("response does not isolate bootstrap-container failures:\n%s", body) + } + + scriptPath := filepath.Join(t.TempDir(), "generated-ztp.sh") + if err := os.WriteFile(scriptPath, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if output, err := exec.Command("bash", "-n", scriptPath).CombinedOutput(); err != nil { + t.Fatalf("generated script is not valid Bash: %v: %s", err, output) + } +} + +func TestGeneratedHandlerRejectsMissingControlKubeconfig(t *testing.T) { + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := networkingv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects( + &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "switch-1"}, + Spec: networkingv1alpha1.SwitchSpec{ + ZTP: &networkingv1alpha1.ZTP{ + SourceAddress: "192.0.2.10", + }, + Containers: []networkingv1alpha1.Container{{ + Name: "wirelet", + Image: "example.invalid/wirelet:latest", + InjectControlKubeconfig: true, + }}, + }, + }, + ).Build() + + mux := http.NewServeMux() + RegisterGenerated(mux, c, GeneratedOptions{}) + req := httptest.NewRequest(http.MethodGet, "http://provisioning.example/ztp", nil) + req.RemoteAddr = matchingSwitchRemoteAddr + response := httptest.NewRecorder() + + mux.ServeHTTP(response, req) + + if response.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d: %s", response.Code, http.StatusInternalServerError, response.Body.String()) + } + if !strings.Contains(response.Body.String(), "bootstrap-control-kubeconfig-file") { + t.Errorf("response = %q, want missing kubeconfig error", response.Body.String()) + } +} + +func TestRenderGeneratedScriptDefaultsHostnameToSwitchName(t *testing.T) { + switchObject := &networkingv1alpha1.Switch{ + ObjectMeta: metav1.ObjectMeta{Name: "leaf-1"}, + } + + script, err := renderGeneratedScript(switchObject, "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(script, "config hostname 'leaf-1'") { + t.Errorf("generated script does not default hostname to switch name:\n%s", script) + } +} + +func TestShellQuoteRoundTripsThroughBash(t *testing.T) { + tests := map[string]string{ + "empty": "", + "simple": "simple", + "spaces": "contains spaces", + "single quote": "a'b", + "command expansion": `$(echo unsafe)`, + } + + for name, value := range tests { + t.Run(name, func(t *testing.T) { + output, err := exec.Command("bash", "-c", "printf '%s' "+shellQuote(value)).CombinedOutput() + if err != nil { + t.Fatalf("executing quoted value: %v: %s", err, output) + } + if got := string(output); got != value { + t.Errorf("round-trip = %q, want %q", got, value) + } + }) + } +}