diff --git a/api/machine.go b/api/machine.go index dfbcc7c..a7fe307 100644 --- a/api/machine.go +++ b/api/machine.go @@ -96,8 +96,7 @@ const ( type VolumeType string const ( - VolumeSocketType VolumeType = "socket" - VolumeFileType VolumeType = "file" + VolumeFileType VolumeType = "file" ) type NetworkInterfaceSpec struct { diff --git a/cmd/cloud-hypervisor-provider/app/app.go b/cmd/cloud-hypervisor-provider/app/app.go index 7c21927..8ccdc17 100644 --- a/cmd/cloud-hypervisor-provider/app/app.go +++ b/cmd/cloud-hypervisor-provider/app/app.go @@ -15,9 +15,9 @@ import ( "github.com/ironcore-dev/cloud-hypervisor-provider/internal/controllers" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/host" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/mcr" - "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/networkinterface/options" + nicoptions "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/networkinterface/options" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume" - "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph" + cephoptions "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph/options" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/localdisk" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/raw" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/server" @@ -52,9 +52,8 @@ type Options struct { CloudHypervisorBinPath string CloudHypervisorFirmwarePath string - QMPSocketPath string - - NicPlugin *options.Options + CephPlugin *cephoptions.Options + NicPlugin *nicoptions.Options } func (o *Options) AddFlags(fs *pflag.FlagSet) { @@ -74,13 +73,6 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Path to the directory of the machine store.", ) - fs.StringVar( - &o.QMPSocketPath, - "qmp-socket-path", - "/run/chp/qmp/sock", - "Path to the qmp socket.", - ) - fs.StringVar( &o.CloudHypervisorBinPath, "cloud-hypervisor-bin-path", @@ -101,7 +93,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { "Supported machine classes (format: name,cpu,memory)", ) - o.NicPlugin = options.NewDefaultOptions() + o.CephPlugin = cephoptions.NewDefaultOptions() + o.CephPlugin.AddFlags(fs) + + o.NicPlugin = nicoptions.NewDefaultOptions() o.NicPlugin.AddFlags(fs) } @@ -184,20 +179,15 @@ func Run(ctx context.Context, opts Options) error { return err } - qmpProvider, err := ceph.QMPProvider( - ctx, - log.WithName("ceph-volume-plugin"), - hostPaths, - opts.QMPSocketPath, - ) + cephPlugin, err := opts.CephPlugin.VolumePlugin(ctx, log.WithName("ceph-volume-plugin"), hostPaths) if err != nil { - setupLog.Error(err, "failed to initialize qmp provider") + setupLog.Error(err, "failed to initialize ceph volume plugin") return err } pluginManager := volume.NewPluginManager() if err := pluginManager.InitPlugins(hostPaths, []volume.Plugin{ - ceph.NewPlugin(qmpProvider), + cephPlugin, localdisk.NewPlugin(rawInst, imgCache), }); err != nil { setupLog.Error(err, "failed to initialize plugins") diff --git a/dev-vm/chp-dev.yaml b/dev-vm/chp-dev.yaml index 4526381..8b30419 100644 --- a/dev-vm/chp-dev.yaml +++ b/dev-vm/chp-dev.yaml @@ -258,17 +258,12 @@ provision: cat > /etc/tmpfiles.d/chp.conf <<'EOF' d /run/chp 0775 chp chp - - d /run/chp/qmp 0775 chp chp - EOF systemd-tmpfiles --create /etc/tmpfiles.d/chp.conf systemctl daemon-reload systemctl enable cloud-hypervisor-prepare.service systemctl start cloud-hypervisor-prepare.service - - apt-get install -y qemu-utils - systemctl enable chp-qmp.service - systemctl start chp-qmp.service - mode: system script: | #!/bin/bash @@ -318,12 +313,6 @@ provision: permissions: 644 file: url: cloud-hypervisor-prepare.service - - mode: data - path: /etc/systemd/system/chp-qmp.service - owner: "root:root" - permissions: 644 - file: - url: chp-qmp.service # provision: # # `system` is executed with root privileges # - mode: system diff --git a/dev-vm/chp-qmp.service b/dev-vm/chp-qmp.service deleted file mode 100644 index 1a6b8bf..0000000 --- a/dev-vm/chp-qmp.service +++ /dev/null @@ -1,23 +0,0 @@ -[Unit] -Description=Cloud Hypervisor Provider QMP monitor (qemu-storage-daemon) -After=network-online.target systemd-tmpfiles-setup.service -Wants=network-online.target - -[Service] -Type=simple -User=chp -Group=chp - -ExecStart=/usr/bin/qemu-storage-daemon \ - --chardev socket,id=qmpsock,path=/run/chp/qmp/sock,server=on,wait=off \ - --monitor chardev=qmpsock -ExecStartPost=/bin/sh -c 'for i in $(seq 1 50); do [ -S /run/chp/qmp/sock ] && { chmod 0660 /run/chp/qmp/sock; exit 0; }; sleep 0.1; done' - -Restart=on-failure -RestartSec=1 - -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/go.mod b/go.mod index 7ba3975..ff889d8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.26.4 require ( github.com/blang/semver/v4 v4.0.0 - github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 github.com/getkin/kin-openapi v0.152.0 github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 @@ -37,7 +36,6 @@ require ( github.com/containerd/log v0.2.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/emicklei/go-restful/v3 v3.14.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect @@ -93,7 +91,6 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.58.0 // indirect golang.org/x/mod v0.42.0 // indirect golang.org/x/net v0.61.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect diff --git a/go.sum b/go.sum index c746928..a2d105c 100644 --- a/go.sum +++ b/go.sum @@ -35,10 +35,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8 h1:ZQhCuAXz9mLpTkBCwtS7ajWp3lU7pqiaOM0fxzIyQFc= -github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4= -github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 h1:rh6rt8pF5U4iyQ86h6lRDenJoX4ht2wFnZXB9ogIrIM= -github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242/go.mod h1:LGHUtlhsY4vRGM6AHejEQKVI5e3eHbSylMHwTSpQtVw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= @@ -253,8 +249,6 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= -golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= diff --git a/internal/controllers/machine_controller.go b/internal/controllers/machine_controller.go index fb1e41c..896c9a3 100644 --- a/internal/controllers/machine_controller.go +++ b/internal/controllers/machine_controller.go @@ -240,7 +240,7 @@ func (r *MachineReconciler) deleteMachine(ctx context.Context, log logr.Logger, } log.V(2).Info("Delete volume", "name", vol.Name, "plugin", plugin.Name()) - if err := plugin.Delete(ctx, vol.Name, machine.ID); err != nil { + if err := plugin.Delete(ctx, vol, machine.ID); err != nil { return fmt.Errorf("failed to delete volume %s: %w", vol.Name, err) } } @@ -285,7 +285,7 @@ func (r *MachineReconciler) reconcileVolumes(ctx context.Context, log logr.Logge if vol.DeletedAt != nil { if status.State != api.VolumeStateAttached { log.V(2).Info("Delete not attached volume", "name", vol.Name) - if err := plugin.Delete(ctx, vol.Name, machine.ID); err != nil { + if err := plugin.Delete(ctx, vol, machine.ID); err != nil { return fmt.Errorf("failed to delete volume %s: %w", vol.Name, err) } continue @@ -393,7 +393,14 @@ func (r *MachineReconciler) attachDetachDisks( } log.V(1).Info("Removed disk", "disk", vol.Name) - updatedVolumeStatus = append(updatedVolumeStatus, status) + plugin, err := r.VolumePluginManager.FindPluginBySpec(vol) + if err != nil { + return fmt.Errorf("failed to find plugin for volume %s: %w", vol.Name, err) + } + if err := plugin.Delete(ctx, vol, machine.ID); err != nil { + return fmt.Errorf("failed to cleanup volume %s: %w", vol.Name, err) + } + log.V(1).Info("Cleaned up volume", "disk", vol.Name) continue } diff --git a/internal/plugins/volume/ceph/ceph.go b/internal/plugins/volume/ceph/ceph.go index 78a7988..923e9ae 100644 --- a/internal/plugins/volume/ceph/ceph.go +++ b/internal/plugins/volume/ceph/ceph.go @@ -9,9 +9,7 @@ import ( "net" "os" "strings" - "time" - "github.com/digitalocean/go-qemu/qmp" "github.com/go-logr/logr" "github.com/ironcore-dev/cloud-hypervisor-provider/api" "github.com/ironcore-dev/cloud-hypervisor-provider/internal/host" @@ -44,47 +42,24 @@ type validatedVolume struct { encryptionKey *string } -type Provider interface { +type backend interface { + VolumeType() api.VolumeType Mount(ctx context.Context, machineID string, volume *validatedVolume) (string, error) - Unmount(ctx context.Context, machineID string, volumeID string) error + Unmount(ctx context.Context, machineID string, volume *validatedVolume) error } -func QMPProvider(ctx context.Context, log logr.Logger, paths host.Paths, socket string) (Provider, error) { - monitor, err := qmp.NewSocketMonitor("unix", socket, 2*time.Second) - if err != nil { - return nil, fmt.Errorf("failed to connect to qmp monitor: %w", err) - } - - go func() { - // TODO - _ = monitor.Connect() - defer func() { - // TODO - _ = monitor.Disconnect() - }() - - stream, _ := monitor.Events(ctx) - for e := range stream { - log.V(1).Info(fmt.Sprintf("EVENT: %s", e.Event)) - } - }() - - return &QMP{ - log: log, - paths: paths, - monitor: monitor, - }, nil +// NewRBDPlugin creates a volume.Plugin backed by host-level rbd map/unmap. +func NewRBDPlugin(log logr.Logger, paths host.Paths) volume.Plugin { + return newPlugin(newRBDProvider(log, paths)) } type plugin struct { - provider Provider - host volume.Host + be backend + host volume.Host } -func NewPlugin(provider Provider) volume.Plugin { - return &plugin{ - provider: provider, - } +func newPlugin(be backend) volume.Plugin { + return &plugin{be: be} } func (p *plugin) Init(host volume.Host) error { @@ -181,14 +156,14 @@ func (p *plugin) Apply(ctx context.Context, spec *api.VolumeSpec, machineID stri return nil, fmt.Errorf("failed to get volume data: %w", err) } - path, err := p.provider.Mount(ctx, machineID, volumeData) + path, err := p.be.Mount(ctx, machineID, volumeData) if err != nil { return nil, fmt.Errorf("failed to mount volume: %w", err) } return &api.VolumeStatus{ Name: spec.Name, - Type: api.VolumeSocketType, + Type: p.be.VolumeType(), Path: path, Handle: volumeData.handle, State: api.VolumeStatePrepared, @@ -237,10 +212,43 @@ func (p *plugin) validateVolume(spec *api.VolumeSpec) (vData *validatedVolume, e return vData, nil } -func (p *plugin) Delete(ctx context.Context, computeVolumeName string, machineID string) error { - if err := p.provider.Unmount(ctx, machineID, computeVolumeName); err != nil { - return fmt.Errorf("failed to unmount volume %q: %w", computeVolumeName, err) +func (p *plugin) Delete(ctx context.Context, spec *api.VolumeSpec, machineID string) error { + volume, err := p.volumeForUnmount(spec) + if err != nil { + return fmt.Errorf("failed to get volume data for deletion: %w", err) } - return os.RemoveAll(p.host.MachineVolumeDir(machineID, cephDriverName, computeVolumeName)) + if err := p.be.Unmount(ctx, machineID, volume); err != nil { + return fmt.Errorf("failed to unmount volume %q: %w", spec.Name, err) + } + + return os.RemoveAll(p.host.MachineVolumeDir(machineID, cephDriverName, spec.Name)) +} + +func (p *plugin) volumeForUnmount(spec *api.VolumeSpec) (*validatedVolume, error) { + if spec == nil { + return nil, fmt.Errorf("volume is nil") + } + connection := spec.Connection + if connection == nil { + return nil, fmt.Errorf("volume does not specify connection") + } + if connection.Driver != cephDriverName { + return nil, fmt.Errorf("volume connection specifies invalid driver %q", connection.Driver) + } + if connection.Attributes == nil { + return nil, fmt.Errorf("volume connection does not specify attributes") + } + if connection.Handle == "" { + return nil, fmt.Errorf("volume connection does not specify handle") + } + + volume := &validatedVolume{ + name: spec.Name, + handle: connection.Handle, + } + if err := readVolumeAttributes(connection.Attributes, volume); err != nil { + return nil, fmt.Errorf("error reading volume attributes: %w", err) + } + return volume, nil } diff --git a/internal/plugins/volume/ceph/options/options.go b/internal/plugins/volume/ceph/options/options.go new file mode 100644 index 0000000..2b45cf1 --- /dev/null +++ b/internal/plugins/volume/ceph/options/options.go @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package options + +import ( + "context" + "fmt" + "sort" + + "github.com/go-logr/logr" + "github.com/ironcore-dev/cloud-hypervisor-provider/internal/host" + "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume" + "github.com/spf13/pflag" +) + +type TypeOptions interface { + PluginName() string + AddFlags(fs *pflag.FlagSet) + VolumePlugin(ctx context.Context, log logr.Logger, paths host.Paths) (volume.Plugin, error) +} + +type TypeOptionsRegistry struct { + nameToPluginOpts map[string]typeOptionsAndPriority +} + +type typeOptionsAndPriority struct { + TypeOptions + priority int +} + +func NewTypeOptionsRegistry() *TypeOptionsRegistry { + return &TypeOptionsRegistry{ + nameToPluginOpts: make(map[string]typeOptionsAndPriority), + } +} + +func (r *TypeOptionsRegistry) Register(pluginOpts TypeOptions, priority int) error { + pluginName := pluginOpts.PluginName() + if _, ok := r.nameToPluginOpts[pluginName]; ok { + return fmt.Errorf("plugin %q already registered", pluginName) + } + + r.nameToPluginOpts[pluginName] = typeOptionsAndPriority{pluginOpts, priority} + return nil +} + +func (r *TypeOptionsRegistry) PluginNames() []string { + names := make([]string, 0, len(r.nameToPluginOpts)) + for name := range r.nameToPluginOpts { + names = append(names, name) + } + sort.Slice(names, func(i, j int) bool { + return r.nameToPluginOpts[names[i]].priority < r.nameToPluginOpts[names[j]].priority + }) + return names +} + +func (r *TypeOptionsRegistry) DefaultPluginName() string { + var ( + pluginName string + maxPriority *int + ) + for name, entry := range r.nameToPluginOpts { + if maxPriority == nil || *maxPriority > entry.priority { + pluginName = name + maxPriority = new(entry.priority) + } + } + return pluginName +} + +func (r *TypeOptionsRegistry) ForeachPluginTypeOpts(f func(pluginName string, pluginOpts TypeOptions) bool) { + for pluginName, pluginOpts := range r.nameToPluginOpts { + if !f(pluginName, pluginOpts) { + break + } + } +} + +func (r *TypeOptionsRegistry) PluginTypeOptsByName(pluginName string) (TypeOptions, error) { + pluginOpts, ok := r.nameToPluginOpts[pluginName] + if !ok { + return nil, fmt.Errorf("no plugin options for plugin name %q", pluginName) + } + + return pluginOpts, nil +} + +type Options struct { + PluginName string + registry *TypeOptionsRegistry +} + +func NewOptions(registry *TypeOptionsRegistry) *Options { + return &Options{ + registry: registry, + } +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar( + &o.PluginName, "ceph-plugin-name", + o.registry.DefaultPluginName(), + fmt.Sprintf("Name of the ceph volume plugin to use. Available: %v", + o.registry.PluginNames()), + ) + o.registry.ForeachPluginTypeOpts(func(_ string, pluginOpts TypeOptions) bool { + pluginOpts.AddFlags(fs) + return true + }) +} + +func (o *Options) VolumePlugin(ctx context.Context, log logr.Logger, paths host.Paths) (volume.Plugin, error) { + pluginOpts, err := o.registry.PluginTypeOptsByName(o.PluginName) + if err != nil { + return nil, err + } + + return pluginOpts.VolumePlugin(ctx, log, paths) +} + +var DefaultPluginTypeRegistry = NewTypeOptionsRegistry() + +func NewDefaultOptions() *Options { + return NewOptions(DefaultPluginTypeRegistry) +} diff --git a/internal/plugins/volume/ceph/options/rbd.go b/internal/plugins/volume/ceph/options/rbd.go new file mode 100644 index 0000000..7d057b6 --- /dev/null +++ b/internal/plugins/volume/ceph/options/rbd.go @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package options + +import ( + "context" + + "github.com/go-logr/logr" + "github.com/ironcore-dev/cloud-hypervisor-provider/internal/host" + "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume" + "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph" + "github.com/spf13/pflag" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +type rbdOptions struct{} + +func (o *rbdOptions) PluginName() string { + return "rbd" +} + +func (o *rbdOptions) AddFlags(_ *pflag.FlagSet) {} + +func (o *rbdOptions) VolumePlugin(_ context.Context, log logr.Logger, paths host.Paths) (volume.Plugin, error) { + return ceph.NewRBDPlugin(log, paths), nil +} + +func init() { + utilruntime.Must(DefaultPluginTypeRegistry.Register(&rbdOptions{}, 5)) +} diff --git a/internal/plugins/volume/ceph/qmp.go b/internal/plugins/volume/ceph/qmp.go deleted file mode 100644 index 1114609..0000000 --- a/internal/plugins/volume/ceph/qmp.go +++ /dev/null @@ -1,378 +0,0 @@ -// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and IronCore contributors -// SPDX-License-Identifier: Apache-2.0 - -package ceph - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/digitalocean/go-qemu/qmp" - "github.com/go-logr/logr" - "github.com/ironcore-dev/cloud-hypervisor-provider/internal/host" -) - -type QMP struct { - log logr.Logger - paths host.Paths - monitor *qmp.SocketMonitor -} - -func (q *QMP) Mount(_ context.Context, machineID string, volume *validatedVolume) (string, error) { - volumeDir := q.volumeDir(machineID, volume.handle) - if err := os.MkdirAll(volumeDir, os.ModePerm); err != nil { - return "", err - } - - log := q.log.WithValues("machineID", machineID, "volumeID", volume.handle) - socketPath := filepath.Join(volumeDir, "socket") - - log.V(2).Info("Checking ceph conf") - confPath, err := q.createCephConf(log, machineID, volume) - if err != nil { - return "", fmt.Errorf("error creating ceph conf: %w", err) - } - - handle := fmt.Sprintf("ceph-%s", volume.name) - - if _, err := q.queryBlockNode(handle); err != nil { - if !errors.Is(err, ErrNotFound) { - return "", fmt.Errorf("error querying block device: %w", err) - } - - if err := q.addBlockDev(volume, confPath); err != nil { - return "", fmt.Errorf("error adding block device: %w", err) - } - } - - if _, err := q.queryBlockExports(handle); err != nil { - if !errors.Is(err, ErrNotFound) { - return "", fmt.Errorf("error querying block device: %w", err) - } - - if err := q.exportBlockDev(handle, socketPath); err != nil { - return "", fmt.Errorf("error adding block device: %w", err) - } - } - - return socketPath, nil -} - -func (q *QMP) Unmount(_ context.Context, machineID string, volumeName string) error { - - handle := fmt.Sprintf("ceph-%s", volumeName) - - if _, err := q.queryBlockExports(handle); err != nil { - if !errors.Is(err, ErrNotFound) { - return fmt.Errorf("error querying block device: %w", err) - } - } else { - if err := q.deleteExportBlockDev(handle); err != nil { - return fmt.Errorf("error deleting block device export: %w", err) - } - } - - if _, err := q.queryBlockNode(handle); err != nil { - if !errors.Is(err, ErrNotFound) { - return fmt.Errorf("error querying block device: %w", err) - } - } else { - if err := q.deleteBlockDev(handle); err != nil { - return fmt.Errorf("error deleting block device: %w", err) - } - } - - return nil - -} - -func (q *QMP) volumeDir(machineID string, volumeHandle string) string { - return q.paths.MachineVolumeDir(machineID, cephDriverName, volumeHandle) -} - -func (q *QMP) createCephConf(log logr.Logger, machineID string, volume *validatedVolume) (string, error) { - confPath := filepath.Join( - q.volumeDir(machineID, volume.handle), - "ceph.conf", - ) - keyPath := filepath.Join( - q.volumeDir(machineID, volume.handle), - "ceph.key", - ) - - log.V(2).Info("Creating ceph conf", "confPath", confPath) - confFile, err := os.OpenFile(confPath, os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - return "", fmt.Errorf("error opening conf file %s: %w", confPath, err) - } - - confData := fmt.Sprintf( - "[global]\nmon_host = %s \n\n[client.%s]\nkeyring = %s\n", - strings.Join(volume.monitors, ","), - volume.userID, - keyPath, - ) - _, err = confFile.WriteString(confData) - if err != nil { - return "", fmt.Errorf("error writing to conf file %s: %w", confPath, err) - } - - log.V(1).Info("Creating ceph key", "keyPath", keyPath) - keyFile, err := os.OpenFile(keyPath, os.O_CREATE|os.O_WRONLY, os.ModePerm) - if err != nil { - return "", fmt.Errorf("error opening key file %s: %w", keyPath, err) - } - - keyData := fmt.Sprintf("[client.%s]\nkey = %s\n", volume.userID, volume.userKey) - _, err = keyFile.WriteString(keyData) - if err != nil { - return "", fmt.Errorf("error writing to key file %s: %w", keyPath, err) - } - - return confPath, nil -} - -type BlockdevAddArguments struct { - NodeName string `json:"node-name"` - Driver string `json:"driver"` - Pool string `json:"pool"` - Image string `json:"image"` - User string `json:"user"` - Conf string `json:"conf"` - Discard string `json:"discard"` - Cache struct { - Direct bool `json:"direct"` - } `json:"cache"` -} - -type BlockExportAddArguments struct { - ID string `json:"id"` - NodeName string `json:"node-name"` - Type string `json:"type"` - Addr struct { - Type string `json:"type"` - Path string `json:"path"` - } `json:"addr"` - Writable bool `json:"writable"` -} - -type DeleteExportBlockDevArguments struct { - ID string `json:"id"` -} - -type DeleteBlockDevArguments struct { - Node string `json:"node-name"` -} - -type QMPRequest[T any] struct { - Execute string `json:"execute"` - Arguments T `json:"arguments,omitempty"` -} - -var ( - ErrNotFound = errors.New("not found") -) - -// nolint: unparam -func (q *QMP) queryBlockNode(nodeName string) (*BlockDevice, error) { - cmd, err := json.Marshal(QMPRequest[any]{ - Execute: "query-named-block-nodes", - }) - if err != nil { - return nil, fmt.Errorf("error marshalling cmd: %w", err) - } - - res, err := q.monitor.Run(cmd) - if err != nil { - return nil, fmt.Errorf("error executing cmd: %w", err) - } - - var devs BlockDevicesResponse - if err := json.Unmarshal(res, &devs); err != nil { - return nil, fmt.Errorf("error unmarshalling response: %w", err) - } - - for _, dev := range devs.Data { - if dev.NodeName == nodeName { - return &dev, nil - } - } - return nil, ErrNotFound -} - -// nolint: unparam -func (q *QMP) queryBlockExports(nodeName string) (*BlockExportNode, error) { - cmd, err := json.Marshal(QMPRequest[any]{ - Execute: "query-block-exports", - }) - if err != nil { - return nil, fmt.Errorf("error marshalling cmd: %w", err) - } - - res, err := q.monitor.Run(cmd) - if err != nil { - return nil, fmt.Errorf("error executing cmd: %w", err) - } - - var devs BlockExportResponse - if err := json.Unmarshal(res, &devs); err != nil { - return nil, fmt.Errorf("error unmarshalling response: %w", err) - } - - for _, dev := range devs.Data { - if dev.ID == nodeName { - return &dev, nil - } - } - return nil, ErrNotFound -} - -func (q *QMP) addBlockDev(volume *validatedVolume, confPath string) error { - cmd, err := json.Marshal(QMPRequest[BlockdevAddArguments]{ - Execute: "blockdev-add", - Arguments: BlockdevAddArguments{ - NodeName: fmt.Sprintf("ceph-%s", volume.name), - Driver: "rbd", - Pool: volume.pool, - Image: volume.image, - User: volume.userID, - Conf: confPath, - Discard: "unmap", - Cache: struct { - Direct bool `json:"direct"` - }{Direct: true}, - }, - }) - if err != nil { - return fmt.Errorf("error marshalling cmd: %w", err) - } - - if _, err := q.monitor.Run(cmd); err != nil { - return fmt.Errorf("error executing cmd: %w", err) - } - - return nil -} - -func (q *QMP) deleteBlockDev(handle string) error { - cmd, err := json.Marshal(QMPRequest[DeleteBlockDevArguments]{ - Execute: "blockdev-del", - Arguments: DeleteBlockDevArguments{ - Node: handle, - }, - }) - if err != nil { - return fmt.Errorf("error marshalling cmd: %w", err) - } - - if _, err := q.monitor.Run(cmd); err != nil { - return fmt.Errorf("error executing cmd: %w", err) - } - - return nil -} - -func (q *QMP) exportBlockDev(handle string, socketPath string) error { - cmd, err := json.Marshal(QMPRequest[BlockExportAddArguments]{ - Execute: "block-export-add", - Arguments: BlockExportAddArguments{ - ID: handle, - NodeName: handle, - Type: "vhost-user-blk", - Addr: struct { - Type string `json:"type"` - Path string `json:"path"` - }{ - Type: "unix", - Path: socketPath, - }, - Writable: true, - }, - }) - if err != nil { - return fmt.Errorf("error marshalling cmd: %w", err) - } - - if _, err := q.monitor.Run(cmd); err != nil { - return fmt.Errorf("error executing cmd: %w", err) - } - - return nil -} - -func (q *QMP) deleteExportBlockDev(nodeName string) error { - cmd, err := json.Marshal(QMPRequest[DeleteExportBlockDevArguments]{ - Execute: "block-export-del", - Arguments: DeleteExportBlockDevArguments{ - ID: nodeName, - }, - }) - if err != nil { - return fmt.Errorf("error marshalling cmd: %w", err) - } - - if _, err := q.monitor.Run(cmd); err != nil { - return fmt.Errorf("error executing cmd: %w", err) - } - - return nil -} - -type BlockExportResponse struct { - Data []BlockExportNode `json:"return"` -} - -type BlockExportNode struct { - NodeName string `json:"node-name"` - ShuttingDown bool `json:"shutting-down"` - Type string `json:"type"` - ID string `json:"id"` -} - -type BlockDevicesResponse struct { - Data []BlockDevice `json:"return"` -} - -type BlockDevice struct { - IOPSRd int `json:"iops_rd"` - IOPSWr int `json:"iops_wr"` - IOPS int `json:"iops"` - BPSRd int `json:"bps_rd"` - BPSWr int `json:"bps_wr"` - BPS int `json:"bps"` - WriteThreshold int `json:"write_threshold"` - DetectZeroes string `json:"detect_zeroes"` - NodeName string `json:"node-name"` - BackingFileDepth int `json:"backing_file_depth"` - Drv string `json:"drv"` - RO bool `json:"ro"` - Encrypted bool `json:"encrypted"` - Image BlockImage `json:"image"` - File string `json:"file"` - Cache BlockCache `json:"cache"` -} - -type BlockImage struct { - VirtualSize int64 `json:"virtual-size"` - Filename string `json:"filename"` - ClusterSize int64 `json:"cluster-size"` - Format string `json:"format"` - DirtyFlag bool `json:"dirty-flag"` - FormatSpecific FormatSpecificDetail `json:"format-specific"` -} - -type FormatSpecificDetail struct { - Type string `json:"type"` - Data interface{} `json:"data"` // empty object, or make a real struct if known -} - -type BlockCache struct { - NoFlush bool `json:"no-flush"` - Direct bool `json:"direct"` - Writeback bool `json:"writeback"` -} diff --git a/internal/plugins/volume/ceph/rbd.go b/internal/plugins/volume/ceph/rbd.go new file mode 100644 index 0000000..f709e1c --- /dev/null +++ b/internal/plugins/volume/ceph/rbd.go @@ -0,0 +1,201 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package ceph + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/go-logr/logr" + "github.com/ironcore-dev/cloud-hypervisor-provider/api" + "github.com/ironcore-dev/cloud-hypervisor-provider/internal/host" +) + +type rbdProvider struct { + log logr.Logger + paths host.Paths +} + +func newRBDProvider(log logr.Logger, paths host.Paths) *rbdProvider { + return &rbdProvider{ + log: log, + paths: paths, + } +} + +func (r *rbdProvider) VolumeType() api.VolumeType { + return api.VolumeFileType +} + +func (r *rbdProvider) Mount(ctx context.Context, machineID string, volume *validatedVolume) (string, error) { + volumeDir := r.volumeDir(machineID, volume.name) + if err := os.MkdirAll(volumeDir, 0o700); err != nil { + return "", err + } + + log := r.log.WithValues("machineID", machineID, "volumeID", volume.handle) + + log.V(2).Info("Creating ceph conf") + confPath, err := r.createCephConf(log, machineID, volume) + if err != nil { + return "", fmt.Errorf("error creating ceph conf: %w", err) + } + + devicePath, err := r.findMappedDevice(volume.pool, volume.image) + if err != nil { + return "", fmt.Errorf("error checking existing rbd mappings: %w", err) + } + + if devicePath != "" { + log.V(1).Info("RBD image already mapped", "device", devicePath) + return devicePath, nil + } + + devicePath, err = r.mapDevice(ctx, volume, confPath) + if err != nil { + return "", fmt.Errorf("error mapping rbd device: %w", err) + } + + log.V(1).Info("Mapped RBD image to device", "device", devicePath) + return devicePath, nil +} + +func (r *rbdProvider) Unmount(ctx context.Context, machineID string, volume *validatedVolume) error { + log := r.log.WithValues("machineID", machineID, "volumeName", volume.name) + + device, err := r.findMappedDevice(volume.pool, volume.image) + if err != nil { + return fmt.Errorf("error querying rbd mappings: %w", err) + } + + if device == "" { + log.V(1).Info("RBD image not currently mapped, nothing to unmap", "pool", volume.pool, "image", volume.image) + return nil + } + + log.V(1).Info("Unmapping RBD device", "device", device) + if err := r.unmapDevice(ctx, device); err != nil { + return fmt.Errorf("error unmapping rbd device %s: %w", device, err) + } + return nil +} + +func (r *rbdProvider) volumeDir(machineID string, volumeHandle string) string { + return r.paths.MachineVolumeDir(machineID, cephDriverName, volumeHandle) +} + +func (r *rbdProvider) createCephConf(log logr.Logger, machineID string, volume *validatedVolume) (string, error) { + confPath := filepath.Join( + r.volumeDir(machineID, volume.name), + "ceph.conf", + ) + keyPath := filepath.Join( + r.volumeDir(machineID, volume.name), + "ceph.key", + ) + + log.V(2).Info("Creating ceph conf", "confPath", confPath) + confData := fmt.Sprintf( + "[global]\nmon_host = %s \n\n[client.%s]\nkeyring = %s\n", + strings.Join(volume.monitors, ","), + volume.userID, + keyPath, + ) + if err := os.WriteFile(confPath, []byte(confData), 0o600); err != nil { + return "", fmt.Errorf("error writing conf file %s: %w", confPath, err) + } + if err := os.Chmod(confPath, 0o600); err != nil { + return "", fmt.Errorf("error setting permissions on conf file %s: %w", confPath, err) + } + + log.V(1).Info("Creating ceph key", "keyPath", keyPath) + keyData := fmt.Sprintf("[client.%s]\nkey = %s\n", volume.userID, volume.userKey) + if err := os.WriteFile(keyPath, []byte(keyData), 0o600); err != nil { + return "", fmt.Errorf("error writing key file %s: %w", keyPath, err) + } + if err := os.Chmod(keyPath, 0o600); err != nil { + return "", fmt.Errorf("error setting permissions on key file %s: %w", keyPath, err) + } + + return confPath, nil +} + +func (r *rbdProvider) mapDevice(ctx context.Context, volume *validatedVolume, confPath string) (string, error) { + imageSpec := fmt.Sprintf("%s/%s", volume.pool, volume.image) + + args := []string{ + "map", imageSpec, + "--id", volume.userID, + "-c", confPath, + } + + cmd := exec.CommandContext(ctx, "rbd", args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("rbd map failed: %w, stderr: %s", err, stderr.String()) + } + + devicePath := strings.TrimSpace(stdout.String()) + if devicePath == "" { + return "", fmt.Errorf("rbd map returned empty device path") + } + + return devicePath, nil +} + +func (r *rbdProvider) unmapDevice(ctx context.Context, devicePath string) error { + cmd := exec.CommandContext(ctx, "rbd", "unmap", devicePath) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("rbd unmap failed: %w, stderr: %s", err, stderr.String()) + } + + return nil +} + +// rbdMappedEntry represents a single device from `rbd showmapped --format json`. +type rbdMappedEntry struct { + Pool string `json:"pool"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Snap string `json:"snap"` + Device string `json:"device"` +} + +// findMappedDevice returns the kernel device path for an already-mapped RBD +// image, or "" if the image is not currently mapped. +func (r *rbdProvider) findMappedDevice(pool, image string) (string, error) { + output, err := exec.Command("rbd", "showmapped", "--format", "json").Output() + if err != nil { + return "", fmt.Errorf("rbd showmapped failed: %w", err) + } + + if len(bytes.TrimSpace(output)) == 0 { + return "", nil + } + + var mapped []rbdMappedEntry + if err := json.Unmarshal(output, &mapped); err != nil { + return "", fmt.Errorf("error parsing rbd showmapped output: %w", err) + } + + for _, entry := range mapped { + if entry.Pool == pool && entry.Name == image { + return entry.Device, nil + } + } + + return "", nil +} diff --git a/internal/plugins/volume/localdisk/localdisk.go b/internal/plugins/volume/localdisk/localdisk.go index ddfc4aa..676141b 100644 --- a/internal/plugins/volume/localdisk/localdisk.go +++ b/internal/plugins/volume/localdisk/localdisk.go @@ -122,8 +122,8 @@ func (p *plugin) Apply(ctx context.Context, spec *api.VolumeSpec, machineID stri }, nil } -func (p *plugin) Delete(_ context.Context, computeVolumeName string, machineID string) error { - return os.RemoveAll(p.host.MachineVolumeDir(machineID, utilstrings.EscapeQualifiedName(pluginName), computeVolumeName)) +func (p *plugin) Delete(_ context.Context, spec *api.VolumeSpec, machineID string) error { + return os.RemoveAll(p.host.MachineVolumeDir(machineID, utilstrings.EscapeQualifiedName(pluginName), spec.Name)) } func generateWWN(machineID, diskName string) string { diff --git a/internal/plugins/volume/plugins.go b/internal/plugins/volume/plugins.go index 3877559..4cdde38 100644 --- a/internal/plugins/volume/plugins.go +++ b/internal/plugins/volume/plugins.go @@ -25,7 +25,7 @@ type Plugin interface { CanSupport(spec *api.VolumeSpec) bool Apply(ctx context.Context, spec *api.VolumeSpec, machineID string) (*api.VolumeStatus, error) - Delete(ctx context.Context, computeVolumeName string, machineID string) error + Delete(ctx context.Context, spec *api.VolumeSpec, machineID string) error } type PluginManager struct { diff --git a/internal/vmm/manager.go b/internal/vmm/manager.go index b2df7cd..9621991 100644 --- a/internal/vmm/manager.go +++ b/internal/vmm/manager.go @@ -378,17 +378,9 @@ func (m *Manager) Create(ctx context.Context, machine *api.Machine) error { } disk := client.DiskConfig{ - Id: new(vol.Handle), - } - - switch vol.Type { - case api.VolumeSocketType: - disk.VhostUser = new(true) - disk.VhostSocket = new(vol.Path) - disk.Readonly = new(false) - case api.VolumeFileType: - disk.Path = new(vol.Path) - disk.ImageType = new(client.Raw) + Id: new(vol.Handle), + Path: new(vol.Path), + ImageType: new(client.Raw), } disks = append(disks, disk) @@ -524,18 +516,9 @@ func (m *Manager) AttachDisk(ctx context.Context, instanceID string, volume *api } disk := client.DiskConfig{ - Id: new(volume.Handle), - } - - switch volume.Type { - case api.VolumeSocketType: - disk.VhostUser = new(true) - disk.VhostSocket = new(volume.Path) - disk.Readonly = new(false) - case api.VolumeFileType: - disk.Path = new(volume.Path) - // See Create: file-backed disks must declare their image type. - disk.ImageType = new(client.Raw) + Id: new(volume.Handle), + Path: new(volume.Path), + ImageType: new(client.Raw), } resp, err := apiClient.PutVmAddDiskWithResponse(ctx, disk)