Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions api/machine.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,7 @@ const (
type VolumeType string

const (
VolumeSocketType VolumeType = "socket"
VolumeFileType VolumeType = "file"
VolumeFileType VolumeType = "file"
)

type NetworkInterfaceSpec struct {
Expand Down
32 changes: 11 additions & 21 deletions cmd/cloud-hypervisor-provider/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ import (
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/controllers"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/host"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/mcr"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/networkinterface/options"
nicoptions "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/networkinterface/options"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph"
cephoptions "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph/options"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/localdisk"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/raw"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/server"
Expand Down Expand Up @@ -52,9 +52,8 @@ type Options struct {
CloudHypervisorBinPath string
CloudHypervisorFirmwarePath string

QMPSocketPath string

NicPlugin *options.Options
CephPlugin *cephoptions.Options
NicPlugin *nicoptions.Options
}

func (o *Options) AddFlags(fs *pflag.FlagSet) {
Expand All @@ -74,13 +73,6 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
"Path to the directory of the machine store.",
)

fs.StringVar(
&o.QMPSocketPath,
"qmp-socket-path",
"/run/chp/qmp/sock",
"Path to the qmp socket.",
)

fs.StringVar(
&o.CloudHypervisorBinPath,
"cloud-hypervisor-bin-path",
Expand All @@ -101,7 +93,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
"Supported machine classes (format: name,cpu,memory)",
)

o.NicPlugin = options.NewDefaultOptions()
o.CephPlugin = cephoptions.NewDefaultOptions()
o.CephPlugin.AddFlags(fs)

o.NicPlugin = nicoptions.NewDefaultOptions()
o.NicPlugin.AddFlags(fs)
}

Expand Down Expand Up @@ -184,20 +179,15 @@ func Run(ctx context.Context, opts Options) error {
return err
}

qmpProvider, err := ceph.QMPProvider(
ctx,
log.WithName("ceph-volume-plugin"),
hostPaths,
opts.QMPSocketPath,
)
cephPlugin, err := opts.CephPlugin.VolumePlugin(ctx, log.WithName("ceph-volume-plugin"), hostPaths)
if err != nil {
setupLog.Error(err, "failed to initialize qmp provider")
setupLog.Error(err, "failed to initialize ceph volume plugin")
return err
}

pluginManager := volume.NewPluginManager()
if err := pluginManager.InitPlugins(hostPaths, []volume.Plugin{
ceph.NewPlugin(qmpProvider),
cephPlugin,
localdisk.NewPlugin(rawInst, imgCache),
}); err != nil {
setupLog.Error(err, "failed to initialize plugins")
Expand Down
11 changes: 0 additions & 11 deletions dev-vm/chp-dev.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -258,17 +258,12 @@ provision:

cat > /etc/tmpfiles.d/chp.conf <<'EOF'
d /run/chp 0775 chp chp -
d /run/chp/qmp 0775 chp chp -
EOF
systemd-tmpfiles --create /etc/tmpfiles.d/chp.conf

systemctl daemon-reload
systemctl enable cloud-hypervisor-prepare.service
systemctl start cloud-hypervisor-prepare.service

apt-get install -y qemu-utils
systemctl enable chp-qmp.service
systemctl start chp-qmp.service
- mode: system
script: |
#!/bin/bash
Expand Down Expand Up @@ -318,12 +313,6 @@ provision:
permissions: 644
file:
url: cloud-hypervisor-prepare.service
- mode: data
path: /etc/systemd/system/chp-qmp.service
owner: "root:root"
permissions: 644
file:
url: chp-qmp.service
# provision:
# # `system` is executed with root privileges
# - mode: system
Expand Down
23 changes: 0 additions & 23 deletions dev-vm/chp-qmp.service

This file was deleted.

3 changes: 0 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ go 1.26.4

require (
github.com/blang/semver/v4 v4.0.0
github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242
github.com/getkin/kin-openapi v0.152.0
github.com/go-logr/logr v1.4.4
github.com/google/uuid v1.6.0
Expand Down Expand Up @@ -37,7 +36,6 @@ require (
github.com/containerd/log v0.2.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/emicklei/go-restful/v3 v3.14.0 // indirect
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
Expand Down Expand Up @@ -93,7 +91,6 @@ require (
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/crypto v0.58.0 // indirect
golang.org/x/mod v0.42.0 // indirect
golang.org/x/net v0.61.0 // indirect
golang.org/x/oauth2 v0.37.0 // indirect
Expand Down
6 changes: 0 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8 h1:ZQhCuAXz9mLpTkBCwtS7ajWp3lU7pqiaOM0fxzIyQFc=
github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 h1:rh6rt8pF5U4iyQ86h6lRDenJoX4ht2wFnZXB9ogIrIM=
github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242/go.mod h1:LGHUtlhsY4vRGM6AHejEQKVI5e3eHbSylMHwTSpQtVw=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
Expand Down Expand Up @@ -253,8 +249,6 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=
Expand Down
13 changes: 10 additions & 3 deletions internal/controllers/machine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ func (r *MachineReconciler) deleteMachine(ctx context.Context, log logr.Logger,
}

log.V(2).Info("Delete volume", "name", vol.Name, "plugin", plugin.Name())
if err := plugin.Delete(ctx, vol.Name, machine.ID); err != nil {
if err := plugin.Delete(ctx, vol, machine.ID); err != nil {
return fmt.Errorf("failed to delete volume %s: %w", vol.Name, err)
}
}
Expand Down Expand Up @@ -285,7 +285,7 @@ func (r *MachineReconciler) reconcileVolumes(ctx context.Context, log logr.Logge
if vol.DeletedAt != nil {
if status.State != api.VolumeStateAttached {
log.V(2).Info("Delete not attached volume", "name", vol.Name)
if err := plugin.Delete(ctx, vol.Name, machine.ID); err != nil {
if err := plugin.Delete(ctx, vol, machine.ID); err != nil {
return fmt.Errorf("failed to delete volume %s: %w", vol.Name, err)
}
continue
Expand Down Expand Up @@ -393,7 +393,14 @@ func (r *MachineReconciler) attachDetachDisks(
}
log.V(1).Info("Removed disk", "disk", vol.Name)

updatedVolumeStatus = append(updatedVolumeStatus, status)
plugin, err := r.VolumePluginManager.FindPluginBySpec(vol)
if err != nil {
return fmt.Errorf("failed to find plugin for volume %s: %w", vol.Name, err)
}
if err := plugin.Delete(ctx, vol, machine.ID); err != nil {
return fmt.Errorf("failed to cleanup volume %s: %w", vol.Name, err)
}
log.V(1).Info("Cleaned up volume", "disk", vol.Name)
continue
}

Expand Down
90 changes: 49 additions & 41 deletions internal/plugins/volume/ceph/ceph.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@ import (
"net"
"os"
"strings"
"time"

"github.com/digitalocean/go-qemu/qmp"
"github.com/go-logr/logr"
"github.com/ironcore-dev/cloud-hypervisor-provider/api"
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/host"
Expand Down Expand Up @@ -44,47 +42,24 @@ type validatedVolume struct {
encryptionKey *string
}

type Provider interface {
type backend interface {
VolumeType() api.VolumeType
Mount(ctx context.Context, machineID string, volume *validatedVolume) (string, error)
Unmount(ctx context.Context, machineID string, volumeID string) error
Unmount(ctx context.Context, machineID string, volume *validatedVolume) error
}

func QMPProvider(ctx context.Context, log logr.Logger, paths host.Paths, socket string) (Provider, error) {
monitor, err := qmp.NewSocketMonitor("unix", socket, 2*time.Second)
if err != nil {
return nil, fmt.Errorf("failed to connect to qmp monitor: %w", err)
}

go func() {
// TODO
_ = monitor.Connect()
defer func() {
// TODO
_ = monitor.Disconnect()
}()

stream, _ := monitor.Events(ctx)
for e := range stream {
log.V(1).Info(fmt.Sprintf("EVENT: %s", e.Event))
}
}()

return &QMP{
log: log,
paths: paths,
monitor: monitor,
}, nil
// NewRBDPlugin creates a volume.Plugin backed by host-level rbd map/unmap.
func NewRBDPlugin(log logr.Logger, paths host.Paths) volume.Plugin {
return newPlugin(newRBDProvider(log, paths))
}

type plugin struct {
provider Provider
host volume.Host
be backend
host volume.Host
}

func NewPlugin(provider Provider) volume.Plugin {
return &plugin{
provider: provider,
}
func newPlugin(be backend) volume.Plugin {
return &plugin{be: be}
}

func (p *plugin) Init(host volume.Host) error {
Expand Down Expand Up @@ -181,14 +156,14 @@ func (p *plugin) Apply(ctx context.Context, spec *api.VolumeSpec, machineID stri
return nil, fmt.Errorf("failed to get volume data: %w", err)
}

path, err := p.provider.Mount(ctx, machineID, volumeData)
path, err := p.be.Mount(ctx, machineID, volumeData)
if err != nil {
return nil, fmt.Errorf("failed to mount volume: %w", err)
}

return &api.VolumeStatus{
Name: spec.Name,
Type: api.VolumeSocketType,
Type: p.be.VolumeType(),
Path: path,
Handle: volumeData.handle,
State: api.VolumeStatePrepared,
Expand Down Expand Up @@ -237,10 +212,43 @@ func (p *plugin) validateVolume(spec *api.VolumeSpec) (vData *validatedVolume, e
return vData, nil
}

func (p *plugin) Delete(ctx context.Context, computeVolumeName string, machineID string) error {
if err := p.provider.Unmount(ctx, machineID, computeVolumeName); err != nil {
return fmt.Errorf("failed to unmount volume %q: %w", computeVolumeName, err)
func (p *plugin) Delete(ctx context.Context, spec *api.VolumeSpec, machineID string) error {
volume, err := p.volumeForUnmount(spec)
if err != nil {
return fmt.Errorf("failed to get volume data for deletion: %w", err)
}

return os.RemoveAll(p.host.MachineVolumeDir(machineID, cephDriverName, computeVolumeName))
if err := p.be.Unmount(ctx, machineID, volume); err != nil {
return fmt.Errorf("failed to unmount volume %q: %w", spec.Name, err)
}

return os.RemoveAll(p.host.MachineVolumeDir(machineID, cephDriverName, spec.Name))
}

func (p *plugin) volumeForUnmount(spec *api.VolumeSpec) (*validatedVolume, error) {
if spec == nil {
return nil, fmt.Errorf("volume is nil")
}
connection := spec.Connection
if connection == nil {
return nil, fmt.Errorf("volume does not specify connection")
}
if connection.Driver != cephDriverName {
return nil, fmt.Errorf("volume connection specifies invalid driver %q", connection.Driver)
}
if connection.Attributes == nil {
return nil, fmt.Errorf("volume connection does not specify attributes")
}
if connection.Handle == "" {
return nil, fmt.Errorf("volume connection does not specify handle")
}

volume := &validatedVolume{
name: spec.Name,
handle: connection.Handle,
}
if err := readVolumeAttributes(connection.Attributes, volume); err != nil {
return nil, fmt.Errorf("error reading volume attributes: %w", err)
}
return volume, nil
}
Loading
Loading