Skip to content

Commit 53c90cc

Browse files
committed
Use kernel to mount rbd disks to drop sidecar daemon.
Signed-off-by: Lukas Frank <lukas.frank@sap.com>
1 parent 242709c commit 53c90cc

13 files changed

Lines changed: 404 additions & 505 deletions

File tree

‎api/machine.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,7 @@ const (
9696
type VolumeType string
9797

9898
const (
99-
VolumeSocketType VolumeType = "socket"
100-
VolumeFileType VolumeType = "file"
99+
VolumeFileType VolumeType = "file"
101100
)
102101

103102
type NetworkInterfaceSpec struct {

‎cmd/cloud-hypervisor-provider/app/app.go‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ import (
1515
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/controllers"
1616
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/host"
1717
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/mcr"
18-
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/networkinterface/options"
18+
nicoptions "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/networkinterface/options"
1919
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume"
20-
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph"
20+
cephoptions "github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/ceph/options"
2121
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/plugins/volume/localdisk"
2222
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/raw"
2323
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/server"
@@ -52,9 +52,8 @@ type Options struct {
5252
CloudHypervisorBinPath string
5353
CloudHypervisorFirmwarePath string
5454

55-
QMPSocketPath string
56-
57-
NicPlugin *options.Options
55+
CephPlugin *cephoptions.Options
56+
NicPlugin *nicoptions.Options
5857
}
5958

6059
func (o *Options) AddFlags(fs *pflag.FlagSet) {
@@ -74,13 +73,6 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
7473
"Path to the directory of the machine store.",
7574
)
7675

77-
fs.StringVar(
78-
&o.QMPSocketPath,
79-
"qmp-socket-path",
80-
"/run/chp/qmp/sock",
81-
"Path to the qmp socket.",
82-
)
83-
8476
fs.StringVar(
8577
&o.CloudHypervisorBinPath,
8678
"cloud-hypervisor-bin-path",
@@ -101,7 +93,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
10193
"Supported machine classes (format: name,cpu,memory)",
10294
)
10395

104-
o.NicPlugin = options.NewDefaultOptions()
96+
o.CephPlugin = cephoptions.NewDefaultOptions()
97+
o.CephPlugin.AddFlags(fs)
98+
99+
o.NicPlugin = nicoptions.NewDefaultOptions()
105100
o.NicPlugin.AddFlags(fs)
106101
}
107102

@@ -184,20 +179,15 @@ func Run(ctx context.Context, opts Options) error {
184179
return err
185180
}
186181

187-
qmpProvider, err := ceph.QMPProvider(
188-
ctx,
189-
log.WithName("ceph-volume-plugin"),
190-
hostPaths,
191-
opts.QMPSocketPath,
192-
)
182+
cephPlugin, err := opts.CephPlugin.VolumePlugin(ctx, log.WithName("ceph-volume-plugin"), hostPaths)
193183
if err != nil {
194-
setupLog.Error(err, "failed to initialize qmp provider")
184+
setupLog.Error(err, "failed to initialize ceph volume plugin")
195185
return err
196186
}
197187

198188
pluginManager := volume.NewPluginManager()
199189
if err := pluginManager.InitPlugins(hostPaths, []volume.Plugin{
200-
ceph.NewPlugin(qmpProvider),
190+
cephPlugin,
201191
localdisk.NewPlugin(rawInst, imgCache),
202192
}); err != nil {
203193
setupLog.Error(err, "failed to initialize plugins")

‎dev-vm/chp-dev.yaml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -258,17 +258,12 @@ provision:
258258
259259
cat > /etc/tmpfiles.d/chp.conf <<'EOF'
260260
d /run/chp 0775 chp chp -
261-
d /run/chp/qmp 0775 chp chp -
262261
EOF
263262
systemd-tmpfiles --create /etc/tmpfiles.d/chp.conf
264263
265264
systemctl daemon-reload
266265
systemctl enable cloud-hypervisor-prepare.service
267266
systemctl start cloud-hypervisor-prepare.service
268-
269-
apt-get install -y qemu-utils
270-
systemctl enable chp-qmp.service
271-
systemctl start chp-qmp.service
272267
- mode: system
273268
script: |
274269
#!/bin/bash
@@ -318,12 +313,6 @@ provision:
318313
permissions: 644
319314
file:
320315
url: cloud-hypervisor-prepare.service
321-
- mode: data
322-
path: /etc/systemd/system/chp-qmp.service
323-
owner: "root:root"
324-
permissions: 644
325-
file:
326-
url: chp-qmp.service
327316
# provision:
328317
# # `system` is executed with root privileges
329318
# - mode: system

‎dev-vm/chp-qmp.service‎

Lines changed: 0 additions & 23 deletions
This file was deleted.

‎go.mod‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ go 1.26.4
44

55
require (
66
github.com/blang/semver/v4 v4.0.0
7-
github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242
87
github.com/getkin/kin-openapi v0.152.0
98
github.com/go-logr/logr v1.4.4
109
github.com/google/uuid v1.6.0
@@ -37,7 +36,6 @@ require (
3736
github.com/containerd/log v0.2.0 // indirect
3837
github.com/containerd/platforms v0.2.1 // indirect
3938
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
40-
github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8 // indirect
4139
github.com/distribution/reference v0.6.0 // indirect
4240
github.com/emicklei/go-restful/v3 v3.14.0 // indirect
4341
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
@@ -93,7 +91,6 @@ require (
9391
go.uber.org/zap v1.28.0 // indirect
9492
go.yaml.in/yaml/v2 v2.4.4 // indirect
9593
go.yaml.in/yaml/v3 v3.0.5 // indirect
96-
golang.org/x/crypto v0.58.0 // indirect
9794
golang.org/x/mod v0.42.0 // indirect
9895
golang.org/x/net v0.61.0 // indirect
9996
golang.org/x/oauth2 v0.37.0 // indirect

‎go.sum‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
3535
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3636
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
3737
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
38-
github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8 h1:ZQhCuAXz9mLpTkBCwtS7ajWp3lU7pqiaOM0fxzIyQFc=
39-
github.com/digitalocean/go-libvirt v0.0.0-20261013170128-efbbdf578ac8/go.mod h1:qb0Ofa71d3oXARQf633h2tNaeBxLsVxuDp+jcsVO2+4=
40-
github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 h1:rh6rt8pF5U4iyQ86h6lRDenJoX4ht2wFnZXB9ogIrIM=
41-
github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242/go.mod h1:LGHUtlhsY4vRGM6AHejEQKVI5e3eHbSylMHwTSpQtVw=
4238
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
4339
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
4440
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
@@ -253,8 +249,6 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
253249
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
254250
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
255251
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
256-
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
257-
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
258252
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
259253
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
260254
golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw=

‎internal/controllers/machine_controller.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,14 @@ func (r *MachineReconciler) attachDetachDisks(
393393
}
394394
log.V(1).Info("Removed disk", "disk", vol.Name)
395395

396-
updatedVolumeStatus = append(updatedVolumeStatus, status)
396+
plugin, err := r.VolumePluginManager.FindPluginBySpec(vol)
397+
if err != nil {
398+
return fmt.Errorf("failed to find plugin for volume %s: %w", vol.Name, err)
399+
}
400+
if err := plugin.Delete(ctx, vol.Name, machine.ID); err != nil {
401+
return fmt.Errorf("failed to cleanup volume %s: %w", vol.Name, err)
402+
}
403+
log.V(1).Info("Cleaned up volume", "disk", vol.Name)
397404
continue
398405
}
399406

‎internal/plugins/volume/ceph/ceph.go‎

Lines changed: 12 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,7 @@ import (
99
"net"
1010
"os"
1111
"strings"
12-
"time"
1312

14-
"github.com/digitalocean/go-qemu/qmp"
1513
"github.com/go-logr/logr"
1614
"github.com/ironcore-dev/cloud-hypervisor-provider/api"
1715
"github.com/ironcore-dev/cloud-hypervisor-provider/internal/host"
@@ -44,47 +42,24 @@ type validatedVolume struct {
4442
encryptionKey *string
4543
}
4644

47-
type Provider interface {
45+
type backend interface {
46+
VolumeType() api.VolumeType
4847
Mount(ctx context.Context, machineID string, volume *validatedVolume) (string, error)
4948
Unmount(ctx context.Context, machineID string, volumeID string) error
5049
}
5150

52-
func QMPProvider(ctx context.Context, log logr.Logger, paths host.Paths, socket string) (Provider, error) {
53-
monitor, err := qmp.NewSocketMonitor("unix", socket, 2*time.Second)
54-
if err != nil {
55-
return nil, fmt.Errorf("failed to connect to qmp monitor: %w", err)
56-
}
57-
58-
go func() {
59-
// TODO
60-
_ = monitor.Connect()
61-
defer func() {
62-
// TODO
63-
_ = monitor.Disconnect()
64-
}()
65-
66-
stream, _ := monitor.Events(ctx)
67-
for e := range stream {
68-
log.V(1).Info(fmt.Sprintf("EVENT: %s", e.Event))
69-
}
70-
}()
71-
72-
return &QMP{
73-
log: log,
74-
paths: paths,
75-
monitor: monitor,
76-
}, nil
51+
// NewRBDPlugin creates a volume.Plugin backed by host-level rbd map/unmap.
52+
func NewRBDPlugin(log logr.Logger, paths host.Paths) volume.Plugin {
53+
return newPlugin(newRBDProvider(log, paths))
7754
}
7855

7956
type plugin struct {
80-
provider Provider
81-
host volume.Host
57+
be backend
58+
host volume.Host
8259
}
8360

84-
func NewPlugin(provider Provider) volume.Plugin {
85-
return &plugin{
86-
provider: provider,
87-
}
61+
func newPlugin(be backend) volume.Plugin {
62+
return &plugin{be: be}
8863
}
8964

9065
func (p *plugin) Init(host volume.Host) error {
@@ -181,14 +156,14 @@ func (p *plugin) Apply(ctx context.Context, spec *api.VolumeSpec, machineID stri
181156
return nil, fmt.Errorf("failed to get volume data: %w", err)
182157
}
183158

184-
path, err := p.provider.Mount(ctx, machineID, volumeData)
159+
path, err := p.be.Mount(ctx, machineID, volumeData)
185160
if err != nil {
186161
return nil, fmt.Errorf("failed to mount volume: %w", err)
187162
}
188163

189164
return &api.VolumeStatus{
190165
Name: spec.Name,
191-
Type: api.VolumeSocketType,
166+
Type: p.be.VolumeType(),
192167
Path: path,
193168
Handle: volumeData.handle,
194169
State: api.VolumeStatePrepared,
@@ -238,7 +213,7 @@ func (p *plugin) validateVolume(spec *api.VolumeSpec) (vData *validatedVolume, e
238213
}
239214

240215
func (p *plugin) Delete(ctx context.Context, computeVolumeName string, machineID string) error {
241-
if err := p.provider.Unmount(ctx, machineID, computeVolumeName); err != nil {
216+
if err := p.be.Unmount(ctx, machineID, computeVolumeName); err != nil {
242217
return fmt.Errorf("failed to unmount volume %q: %w", computeVolumeName, err)
243218
}
244219

0 commit comments

Comments
 (0)