diff --git a/g3doc/user_guide/checkpoint_restore.md b/g3doc/user_guide/checkpoint_restore.md index 6c94be1b46..48aa1bde81 100644 --- a/g3doc/user_guide/checkpoint_restore.md +++ b/g3doc/user_guide/checkpoint_restore.md @@ -150,6 +150,25 @@ docker start --checkpoint `--checkpoint-dir` flag but this will be required when restoring from a checkpoint made in another container. +## Networking + +Checkpoint/restore is supported with `--network=sandbox` (default), +`--network=none`, and `--network=host`. + +With `--network=host`, host sockets cannot be saved, so: + +- Checkpoint with `--leave-running` does not touch the running sandbox's + sockets. It keeps using them as before. +- TCP listening sockets are re-created during restore and keep accepting new + connections. Connections that were pending in the backlog at checkpoint time + are lost. If the listen address cannot be bound on the restoring host, the + restore fails. +- Sockets that were connected at checkpoint time return `ECONNABORTED`, and + `epoll_wait` on them returns `EPOLLERR | EPOLLHUP` immediately. Applications + must reconnect after restore. +- Network configuration visible inside the sandbox (interface statistics, TCP + buffer sizes) reflects the host the sandbox was restored on. + ## Checkpoint & Restore with different CPU features When restoring a state file, gVisor verifies that the target host machine diff --git a/pkg/sentry/socket/hostinet/BUILD b/pkg/sentry/socket/hostinet/BUILD index 96cbddd323..58b3d8de1f 100644 --- a/pkg/sentry/socket/hostinet/BUILD +++ b/pkg/sentry/socket/hostinet/BUILD @@ -1,15 +1,27 @@ -load("//tools:defs.bzl", "go_library") +load("//tools:defs.bzl", "go_library", "go_test") package( default_applicable_licenses = ["//:license"], licenses = ["notice"], ) +go_test( + name = "hostinet_test", + size = "small", + srcs = ["stack_test.go"], + library = ":hostinet", + deps = [ + "//pkg/abi/linux", + "@org_golang_x_sys//unix:go_default_library", + ], +) + go_library( name = "hostinet", srcs = [ "hostinet.go", "netlink.go", + "save_restore.go", "socket.go", "socket_unsafe.go", "sockopt.go", diff --git a/pkg/sentry/socket/hostinet/save_restore.go b/pkg/sentry/socket/hostinet/save_restore.go new file mode 100644 index 0000000000..97191b6999 --- /dev/null +++ b/pkg/sentry/socket/hostinet/save_restore.go @@ -0,0 +1,162 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinet + +import ( + "context" + "fmt" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/fdnotifier" + "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/sync" +) + +// Host socket fds cannot cross a checkpoint boundary, so Socket.fd is not +// saved and the checkpointed sandbox retains ownership of it. +// +// Listening sockets carry no connection state, so beforeSave records what is +// needed to re-create them and Stack.Restore re-creates the host socket with +// socket/bind/listen. Connections that were pending in the backlog are lost. +// +// All other sockets are restored with fd -1, so host socket operations fail +// with ECONNABORTED and Readiness reports hangup/error to wake pollers +// immediately. + +// listenerState describes a listening host socket so restore can re-create it. +// +// +stateify savable +type listenerState struct { + addr []byte + backlog int32 + reuseAddr int32 + reusePort int32 + v6Only int32 + bindToDevice string +} + +var restoredListeners struct { + mu sync.Mutex + sockets []*Socket +} + +// beforeSave is invoked by stateify. +func (s *Socket) beforeSave() { + s.savedListener = nil + if s.fd < 0 { + return + } + accepting, err := unix.GetsockoptInt(s.fd, unix.SOL_SOCKET, unix.SO_ACCEPTCONN) + if err != nil || accepting == 0 { + return + } + addr, err := getsockname(s.fd) + if err != nil { + panic(fmt.Sprintf("getsockname on listening host socket failed during save: %v", err)) + } + l := &listenerState{ + addr: addr, + backlog: s.listenBacklog.Load(), + } + if v, err := unix.GetsockoptInt(s.fd, unix.SOL_SOCKET, unix.SO_REUSEADDR); err == nil { + l.reuseAddr = int32(v) + } + if v, err := unix.GetsockoptInt(s.fd, unix.SOL_SOCKET, unix.SO_REUSEPORT); err == nil { + l.reusePort = int32(v) + } + if s.family == unix.AF_INET6 { + if v, err := unix.GetsockoptInt(s.fd, unix.IPPROTO_IPV6, unix.IPV6_V6ONLY); err == nil { + l.v6Only = int32(v) + } + } + if dev, err := unix.GetsockoptString(s.fd, unix.SOL_SOCKET, unix.SO_BINDTODEVICE); err == nil { + l.bindToDevice = dev + } + s.savedListener = l +} + +// afterLoad is invoked by stateify. +func (s *Socket) afterLoad(context.Context) { + s.fd = -1 + if s.savedListener != nil { + restoredListeners.mu.Lock() + restoredListeners.sockets = append(restoredListeners.sockets, s) + restoredListeners.mu.Unlock() + } +} + +// restoreListeners re-creates the host sockets for saved listening sockets. +func restoreListeners() { + restoredListeners.mu.Lock() + sockets := restoredListeners.sockets + restoredListeners.sockets = nil + restoredListeners.mu.Unlock() + for _, s := range sockets { + if err := s.restoreListener(); err != nil { + log.Warningf("Failed to restore listening host socket (family=%d, addr=%x): %v", s.family, s.savedListener.addr, err) + s.savedListener = nil + } + } +} + +// restoreListener attempts to re-create a saved listening socket. If any host +// operation fails, the socket is left unrestored with fd -1 and restore +// continues with other sockets. +func (s *Socket) restoreListener() error { + l := s.savedListener + fd, err := unix.Socket(s.family, int(s.stype)|unix.SOCK_NONBLOCK|unix.SOCK_CLOEXEC, s.protocol) + if err != nil { + return fmt.Errorf("creating socket: %w", err) + } + if l.reuseAddr != 0 { + if err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, int(l.reuseAddr)); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("setting SO_REUSEADDR: %w", err) + } + } + if l.reusePort != 0 { + if err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, int(l.reusePort)); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("setting SO_REUSEPORT: %w", err) + } + } + if s.family == unix.AF_INET6 { + if err := unix.SetsockoptInt(fd, unix.IPPROTO_IPV6, unix.IPV6_V6ONLY, int(l.v6Only)); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("setting IPV6_V6ONLY: %w", err) + } + } + if l.bindToDevice != "" { + if err := unix.SetsockoptString(fd, unix.SOL_SOCKET, unix.SO_BINDTODEVICE, l.bindToDevice); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("setting SO_BINDTODEVICE to %q: %w", l.bindToDevice, err) + } + } + if err := bind(fd, l.addr); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("binding: %w", err) + } + if err := unix.Listen(fd, int(l.backlog)); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("listening: %w", err) + } + if err := fdnotifier.AddFD(int32(fd), &s.queue); err != nil { + _ = unix.Close(fd) + return fmt.Errorf("registering with fdnotifier: %w", err) + } + s.fd = fd + s.savedListener = nil + return nil +} diff --git a/pkg/sentry/socket/hostinet/socket.go b/pkg/sentry/socket/hostinet/socket.go index 606d513e62..3d8a42d649 100644 --- a/pkg/sentry/socket/hostinet/socket.go +++ b/pkg/sentry/socket/hostinet/socket.go @@ -129,18 +129,24 @@ type Socket struct { // masks, as when e.g. applications repeatedly call poll() with the same // event mask, or blocking accept() / read() / write() / recvmsg() / // sendmsg() / etc., on the same socket. - persistentEventMu sync.Mutex + persistentEventMu sync.Mutex `state:"nosave"` persistentEventMask atomicbitops.Uint64 persistentEntry waiter.Entry // fd is the host socket fd. It must have O_NONBLOCK, so that operations // will return EWOULDBLOCK instead of blocking on the host. This allows us to // handle blocking behavior independently in the sentry. - fd int + fd int `state:"nosave"` // recvClosed indicates that the socket has been shutdown for reading // (SHUT_RD or SHUT_RDWR). recvClosed atomicbitops.Bool + + // listenBacklog is the backlog passed to the most recent listen(2). + listenBacklog atomicbitops.Int32 + + // savedListener is set by beforeSave if this socket is listening. + savedListener *listenerState } var _ = socket.Socket(&Socket{}) @@ -175,6 +181,9 @@ func newSocket(t *kernel.Task, family int, stype linux.SockType, protocol int, f // Release implements vfs.FileDescriptionImpl.Release. func (s *Socket) Release(ctx context.Context) { kernel.KernelFromContext(ctx).DeleteSocket(&s.vfsfd) + if s.fd < 0 { + return + } fdnotifier.RemoveFD(int32(s.fd)) _ = unix.Close(s.fd) } @@ -186,6 +195,9 @@ func (s *Socket) Epollable() bool { // Ioctl implements vfs.FileDescriptionImpl. func (s *Socket) Ioctl(ctx context.Context, uio usermem.IO, sysno uintptr, args arch.SyscallArguments) (uintptr, error) { + if s.fd < 0 { + return 0, linuxerr.ECONNABORTED + } return ioctl(ctx, s.fd, uio, sysno, args) } @@ -202,6 +214,10 @@ func (s *Socket) Read(ctx context.Context, dst usermem.IOSequence, opts vfs.Read return 0, linuxerr.EOPNOTSUPP } + if s.fd < 0 { + return 0, linuxerr.ECONNABORTED + } + reader := hostfd.GetReadWriterAt(int32(s.fd), -1, opts.Flags) defer hostfd.PutReadWriterAt(reader) n, err := dst.CopyOutFrom(ctx, reader) @@ -226,6 +242,10 @@ func (s *Socket) Write(ctx context.Context, src usermem.IOSequence, opts vfs.Wri return 0, linuxerr.EOPNOTSUPP } + if s.fd < 0 { + return 0, linuxerr.ECONNABORTED + } + writer := hostfd.GetReadWriterAt(int32(s.fd), -1, opts.Flags) defer hostfd.PutReadWriterAt(writer) n, err := src.CopyInTo(ctx, writer) @@ -303,11 +323,18 @@ func (p *socketProvider) Pair(t *kernel.Task, stype linux.SockType, protocol int // Readiness implements waiter.Waitable.Readiness. func (s *Socket) Readiness(mask waiter.EventMask) waiter.EventMask { + if s.fd < 0 { + return waiter.EventHUp | waiter.EventErr | waiter.EventRdHUp + } return fdnotifier.NonBlockingPoll(int32(s.fd), mask) } // EventRegister implements waiter.Waitable.EventRegister. func (s *Socket) EventRegister(e *waiter.Entry) error { + if s.fd < 0 { + s.queue.EventRegister(e) + return nil + } if em, pem := e.Mask(), waiter.EventMask(s.persistentEventMask.Load()); em&^pem != 0 { s.persistentEventMu.Lock() pem = waiter.EventMask(s.persistentEventMask.RacyLoad()) @@ -362,6 +389,9 @@ func (s *Socket) eventUnregisterTransient(e *waiter.Entry) { // Connect implements socket.Socket.Connect. func (s *Socket) Connect(t *kernel.Task, sockaddr []byte, blocking bool) *syserr.Error { + if s.fd < 0 { + return syserr.ErrConnectionAborted + } if len(sockaddr) > sizeofSockaddr { sockaddr = sockaddr[:sizeofSockaddr] } @@ -424,6 +454,9 @@ func (s *Socket) Connect(t *kernel.Task, sockaddr []byte, blocking bool) *syserr // Accept implements socket.Socket.Accept. func (s *Socket) Accept(t *kernel.Task, peerRequested bool, flags int, blocking bool) (int32, linux.SockAddr, uint32, *syserr.Error) { + if s.fd < 0 { + return 0, nil, 0, syserr.ErrConnectionAborted + } var peerAddr linux.SockAddr var peerAddrBuf []byte var peerAddrlen uint32 @@ -484,24 +517,36 @@ func (s *Socket) Accept(t *kernel.Task, peerRequested bool, flags int, blocking // Bind implements socket.Socket.Bind. func (s *Socket) Bind(_ *kernel.Task, sockaddr []byte) *syserr.Error { + if s.fd < 0 { + return syserr.ErrConnectionAborted + } if len(sockaddr) > sizeofSockaddr { sockaddr = sockaddr[:sizeofSockaddr] } - _, _, errno := unix.Syscall(unix.SYS_BIND, uintptr(s.fd), uintptr(firstBytePtr(sockaddr)), uintptr(len(sockaddr))) - if errno != 0 { - return syserr.FromError(errno) + if err := bind(s.fd, sockaddr); err != nil { + return syserr.FromError(err) } return nil } // Listen implements socket.Socket.Listen. func (s *Socket) Listen(_ *kernel.Task, backlog int) *syserr.Error { - return syserr.FromError(unix.Listen(s.fd, backlog)) + if s.fd < 0 { + return syserr.ErrConnectionAborted + } + if err := unix.Listen(s.fd, backlog); err != nil { + return syserr.FromError(err) + } + s.listenBacklog.Store(int32(backlog)) + return nil } // Shutdown implements socket.Socket.Shutdown. func (s *Socket) Shutdown(_ *kernel.Task, how int) *syserr.Error { + if s.fd < 0 { + return syserr.ErrConnectionAborted + } switch how { case unix.SHUT_RD, unix.SHUT_RDWR: // Mark the socket as closed for reading. @@ -559,6 +604,9 @@ func (s *Socket) RecvMsg(t *kernel.Task, dst usermem.IOSequence, flags int, have if flags&^allowedRecvMsgFlags != 0 { return 0, 0, nil, 0, socket.ControlMessages{}, syserr.ErrInvalidArgument } + if s.fd < 0 { + return 0, 0, nil, 0, socket.ControlMessages{}, syserr.ErrConnectionAborted + } var senderAddrBuf []byte var controlBuf []byte @@ -745,6 +793,10 @@ func (s *Socket) SendMsg(t *kernel.Task, src usermem.IOSequence, to []byte, flag return 0, syserr.ErrInvalidArgument } + if s.fd < 0 { + return 0, syserr.ErrConnectionAborted + } + // If the src is zero-length, call SENDTO directly with a null buffer in // order to generate poll/epoll notifications. if src.NumBytes() == 0 { @@ -840,6 +892,12 @@ func translateIOSyscallError(err error) error { // State implements socket.Socket.State. func (s *Socket) State() uint32 { + if s.fd < 0 { + if s.stype == linux.SOCK_STREAM { + return linux.TCP_CLOSE + } + return 0 + } info := linux.TCPInfo{} buf := make([]byte, linux.SizeOfTCPInfo) var err error diff --git a/pkg/sentry/socket/hostinet/socket_unsafe.go b/pkg/sentry/socket/hostinet/socket_unsafe.go index 0152f935c5..02a7126a8a 100644 --- a/pkg/sentry/socket/hostinet/socket_unsafe.go +++ b/pkg/sentry/socket/hostinet/socket_unsafe.go @@ -204,8 +204,32 @@ func getsockopt(fd int, level, name int, opt []byte) ([]byte, error) { return opt[:optlen32], nil } +func bind(fd int, sockaddr []byte) error { + _, _, errno := unix.Syscall(unix.SYS_BIND, uintptr(fd), uintptr(firstBytePtr(sockaddr)), uintptr(len(sockaddr))) + if errno != 0 { + return errno + } + return nil +} + +func getsockname(fd int) ([]byte, error) { + addr := make([]byte, sizeofSockaddr) + addrlen := uint32(len(addr)) + _, _, errno := unix.Syscall(unix.SYS_GETSOCKNAME, uintptr(fd), uintptr(unsafe.Pointer(&addr[0])), uintptr(unsafe.Pointer(&addrlen))) + if errno != 0 { + return nil, errno + } + if addrlen > uint32(len(addr)) { + addrlen = uint32(len(addr)) + } + return addr[:addrlen], nil +} + // GetSockName implements socket.Socket.GetSockName. func (s *Socket) GetSockName(t *kernel.Task) (linux.SockAddr, uint32, *syserr.Error) { + if s.fd < 0 { + return nil, 0, syserr.ErrConnectionAborted + } addr := make([]byte, sizeofSockaddr) addrlen := uint32(len(addr)) _, _, errno := unix.Syscall(unix.SYS_GETSOCKNAME, uintptr(s.fd), uintptr(unsafe.Pointer(&addr[0])), uintptr(unsafe.Pointer(&addrlen))) @@ -217,6 +241,9 @@ func (s *Socket) GetSockName(t *kernel.Task) (linux.SockAddr, uint32, *syserr.Er // GetPeerName implements socket.Socket.GetPeerName. func (s *Socket) GetPeerName(t *kernel.Task) (linux.SockAddr, uint32, *syserr.Error) { + if s.fd < 0 { + return nil, 0, syserr.ErrConnectionAborted + } addr := make([]byte, sizeofSockaddr) addrlen := uint32(len(addr)) _, _, errno := unix.Syscall(unix.SYS_GETPEERNAME, uintptr(s.fd), uintptr(unsafe.Pointer(&addr[0])), uintptr(unsafe.Pointer(&addrlen))) diff --git a/pkg/sentry/socket/hostinet/sockopt.go b/pkg/sentry/socket/hostinet/sockopt.go index 25005de119..ba9545c33d 100644 --- a/pkg/sentry/socket/hostinet/sockopt.go +++ b/pkg/sentry/socket/hostinet/sockopt.go @@ -175,6 +175,14 @@ func (s *Socket) GetSockOpt(t *kernel.Task, level, name int, optValAddr hostarch if !sockOpt.AllowGet { return nil, syserr.ErrInvalidArgument } + if s.fd < 0 { + // Reading SO_ERROR after EPOLLERR must still succeed. + if level == linux.SOL_SOCKET && name == linux.SO_ERROR { + v := primitive.Int32(int32(unix.ECONNABORTED)) + return &v, nil + } + return nil, syserr.ErrConnectionAborted + } var opt []byte if sockOpt.Size > 0 { // Validate size of input buffer. @@ -240,6 +248,9 @@ func (s *Socket) SetSockOpt(t *kernel.Task, level, name int, opt []byte) *syserr if !sockOpt.AllowSet { return syserr.ErrInvalidArgument } + if s.fd < 0 { + return syserr.ErrConnectionAborted + } if sockOpt.Size > 0 { if uint64(len(opt)) < sockOpt.Size { return syserr.ErrInvalidArgument diff --git a/pkg/sentry/socket/hostinet/stack.go b/pkg/sentry/socket/hostinet/stack.go index 8f514c0bcd..99c76d180d 100644 --- a/pkg/sentry/socket/hostinet/stack.go +++ b/pkg/sentry/socket/hostinet/stack.go @@ -48,22 +48,34 @@ var defaultSendBufSize = inet.TCPBufferSize{ Max: 4194304, } -// Stack implements inet.Stack for host sockets. +// Stack implements inet.Stack for host sockets. Most fields are derived from +// the host network configuration. During restore, ReplaceConfig overwrites the +// deserialized Stack with values from the destination host. +// +// +stateify savable type Stack struct { - // Stack is immutable. supportsIPv6 bool tcpRecovery inet.TCPLossRecovery tcpRecvBufSize inet.TCPBufferSize tcpSendBufSize inet.TCPBufferSize tcpSACKEnabled bool - netDevFile *os.File - netSNMPFile *os.File + configured bool `state:"nosave"` + netDevFile *os.File `state:"nosave"` + netSNMPFile *os.File `state:"nosave"` // allowedSocketTypes is the list of allowed socket types - allowedSocketTypes []AllowedSocketType + allowedSocketTypes []AllowedSocketType `state:"nosave"` } // Destroy implements inet.Stack.Destroy. -func (*Stack) Destroy() { +func (s *Stack) Destroy() { + if s.netDevFile != nil { + _ = s.netDevFile.Close() + s.netDevFile = nil + } + if s.netSNMPFile != nil { + _ = s.netSNMPFile.Close() + s.netSNMPFile = nil + } } // NewStack returns an empty Stack containing no configuration. @@ -73,6 +85,9 @@ func NewStack() *Stack { // Configure sets up the stack using the current state of the host network. func (s *Stack) Configure(allowRawSockets bool) error { + if s.configured { + return nil + } if _, err := os.Stat("/proc/net/if_inet6"); err == nil { s.supportsIPv6 = true } @@ -117,6 +132,7 @@ func (s *Stack) Configure(allowRawSockets bool) error { s.allowedSocketTypes = append(s.allowedSocketTypes, AllowedRawSocketTypes...) } + s.configured = true return nil } @@ -407,10 +423,30 @@ func (*Stack) RemoveRoute(ctx context.Context, msg *nlmsg.Message) *syserr.Error func (*Stack) Pause() {} // Restore implements inet.Stack.Restore. -func (*Stack) Restore() {} +func (*Stack) Restore() { + restoreListeners() +} -// ReplaceConfig implements inet.Stack.ReplaceConfig. -func (s *Stack) ReplaceConfig(_ inet.Stack) {} +// ReplaceConfig implements inet.Stack.ReplaceConfig. It takes ownership of +// the freshly configured stack's host state, including its proc net files. +func (s *Stack) ReplaceConfig(st inet.Stack) { + newStack, ok := st.(*Stack) + if !ok { + panic(fmt.Sprintf("hostinet.Stack cannot replace config from %T", st)) + } + s.Destroy() + s.supportsIPv6 = newStack.supportsIPv6 + s.tcpRecovery = newStack.tcpRecovery + s.tcpRecvBufSize = newStack.tcpRecvBufSize + s.tcpSendBufSize = newStack.tcpSendBufSize + s.tcpSACKEnabled = newStack.tcpSACKEnabled + s.netDevFile = newStack.netDevFile + s.netSNMPFile = newStack.netSNMPFile + newStack.netDevFile = nil + newStack.netSNMPFile = nil + s.allowedSocketTypes = newStack.allowedSocketTypes + s.configured = newStack.configured +} // Resume implements inet.Stack.Resume. func (*Stack) Resume() {} diff --git a/pkg/sentry/socket/hostinet/stack_test.go b/pkg/sentry/socket/hostinet/stack_test.go new file mode 100644 index 0000000000..5df7a5602d --- /dev/null +++ b/pkg/sentry/socket/hostinet/stack_test.go @@ -0,0 +1,145 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinet + +import ( + "testing" + + "golang.org/x/sys/unix" + "gvisor.dev/gvisor/pkg/abi/linux" +) + +func TestRestoreListenersContinuesAfterFailure(t *testing.T) { + fd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM|unix.SOCK_NONBLOCK|unix.SOCK_CLOEXEC, unix.IPPROTO_TCP) + if err != nil { + t.Fatalf("Socket: %v", err) + } + t.Cleanup(func() { + _ = unix.Close(fd) + }) + if err := unix.Bind(fd, &unix.SockaddrInet4{Addr: [4]byte{127, 0, 0, 1}}); err != nil { + t.Fatalf("Bind: %v", err) + } + if err := unix.Listen(fd, 1); err != nil { + t.Fatalf("Listen: %v", err) + } + addr, err := getsockname(fd) + if err != nil { + t.Fatalf("getsockname: %v", err) + } + + s := &Socket{ + family: unix.AF_INET, + stype: linux.SOCK_STREAM, + protocol: unix.IPPROTO_TCP, + fd: -1, + savedListener: &listenerState{ + addr: addr, + backlog: 1, + }, + } + + restoredListeners.mu.Lock() + restoredListeners.sockets = []*Socket{s} + restoredListeners.mu.Unlock() + restoreListeners() + + if s.fd != -1 { + t.Errorf("fd after failed listener restore: got %d, want -1", s.fd) + _ = unix.Close(s.fd) + } + if s.savedListener != nil { + t.Errorf("savedListener after failed listener restore: got %+v, want nil", s.savedListener) + } +} + +// TestReplaceConfigTransfersHostState verifies that ReplaceConfig transfers +// host-derived configuration into a deserialized stack. +func TestReplaceConfigTransfersHostState(t *testing.T) { + fresh := NewStack() + if err := fresh.Configure(true /* allowRawSockets */); err != nil { + t.Fatalf("fresh.Configure: %v", err) + } + if fresh.netDevFile == nil { + t.Fatalf("fresh.netDevFile is nil after Configure") + } + freshDev := fresh.netDevFile + freshSNMP := fresh.netSNMPFile + freshAllowedLen := len(fresh.allowedSocketTypes) + freshTCPRecvBufSize := fresh.tcpRecvBufSize + if len(fresh.allowedSocketTypes) == 0 { + t.Fatalf("fresh.allowedSocketTypes is empty after Configure(true)") + } + t.Cleanup(fresh.Destroy) + + restored := &Stack{} + restored.ReplaceConfig(fresh) + t.Cleanup(restored.Destroy) + + if restored.netDevFile != freshDev { + t.Errorf("netDevFile not transferred: got %v want %v", restored.netDevFile, freshDev) + } + if restored.netSNMPFile != freshSNMP { + t.Errorf("netSNMPFile not transferred: got %v want %v", restored.netSNMPFile, freshSNMP) + } + if fresh.netDevFile != nil { + t.Errorf("fresh.netDevFile not cleared after transfer: %v", fresh.netDevFile) + } + if fresh.netSNMPFile != nil { + t.Errorf("fresh.netSNMPFile not cleared after transfer: %v", fresh.netSNMPFile) + } + if !restored.configured { + t.Errorf("configured not propagated") + } + if len(restored.allowedSocketTypes) != freshAllowedLen { + t.Errorf("allowedSocketTypes len mismatch: got %d want %d", + len(restored.allowedSocketTypes), freshAllowedLen) + } + if restored.tcpRecvBufSize != freshTCPRecvBufSize { + t.Errorf("tcpRecvBufSize mismatch: got %+v want %+v", + restored.tcpRecvBufSize, freshTCPRecvBufSize) + } + + if _, err := restored.netDevFile.Stat(); err != nil { + t.Errorf("netDevFile.Stat after ReplaceConfig: %v", err) + } +} + +// TestConfigureIdempotent verifies that only the first Configure call takes +// effect. +func TestConfigureIdempotent(t *testing.T) { + s := NewStack() + if err := s.Configure(false /* allowRawSockets */); err != nil { + t.Fatalf("first Configure: %v", err) + } + t.Cleanup(s.Destroy) + firstDev := s.netDevFile + firstSNMP := s.netSNMPFile + firstAllowedLen := len(s.allowedSocketTypes) + + if err := s.Configure(true /* allowRawSockets */); err != nil { + t.Fatalf("second Configure: %v", err) + } + if s.netDevFile != firstDev { + t.Errorf("second Configure reopened netDevFile") + } + if s.netSNMPFile != firstSNMP { + t.Errorf("second Configure reopened netSNMPFile") + } + if len(s.allowedSocketTypes) != firstAllowedLen { + t.Errorf("second Configure mutated allowedSocketTypes: got %d want %d", + len(s.allowedSocketTypes), firstAllowedLen) + } +} diff --git a/pkg/waiter/waiter.go b/pkg/waiter/waiter.go index 785992fcda..f38989abb0 100644 --- a/pkg/waiter/waiter.go +++ b/pkg/waiter/waiter.go @@ -211,6 +211,8 @@ func NewFunctionEntry(mask EventMask, fn func(EventMask)) (e Entry) { } // NoopListener is an EventListener that does nothing. +// +// +stateify savable type NoopListener struct{} // NotifyEvent implements EventListener.NotifyEvent. diff --git a/runsc/boot/restore.go b/runsc/boot/restore.go index 487d0463f6..817f33f9a0 100644 --- a/runsc/boot/restore.go +++ b/runsc/boot/restore.go @@ -15,7 +15,6 @@ package boot import ( - "errors" "fmt" "io" "os" @@ -108,6 +107,8 @@ const ( // annotationSaveRestoreExecTimeout is the timeout to use for the save/restore // exec binary. annotationSaveRestoreExecTimeout = annotationCheckpointPrefix + "save-restore-exec-timeout" + + networkKey = "network" ) // GetAnnotationCheckpointPath returns the checkpoint path specified in the @@ -351,13 +352,19 @@ func (r *restorer) restoreContainerInfo(l *Loader, info *containerInfo) error { return nil } -func createNetworkStackForRestore(l *Loader) inet.Stack { +func createNetworkStackForRestore(l *Loader) (inet.Stack, error) { // Save the current network stack to slap on top of the one that was restored. curNetwork := l.k.RootNetworkNamespace().Stack() if _, ok := curNetwork.(*netstack.Stack); ok { - return curNetwork + return curNetwork, nil + } + if h, ok := curNetwork.(*hostinet.Stack); ok { + if err := h.Configure(l.root.conf.EnableRaw); err != nil { + return nil, fmt.Errorf("configuring hostinet during restore: %w", err) + } + return h, nil } - return hostinet.NewStack() + return hostinet.NewStack(), nil } func (r *restorer) restore(l *Loader) error { @@ -368,15 +375,26 @@ func (r *restorer) restore(l *Loader) error { if err := specutils.RestoreValidateSpec(r.checkpointedSpecs, l.GetContainerSpecs(), l.root.conf); err != nil { return fmt.Errorf("failed to handle restore spec validation: %w", err) } - if l.root.conf.Network != config.NetworkSandbox && l.root.conf.Network != config.NetworkNone { - // TODO(gvisor.dev/issues/6243): save/restore not supported w/ hostinet - return errors.New("checkpoint not supported when using hostinet") + if l.root.conf.Network != config.NetworkSandbox && l.root.conf.Network != config.NetworkNone && l.root.conf.Network != config.NetworkHost { + return fmt.Errorf("checkpoint not supported when using %s networking", l.root.conf.Network) + } + // Checkpoints without the network key predate hostinet support. + savedNetwork, ok := r.metadata[networkKey] + if !ok { + savedNetwork = config.NetworkSandbox.String() + } + savedHost := savedNetwork == config.NetworkHost.String() + if restoreHost := l.root.conf.Network == config.NetworkHost; savedHost != restoreHost { + return fmt.Errorf("checkpoint created with %s networking cannot be restored with %s networking", savedNetwork, l.root.conf.Network) } r.timer.Reached("specs validated") // Create a new root network namespace with the network stack of the // old kernel to preserve the existing network configuration. - oldInetStack := createNetworkStackForRestore(l) + oldInetStack, err := createNetworkStackForRestore(l) + if err != nil { + return err + } r.timer.Reached("netstack created") // Reset the network stack in the network namespace to nil before @@ -677,11 +695,6 @@ func (l *Loader) saveWithOpts(saveOpts *state.SaveOpts, execOpts *control.SaveRe l.k.OnCheckpointAttempt(err) }() - // TODO(gvisor.dev/issues/6243): save/restore not supported w/ hostinet - if l.root.conf.Network == config.NetworkHost { - return errors.New("checkpoint not supported when using hostinet") - } - if saveOpts.Metadata == nil { saveOpts.Metadata = make(map[string]string) } @@ -690,6 +703,8 @@ func (l *Loader) saveWithOpts(saveOpts *state.SaveOpts, execOpts *control.SaveRe // Save runsc version. saveOpts.Metadata[VersionKey] = version.Version() + saveOpts.Metadata[networkKey] = l.root.conf.Network.String() + // Save container specs. specsStr, err := specutils.ConvertSpecsToString(l.GetContainerSpecs()) if err != nil { diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 8b282c4fe0..dfe6d4af65 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -21,6 +21,7 @@ import ( "io" "math" "math/rand" + "net" "os" "os/exec" "path" @@ -1452,6 +1453,391 @@ func TestCheckpointRestoreHostname(t *testing.T) { } } +// TestCheckpointRestoreHostinet does the checkpoint/restore test with host +// networking. +func TestCheckpointRestoreHostinet(t *testing.T) { + app, err := testutil.FindFile("test/cmd/test_app/test_app") + if err != nil { + t.Fatal("error finding test_app:", err) + } + + // Skip overlay because the test app writes its log to a bind-mounted + // host file. + for name, conf := range configs(t, true /* noOverlay */) { + conf.Network = config.NetworkHost + t.Run(name, func(t *testing.T) { + testCheckpointRestoreHostinet(t, conf, app) + }) + } +} + +// TestCheckpointResumeHostinet does the checkpoint --leave-running test with +// host networking. +func TestCheckpointResumeHostinet(t *testing.T) { + app, err := testutil.FindFile("test/cmd/test_app/test_app") + if err != nil { + t.Fatal("error finding test_app:", err) + } + + // Skip overlay because the test app writes its log to a bind-mounted + // host file. + for name, conf := range configs(t, true /* noOverlay */) { + conf.Network = config.NetworkHost + t.Run(name, func(t *testing.T) { + testCheckpointResumeHostinet(t, conf, app) + }) + } +} + +// testCheckpointRestoreHostinet checkpoints a hostinet container and checks +// that connected sockets return ECONNABORTED after restore while the restored +// listener keeps accepting. +func testCheckpointRestoreHostinet(t *testing.T, conf *config.Config, app string) { + dir, err := os.MkdirTemp(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("os.MkdirTemp failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + target, connClosed, stopServer := startHostinetSRServer(t) + defer stopServer() + + logPath := filepath.Join(dir, "hostinet-sr.log") + spec := testutil.NewSpecWithArgs(app, "hostinet-sr", "--file", logPath, "--target", target) + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(conf); err != nil { + t.Fatalf("error starting container: %v", err) + } + + if err := waitForHostinetSRLog(logPath, "SETUP_DONE", "TCP_WRITE OK", "UDP_WRITE OK", "EPOLL_WAIT TIMEOUT"); err != nil { + t.Fatalf("wait for setup: %v", err) + } + lastCount, err := lastHostinetSRCount(logPath) + if err != nil { + t.Fatalf("lastHostinetSRCount pre-restore: %v", err) + } + select { + case <-connClosed: + t.Fatalf("server connection closed before checkpoint") + default: + } + + if err := cont.Checkpoint(conf, dir, sandbox.CheckpointOpts{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + t.Fatalf("error checkpointing container: %v", err) + } + + // The remote peer must observe the connection closing when the + // checkpointed sandbox exits. + select { + case <-connClosed: + case <-time.After(30 * time.Second): + t.Fatalf("remote peer did not observe connection close after checkpoint") + } + + args2 := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont2, err := New(conf, args2) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont2.Destroy() + if err := cont2.Restore(conf, dir, false /* direct */, false /* background */, nil /* networkArgs */); err != nil { + t.Fatalf("error restoring container: %v", err) + } + if !cont2.Sandbox.Restored { + t.Fatalf("sandbox returned wrong value for Sandbox.Restored, got: false, want: true") + } + + if err := waitForHostinetSRLog(logPath, + "TCP_WRITE ERRNO=103", + "TCP_READ ERRNO=103", + "UDP_WRITE ERRNO=103", + "UDP_READ ERRNO=103", + "EPOLL_EVENT_ERR", + "EPOLL_EVENT_HUP", + "SO_ERROR=103", + "NEW_TCP_WRITE OK"); err != nil { + t.Fatalf("wait for post-restore socket state: %v", err) + } + if err := waitForHostinetSRCountAfter(logPath, lastCount); err != nil { + t.Fatalf("wait for post-restore progress: %v", err) + } + + // Dialing the re-created listener must wake the blocked accept. + listenerAddr, err := hostinetSRListenerAddr(logPath) + if err != nil { + t.Fatalf("hostinetSRListenerAddr: %v", err) + } + acceptConn, err := net.Dial("tcp", listenerAddr) + if err != nil { + t.Fatalf("dialing restored listener %q: %v", listenerAddr, err) + } + defer acceptConn.Close() + if err := waitForHostinetSRLog(logPath, "BLOCKING_ACCEPT OK"); err != nil { + t.Fatalf("wait for restored listener accept: %v", err) + } +} + +// testCheckpointResumeHostinet checks that checkpoint with Resume leaves the +// container's sockets working. +func testCheckpointResumeHostinet(t *testing.T, conf *config.Config, app string) { + dir, err := os.MkdirTemp(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("os.MkdirTemp failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + target, connClosed, stopServer := startHostinetSRServer(t) + defer stopServer() + + logPath := filepath.Join(dir, "hostinet-sr.log") + spec := testutil.NewSpecWithArgs(app, "hostinet-sr", "--file", logPath, "--target", target) + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(conf); err != nil { + t.Fatalf("error starting container: %v", err) + } + + if err := waitForHostinetSRLog(logPath, "SETUP_DONE", "TCP_WRITE OK", "UDP_WRITE OK", "EPOLL_WAIT TIMEOUT"); err != nil { + t.Fatalf("wait for setup: %v", err) + } + lastCount, err := lastHostinetSRCount(logPath) + if err != nil { + t.Fatalf("lastHostinetSRCount pre-checkpoint: %v", err) + } + + if err := cont.Checkpoint(conf, dir, sandbox.CheckpointOpts{ + Compression: statefile.CompressionLevelFlateBestSpeed, + Resume: true, + }); err != nil { + t.Fatalf("error checkpointing container: %v", err) + } + + // Wait for a few more iterations and check that no socket died. Idle + // nonblocking reads log EAGAIN, so check specifically for EBADF and + // ECONNABORTED. + if err := waitForHostinetSRCountAfter(logPath, lastCount+2); err != nil { + t.Fatalf("wait for resumed progress: %v", err) + } + b, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("error reading log file: %v", err) + } + if strings.Contains(string(b), "ERRNO=9\n") { + t.Errorf("socket returned EBADF after checkpoint --leave-running:\n%s", b) + } + if strings.Contains(string(b), "ERRNO=103\n") { + t.Errorf("socket returned ECONNABORTED after checkpoint --leave-running:\n%s", b) + } + select { + case <-connClosed: + t.Errorf("server connection closed after checkpoint --leave-running") + default: + } +} + +// TestCheckpointHostinetRestoreNetworkMismatch checks that a checkpoint taken +// with host networking cannot be restored with sandbox networking. +func TestCheckpointHostinetRestoreNetworkMismatch(t *testing.T) { + dir, err := os.MkdirTemp(testutil.TmpDir(), "checkpoint-test") + if err != nil { + t.Fatalf("os.MkdirTemp failed: %v", err) + } + defer os.RemoveAll(dir) + if err := os.Chmod(dir, 0777); err != nil { + t.Fatalf("error chmoding file: %q, %v", dir, err) + } + + conf := testutil.TestConfig(t) + conf.Network = config.NetworkHost + spec := testutil.NewSpecWithArgs("sleep", "1000") + _, bundleDir, cleanup, err := testutil.SetupContainer(spec, conf) + if err != nil { + t.Fatalf("error setting up container: %v", err) + } + defer cleanup() + + args := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont, err := New(conf, args) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont.Destroy() + if err := cont.Start(conf); err != nil { + t.Fatalf("error starting container: %v", err) + } + if err := cont.Checkpoint(conf, dir, sandbox.CheckpointOpts{Compression: statefile.CompressionLevelFlateBestSpeed}); err != nil { + t.Fatalf("error checkpointing container: %v", err) + } + + restoreConf := *conf + restoreConf.Network = config.NetworkSandbox + args2 := Args{ + ID: testutil.RandomContainerID(), + Spec: spec, + BundleDir: bundleDir, + } + cont2, err := New(&restoreConf, args2) + if err != nil { + t.Fatalf("error creating container: %v", err) + } + defer cont2.Destroy() + err = cont2.Restore(&restoreConf, dir, false /* direct */, false /* background */, nil /* networkArgs */) + if err == nil { + t.Fatalf("restore with mismatched network type succeeded, want error") + } + if !strings.Contains(err.Error(), "cannot be restored with") { + t.Errorf("restore failed with %v, want network mismatch error", err) + } +} + +// startHostinetSRServer starts a TCP server and returns its address, a +// channel signaled when it observes a connection closing, and a stop func. +func startHostinetSRServer(t *testing.T) (string, chan struct{}, func()) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + done := make(chan struct{}) + connClosed := make(chan struct{}, 1) + go func() { + defer close(done) + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + io.Copy(io.Discard, c) + // Only the first observed close matters. + select { + case connClosed <- struct{}{}: + default: + } + }() + } + }() + return ln.Addr().String(), connClosed, func() { + ln.Close() + <-done + } +} + +// waitForHostinetSRLog waits for all wanted strings to show up in the +// hostinet-sr test app's log. +func waitForHostinetSRLog(path string, wants ...string) error { + return testutil.Poll(func() error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + got := string(b) + for _, want := range wants { + if !strings.Contains(got, want) { + return fmt.Errorf("log missing %q in:\n%s", want, got) + } + } + return nil + }, 30*time.Second) +} + +// waitForHostinetSRCountAfter waits for the hostinet-sr test app's iteration +// counter to advance past prev. +func waitForHostinetSRCountAfter(path string, prev int) error { + return testutil.Poll(func() error { + count, err := lastHostinetSRCount(path) + if err != nil { + return err + } + if count <= prev { + return fmt.Errorf("last COUNT=%d, want > %d", count, prev) + } + return nil + }, 30*time.Second) +} + +// hostinetSRListenerAddr returns the hostinet-sr test app's logged listener +// address. +func hostinetSRListenerAddr(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + for _, line := range strings.Split(string(b), "\n") { + if addr, ok := strings.CutPrefix(line, "LISTENER_ADDR="); ok { + return addr, nil + } + } + return "", fmt.Errorf("no LISTENER_ADDR line in:\n%s", b) +} + +// lastHostinetSRCount returns the hostinet-sr test app's last logged +// iteration count. +func lastHostinetSRCount(path string) (int, error) { + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + last := -1 + for _, line := range strings.Split(string(b), "\n") { + if !strings.HasPrefix(line, "COUNT=") { + continue + } + n, err := strconv.Atoi(strings.TrimPrefix(line, "COUNT=")) + if err != nil { + return 0, err + } + last = n + } + if last < 0 { + return 0, fmt.Errorf("no COUNT line in:\n%s", b) + } + return last, nil +} + // TestCheckpointRestoreExecKilled checks that exec'd processes are killed // after the container is restored. func TestCheckpointRestoreExecKilled(t *testing.T) { diff --git a/test/cmd/test_app/BUILD b/test/cmd/test_app/BUILD index bfecf6c480..36fafdb2cf 100644 --- a/test/cmd/test_app/BUILD +++ b/test/cmd/test_app/BUILD @@ -10,6 +10,7 @@ go_binary( testonly = 1, srcs = [ "fds.go", + "hostinet_sr.go", "main.go", "zombies.go", ], diff --git a/test/cmd/test_app/hostinet_sr.go b/test/cmd/test_app/hostinet_sr.go new file mode 100644 index 0000000000..294a76ef7b --- /dev/null +++ b/test/cmd/test_app/hostinet_sr.go @@ -0,0 +1,272 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "os" + sys "syscall" + "time" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/runsc/flag" +) + +// hostinetSR opens TCP, UDP, listening, and epoll-registered sockets, then +// continuously logs the result of operations against them so tests can +// observe socket behavior across checkpoint/restore. +type hostinetSR struct { + file string + target string + tcpRaw sys.RawConn + udpRaw sys.RawConn +} + +// Name implements subcommands.Command.Name. +func (*hostinetSR) Name() string { + return "hostinet-sr" +} + +// Synopsis implements subcommands.Command.Synopsis. +func (*hostinetSR) Synopsis() string { + return "exercises hostinet checkpoint/restore sockets" +} + +// Usage implements subcommands.Command.Usage. +func (*hostinetSR) Usage() string { + return "hostinet-sr --file= --target=" +} + +// SetFlags implements subcommands.Command.SetFlags. +func (h *hostinetSR) SetFlags(f *flag.FlagSet) { + f.StringVar(&h.file, "file", "", "file for test output") + f.StringVar(&h.target, "target", "", "TCP server address") +} + +// Execute implements subcommands.Command.Execute. +func (h *hostinetSR) Execute(ctx context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if h.file == "" || h.target == "" { + log.Fatalf("--file and --target are required") + } + + tcpConn, err := net.Dial("tcp", h.target) + if err != nil { + log.Fatalf("Dial(%q): %v", h.target, err) + } + defer tcpConn.Close() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Fatalf("Listen: %v", err) + } + defer listener.Close() + + udpConn, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + log.Fatalf("ListenPacket: %v", err) + } + defer udpConn.Close() + + h.tcpRaw = h.mustRawConn(tcpConn.(*net.TCPConn)) + h.udpRaw = h.mustRawConn(udpConn.(*net.UDPConn)) + + epollFD, err := sys.EpollCreate1(0) + if err != nil { + log.Fatalf("EpollCreate1: %v", err) + } + defer sys.Close(epollFD) + + var tcpFD int + if err := h.tcpRaw.Control(func(fd uintptr) { + tcpFD = int(fd) + }); err != nil { + log.Fatalf("Control: %v", err) + } + if err := sys.EpollCtl(epollFD, sys.EPOLL_CTL_ADD, tcpFD, &sys.EpollEvent{ + Events: sys.EPOLLIN, + Fd: int32(tcpFD), + }); err != nil { + log.Fatalf("EpollCtl: %v", err) + } + + h.logf("LISTENER_ADDR=%s", listener.Addr()) + h.logf("SETUP_DONE") + go h.blockingAccept(listener) + for i := 0; ; i++ { + h.logf("COUNT=%d", i) + h.checkTCP(tcpConn) + h.checkUDP(udpConn) + h.checkEpoll(epollFD) + time.Sleep(200 * time.Millisecond) + } +} + +// checkTCP logs the result of a write and read on the TCP connection, and +// dials a fresh connection once the existing one returns ECONNABORTED. +func (h *hostinetSR) checkTCP(conn net.Conn) { + _, err := conn.Write([]byte("x")) + h.logResult("TCP_WRITE", err) + if errno, ok := errnoOf(err); ok && errno == sys.ECONNABORTED { + newConn, err := net.Dial("tcp", h.target) + h.logResult("NEW_TCP_DIAL", err) + if err == nil { + _, err = newConn.Write([]byte("x")) + h.logResult("NEW_TCP_WRITE", err) + newConn.Close() + } + } + + h.logResult("TCP_READ", rawRead(h.tcpRaw)) +} + +// checkUDP logs the result of a sendto and recvfrom on the UDP socket. +func (h *hostinetSR) checkUDP(conn net.PacketConn) { + h.logResult("UDP_WRITE", rawSendto(h.udpRaw)) + h.logResult("UDP_READ", rawRecvfrom(h.udpRaw)) +} + +// blockingAccept blocks in accept and logs the result when it returns. +func (h *hostinetSR) blockingAccept(listener net.Listener) { + conn, err := listener.Accept() + h.logResult("BLOCKING_ACCEPT", err) + if err == nil { + conn.Close() + } +} + +// rawRead reads one byte directly from the connection's fd. +func rawRead(raw sys.RawConn) error { + var buf [1]byte + var opErr error + err := raw.Read(func(fd uintptr) bool { + _, opErr = sys.Read(int(fd), buf[:]) + return true + }) + if err != nil { + return err + } + return opErr +} + +// rawSendto sends one byte directly to the connection's fd. +func rawSendto(raw sys.RawConn) error { + var opErr error + err := raw.Write(func(fd uintptr) bool { + opErr = sys.Sendto(int(fd), []byte("x"), sys.MSG_DONTWAIT, &sys.SockaddrInet4{ + Port: 9, + Addr: [4]byte{127, 0, 0, 1}, + }) + return true + }) + if err != nil { + return err + } + return opErr +} + +// rawRecvfrom receives one byte directly from the connection's fd. +func rawRecvfrom(raw sys.RawConn) error { + var buf [1]byte + var opErr error + err := raw.Read(func(fd uintptr) bool { + _, _, opErr = sys.Recvfrom(int(fd), buf[:], sys.MSG_DONTWAIT) + return true + }) + if err != nil { + return err + } + return opErr +} + +// checkEpoll logs the result of a non-blocking epoll_wait. +func (h *hostinetSR) checkEpoll(epollFD int) { + events := make([]sys.EpollEvent, 1) + n, err := sys.EpollWait(epollFD, events, 0) + if err != nil { + h.logResult("EPOLL_WAIT", err) + return + } + if n == 0 { + h.logf("EPOLL_WAIT TIMEOUT") + return + } + h.logf("EPOLL_EVENT=0x%x", events[0].Events) + if events[0].Events&sys.EPOLLERR != 0 { + h.logf("EPOLL_EVENT_ERR") + if v, err := sys.GetsockoptInt(int(events[0].Fd), sys.SOL_SOCKET, sys.SO_ERROR); err != nil { + h.logResult("SO_ERROR_GET", err) + } else { + h.logf("SO_ERROR=%d", v) + } + } + if events[0].Events&sys.EPOLLHUP != 0 { + h.logf("EPOLL_EVENT_HUP") + } +} + +// logResult logs the outcome of op, including the errno on failure. +func (h *hostinetSR) logResult(op string, err error) { + if err == nil { + h.logf("%s OK", op) + return + } + if errno, ok := errnoOf(err); ok { + h.logf("%s ERRNO=%d", op, errno) + return + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + h.logf("%s TIMEOUT", op) + return + } + h.logf("%s ERR=%q", op, err.Error()) +} + +// logf appends a line to the output file. +func (h *hostinetSR) logf(format string, args ...any) { + f, err := os.OpenFile(h.file, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + log.Fatalf("OpenFile(%q): %v", h.file, err) + } + defer f.Close() + fmt.Fprintf(f, format+"\n", args...) +} + +// syscallConner is implemented by connection types that expose their fd. +type syscallConner interface { + SyscallConn() (sys.RawConn, error) +} + +// mustRawConn returns the connection's raw fd accessor. +func (h *hostinetSR) mustRawConn(c syscallConner) sys.RawConn { + raw, err := c.SyscallConn() + if err != nil { + log.Fatalf("SyscallConn: %v", err) + } + return raw +} + +// errnoOf unwraps the errno from err, if any. +func errnoOf(err error) (sys.Errno, bool) { + var errno sys.Errno + if errors.As(err, &errno) { + return errno, true + } + return 0, false +} diff --git a/test/cmd/test_app/main.go b/test/cmd/test_app/main.go index c9cdd4fc68..702b75539a 100644 --- a/test/cmd/test_app/main.go +++ b/test/cmd/test_app/main.go @@ -52,6 +52,7 @@ func main() { subcommands.Register(new(fsTreeVerify), "") subcommands.Register(new(assertIsEmpty), "") subcommands.Register(new(gvisorDetect), "") + subcommands.Register(new(hostinetSR), "") subcommands.Register(new(ptyRunner), "") subcommands.Register(new(reaper), "") subcommands.Register(new(syscall), "") diff --git a/test/runner/defs.bzl b/test/runner/defs.bzl index 914885f7bb..89072fde9e 100644 --- a/test/runner/defs.bzl +++ b/test/runner/defs.bzl @@ -314,8 +314,9 @@ def syscall_test_variants( **kwargs ) - # TODO(b/192114729): hostinet is not supported with S/R. - if add_hostinet and not (save or save_resume): + # Connected host sockets do not survive restore, so the save tests are + # skipped and only the save_resume tests are generated. + if add_hostinet and not save: _syscall_test( test = test, platform = default_platform,