Prevent duplicate default routes on multi-NIC DHCP VMs

platform/net/interface_configuration_creator.go

@@ -68,9 +68,10 @@ func (configs StaticInterfaceConfigurations) HasVersion6() bool {
 }
 
 type DHCPInterfaceConfiguration struct {
-	Name         string
-	PostUpRoutes boshsettings.Routes
-	Address      string
+	Name                string
+	PostUpRoutes        boshsettings.Routes
+	Address             string
+	IsDefaultForGateway bool
 }
 
 func (c DHCPInterfaceConfiguration) Version6() string {
@@ -111,6 +112,15 @@ func (configs DHCPInterfaceConfigurations) HasVersion6() bool {
 	return false
 }
 
+func (configs DHCPInterfaceConfigurations) IsDefaultForGateway() bool {
+	for _, config := range configs {
+		if config.IsDefaultForGateway {
+			return true
+		}
+	}
+	return false
+}
+
 type InterfaceConfigurationCreator interface {
 	CreateInterfaceConfigurations(boshsettings.Networks, map[string]string) ([]StaticInterfaceConfiguration, []DHCPInterfaceConfiguration, error)
 }
@@ -214,9 +224,10 @@ func (creator interfaceConfigurationCreator) createInterfaceConfiguration(static
 	if (networkSettings.IsDHCP() || networkSettings.Mac == "") && networkSettings.Alias == "" {
 		creator.logger.Debug(creator.logTag, "Using dhcp networking")
 		dhcpConfigs = append(dhcpConfigs, DHCPInterfaceConfiguration{
-			Name:         ifaceName,
-			PostUpRoutes: networkSettings.Routes,
-			Address:      networkSettings.IP,
+			Name:                ifaceName,
+			PostUpRoutes:        networkSettings.Routes,
+			Address:             networkSettings.IP,
+			IsDefaultForGateway: networkSettings.IsDefaultFor("gateway"),
 		})
 	} else {
 		creator.logger.Debug(creator.logTag, "Using static networking")

platform/net/interface_configuration_creator_test.go

@@ -281,6 +281,45 @@ var _ = Describe("InterfaceConfigurationCreator", func() {
 					})
 				})
 
+				Context("and the dhcp network has default gateway", func() {
+					BeforeEach(func() {
+						dhcpNetwork.Default = []string{"gateway"}
+						networks["foo"] = staticNetwork
+						networks["bar"] = dhcpNetwork
+						interfacesByMAC[staticNetwork.Mac] = "static-interface-name"
+						interfacesByMAC[dhcpNetwork.Mac] = "dhcp-interface-name"
+					})
+
+					It("sets IsDefaultForGateway=true on the DHCP configuration", func() {
+						_, dhcpInterfaceConfigurations, err := interfaceConfigurationCreator.CreateInterfaceConfigurations(networks, interfacesByMAC)
+						Expect(err).ToNot(HaveOccurred())
+
+						Expect(dhcpInterfaceConfigurations).To(ConsistOf(DHCPInterfaceConfiguration{
+							Name:                "dhcp-interface-name",
+							IsDefaultForGateway: true,
+						}))
+					})
+				})
+
+				Context("and the dhcp network does not have default gateway", func() {
+					BeforeEach(func() {
+						networks["foo"] = staticNetwork
+						networks["bar"] = dhcpNetwork
+						interfacesByMAC[staticNetwork.Mac] = "static-interface-name"
+						interfacesByMAC[dhcpNetwork.Mac] = "dhcp-interface-name"
+					})
+
+					It("sets IsDefaultForGateway=false on the DHCP configuration", func() {
+						_, dhcpInterfaceConfigurations, err := interfaceConfigurationCreator.CreateInterfaceConfigurations(networks, interfacesByMAC)
+						Expect(err).ToNot(HaveOccurred())
+
+						Expect(dhcpInterfaceConfigurations).To(ConsistOf(DHCPInterfaceConfiguration{
+							Name:                "dhcp-interface-name",
+							IsDefaultForGateway: false,
+						}))
+					})
+				})
+
 				Context("and some networks have no MAC address", func() {
 					BeforeEach(func() {
 						networks["foo"] = staticNetworkWithoutMAC

platform/net/ubuntu_net_manager.go

@@ -380,6 +380,16 @@ func (net UbuntuNetManager) writeNetworkInterfaces(
 
 	anyChanged := false
 
+	anyIsDefaultForGateway := dhcpConfigs.IsDefaultForGateway()
+	if !anyIsDefaultForGateway {
+		for _, c := range staticConfigs {
+			if c.IsDefaultForGateway {
+				anyIsDefaultForGateway = true
+				break
+			}
+		}
+	}
+
 	dhcpConfigsForOneInterface := make(map[string]DHCPInterfaceConfigurations)
 	for _, dynamicAddressConfiguration := range dhcpConfigs {
 		dhcpConfigsForOneInterface[dynamicAddressConfiguration.Name] = append(
@@ -389,7 +399,8 @@ func (net UbuntuNetManager) writeNetworkInterfaces(
 	}
 
 	for interfaceName, dynamicAddressConfigurations := range dhcpConfigsForOneInterface {
-		changed, err := net.writeDynamicInterfaceConfiguration(dynamicAddressConfigurations, dnsServers, opts)
+		isDefaultGateway := !anyIsDefaultForGateway || dynamicAddressConfigurations.IsDefaultForGateway()
+		changed, err := net.writeDynamicInterfaceConfiguration(dynamicAddressConfigurations, dnsServers, isDefaultGateway, opts)
 		if err != nil {
 			return false, bosherr.WrapError(err, fmt.Sprintf("Updating network configuration for %s", interfaceName))
 		}
@@ -502,7 +513,7 @@ func (net UbuntuNetManager) writeStaticInterfaceConfiguration(config StaticInter
 	return net.fs.ConvergeFileContents(configPath, buffer.Bytes(), opts)
 }
 
-func (net UbuntuNetManager) writeDynamicInterfaceConfiguration(configs DHCPInterfaceConfigurations, dnsServers []string, opts boshsys.ConvergeFileContentsOpts) (bool, error) {
+func (net UbuntuNetManager) writeDynamicInterfaceConfiguration(configs DHCPInterfaceConfigurations, dnsServers []string, isDefaultGateway bool, opts boshsys.ConvergeFileContentsOpts) (bool, error) {
 	var err error
 	// all configs share the same name, so we just use the name from the first config
 	configPath := interfaceConfigurationFile(configs[0].Name)
@@ -531,6 +542,17 @@ func (net UbuntuNetManager) writeDynamicInterfaceConfiguration(configs DHCPInter
 	dhcpSection := &ini.Section{Name: "DHCP"}
 	dhcpSection.AddKey("UseDomains", "yes")
 	dhcpSection.AddKey("UseMTU", "yes")
+	if !isDefaultGateway {
+		// UseRoutes=no prevents systemd-networkd from installing the DHCP-supplied
+		// default gateway (and any classless static routes) for this NIC, so only
+		// the designated default-gateway NIC installs a default route.
+		//
+		// UseGateway=no would be more precise (gateway only, routes kept), but it
+		// was introduced in systemd 250. Ubuntu 22.04 Jammy ships systemd 249, so
+		// UseGateway= is silently ignored on current stemcells. UseRoutes=no has
+		// been supported since systemd 217 and works on all supported stemcells.
+		dhcpSection.AddKey("UseRoutes", "no")
+	}
 	file.AppendSection(dhcpSection)
 
 	// Route Sections

platform/net/ubuntu_net_manager_test.go

@@ -635,6 +635,21 @@ Gateway=3.4.5.6
 `))
 		})
 
+		It("does not write UseRoutes=no for a single DHCP NIC with no explicit default gateway", func() {
+			stubInterfaces(map[string]boshsettings.Network{
+				"eth0": dhcpNetwork,
+			})
+
+			err := netManager.SetupNetworking(boshsettings.Networks{
+				"dhcp-1": dhcpNetwork,
+			}, nil, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			networkConfig := fs.GetFileTestStat("/etc/systemd/network/10_eth0.network")
+			Expect(networkConfig).ToNot(BeNil())
+			Expect(networkConfig.StringContents()).ToNot(ContainSubstring("UseRoutes=no"))
+		})
+
 		It("writes /etc/network/interfaces without dns-namservers if there are no dns servers", func() {
 			staticNetworkWithoutDNS := boshsettings.Network{
 				Type:    "manual",
@@ -993,6 +1008,7 @@ DNS=9.9.9.9
 [DHCP]
 UseDomains=yes
 UseMTU=yes
+UseRoutes=no
 
 `))
 
@@ -1011,6 +1027,103 @@ Gateway=3.4.5.6
 DNS=8.8.8.8
 DNS=9.9.9.9
 
+`))
+		})
+
+		It("writes UseRoutes=no for non-gateway DHCP NIC in multi-NIC setup", func() {
+			primaryDHCPNetwork := boshsettings.Network{
+				Type:    "dynamic",
+				Default: []string{"gateway", "dns"},
+				DNS:     []string{"8.8.8.8"},
+				Mac:     "fake-primary-dhcp-mac-address",
+			}
+			secondaryDHCPNetwork := boshsettings.Network{
+				Type: "dynamic",
+				Mac:  "fake-secondary-dhcp-mac-address",
+			}
+
+			stubInterfaces(map[string]boshsettings.Network{
+				"ethprimary":   primaryDHCPNetwork,
+				"ethsecondary": secondaryDHCPNetwork,
+			})
+
+			err := netManager.SetupNetworking(boshsettings.Networks{
+				"primary-network":   primaryDHCPNetwork,
+				"secondary-network": secondaryDHCPNetwork,
+			}, nil, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			primaryConfig := fs.GetFileTestStat("/etc/systemd/network/10_ethprimary.network")
+			Expect(primaryConfig).ToNot(BeNil())
+			Expect(primaryConfig.StringContents()).To(Equal(`# Generated by bosh-agent
+[Match]
+Name=ethprimary
+
+[Network]
+DHCP=yes
+DNS=8.8.8.8
+
+[DHCP]
+UseDomains=yes
+UseMTU=yes
+
+`))
+
+			secondaryConfig := fs.GetFileTestStat("/etc/systemd/network/10_ethsecondary.network")
+			Expect(secondaryConfig).ToNot(BeNil())
+			Expect(secondaryConfig.StringContents()).To(Equal(`# Generated by bosh-agent
+[Match]
+Name=ethsecondary
+
+[Network]
+DHCP=yes
+DNS=8.8.8.8
+
+[DHCP]
+UseDomains=yes
+UseMTU=yes
+UseRoutes=no
+
+`))
+		})
+
+		It("does not write UseRoutes=no when one of the DHCP networks on a shared interface is the gateway default", func() {
+			sharedMAC := "fake-shared-mac-address"
+			gatewayDHCPNetwork := boshsettings.Network{
+				Type:    "dynamic",
+				Default: []string{"gateway", "dns"},
+				DNS:     []string{"8.8.8.8"},
+				Mac:     sharedMAC,
+			}
+			nonGatewayDHCPNetwork := boshsettings.Network{
+				Type: "dynamic",
+				Mac:  sharedMAC,
+			}
+
+			stubInterfaces(map[string]boshsettings.Network{
+				"ethshared": gatewayDHCPNetwork,
+			})
+
+			err := netManager.SetupNetworking(boshsettings.Networks{
+				"primary-network":   gatewayDHCPNetwork,
+				"secondary-network": nonGatewayDHCPNetwork,
+			}, nil, nil)
+			Expect(err).ToNot(HaveOccurred())
+
+			sharedConfig := fs.GetFileTestStat("/etc/systemd/network/10_ethshared.network")
+			Expect(sharedConfig).ToNot(BeNil())
+			Expect(sharedConfig.StringContents()).To(Equal(`# Generated by bosh-agent
+[Match]
+Name=ethshared
+
+[Network]
+DHCP=yes
+DNS=8.8.8.8
+
+[DHCP]
+UseDomains=yes
+UseMTU=yes
+
 `))
 		})