From 13b0de08d94c5afce18a780164827e815e67db8d Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Tue, 8 Jun 2021 14:17:17 +0100 Subject: [PATCH] Implement automatic route copying from host --- pkg/plugin/network.go | 128 +++++++++++++++++++++++++++++++++++++++--- pkg/plugin/plugin.go | 10 +++- pkg/util/errors.go | 2 + test_env.sh | 23 ++++++-- 4 files changed, 146 insertions(+), 17 deletions(-) diff --git a/pkg/plugin/network.go b/pkg/plugin/network.go index 8bc26fd..b8eaae7 100644 --- a/pkg/plugin/network.go +++ b/pkg/plugin/network.go @@ -203,14 +203,22 @@ func (p *Plugin) CreateEndpoint(ctx context.Context, r CreateEndpointRequest) (C } return fmt.Errorf("failed to get initial IP%v address via DHCP%v: %w", v6str, v6str, err) } + ip, _, err := net.ParseCIDR(info.IP) + if err != nil { + return fmt.Errorf("failed to parse initial IP address: %w", err) + } + hint := p.joinHints[r.EndpointID] if v6 { res.Interface.AddressIPv6 = info.IP + hint.IPv6 = ip // No gateways in DHCPv6! } else { res.Interface.Address = info.IP - p.gatewayHints[r.EndpointID] = info.Gateway + hint.IPv4 = ip + hint.Gateway = info.Gateway } + p.joinHints[r.EndpointID] = hint return nil } @@ -236,7 +244,7 @@ func (p *Plugin) CreateEndpoint(ctx context.Context, r CreateEndpointRequest) (C "endpoint": r.EndpointID[:12], "ip": res.Interface.Address, "ipv6": res.Interface.AddressIPv6, - "hints": fmt.Sprintf("%#v", p.gatewayHints[r.EndpointID]), + "gateway": fmt.Sprintf("%#v", p.joinHints[r.EndpointID].Gateway), }).Info("Endpoint created") return res, nil @@ -286,19 +294,121 @@ func (p *Plugin) Join(ctx context.Context, r JoinRequest) (JoinResponse, error) DstPrefix: opts.Bridge, } - if hint, ok := p.gatewayHints[r.EndpointID]; ok { + hint, ok := p.joinHints[r.EndpointID] + if !ok { + return res, util.ErrNoHint + } + delete(p.joinHints, r.EndpointID) + + if hint.Gateway != "" { log.WithFields(log.Fields{ "network": r.NetworkID[:12], "endpoint": r.EndpointID[:12], "sandbox": r.SandboxKey, - "gateway": hint, - }).Info("[Join] Setting IPv4 gateway retrieved from CreateEndpoint") - res.Gateway = hint - - delete(p.gatewayHints, r.EndpointID) + "gateway": hint.Gateway, + }).Info("[Join] Setting IPv4 gateway retrieved from initial DHCP in CreateEndpoint") + res.Gateway = hint.Gateway + } + + bridge, err := netlink.LinkByName(opts.Bridge) + if err != nil { + return res, fmt.Errorf("failed to get bridge interface: %w", err) + } + + addRoutes := func(v6 bool) error { + family := unix.AF_INET + if v6 { + family = unix.AF_INET6 + } + + routes, err := netlink.RouteListFiltered(family, &netlink.Route{ + LinkIndex: bridge.Attrs().Index, + Type: unix.RTN_UNICAST, + }, netlink.RT_FILTER_OIF|netlink.RT_FILTER_TYPE) + if err != nil { + return fmt.Errorf("failed to list routes: %w", err) + } + + for _, route := range routes { + log.WithFields(log.Fields{ + "route": route, + "type": route.Protocol, + }).Debug("Route") + if route.Dst == nil { + // Default route + switch family { + case unix.AF_INET: + if res.Gateway == "" { + res.Gateway = route.Gw.String() + log.WithFields(log.Fields{ + "network": r.NetworkID[:12], + "endpoint": r.EndpointID[:12], + "sandbox": r.SandboxKey, + "gateway": res.Gateway, + }).Info("[Join] Setting IPv4 gateway retrieved from bridge interface on host routing table") + } + case unix.AF_INET6: + if res.GatewayIPv6 == "" { + res.GatewayIPv6 = route.Gw.String() + log.WithFields(log.Fields{ + "network": r.NetworkID[:12], + "endpoint": r.EndpointID[:12], + "sandbox": r.SandboxKey, + "gateway": res.GatewayIPv6, + }).Info("[Join] Setting IPv6 gateway retrieved from bridge interface on host routing table") + } + } + + continue + } + + if route.Protocol == unix.RTPROT_KERNEL || + (family == unix.AF_INET && route.Dst.Contains(hint.IPv4)) || + (family == unix.AF_INET6 && route.Dst.Contains(hint.IPv6)) { + // Make sure to leave out the default on-link route created automatically for the IP(s) acquired by DHCP + continue + } + + staticRoute := &StaticRoute{ + Destination: route.Dst.String(), + // Default to an on-link route + RouteType: 1, + } + res.StaticRoutes = append(res.StaticRoutes, staticRoute) + + if route.Gw != nil { + staticRoute.RouteType = 0 + staticRoute.NextHop = route.Gw.String() + + log.WithFields(log.Fields{ + "network": r.NetworkID[:12], + "endpoint": r.EndpointID[:12], + "sandbox": r.SandboxKey, + "route": staticRoute.Destination, + "gateway": staticRoute.NextHop, + }).Info("[Join] Adding route (via gateway) retrieved from bridge interface on host routing table") + } else { + log.WithFields(log.Fields{ + "network": r.NetworkID[:12], + "endpoint": r.EndpointID[:12], + "sandbox": r.SandboxKey, + "route": staticRoute.Destination, + }).Info("[Join] Adding on-link route retrieved from bridge interface on host routing table") + } + } + + return nil + } + + if err := addRoutes(false); err != nil { + return res, err + } + if opts.IPv6 { + if err := addRoutes(true); err != nil { + return res, err + } } - // TODO: Try to intelligently copy existing routes from the bridge // TODO: Start a persistent DHCP client log.WithFields(log.Fields{ diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index f3c29a2..462c54a 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -46,12 +46,18 @@ func decodeOpts(input interface{}) (DHCPNetworkOptions, error) { return opts, nil } +type joinHint struct { + IPv4 net.IP + IPv6 net.IP + Gateway string +} + // Plugin is the DHCP network plugin type Plugin struct { docker *docker.Client server http.Server - gatewayHints map[string]string + joinHints map[string]joinHint } // NewPlugin creates a new Plugin @@ -64,7 +70,7 @@ func NewPlugin() (*Plugin, error) { p := Plugin{ docker: client, - gatewayHints: make(map[string]string), + joinHints: make(map[string]joinHint), } mux := http.NewServeMux() diff --git a/pkg/util/errors.go b/pkg/util/errors.go index bd3b8b1..3fe28cf 100644 --- a/pkg/util/errors.go +++ b/pkg/util/errors.go @@ -18,6 +18,8 @@ var ( ErrMACAddress = errors.New("invalid MAC address") // ErrNoLease indicates a DHCP lease was not obtained from udhcpc ErrNoLease = errors.New("udhcpc did not output a lease") + // ErrNoHint indicates missing state from the CreateEndpoint stage in Join + ErrNoHint = errors.New("missing CreateEndpoint hints") ) func ErrToStatus(err error) int { diff --git a/test_env.sh b/test_env.sh index bb679ba..02b4356 100755 --- a/test_env.sh +++ b/test_env.sh @@ -1,8 +1,12 @@ #!/bin/sh BRIDGE=net-dhcp -BRIDGE_IP="10.123.0.1/24" +BRIDGE_IP="10.123.0.1" +DUMMY_IP="10.123.0.3" +MASK="24" DHCP_RANGE="10.123.0.5,10.123.0.254" -BRIDGE_IP6="fd69::1/64" +BRIDGE_IP6="fd69::1" +DUMMY_IP6="fd69::3" +MASK6="64" DHCP6_RANGE="fd69::5,fd69::1000,64" DOMAIN=cool-dhcp @@ -15,13 +19,20 @@ trap quit SIGINT SIGTERM ip link add "$BRIDGE" type bridge ip link set up dev "$BRIDGE" -ip addr add "$BRIDGE_IP" dev "$BRIDGE" -ip addr add "$BRIDGE_IP6" dev "$BRIDGE" +ip addr add "$BRIDGE_IP/$MASK" dev "$BRIDGE" +ip addr add "$BRIDGE_IP6/$MASK6" dev "$BRIDGE" -dnsmasq --no-daemon --conf-file=/dev/null \ +ip route add 10.223.0.0/24 dev "$BRIDGE" +ip route add 10.224.0.0/24 via "$DUMMY_IP" +ip route add fd42::0/64 dev "$BRIDGE" +# TODO: This doesn't work right now because the route is added by Docker before +# router advertisement stuff is done :/ +#ip route add fd43::0/64 via "$DUMMY_IP6" + +dnsmasq --no-daemon --conf-file=/dev/null --dhcp-leasefile=/dev/null \ --port=0 --interface="$BRIDGE" --bind-interfaces \ --domain="$DOMAIN" \ - --dhcp-range="$DHCP_RANGE" --dhcp-leasefile=/dev/null \ + --dhcp-range="$DHCP_RANGE" \ --dhcp-range="$DHCP6_RANGE" --enable-ra quit