Further work reducing file descriptor leaks

The file descriptor problem is multi-tiered... Maintaining pyroute2 NDB
sources in namespaces keeps a proxy process running in each namespace,
wasting a lot of file descriptors on pipes. It also leaks some of these
pipes upon removal of sources! Even the Python Docker client leaks its
sockets! (https://github.com/docker/docker-py/issues/1293)

net-dhcp/__main__.py

@@ -1,6 +1,5 @@
 import logging
 import socketserver
-import resource
 from werkzeug.serving import run_simple
 from . import app
 
@@ -11,7 +10,5 @@ logger = logging.getLogger('net-dhcp')
 logger.setLevel(logging.DEBUG)
 logger.addHandler(fh)
 
-resource.setrlimit(resource.RLIMIT_NOFILE, (1048576, 1048576))
-
 socketserver.TCPServer.allow_reuse_address = True
 run_simple('unix:///run/docker/plugins/net-dhcp.sock', 0, app)

net-dhcp/network.py

@@ -5,10 +5,10 @@ import atexit
 import socket
 import time
 import threading
+import subprocess
 
 import pyroute2
 from pyroute2.netlink.rtnl import rtypes
-from pyroute2.netns.process.proxy import NSPopen
 import docker
 from flask import request, jsonify
 
@@ -66,17 +66,15 @@ def endpoint_container_iface(n, e):
             container = client.containers.get(cid)
             netns = f'/proc/{container.attrs["State"]["Pid"]}/ns/net'
 
-            already = False
-            for source in ndb.sources:
-                if source == netns:
-                    already = True
-                    break
-            if not already:
-                ndb.sources.add(netns=netns)
-
-            for i in ndb.interfaces:
-                if i['address'] == info['MacAddress']:
-                    return i
+            with pyroute2.NetNS(netns) as rtnl:
+                for link in rtnl.get_links():
+                    attrs = dict(link['attrs'])
+                    if attrs['IFLA_ADDRESS'] == info['MacAddress']:
+                        return {
+                            'netns': netns,
+                            'ifname': attrs['IFLA_IFNAME'],
+                            'address': attrs['IFLA_ADDRESS']
+                        }
             break
     return None
 def await_endpoint_container_iface(n, e, timeout=5):
@@ -324,13 +322,9 @@ class ContainerDHCPManager:
             return
 
         logger.info('[dhcp container] Replacing gateway with %s', dhcp.gateway)
-        proc = NSPopen(dhcp.netns, ['/sbin/ip', 'route', 'replace', 'default', 'via', str(dhcp.gateway)])
-        try:
-            if proc.wait(timeout=1) != 0:
-                raise NetDhcpError(f'Failed to replace default route; "ip route" command exited with non-zero code %d', \
-                    proc.returncode)
-        finally:
-            proc.release()
+        subprocess.check_call(['nsenter', f'-n{dhcp.netns}', '--', '/sbin/ip', 'route', 'replace', 'default', 'via',
+            str(dhcp.gateway)], timeout=1, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL)
 
         # TODO: Adding default route with NDB seems to be broken (because of the dst syntax?)
         #for route in ndb.routes:
@@ -384,5 +378,9 @@ class ContainerDHCPManager:
                         self.dhcp6.iface['ifname'], self.dhcp.netns)
                     self.dhcp6.finish(timeout=1)
             finally:
-                ndb.sources.remove(self.dhcp.netns)
                 self._thread.join()
+
+                # we have to do this since the docker client leaks sockets...
+                global client
+                client.close()
+                client = docker.from_env()

net-dhcp/udhcpc.py

@@ -12,7 +12,6 @@ import logging
 
 from eventfd import EventFD
 import posix_ipc
-from pyroute2.netns.process.proxy import NSPopen
 
 HANDLER_SCRIPT = path.join(path.dirname(__file__), 'udhcpc_handler.py')
 AWAIT_INTERVAL = 0.1
@@ -30,7 +29,7 @@ class DHCPClientError(Exception):
     pass
 
 def _nspopen_wrapper(netns):
-    return lambda *args, **kwargs: NSPopen(netns, *args, **kwargs)
+    return lambda cmd, *args, **kwargs: subprocess.Popen(['nsenter', f'-n{netns}', '--'] + cmd, *args, **kwargs)
 class DHCPClient:
     def __init__(self, iface, v6=False, once=False, hostname=None, event_listener=None):
         self.iface = iface
@@ -41,8 +40,8 @@ class DHCPClient:
             self.event_listeners.append(event_listener)
 
         self.netns = None
-        if iface['target'] and iface['target'] != 'localhost':
-            self.netns = iface['target']
+        if 'netns' in iface:
+            self.netns = iface['netns']
             logger.debug('udhcpc using netns %s', self.netns)
 
         Popen = _nspopen_wrapper(self.netns) if self.netns else subprocess.Popen
@@ -67,7 +66,8 @@ class DHCPClient:
         self._suffix = '6' if v6 else ''
         self._event_queue = posix_ipc.MessageQueue(f'/udhcpc{self._suffix}_{iface["address"].replace(":", "_")}', \
             flags=os.O_CREAT | os.O_EXCL, max_messages=2, max_message_size=1024)
-        self.proc = Popen(cmdline, env={'EVENT_QUEUE': self._event_queue.name})
+        self.proc = Popen(cmdline, env={'EVENT_QUEUE': self._event_queue.name}, stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, preexec_fn=close_fds)
         if hostname:
             logger.debug('[udhcpc%s#%d] using hostname "%s"', self._suffix, self.proc.pid, hostname)
 
@@ -142,9 +142,6 @@ class DHCPClient:
 
             return self.ip
         finally:
-            if self.netns:
-                self.proc.release()
-
             self._shutdown_event.set()
             self._event_thread.join()
             self._event_queue.close()