Further work reducing file descriptor leaks

The file descriptor problem is multi-tiered... Maintaining pyroute2 NDB
sources in namespaces keeps a proxy process running in each namespace,
wasting a lot of file descriptors on pipes. It also leaks some of these
pipes upon removal of sources! Even the Python Docker client leaks its
sockets! (https://github.com/docker/docker-py/issues/1293)
This commit is contained in:
Jack O'Sullivan 2019-09-22 03:07:50 +01:00
parent 0711a747bd
commit 7afa8efbd6
3 changed files with 23 additions and 31 deletions

View File

@ -1,6 +1,5 @@
import logging
import socketserver
import resource
from werkzeug.serving import run_simple
from . import app
@ -11,7 +10,5 @@ logger = logging.getLogger('net-dhcp')
logger.setLevel(logging.DEBUG)
logger.addHandler(fh)
resource.setrlimit(resource.RLIMIT_NOFILE, (1048576, 1048576))
socketserver.TCPServer.allow_reuse_address = True
run_simple('unix:///run/docker/plugins/net-dhcp.sock', 0, app)

View File

@ -5,10 +5,10 @@ import atexit
import socket
import time
import threading
import subprocess
import pyroute2
from pyroute2.netlink.rtnl import rtypes
from pyroute2.netns.process.proxy import NSPopen
import docker
from flask import request, jsonify
@ -66,17 +66,15 @@ def endpoint_container_iface(n, e):
container = client.containers.get(cid)
netns = f'/proc/{container.attrs["State"]["Pid"]}/ns/net'
already = False
for source in ndb.sources:
if source == netns:
already = True
break
if not already:
ndb.sources.add(netns=netns)
for i in ndb.interfaces:
if i['address'] == info['MacAddress']:
return i
with pyroute2.NetNS(netns) as rtnl:
for link in rtnl.get_links():
attrs = dict(link['attrs'])
if attrs['IFLA_ADDRESS'] == info['MacAddress']:
return {
'netns': netns,
'ifname': attrs['IFLA_IFNAME'],
'address': attrs['IFLA_ADDRESS']
}
break
return None
def await_endpoint_container_iface(n, e, timeout=5):
@ -324,13 +322,9 @@ class ContainerDHCPManager:
return
logger.info('[dhcp container] Replacing gateway with %s', dhcp.gateway)
proc = NSPopen(dhcp.netns, ['/sbin/ip', 'route', 'replace', 'default', 'via', str(dhcp.gateway)])
try:
if proc.wait(timeout=1) != 0:
raise NetDhcpError(f'Failed to replace default route; "ip route" command exited with non-zero code %d', \
proc.returncode)
finally:
proc.release()
subprocess.check_call(['nsenter', f'-n{dhcp.netns}', '--', '/sbin/ip', 'route', 'replace', 'default', 'via',
str(dhcp.gateway)], timeout=1, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
# TODO: Adding default route with NDB seems to be broken (because of the dst syntax?)
#for route in ndb.routes:
@ -384,5 +378,9 @@ class ContainerDHCPManager:
self.dhcp6.iface['ifname'], self.dhcp.netns)
self.dhcp6.finish(timeout=1)
finally:
ndb.sources.remove(self.dhcp.netns)
self._thread.join()
# we have to do this since the docker client leaks sockets...
global client
client.close()
client = docker.from_env()

View File

@ -12,7 +12,6 @@ import logging
from eventfd import EventFD
import posix_ipc
from pyroute2.netns.process.proxy import NSPopen
HANDLER_SCRIPT = path.join(path.dirname(__file__), 'udhcpc_handler.py')
AWAIT_INTERVAL = 0.1
@ -30,7 +29,7 @@ class DHCPClientError(Exception):
pass
def _nspopen_wrapper(netns):
return lambda *args, **kwargs: NSPopen(netns, *args, **kwargs)
return lambda cmd, *args, **kwargs: subprocess.Popen(['nsenter', f'-n{netns}', '--'] + cmd, *args, **kwargs)
class DHCPClient:
def __init__(self, iface, v6=False, once=False, hostname=None, event_listener=None):
self.iface = iface
@ -41,8 +40,8 @@ class DHCPClient:
self.event_listeners.append(event_listener)
self.netns = None
if iface['target'] and iface['target'] != 'localhost':
self.netns = iface['target']
if 'netns' in iface:
self.netns = iface['netns']
logger.debug('udhcpc using netns %s', self.netns)
Popen = _nspopen_wrapper(self.netns) if self.netns else subprocess.Popen
@ -67,7 +66,8 @@ class DHCPClient:
self._suffix = '6' if v6 else ''
self._event_queue = posix_ipc.MessageQueue(f'/udhcpc{self._suffix}_{iface["address"].replace(":", "_")}', \
flags=os.O_CREAT | os.O_EXCL, max_messages=2, max_message_size=1024)
self.proc = Popen(cmdline, env={'EVENT_QUEUE': self._event_queue.name})
self.proc = Popen(cmdline, env={'EVENT_QUEUE': self._event_queue.name}, stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, preexec_fn=close_fds)
if hostname:
logger.debug('[udhcpc%s#%d] using hostname "%s"', self._suffix, self.proc.pid, hostname)
@ -142,9 +142,6 @@ class DHCPClient:
return self.ip
finally:
if self.netns:
self.proc.release()
self._shutdown_event.set()
self._event_thread.join()
self._event_queue.close()