From c01112e4b77d633f33637caf5f31f04a1cf67186 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 3 Sep 2019 11:56:56 -0400
Subject: [PATCH 01/27] tracy lock contention testing and other fun things

---
 CMakeLists.txt                       |  16 +++-
 Makefile                             |   4 +-
 llarp/handlers/exit.cpp              |  29 +++-----
 llarp/handlers/tun.cpp               | 107 ++++++++++++++++-----------
 llarp/iwp/linklayer.cpp              |   6 +-
 llarp/link/server.cpp                |  40 +++++-----
 llarp/link/server.hpp                |  17 +++--
 llarp/net/ip_range_map.hpp           |  89 ++++++++++++++++++++++
 llarp/net/net.cpp                    |  46 ++++++++++++
 llarp/net/net.hpp                    |  27 ++++++-
 llarp/net/net_int.hpp                |  12 +++
 llarp/path/pathbuilder.cpp           |   3 +-
 llarp/router/i_rc_lookup_handler.hpp |   3 +
 llarp/router/rc_lookup_handler.cpp   |   6 ++
 llarp/router/rc_lookup_handler.hpp   |   3 +
 llarp/router/router.cpp              |  11 ++-
 llarp/service/endpoint.cpp           |  15 ++++
 llarp/service/endpoint.hpp           |   7 +-
 llarp/service/outbound_context.cpp   |   6 ++
 llarp/util/bits.hpp                  |   7 ++
 llarp/util/threading.hpp             |  21 +++++-
 llarp/utp/linklayer.cpp              |   4 +-
 22 files changed, 374 insertions(+), 105 deletions(-)
 create mode 100644 llarp/net/ip_range_map.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4e09a39b6..575acacd6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -27,6 +27,7 @@ option(WITH_SHARED "build shared library")
 option(WITH_COVERAGE "generate coverage data")
 option(USE_SHELLHOOKS "enable shell hooks on compile time (dangerous)" OFF)
 option(WARNINGS_AS_ERRORS "treat all warnings as errors. turn off for development, on for release" OFF)
+option(TRACY_ROOT "include tracy profiler source")
 
 include(cmake/target_link_libraries_system.cmake)
 include(cmake/add_import_library.cmake)
@@ -195,6 +196,11 @@ add_definitions("-DGIT_REV=\"${GIT_VERSION_REAL}\"")
 
 set(EXE lokinet)
 set(EXE_SRC daemon/main.cpp)
+if(TRACY_ROOT)
+  include_directories(${TRACY_ROOT})
+  add_definitions(-DTRACY_ENABLE)
+  list(APPEND EXE_SRC ${TRACY_ROOT}/TracyClient.cpp)
+endif()
 
 # HeapAlloc(2) on Windows was significantly revamped in 2009
 # but the old algorithm isn't too bad either
@@ -223,6 +229,9 @@ else()
 endif()
 
 set(LIBS ${MALLOC_LIB} ${FS_LIB} ${LIBUV_LIBRARY})
+if(TRACY_ROOT)
+  list(APPEND LIBS -ldl)
+endif()
 
 add_subdirectory(crypto)
 add_subdirectory(libutp)
@@ -240,6 +249,7 @@ else()
   target_link_libraries(${ABYSS_EXE} PUBLIC ${ABYSS_LIB} ${STATIC_LIB} ws2_32)
 endif(NOT WIN32)
 
+
 # Why does abyss not inherit the existing include folders?
 target_include_directories(${ABYSS_LIB} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/${ABYSS}/include" llarp vendor/nlohmann/include include crypto/include)
 target_include_directories(${ABYSS_EXE} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/${ABYSS}/include" llarp vendor/nlohmann/include include crypto/include)
@@ -262,7 +272,11 @@ if(SHADOW)
 else()
   if(NOT WIN32)
     add_executable(${EXE} ${EXE_SRC})
-    add_executable(lokinet-rcutil daemon/rcutil.cpp)
+    if(TRACY_ROOT)
+      add_executable(lokinet-rcutil daemon/rcutil.cpp ${TRACY_ROOT}/TracyClient.cpp)
+    else()
+      add_executable(lokinet-rcutil daemon/rcutil.cpp)
+    endif()
   elseif(NOT MSVC_VERSION)
     add_executable(${EXE} ${EXE_SRC} llarp/win32/version.rc)
     add_executable(lokinet-rcutil daemon/rcutil.cpp llarp/win32/version.rc)
diff --git a/Makefile b/Makefile
index f82077ddb..fc6e9d21c 100644
--- a/Makefile
+++ b/Makefile
@@ -80,6 +80,8 @@ COVERAGE ?= OFF
 COVERAGE_OUTDIR ?= "$(TMPDIR)/lokinet-coverage"
 # enable ASAN
 ASAN ?= OFF
+# tracy profiler
+TRACY_ROOT ?=
 
 # cmake generator type
 CMAKE_GEN ?= Unix Makefiles
@@ -104,7 +106,7 @@ ANALYZE_CONFIG_CMD = $(shell gecho -n "cd '$(BUILD_ROOT)' && " ; gecho -n "$(SCA
 
 COVERAGE_CONFIG_CMD = $(shell gecho -n "cd '$(BUILD_ROOT)' && " ; gecho -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DWITH_COVERAGE=yes -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
 else
-CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_SHELLHOOKS=$(SHELL_HOOKS) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
+CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_SHELLHOOKS=$(SHELL_HOOKS) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DTRACY_ROOT=$(TRACY_ROOT) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
 CONFIG_CMD_WINDOWS = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=ON -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_SHELLHOOKS=$(SHELL_HOOKS) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
 
 ANALYZE_CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "$(SCAN_BUILD) cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DASAN=$(ASAN) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
diff --git a/llarp/handlers/exit.cpp b/llarp/handlers/exit.cpp
index ca68b9f60..c0f6c944f 100644
--- a/llarp/handlers/exit.cpp
+++ b/llarp/handlers/exit.cpp
@@ -5,6 +5,7 @@
 #include <path/path_context.hpp>
 #include <router/abstractrouter.hpp>
 #include <util/str.hpp>
+#include <util/bits.hpp>
 
 #include <cassert>
 
@@ -547,6 +548,11 @@ namespace llarp
       }
       if(k == "ifaddr")
       {
+        if(!m_OurRange.FromString(v))
+        {
+          LogError(Name(), " has invalid address range: ", v);
+          return false;
+        }
         auto pos = v.find("/");
         if(pos == std::string::npos)
         {
@@ -558,26 +564,9 @@ namespace llarp
         // string, or just a plain char array?
         strncpy(m_Tun.ifaddr, host_str.c_str(), sizeof(m_Tun.ifaddr) - 1);
         m_Tun.netmask = std::atoi(nmask_str.c_str());
-
-        huint32_t ip;
-        if(ip.FromString(host_str))
-        {
-          m_IfAddr                = net::IPPacket::ExpandV4(ip);
-          m_OurRange.netmask_bits = netmask_ipv6_bits(m_Tun.netmask + 96);
-        }
-        else if(m_IfAddr.FromString(host_str))
-        {
-          m_UseV6                 = true;
-          m_OurRange.netmask_bits = netmask_ipv6_bits(m_Tun.netmask);
-        }
-        else
-        {
-          LogError(Name(), " invalid ifaddr: ", v);
-          return false;
-        }
-        m_OurRange.addr = m_IfAddr;
-        m_NextAddr      = m_IfAddr;
-        m_HigestAddr    = m_IfAddr | (~m_OurRange.netmask_bits);
+        m_IfAddr      = m_OurRange.addr;
+        m_NextAddr    = m_IfAddr;
+        m_HigestAddr  = m_OurRange.HighestAddr();
         LogInfo(Name(), " set ifaddr range to ", m_Tun.ifaddr, "/",
                 m_Tun.netmask, " lo=", m_IfAddr, " hi=", m_HigestAddr);
       }
diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp
index ca54318e6..e829dd2b0 100644
--- a/llarp/handlers/tun.cpp
+++ b/llarp/handlers/tun.cpp
@@ -18,6 +18,8 @@
 
 #include <util/str.hpp>
 
+#include <absl/strings/ascii.h>
+
 namespace llarp
 {
   namespace handlers
@@ -143,18 +145,39 @@ namespace llarp
       // the keyfile
       if(k == "exit-node")
       {
+        IPRange exitRange;
         llarp::RouterID exitRouter;
-        if(!(exitRouter.FromString(v)
-             || HexDecode(v.c_str(), exitRouter.begin(), exitRouter.size())))
+        std::string routerStr;
+        const auto pos = v.find(",");
+        if(pos != std::string::npos)
+        {
+          auto range_str = v.substr(1 + pos);
+          if(!exitRange.FromString(range_str))
+          {
+            LogError("bad exit range: '", range_str, "'");
+            return false;
+          }
+          routerStr = v.substr(0, pos);
+        }
+        else
         {
-          llarp::LogError(Name(), " bad exit router key: ", v);
+          routerStr = v;
+        }
+        absl::StripAsciiWhitespace(&routerStr);
+        if(!(exitRouter.FromString(routerStr)
+             || HexDecode(routerStr.c_str(), exitRouter.begin(),
+                          exitRouter.size())))
+        {
+          llarp::LogError(Name(), " bad exit router key: ", routerStr);
           return false;
         }
-        m_Exit = std::make_shared< llarp::exit::ExitSession >(
+        auto exit = std::make_shared< llarp::exit::ExitSession >(
             exitRouter,
             util::memFn(&TunEndpoint::QueueInboundPacketForExit, this),
             m_router, numPaths, numHops, ShouldBundleRC());
-        llarp::LogInfo(Name(), " using exit at ", exitRouter);
+        m_ExitMap.Insert(exitRange, exit);
+        llarp::LogInfo(Name(), " using exit at ", exitRouter, " for ",
+                       exitRange);
       }
       if(k == "local-dns")
       {
@@ -288,13 +311,11 @@ namespace llarp
     {
       auto self = shared_from_this();
       FlushSend();
-      if(m_Exit)
-      {
-        RouterLogic()->queue_func([=] {
-          self->m_Exit->FlushUpstream();
-          self->Router()->PumpLL();
-        });
-      }
+      RouterLogic()->queue_func([=] {
+        self->m_ExitMap.ForEachValue(
+            [](const auto &exit) { exit->FlushUpstream(); });
+        self->Router()->PumpLL();
+      });
       RouterLogic()->queue_func([=]() {
         self->Pump(self->Now());
         self->Router()->PumpLL();
@@ -487,8 +508,8 @@ namespace llarp
     TunEndpoint::ResetInternalState()
     {
       service::Endpoint::ResetInternalState();
-      if(m_Exit)
-        m_Exit->ResetInternalState();
+      m_ExitMap.ForEachValue(
+          [](const auto &exit) { exit->ResetInternalState(); });
     }
 
     bool
@@ -550,11 +571,11 @@ namespace llarp
         llarp::LogWarn("Couldn't start endpoint");
         return false;
       }
-      if(m_Exit)
-      {
-        for(const auto &snode : SnodeBlacklist())
-          m_Exit->BlacklistSnode(snode);
-      }
+      const auto blacklist = SnodeBlacklist();
+      m_ExitMap.ForEachValue([blacklist](const auto &exit) {
+        for(const auto &snode : blacklist)
+          exit->BlacklistSnode(snode);
+      });
       return SetupNetworking();
     }
 
@@ -639,7 +660,7 @@ namespace llarp
 
       m_NextIP        = m_OurIP;
       m_OurRange.addr = m_OurIP;
-      m_MaxIP         = m_OurIP | (~m_OurRange.netmask_bits);
+      m_MaxIP         = m_OurRange.HighestAddr();
       llarp::LogInfo(Name(), " set ", tunif.ifname, " to have address ",
                      m_OurIP);
       llarp::LogInfo(Name(), " allocated up to ", m_MaxIP, " on range ",
@@ -690,19 +711,17 @@ namespace llarp
     {
       // call tun code in endpoint logic in case of network isolation
       // EndpointLogic()->queue_job({this, handleTickTun});
-      if(m_Exit)
-      {
-        EnsureRouterIsKnown(m_Exit->Endpoint());
-        m_Exit->Tick(now);
-      }
+      m_ExitMap.ForEachValue([&](const auto &exit) {
+        EnsureRouterIsKnown(exit->Endpoint());
+        exit->Tick(now);
+      });
       Endpoint::Tick(now);
     }
 
     bool
     TunEndpoint::Stop()
     {
-      if(m_Exit)
-        m_Exit->Stop();
+      m_ExitMap.ForEachValue([](const auto &exit) { exit->Stop(); });
       return llarp::service::Endpoint::Stop();
     }
 
@@ -721,22 +740,26 @@ namespace llarp
         auto itr = m_IPToAddr.find(dst);
         if(itr == m_IPToAddr.end())
         {
-          if(m_Exit && pkt.IsV4() && !llarp::IsIPv4Bogon(pkt.dstv4()))
+          const auto exits = m_ExitMap.FindAll(dst);
+          if(exits.empty())
           {
-            pkt.UpdateIPv4Address({0}, xhtonl(pkt.dstv4()));
-            m_Exit->QueueUpstreamTraffic(std::move(pkt),
-                                         llarp::routing::ExitPadSize);
+            llarp::LogWarn(Name(), " has no exit mapped for ", dst);
+            return;
           }
-          else if(m_Exit && pkt.IsV6())
+          for(const auto &exit : exits)
           {
-            pkt.UpdateIPv6Address({0}, pkt.dstv6());
-            m_Exit->QueueUpstreamTraffic(std::move(pkt),
+            if(pkt.IsV4() && !llarp::IsIPv4Bogon(pkt.dstv4()))
+            {
+              pkt.UpdateIPv4Address({0}, xhtonl(pkt.dstv4()));
+              exit->QueueUpstreamTraffic(std::move(pkt),
                                          llarp::routing::ExitPadSize);
-          }
-          else
-          {
-            llarp::LogWarn(Name(), " has no endpoint for ", dst);
-            llarp::DumpBuffer(pkt.ConstBuffer());
+            }
+            else if(pkt.IsV6())
+            {
+              pkt.UpdateIPv6Address({0}, pkt.dstv6());
+              exit->QueueUpstreamTraffic(std::move(pkt),
+                                         llarp::routing::ExitPadSize);
+            }
           }
           return;
         }
@@ -913,10 +936,8 @@ namespace llarp
       // flush user to network
       self->FlushSend();
       // flush exit traffic queues if it's there
-      if(self->m_Exit)
-      {
-        self->m_Exit->FlushDownstream();
-      }
+      self->m_ExitMap.ForEachValue(
+          [](const auto &exit) { exit->FlushDownstream(); });
       // flush network to user
       self->m_NetworkToUserPktQueue.Process([tun](net::IPPacket &pkt) {
         if(!llarp_ev_tun_async_write(tun, pkt.Buffer()))
diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index 57d0c499a..84e6db456 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -24,7 +24,7 @@ namespace llarp
     {
       std::set< RouterID > sessions;
       {
-        Lock l(&m_AuthedLinksMutex);
+        ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
         auto itr = m_AuthedLinks.begin();
         while(itr != m_AuthedLinks.end())
         {
@@ -34,7 +34,7 @@ namespace llarp
       }
       ILinkLayer::Pump();
       {
-        Lock l(&m_AuthedLinksMutex);
+        ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
         for(const auto& pk : sessions)
         {
           if(m_AuthedLinks.count(pk) == 0)
@@ -79,7 +79,7 @@ namespace llarp
       auto itr = m_AuthedAddrs.find(from);
       if(itr == m_AuthedAddrs.end())
       {
-        util::Lock lock(&m_PendingMutex);
+        // ACQUIRE_LOCK(Lock_t lock , m_PendingMutex);
         if(m_Pending.count(from) == 0)
         {
           if(not permitInbound)
diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp
index 8f0cc2d2d..0d47ccced 100644
--- a/llarp/link/server.cpp
+++ b/llarp/link/server.cpp
@@ -29,7 +29,7 @@ namespace llarp
   bool
   ILinkLayer::HasSessionTo(const RouterID& id)
   {
-    Lock l(&m_AuthedLinksMutex);
+    ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
     return m_AuthedLinks.find(id) != m_AuthedLinks.end();
   }
 
@@ -39,7 +39,7 @@ namespace llarp
   {
     std::vector< std::shared_ptr< ILinkSession > > sessions;
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       if(m_AuthedLinks.size() == 0)
         return;
       const size_t sz = randint() % m_AuthedLinks.size();
@@ -75,7 +75,7 @@ namespace llarp
   {
     std::shared_ptr< ILinkSession > session;
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto itr = m_AuthedLinks.find(pk);
       if(itr == m_AuthedLinks.end())
         return false;
@@ -89,7 +89,7 @@ namespace llarp
   {
     std::vector< std::shared_ptr< ILinkSession > > sessions;
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto itr = m_AuthedLinks.begin();
       while(itr != m_AuthedLinks.end())
       {
@@ -125,7 +125,7 @@ namespace llarp
   {
     auto _now = Now();
     {
-      Lock lock(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto itr = m_AuthedLinks.begin();
       while(itr != m_AuthedLinks.end())
       {
@@ -144,7 +144,7 @@ namespace llarp
       }
     }
     {
-      Lock lock(&m_PendingMutex);
+      ACQUIRE_LOCK(Lock_t l, m_PendingMutex);
 
       auto itr = m_Pending.begin();
       while(itr != m_Pending.end())
@@ -172,8 +172,8 @@ namespace llarp
   bool
   ILinkLayer::MapAddr(const RouterID& pk, ILinkSession* s)
   {
-    Lock l_authed(&m_AuthedLinksMutex);
-    Lock l_pending(&m_PendingMutex);
+    ACQUIRE_LOCK(Lock_t l_authed, m_AuthedLinksMutex);
+    ACQUIRE_LOCK(Lock_t l_pending, m_PendingMutex);
     llarp::Addr addr = s->GetRemoteEndpoint();
     auto itr         = m_Pending.find(addr);
     if(itr != m_Pending.end())
@@ -213,7 +213,7 @@ namespace llarp
     std::vector< util::StatusObject > pending, established;
 
     {
-      Lock l(&m_PendingMutex);
+      ACQUIRE_LOCK(Lock_t l, m_PendingMutex);
       std::transform(m_Pending.cbegin(), m_Pending.cend(),
                      std::back_inserter(pending),
                      [](const auto& item) -> util::StatusObject {
@@ -221,7 +221,7 @@ namespace llarp
                      });
     }
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       std::transform(m_AuthedLinks.cbegin(), m_AuthedLinks.cend(),
                      std::back_inserter(established),
                      [](const auto& item) -> util::StatusObject {
@@ -241,7 +241,7 @@ namespace llarp
   ILinkLayer::TryEstablishTo(RouterContact rc)
   {
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       if(m_AuthedLinks.count(rc.pubkey) >= MaxSessionsPerKey)
         return false;
     }
@@ -250,7 +250,7 @@ namespace llarp
       return false;
     const llarp::Addr addr(to);
     {
-      Lock l(&m_PendingMutex);
+      ACQUIRE_LOCK(Lock_t l, m_PendingMutex);
       if(m_Pending.count(addr) >= MaxSessionsPerKey)
         return false;
     }
@@ -275,7 +275,7 @@ namespace llarp
   ILinkLayer::Tick(llarp_time_t now)
   {
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto itr = m_AuthedLinks.begin();
       while(itr != m_AuthedLinks.end())
       {
@@ -285,7 +285,7 @@ namespace llarp
     }
 
     {
-      Lock l(&m_PendingMutex);
+      ACQUIRE_LOCK(Lock_t l, m_PendingMutex);
       auto itr = m_Pending.begin();
       while(itr != m_Pending.end())
       {
@@ -301,7 +301,7 @@ namespace llarp
     if(m_Logic && tick_id)
       m_Logic->remove_call(tick_id);
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto itr = m_AuthedLinks.begin();
       while(itr != m_AuthedLinks.end())
       {
@@ -310,7 +310,7 @@ namespace llarp
       }
     }
     {
-      Lock l(&m_PendingMutex);
+      ACQUIRE_LOCK(Lock_t l, m_PendingMutex);
       auto itr = m_Pending.begin();
       while(itr != m_Pending.end())
       {
@@ -323,7 +323,7 @@ namespace llarp
   void
   ILinkLayer::CloseSessionTo(const RouterID& remote)
   {
-    Lock l(&m_AuthedLinksMutex);
+    ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
     RouterID r = remote;
     llarp::LogInfo("Closing all to ", r);
     auto range = m_AuthedLinks.equal_range(r);
@@ -338,7 +338,7 @@ namespace llarp
   void
   ILinkLayer::KeepAliveSessionTo(const RouterID& remote)
   {
-    Lock l(&m_AuthedLinksMutex);
+    ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
     auto range = m_AuthedLinks.equal_range(remote);
     auto itr   = range.first;
     while(itr != range.second)
@@ -355,7 +355,7 @@ namespace llarp
   {
     ILinkSession* s = nullptr;
     {
-      Lock l(&m_AuthedLinksMutex);
+      ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto range = m_AuthedLinks.equal_range(remote);
       auto itr   = range.first;
       // pick lowest backlog session
@@ -431,7 +431,7 @@ namespace llarp
   ILinkLayer::PutSession(const std::shared_ptr< ILinkSession >& s)
   {
     static constexpr size_t MaxSessionsPerEndpoint = 5;
-    Lock lock(&m_PendingMutex);
+    ACQUIRE_LOCK(Lock_t lock, m_PendingMutex);
     llarp::Addr addr = s->GetRemoteEndpoint();
     if(m_Pending.count(addr) >= MaxSessionsPerEndpoint)
       return false;
diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp
index be25c4fdc..1650f3c14 100644
--- a/llarp/link/server.hpp
+++ b/llarp/link/server.hpp
@@ -237,9 +237,13 @@ namespace llarp
     const SecretKey& m_RouterEncSecret;
 
    protected:
-    using Lock  = util::Lock;
-    using Mutex = util::Mutex;
-
+#ifdef TRACY_ENABLE
+    using Lock_t  = std::lock_guard< LockableBase(std::mutex) >;
+    using Mutex_t = std::mutex;
+#else
+    using Lock_t  = util::NullLock;
+    using Mutex_t = util::NullMutex;
+#endif
     bool
     PutSession(const std::shared_ptr< ILinkSession >& s);
 
@@ -255,10 +259,11 @@ namespace llarp
     using Pending =
         std::unordered_multimap< llarp::Addr, std::shared_ptr< ILinkSession >,
                                  llarp::Addr::Hash >;
-
-    mutable Mutex m_AuthedLinksMutex ACQUIRED_BEFORE(m_PendingMutex);
+    mutable DECLARE_LOCK(Mutex_t, m_AuthedLinksMutex,
+                         ACQUIRED_BEFORE(m_PendingMutex));
     AuthedLinks m_AuthedLinks GUARDED_BY(m_AuthedLinksMutex);
-    mutable Mutex m_PendingMutex ACQUIRED_AFTER(m_AuthedLinksMutex);
+    mutable DECLARE_LOCK(Mutex_t, m_PendingMutex,
+                         ACQUIRED_AFTER(m_AuthedLinksMutex));
     Pending m_Pending GUARDED_BY(m_PendingMutex);
   };
 
diff --git a/llarp/net/ip_range_map.hpp b/llarp/net/ip_range_map.hpp
new file mode 100644
index 000000000..f54d6196f
--- /dev/null
+++ b/llarp/net/ip_range_map.hpp
@@ -0,0 +1,89 @@
+#ifndef LLARP_NET_IP_RANGE_MAP_HPP
+#define LLARP_NET_IP_RANGE_MAP_HPP
+
+#include <net/ip.hpp>
+
+namespace llarp
+{
+  namespace net
+  {
+    /// a container that maps an ip range to a value that allows you to lookup
+    /// key by range hit
+    /// TODO: do some kind of magic shit to ensure near constant time for
+    /// lookups
+    template < typename Value_t >
+    struct IPRangeMap
+    {
+      using Range_t = IPRange;
+      using IP_t    = Range_t::Addr_t;
+
+      using Entry_t     = std::pair< Range_t, Value_t >;
+      using Container_t = std::forward_list< Entry_t >;
+
+      /// get a set of all values
+      std::set< Value_t >
+      Values() const
+      {
+        std::set< Value_t > all;
+        for(const auto &entry : m_Entries)
+          all.insert(entry.second);
+        return all;
+      }
+
+      void
+      ForEachValue(std::function< void(const Value_t &) > functor) const
+      {
+        for(const auto &entry : m_Entries)
+          functor(entry.second);
+      }
+
+      /// convert all values into type T
+      template < typename T, typename Transformer >
+      std::set< T >
+      TransformValues(Transformer transform) const
+      {
+        std::set< T > transformed;
+        for(const auto &entry : m_Entries)
+        {
+          T val = transform(entry.second);
+          transformed.insert(std::move(val));
+        }
+        return transformed;
+      }
+
+      /// return a set of all values who's range contains this IP
+      std::set< Value_t >
+      FindAll(const IP_t &addr) const
+      {
+        std::set< Value_t > found;
+        for(const auto &entry : m_Entries)
+        {
+          if(entry.first.Contains(addr))
+            found.insert(entry.second);
+        }
+        return found;
+      }
+
+      struct CompareEntry
+      {
+        bool
+        operator()(const Entry_t &left, const Entry_t &right) const
+        {
+          return left.first < right.first;
+        }
+      };
+
+      void
+      Insert(const Range_t &addr, const Value_t &val)
+      {
+        m_Entries.emplace_front(addr, val);
+        m_Entries.sort(CompareEntry{});
+      }
+
+     private:
+      Container_t m_Entries;
+    };
+  }  // namespace net
+}  // namespace llarp
+
+#endif
\ No newline at end of file
diff --git a/llarp/net/net.cpp b/llarp/net/net.cpp
index f55916912..b7329ba7c 100644
--- a/llarp/net/net.cpp
+++ b/llarp/net/net.cpp
@@ -987,6 +987,52 @@ namespace llarp
     return Contains(net::IPPacket::ExpandV4(ip));
   }
 
+  bool
+  IPRange::FromString(std::string str)
+  {
+    const auto colinpos = str.find(":");
+    const auto slashpos = str.find("/");
+    std::string bitsstr;
+    if(slashpos != std::string::npos)
+    {
+      bitsstr = str.substr(slashpos + 1);
+      str     = str.substr(0, slashpos);
+    }
+    if(colinpos == std::string::npos)
+    {
+      huint32_t ip;
+      if(!ip.FromString(str))
+        return false;
+      addr = net::IPPacket::ExpandV4(ip);
+      if(!bitsstr.empty())
+      {
+        auto bits = atoi(bitsstr.c_str());
+        if(bits < 0 || bits > 32)
+          return false;
+        netmask_bits = netmask_ipv6_bits(96 + bits);
+      }
+      else
+        netmask_bits = netmask_ipv6_bits(128);
+    }
+    else
+    {
+      if(!addr.FromString(str))
+        return false;
+      if(!bitsstr.empty())
+      {
+        auto bits = atoi(bitsstr.c_str());
+        if(bits < 0 || bits > 128)
+          return false;
+        netmask_bits = netmask_ipv6_bits(bits);
+      }
+      else
+      {
+        netmask_bits = netmask_ipv6_bits(128);
+      }
+    }
+    return true;
+  }
+
   std::string
   IPRange::ToString() const
   {
diff --git a/llarp/net/net.hpp b/llarp/net/net.hpp
index 044e512f6..3e6325dc1 100644
--- a/llarp/net/net.hpp
+++ b/llarp/net/net.hpp
@@ -60,12 +60,13 @@ namespace llarp
 {
   struct IPRange
   {
-    huint128_t addr;
-    huint128_t netmask_bits;
+    using Addr_t            = huint128_t;
+    huint128_t addr         = {0};
+    huint128_t netmask_bits = {0};
 
     /// return true if ip is contained in this ip range
     bool
-    Contains(const huint128_t& ip) const
+    Contains(const Addr_t& ip) const
     {
       return (addr & netmask_bits) == (ip & netmask_bits);
     }
@@ -79,8 +80,28 @@ namespace llarp
       return out << a.ToString();
     }
 
+    /// get the highest address on this range
+    huint128_t
+    HighestAddr() const
+    {
+      return (addr & netmask_bits)
+          + (huint128_t{1} << (128 - bits::count_bits_128(netmask_bits.h)))
+          - huint128_t{1};
+    }
+
+    bool
+    operator<(const IPRange& other) const
+    {
+      return (this->addr & this->netmask_bits)
+          < (other.addr & other.netmask_bits)
+          || this->netmask_bits < other.netmask_bits;
+    }
+
     std::string
     ToString() const;
+
+    bool
+    FromString(std::string str);
   };
 
   huint128_t
diff --git a/llarp/net/net_int.hpp b/llarp/net/net_int.hpp
index 5421abc4d..44c39f366 100644
--- a/llarp/net/net_int.hpp
+++ b/llarp/net/net_int.hpp
@@ -40,6 +40,18 @@ namespace llarp
       return huint_t{UInt_t{h | x.h}};
     }
 
+    constexpr huint_t
+    operator-(huint_t x) const
+    {
+      return huint_t{UInt_t{h - x.h}};
+    }
+
+    constexpr huint_t
+    operator+(huint_t x) const
+    {
+      return huint_t{UInt_t{h + x.h}};
+    }
+
     constexpr huint_t
     operator^(huint_t x) const
     {
diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp
index a1026cca7..434704037 100644
--- a/llarp/path/pathbuilder.cpp
+++ b/llarp/path/pathbuilder.cpp
@@ -214,7 +214,8 @@ namespace llarp
               if(s && s->IsEstablished() && isOutbound && !got)
               {
                 const RouterContact rc = s->GetRemoteRC();
-                if(got || exclude.count(rc.pubkey))
+                if(got || exclude.count(rc.pubkey)
+                   || m_router->IsBootstrapNode(rc.pubkey))
                   return;
                 cur = rc;
                 got = true;
diff --git a/llarp/router/i_rc_lookup_handler.hpp b/llarp/router/i_rc_lookup_handler.hpp
index 33ef5581f..15920df87 100644
--- a/llarp/router/i_rc_lookup_handler.hpp
+++ b/llarp/router/i_rc_lookup_handler.hpp
@@ -56,6 +56,9 @@ namespace llarp
 
     virtual void
     ExploreNetwork() = 0;
+
+    virtual size_t
+    NumberOfStrictConnectRouters() const = 0;
   };
 
 }  // namespace llarp
diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp
index 2f4b05322..e75ee59c0 100644
--- a/llarp/router/rc_lookup_handler.cpp
+++ b/llarp/router/rc_lookup_handler.cpp
@@ -149,6 +149,12 @@ namespace llarp
     return true;
   }
 
+  size_t
+  RCLookupHandler::NumberOfStrictConnectRouters() const
+  {
+    return _strictConnectPubkeys.size();
+  }
+
   bool
   RCLookupHandler::GetRandomWhitelistRouter(RouterID &router) const
   {
diff --git a/llarp/router/rc_lookup_handler.hpp b/llarp/router/rc_lookup_handler.hpp
index a94a60a1c..534cb3fbe 100644
--- a/llarp/router/rc_lookup_handler.hpp
+++ b/llarp/router/rc_lookup_handler.hpp
@@ -64,6 +64,9 @@ namespace llarp
     void
     ExploreNetwork() override;
 
+    size_t
+    NumberOfStrictConnectRouters() const override;
+
     void
     Init(llarp_dht_context *dht, llarp_nodedb *nodedb,
          std::shared_ptr< llarp::thread::ThreadPool > threadpool,
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index a7475cfc8..889058f2e 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -697,9 +697,16 @@ namespace llarp
 
       _rcLookupHandler.ExploreNetwork();
     }
-    if(connected < _outboundSessionMaker.minConnectedRouters)
+    size_t connectToNum      = _outboundSessionMaker.minConnectedRouters;
+    const auto strictConnect = _rcLookupHandler.NumberOfStrictConnectRouters();
+    if(strictConnect > 0 && connectToNum > strictConnect)
     {
-      size_t dlt = _outboundSessionMaker.minConnectedRouters - connected;
+      connectToNum = strictConnect;
+    }
+
+    if(connected < connectToNum)
+    {
+      size_t dlt = connectToNum - connected;
       LogInfo("connecting to ", dlt, " random routers to keep alive");
       _outboundSessionMaker.ConnectToRandomRouters(dlt, now);
     }
diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp
index a30bf1fca..8e350ec2b 100644
--- a/llarp/service/endpoint.cpp
+++ b/llarp/service/endpoint.cpp
@@ -627,9 +627,24 @@ namespace llarp
       std::set< RouterID > exclude = prev;
       for(const auto& snode : SnodeBlacklist())
         exclude.insert(snode);
+      if(hop == 0)
+      {
+        const auto exits = GetExitRouters();
+        // exclude exit node as first hop in any paths
+        exclude.insert(exits.begin(), exits.end());
+      }
       return path::Builder::SelectHop(db, exclude, cur, hop, roles);
     }
 
+    std::set< RouterID >
+    Endpoint::GetExitRouters() const
+    {
+      return m_ExitMap.TransformValues< RouterID >(
+          [](const exit::BaseSession_ptr& ptr) -> RouterID {
+            return ptr->Endpoint();
+          });
+    }
+
     bool
     Endpoint::ShouldBundleRC() const
     {
diff --git a/llarp/service/endpoint.hpp b/llarp/service/endpoint.hpp
index b40dfc345..fa006d2d7 100644
--- a/llarp/service/endpoint.hpp
+++ b/llarp/service/endpoint.hpp
@@ -4,6 +4,7 @@
 #include <dht/messages/gotrouter.hpp>
 #include <ev/ev.h>
 #include <exit/session.hpp>
+#include <net/ip_range_map.hpp>
 #include <net/net.hpp>
 #include <path/path.hpp>
 #include <path/pathbuilder.hpp>
@@ -145,6 +146,10 @@ namespace llarp
       std::string
       Name() const override;
 
+      /// get a set of all the routers we use as exit node
+      std::set< RouterID >
+      GetExitRouters() const;
+
       bool
       ShouldPublishDescriptors(llarp_time_t now) const override;
 
@@ -417,7 +422,7 @@ namespace llarp
      protected:
       IDataHandler* m_DataHandler = nullptr;
       Identity m_Identity;
-      std::shared_ptr< exit::BaseSession > m_Exit;
+      net::IPRangeMap< exit::BaseSession_ptr > m_ExitMap;
       hooks::Backend_ptr m_OnUp;
       hooks::Backend_ptr m_OnDown;
       hooks::Backend_ptr m_OnReady;
diff --git a/llarp/service/outbound_context.cpp b/llarp/service/outbound_context.cpp
index 9849921a2..616418fe6 100644
--- a/llarp/service/outbound_context.cpp
+++ b/llarp/service/outbound_context.cpp
@@ -301,6 +301,12 @@ namespace llarp
       exclude.insert(m_NextIntro.router);
       for(const auto& snode : m_Endpoint->SnodeBlacklist())
         exclude.insert(snode);
+      if(hop == 0)
+      {
+        // exclude any exits as our first hop
+        const auto exits = m_Endpoint->GetExitRouters();
+        exclude.insert(exits.begin(), exits.end());
+      }
       if(hop == numHops - 1)
       {
         m_Endpoint->EnsureRouterIsKnown(m_NextIntro.router);
diff --git a/llarp/util/bits.hpp b/llarp/util/bits.hpp
index 310650e23..c538ab7d8 100644
--- a/llarp/util/bits.hpp
+++ b/llarp/util/bits.hpp
@@ -23,6 +23,13 @@ namespace llarp
       return std::bitset< std::numeric_limits< Int_t >::digits >(i).count();
     }
 
+    constexpr std::size_t
+    count_bits_128(const absl::uint128& i)
+    {
+      return count_bits(absl::Uint128High64(i))
+          + count_bits(absl::Uint128Low64(i));
+    }
+
     template < typename InputIt >
     constexpr std::size_t
     __count_array_bits(InputIt begin, InputIt end)
diff --git a/llarp/util/threading.hpp b/llarp/util/threading.hpp
index 05113a2b3..a6ba45d6a 100644
--- a/llarp/util/threading.hpp
+++ b/llarp/util/threading.hpp
@@ -13,6 +13,22 @@ using pid_t = int;
 #include <unistd.h>
 #endif
 
+#ifdef TRACY_ENABLE
+#include "Tracy.hpp"
+#define DECLARE_LOCK(type, var, ...) TracyLockable(type, var)
+#define ACQUIRE_LOCK(lock, mtx) lock(mtx)
+#else
+#define DECLARE_LOCK(type, var, ...) type var __VA_ARGS__
+#define ACQUIRE_LOCK(lock, mtx) lock(&mtx)
+#endif
+
+#ifdef YOLO_DISABLE_LOCKING
+#warning \
+    "!!! locking disabled !!! This may cause hella crashes, please mind the gap."
+#undef ACQUIRE_LOCK
+#define ACQUIRE_LOCK(lock, mtx)
+#endif
+
 namespace llarp
 {
   namespace util
@@ -36,8 +52,9 @@ namespace llarp
       }
     };
 
-    using Mutex          = absl::Mutex;
-    using Lock           = absl::MutexLock;
+    using Mutex = absl::Mutex;
+    using Lock  = absl::MutexLock;
+
     using ReleasableLock = absl::ReleasableMutexLock;
     using Condition      = absl::CondVar;
 
diff --git a/llarp/utp/linklayer.cpp b/llarp/utp/linklayer.cpp
index b0b589fc5..41a968255 100644
--- a/llarp/utp/linklayer.cpp
+++ b/llarp/utp/linklayer.cpp
@@ -206,7 +206,7 @@ namespace llarp
       utp_issue_deferred_acks(_utp_ctx);
       std::set< RouterID > sessions;
       {
-        Lock l(&m_AuthedLinksMutex);
+        ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
         auto itr = m_AuthedLinks.begin();
         while(itr != m_AuthedLinks.end())
         {
@@ -216,7 +216,7 @@ namespace llarp
       }
       ILinkLayer::Pump();
       {
-        Lock l(&m_AuthedLinksMutex);
+        ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
         for(const auto& pk : sessions)
         {
           if(m_AuthedLinks.count(pk) == 0)

From e9f01923d9935d008982fafdbb449bff1fa08386 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 3 Sep 2019 16:22:52 -0400
Subject: [PATCH 02/27] dont nack replayed messages, tweak transmission
 parameters.

this yields more efficient througput. probably.
---
 llarp/iwp/session.cpp | 17 ++++++++++-------
 llarp/iwp/session.hpp |  8 ++++----
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index e14ccc2fb..bd062366d 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -548,13 +548,16 @@ namespace llarp
       auto itr      = m_RXMsgs.find(rxid);
       if(itr == m_RXMsgs.end())
       {
-        LogDebug("no rxid=", rxid, " for ", m_RemoteAddr);
-        std::vector< byte_t > nack = {
-            LLARP_PROTO_VERSION, Command::eNACK, 0, 0, 0, 0, 0, 0, 0, 0};
-        htobe64buf(nack.data() + 2, rxid);
-        AddRandomPadding(nack);
-        const llarp_buffer_t nackbuf(nack);
-        EncryptAndSend(nackbuf);
+        if(m_ReplayFilter.find(rxid) == m_ReplayFilter.end())
+        {
+          LogDebug("no rxid=", rxid, " for ", m_RemoteAddr);
+          std::vector< byte_t > nack = {
+              LLARP_PROTO_VERSION, Command::eNACK, 0, 0, 0, 0, 0, 0, 0, 0};
+          htobe64buf(nack.data() + 2, rxid);
+          AddRandomPadding(nack);
+          const llarp_buffer_t nackbuf(nack);
+          EncryptAndSend(nackbuf);
+        }
         return;
       }
 
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index fec572579..3a48eded4 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -17,13 +17,13 @@ namespace llarp
                      public std::enable_shared_from_this< Session >
     {
       /// Time how long we try delivery for
-      static constexpr llarp_time_t DeliveryTimeout = 5000;
-      /// How long to keep a replay window for
-      static constexpr llarp_time_t ReplayWindow = (DeliveryTimeout * 3) / 2;
+      static constexpr llarp_time_t DeliveryTimeout = 2000;
       /// Time how long we wait to recieve a message
       static constexpr llarp_time_t RecievalTimeout = (DeliveryTimeout * 8) / 5;
+      /// How long to keep a replay window for
+      static constexpr llarp_time_t ReplayWindow = (RecievalTimeout * 3) / 2;
       /// How often to acks RX messages
-      static constexpr llarp_time_t ACKResendInterval = 500;
+      static constexpr llarp_time_t ACKResendInterval = 250;
       /// How often to retransmit TX fragments
       static constexpr llarp_time_t TXFlushInterval =
           (ACKResendInterval * 3) / 2;

From 23d76e3600a601210b6dd2f2dcdffe445ee99516 Mon Sep 17 00:00:00 2001
From: Michael <michael@loki.network>
Date: Tue, 3 Sep 2019 23:25:37 +0100
Subject: [PATCH 03/27] Abort when a null mutex is locked from a different
 thread (in debug mode only)

---
 llarp/util/thread/threading.hpp | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/llarp/util/thread/threading.hpp b/llarp/util/thread/threading.hpp
index 05113a2b3..fdc7dee67 100644
--- a/llarp/util/thread/threading.hpp
+++ b/llarp/util/thread/threading.hpp
@@ -3,8 +3,12 @@
 
 #include <absl/synchronization/barrier.h>
 #include <absl/synchronization/mutex.h>
+#include <absl/types/optional.h>
 #include <absl/time/time.h>
 
+#include <iostream>
+#include <thread>
+
 #if defined(WIN32) && !defined(__GNUC__)
 #include <process.h>
 using pid_t = int;
@@ -20,6 +24,29 @@ namespace llarp
     /// a mutex that does nothing
     struct LOCKABLE NullMutex
     {
+#ifdef LOKINET_DEBUG
+      mutable absl::optional< std::thread::id > m_id;
+      void
+      lock() const
+      {
+        if(!m_id)
+        {
+          m_id.emplace(std::this_thread::get_id());
+        }
+        else if(m_id.value() != std::this_thread::get_id())
+        {
+          std::cerr << "NullMutex " << this << " was locked by "
+                    << std::this_thread::get_id()
+                    << " and was previously locked by " << m_id.value() << "\n";
+          std::abort();
+        }
+      }
+#else
+      void
+      lock() const
+      {
+      }
+#endif
     };
 
     /// a lock that does nothing
@@ -28,6 +55,7 @@ namespace llarp
       NullLock(ABSL_ATTRIBUTE_UNUSED const NullMutex* mtx)
           EXCLUSIVE_LOCK_FUNCTION(mtx)
       {
+        mtx->lock();
       }
 
       ~NullLock() UNLOCK_FUNCTION()

From 5cdd92e2a376a2f2a51c1a413432c99901fc1b74 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Wed, 4 Sep 2019 08:24:17 -0400
Subject: [PATCH 04/27] remove more locking and make it safe

---
 llarp/messages/relay_commit.cpp | 36 +++++++++++++++++++++------------
 llarp/messages/relay_status.hpp |  3 ++-
 llarp/path/path_context.cpp     | 24 +++++++++++-----------
 llarp/path/path_context.hpp     | 13 ++++++++----
 4 files changed, 46 insertions(+), 30 deletions(-)

diff --git a/llarp/messages/relay_commit.cpp b/llarp/messages/relay_commit.cpp
index 51a08cfb9..8abebf9b8 100644
--- a/llarp/messages/relay_commit.cpp
+++ b/llarp/messages/relay_commit.cpp
@@ -238,6 +238,15 @@ namespace llarp
     static void
     SendLRCM(std::shared_ptr< LRCMFrameDecrypt > self)
     {
+      if(self->context->HasTransitHop(self->hop->info))
+      {
+        llarp::LogError("duplicate transit hop", self->hop->info);
+        OnForwardLRCMResult(self->context->Router(), self->hop->info.rxID,
+                            self->hop->info.downstream, self->hop->pathKey,
+                            SendStatus::Congestion);
+        self->hop = nullptr;
+        return;
+      }
       if(!self->context->Router()->ConnectionToRouterAllowed(
              self->hop->info.upstream))
       {
@@ -287,15 +296,21 @@ namespace llarp
     static void
     SendPathConfirm(std::shared_ptr< LRCMFrameDecrypt > self)
     {
-      // persist session to downstream until path expiration
-      self->context->Router()->PersistSessionUntil(
-          self->hop->info.downstream, self->hop->ExpireTime() + 10000);
-      // put hop
-      self->context->PutTransitHop(self->hop);
-
       // send path confirmation
       // TODO: other status flags?
       uint64_t status = LR_StatusRecord::SUCCESS;
+      if(self->context->HasTransitHop(self->hop->info))
+      {
+        status = LR_StatusRecord::FAIL_DUPLICATE_HOP;
+      }
+      else
+      {
+        // persist session to downstream until path expiration
+        self->context->Router()->PersistSessionUntil(
+            self->hop->info.downstream, self->hop->ExpireTime() + 10000);
+        // put hop
+        self->context->PutTransitHop(self->hop);
+      }
 
       if(!LR_StatusMessage::CreateAndSend(
              self->context->Router(), self->hop->info.rxID,
@@ -332,12 +347,7 @@ namespace llarp
       info.txID     = self->record.txid;
       info.rxID     = self->record.rxid;
       info.upstream = self->record.nextHop;
-      if(self->context->HasTransitHop(info))
-      {
-        llarp::LogError("duplicate transit hop ", info);
-        self->decrypter = nullptr;
-        return;
-      }
+
       // generate path key as we are in a worker thread
       auto crypto = CryptoManager::instance();
       if(!crypto->dh_server(self->hop->pathKey, self->record.commkey,
@@ -387,7 +397,7 @@ namespace llarp
       {
         // we are the farthest hop
         llarp::LogDebug("We are the farthest hop for ", info);
-        // send a LRAM down the path
+        // send a LRSM down the path
         self->context->logic()->queue_func([=]() {
           SendPathConfirm(self);
           self->decrypter = nullptr;
diff --git a/llarp/messages/relay_status.hpp b/llarp/messages/relay_status.hpp
index 2e744f8dd..77ad47592 100644
--- a/llarp/messages/relay_status.hpp
+++ b/llarp/messages/relay_status.hpp
@@ -23,7 +23,7 @@ namespace llarp
 
   struct LR_StatusRecord
   {
-    static constexpr uint64_t SUCCESS               = 1;
+    static constexpr uint64_t SUCCESS               = 1 << 0;
     static constexpr uint64_t FAIL_TIMEOUT          = 1 << 1;
     static constexpr uint64_t FAIL_CONGESTION       = 1 << 2;
     static constexpr uint64_t FAIL_DEST_UNKNOWN     = 1 << 3;
@@ -31,6 +31,7 @@ namespace llarp
     static constexpr uint64_t FAIL_MALFORMED_RECORD = 1 << 5;
     static constexpr uint64_t FAIL_DEST_INVALID     = 1 << 6;
     static constexpr uint64_t FAIL_CANNOT_CONNECT   = 1 << 7;
+    static constexpr uint64_t FAIL_DUPLICATE_HOP    = 1 << 8;
 
     uint64_t status  = 0;
     uint64_t version = 0;
diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp
index 84a19f575..0c771cc0b 100644
--- a/llarp/path/path_context.cpp
+++ b/llarp/path/path_context.cpp
@@ -88,7 +88,7 @@ namespace llarp
     HopHandler_ptr
     MapGet(Map_t& map, const Key_t& k, CheckValue_t check, GetFunc_t get)
     {
-      util::Lock lock(&map.first);
+      Map_t::Lock_t lock(&map.first);
       auto range = map.second.equal_range(k);
       for(auto i = range.first; i != range.second; ++i)
       {
@@ -102,7 +102,7 @@ namespace llarp
     bool
     MapHas(Map_t& map, const Key_t& k, CheckValue_t check)
     {
-      util::Lock lock(&map.first);
+      Map_t::Lock_t lock(&map.first);
       auto range = map.second.equal_range(k);
       for(auto i = range.first; i != range.second; ++i)
       {
@@ -116,7 +116,7 @@ namespace llarp
     void
     MapPut(Map_t& map, const Key_t& k, const Value_t& v)
     {
-      util::Lock lock(&map.first);
+      Map_t::Lock_t lock(&map.first);
       map.second.emplace(k, v);
     }
 
@@ -124,7 +124,7 @@ namespace llarp
     void
     MapIter(Map_t& map, Visit_t v)
     {
-      util::Lock lock(map.first);
+      Map_t::Lock_t lock(map.first);
       for(const auto& item : map.second)
         v(item);
     }
@@ -133,7 +133,7 @@ namespace llarp
     void
     MapDel(Map_t& map, const Key_t& k, Check_t check)
     {
-      util::Lock lock(map.first);
+      Map_t::Lock_t lock(map.first);
       auto range = map.second.equal_range(k);
       for(auto i = range.first; i != range.second;)
       {
@@ -190,7 +190,7 @@ namespace llarp
     PathContext::TransitHopPreviousIsRouter(const PathID_t& path,
                                             const RouterID& otherRouter)
     {
-      util::Lock lock(&m_TransitPaths.first);
+      decltype(m_TransitPaths)::Lock_t lock(&m_TransitPaths.first);
       auto itr = m_TransitPaths.second.find(path);
       if(itr == m_TransitPaths.second.end())
         return false;
@@ -214,7 +214,7 @@ namespace llarp
     PathContext::GetLocalPathSet(const PathID_t& id)
     {
       auto& map = m_OurPaths;
-      util::Lock lock(&map.first);
+      decltype(m_OurPaths)::Lock_t lock(&map.first);
       auto itr = map.second.find(id);
       if(itr != map.second.end())
       {
@@ -241,7 +241,7 @@ namespace llarp
       RouterID us(OurRouterID());
       auto& map = m_TransitPaths;
       {
-        util::Lock lock(&map.first);
+        decltype(m_TransitPaths)::Lock_t lock(&map.first);
         auto range = map.second.equal_range(id);
         for(auto i = range.first; i != range.second; ++i)
         {
@@ -263,7 +263,7 @@ namespace llarp
     PathContext::ExpirePaths(llarp_time_t now)
     {
       {
-        util::Lock lock(&m_TransitPaths.first);
+        decltype(m_TransitPaths)::Lock_t lock(&m_TransitPaths.first);
         auto& map = m_TransitPaths.second;
         auto itr  = map.begin();
         while(itr != map.end())
@@ -277,7 +277,7 @@ namespace llarp
         }
       }
       {
-        util::Lock lock(&m_OurPaths.first);
+        decltype(m_OurPaths)::Lock_t lock(&m_OurPaths.first);
         auto& map = m_OurPaths.second;
         for(auto& item : map)
         {
@@ -299,7 +299,7 @@ namespace llarp
       const RouterID us(OurRouterID());
       auto& map = m_TransitPaths;
       {
-        util::Lock lock(&map.first);
+        decltype(m_TransitPaths)::Lock_t lock(&map.first);
         auto range = map.second.equal_range(id);
         for(auto i = range.first; i != range.second; ++i)
         {
@@ -313,7 +313,7 @@ namespace llarp
     void
     PathContext::RemovePathSet(PathSet_ptr set)
     {
-      util::Lock lock(&m_OurPaths.first);
+      decltype(m_OurPaths)::Lock_t lock(&m_OurPaths.first);
       auto& map = m_OurPaths.second;
       auto itr  = map.begin();
       while(itr != map.end())
diff --git a/llarp/path/path_context.hpp b/llarp/path/path_context.hpp
index fce0f1454..2c452e659 100644
--- a/llarp/path/path_context.hpp
+++ b/llarp/path/path_context.hpp
@@ -102,14 +102,17 @@ namespace llarp
 
       struct SyncTransitMap_t
       {
-        util::Mutex first;  // protects second
+        using Mutex_t = util::NullMutex;
+        using Lock_t  = util::NullLock;
+
+        Mutex_t first;  // protects second
         TransitHopsMap_t second GUARDED_BY(first);
 
         void
         ForEach(std::function< void(const TransitHop_ptr&) > visit)
             LOCKS_EXCLUDED(first)
         {
-          util::Lock lock(&first);
+          Lock_t lock(&first);
           for(const auto& item : second)
             visit(item.second);
         }
@@ -120,13 +123,15 @@ namespace llarp
 
       struct SyncOwnedPathsMap_t
       {
-        util::Mutex first;  // protects second
+        using Mutex_t = util::Mutex;
+        using Lock_t  = util::Lock;
+        Mutex_t first;  // protects second
         OwnedPathsMap_t second GUARDED_BY(first);
 
         void
         ForEach(std::function< void(const PathSet_ptr&) > visit)
         {
-          util::Lock lock(&first);
+          Lock_t lock(&first);
           for(const auto& item : second)
             visit(item.second);
         }

From 12314e8d0021f4cbbc2de452a41c59d297dd3b30 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Wed, 4 Sep 2019 08:41:07 -0400
Subject: [PATCH 05/27] ensure no crash on quit

---
 daemon/main.cpp             |  2 ++
 llarp/context.cpp           |  3 +-
 llarp/path/path_context.cpp | 62 ++++++++++++++++++++-----------------
 3 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/daemon/main.cpp b/daemon/main.cpp
index cbac238dd..c1236bb90 100644
--- a/daemon/main.cpp
+++ b/daemon/main.cpp
@@ -27,7 +27,9 @@ void
 handle_signal(int sig)
 {
   if(ctx)
+  {
     llarp_main_signal(ctx, sig);
+  }
 }
 
 #ifdef _WIN32
diff --git a/llarp/context.cpp b/llarp/context.cpp
index 509902a4d..9abbddf06 100644
--- a/llarp/context.cpp
+++ b/llarp/context.cpp
@@ -433,7 +433,8 @@ extern "C"
   void
   llarp_main_signal(struct llarp_main *ptr, int sig)
   {
-    ptr->ctx->HandleSignal(sig);
+    ptr->ctx->logic->queue_func(
+        std::bind(llarp::Context::HandleSignal, ptr->ctx.get(), sig));
   }
 
   int
diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp
index 0c771cc0b..ff556a384 100644
--- a/llarp/path/path_context.cpp
+++ b/llarp/path/path_context.cpp
@@ -83,12 +83,12 @@ namespace llarp
 
       return true;
     }
-    template < typename Map_t, typename Key_t, typename CheckValue_t,
-               typename GetFunc_t >
+    template < typename Lock_t, typename Map_t, typename Key_t,
+               typename CheckValue_t, typename GetFunc_t >
     HopHandler_ptr
     MapGet(Map_t& map, const Key_t& k, CheckValue_t check, GetFunc_t get)
     {
-      Map_t::Lock_t lock(&map.first);
+      Lock_t lock(&map.first);
       auto range = map.second.equal_range(k);
       for(auto i = range.first; i != range.second; ++i)
       {
@@ -98,11 +98,12 @@ namespace llarp
       return nullptr;
     }
 
-    template < typename Map_t, typename Key_t, typename CheckValue_t >
+    template < typename Lock_t, typename Map_t, typename Key_t,
+               typename CheckValue_t >
     bool
     MapHas(Map_t& map, const Key_t& k, CheckValue_t check)
     {
-      Map_t::Lock_t lock(&map.first);
+      Lock_t lock(&map.first);
       auto range = map.second.equal_range(k);
       for(auto i = range.first; i != range.second; ++i)
       {
@@ -112,28 +113,30 @@ namespace llarp
       return false;
     }
 
-    template < typename Map_t, typename Key_t, typename Value_t >
+    template < typename Lock_t, typename Map_t, typename Key_t,
+               typename Value_t >
     void
     MapPut(Map_t& map, const Key_t& k, const Value_t& v)
     {
-      Map_t::Lock_t lock(&map.first);
+      Lock_t lock(&map.first);
       map.second.emplace(k, v);
     }
 
-    template < typename Map_t, typename Visit_t >
+    template < typename Lock_t, typename Map_t, typename Visit_t >
     void
     MapIter(Map_t& map, Visit_t v)
     {
-      Map_t::Lock_t lock(map.first);
+      Lock_t lock(map.first);
       for(const auto& item : map.second)
         v(item);
     }
 
-    template < typename Map_t, typename Key_t, typename Check_t >
+    template < typename Lock_t, typename Map_t, typename Key_t,
+               typename Check_t >
     void
     MapDel(Map_t& map, const Key_t& k, Check_t check)
     {
-      Map_t::Lock_t lock(map.first);
+      Lock_t lock(map.first);
       auto range = map.second.equal_range(k);
       for(auto i = range.first; i != range.second;)
       {
@@ -148,23 +151,24 @@ namespace llarp
     PathContext::AddOwnPath(PathSet_ptr set, Path_ptr path)
     {
       set->AddPath(path);
-      MapPut(m_OurPaths, path->TXID(), set);
-      MapPut(m_OurPaths, path->RXID(), set);
+      MapPut< SyncOwnedPathsMap_t::Lock_t >(m_OurPaths, path->TXID(), set);
+      MapPut< SyncOwnedPathsMap_t::Lock_t >(m_OurPaths, path->RXID(), set);
     }
 
     bool
     PathContext::HasTransitHop(const TransitHopInfo& info)
     {
-      return MapHas(m_TransitPaths, info.txID,
-                    [info](const std::shared_ptr< TransitHop >& hop) -> bool {
-                      return info == hop->info;
-                    });
+      return MapHas< SyncTransitMap_t::Lock_t >(
+          m_TransitPaths, info.txID,
+          [info](const std::shared_ptr< TransitHop >& hop) -> bool {
+            return info == hop->info;
+          });
     }
 
     HopHandler_ptr
     PathContext::GetByUpstream(const RouterID& remote, const PathID_t& id)
     {
-      auto own = MapGet(
+      auto own = MapGet< SyncOwnedPathsMap_t::Lock_t >(
           m_OurPaths, id,
           [](const PathSet_ptr) -> bool {
             // TODO: is this right?
@@ -176,7 +180,7 @@ namespace llarp
       if(own)
         return own;
 
-      return MapGet(
+      return MapGet< SyncTransitMap_t::Lock_t >(
           m_TransitPaths, id,
           [remote](const std::shared_ptr< TransitHop >& hop) -> bool {
             return hop->info.upstream == remote;
@@ -190,7 +194,7 @@ namespace llarp
     PathContext::TransitHopPreviousIsRouter(const PathID_t& path,
                                             const RouterID& otherRouter)
     {
-      decltype(m_TransitPaths)::Lock_t lock(&m_TransitPaths.first);
+      SyncTransitMap_t::Lock_t lock(&m_TransitPaths.first);
       auto itr = m_TransitPaths.second.find(path);
       if(itr == m_TransitPaths.second.end())
         return false;
@@ -200,7 +204,7 @@ namespace llarp
     HopHandler_ptr
     PathContext::GetByDownstream(const RouterID& remote, const PathID_t& id)
     {
-      return MapGet(
+      return MapGet< SyncTransitMap_t::Lock_t >(
           m_TransitPaths, id,
           [remote](const std::shared_ptr< TransitHop >& hop) -> bool {
             return hop->info.downstream == remote;
@@ -214,7 +218,7 @@ namespace llarp
     PathContext::GetLocalPathSet(const PathID_t& id)
     {
       auto& map = m_OurPaths;
-      decltype(m_OurPaths)::Lock_t lock(&map.first);
+      SyncOwnedPathsMap_t::Lock_t lock(&map.first);
       auto itr = map.second.find(id);
       if(itr != map.second.end())
       {
@@ -241,7 +245,7 @@ namespace llarp
       RouterID us(OurRouterID());
       auto& map = m_TransitPaths;
       {
-        decltype(m_TransitPaths)::Lock_t lock(&map.first);
+        SyncTransitMap_t::Lock_t lock(&map.first);
         auto range = map.second.equal_range(id);
         for(auto i = range.first; i != range.second; ++i)
         {
@@ -255,15 +259,15 @@ namespace llarp
     void
     PathContext::PutTransitHop(std::shared_ptr< TransitHop > hop)
     {
-      MapPut(m_TransitPaths, hop->info.txID, hop);
-      MapPut(m_TransitPaths, hop->info.rxID, hop);
+      MapPut< SyncTransitMap_t::Lock_t >(m_TransitPaths, hop->info.txID, hop);
+      MapPut< SyncTransitMap_t::Lock_t >(m_TransitPaths, hop->info.rxID, hop);
     }
 
     void
     PathContext::ExpirePaths(llarp_time_t now)
     {
       {
-        decltype(m_TransitPaths)::Lock_t lock(&m_TransitPaths.first);
+        SyncTransitMap_t::Lock_t lock(&m_TransitPaths.first);
         auto& map = m_TransitPaths.second;
         auto itr  = map.begin();
         while(itr != map.end())
@@ -277,7 +281,7 @@ namespace llarp
         }
       }
       {
-        decltype(m_OurPaths)::Lock_t lock(&m_OurPaths.first);
+        SyncOwnedPathsMap_t::Lock_t lock(&m_OurPaths.first);
         auto& map = m_OurPaths.second;
         for(auto& item : map)
         {
@@ -299,7 +303,7 @@ namespace llarp
       const RouterID us(OurRouterID());
       auto& map = m_TransitPaths;
       {
-        decltype(m_TransitPaths)::Lock_t lock(&map.first);
+        SyncTransitMap_t::Lock_t lock(&map.first);
         auto range = map.second.equal_range(id);
         for(auto i = range.first; i != range.second; ++i)
         {
@@ -313,7 +317,7 @@ namespace llarp
     void
     PathContext::RemovePathSet(PathSet_ptr set)
     {
-      decltype(m_OurPaths)::Lock_t lock(&m_OurPaths.first);
+      SyncOwnedPathsMap_t::Lock_t lock(&m_OurPaths.first);
       auto& map = m_OurPaths.second;
       auto itr  = map.begin();
       while(itr != map.end())

From d11321366cd914ab6311d519ea4e7c6441329b16 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Wed, 4 Sep 2019 08:43:50 -0400
Subject: [PATCH 06/27] make it compile

---
 llarp/context.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llarp/context.cpp b/llarp/context.cpp
index 9abbddf06..9b8d71ded 100644
--- a/llarp/context.cpp
+++ b/llarp/context.cpp
@@ -434,7 +434,7 @@ extern "C"
   llarp_main_signal(struct llarp_main *ptr, int sig)
   {
     ptr->ctx->logic->queue_func(
-        std::bind(llarp::Context::HandleSignal, ptr->ctx.get(), sig));
+        std::bind(&llarp::Context::HandleSignal, ptr->ctx.get(), sig));
   }
 
   int

From bde5195315b2b3ad50c9d4152a020d473b645175 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Wed, 4 Sep 2019 09:46:53 -0400
Subject: [PATCH 07/27] change default url for bootstrap

---
 win32-setup/lokinet-win32.iss | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/win32-setup/lokinet-win32.iss b/win32-setup/lokinet-win32.iss
index 586ccd071..224e73dfe 100644
--- a/win32-setup/lokinet-win32.iss
+++ b/win32-setup/lokinet-win32.iss
@@ -217,7 +217,7 @@ Filename: "{app}\{#MyAppExeName}"; Flags: nowait postinstall skipifsilent; Descr
 Filename: "{tmp}\7z.exe"; Parameters: "x tuntapv9.7z"; WorkingDir: "{app}"; Flags: runascurrentuser waituntilterminated skipifdoesntexist; Description: "extract TUN/TAP-v9 driver"; StatusMsg: "Extracting driver..."; OnlyBelowVersion: 0, 6.0
 Filename: "{tmp}\7z.exe"; Parameters: "x tuntapv9_n6.7z"; WorkingDir: "{app}"; Flags: runascurrentuser waituntilterminated skipifdoesntexist; Description: "extract TUN/TAP-v9 driver"; StatusMsg: "Extracting driver..."; MinVersion: 0, 6.0
 Filename: "{tmp}\7z.exe"; Parameters: "x inet6.7z"; WorkingDir: "{app}"; Flags: skipifdoesntexist runascurrentuser waituntilterminated skipifdoesntexist; Description: "extract inet6 driver"; StatusMsg: "Extracting IPv6 driver..."; MinVersion: 0, 5.0; OnlyBelowVersion: 0, 5.1
-Filename: "{tmp}\lokinet-bootstrap.exe"; Parameters:"https://i2p.rocks/bootstrap.signed {userappdata}\.lokinet\bootstrap.signed"; WorkingDir: "{app}"; Flags: runascurrentuser waituntilterminated; Description: "bootstrap dht"; StatusMsg: "Downloading initial RC..."
+Filename: "{tmp}\lokinet-bootstrap.exe"; Parameters:"https://seed.lokinet.org/bootstrap.signed {userappdata}\.lokinet\bootstrap.signed"; WorkingDir: "{app}"; Flags: runascurrentuser waituntilterminated; Description: "bootstrap dht"; StatusMsg: "Downloading initial RC..."
 ; then ask to install drivers
 Filename: "{app}\tap-windows-9.9.2\install.bat"; WorkingDir: "{app}\tap-windows-9.9.2\"; Flags: runascurrentuser waituntilterminated skipifdoesntexist; Description: "Install TUN/TAP-v9 driver"; StatusMsg: "Installing driver..."; OnlyBelowVersion: 0, 6.0; Check: not IsTapInstalled
 Filename: "{app}\tap-windows-9.21.2\install.bat"; WorkingDir: "{app}\tap-windows-9.21.2\"; Flags: runascurrentuser waituntilterminated skipifdoesntexist; Description: "Install TUN/TAP-v9 driver"; StatusMsg: "Installing driver..."; MinVersion: 0, 6.0; Check: not IsTapInstalled

From 3b1a0b28353e570f13249bb156374d3ef4e7a8a0 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 09:21:35 -0400
Subject: [PATCH 08/27] add multi-ack and change protocol handshake to pin
 identity key

---
 llarp/constants/link_layer.hpp |   2 +-
 llarp/iwp/message_buffer.cpp   |  20 +++--
 llarp/iwp/message_buffer.hpp   |   7 +-
 llarp/iwp/session.cpp          | 139 ++++++++++++++++++++++++++++-----
 llarp/iwp/session.hpp          |  11 +++
 5 files changed, 147 insertions(+), 32 deletions(-)

diff --git a/llarp/constants/link_layer.hpp b/llarp/constants/link_layer.hpp
index b09ee5e4c..74c32950c 100644
--- a/llarp/constants/link_layer.hpp
+++ b/llarp/constants/link_layer.hpp
@@ -6,5 +6,5 @@
 
 constexpr size_t MAX_LINK_MSG_SIZE                = 8192;
 constexpr llarp_time_t DefaultLinkSessionLifetime = 60 * 1000;
-constexpr size_t MaxSendQueueSize                 = 128;
+constexpr size_t MaxSendQueueSize                 = 1024;
 #endif
diff --git a/llarp/iwp/message_buffer.cpp b/llarp/iwp/message_buffer.cpp
index a9578908c..ab610edb2 100644
--- a/llarp/iwp/message_buffer.cpp
+++ b/llarp/iwp/message_buffer.cpp
@@ -143,22 +143,20 @@ namespace llarp
     std::vector< byte_t >
     InboundMessage::ACKS() const
     {
-      std::vector< byte_t > acks{LLARP_PROTO_VERSION,
-                                 Command::eACKS,
-                                 0,
-                                 0,
-                                 0,
-                                 0,
-                                 0,
-                                 0,
-                                 0,
-                                 0,
-                                 uint8_t{(uint8_t)m_Acks.to_ulong()}};
+      std::vector< byte_t > acks{
+          LLARP_PROTO_VERSION, Command::eACKS, 0, 0, 0, 0, 0, 0, 0, 0,
+          AcksBitmask()};
 
       htobe64buf(acks.data() + 2, m_MsgID);
       return acks;
     }
 
+    byte_t
+    InboundMessage::AcksBitmask() const
+    {
+      return byte_t{(byte_t)m_Acks.to_ulong()};
+    }
+
     bool
     InboundMessage::IsCompleted() const
     {
diff --git a/llarp/iwp/message_buffer.hpp b/llarp/iwp/message_buffer.hpp
index 7671506c2..f2e6cf948 100644
--- a/llarp/iwp/message_buffer.hpp
+++ b/llarp/iwp/message_buffer.hpp
@@ -24,7 +24,9 @@ namespace llarp
       /// negative ack
       eNACK = 4,
       /// close session
-      eCLOS = 5
+      eCLOS = 5,
+      /// multiack
+      eMACK = 6,
     };
 
     static constexpr size_t FragmentSize = 1024;
@@ -97,6 +99,9 @@ namespace llarp
       bool
       Verify() const;
 
+      byte_t
+      AcksBitmask() const;
+
       bool
       ShouldSendACKS(llarp_time_t now) const;
 
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index 8efab90e1..e3a4125bc 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -30,6 +30,8 @@ namespace llarp
     {
       token.Zero();
       GotLIM = util::memFn(&Session::GotOutboundLIM, this);
+      CryptoManager::instance()->shorthash(m_SessionKey,
+                                           llarp_buffer_t(rc.pubkey));
     }
 
     Session::Session(LinkLayer* p, Addr from)
@@ -40,7 +42,9 @@ namespace llarp
         , m_RemoteAddr{from}
     {
       token.Randomize();
-      GotLIM = util::memFn(&Session::GotInboundLIM, this);
+      GotLIM          = util::memFn(&Session::GotInboundLIM, this);
+      const PubKey pk = m_Parent->GetOurRC().pubkey;
+      CryptoManager::instance()->shorthash(m_SessionKey, llarp_buffer_t(pk));
     }
 
     void
@@ -54,9 +58,10 @@ namespace llarp
     bool
     Session::GotInboundLIM(const LinkIntroMessage* msg)
     {
-      if(msg->rc.enckey != m_RemoteOnionKey)
+      if(msg->rc.pubkey != m_ExpectedIdent)
       {
-        LogError("key missmatch");
+        LogError("ident key missmatch from ", m_RemoteAddr, " ", msg->rc.pubkey,
+                 " != ", m_ExpectedIdent);
         return false;
       }
       m_State    = State::Ready;
@@ -159,6 +164,8 @@ namespace llarp
     Session::SendMessageBuffer(const llarp_buffer_t& buf,
                                ILinkSession::CompletionHandler completed)
     {
+      if(m_TXMsgs.size() >= MaxSendQueueSize)
+        return false;
       const auto now   = m_Parent->Now();
       const auto msgid = m_TXID++;
       auto& msg =
@@ -173,6 +180,31 @@ namespace llarp
       return true;
     }
 
+    void
+    Session::SendMACK()
+    {
+      // send multi acks
+      while(!m_SendMACKS.empty())
+      {
+        byte_t numAcks = std::min(m_SendMACKS.size(), MaxACKSInMACK);
+        std::vector< byte_t > mack;
+        mack.resize(2 + 1 + (numAcks * sizeof(uint64_t)));
+        mack[0]     = LLARP_PROTO_VERSION;
+        mack[1]     = Command::eMACK;
+        mack[2]     = numAcks;
+        byte_t* ptr = mack.data() + 3;
+        while(numAcks > 0)
+        {
+          htobe64buf(ptr, m_SendMACKS.back());
+          m_SendMACKS.pop_back();
+          numAcks--;
+          ptr += sizeof(uint64_t);
+        }
+        const llarp_buffer_t buf(mack);
+        EncryptAndSend(buf);
+      }
+    }
+
     void
     Session::Pump()
     {
@@ -181,6 +213,7 @@ namespace llarp
       {
         if(ShouldPing())
           SendKeepAlive();
+        SendMACK();
         for(auto& item : m_RXMsgs)
         {
           if(item.second.ShouldSendACKS(now))
@@ -267,6 +300,7 @@ namespace llarp
         {
           if(itr->second.IsTimedOut(now))
           {
+            m_ReplayFilter.emplace(itr->first, now);
             itr = m_RXMsgs.erase(itr);
           }
           else
@@ -288,13 +322,12 @@ namespace llarp
       }
     }
 
-    using Introduction = AlignedBuffer< 64 >;
+    using Introduction = AlignedBuffer< PubKey::SIZE + PubKey::SIZE
+                                        + TunnelNonce::SIZE + Signature::SIZE >;
 
     void
     Session::GenerateAndSendIntro()
     {
-      Introduction intro;
-
       TunnelNonce N;
       N.Randomize();
       if(not CryptoManager::instance()->transport_dh_client(
@@ -305,16 +338,25 @@ namespace llarp
                  m_RemoteAddr);
         return;
       }
-      const auto pk = m_Parent->RouterEncryptionSecret().toPublic();
-      std::copy_n(pk.begin(), pk.size(), intro.begin());
-      std::copy(N.begin(), N.end(), intro.begin() + PubKey::SIZE);
-      LogDebug("pk=", pk.ToHex(), " N=", N.ToHex(),
-               " remote-pk=", m_ChosenAI.pubkey.ToHex());
       std::vector< byte_t > req;
-      std::copy_n(intro.begin(), intro.size(), std::back_inserter(req));
+      req.resize(Introduction::SIZE - Signature::SIZE);
+      const auto pk   = m_Parent->GetOurRC().pubkey;
+      const auto e_pk = m_Parent->RouterEncryptionSecret().toPublic();
+      auto itr        = req.begin();
+      std::copy_n(pk.begin(), pk.size(), itr);
+      itr += pk.size();
+      std::copy_n(e_pk.begin(), e_pk.size(), itr);
+      itr += e_pk.size();
+      std::copy(N.begin(), N.end(), itr);
+      Signature Z;
+      llarp_buffer_t signbuf(req);
+      m_Parent->Sign(Z, signbuf);
+      req.resize(Introduction::SIZE);
+      std::copy_n(Z.begin(), Z.size(),
+                  req.begin() + (Introduction::SIZE - Signature::SIZE));
       AddRandomPadding(req);
       const llarp_buffer_t buf(req);
-      Send_LL(buf);
+      EncryptAndSend(buf);
       m_State = State::Introduction;
       LogDebug("sent intro to ", m_RemoteAddr);
     }
@@ -353,9 +395,23 @@ namespace llarp
         LogWarn("intro too small from ", m_RemoteAddr);
         return;
       }
+      byte_t* ptr = buf.base;
       TunnelNonce N;
-      std::copy_n(buf.base, PubKey::SIZE, m_RemoteOnionKey.begin());
-      std::copy_n(buf.base + PubKey::SIZE, TunnelNonce::SIZE, N.begin());
+      std::copy_n(ptr, PubKey::SIZE, m_ExpectedIdent.begin());
+      ptr += PubKey::SIZE;
+      std::copy_n(ptr, PubKey::SIZE, m_RemoteOnionKey.begin());
+      ptr += PubKey::SIZE;
+      std::copy_n(ptr, TunnelNonce::SIZE, N.begin());
+      ptr += TunnelNonce::SIZE;
+      Signature Z;
+      std::copy_n(ptr, Z.size(), Z.begin());
+      const llarp_buffer_t verifybuf(buf.base,
+                                     Introduction::SIZE - Signature::SIZE);
+      if(!CryptoManager::instance()->verify(m_ExpectedIdent, verifybuf, Z))
+      {
+        LogError("intro verify failed from ", m_RemoteAddr);
+        return;
+      }
       const PubKey pk = m_Parent->TransportSecretKey().toPublic();
       LogDebug("got intro: remote-pk=", m_RemoteOnionKey.ToHex(),
                " N=", N.ToHex(), " local-pk=", pk.ToHex(), " sz=", buf.sz);
@@ -367,7 +423,8 @@ namespace llarp
         return;
       }
       std::vector< byte_t > reply;
-      std::copy_n(token.begin(), token.size(), std::back_inserter(reply));
+      reply.resize(token.size());
+      std::copy_n(token.begin(), token.size(), reply.begin());
       AddRandomPadding(reply);
       const llarp_buffer_t pkt(reply);
       m_LastRX = m_Parent->Now();
@@ -478,10 +535,46 @@ namespace llarp
         case Command::eCLOS:
           HandleCLOS(std::move(result));
           return;
+        case Command::eMACK:
+          HandleMACK(std::move(result));
+          return;
       }
       LogError("invalid command ", int(result[1]));
     }
 
+    void
+    Session::HandleMACK(std::vector< byte_t > data)
+    {
+      if(data.size() < 3)
+      {
+        LogError("impossibly short mack from ", m_RemoteAddr);
+        return;
+      }
+      byte_t numAcks = data[2];
+      if(data.size() < ((numAcks * sizeof(uint64_t)) + 3))
+      {
+        LogError("short mack from ", m_RemoteAddr);
+        return;
+      }
+      byte_t* ptr = data.data() + 3;
+      while(numAcks > 0)
+      {
+        uint64_t acked = bufbe64toh(ptr);
+        auto itr       = m_TXMsgs.find(acked);
+        if(itr != m_TXMsgs.end())
+        {
+          itr->second.Completed();
+          m_TXMsgs.erase(itr);
+        }
+        else
+        {
+          LogDebug("ignored mack for txid=", acked, " from ", m_RemoteAddr);
+        }
+        ptr += sizeof(uint64_t);
+        numAcks--;
+      }
+    }
+
     void
     Session::HandleNACK(std::vector< byte_t > data)
     {
@@ -549,6 +642,11 @@ namespace llarp
       auto itr      = m_RXMsgs.find(rxid);
       if(itr == m_RXMsgs.end())
       {
+        if(m_ReplayFilter.find(rxid) != m_ReplayFilter.end())
+        {
+          // don't nack if we already got it
+          return;
+        }
         LogDebug("no rxid=", rxid, " for ", m_RemoteAddr);
         std::vector< byte_t > nack = {
             LLARP_PROTO_VERSION, Command::eNACK, 0, 0, 0, 0, 0, 0, 0, 0};
@@ -566,14 +664,13 @@ namespace llarp
 
       if(itr->second.IsCompleted())
       {
-        itr->second.SendACKS(util::memFn(&Session::EncryptAndSend, this),
-                             m_Parent->Now());
         if(itr->second.Verify())
         {
           auto msg = std::move(itr->second);
           const llarp_buffer_t buf(msg.m_Data.data(), msg.m_Size);
           m_Parent->HandleMessage(this, buf);
           m_ReplayFilter.emplace(itr->first, m_Parent->Now());
+          m_SendMACKS.emplace_back(itr->first);
         }
         else
         {
@@ -648,6 +745,7 @@ namespace llarp
     void
     Session::Recv_LL(const llarp_buffer_t& buf)
     {
+      std::vector< byte_t > data;
       switch(m_State)
       {
         case State::Initial:
@@ -655,7 +753,10 @@ namespace llarp
           {
             // initial data
             // enter introduction phase
-            HandleGotIntro(buf);
+            if(DecryptMessage(buf, data))
+              HandleGotIntro(llarp_buffer_t(data));
+            else
+              LogError("bad intro from ", m_RemoteAddr);
           }
           else
           {
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index fec572579..f729601a9 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -32,6 +32,8 @@ namespace llarp
       /// How long we wait for a session to die with no tx from them
       static constexpr llarp_time_t SessionAliveTimeout =
           (PingInterval * 13) / 3;
+      /// maximum number of messages we can ack in a multiack
+      static constexpr size_t MaxACKSInMACK = 1024 / 8;
 
       /// outbound session
       Session(LinkLayer* parent, RouterContact rc, AddressInfo ai);
@@ -149,6 +151,7 @@ namespace llarp
       /// session token
       AlignedBuffer< 24 > token;
 
+      PubKey m_ExpectedIdent;
       PubKey m_RemoteOnionKey;
 
       llarp_time_t m_LastTX = 0;
@@ -161,6 +164,8 @@ namespace llarp
 
       /// maps rxid to time recieved
       std::unordered_map< uint64_t, llarp_time_t > m_ReplayFilter;
+      /// list of rx messages to send in next set of multiacks
+      std::vector< uint64_t > m_SendMACKS;
 
       void
       HandleGotIntro(const llarp_buffer_t& buf);
@@ -180,6 +185,9 @@ namespace llarp
       bool
       DecryptMessage(const llarp_buffer_t& buf, std::vector< byte_t >& result);
 
+      void
+      SendMACK();
+
       void
       GenerateAndSendIntro();
 
@@ -212,6 +220,9 @@ namespace llarp
 
       void
       HandleCLOS(std::vector< byte_t > msg);
+
+      void
+      HandleMACK(std::vector< byte_t > msg);
     };
   }  // namespace iwp
 }  // namespace llarp

From a4160006abce1c0e5b0323dda519f897d74fec89 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 09:34:59 -0400
Subject: [PATCH 09/27] make it compile

---
 llarp/iwp/session.cpp | 6 ++++--
 llarp/iwp/session.hpp | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index e3a4125bc..43afca2a5 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -186,12 +186,14 @@ namespace llarp
       // send multi acks
       while(!m_SendMACKS.empty())
       {
-        byte_t numAcks = std::min(m_SendMACKS.size(), MaxACKSInMACK);
+        const auto sz  = m_SendMACKS.size();
+        const auto max = Session::MaxACKSInMACK;
+        auto numAcks   = std::min(sz, max);
         std::vector< byte_t > mack;
         mack.resize(2 + 1 + (numAcks * sizeof(uint64_t)));
         mack[0]     = LLARP_PROTO_VERSION;
         mack[1]     = Command::eMACK;
-        mack[2]     = numAcks;
+        mack[2]     = byte_t{numAcks};
         byte_t* ptr = mack.data() + 3;
         while(numAcks > 0)
         {
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index f729601a9..71ea3f22f 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -33,7 +33,7 @@ namespace llarp
       static constexpr llarp_time_t SessionAliveTimeout =
           (PingInterval * 13) / 3;
       /// maximum number of messages we can ack in a multiack
-      static constexpr size_t MaxACKSInMACK = 1024 / 8;
+      static constexpr std::size_t MaxACKSInMACK = 1024 / sizeof(uint64_t);
 
       /// outbound session
       Session(LinkLayer* parent, RouterContact rc, AddressInfo ai);

From 0d3c07999a2df27555aff5d3943a6aa8a268e381 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 09:36:46 -0400
Subject: [PATCH 10/27] break it again just to be sure

---
 llarp/iwp/message_buffer.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llarp/iwp/message_buffer.hpp b/llarp/iwp/message_buffer.hpp
index f2e6cf948..c2c1dcb6f 100644
--- a/llarp/iwp/message_buffer.hpp
+++ b/llarp/iwp/message_buffer.hpp
@@ -23,10 +23,10 @@ namespace llarp
       eACKS = 3,
       /// negative ack
       eNACK = 4,
-      /// close session
-      eCLOS = 5,
       /// multiack
-      eMACK = 6,
+      eMACK = 5,
+      /// close session
+      eCLOS = 6,
     };
 
     static constexpr size_t FragmentSize = 1024;

From 88cde21b9b5a23b845c9ceedff2e7cbdefa129a1 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 10:57:01 -0400
Subject: [PATCH 11/27] multithreaded iwp cryptography

---
 llarp/iwp/linklayer.cpp |  22 +++++-
 llarp/iwp/linklayer.hpp |   8 ++
 llarp/iwp/session.cpp   | 169 +++++++++++++++++++++++++---------------
 llarp/iwp/session.hpp   |  14 ++++
 4 files changed, 150 insertions(+), 63 deletions(-)

diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index 57d0c499a..b2e308733 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -14,10 +14,14 @@ namespace llarp
         : ILinkLayer(routerEncSecret, getrc, h, sign, est, reneg, timeout,
                      closed)
         , permitInbound{allowInbound}
+        , m_CryptoWorker(4, 1024 * 8, "iwp-worker")
     {
     }
 
-    LinkLayer::~LinkLayer() = default;
+    LinkLayer::~LinkLayer()
+    {
+      m_CryptoWorker.stop();
+    }
 
     void
     LinkLayer::Pump()
@@ -66,10 +70,24 @@ namespace llarp
       return 2;
     }
 
+    void
+    LinkLayer::QueueWork(std::function< void(void) > func)
+    {
+      m_CryptoWorker.addJob(func);
+    }
+
     bool
     LinkLayer::Start(std::shared_ptr< Logic > l)
     {
-      return ILinkLayer::Start(l);
+      if(!ILinkLayer::Start(l))
+        return false;
+      return m_CryptoWorker.start();
+    }
+
+    void
+    LinkLayer::Stop()
+    {
+      ILinkLayer::Stop();
     }
 
     void
diff --git a/llarp/iwp/linklayer.hpp b/llarp/iwp/linklayer.hpp
index 13ecf3ecf..4ae305ddf 100644
--- a/llarp/iwp/linklayer.hpp
+++ b/llarp/iwp/linklayer.hpp
@@ -6,6 +6,7 @@
 #include <crypto/encrypted.hpp>
 #include <crypto/types.hpp>
 #include <link/server.hpp>
+#include <util/thread/thread_pool.hpp>
 
 namespace llarp
 {
@@ -37,6 +38,9 @@ namespace llarp
       const char *
       Name() const override;
 
+      void
+      Stop() override;
+
       uint16_t
       Rank() const override;
 
@@ -49,9 +53,13 @@ namespace llarp
       void
       UnmapAddr(const Addr &addr);
 
+      void
+      QueueWork(std::function< void(void) > work);
+
      private:
       std::unordered_map< Addr, RouterID, Addr::Hash > m_AuthedAddrs;
       const bool permitInbound;
+      thread::ThreadPool m_CryptoWorker;
     };
 
     using LinkLayer_ptr = std::shared_ptr< LinkLayer >;
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index 43afca2a5..736b9c4df 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -123,26 +123,36 @@ namespace llarp
     void
     Session::EncryptAndSend(const llarp_buffer_t& data)
     {
-      std::vector< byte_t > pkt;
-      pkt.resize(data.sz + PacketOverhead);
-      CryptoManager::instance()->randbytes(pkt.data(), pkt.size());
-      llarp_buffer_t pktbuf(pkt);
-      pktbuf.base += PacketOverhead;
-      pktbuf.cur = pktbuf.base;
-      pktbuf.sz -= PacketOverhead;
-      byte_t* nonce_ptr = pkt.data() + HMACSIZE;
-
-      CryptoManager::instance()->xchacha20_alt(pktbuf, data, m_SessionKey,
-                                               nonce_ptr);
-
-      pktbuf.base = nonce_ptr;
-      pktbuf.sz   = data.sz + 32;
-      CryptoManager::instance()->hmac(pkt.data(), pktbuf, m_SessionKey);
+      m_EncryptNext.emplace_back(data.sz + PacketOverhead);
+      auto& pkt = m_EncryptNext.back();
+      std::copy_n(data.base, data.sz, pkt.begin() + PacketOverhead);
+      if(!IsEstablished())
+        EncryptWorker(std::move(m_EncryptNext));
+    }
 
-      pktbuf.base = pkt.data();
-      pktbuf.cur  = pkt.data();
-      pktbuf.sz   = pkt.size();
-      Send_LL(pktbuf);
+    void
+    Session::EncryptWorker(CryptoQueue_t msgs)
+    {
+      LogDebug("encrypt worker ", msgs.size(), " messages");
+      for(auto& pkt : msgs)
+      {
+        llarp_buffer_t pktbuf(pkt);
+        byte_t* nonce_ptr = pkt.data() + HMACSIZE;
+        CryptoManager::instance()->randbytes(nonce_ptr,
+                                             PacketOverhead - HMACSECSIZE);
+        pktbuf.base += PacketOverhead;
+        pktbuf.cur = pktbuf.base;
+        pktbuf.sz -= PacketOverhead;
+        CryptoManager::instance()->xchacha20_alt(pktbuf, pktbuf, m_SessionKey,
+                                                 nonce_ptr);
+        pktbuf.base = nonce_ptr;
+        pktbuf.sz   = pkt.size() - HMACSIZE;
+        CryptoManager::instance()->hmac(pkt.data(), pktbuf, m_SessionKey);
+        pktbuf.base = pkt.data();
+        pktbuf.cur  = pkt.data();
+        pktbuf.sz   = pkt.size();
+        Send_LL(pktbuf);
+      }
     }
 
     void
@@ -233,6 +243,16 @@ namespace llarp
           }
         }
       }
+
+      if(!m_EncryptNext.empty())
+        m_Parent->QueueWork(std::bind(&Session::EncryptWorker,
+                                      shared_from_this(),
+                                      std::move(m_EncryptNext)));
+
+      if(!m_DecryptNext.empty())
+        m_Parent->QueueWork(std::bind(&Session::DecryptWorker,
+                                      shared_from_this(),
+                                      std::move(m_DecryptNext)));
     }
 
     bool
@@ -332,14 +352,6 @@ namespace llarp
     {
       TunnelNonce N;
       N.Randomize();
-      if(not CryptoManager::instance()->transport_dh_client(
-             m_SessionKey, m_ChosenAI.pubkey,
-             m_Parent->RouterEncryptionSecret(), N))
-      {
-        LogError("failed to transport_dh_client on outbound session to ",
-                 m_RemoteAddr);
-        return;
-      }
       std::vector< byte_t > req;
       req.resize(Introduction::SIZE - Signature::SIZE);
       const auto pk   = m_Parent->GetOurRC().pubkey;
@@ -360,6 +372,14 @@ namespace llarp
       const llarp_buffer_t buf(req);
       EncryptAndSend(buf);
       m_State = State::Introduction;
+      if(not CryptoManager::instance()->transport_dh_client(
+             m_SessionKey, m_ChosenAI.pubkey,
+             m_Parent->RouterEncryptionSecret(), N))
+      {
+        LogError("failed to transport_dh_client on outbound session to ",
+                 m_RemoteAddr);
+        return;
+      }
       LogDebug("sent intro to ", m_RemoteAddr);
     }
 
@@ -504,44 +524,71 @@ namespace llarp
     void
     Session::HandleSessionData(const llarp_buffer_t& buf)
     {
-      std::vector< byte_t > result;
-      if(not DecryptMessage(buf, result))
-      {
-        LogError("failed to decrypt session data from ", m_RemoteAddr);
-        return;
-      }
-      if(result[0] != LLARP_PROTO_VERSION)
+      m_DecryptNext.emplace_back(buf.sz);
+      auto& pkt = m_DecryptNext.back();
+      std::copy_n(buf.base, buf.sz, pkt.begin());
+    }
+
+    void
+    Session::DecryptWorker(CryptoQueue_t msgs)
+    {
+      CryptoQueue_t recvMsgs;
+      for(const auto& pkt : msgs)
       {
-        LogError("protocol version missmatch ", int(result[0]),
-                 " != ", LLARP_PROTO_VERSION);
-        return;
+        std::vector< byte_t > result;
+        const llarp_buffer_t buf(pkt);
+        if(not DecryptMessage(buf, result))
+        {
+          LogError("failed to decrypt session data from ", m_RemoteAddr);
+          continue;
+        }
+        if(result[0] != LLARP_PROTO_VERSION)
+        {
+          LogError("protocol version missmatch ", int(result[0]),
+                   " != ", LLARP_PROTO_VERSION);
+          continue;
+        }
+        recvMsgs.emplace_back(std::move(result));
       }
-      LogDebug("command ", int(result[1]), " from ", m_RemoteAddr);
-      switch(result[1])
+      LogDebug("decrypted ", recvMsgs.size(), " packets from ", m_RemoteAddr);
+      m_Parent->logic()->queue_func(std::bind(
+          &Session::HandlePlaintext, shared_from_this(), std::move(recvMsgs)));
+    }
+
+    void
+    Session::HandlePlaintext(CryptoQueue_t msgs)
+    {
+      for(auto& result : msgs)
       {
-        case Command::eXMIT:
-          HandleXMIT(std::move(result));
-          return;
-        case Command::eDATA:
-          HandleDATA(std::move(result));
-          return;
-        case Command::eACKS:
-          HandleACKS(std::move(result));
-          return;
-        case Command::ePING:
-          HandlePING(std::move(result));
-          return;
-        case Command::eNACK:
-          HandleNACK(std::move(result));
-          return;
-        case Command::eCLOS:
-          HandleCLOS(std::move(result));
-          return;
-        case Command::eMACK:
-          HandleMACK(std::move(result));
-          return;
+        LogDebug("command ", int(result[1]), " from ", m_RemoteAddr);
+        switch(result[1])
+        {
+          case Command::eXMIT:
+            HandleXMIT(std::move(result));
+            break;
+          case Command::eDATA:
+            HandleDATA(std::move(result));
+            break;
+          case Command::eACKS:
+            HandleACKS(std::move(result));
+            break;
+          case Command::ePING:
+            HandlePING(std::move(result));
+            break;
+          case Command::eNACK:
+            HandleNACK(std::move(result));
+            break;
+          case Command::eCLOS:
+            HandleCLOS(std::move(result));
+            break;
+          case Command::eMACK:
+            HandleMACK(std::move(result));
+            break;
+          default:
+            LogError("invalid command ", int(result[1]), " from ",
+                     m_RemoteAddr);
+        }
       }
-      LogError("invalid command ", int(result[1]));
     }
 
     void
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index 71ea3f22f..aaa68b21b 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -167,6 +167,20 @@ namespace llarp
       /// list of rx messages to send in next set of multiacks
       std::vector< uint64_t > m_SendMACKS;
 
+      using CryptoQueue_t = std::vector< std::vector< byte_t > >;
+
+      CryptoQueue_t m_EncryptNext;
+      CryptoQueue_t m_DecryptNext;
+
+      void
+      EncryptWorker(CryptoQueue_t msgs);
+
+      void
+      DecryptWorker(CryptoQueue_t msgs);
+
+      void
+      HandlePlaintext(CryptoQueue_t msgs);
+
       void
       HandleGotIntro(const llarp_buffer_t& buf);
 

From 4bf6882c8ae5a16b17937f0598e19af29f16c1fb Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 13:39:09 -0400
Subject: [PATCH 12/27] more async cryptography

---
 llarp/ev/ev.hpp               |   3 +
 llarp/ev/ev_libuv.cpp         |  45 +++++++++++++++
 llarp/ev/ev_libuv.hpp         |   3 +
 llarp/iwp/linklayer.cpp       |  22 +------
 llarp/iwp/linklayer.hpp       |   7 ---
 llarp/link/i_link_manager.hpp |   3 +-
 llarp/link/link_manager.cpp   |   7 ++-
 llarp/link/link_manager.hpp   |   3 +-
 llarp/link/server.cpp         |   6 +-
 llarp/link/server.hpp         |   8 ++-
 llarp/path/ihophandler.hpp    |  46 +++++++++++++--
 llarp/path/path.cpp           | 106 +++++++++++++++++++++++++---------
 llarp/path/path.hpp           |  28 +++++----
 llarp/path/path_context.cpp   |   7 +++
 llarp/path/path_context.hpp   |   3 +
 llarp/path/pathset.cpp        |   7 +++
 llarp/path/pathset.hpp        |   3 +
 llarp/path/transit_hop.cpp    | 106 ++++++++++++++++++++++++++--------
 llarp/path/transit_hop.hpp    |  28 ++++++---
 llarp/router/router.cpp       |   9 ++-
 test/link/test_llarp_link.cpp |  15 ++++-
 21 files changed, 351 insertions(+), 114 deletions(-)

diff --git a/llarp/ev/ev.hpp b/llarp/ev/ev.hpp
index 9c38d2ac1..dbbd16440 100644
--- a/llarp/ev/ev.hpp
+++ b/llarp/ev/ev.hpp
@@ -744,6 +744,9 @@ struct llarp_ev_loop
   virtual int
   tick(int ms) = 0;
 
+  virtual bool
+  add_ticker(std::function< void(void) > ticker) = 0;
+
   virtual void
   stop() = 0;
 
diff --git a/llarp/ev/ev_libuv.cpp b/llarp/ev/ev_libuv.cpp
index 732770dda..bddfc6791 100644
--- a/llarp/ev/ev_libuv.cpp
+++ b/llarp/ev/ev_libuv.cpp
@@ -312,6 +312,39 @@ namespace libuv
     }
   };
 
+  struct ticker_glue : public glue
+  {
+    std::function< void(void) > func;
+
+    ticker_glue(uv_loop_t* loop, std::function< void(void) > tick) : func(tick)
+    {
+      m_Ticker.data = this;
+      uv_check_init(loop, &m_Ticker);
+    }
+
+    static void
+    OnTick(uv_check_t* t)
+    {
+      static_cast< ticker_glue* >(t->data)->func();
+    }
+
+    bool
+    Start()
+    {
+      return uv_check_start(&m_Ticker, &OnTick) != -1;
+    }
+
+    void
+    Close() override
+    {
+      uv_check_stop(&m_Ticker);
+      m_Ticker.data = nullptr;
+      delete this;
+    }
+
+    uv_check_t m_Ticker;
+  };
+
   struct udp_glue : public glue
   {
     uv_udp_t m_Handle;
@@ -737,6 +770,18 @@ namespace libuv
     return false;
   }
 
+  bool
+  Loop::add_ticker(std::function< void(void) > func)
+  {
+    auto* ticker = new ticker_glue(m_Impl.get(), func);
+    if(ticker->Start())
+    {
+      return true;
+    }
+    delete ticker;
+    return false;
+  }
+
   bool
   Loop::udp_close(llarp_udp_io* udp)
   {
diff --git a/llarp/ev/ev_libuv.hpp b/llarp/ev/ev_libuv.hpp
index 3ad9a6389..ab19fec73 100644
--- a/llarp/ev/ev_libuv.hpp
+++ b/llarp/ev/ev_libuv.hpp
@@ -78,6 +78,9 @@ namespace libuv
       return nullptr;
     }
 
+    bool
+    add_ticker(std::function< void(void) > ticker) override;
+
     /// register event listener
     bool
     add_ev(llarp::ev_io*, bool) override
diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index b2e308733..7704dcc3e 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -14,14 +14,10 @@ namespace llarp
         : ILinkLayer(routerEncSecret, getrc, h, sign, est, reneg, timeout,
                      closed)
         , permitInbound{allowInbound}
-        , m_CryptoWorker(4, 1024 * 8, "iwp-worker")
     {
     }
 
-    LinkLayer::~LinkLayer()
-    {
-      m_CryptoWorker.stop();
-    }
+    LinkLayer::~LinkLayer() = default;
 
     void
     LinkLayer::Pump()
@@ -73,21 +69,7 @@ namespace llarp
     void
     LinkLayer::QueueWork(std::function< void(void) > func)
     {
-      m_CryptoWorker.addJob(func);
-    }
-
-    bool
-    LinkLayer::Start(std::shared_ptr< Logic > l)
-    {
-      if(!ILinkLayer::Start(l))
-        return false;
-      return m_CryptoWorker.start();
-    }
-
-    void
-    LinkLayer::Stop()
-    {
-      ILinkLayer::Stop();
+      m_Worker->addJob(func);
     }
 
     void
diff --git a/llarp/iwp/linklayer.hpp b/llarp/iwp/linklayer.hpp
index 4ae305ddf..f481ddf61 100644
--- a/llarp/iwp/linklayer.hpp
+++ b/llarp/iwp/linklayer.hpp
@@ -22,9 +22,6 @@ namespace llarp
 
       ~LinkLayer() override;
 
-      bool
-      Start(std::shared_ptr< Logic > l) override;
-
       std::shared_ptr< ILinkSession >
       NewOutboundSession(const RouterContact &rc,
                          const AddressInfo &ai) override;
@@ -38,9 +35,6 @@ namespace llarp
       const char *
       Name() const override;
 
-      void
-      Stop() override;
-
       uint16_t
       Rank() const override;
 
@@ -59,7 +53,6 @@ namespace llarp
      private:
       std::unordered_map< Addr, RouterID, Addr::Hash > m_AuthedAddrs;
       const bool permitInbound;
-      thread::ThreadPool m_CryptoWorker;
     };
 
     using LinkLayer_ptr = std::shared_ptr< LinkLayer >;
diff --git a/llarp/link/i_link_manager.hpp b/llarp/link/i_link_manager.hpp
index 55d32dd4a..d42c9e98d 100644
--- a/llarp/link/i_link_manager.hpp
+++ b/llarp/link/i_link_manager.hpp
@@ -42,7 +42,8 @@ namespace llarp
     AddLink(LinkLayer_ptr link, bool inbound = false) = 0;
 
     virtual bool
-    StartLinks(Logic_ptr logic) = 0;
+    StartLinks(Logic_ptr logic,
+               std::shared_ptr< thread::ThreadPool > worker) = 0;
 
     virtual void
     Stop() = 0;
diff --git a/llarp/link/link_manager.cpp b/llarp/link/link_manager.cpp
index 69b58dc0a..09aee0d4d 100644
--- a/llarp/link/link_manager.cpp
+++ b/llarp/link/link_manager.cpp
@@ -89,12 +89,13 @@ namespace llarp
   }
 
   bool
-  LinkManager::StartLinks(Logic_ptr logic)
+  LinkManager::StartLinks(Logic_ptr logic,
+                          std::shared_ptr< thread::ThreadPool > worker)
   {
     LogInfo("starting ", outboundLinks.size(), " outbound links");
     for(const auto &link : outboundLinks)
     {
-      if(!link->Start(logic))
+      if(!link->Start(logic, worker))
       {
         LogWarn("outbound link '", link->Name(), "' failed to start");
         return false;
@@ -107,7 +108,7 @@ namespace llarp
       LogInfo("starting ", inboundLinks.size(), " inbound links");
       for(const auto &link : inboundLinks)
       {
-        if(!link->Start(logic))
+        if(!link->Start(logic, worker))
         {
           LogWarn("Link ", link->Name(), " failed to start");
           return false;
diff --git a/llarp/link/link_manager.hpp b/llarp/link/link_manager.hpp
index 877a65816..103ef5869 100644
--- a/llarp/link/link_manager.hpp
+++ b/llarp/link/link_manager.hpp
@@ -40,7 +40,8 @@ namespace llarp
     AddLink(LinkLayer_ptr link, bool inbound = false) override;
 
     bool
-    StartLinks(Logic_ptr logic) override;
+    StartLinks(Logic_ptr logic,
+               std::shared_ptr< thread::ThreadPool > worker) override;
 
     void
     Stop() override;
diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp
index 8f0cc2d2d..944b2ebc2 100644
--- a/llarp/link/server.cpp
+++ b/llarp/link/server.cpp
@@ -264,9 +264,11 @@ namespace llarp
   }
 
   bool
-  ILinkLayer::Start(std::shared_ptr< Logic > l)
+  ILinkLayer::Start(std::shared_ptr< Logic > l,
+                    std::shared_ptr< thread::ThreadPool > worker)
   {
-    m_Logic = l;
+    m_Worker = worker;
+    m_Logic  = l;
     ScheduleTick(100);
     return true;
   }
diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp
index 499040db2..4d98af8ed 100644
--- a/llarp/link/server.hpp
+++ b/llarp/link/server.hpp
@@ -122,8 +122,9 @@ namespace llarp
     bool
     TryEstablishTo(RouterContact rc);
 
-    virtual bool
-    Start(std::shared_ptr< llarp::Logic > l);
+    bool
+    Start(std::shared_ptr< llarp::Logic > l,
+          std::shared_ptr< thread::ThreadPool > worker);
 
     virtual void
     Stop();
@@ -243,7 +244,8 @@ namespace llarp
     bool
     PutSession(const std::shared_ptr< ILinkSession >& s);
 
-    std::shared_ptr< llarp::Logic > m_Logic = nullptr;
+    std::shared_ptr< llarp::Logic > m_Logic               = nullptr;
+    std::shared_ptr< llarp::thread::ThreadPool > m_Worker = nullptr;
     llarp_ev_loop_ptr m_Loop;
     Addr m_ourAddr;
     llarp_udp_io m_udp;
diff --git a/llarp/path/ihophandler.hpp b/llarp/path/ihophandler.hpp
index ea672d74b..c0495e57c 100644
--- a/llarp/path/ihophandler.hpp
+++ b/llarp/path/ihophandler.hpp
@@ -4,6 +4,8 @@
 #include <crypto/types.hpp>
 #include <util/types.hpp>
 #include <crypto/encrypted_frame.hpp>
+#include <messages/relay.hpp>
+#include <vector>
 
 #include <memory>
 
@@ -22,6 +24,9 @@ namespace llarp
   {
     struct IHopHandler
     {
+      using TrafficEvent_t = std::pair< std::vector< byte_t >, TunnelNonce >;
+      using TrafficQueue_t = std::vector< TrafficEvent_t >;
+
       virtual ~IHopHandler() = default;
 
       virtual bool
@@ -35,14 +40,30 @@ namespace llarp
       SendRoutingMessage(const routing::IMessage& msg, AbstractRouter* r) = 0;
 
       // handle data in upstream direction
-      virtual bool
+      bool
       HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                     AbstractRouter* r) = 0;
+                     AbstractRouter*)
+      {
+        m_UpstreamQueue.emplace_back();
+        auto& pkt = m_UpstreamQueue.back();
+        pkt.first.resize(X.sz);
+        std::copy_n(X.base, X.sz, pkt.first.begin());
+        pkt.second = Y;
+        return true;
+      }
 
       // handle data in downstream direction
-      virtual bool
+      bool
       HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                       AbstractRouter* r) = 0;
+                       AbstractRouter*)
+      {
+        m_DownstreamQueue.emplace_back();
+        auto& pkt = m_DownstreamQueue.back();
+        pkt.first.resize(X.sz);
+        std::copy_n(X.base, X.sz, pkt.first.begin());
+        pkt.second = Y;
+        return true;
+      }
 
       /// return timestamp last remote activity happened at
       virtual llarp_time_t
@@ -57,9 +78,26 @@ namespace llarp
       {
         return m_SequenceNum++;
       }
+      virtual void
+      FlushQueues(AbstractRouter* r) = 0;
 
      protected:
       uint64_t m_SequenceNum = 0;
+      TrafficQueue_t m_UpstreamQueue;
+      TrafficQueue_t m_DownstreamQueue;
+
+      virtual void
+      UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) = 0;
+
+      virtual void
+      DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) = 0;
+
+      virtual void
+      HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
+                        AbstractRouter* r) = 0;
+      virtual void
+      HandleAllDownstream(std::vector< RelayDownstreamMessage > msgs,
+                          AbstractRouter* r) = 0;
     };
 
     using HopHandler_ptr = std::shared_ptr< IHopHandler >;
diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp
index 381ec6f02..d24ca336b 100644
--- a/llarp/path/path.cpp
+++ b/llarp/path/path.cpp
@@ -363,24 +363,55 @@ namespace llarp
       }
     }
 
-    bool
-    Path::HandleUpstream(const llarp_buffer_t& buf, const TunnelNonce& Y,
-                         AbstractRouter* r)
+    void
+    Path::HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
+                            AbstractRouter* r)
     {
-      TunnelNonce n = Y;
-      for(const auto& hop : hops)
+      for(const auto& msg : msgs)
       {
-        CryptoManager::instance()->xchacha20(buf, hop.shared, n);
-        n ^= hop.nonceXOR;
+        if(!r->SendToOrQueue(Upstream(), &msg))
+        {
+          LogDebug("failed to send upstream to ", Upstream());
+        }
       }
-      RelayUpstreamMessage msg;
-      msg.X      = buf;
-      msg.Y      = Y;
-      msg.pathid = TXID();
-      if(r->SendToOrQueue(Upstream(), &msg))
-        return true;
-      LogError("send to ", Upstream(), " failed");
-      return false;
+    }
+
+    void
+    Path::UpstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
+    {
+      std::vector< RelayUpstreamMessage > sendmsgs(msgs.size());
+      size_t idx = 0;
+      for(const auto& ev : msgs)
+      {
+        const llarp_buffer_t buf(ev.first);
+        TunnelNonce n = ev.second;
+        for(const auto& hop : hops)
+        {
+          CryptoManager::instance()->xchacha20(buf, hop.shared, n);
+          n ^= hop.nonceXOR;
+        }
+        auto& msg  = sendmsgs[idx];
+        msg.X      = buf;
+        msg.Y      = ev.second;
+        msg.pathid = TXID();
+        ++idx;
+      }
+      r->logic()->queue_func(std::bind(&Path::HandleAllUpstream,
+                                       shared_from_this(), std::move(sendmsgs),
+                                       r));
+    }
+
+    void
+    Path::FlushQueues(AbstractRouter* r)
+    {
+      if(!m_UpstreamQueue.empty())
+        r->threadpool()->addJob(std::bind(&Path::UpstreamWork,
+                                          shared_from_this(),
+                                          std::move(m_UpstreamQueue), r));
+      if(!m_DownstreamQueue.empty())
+        r->threadpool()->addJob(std::bind(&Path::DownstreamWork,
+                                          shared_from_this(),
+                                          std::move(m_DownstreamQueue), r));
     }
 
     bool
@@ -406,20 +437,43 @@ namespace llarp
       return ss.str();
     }
 
-    bool
-    Path::HandleDownstream(const llarp_buffer_t& buf, const TunnelNonce& Y,
-                           AbstractRouter* r)
+    void
+    Path::DownstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
     {
-      TunnelNonce n = Y;
-      for(const auto& hop : hops)
+      std::vector< RelayDownstreamMessage > sendMsgs(msgs.size());
+      size_t idx = 0;
+      for(auto& ev : msgs)
       {
-        n ^= hop.nonceXOR;
-        CryptoManager::instance()->xchacha20(buf, hop.shared, n);
+        const llarp_buffer_t buf(ev.first);
+        sendMsgs[idx].Y = ev.second;
+        for(const auto& hop : hops)
+        {
+          sendMsgs[idx].Y ^= hop.nonceXOR;
+          CryptoManager::instance()->xchacha20(buf, hop.shared,
+                                               sendMsgs[idx].Y);
+        }
+        sendMsgs[idx].X = buf;
+        ++idx;
+      }
+      r->logic()->queue_func(std::bind(&Path::HandleAllDownstream,
+                                       shared_from_this(), std::move(sendMsgs),
+                                       r));
+    }
+
+    void
+    Path::HandleAllDownstream(std::vector< RelayDownstreamMessage > msgs,
+                              AbstractRouter* r)
+    {
+      for(const auto& msg : msgs)
+      {
+        const llarp_buffer_t buf(msg.X);
+        if(!HandleRoutingMessage(buf, r))
+        {
+          LogWarn("failed to handle downstream message");
+          continue;
+        }
+        m_LastRecvMessage = r->Now();
       }
-      if(!HandleRoutingMessage(buf, r))
-        return false;
-      m_LastRecvMessage = r->Now();
-      return true;
     }
 
     bool
diff --git a/llarp/path/path.hpp b/llarp/path/path.hpp
index 9d7f275a1..e1a9b45c7 100644
--- a/llarp/path/path.hpp
+++ b/llarp/path/path.hpp
@@ -286,16 +286,6 @@ namespace llarp
       bool
       HandleRoutingMessage(const llarp_buffer_t& buf, AbstractRouter* r);
 
-      // handle data in upstream direction
-      bool
-      HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                     AbstractRouter* r) override;
-
-      // handle data in downstream direction
-      bool
-      HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                       AbstractRouter* r) override;
-
       bool
       IsReady() const;
 
@@ -334,6 +324,24 @@ namespace llarp
       bool
       SendExitClose(const routing::CloseExitMessage& msg, AbstractRouter* r);
 
+      void
+      FlushQueues(AbstractRouter* r) override;
+
+     protected:
+      void
+      UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+
+      void
+      DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+
+      void
+      HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
+                        AbstractRouter* r) override;
+
+      void
+      HandleAllDownstream(std::vector< RelayDownstreamMessage > msgs,
+                          AbstractRouter* r) override;
+
      private:
       /// call obtained exit hooks
       bool
diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp
index 84a19f575..3a53b3d79 100644
--- a/llarp/path/path_context.cpp
+++ b/llarp/path/path_context.cpp
@@ -252,6 +252,13 @@ namespace llarp
       return nullptr;
     }
 
+    void
+    PathContext::Pump()
+    {
+      m_TransitPaths.ForEach([&](auto& ptr) { ptr->FlushQueues(m_Router); });
+      m_OurPaths.ForEach([&](auto& ptr) { ptr->FlushQueues(m_Router); });
+    }
+
     void
     PathContext::PutTransitHop(std::shared_ptr< TransitHop > hop)
     {
diff --git a/llarp/path/path_context.hpp b/llarp/path/path_context.hpp
index fce0f1454..0234fb57b 100644
--- a/llarp/path/path_context.hpp
+++ b/llarp/path/path_context.hpp
@@ -37,6 +37,9 @@ namespace llarp
       void
       ExpirePaths(llarp_time_t now);
 
+      void
+      Pump();
+
       void
       AllowTransit();
 
diff --git a/llarp/path/pathset.cpp b/llarp/path/pathset.cpp
index 5514d51e2..79b3af3b4 100644
--- a/llarp/path/pathset.cpp
+++ b/llarp/path/pathset.cpp
@@ -375,5 +375,12 @@ namespace llarp
       return nullptr;
     }
 
+    void
+    PathSet::FlushQueues(AbstractRouter* r)
+
+    {
+      ForEachPath([r](const Path_ptr& ptr) { ptr->FlushQueues(r); });
+    }
+
   }  // namespace path
 }  // namespace llarp
diff --git a/llarp/path/pathset.hpp b/llarp/path/pathset.hpp
index 802351d63..ed59cb532 100644
--- a/llarp/path/pathset.hpp
+++ b/llarp/path/pathset.hpp
@@ -275,6 +275,9 @@ namespace llarp
         }
       }
 
+      void
+      FlushQueues(AbstractRouter* r);
+
       size_t numPaths;
 
      protected:
diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp
index b9a46c24c..858b8f1f1 100644
--- a/llarp/path/transit_hop.cpp
+++ b/llarp/path/transit_hop.cpp
@@ -114,39 +114,96 @@ namespace llarp
       return HandleDownstream(buf, N, r);
     }
 
-    bool
-    TransitHop::HandleDownstream(const llarp_buffer_t& buf,
-                                 const TunnelNonce& Y, AbstractRouter* r)
+    void
+    TransitHop::DownstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
     {
-      RelayDownstreamMessage msg;
-      msg.pathid = info.rxID;
-      msg.Y      = Y ^ nonceXOR;
-      CryptoManager::instance()->xchacha20(buf, pathKey, Y);
-      msg.X = buf;
-      llarp::LogDebug("relay ", msg.X.size(), " bytes downstream from ",
-                      info.upstream, " to ", info.downstream);
-      return r->SendToOrQueue(info.downstream, &msg);
+      std::vector< RelayDownstreamMessage > sendmsgs(msgs.size());
+      size_t idx = 0;
+      for(auto& ev : msgs)
+      {
+        const llarp_buffer_t buf(ev.first);
+        auto& msg  = sendmsgs[idx];
+        msg.pathid = info.rxID;
+        msg.Y      = ev.second ^ nonceXOR;
+        CryptoManager::instance()->xchacha20(buf, pathKey, ev.second);
+        msg.X = buf;
+        llarp::LogDebug("relay ", msg.X.size(), " bytes downstream from ",
+                        info.upstream, " to ", info.downstream);
+        ++idx;
+      }
+      r->logic()->queue_func(std::bind(&TransitHop::HandleAllDownstream,
+                                       shared_from_this(), std::move(sendmsgs),
+                                       r));
     }
 
-    bool
-    TransitHop::HandleUpstream(const llarp_buffer_t& buf, const TunnelNonce& Y,
-                               AbstractRouter* r)
+    void
+    TransitHop::UpstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
+    {
+      std::vector< RelayUpstreamMessage > sendmsgs(msgs.size());
+      size_t idx = 0;
+      for(auto& ev : msgs)
+      {
+        const llarp_buffer_t buf(ev.first);
+        auto& msg = sendmsgs[idx];
+        CryptoManager::instance()->xchacha20(buf, pathKey, ev.second);
+        msg.pathid = info.txID;
+        msg.Y      = ev.second ^ nonceXOR;
+        msg.X      = buf;
+        ++idx;
+      }
+      r->logic()->queue_func(std::bind(&TransitHop::HandleAllUpstream,
+                                       shared_from_this(), std::move(sendmsgs),
+                                       r));
+    }
+
+    void
+    TransitHop::HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
+                                  AbstractRouter* r)
     {
-      CryptoManager::instance()->xchacha20(buf, pathKey, Y);
       if(IsEndpoint(r->pubkey()))
       {
-        m_LastActivity = r->Now();
-        return r->ParseRoutingMessageBuffer(buf, this, info.rxID);
+        for(const auto& msg : msgs)
+        {
+          const llarp_buffer_t buf(msg.X);
+          if(!r->ParseRoutingMessageBuffer(buf, this, info.rxID))
+            continue;
+          m_LastActivity = r->Now();
+        }
       }
+      else
+      {
+        for(const auto& msg : msgs)
+        {
+          llarp::LogDebug("relay ", msg.X.size(), " bytes upstream from ",
+                          info.downstream, " to ", info.upstream);
+          r->SendToOrQueue(info.upstream, &msg);
+        }
+      }
+    }
 
-      RelayUpstreamMessage msg;
-      msg.pathid = info.txID;
-      msg.Y      = Y ^ nonceXOR;
+    void
+    TransitHop::HandleAllDownstream(std::vector< RelayDownstreamMessage > msgs,
+                                    AbstractRouter* r)
+    {
+      for(const auto& msg : msgs)
+      {
+        llarp::LogDebug("relay ", msg.X.size(), " bytes downstream from ",
+                        info.downstream, " to ", info.upstream);
+        r->SendToOrQueue(info.downstream, &msg);
+      }
+    }
 
-      msg.X = buf;
-      llarp::LogDebug("relay ", msg.X.size(), " bytes upstream from ",
-                      info.downstream, " to ", info.upstream);
-      return r->SendToOrQueue(info.upstream, &msg);
+    void
+    TransitHop::FlushQueues(AbstractRouter* r)
+    {
+      if(!m_UpstreamQueue.empty())
+        r->threadpool()->addJob(std::bind(&TransitHop::UpstreamWork,
+                                          shared_from_this(),
+                                          std::move(m_UpstreamQueue), r));
+      if(!m_DownstreamQueue.empty())
+        r->threadpool()->addJob(std::bind(&TransitHop::DownstreamWork,
+                                          shared_from_this(),
+                                          std::move(m_DownstreamQueue), r));
     }
 
     bool
@@ -228,7 +285,6 @@ namespace llarp
         {
           if(SendRoutingMessage(reply, r))
           {
-            r->PumpLL();
             ep->Close();
             return true;
           }
diff --git a/llarp/path/transit_hop.hpp b/llarp/path/transit_hop.hpp
index eb958eb50..c29a5b6b4 100644
--- a/llarp/path/transit_hop.hpp
+++ b/llarp/path/transit_hop.hpp
@@ -79,7 +79,9 @@ namespace llarp
       return info.print(out, -1, -1);
     }
 
-    struct TransitHop : public IHopHandler, public routing::IMessageHandler
+    struct TransitHop : public IHopHandler,
+                        public routing::IMessageHandler,
+                        std::enable_shared_from_this< TransitHop >
     {
       TransitHop();
 
@@ -193,15 +195,23 @@ namespace llarp
       bool
       HandleDHTMessage(const dht::IMessage& msg, AbstractRouter* r) override;
 
-      // handle data in upstream direction
-      bool
-      HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                     AbstractRouter* r) override;
+      void
+      FlushQueues(AbstractRouter* r) override;
 
-      // handle data in downstream direction
-      bool
-      HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                       AbstractRouter* r) override;
+     protected:
+      void
+      UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+
+      void
+      DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+
+      void
+      HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
+                        AbstractRouter* r) override;
+
+      void
+      HandleAllDownstream(std::vector< RelayDownstreamMessage > msgs,
+                          AbstractRouter* r) override;
 
      private:
       void
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 9a72d542b..ef55a3942 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -22,6 +22,7 @@
 #include <util/metrics/metrics.hpp>
 #include <util/str.hpp>
 #include <utp/utp.hpp>
+#include <ev/ev.hpp>
 
 #include <fstream>
 #include <cstdlib>
@@ -164,6 +165,9 @@ namespace llarp
   void
   Router::PumpLL()
   {
+    _logic->tick(time_now_ms());
+    paths.Pump();
+    _logic->tick(time_now_ms());
     _linkManager.PumpLinks();
   }
 
@@ -913,7 +917,7 @@ namespace llarp
       return false;
     }
     _outboundSessionMaker.SetOurRouter(pubkey());
-    if(!_linkManager.StartLinks(_logic))
+    if(!_linkManager.StartLinks(_logic, cryptoworker))
     {
       LogWarn("One or more links failed to start.");
       return false;
@@ -987,7 +991,7 @@ namespace llarp
     }
 
     LogInfo("have ", nodedb->num_loaded(), " routers");
-
+    _netloop->add_ticker(std::bind(&Router::PumpLL, this));
     ScheduleTicker(1000);
     _running.store(true);
     _startedAt = Now();
@@ -1039,6 +1043,7 @@ namespace llarp
     _exitContext.Stop();
     if(rpcServer)
       rpcServer->Stop();
+    paths.Pump();
     _linkManager.PumpLinks();
     _logic->call_later({200, this, &RouterAfterStopIssued});
   }
diff --git a/test/link/test_llarp_link.cpp b/test/link/test_llarp_link.cpp
index 958e85da8..c8ac1955f 100644
--- a/test/link/test_llarp_link.cpp
+++ b/test/link/test_llarp_link.cpp
@@ -30,6 +30,8 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
       rc.enckey = encryptionKey.toPublic();
     }
 
+    std::shared_ptr<thread::ThreadPool> worker;
+
     SecretKey signingKey;
     SecretKey encryptionKey;
 
@@ -37,6 +39,12 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
 
     bool gotLIM = false;
 
+    void Setup()
+    {
+        worker = std::make_shared<thread::ThreadPool>(1, 128, "test-worker");
+        worker->start();
+    }
+
     const RouterContact&
     GetRC() const
     {
@@ -85,7 +93,7 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
         return false;
       if(!rc.Sign(signingKey))
         return false;
-      return link->Start(logic);
+      return link->Start(logic, worker);
     }
 
     void
@@ -93,6 +101,8 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
     {
       if(link)
         link->Stop();
+      if(worker)
+        worker->stop();
     }
 
     void
@@ -100,6 +110,7 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
     {
       Stop();
       link.reset();
+      worker.reset();
     }
   };
 
@@ -125,6 +136,8 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
     RouterContact::Lifetime    = 500;
     netLoop                    = llarp_make_ev_loop();
     m_logic.reset(new Logic());
+    Alice.Setup();
+    Bob.Setup();
   }
 
   void

From bcf9135da6d4caf7e35ba5c5b83363c092254dee Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 16:20:55 -0400
Subject: [PATCH 13/27] testnet

---
 llarp/path/pathbuilder.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp
index 80b6a716d..4ef91fb9b 100644
--- a/llarp/path/pathbuilder.cpp
+++ b/llarp/path/pathbuilder.cpp
@@ -214,8 +214,12 @@ namespace llarp
               if(s && s->IsEstablished() && isOutbound && !got)
               {
                 const RouterContact rc = s->GetRemoteRC();
+#ifdef TESTNET
+                if(got || exclude.count(rc.pubkey))
+#else
                 if(got || exclude.count(rc.pubkey)
                    || m_router->IsBootstrapNode(rc.pubkey))
+#endif
                   return;
                 cur = rc;
                 got = true;

From e3bb59707ee158e98557a199f91da1bc49399a0c Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 5 Sep 2019 17:28:50 -0400
Subject: [PATCH 14/27] more

---
 llarp/context.cpp           |  2 +-
 llarp/ev/ev.cpp             |  1 -
 llarp/ev/ev_libuv.cpp       | 10 ++++++++--
 llarp/handlers/exit.cpp     |  3 ++-
 llarp/handlers/tun.cpp      | 25 ++++++++++++++-----------
 llarp/link/server.hpp       | 11 ++++++++---
 llarp/router/router.cpp     |  3 +--
 llarp/util/thread/logic.cpp | 10 ++++++++--
 llarp/util/thread/logic.hpp |  7 ++++---
 9 files changed, 46 insertions(+), 26 deletions(-)

diff --git a/llarp/context.cpp b/llarp/context.cpp
index 9b8d71ded..894454602 100644
--- a/llarp/context.cpp
+++ b/llarp/context.cpp
@@ -43,6 +43,7 @@ namespace llarp
   bool
   Context::Configure()
   {
+    logic = std::make_shared< Logic >();
     // llarp::LogInfo("loading config at ", configfile);
     if(!config->Load(configfile.c_str()))
     {
@@ -197,7 +198,6 @@ __        ___    ____  _   _ ___ _   _  ____
     llarp::LogInfo(LLARP_VERSION, " ", LLARP_RELEASE_MOTTO);
     llarp::LogInfo("starting up");
     mainloop = llarp_make_ev_loop();
-    logic    = std::make_shared< Logic >();
 
     if(debug)
     {
diff --git a/llarp/ev/ev.cpp b/llarp/ev/ev.cpp
index 54366cde1..e2655b473 100644
--- a/llarp/ev/ev.cpp
+++ b/llarp/ev/ev.cpp
@@ -44,7 +44,6 @@ llarp_ev_loop_run_single_process(llarp_ev_loop_ptr ev,
     {
       ev->update_time();
       logic->tick_async(ev->time_now());
-      llarp_threadpool_tick(logic->thread);
     }
     llarp::LogContext::Instance().logStream->Tick(ev->time_now());
   }
diff --git a/llarp/ev/ev_libuv.cpp b/llarp/ev/ev_libuv.cpp
index bddfc6791..a4e0b4735 100644
--- a/llarp/ev/ev_libuv.cpp
+++ b/llarp/ev/ev_libuv.cpp
@@ -98,9 +98,14 @@ namespace libuv
     OnRead(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf)
     {
       if(nread >= 0)
-        static_cast< conn_glue* >(stream->data)->Read(buf->base, nread);
+      {
+        auto* conn = static_cast< conn_glue* >(stream->data);
+        conn->Read(buf->base, nread);
+      }
       else if(nread < 0)
+      {
         static_cast< conn_glue* >(stream->data)->Close();
+      }
       delete[] buf->base;
     }
 
@@ -257,7 +262,8 @@ namespace libuv
     static void
     OnTick(uv_check_t* t)
     {
-      static_cast< conn_glue* >(t->data)->Tick();
+      auto* conn = static_cast< conn_glue* >(t->data);
+      conn->Tick();
     }
 
     void
diff --git a/llarp/handlers/exit.cpp b/llarp/handlers/exit.cpp
index c0f6c944f..5b207056f 100644
--- a/llarp/handlers/exit.cpp
+++ b/llarp/handlers/exit.cpp
@@ -22,7 +22,8 @@ namespace llarp
     static void
     ExitHandlerFlush(llarp_tun_io *tun)
     {
-      static_cast< ExitEndpoint * >(tun->user)->Flush();
+      auto *ep = static_cast< ExitEndpoint * >(tun->user);
+      ep->GetRouter()->logic()->queue_func(std::bind(&ExitEndpoint::Flush, ep));
     }
 
     ExitEndpoint::ExitEndpoint(const std::string &name, AbstractRouter *r)
diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp
index e1238ffec..27fa27c53 100644
--- a/llarp/handlers/tun.cpp
+++ b/llarp/handlers/tun.cpp
@@ -709,13 +709,13 @@ namespace llarp
     void
     TunEndpoint::Tick(llarp_time_t now)
     {
-      // call tun code in endpoint logic in case of network isolation
-      // EndpointLogic()->queue_job({this, handleTickTun});
-      m_ExitMap.ForEachValue([&](const auto &exit) {
-        EnsureRouterIsKnown(exit->Endpoint());
-        exit->Tick(now);
+      EndpointLogic()->queue_func([&]() {
+        m_ExitMap.ForEachValue([&](const auto &exit) {
+          this->EnsureRouterIsKnown(exit->Endpoint());
+          exit->Tick(now);
+        });
+        Endpoint::Tick(now);
       });
-      Endpoint::Tick(now);
     }
 
     bool
@@ -934,10 +934,13 @@ namespace llarp
       // called in the isolated network thread
       auto *self = static_cast< TunEndpoint * >(tun->user);
       // flush user to network
-      self->FlushSend();
+      self->EndpointLogic()->queue_func(
+          std::bind(&TunEndpoint::FlushSend, self));
       // flush exit traffic queues if it's there
-      self->m_ExitMap.ForEachValue(
-          [](const auto &exit) { exit->FlushDownstream(); });
+      self->EndpointLogic()->queue_func([self] {
+        self->m_ExitMap.ForEachValue(
+            [](const auto &exit) { exit->FlushDownstream(); });
+      });
       // flush network to user
       self->m_NetworkToUserPktQueue.Process([tun](net::IPPacket &pkt) {
         if(!llarp_ev_tun_async_write(tun, pkt.Buffer()))
@@ -950,9 +953,9 @@ namespace llarp
     {
       // called for every packet read from user in isolated network thread
       auto *self = static_cast< TunEndpoint * >(tun->user);
-      const ManagedBuffer buf(b);
+      const ManagedBuffer pkt(b);
       self->m_UserToNetworkPktQueue.EmplaceIf(
-          [&buf](net::IPPacket &pkt) -> bool { return pkt.Load(buf); });
+          [&pkt](net::IPPacket &p) -> bool { return p.Load(pkt); });
     }
 
     TunEndpoint::~TunEndpoint() = default;
diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp
index c7b543d9f..3ef387719 100644
--- a/llarp/link/server.hpp
+++ b/llarp/link/server.hpp
@@ -79,7 +79,8 @@ namespace llarp
     static void
     udp_tick(llarp_udp_io* udp)
     {
-      static_cast< ILinkLayer* >(udp->user)->Pump();
+      ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
+      link->logic()->queue_func([link]() { link->Pump(); });
     }
 
     static void
@@ -90,11 +91,15 @@ namespace llarp
         llarp::LogWarn("no udp set");
         return;
       }
+      std::vector< byte_t > pkt(buf.underlying.sz);
+      std::copy_n(buf.underlying.base, buf.underlying.sz, pkt.begin());
       const llarp::Addr srcaddr(*from);
       // maybe check from too?
       // no it's never null
-      static_cast< ILinkLayer* >(udp->user)->RecvFrom(
-          srcaddr, buf.underlying.base, buf.underlying.sz);
+      ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
+      link->logic()->queue_func([link, srcaddr, pkt]() {
+        link->RecvFrom(srcaddr, pkt.data(), pkt.size());
+      });
     }
 
     void
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 643e0c078..6fb0d1973 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -294,7 +294,7 @@ namespace llarp
       return;
     auto *self          = static_cast< Router * >(user);
     self->ticker_job_id = 0;
-    self->Tick();
+    self->logic()->queue_func(std::bind(&Router::Tick, self));
     self->ScheduleTicker(orig);
   }
 
@@ -998,7 +998,6 @@ namespace llarp
     }
 
     LogInfo("have ", nodedb->num_loaded(), " routers");
-    _netloop->add_ticker(std::bind(&Router::PumpLL, this));
     ScheduleTicker(1000);
     _running.store(true);
     _startedAt = Now();
diff --git a/llarp/util/thread/logic.cpp b/llarp/util/thread/logic.cpp
index 4ff30debd..ed4fd0532 100644
--- a/llarp/util/thread/logic.cpp
+++ b/llarp/util/thread/logic.cpp
@@ -13,11 +13,17 @@ namespace llarp
     llarp_threadpool_tick(this->thread);
   }
 
+  Logic::~Logic()
+  {
+    llarp_threadpool_stop(this->thread);
+    llarp_threadpool_join(this->thread);
+    llarp_free_threadpool(&this->thread);
+  }
+
   void
   Logic::tick_async(llarp_time_t now)
   {
     llarp_timer_tick_all_async(this->timer, this->thread, now);
-    llarp_threadpool_tick(this->thread);
   }
 
   void
@@ -97,7 +103,7 @@ namespace llarp
   bool
   Logic::can_flush() const
   {
-    return ourID == std::this_thread::get_id();
+    return false;
   }
 
 }  // namespace llarp
diff --git a/llarp/util/thread/logic.hpp b/llarp/util/thread/logic.hpp
index f29e1dcda..6e36898ed 100644
--- a/llarp/util/thread/logic.hpp
+++ b/llarp/util/thread/logic.hpp
@@ -12,15 +12,16 @@ namespace llarp
    public:
     struct llarp_threadpool* thread;
     struct llarp_timer_context* timer;
-    const std::thread::id ourID;
 
     Logic()
-        : thread(llarp_init_same_process_threadpool())
+        : thread(llarp_init_threadpool(1, "llarp-logic"))
         , timer(llarp_init_timer())
-        , ourID(std::this_thread::get_id())
     {
+      llarp_threadpool_start(thread);
     }
 
+    ~Logic();
+
     /// single threaded tick
     void
     tick(llarp_time_t now);

From d04762ab4977c96ad031d931d199359a99b49dfe Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Thu, 12 Sep 2019 10:34:27 -0400
Subject: [PATCH 15/27] unstaged changed

---
 llarp/iwp/message_buffer.cpp |  84 +++++++---------
 llarp/iwp/message_buffer.hpp |  17 ++--
 llarp/iwp/session.cpp        | 186 ++++++++++++++++++-----------------
 llarp/iwp/session.hpp        |  30 +++---
 llarp/link/session.hpp       |   8 +-
 llarp/net/ip_range_map.hpp   |   2 +-
 6 files changed, 158 insertions(+), 169 deletions(-)

diff --git a/llarp/iwp/message_buffer.cpp b/llarp/iwp/message_buffer.cpp
index ab610edb2..33adcb624 100644
--- a/llarp/iwp/message_buffer.cpp
+++ b/llarp/iwp/message_buffer.cpp
@@ -6,28 +6,28 @@ namespace llarp
 {
   namespace iwp
   {
-    OutboundMessage::OutboundMessage(uint64_t msgid, const llarp_buffer_t &pkt,
+    OutboundMessage::OutboundMessage(uint64_t msgid,
+                                     ILinkSession::Message_t msg,
                                      llarp_time_t now,
                                      ILinkSession::CompletionHandler handler)
-        : m_Size{(uint16_t)std::min(pkt.sz, MAX_LINK_MSG_SIZE)}
+        : m_Data{std::move(msg)}
         , m_MsgID{msgid}
         , m_Completed{handler}
         , m_StartedAt{now}
     {
-      m_Data.Zero();
-      std::copy_n(pkt.base, m_Size, m_Data.begin());
-      const llarp_buffer_t buf(m_Data.data(), m_Size);
-      CryptoManager::instance()->shorthash(digest, buf);
+      const llarp_buffer_t buf(m_Data);
+      CryptoManager::instance()->shorthash(m_Digest, buf);
     }
 
-    std::vector< byte_t >
+    ILinkSession::Packet_t
     OutboundMessage::XMIT() const
     {
-      std::vector< byte_t > xmit{
-          LLARP_PROTO_VERSION, Command::eXMIT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-      htobe16buf(xmit.data() + 2, m_Size);
+      ILinkSession::Packet_t xmit(12 + 32);
+      xmit[0] = LLARP_PROTO_VERSION;
+      xmit[1] = Command::eXMIT;
+      htobe16buf(xmit.data() + 2, m_Data.size());
       htobe64buf(xmit.data() + 4, m_MsgID);
-      std::copy(digest.begin(), digest.end(), std::back_inserter(xmit));
+      std::copy_n(m_Digest.begin(), m_Digest.size(), xmit.begin() + 12);
       return xmit;
     }
 
@@ -55,35 +55,27 @@ namespace llarp
 
     void
     OutboundMessage::FlushUnAcked(
-        std::function< void(const llarp_buffer_t &) > sendpkt, llarp_time_t now)
+        std::function< void(ILinkSession::Packet_t) > sendpkt, llarp_time_t now)
     {
-      uint16_t idx = 0;
-      while(idx < m_Size)
+      /// overhead for a data packet in plaintext
+      static constexpr size_t Overhead = 12;
+      uint16_t idx                     = 0;
+      const auto datasz                = m_Data.size();
+      while(idx < datasz)
       {
         if(not m_Acks[idx / FragmentSize])
         {
-          std::vector< byte_t > frag{LLARP_PROTO_VERSION,
-                                     Command::eDATA,
-                                     0,
-                                     0,
-                                     0,
-                                     0,
-                                     0,
-                                     0,
-                                     0,
-                                     0,
-                                     0,
-                                     0};
+          const size_t fragsz =
+              idx + FragmentSize < datasz ? FragmentSize : datasz - idx;
+          ILinkSession::Packet_t frag(fragsz + Overhead);
+
+          frag[0] = LLARP_PROTO_VERSION;
+          frag[1] = Command::eDATA;
           htobe16buf(frag.data() + 2, idx);
           htobe64buf(frag.data() + 4, m_MsgID);
-          const size_t fragsz =
-              idx + FragmentSize < m_Size ? FragmentSize : m_Size - idx;
-          const auto sz = frag.size();
-          frag.resize(sz + fragsz);
           std::copy(m_Data.begin() + idx, m_Data.begin() + idx + fragsz,
-                    frag.begin() + sz);
-          const llarp_buffer_t pkt(frag);
-          sendpkt(pkt);
+                    frag.begin() + Overhead);
+          sendpkt(std::move(frag));
         }
         idx += FragmentSize;
       }
@@ -93,7 +85,8 @@ namespace llarp
     bool
     OutboundMessage::IsTransmitted() const
     {
-      for(uint16_t idx = 0; idx < m_Size; idx += FragmentSize)
+      const auto sz = m_Data.size();
+      for(uint16_t idx = 0; idx < sz; idx += FragmentSize)
       {
         if(!m_Acks.test(idx / FragmentSize))
           return false;
@@ -140,14 +133,14 @@ namespace llarp
       m_LastActiveAt = now;
     }
 
-    std::vector< byte_t >
+    ILinkSession::Packet_t
     InboundMessage::ACKS() const
     {
-      std::vector< byte_t > acks{
-          LLARP_PROTO_VERSION, Command::eACKS, 0, 0, 0, 0, 0, 0, 0, 0,
-          AcksBitmask()};
-
+      ILinkSession::Packet_t acks(9);
+      acks[0] = LLARP_PROTO_VERSION;
+      acks[1] = Command::eACKS;
       htobe64buf(acks.data() + 2, m_MsgID);
+      acks[8] = AcksBitmask();
       return acks;
     }
 
@@ -183,12 +176,11 @@ namespace llarp
 
     void
     InboundMessage::SendACKS(
-        std::function< void(const llarp_buffer_t &) > sendpkt, llarp_time_t now)
+        std::function< void(ILinkSession::Packet_t) > sendpkt, llarp_time_t now)
     {
       auto acks = ACKS();
       AddRandomPadding(acks);
-      const llarp_buffer_t pkt(acks);
-      sendpkt(pkt);
+      sendpkt(std::move(acks));
       m_LastACKSent = now;
     }
 
@@ -198,13 +190,7 @@ namespace llarp
       ShortHash gotten;
       const llarp_buffer_t buf(m_Data.data(), m_Size);
       CryptoManager::instance()->shorthash(gotten, buf);
-      LogDebug("gotten=", gotten.ToHex());
-      if(gotten != m_Digset)
-      {
-        DumpBuffer(buf);
-        return false;
-      }
-      return true;
+      return gotten == m_Digset;
     }
 
   }  // namespace iwp
diff --git a/llarp/iwp/message_buffer.hpp b/llarp/iwp/message_buffer.hpp
index c2c1dcb6f..e09883094 100644
--- a/llarp/iwp/message_buffer.hpp
+++ b/llarp/iwp/message_buffer.hpp
@@ -34,27 +34,26 @@ namespace llarp
     struct OutboundMessage
     {
       OutboundMessage() = default;
-      OutboundMessage(uint64_t msgid, const llarp_buffer_t &pkt,
+      OutboundMessage(uint64_t msgid, ILinkSession::Message_t data,
                       llarp_time_t now,
                       ILinkSession::CompletionHandler handler);
 
-      AlignedBuffer< MAX_LINK_MSG_SIZE > m_Data;
-      uint16_t m_Size  = 0;
+      ILinkSession::Message_t m_Data;
       uint64_t m_MsgID = 0;
       std::bitset< MAX_LINK_MSG_SIZE / FragmentSize > m_Acks;
       ILinkSession::CompletionHandler m_Completed;
       llarp_time_t m_LastFlush = 0;
-      ShortHash digest;
+      ShortHash m_Digest;
       llarp_time_t m_StartedAt = 0;
 
-      std::vector< byte_t >
+      ILinkSession::Packet_t
       XMIT() const;
 
       void
       Ack(byte_t bitmask);
 
       void
-      FlushUnAcked(std::function< void(const llarp_buffer_t &) > sendpkt,
+      FlushUnAcked(std::function< void(ILinkSession::Packet_t) > sendpkt,
                    llarp_time_t now);
 
       bool
@@ -88,7 +87,7 @@ namespace llarp
       std::bitset< MAX_LINK_MSG_SIZE / FragmentSize > m_Acks;
 
       void
-      HandleData(uint16_t idx, const llarp_buffer_t &buf, llarp_time_t now);
+      HandleData(uint16_t idx, const llarp_buffer_t& buf, llarp_time_t now);
 
       bool
       IsCompleted() const;
@@ -106,10 +105,10 @@ namespace llarp
       ShouldSendACKS(llarp_time_t now) const;
 
       void
-      SendACKS(std::function< void(const llarp_buffer_t &) > sendpkt,
+      SendACKS(std::function< void(ILinkSession::Packet_t) > sendpkt,
                llarp_time_t now);
 
-      std::vector< byte_t >
+      ILinkSession::Packet_t
       ACKS() const;
     };
 
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index 7a719bf5d..c8c811f5d 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -8,15 +8,23 @@ namespace llarp
 {
   namespace iwp
   {
-    static constexpr size_t PacketOverhead = HMACSIZE + TUNNONCESIZE;
-
-    void
-    AddRandomPadding(std::vector< byte_t >& pkt, size_t min, size_t variance)
+    ILinkSession::Packet_t
+    CreatePacket(Command cmd, size_t plainsize, size_t minpad, size_t variance)
     {
-      const auto sz        = pkt.size();
-      const size_t randpad = min + randint() % variance;
-      pkt.resize(sz + randpad);
-      CryptoManager::instance()->randbytes(pkt.data() + sz, randpad);
+      const size_t pad = minpad > 0 ?: min + randint() % variance : 0;
+      ILinkSession::Packet_t pkt(PacketOverhead + plainsize + pad + 2);
+      // randomize pad
+      if(pad)
+      {
+        CryptoManager::instance()->randbytes(
+            pkt.data() + PacketOverhead + 2 + plainsize, pad);
+      }
+      // randomize nounce
+      CryptoManager::instance()->randbytes(pkt.data() + HMACSIZE,
+                                           TUNNOUNCESIZE);
+      pkt[PacketOverhead]     = LLARP_PROTO_VERSION;
+      pkt[PacketOverhead + 1] = cmd;
+      return pkt;
     }
 
     Session::Session(LinkLayer* p, RouterContact rc, AddressInfo ai)
@@ -121,11 +129,9 @@ namespace llarp
     }
 
     void
-    Session::EncryptAndSend(const llarp_buffer_t& data)
+    Session::EncryptAndSend(ILinkSession::Packet_t data)
     {
-      m_EncryptNext.emplace_back(data.sz + PacketOverhead);
-      auto& pkt = m_EncryptNext.back();
-      std::copy_n(data.base, data.sz, pkt.begin() + PacketOverhead);
+      m_EncryptNext.emplace_back(std::move(data));
       if(!IsEstablished())
         EncryptWorker(std::move(m_EncryptNext));
     }
@@ -160,10 +166,8 @@ namespace llarp
     {
       if(m_State == State::Closed)
         return;
-      std::vector< byte_t > close_msg = {LLARP_PROTO_VERSION, Command::eCLOS};
-      AddRandomPadding(close_msg);
-      const llarp_buffer_t buf(close_msg);
-      EncryptAndSend(buf);
+      auto close_msg = CreatePacket(Command::eCLOS, 0, 16, 16);
+      EncryptAndSend(std::move(close_msg));
       if(m_State == State::Ready)
         m_Parent->UnmapAddr(m_RemoteAddr);
       m_State = State::Closed;
@@ -171,7 +175,7 @@ namespace llarp
     }
 
     bool
-    Session::SendMessageBuffer(const llarp_buffer_t& buf,
+    Session::SendMessageBuffer(ILinkSession::Message_t buf,
                                ILinkSession::CompletionHandler completed)
     {
       if(m_TXMsgs.size() >= MaxSendQueueSize)
@@ -179,12 +183,12 @@ namespace llarp
       const auto now   = m_Parent->Now();
       const auto msgid = m_TXID++;
       auto& msg =
-          m_TXMsgs.emplace(msgid, OutboundMessage{msgid, buf, now, completed})
+          m_TXMsgs
+              .emplace(msgid,
+                       OutboundMessage{msgid, std::move(buf), now, completed})
               .first->second;
       auto xmit = msg.XMIT();
-      AddRandomPadding(xmit);
-      const llarp_buffer_t pkt(xmit);
-      EncryptAndSend(pkt);
+      EncryptAndSend(pstd::move(xmit));
       msg.FlushUnAcked(util::memFn(&Session::EncryptAndSend, this), now);
       LogDebug("send message ", msgid);
       return true;
@@ -199,12 +203,10 @@ namespace llarp
         const auto sz  = m_SendMACKS.size();
         const auto max = Session::MaxACKSInMACK;
         auto numAcks   = std::min(sz, max);
-        std::vector< byte_t > mack;
-        mack.resize(2 + 1 + (numAcks * sizeof(uint64_t)));
-        mack[0]     = LLARP_PROTO_VERSION;
-        mack[1]     = Command::eMACK;
-        mack[2]     = byte_t{numAcks};
-        byte_t* ptr = mack.data() + 3;
+        auto mack =
+            CreatePacket(Command::eMACK, numAcks * sizeof(uint64_t), 0, 0);
+        mack[PacketOverhead + 2] = byte_t{numAcks};
+        byte_t* ptr              = mack.data() + 3 + PacketOverhead;
         while(numAcks > 0)
         {
           htobe64buf(ptr, m_SendMACKS.back());
@@ -212,8 +214,7 @@ namespace llarp
           numAcks--;
           ptr += sizeof(uint64_t);
         }
-        const llarp_buffer_t buf(mack);
-        EncryptAndSend(buf);
+        EncryptAndSend(std::move(mack));
       }
     }
 
@@ -243,16 +244,16 @@ namespace llarp
           }
         }
       }
-
+      auto self = shared_from_this();
       if(!m_EncryptNext.empty())
-        m_Parent->QueueWork(std::bind(&Session::EncryptWorker,
-                                      shared_from_this(),
-                                      std::move(m_EncryptNext)));
+        m_Parent->QueueWork([self, data = std::move(m_EncryptNext)] {
+          self->EncryptWorker(data);
+        });
 
       if(!m_DecryptNext.empty())
-        m_Parent->QueueWork(std::bind(&Session::DecryptWorker,
-                                      shared_from_this(),
-                                      std::move(m_DecryptNext)));
+        m_Parent->QueueWork([self, data = std::move(m_DecryptNext)] {
+          self->DecryptWorker(data);
+        });
     }
 
     bool
@@ -353,7 +354,7 @@ namespace llarp
       TunnelNonce N;
       N.Randomize();
       std::vector< byte_t > req;
-      req.resize(Introduction::SIZE - Signature::SIZE);
+      req.reserve(Introduction::SIZE - Signature::SIZE);
       const auto pk   = m_Parent->GetOurRC().pubkey;
       const auto e_pk = m_Parent->RouterEncryptionSecret().toPublic();
       auto itr        = req.begin();
@@ -365,12 +366,10 @@ namespace llarp
       Signature Z;
       llarp_buffer_t signbuf(req);
       m_Parent->Sign(Z, signbuf);
-      req.resize(Introduction::SIZE);
+      req.reserve(Introduction::SIZE);
       std::copy_n(Z.begin(), Z.size(),
                   req.begin() + (Introduction::SIZE - Signature::SIZE));
-      AddRandomPadding(req);
-      const llarp_buffer_t buf(req);
-      EncryptAndSend(buf);
+      EncryptAndSend(std::move(req));
       m_State = State::Introduction;
       if(not CryptoManager::instance()->transport_dh_client(
              m_SessionKey, m_ChosenAI.pubkey,
@@ -384,22 +383,21 @@ namespace llarp
     }
 
     void
-    Session::HandleCreateSessionRequest(const llarp_buffer_t& buf)
+    Session::HandleCreateSessionRequest(Packet_t pkt)
     {
-      std::vector< byte_t > result;
-      if(not DecryptMessage(buf, result))
+      if(not DecryptMessageInPlace(pkt))
       {
         LogError("failed to decrypt session request from ", m_RemoteAddr);
         return;
       }
-      if(result.size() < token.size())
+      if(pkt.size() < token.size() + PacketOverhead)
       {
         LogError("bad session request size, ", result.size(), " < ",
-                 token.size(), " from ", m_RemoteAddr);
+                 token.size() + PacketOverhead, " from ", m_RemoteAddr);
         return;
       }
-      if(not std::equal(result.begin(), result.begin() + token.size(),
-                        token.begin()))
+      const auto begin = pkt.begin() + PacketOverhead;
+      if(not std::equal(begin, begin + token.size(), token.begin()))
       {
         LogError("token missmatch from ", m_RemoteAddr);
         return;
@@ -410,14 +408,14 @@ namespace llarp
     }
 
     void
-    Session::HandleGotIntro(const llarp_buffer_t& buf)
+    Session::HandleGotIntro(Packet_t pkt)
     {
-      if(buf.sz < Introduction::SIZE)
+      if(pkt.size() < Introduction::SIZE + PacketOverhead)
       {
         LogWarn("intro too small from ", m_RemoteAddr);
         return;
       }
-      byte_t* ptr = buf.base;
+      byte_t* ptr = pkt.begin() + PacketOverhead;
       TunnelNonce N;
       std::copy_n(ptr, PubKey::SIZE, m_ExpectedIdent.begin());
       ptr += PubKey::SIZE;
@@ -427,7 +425,7 @@ namespace llarp
       ptr += TunnelNonce::SIZE;
       Signature Z;
       std::copy_n(ptr, Z.size(), Z.begin());
-      const llarp_buffer_t verifybuf(buf.base,
+      const llarp_buffer_t verifybuf(pkt.begin() + PacketOverhead,
                                      Introduction::SIZE - Signature::SIZE);
       if(!CryptoManager::instance()->verify(m_ExpectedIdent, verifybuf, Z))
       {
@@ -436,7 +434,7 @@ namespace llarp
       }
       const PubKey pk = m_Parent->TransportSecretKey().toPublic();
       LogDebug("got intro: remote-pk=", m_RemoteOnionKey.ToHex(),
-               " N=", N.ToHex(), " local-pk=", pk.ToHex(), " sz=", buf.sz);
+               " N=", N.ToHex(), " local-pk=", pk.ToHex());
       if(not CryptoManager::instance()->transport_dh_server(
              m_SessionKey, m_RemoteOnionKey, m_Parent->TransportSecretKey(), N))
       {
@@ -444,49 +442,53 @@ namespace llarp
                  m_RemoteAddr);
         return;
       }
-      std::vector< byte_t > reply;
-      reply.resize(token.size());
-      std::copy_n(token.begin(), token.size(), reply.begin());
-      AddRandomPadding(reply);
-      const llarp_buffer_t pkt(reply);
+      std::vector< byte_t > reply(token.size() + PacketOverhead);
+      // random nonce
+      CryptoManger::instance()->rand_bytes(reply.data() + HMACSIZE,
+                                           TUNNONCESIZE);
+      // set token
+      std::copy_n(token.begin(), token.size(), reply.begin() + PacketOverhead);
       m_LastRX = m_Parent->Now();
-      EncryptAndSend(pkt);
+      EncryptAndSend(std::move(reply));
       LogDebug("sent intro ack to ", m_RemoteAddr);
       m_State = State::Introduction;
     }
 
     void
-    Session::HandleGotIntroAck(const llarp_buffer_t& buf)
+    Session::HandleGotIntroAck(Packet_t pkt)
     {
-      std::vector< byte_t > reply;
-      if(not DecryptMessage(buf, reply))
+      if(pkt.size() < token.size() + PacketOverhead)
       {
-        LogError("intro ack decrypt failed from ", m_RemoteAddr);
+        LogError("bad intro ack size ", reply.size(), " < ",
+                 token.size() + PacketOverhead, " from ", m_RemoteAddr);
         return;
       }
-      if(reply.size() < token.size())
+      std::vector< byte_t > reply(token.size() + PacketOverhead);
+      if(not DecryptMessageInPlace(pkt))
       {
-        LogError("bad intro ack size ", reply.size(), " < ", token.size(),
-                 " from ", m_RemoteAddr);
+        LogError("intro ack decrypt failed from ", m_RemoteAddr);
         return;
       }
       m_LastRX = m_Parent->Now();
-      std::copy_n(reply.begin(), token.size(), token.begin());
-      const llarp_buffer_t pkt(token);
-      EncryptAndSend(pkt);
+      std::copy_n(pkt.begin() + PacketOverhead, token.size(), token.begin());
+      std::copy_n(token.begin(), token.size(), reply.begin() + PacketOverhead);
+      // random nounce
+      CryptoManager::instance()->rand_bytes(reply.data() + HMACSIZE,
+                                            TUNNONCESIZE);
+      EncryptAndSend(std::move(reply));
       LogDebug("sent session request to ", m_RemoteAddr);
       m_State = State::LinkIntro;
     }
 
     bool
-    Session::DecryptMessage(const llarp_buffer_t& buf,
-                            std::vector< byte_t >& result)
+    Session::DecryptMessageInPlace(Packet_t& pkt)
     {
-      if(buf.sz <= PacketOverhead)
+      if(pkt.size() <= PacketOverhead)
       {
-        LogError("packet too small ", buf.sz);
+        LogError("packet too small from ", m_RemoteAddr);
         return false;
       }
+      const llarp_buffer_t buf(pkt);
       ShortHash H;
       llarp_buffer_t curbuf(buf.base, buf.sz);
       curbuf.base += ShortHash::SIZE;
@@ -503,14 +505,11 @@ namespace llarp
                  m_RemoteAddr, " state=", int(m_State), " size=", buf.sz);
         return false;
       }
-      const byte_t* nonce_ptr = curbuf.base;
+      const TunnelNounce N{curbuf.base};
       curbuf.base += 32;
       curbuf.sz -= 32;
-      result.resize(buf.sz - PacketOverhead);
-      const llarp_buffer_t outbuf(result);
       LogDebug("decrypt: ", result.size(), " bytes from ", m_RemoteAddr);
-      return CryptoManager::instance()->xchacha20_alt(outbuf, curbuf,
-                                                      m_SessionKey, nonce_ptr);
+      return CryptoManager::instance()->xchacha20(curbuf, m_SessionKey, N);
     }
 
     void
@@ -522,29 +521,33 @@ namespace llarp
     }
 
     void
-    Session::HandleSessionData(const llarp_buffer_t& buf)
+    Session::HandleSessionData(Packet_t pkt)
     {
-      m_DecryptNext.emplace_back(buf.sz);
-      auto& pkt = m_DecryptNext.back();
-      std::copy_n(buf.base, buf.sz, pkt.begin());
+      m_DecryptNext.emplace_back(pkt);
+      // pump rx if we are big enough
+      if(m_DecryptNext.size() >= RXPumpSize)
+      {
+        auto self = shared_from_this();
+        m_Parent->QueueWork([self, data = std::move(m_DecryptNext)] {
+          self->DecryptWorker(data);
+        });
+      }
     }
 
     void
     Session::DecryptWorker(CryptoQueue_t msgs)
     {
       CryptoQueue_t recvMsgs;
-      for(const auto& pkt : msgs)
+      for(auto& pkt : msgs)
       {
-        std::vector< byte_t > result;
-        const llarp_buffer_t buf(pkt);
-        if(not DecryptMessage(buf, result))
+        if(not DecryptMessageInPlace(pkt))
         {
           LogError("failed to decrypt session data from ", m_RemoteAddr);
           continue;
         }
-        if(result[0] != LLARP_PROTO_VERSION)
+        if(pkt[PacketOverhead] != LLARP_PROTO_VERSION)
         {
-          LogError("protocol version missmatch ", int(result[0]),
+          LogError("protocol version missmatch ", int(result[PacketOverhead]),
                    " != ", LLARP_PROTO_VERSION);
           continue;
         }
@@ -560,8 +563,7 @@ namespace llarp
     {
       for(auto& result : msgs)
       {
-        LogDebug("command ", int(result[1]), " from ", m_RemoteAddr);
-        switch(result[1])
+        switch(result[PacketOverhead + 1])
         {
           case Command::eXMIT:
             HandleXMIT(std::move(result));
@@ -585,8 +587,8 @@ namespace llarp
             HandleMACK(std::move(result));
             break;
           default:
-            LogError("invalid command ", int(result[1]), " from ",
-                     m_RemoteAddr);
+            LogError("invalid command ", int(result[PacketOverhead + 1]),
+                     " from ", m_RemoteAddr);
         }
       }
     }
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index e2982fd7a..6d235276b 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -9,9 +9,11 @@ namespace llarp
 {
   namespace iwp
   {
-    void
-    AddRandomPadding(std::vector< byte_t >& pkt, size_t min = 16,
-                     size_t variance = 16);
+    static constexpr size_t PacketOverhead = HMACSIZE + TUNNONCESIZE;
+    /// creates a packet with plaintext size + wire overhead + random pad
+    ILinkSession::Packet_t
+    CreatePacket(Command cmd, size_t plainsize, size_t min_pad = 16,
+                 size_t pad_variance = 16);
 
     struct Session : public ILinkSession,
                      public std::enable_shared_from_this< Session >
@@ -55,14 +57,13 @@ namespace llarp
       Tick(llarp_time_t now) override;
 
       bool
-      SendMessageBuffer(const llarp_buffer_t& buf,
+      SendMessageBuffer(ILinkSession::Message_t msg,
                         CompletionHandler resultHandler) override;
 
       void
       Send_LL(const llarp_buffer_t& pkt);
 
-      void
-      EncryptAndSend(const llarp_buffer_t& data);
+      void EncryptAndSend(ILinkSession::Packet_t);
 
       void
       Start() override;
@@ -70,8 +71,7 @@ namespace llarp
       void
       Close() override;
 
-      void
-      Recv_LL(const llarp_buffer_t& pkt) override;
+      void Recv_LL(ILinkSession::Packet_t) override;
 
       bool
       SendKeepAlive() override;
@@ -167,7 +167,7 @@ namespace llarp
       /// list of rx messages to send in next set of multiacks
       std::vector< uint64_t > m_SendMACKS;
 
-      using CryptoQueue_t = std::vector< std::vector< byte_t > >;
+      using CryptoQueue_t = std::vector< Packet_t >;
 
       CryptoQueue_t m_EncryptNext;
       CryptoQueue_t m_DecryptNext;
@@ -182,22 +182,22 @@ namespace llarp
       HandlePlaintext(CryptoQueue_t msgs);
 
       void
-      HandleGotIntro(const llarp_buffer_t& buf);
+      HandleGotIntro(Packet_t pkt);
 
       void
-      HandleGotIntroAck(const llarp_buffer_t& buf);
+      HandleGotIntroAck(Packet_t pkt);
 
       void
-      HandleCreateSessionRequest(const llarp_buffer_t& buf);
+      HandleCreateSessionRequest(Packet_t pkt);
 
       void
-      HandleAckSession(const llarp_buffer_t& buf);
+      HandleAckSession(Packet_t pkt);
 
       void
-      HandleSessionData(const llarp_buffer_t& buf);
+      HandleSessionData(Packet_t pkt);
 
       bool
-      DecryptMessage(const llarp_buffer_t& buf, std::vector< byte_t >& result);
+      DecryptMessageInPlace(Packet_t& pkt);
 
       void
       SendMACK();
diff --git a/llarp/link/session.hpp b/llarp/link/session.hpp
index 8647b6ca9..9ff8f0380 100644
--- a/llarp/link/session.hpp
+++ b/llarp/link/session.hpp
@@ -44,9 +44,12 @@ namespace llarp
     /// message delivery result hook function
     using CompletionHandler = std::function< void(DeliveryStatus) >;
 
+    using Packet_t  = std::vector< byte_t >;
+    using Message_t = std::vector< byte_t >;
+
     /// send a message buffer to the remote endpoint
     virtual bool
-    SendMessageBuffer(const llarp_buffer_t &, CompletionHandler handler) = 0;
+    SendMessageBuffer(Message_t, CompletionHandler handler) = 0;
 
     /// start the connection
     virtual void
@@ -57,8 +60,7 @@ namespace llarp
 
     /// recv packet on low layer
     /// not used by utp
-    virtual void
-    Recv_LL(const llarp_buffer_t &)
+    virtual void Recv_LL(Packet_t)
     {
     }
 
diff --git a/llarp/net/ip_range_map.hpp b/llarp/net/ip_range_map.hpp
index f54d6196f..d598424c0 100644
--- a/llarp/net/ip_range_map.hpp
+++ b/llarp/net/ip_range_map.hpp
@@ -37,7 +37,7 @@ namespace llarp
           functor(entry.second);
       }
 
-      /// convert all values into type T
+      /// convert all values into type T using a transformer
       template < typename T, typename Transformer >
       std::set< T >
       TransformValues(Transformer transform) const

From ac2a2aed1d11b847ee091fd51f6c7255cf628b0e Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Thu, 12 Sep 2019 14:19:25 -0400
Subject: [PATCH 16/27] gut libutp and finish making things compile and pass
 tests

---
 CMakeLists.txt                  |    8 +-
 cmake/unix.cmake                |    2 +-
 libabyss/main.cpp               |    2 +-
 libutp/CMakeLists.txt           |   19 -
 libutp/LICENSE                  |   19 -
 libutp/Makefile                 |   48 -
 libutp/README.md                |   68 -
 libutp/libutp_inet_ntop.cpp     |   57 -
 libutp/libutp_inet_ntop.h       |   63 -
 libutp/parse_log.py             |  295 ---
 libutp/utp_api.cpp              |  139 --
 libutp/utp_callbacks.cpp        |  208 --
 libutp/utp_callbacks.h          |   47 -
 libutp/utp_hash.cpp             |  246 --
 libutp/utp_hash.h               |  209 --
 libutp/utp_internal.cpp         | 4067 -------------------------------
 libutp/utp_internal.h           |  161 --
 libutp/utp_packedsockaddr.cpp   |  166 --
 libutp/utp_packedsockaddr.h     |   60 -
 libutp/utp_templates.h          |  195 --
 libutp/utp_utils.cpp            |  307 ---
 libutp/utp_utils.h              |   27 -
 llarp/CMakeLists.txt            |    7 +-
 llarp/iwp/linklayer.cpp         |    5 +-
 llarp/iwp/linklayer.hpp         |    2 +-
 llarp/iwp/message_buffer.cpp    |   36 +-
 llarp/iwp/session.cpp           |  165 +-
 llarp/iwp/session.hpp           |   12 +-
 llarp/link/factory.cpp          |    9 -
 llarp/link/server.cpp           |   36 +-
 llarp/link/server.hpp           |   31 +-
 llarp/nodedb.cpp                |   31 +-
 llarp/router/router.cpp         |    1 -
 llarp/util/thread/logic.cpp     |    9 +-
 llarp/util/thread/logic.hpp     |    2 +-
 llarp/util/thread/threading.cpp |    3 +-
 llarp/util/thread/threadpool.h  |    9 -
 llarp/utp/inbound_message.cpp   |   27 -
 llarp/utp/inbound_message.hpp   |   86 -
 llarp/utp/linklayer.cpp         |  341 ---
 llarp/utp/linklayer.hpp         |  105 -
 llarp/utp/session.cpp           |  810 ------
 llarp/utp/session.hpp           |  301 ---
 llarp/utp/utp.cpp               |   35 -
 llarp/utp/utp.hpp               |   28 -
 test/link/test_llarp_link.cpp   |  206 +-
 test/test_libabyss.cpp          |    3 -
 47 files changed, 176 insertions(+), 8537 deletions(-)
 delete mode 100644 libutp/CMakeLists.txt
 delete mode 100644 libutp/LICENSE
 delete mode 100644 libutp/Makefile
 delete mode 100644 libutp/README.md
 delete mode 100644 libutp/libutp_inet_ntop.cpp
 delete mode 100644 libutp/libutp_inet_ntop.h
 delete mode 100644 libutp/parse_log.py
 delete mode 100644 libutp/utp_api.cpp
 delete mode 100644 libutp/utp_callbacks.cpp
 delete mode 100644 libutp/utp_callbacks.h
 delete mode 100644 libutp/utp_hash.cpp
 delete mode 100644 libutp/utp_hash.h
 delete mode 100644 libutp/utp_internal.cpp
 delete mode 100644 libutp/utp_internal.h
 delete mode 100644 libutp/utp_packedsockaddr.cpp
 delete mode 100644 libutp/utp_packedsockaddr.h
 delete mode 100644 libutp/utp_templates.h
 delete mode 100644 libutp/utp_utils.cpp
 delete mode 100644 libutp/utp_utils.h
 delete mode 100644 llarp/utp/inbound_message.cpp
 delete mode 100644 llarp/utp/inbound_message.hpp
 delete mode 100644 llarp/utp/linklayer.cpp
 delete mode 100644 llarp/utp/linklayer.hpp
 delete mode 100644 llarp/utp/session.cpp
 delete mode 100644 llarp/utp/session.hpp
 delete mode 100644 llarp/utp/utp.cpp
 delete mode 100644 llarp/utp/utp.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 528d0bcea..c2dff9452 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -66,12 +66,7 @@ include(cmake/solaris.cmake)
 include(cmake/unix.cmake)
 include(cmake/win32.cmake)
 
-if(WIN32)
-  set(CMAKE_CXX_STANDARD 17)
-else()
-  set(CMAKE_CXX_STANDARD 14)
-endif(WIN32)
-
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_C_STANDARD 99)
@@ -248,7 +243,6 @@ if(TRACY_ROOT)
 endif()
 
 add_subdirectory(crypto)
-add_subdirectory(libutp)
 add_subdirectory(llarp)
 add_subdirectory(libabyss)
 
diff --git a/cmake/unix.cmake b/cmake/unix.cmake
index f894eaefa..ecc4a1ad5 100644
--- a/cmake/unix.cmake
+++ b/cmake/unix.cmake
@@ -73,7 +73,7 @@ else()
 endif()
 
 
-set(EXE_LIBS ${STATIC_LIB} libutp)
+set(EXE_LIBS ${STATIC_LIB})
 
 if(RELEASE_MOTTO)
   add_definitions(-DLLARP_RELEASE_MOTTO="${RELEASE_MOTTO}")
diff --git a/libabyss/main.cpp b/libabyss/main.cpp
index 95395015b..626ffb719 100644
--- a/libabyss/main.cpp
+++ b/libabyss/main.cpp
@@ -39,7 +39,7 @@ struct DemoCall : public abyss::http::IRPCClientHandler
   bool HandleResponse(abyss::http::RPC_Response) override
   {
     llarp::LogInfo("response get");
-    m_Logic->queue_func(m_Callback);
+    m_Logic->queue_func([=]() { m_Callback(); });
     return true;
   }
 
diff --git a/libutp/CMakeLists.txt b/libutp/CMakeLists.txt
deleted file mode 100644
index e089e94ec..000000000
--- a/libutp/CMakeLists.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-set(UTP_SRC
-  utp_callbacks.cpp
-  utp_utils.cpp
-  utp_internal.cpp
-  utp_api.cpp
-  utp_packedsockaddr.cpp
-  utp_hash.cpp
-)
-
-if(WIN32)
-  list(APPEND UTP_SRC libutp_inet_ntop.cpp)
-endif(WIN32)
-
-add_library(libutp STATIC ${UTP_SRC})
-if(WIN32)
-  target_link_libraries(libutp ws2_32)
-endif(WIN32)
-target_include_directories(libutp PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CORE_INCLUDE})
-add_log_tag(libutp)
diff --git a/libutp/LICENSE b/libutp/LICENSE
deleted file mode 100644
index 7f6e16c5f..000000000
--- a/libutp/LICENSE
+++ /dev/null
@@ -1,19 +0,0 @@
-Copyright (c) 2010-2013 BitTorrent, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
diff --git a/libutp/Makefile b/libutp/Makefile
deleted file mode 100644
index 818f3095a..000000000
--- a/libutp/Makefile
+++ /dev/null
@@ -1,48 +0,0 @@
-OBJS     = utp_internal.o utp_utils.o utp_hash.o utp_callbacks.o utp_api.o utp_packedsockaddr.o
-CFLAGS   = -Wall -DPOSIX -g -fno-exceptions $(OPT)
-OPT ?= -O3
-CXXFLAGS = $(CFLAGS) -fPIC -fno-rtti
-CC       = gcc
-CXX      = g++
-
-CXXFLAGS += -Wno-sign-compare
-CXXFLAGS += -fpermissive
-
-# Uncomment to enable utp_get_stats(), and a few extra sanity checks
-#CFLAGS += -D_DEBUG
-
-# Uncomment to enable debug logging
-#CFLAGS += -DUTP_DEBUG_LOGGING
-
-# Dynamically determine if librt is available.  If so, assume we need to link
-# against it for clock_gettime(2).  This is required for clean builds on OSX;
-# see <https://github.com/bittorrent/libutp/issues/1> for more.  This should
-# probably be ported to CMake at some point, but is suitable for now.
-lrt := $(shell echo 'int main() {}' | $(CC) -xc -o /dev/null - -lrt >/dev/null 2>&1; echo $$?)
-ifeq ($(strip $(lrt)),0)
-  LDFLAGS += -lrt
-endif
-
-all: libutp.so libutp.a ucat ucat-static
-
-libutp.so: $(OBJS)
-	$(CXX) $(CXXFLAGS) -o libutp.so -shared $(OBJS)
-
-libutp.a: $(OBJS)
-	ar rvs libutp.a $(OBJS)
-
-ucat: ucat.o libutp.so
-	$(CC) $(CFLAGS) -o ucat ucat.o -L. -lutp $(LDFLAGS)
-
-ucat-static: ucat.o libutp.a
-	$(CXX) $(CXXFLAGS) -o ucat-static ucat.o libutp.a $(LDFLAGS)
-
-clean:
-	rm -f *.o libutp.so libutp.a ucat ucat-static
-
-tags: $(shell ls *.cpp *.h)
-	rm -f tags
-	ctags *.cpp *.h
-
-anyway: clean all
-.PHONY: clean all anyway
diff --git a/libutp/README.md b/libutp/README.md
deleted file mode 100644
index a8e20ecac..000000000
--- a/libutp/README.md
+++ /dev/null
@@ -1,68 +0,0 @@
-# libutp - The uTorrent Transport Protocol library.
-Copyright (c) 2010 BitTorrent, Inc.
-
-uTP is a TCP-like implementation of [LEDBAT][ledbat] documented as a BitTorrent
-extension in [BEP-29][bep29]. uTP provides reliable, ordered delivery
-while maintaining minimum extra delay. It is implemented on top of UDP to be
-cross-platform and functional today. As a result, uTP is the primary transport
-for uTorrent peer-to-peer connections.
-
-uTP is written in C++, but the external interface is strictly C (ANSI C89).
-
-## The Interface
-
-The uTP socket interface is a bit different from the Berkeley socket API to
-avoid the need for our own select() implementation, and to make it easier to
-write event-based code with minimal buffering.
-
-When you create a uTP socket, you register a set of callbacks. Most notably, the
-on_read callback is a reactive callback which occurs when bytes arrive off the
-network. The write side of the socket is proactive, and you call UTP_Write to
-indicate the number of bytes you wish to write. As packets are created, the
-on_write callback is called for each packet, so you can fill the buffers with
-data.
-
-The libutp interface is not thread-safe. It was designed for use in a
-single-threaded asyncronous context, although with proper synchronization
-it may be used from a multi-threaded environment as well.
-
-See utp.h for more details and other API documentation.
-
-## Example
-
-See ucat.c. Build with:
- 
-   make ucat
-
-## Building
-
-uTP has been known to build on Windows with MSVC and on linux and OS X with gcc.
-On Windows, use the MSVC project files (utp.sln, and friends). On other platforms,
-building the shared library is as simple as:
-
-    make
-
-To build one of the examples, which will statically link in everything it needs
-from libutp:
-
-    cd utp_test && make
-
-## Packaging and API
-
-The libutp API is considered unstable, and probably always will be. We encourage
-you to test with the version of libutp you have, and be mindful when upgrading.
-For this reason, it is probably also a good idea to bundle libutp with your
-application.
-
-## License
-
-libutp is released under the [MIT][lic] license.
-
-## Related Work
-
-Research and analysis of congestion control mechanisms can be found [here.][survey]
-
-[ledbat]: http://datatracker.ietf.org/wg/ledbat/charter/
-[bep29]: http://www.bittorrent.org/beps/bep_0029.html
-[lic]: http://www.opensource.org/licenses/mit-license.php
-[survey]: http://datatracker.ietf.org/doc/draft-ietf-ledbat-survey/
diff --git a/libutp/libutp_inet_ntop.cpp b/libutp/libutp_inet_ntop.cpp
deleted file mode 100644
index 1dec071af..000000000
--- a/libutp/libutp_inet_ntop.cpp
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <wspiapi.h>
-#include "libutp_inet_ntop.h"
-
-// we already have our own definition of these
-// -despair
-#if _WIN32_WINNT < 0x600
-namespace
-{
-  extern "C"
-  {
-    const char *
-    inet_ntop(int af, const void *src, char *dst, size_t size);
-    int
-    inet_pton(int af, const char *src, void *dst);
-  }
-}  // namespace
-#endif
-//######################################################################
-const char *
-libutp::inet_ntop(int af, const void *src, char *dest, size_t length)
-{
-  return ::inet_ntop(af, (void *)src, dest, length);
-}
-
-//######################################################################
-int
-libutp::inet_pton(int af, const char *src, void *dest)
-{
-  return ::inet_pton(af, src, dest);
-}
diff --git a/libutp/libutp_inet_ntop.h b/libutp/libutp_inet_ntop.h
deleted file mode 100644
index e110245be..000000000
--- a/libutp/libutp_inet_ntop.h
+++ /dev/null
@@ -1,63 +0,0 @@
-#ifndef LIBUTP_INET_NTOP_H
-#define LIBUTP_INET_NTOP_H
-
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-// About us linking the system inet_pton and inet_ntop symbols:
-// 1) These symbols are usually defined on POSIX systems
-// 2) They are not defined on Windows versions earlier than Vista
-// Defined in:
-// ut_utils/src/sockaddr.cpp
-// libutp/win32_inet_ntop.obj
-// 
-
-#if defined(_WIN32_WINNT)
-#if _WIN32_WINNT >= 0x600 // Win32, post-XP
-#include <ws2tcpip.h> // for inet_ntop, inet_pton
-#define INET_NTOP inet_ntop
-#define INET_PTON inet_pton
-#else
-#define INET_NTOP libutp::inet_ntop // Win32, pre-XP: Use ours
-#define INET_PTON libutp::inet_pton
-#endif
-#else // not WIN32
-#include <arpa/inet.h> // for inet_ntop, inet_pton
-#define INET_NTOP inet_ntop
-#define INET_PTON inet_pton
-#endif
-
-//######################################################################
-//######################################################################
-namespace libutp {
-
-
-//######################################################################
-const char *inet_ntop(int af, const void *src, char *dest, size_t length);
-
-//######################################################################
-int inet_pton(int af, const char* src, void* dest);
-
-
-} //namespace libutp
-
-#endif // LIBUTP_INET_NTOP_H
\ No newline at end of file
diff --git a/libutp/parse_log.py b/libutp/parse_log.py
deleted file mode 100644
index b4be9abfb..000000000
--- a/libutp/parse_log.py
+++ /dev/null
@@ -1,295 +0,0 @@
-import os
-import sys
-
-# usage: parse_log.py log-file [socket-index to focus on]
-
-
-socket_filter = None
-if len(sys.argv) >= 3:
-    socket_filter = sys.argv[2].strip()
-
-if socket_filter is None:
-    print "scanning for socket with the most packets"
-    file = open(sys.argv[1], 'rb')
-
-    sockets = {}
-
-    for l in file:
-        if not 'our_delay' in l:
-            continue
-
-        try:
-            a = l.strip().split(" ")
-            socket_index = a[1][:-1]
-        except:
-            continue
-
-        # msvc's runtime library doesn't prefix pointers
-        # with '0x'
-#        if socket_index[:2] != '0x':
-#            continue
-
-        if socket_index in sockets:
-            sockets[socket_index] += 1
-        else:
-            sockets[socket_index] = 1
-
-    items = sockets.items()
-    items.sort(lambda x, y: y[1] - x[1])
-
-    count = 0
-    for i in items:
-        print '%s: %d' % (i[0], i[1])
-        count += 1
-        if count > 5:
-            break
-
-    file.close()
-    socket_filter = items[0][0]
-    print '\nfocusing on socket %s' % socket_filter
-
-file = open(sys.argv[1], 'rb')
-out_file = 'utp.out%s' % socket_filter
-out = open(out_file, 'wb')
-
-delay_samples = 'dots lc rgb "blue"'
-delay_base = 'steps lw 2 lc rgb "purple"'
-target_delay = 'steps lw 2 lc rgb "red"'
-off_target = 'dots lc rgb "blue"'
-cwnd = 'steps lc rgb "green"'
-window_size = 'steps lc rgb "sea-green"'
-rtt = 'lines lc rgb "light-blue"'
-
-metrics = {
-    'our_delay': ['our delay (ms)', 'x1y2', delay_samples],
-    'upload_rate': ['send rate (B/s)', 'x1y1', 'lines'],
-    'max_window': ['cwnd (B)', 'x1y1', cwnd],
-    'target_delay': ['target delay (ms)', 'x1y2', target_delay],
-    'cur_window': ['bytes in-flight (B)', 'x1y1', window_size],
-    'cur_window_packets': ['number of packets in-flight', 'x1y2', 'steps'],
-    'packet_size': ['current packet size (B)', 'x1y2', 'steps'],
-    'rtt': ['rtt (ms)', 'x1y2', rtt],
-    'off_target': ['off-target (ms)', 'x1y2', off_target],
-    'delay_sum': ['delay sum (ms)', 'x1y2', 'steps'],
-    'their_delay': ['their delay (ms)', 'x1y2', delay_samples],
-    'get_microseconds': ['clock (us)', 'x1y1', 'steps'],
-    'wnduser': ['advertised window size (B)', 'x1y1', 'steps'],
-
-    'delay_base': ['delay base (us)', 'x1y1', delay_base],
-    'their_delay_base': ['their delay base (us)', 'x1y1', delay_base],
-    'their_actual_delay': ['their actual delay (us)', 'x1y1', delay_samples],
-    'actual_delay': ['actual_delay (us)', 'x1y1', delay_samples]
-}
-
-histogram_quantization = 1
-socket_index = None
-
-columns = []
-
-begin = None
-
-title = "-"
-packet_loss = 0
-packet_timeout = 0
-
-delay_histogram = {}
-window_size = {'0': 0, '1': 0}
-
-# [35301484] 0x00ec1190: actual_delay:1021583 our_delay:102 their_delay:-1021345 off_target:297 max_window:2687 upload_rate:18942 delay_base:1021481154 delay_sum:-1021242 target_delay:400 acked_bytes:1441 cur_window:2882 scaled_gain:2.432
-
-counter = 0
-
-print "reading log file"
-
-for l in file:
-    if "UTP_Connect" in l:
-        title = l[:-2]
-        if socket_filter != None:
-            title += ' socket: %s' % socket_filter
-        else:
-            title += ' sum of all sockets'
-        continue
-
-    try:
-        a = l.strip().split(" ")
-        t = a[0][1:-1]
-        socket_index = a[1][:-1]
-    except:
-        continue
-#    if socket_index[:2] != '0x':
-#        continue
-
-    if socket_filter != None and socket_index != socket_filter:
-        continue
-
-    counter += 1
-    if (counter % 300 == 0):
-        print "\r%d  " % counter,
-
-    if "lost." in l:
-        packet_loss = packet_loss + 1
-        continue
-    if "Packet timeout" in l:
-        packet_timeout = packet_timeout + 1
-        continue
-    if "our_delay:" not in l:
-        continue
-
-# used for Logf timestamps
-#    t, m = t.split(".")
-#    t = time.strptime(t, "%H:%M:%S")
-#    t = list(t)
-#    t[0] += 107
-#    t = tuple(t)
-#    m = float(m)
-#    m /= 1000.0
-#    t = time.mktime(t) + m
-
-# used for tick count timestamps
-    t = int(t)
-
-    if begin is None:
-        begin = t
-    t = t - begin
-    # print time. Convert from milliseconds to seconds
-    print >>out, '%f\t' % (float(t)/1000.),
-
-    # if t > 200000:
-    #    break
-
-    fill_columns = not columns
-    for i in a[2:]:
-        try:
-            n, v = i.split(':')
-        except:
-            continue
-        v = float(v)
-        if n == "our_delay":
-            bucket = v / histogram_quantization
-            delay_histogram[bucket] = 1 + delay_histogram.get(bucket, 0)
-        if not n in metrics:
-            continue
-        if fill_columns:
-            columns.append(n)
-        if n == "max_window":
-            window_size[socket_index] = v
-            print >>out, '%f\t' % int(
-                reduce(lambda a, b: a+b, window_size.values())),
-        else:
-            print >>out, '%f\t' % v,
-    print >>out, float(packet_loss * 8000), float(packet_timeout * 8000)
-    packet_loss = 0
-    packet_timeout = 0
-
-out.close()
-
-out = open('%s.histogram' % out_file, 'wb')
-for d, f in delay_histogram.iteritems():
-    print >>out, float(d*histogram_quantization) + \
-        histogram_quantization / 2, f
-out.close()
-
-
-plot = [
-    {
-        'data': ['upload_rate', 'max_window', 'cur_window', 'wnduser', 'cur_window_packets', 'packet_size', 'rtt'],
-        'title': 'send-packet-size',
-        'y1': 'Bytes',
-        'y2': 'Time (ms)'
-    },
-    {
-        'data': ['our_delay', 'max_window', 'target_delay', 'cur_window', 'wnduser', 'cur_window_packets'],
-        'title': 'uploading',
-        'y1': 'Bytes',
-        'y2': 'Time (ms)'
-    },
-    {
-        'data': ['our_delay', 'max_window', 'target_delay', 'cur_window', 'cur_window_packets'],
-        'title': 'uploading_packets',
-        'y1': 'Bytes',
-        'y2': 'Time (ms)'
-    },
-    {
-        'data': ['get_microseconds'],
-        'title': 'timer',
-        'y1': 'Time microseconds',
-        'y2': 'Time (ms)'
-    },
-    {
-        'data': ['their_delay', 'target_delay', 'rtt'],
-        'title': 'their_delay',
-        'y1': '',
-        'y2': 'Time (ms)'
-    },
-    {
-        'data': ['their_actual_delay', 'their_delay_base'],
-        'title': 'their_delay_base',
-        'y1': 'Time (us)',
-        'y2': ''
-    },
-    {
-        'data': ['our_delay', 'target_delay', 'rtt'],
-        'title': 'our-delay',
-        'y1': '',
-        'y2': 'Time (ms)'
-    },
-    {
-        'data': ['actual_delay', 'delay_base'],
-        'title': 'our_delay_base',
-        'y1': 'Time (us)',
-        'y2': ''
-    }
-]
-
-out = open('utp.gnuplot', 'w+')
-
-files = ''
-
-#print >>out, 'set xtics 0, 20'
-print >>out, "set term png size 1280,800"
-print >>out, 'set output "%s.delays.png"' % out_file
-print >>out, 'set xrange [0:250]'
-print >>out, 'set xlabel "delay (ms)"'
-print >>out, 'set boxwidth 1'
-print >>out, 'set style fill solid'
-print >>out, 'set ylabel "number of packets"'
-print >>out, 'plot "%s.histogram" using 1:2 with boxes' % out_file
-
-print >>out, "set style data steps"
-#print >>out, "set yrange [0:*]"
-print >>out, "set y2range [*:*]"
-files += out_file + '.delays.png '
-# set hidden3d
-# set title "Peer bandwidth distribution"
-# set xlabel "Ratio"
-
-for p in plot:
-    print >>out, 'set title "%s %s"' % (p['title'], title)
-    print >>out, 'set xlabel "time (s)"'
-    print >>out, 'set ylabel "%s"' % p['y1']
-    print >>out, "set tics nomirror"
-    print >>out, 'set y2tics'
-    print >>out, 'set y2label "%s"' % p['y2']
-    print >>out, 'set xrange [0:*]'
-    print >>out, "set key box"
-    print >>out, "set term png size 1280,800"
-    print >>out, 'set output "%s-%s.png"' % (out_file, p['title'])
-    files += '%s-%s.png ' % (out_file, p['title'])
-
-    comma = ''
-    print >>out, "plot",
-
-    for c in p['data']:
-        if not c in metrics:
-            continue
-        i = columns.index(c)
-        print >>out, '%s"%s" using 1:%d title "%s-%s" axes %s with %s' % (
-            comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]),
-        comma = ', '
-    print >>out, ''
-
-out.close()
-
-os.system("gnuplot utp.gnuplot")
-
-os.system("open %s" % files)
diff --git a/libutp/utp_api.cpp b/libutp/utp_api.cpp
deleted file mode 100644
index 63aff189c..000000000
--- a/libutp/utp_api.cpp
+++ /dev/null
@@ -1,139 +0,0 @@
-// vim:set ts=4 sw=4 ai:
-
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <stdio.h>
-#include "utp_internal.h"
-#include "utp_utils.h"
-
-extern "C" {
-
-const char * utp_callback_names[] = {
-	"UTP_ON_FIREWALL",
-	"UTP_ON_ACCEPT",
-	"UTP_ON_CONNECT",
-	"UTP_ON_ERROR",
-	"UTP_ON_READ",
-	"UTP_ON_OVERHEAD_STATISTICS",
-	"UTP_ON_STATE_CHANGE",
-	"UTP_GET_READ_BUFFER_SIZE",
-	"UTP_ON_DELAY_SAMPLE",
-	"UTP_GET_UDP_MTU",
-	"UTP_GET_UDP_OVERHEAD",
-	"UTP_GET_MILLISECONDS",
-	"UTP_GET_MICROSECONDS",
-	"UTP_GET_RANDOM",
-	"UTP_LOG",
-	"UTP_SENDTO",
-};
-
-const char * utp_error_code_names[] = {
-	"UTP_ECONNREFUSED",
-	"UTP_ECONNRESET",
-	"UTP_ETIMEDOUT",
-};
-
-const char *utp_state_names[] = {
-	NULL,
-	"UTP_STATE_CONNECT",
-	"UTP_STATE_WRITABLE",
-	"UTP_STATE_EOF",
-	"UTP_STATE_DESTROYING",
-};
-
-struct_utp_context::struct_utp_context()
-	: userdata(NULL)
-	, current_ms(0)
-	, last_utp_socket(NULL)
-	, log_normal(false)
-	, log_mtu(false)
-	, log_debug(false)
-{
-	memset(&context_stats, 0, sizeof(context_stats));
-	memset(callbacks, 0, sizeof(callbacks));
-	target_delay = CCONTROL_TARGET;
-	utp_sockets = new UTPSocketHT;
-
-	callbacks[UTP_GET_UDP_MTU]      = &utp_default_get_udp_mtu;
-	callbacks[UTP_GET_UDP_OVERHEAD] = &utp_default_get_udp_overhead;
-	callbacks[UTP_GET_MILLISECONDS] = &utp_default_get_milliseconds;
-	callbacks[UTP_GET_MICROSECONDS] = &utp_default_get_microseconds;
-	callbacks[UTP_GET_RANDOM]       = &utp_default_get_random;
-
-	// 1 MB of receive buffer (i.e. max bandwidth delay product)
-	// means that from  a peer with 200 ms RTT, we cannot receive
-	// faster than 5 MB/s
-	// from a peer with 10 ms RTT, we cannot receive faster than
-	// 100 MB/s. This is assumed to be good enough, since bandwidth
-	// often is proportional to RTT anyway
-	// when setting a download rate limit, all sockets should have
-	// their receive buffer set much lower, to say 60 kiB or so
-	opt_rcvbuf = opt_sndbuf = 1024 * 1024;
-	last_check = 0;
-}
-
-struct_utp_context::~struct_utp_context() {
-	delete this->utp_sockets;
-}
-
-utp_context* utp_init (int version)
-{
-	assert(version == 2);
-	if (version != 2)
-		return NULL;
-	utp_context *ctx = new utp_context;
-	return ctx;
-}
-
-void utp_destroy(utp_context *ctx) {
-	assert(ctx);
-	if (ctx) delete ctx;
-}
-
-void utp_set_callback(utp_context *ctx, int callback_name, utp_callback_t *proc) {
-	assert(ctx);
-	if (ctx) ctx->callbacks[callback_name] = proc;
-}
-
-void* utp_context_set_userdata(utp_context *ctx, void *userdata) {
-	assert(ctx);
-	if (ctx) ctx->userdata = userdata;
-	return ctx ? ctx->userdata : NULL;
-}
-
-void* utp_context_get_userdata(utp_context *ctx) {
-	assert(ctx);
-	return ctx ? ctx->userdata : NULL;
-}
-
-utp_context_stats* utp_get_context_stats(utp_context *ctx) {
-	assert(ctx);
-	return ctx ? &ctx->context_stats : NULL;
-}
-
-ssize_t utp_write(utp_socket *socket, void *buf, size_t len) {
-	struct utp_iovec iovec = { buf, len };
-	return utp_writev(socket, &iovec, 1);
-}
-
-}
diff --git a/libutp/utp_callbacks.cpp b/libutp/utp_callbacks.cpp
deleted file mode 100644
index 9540d8c40..000000000
--- a/libutp/utp_callbacks.cpp
+++ /dev/null
@@ -1,208 +0,0 @@
-// vim:set ts=4 sw=4 ai:
-
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "utp_callbacks.h"
-
-int utp_call_on_firewall(utp_context *ctx, const struct sockaddr *address, socklen_t address_len)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_FIREWALL]) return 0;
-	args.callback_type = UTP_ON_FIREWALL;
-	args.context = ctx;
-	args.socket = NULL;
-	args.address = address;
-	args.address_len = address_len;
-	return (int)ctx->callbacks[UTP_ON_FIREWALL](&args);
-}
-
-void utp_call_on_accept(utp_context *ctx, utp_socket *socket, const struct sockaddr *address, socklen_t address_len)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_ACCEPT]) return;
-	args.callback_type = UTP_ON_ACCEPT;
-	args.context = ctx;
-	args.socket = socket;
-	args.address = address;
-	args.address_len = address_len;
-	ctx->callbacks[UTP_ON_ACCEPT](&args);
-}
-
-void utp_call_on_connect(utp_context *ctx, utp_socket *socket)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_CONNECT]) return;
-	args.callback_type = UTP_ON_CONNECT;
-	args.context = ctx;
-	args.socket = socket;
-	ctx->callbacks[UTP_ON_CONNECT](&args);
-}
-
-void utp_call_on_error(utp_context *ctx, utp_socket *socket, int error_code)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_ERROR]) return;
-	args.callback_type = UTP_ON_ERROR;
-	args.context = ctx;
-	args.socket = socket;
-	args.error_code = error_code;
-	ctx->callbacks[UTP_ON_ERROR](&args);
-}
-
-void utp_call_on_read(utp_context *ctx, utp_socket *socket, const byte *buf, size_t len)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_READ]) return;
-	args.callback_type = UTP_ON_READ;
-	args.context = ctx;
-	args.socket = socket;
-	args.buf = buf;
-	args.len = len;
-	ctx->callbacks[UTP_ON_READ](&args);
-}
-
-void utp_call_on_overhead_statistics(utp_context *ctx, utp_socket *socket, int send, size_t len, int type)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_OVERHEAD_STATISTICS]) return;
-	args.callback_type = UTP_ON_OVERHEAD_STATISTICS;
-	args.context = ctx;
-	args.socket = socket;
-	args.send = send;
-	args.len = len;
-	args.type = type;
-	ctx->callbacks[UTP_ON_OVERHEAD_STATISTICS](&args);
-}
-
-void utp_call_on_delay_sample(utp_context *ctx, utp_socket *socket, int sample_ms)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_DELAY_SAMPLE]) return;
-	args.callback_type = UTP_ON_DELAY_SAMPLE;
-	args.context = ctx;
-	args.socket = socket;
-	args.sample_ms = sample_ms;
-	ctx->callbacks[UTP_ON_DELAY_SAMPLE](&args);
-}
-
-void utp_call_on_state_change(utp_context *ctx, utp_socket *socket, int state)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_ON_STATE_CHANGE]) return;
-	args.callback_type = UTP_ON_STATE_CHANGE;
-	args.context = ctx;
-	args.socket = socket;
-	args.state = state;
-	ctx->callbacks[UTP_ON_STATE_CHANGE](&args);
-}
-
-uint16 utp_call_get_udp_mtu(utp_context *ctx, utp_socket *socket, const struct sockaddr *address, socklen_t address_len)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_GET_UDP_MTU]) return 0;
-	args.callback_type = UTP_GET_UDP_MTU;
-	args.context = ctx;
-	args.socket = socket;
-	args.address = address;
-	args.address_len = address_len;
-	return (uint16)ctx->callbacks[UTP_GET_UDP_MTU](&args);
-}
-
-uint16 utp_call_get_udp_overhead(utp_context *ctx, utp_socket *socket, const struct sockaddr *address, socklen_t address_len)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_GET_UDP_OVERHEAD]) return 0;
-	args.callback_type = UTP_GET_UDP_OVERHEAD;
-	args.context = ctx;
-	args.socket = socket;
-	args.address = address;
-	args.address_len = address_len;
-	return (uint16)ctx->callbacks[UTP_GET_UDP_OVERHEAD](&args);
-}
-
-uint64 utp_call_get_milliseconds(utp_context *ctx, utp_socket *socket)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_GET_MILLISECONDS]) return 0;
-	args.callback_type = UTP_GET_MILLISECONDS;
-	args.context = ctx;
-	args.socket = socket;
-	return ctx->callbacks[UTP_GET_MILLISECONDS](&args);
-}
-
-uint64 utp_call_get_microseconds(utp_context *ctx, utp_socket *socket)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_GET_MICROSECONDS]) return 0;
-	args.callback_type = UTP_GET_MICROSECONDS;
-	args.context = ctx;
-	args.socket = socket;
-	return ctx->callbacks[UTP_GET_MICROSECONDS](&args);
-}
-
-uint32 utp_call_get_random(utp_context *ctx, utp_socket *socket)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_GET_RANDOM]) return 0;
-	args.callback_type = UTP_GET_RANDOM;
-	args.context = ctx;
-	args.socket = socket;
-	return (uint32)ctx->callbacks[UTP_GET_RANDOM](&args);
-}
-
-size_t utp_call_get_read_buffer_size(utp_context *ctx, utp_socket *socket)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_GET_READ_BUFFER_SIZE]) return 0;
-	args.callback_type = UTP_GET_READ_BUFFER_SIZE;
-	args.context = ctx;
-	args.socket = socket;
-	return (size_t)ctx->callbacks[UTP_GET_READ_BUFFER_SIZE](&args);
-}
-
-void utp_call_log(utp_context *ctx, utp_socket *socket, const byte *buf)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_LOG]) return;
-	args.callback_type = UTP_LOG;
-	args.context = ctx;
-	args.socket = socket;
-	args.buf = buf;
-	ctx->callbacks[UTP_LOG](&args);
-}
-
-void utp_call_sendto(utp_context *ctx, utp_socket *socket, const byte *buf, size_t len, const struct sockaddr *address, socklen_t address_len, uint32 flags)
-{
-	utp_callback_arguments args;
-	if (!ctx->callbacks[UTP_SENDTO]) return;
-	args.callback_type = UTP_SENDTO;
-	args.context = ctx;
-	args.socket = socket;
-	args.buf = buf;
-	args.len = len;
-	args.address = address;
-	args.address_len = address_len;
-	args.flags = flags;
-	ctx->callbacks[UTP_SENDTO](&args);
-}
-
diff --git a/libutp/utp_callbacks.h b/libutp/utp_callbacks.h
deleted file mode 100644
index 649e7e14f..000000000
--- a/libutp/utp_callbacks.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef __UTP_CALLBACKS_H__
-#define __UTP_CALLBACKS_H__
-
-#include "utp.h"
-#include "utp_internal.h"
-
-// Generated by running:  grep ^[a-z] utp_callbacks.cpp | sed 's/$/;/'
-int utp_call_on_firewall(utp_context *ctx, const struct sockaddr *address, socklen_t address_len);
-void utp_call_on_accept(utp_context *ctx, utp_socket *s, const struct sockaddr *address, socklen_t address_len);
-void utp_call_on_connect(utp_context *ctx, utp_socket *s);
-void utp_call_on_error(utp_context *ctx, utp_socket *s, int error_code);
-void utp_call_on_read(utp_context *ctx, utp_socket *s, const byte *buf, size_t len);
-void utp_call_on_overhead_statistics(utp_context *ctx, utp_socket *s, int send, size_t len, int type);
-void utp_call_on_delay_sample(utp_context *ctx, utp_socket *s, int sample_ms);
-void utp_call_on_state_change(utp_context *ctx, utp_socket *s, int state);
-uint16 utp_call_get_udp_mtu(utp_context *ctx, utp_socket *s, const struct sockaddr *address, socklen_t address_len);
-uint16 utp_call_get_udp_overhead(utp_context *ctx, utp_socket *s, const struct sockaddr *address, socklen_t address_len);
-uint64 utp_call_get_milliseconds(utp_context *ctx, utp_socket *s);
-uint64 utp_call_get_microseconds(utp_context *ctx, utp_socket *s);
-uint32 utp_call_get_random(utp_context *ctx, utp_socket *s);
-size_t utp_call_get_read_buffer_size(utp_context *ctx, utp_socket *s);
-void utp_call_log(utp_context *ctx, utp_socket *s, const byte *buf);
-void utp_call_sendto(utp_context *ctx, utp_socket *s, const byte *buf, size_t len, const struct sockaddr *address, socklen_t address_len, uint32 flags);
-
-#endif // __UTP_CALLBACKS_H__
diff --git a/libutp/utp_hash.cpp b/libutp/utp_hash.cpp
deleted file mode 100644
index a4a71d906..000000000
--- a/libutp/utp_hash.cpp
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "utp_hash.h"
-#include "utp_types.h"
-
-#define LIBUTP_HASH_UNUSED ((utp_link_t)-1)
-
-#ifdef STRICT_ALIGN
-inline uint32 Read32(const void *p)
-{
-	uint32 tmp;
-	memcpy(&tmp, p, sizeof tmp);
-	return tmp;
-}
-
-#else
-inline uint32 Read32(const void *p) { return *(uint32*)p; }
-#endif
-
-
-// Get the amount of memory required for the hash parameters and the bucket set
-// Waste a space for an unused bucket in order to ensure the following managed memory have 32-bit aligned addresses
-// TODO:  make this 64-bit clean
-#define BASE_SIZE(bc) (sizeof(utp_hash_t) + sizeof(utp_link_t) * ((bc) + 1))
-
-// Get a pointer to the base of the structure array managed by the hash table
-#define get_bep(h) ((byte*)(h)) + BASE_SIZE((h)->N)
-
-// Get the address of the information associated with a specific structure in the array,
-// given the address of the base of the structure.
-// This assumes a utp_link_t link member is at the end of the structure.
-// Given compilers filling out the memory to a 32-bit clean value, this may mean that
-// the location named in the structure may not be the location actually used by the hash table,
-// since the compiler may have padded the end of the structure with 2 bytes after the utp_link_t member.
-// TODO: this macro should not require that the variable pointing at the hash table be named 'hash'
-#define ptr_to_link(p) (utp_link_t *) (((byte *) (p)) + hash->E - sizeof(utp_link_t))
-
-// Calculate how much to allocate for a hash table with bucket count, total size, and structure count
-// TODO:  make this 64-bit clean
-#define ALLOCATION_SIZE(bc, ts, sc) (BASE_SIZE((bc)) + (ts) * (sc))
-
-utp_hash_t *utp_hash_create(int N, int key_size, int total_size, int initial, utp_hash_compute_t hashfun, utp_hash_equal_t compfun)
-{
-	// Must have odd number of hash buckets (prime number is best)
-	assert(N % 2);
-	// Ensure structures will be at aligned memory addresses
-	// TODO:  make this 64-bit clean
-	assert(0 == (total_size % 4));
-
-	int size = ALLOCATION_SIZE(N, total_size, initial);
-	utp_hash_t *hash = (utp_hash_t *) malloc( size );
-	memset( hash, 0, size );
-
-	for (int i = 0; i < N + 1; ++i)
-		hash->inits[i] = LIBUTP_HASH_UNUSED;
-	hash->N = N;
-	hash->K = key_size;
-	hash->E = total_size;
-	hash->hash_compute = hashfun;
-	hash->hash_equal = compfun;
-	hash->allocated = initial;
-	hash->count = 0;
-	hash->used = 0;
-	hash->free = LIBUTP_HASH_UNUSED;
-	return hash;
-}
-
-uint utp_hash_mem(const void *keyp, size_t keysize)
-{
-	uint hash = 0;
-	uint n = keysize;
-	while (n >= 4) {
-		hash ^= Read32(keyp);
-		keyp = (byte*)keyp + sizeof(uint32);
-		hash = (hash << 13) | (hash >> 19);
-		n -= 4;
-	}
-	while (n != 0) {
-		hash ^= *(byte*)keyp;
-		keyp = (byte*)keyp + sizeof(byte);
-		hash = (hash << 8) | (hash >> 24);
-		n--;
-	}
-	return hash;
-}
-
-uint utp_hash_mkidx(utp_hash_t *hash, const void *keyp)
-{
-	// Generate a key from the hash
-	return hash->hash_compute(keyp, hash->K) % hash->N;
-}
-
-static inline bool compare(byte *a, byte *b,int n)
-{
-	assert(n >= 4);
-	if (Read32(a) != Read32(b)) return false;
-	return memcmp(a+4, b+4, n-4) == 0;
-}
-
-#define COMPARE(h,k1,k2,ks) (((h)->hash_equal) ? (h)->hash_equal((void*)k1,(void*)k2,ks) : compare(k1,k2,ks))
-
-// Look-up a key in the hash table.
-// Returns NULL if not found
-void *utp_hash_lookup(utp_hash_t *hash, const void *key)
-{
-	utp_link_t idx = utp_hash_mkidx(hash, key);
-
-	// base pointer
-	byte *bep = get_bep(hash);
-
-	utp_link_t cur = hash->inits[idx];
-	while (cur != LIBUTP_HASH_UNUSED) {
-		byte *key2 = bep + (cur * hash->E);
-		if (COMPARE(hash, (byte*)key, key2, hash->K))
-			return key2;
-		cur = *ptr_to_link(key2);
-	}
-
-	return NULL;
-}
-
-// Add a new element to the hash table.
-// Returns a pointer to the new element.
-// This assumes the element is not already present!
-void *utp_hash_add(utp_hash_t **hashp, const void *key)
-{
-	//Allocate a new entry
-	byte *elemp;
-	utp_link_t elem;
-	utp_hash_t *hash = *hashp;
-	utp_link_t idx = utp_hash_mkidx(hash, key);
-
-	if ((elem=hash->free) == LIBUTP_HASH_UNUSED) {
-		utp_link_t all = hash->allocated;
-		if (hash->used == all) {
-			utp_hash_t *nhash;
-			if (all <= (LIBUTP_HASH_UNUSED/2)) {
-				all *= 2;
-			} else if (all != LIBUTP_HASH_UNUSED) {
-				all  = LIBUTP_HASH_UNUSED;
-			} else {
-				// too many items! can't grow!
-				assert(0);
-				return NULL;
-			}
-			// otherwise need to allocate.
-			nhash = (utp_hash_t*)realloc(hash, ALLOCATION_SIZE(hash->N, hash->E, all));
-			if (!nhash) {
-				// out of memory (or too big to allocate)
-				assert(nhash);
-				return NULL;
-			}
-			hash = *hashp = nhash;
-			hash->allocated = all;
-		}
-
-		elem = hash->used++;
-		elemp = get_bep(hash) + elem * hash->E;
-	} else {
-		elemp = get_bep(hash) + elem * hash->E;
-		hash->free = *ptr_to_link(elemp);
-	}
-
-	*ptr_to_link(elemp) = hash->inits[idx];
-	hash->inits[idx] = elem;
-	hash->count++;
-
-	// copy key into it
-	memcpy(elemp, key, hash->K);
-	return elemp;
-}
-
-// Delete an element from the utp_hash_t
-// Returns a pointer to the already deleted element.
-void *utp_hash_del(utp_hash_t *hash, const void *key)
-{
-	utp_link_t idx = utp_hash_mkidx(hash, key);
-
-	// base pointer
-	byte *bep = get_bep(hash);
-
-	utp_link_t *curp = &hash->inits[idx];
-	utp_link_t cur;
-	while ((cur=*curp) != LIBUTP_HASH_UNUSED) {
-		byte *key2 = bep + (cur * hash->E);
-		if (COMPARE(hash,(byte*)key,(byte*)key2, hash->K )) {
-			// found an item that matched. unlink it
-			*curp = *ptr_to_link(key2);
-			// Insert into freelist
-			*ptr_to_link(key2) = hash->free;
-			hash->free = cur;
-			hash->count--;
-			return key2;
-		}
-		curp = ptr_to_link(key2);
-	}
-
-	return NULL;
-}
-
-void *utp_hash_iterate(utp_hash_t *hash, utp_hash_iterator_t *iter)
-{
-	utp_link_t elem;
-
-	if ((elem=iter->elem) == LIBUTP_HASH_UNUSED) {
-		// Find a bucket with an element
-		utp_link_t buck = iter->bucket + 1;
-		for(;;) {
-			if (buck >= hash->N)
-				return NULL;
-			if ((elem = hash->inits[buck]) != LIBUTP_HASH_UNUSED)
-				break;
-			buck++;
-		}
-		iter->bucket = buck;
-	}
-
-	byte *elemp = get_bep(hash) + (elem * hash->E);
-	iter->elem = *ptr_to_link(elemp);
-	return elemp;
-}
-
-void utp_hash_free_mem(utp_hash_t* hash)
-{
-	free(hash);
-}
diff --git a/libutp/utp_hash.h b/libutp/utp_hash.h
deleted file mode 100644
index b33acbdae..000000000
--- a/libutp/utp_hash.h
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef __UTP_HASH_H__
-#define __UTP_HASH_H__
-
-#include <string.h>  // memset
-#include <stdlib.h>  // malloc
-
-#include "utp_types.h"
-#include "utp_templates.h"
-
-// TODO: make utp_link_t a template parameter to HashTable
-typedef uint32 utp_link_t;
-
-#ifdef _MSC_VER
-// Silence the warning about the C99-compliant zero-length array at the end of
-// the structure
-#pragma warning(disable : 4200)
-#endif
-
-#include <absl/base/attributes.h>
-
-typedef uint32 (*utp_hash_compute_t)(const void *keyp, size_t keysize);
-typedef uint (*utp_hash_equal_t)(const void *key_a, const void *key_b,
-                                 size_t keysize);
-
-// In memory the HashTable is laid out as follows:
-//  ---------------------------- low
-//  | hash table data members  |
-//  ----------------------------  _
-//  |  indices                 |  ^
-//  |  .                       |  |  utp_link_t indices into the key-values.
-//  |  .                       |  .
-//  ----------------------------  -  <----- bep
-//  |  keys and values         |     each key-value pair has size total_size
-//  |  .                       |
-//  |  .                       |
-//  ---------------------------- high
-//
-// The code depends on the ability of the compiler to pad the length
-// of the hash table data members structure to
-// a length divisible by 32-bits with no remainder.
-//
-// Since the number of hash buckets (indices) should be odd, the code
-// asserts this and adds one to the hash bucket count to ensure that the
-// following key-value pairs array starts on a 32-bit boundary.
-//
-// The key-value pairs array should start on a 32-bit boundary, otherwise
-// processors like the ARM will silently mangle 32-bit data in these structures
-// (e.g., turning 0xABCD into 0XCDAB when moving a value from memory to register
-// when the memory address is 16 bits offset from a 32-bit boundary),
-// also, the value will be stored at an address two bytes lower than the address
-// value would ordinarily indicate.
-//
-// The key-value pair is of type T. The first field in T must
-// be the key, i.e., the first K bytes of T contains the key.
-// total_size = sizeof(T) and thus sizeof(T) >= sizeof(K)
-//
-// N is the number of buckets.
-//
-struct utp_hash_t
-{
-  utp_link_t N;
-  byte K;
-  byte E;
-  size_t count;
-  utp_hash_compute_t hash_compute;
-  utp_hash_equal_t hash_equal;
-  utp_link_t allocated;
-  utp_link_t used;
-  utp_link_t free;
-  utp_link_t inits[0];
-};
-
-#ifdef _MSC_VER
-#pragma warning(default : 4200)
-#endif
-
-struct utp_hash_iterator_t
-{
-  utp_link_t bucket;
-  utp_link_t elem;
-
-  utp_hash_iterator_t() : bucket(0xffffffff), elem(0xffffffff)
-  {
-  }
-};
-
-uint
-utp_hash_mem(const void *keyp, size_t keysize);
-uint
-utp_hash_comp(const void *key_a, const void *key_b, size_t keysize);
-
-utp_hash_t *
-utp_hash_create(int N, int key_size, int total_size, int initial,
-                utp_hash_compute_t hashfun = utp_hash_mem,
-                utp_hash_equal_t eqfun     = NULL);
-void *
-utp_hash_lookup(utp_hash_t *hash, const void *key);
-void *
-utp_hash_add(utp_hash_t **hashp, const void *key);
-void *
-utp_hash_del(utp_hash_t *hash, const void *key);
-
-void *
-utp_hash_iterate(utp_hash_t *hash, utp_hash_iterator_t *iter);
-void
-utp_hash_free_mem(utp_hash_t *hash);
-
-/*
-        This HashTable requires that T have at least
-   sizeof(K)+sizeof(utp_link_t) bytes. Usually done like this:
-
-        struct K {
-                int whatever;
-        };
-
-        struct T {
-                K wtf;
-                utp_link_t link; // also wtf
-        };
-*/
-
-template < typename K, typename T >
-class utpHashTable
-{
-  utp_hash_t *hash;
-
- public:
-  static uint
-  compare(const void *k1, const void *k2, ABSL_ATTRIBUTE_UNUSED size_t ks)
-  {
-    return *((K *)k1) == *((K *)k2);
-  }
-  static uint32
-  compute_hash(const void *k, ABSL_ATTRIBUTE_UNUSED size_t ks)
-  {
-    return ((K *)k)->compute_hash();
-  }
-  void
-  Init()
-  {
-    hash = NULL;
-  }
-  bool
-  Allocated()
-  {
-    return (hash != NULL);
-  }
-  void
-  Free()
-  {
-    utp_hash_free_mem(hash);
-    hash = NULL;
-  }
-  void
-  Create(int N, int initial)
-  {
-    hash = utp_hash_create(N, sizeof(K), sizeof(T), initial, &compute_hash,
-                           &compare);
-  }
-  T *
-  Lookup(const K &key)
-  {
-    return (T *)utp_hash_lookup(hash, &key);
-  }
-  T *
-  Add(const K &key)
-  {
-    return (T *)utp_hash_add(&hash, &key);
-  }
-  T *
-  Delete(const K &key)
-  {
-    return (T *)utp_hash_del(hash, &key);
-  }
-  T *
-  Iterate(utp_hash_iterator_t &iterator)
-  {
-    return (T *)utp_hash_iterate(hash, &iterator);
-  }
-  size_t
-  GetCount()
-  {
-    return hash->count;
-  }
-};
-
-#endif  //__UTP_HASH_H__
diff --git a/libutp/utp_internal.cpp b/libutp/utp_internal.cpp
deleted file mode 100644
index 1529fce64..000000000
--- a/libutp/utp_internal.cpp
+++ /dev/null
@@ -1,4067 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <stdio.h>
-#include <assert.h>
-#include <string.h>
-#include <string.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <limits.h>  // for UINT_MAX
-#include <time.h>
-
-#include "utp_types.h"
-#include "utp_packedsockaddr.h"
-#include "utp_internal.h"
-#include "utp_hash.h"
-
-#define TIMEOUT_CHECK_INTERVAL 500
-
-// number of bytes to increase max window size by, per RTT. This is
-// scaled down linearly proportional to off_target. i.e. if all packets
-// in one window have 0 delay, window size will increase by this number.
-// Typically it's less. TCP increases one MSS per RTT, which is 1500
-#define MAX_CWND_INCREASE_BYTES_PER_RTT 3000
-#define CUR_DELAY_SIZE 3
-// experiments suggest that a clock skew of 10 ms per 325 seconds
-// is not impossible. Reset delay_base every 13 minutes. The clock
-// skew is dealt with by observing the delay base in the other
-// direction, and adjusting our own upwards if the opposite direction
-// delay base keeps going down
-#define DELAY_BASE_HISTORY 13
-#define MAX_WINDOW_DECAY 100  // ms
-
-#define REORDER_BUFFER_SIZE 32
-#define REORDER_BUFFER_MAX_SIZE 1024
-#define OUTGOING_BUFFER_MAX_SIZE 1024
-
-#define PACKET_SIZE 1435
-
-// this is the minimum max_window value. It can never drop below this
-#define MIN_WINDOW_SIZE 10
-
-// if we receive 4 or more duplicate acks, we resend the packet
-// that hasn't been acked yet
-#define DUPLICATE_ACKS_BEFORE_RESEND 3
-
-// Allow a reception window of at least 3 ack_nrs behind seq_nr
-// A non-SYN packet with an ack_nr difference greater than this is
-// considered suspicious and ignored
-#define ACK_NR_ALLOWED_WINDOW DUPLICATE_ACKS_BEFORE_RESEND
-
-#define RST_INFO_TIMEOUT 10000
-#define RST_INFO_LIMIT 1000
-// 29 seconds determined from measuring many home NAT devices
-#define KEEPALIVE_INTERVAL 29000
-
-#define SEQ_NR_MASK 0xFFFF
-#define ACK_NR_MASK 0xFFFF
-#define TIMESTAMP_MASK 0xFFFFFFFF
-
-#define DIV_ROUND_UP(num, denom) ((num + denom - 1) / denom)
-
-// The totals are derived from the following data:
-//  45: IPv6 address including embedded IPv4 address
-//  11: Scope Id
-//   2: Brackets around IPv6 address when port is present
-//   6: Port (including colon)
-//   1: Terminating null byte
-char addrbuf[65];
-#define addrfmt(x, s) x.fmt(s, sizeof(s))
-
-#if(defined(__SVR4) && defined(__sun))
-#pragma pack(1)
-#else
-#pragma pack(push, 1)
-#endif
-
-// these packet sizes are including the uTP header wich
-// is either 20 or 23 bytes depending on version
-#define PACKET_SIZE_EMPTY_BUCKET 0
-#define PACKET_SIZE_EMPTY 23
-#define PACKET_SIZE_SMALL_BUCKET 1
-#define PACKET_SIZE_SMALL 373
-#define PACKET_SIZE_MID_BUCKET 2
-#define PACKET_SIZE_MID 723
-#define PACKET_SIZE_BIG_BUCKET 3
-#define PACKET_SIZE_BIG 1400
-#define PACKET_SIZE_HUGE_BUCKET 4
-
-struct PACKED_ATTRIBUTE PacketFormatV1
-{
-  // packet_type (4 high bits)
-  // protocol version (4 low bits)
-  byte ver_type;
-  byte
-  version() const
-  {
-    return ver_type & 0xf;
-  }
-  byte
-  type() const
-  {
-    return ver_type >> 4;
-  }
-  void
-  set_version(byte v)
-  {
-    ver_type = (ver_type & 0xf0) | (v & 0xf);
-  }
-  void
-  set_type(byte t)
-  {
-    ver_type = (ver_type & 0xf) | (t << 4);
-  }
-
-  // Type of the first extension header
-  byte ext;
-  // connection ID
-  uint16_big connid;
-  uint32_big tv_usec;
-  uint32_big reply_micro;
-  // receive window size in bytes
-  uint32_big windowsize;
-  // Sequence number
-  uint16_big seq_nr;
-  // Acknowledgment number
-  uint16_big ack_nr;
-};
-
-struct PACKED_ATTRIBUTE PacketFormatAckV1
-{
-  PacketFormatV1 pf;
-  byte ext_next;
-  byte ext_len;
-  byte acks[4];
-};
-
-#if(defined(__SVR4) && defined(__sun))
-#pragma pack(0)
-#else
-#pragma pack(pop)
-#endif
-
-enum
-{
-  ST_DATA  = 0,   // Data packet.
-  ST_FIN   = 1,   // Finalize the connection. This is the last packet.
-  ST_STATE = 2,   // State packet. Used to transmit an ACK with no data.
-  ST_RESET = 3,   // Terminate connection forcefully.
-  ST_SYN   = 4,   // Connect SYN
-  ST_NUM_STATES,  // used for bounds checking
-};
-
-enum CONN_STATE
-{
-  CS_UNINITIALIZED = 0,
-  CS_IDLE,
-  CS_SYN_SENT,
-  CS_SYN_RECV,
-  CS_CONNECTED,
-  CS_CONNECTED_FULL,
-  CS_RESET,
-  CS_DESTROY
-};
-
-#if UTP_DEBUG_LOGGING
-static const cstr flagnames[] = {"ST_DATA", "ST_FIN", "ST_STATE", "ST_RESET",
-                                 "ST_SYN"};
-
-static const cstr statenames[] = {
-    "UNINITIALIZED",  "IDLE",          "SYN_SENT", "SYN_RECV", "CONNECTED",
-    "CONNECTED_FULL", "DESTROY_DELAY", "RESET",    "DESTROY"};
-
-#endif
-
-struct OutgoingPacket
-{
-  size_t length;
-  size_t payload;
-  uint64 time_sent;  // microseconds
-  uint transmissions : 31;
-  bool need_resend : 1;
-  byte data[1];
-};
-
-struct SizableCircularBuffer
-{
-  // This is the mask. Since it's always a power of 2, adding 1 to this value
-  // will return the size.
-  size_t mask;
-  // This is the elements that the circular buffer points to
-  void **elements;
-
-  void *
-  get(size_t i) const
-  {
-    assert(elements);
-    return elements ? elements[i & mask] : NULL;
-  }
-  void
-  put(size_t i, void *data)
-  {
-    assert(elements);
-    elements[i & mask] = data;
-  }
-
-  void
-  grow(size_t item, size_t index);
-  void
-  ensure_size(size_t item, size_t index)
-  {
-    if(index > mask)
-      grow(item, index);
-  }
-  size_t
-  size()
-  {
-    return mask + 1;
-  }
-};
-
-// Item contains the element we want to make space for
-// index is the index in the list.
-void
-SizableCircularBuffer::grow(size_t item, size_t index)
-{
-  // Figure out the new size.
-  size_t size = mask + 1;
-  do
-    size *= 2;
-  while(index >= size);
-
-  // Allocate the new buffer
-  void **buf = (void **)calloc(size, sizeof(void *));
-
-  size--;
-
-  // Copy elements from the old buffer to the new buffer
-  for(size_t i = 0; i <= mask; i++)
-  {
-    buf[(item - index + i) & size] = get(item - index + i);
-  }
-
-  // Swap to the newly allocated buffer
-  mask = size;
-  free(elements);
-  elements = buf;
-}
-
-// compare if lhs is less than rhs, taking wrapping
-// into account. if lhs is close to UINT_MAX and rhs
-// is close to 0, lhs is assumed to have wrapped and
-// considered smaller
-bool
-wrapping_compare_less(uint32 lhs, uint32 rhs, uint32 mask)
-{
-  // distance walking from lhs to rhs, downwards
-  const uint32 dist_down = (lhs - rhs) & mask;
-  // distance walking from lhs to rhs, upwards
-  const uint32 dist_up = (rhs - lhs) & mask;
-
-  // if the distance walking up is shorter, lhs
-  // is less than rhs. If the distance walking down
-  // is shorter, then rhs is less than lhs
-  return dist_up < dist_down;
-}
-
-struct DelayHist
-{
-  uint32 delay_base;
-
-  // this is the history of delay samples,
-  // normalized by using the delay_base. These
-  // values are always greater than 0 and measures
-  // the queuing delay in microseconds
-  uint32 cur_delay_hist[CUR_DELAY_SIZE];
-  size_t cur_delay_idx;
-
-  // this is the history of delay_base. It's
-  // a number that doesn't have an absolute meaning
-  // only relative. It doesn't make sense to initialize
-  // it to anything other than values relative to
-  // what's been seen in the real world.
-  uint32 delay_base_hist[DELAY_BASE_HISTORY];
-  size_t delay_base_idx;
-  // the time when we last stepped the delay_base_idx
-  uint64 delay_base_time;
-
-  bool delay_base_initialized;
-
-  void
-  clear(uint64 current_ms)
-  {
-    delay_base_initialized = false;
-    delay_base             = 0;
-    cur_delay_idx          = 0;
-    delay_base_idx         = 0;
-    delay_base_time        = current_ms;
-    for(size_t i = 0; i < CUR_DELAY_SIZE; i++)
-    {
-      cur_delay_hist[i] = 0;
-    }
-    for(size_t i = 0; i < DELAY_BASE_HISTORY; i++)
-    {
-      delay_base_hist[i] = 0;
-    }
-  }
-
-  void
-  shift(const uint32 offset)
-  {
-    // the offset should never be "negative"
-    // assert(offset < 0x10000000);
-
-    // increase all of our base delays by this amount
-    // this is used to take clock skew into account
-    // by observing the other side's changes in its base_delay
-    for(size_t i = 0; i < DELAY_BASE_HISTORY; i++)
-    {
-      delay_base_hist[i] += offset;
-    }
-    delay_base += offset;
-  }
-
-  void
-  add_sample(const uint32 sample, uint64 current_ms)
-  {
-    // The two clocks (in the two peers) are assumed not to
-    // progress at the exact same rate. They are assumed to be
-    // drifting, which causes the delay samples to contain
-    // a systematic error, either they are under-
-    // estimated or over-estimated. This is why we update the
-    // delay_base every two minutes, to adjust for this.
-
-    // This means the values will keep drifting and eventually wrap.
-    // We can cross the wrapping boundry in two directions, either
-    // going up, crossing the highest value, or going down, crossing 0.
-
-    // if the delay_base is close to the max value and sample actually
-    // wrapped on the other end we would see something like this:
-    // delay_base = 0xffffff00, sample = 0x00000400
-    // sample - delay_base = 0x500 which is the correct difference
-
-    // if the delay_base is instead close to 0, and we got an even lower
-    // sample (that will eventually update the delay_base), we may see
-    // something like this:
-    // delay_base = 0x00000400, sample = 0xffffff00
-    // sample - delay_base = 0xfffffb00
-    // this needs to be interpreted as a negative number and the actual
-    // recorded delay should be 0.
-
-    // It is important that all arithmetic that assume wrapping
-    // is done with unsigned intergers. Signed integers are not guaranteed
-    // to wrap the way unsigned integers do. At least GCC takes advantage
-    // of this relaxed rule and won't necessarily wrap signed ints.
-
-    // remove the clock offset and propagation delay.
-    // delay base is min of the sample and the current
-    // delay base. This min-operation is subject to wrapping
-    // and care needs to be taken to correctly choose the
-    // true minimum.
-
-    // specifically the problem case is when delay_base is very small
-    // and sample is very large (because it wrapped past zero), sample
-    // needs to be considered the smaller
-
-    if(!delay_base_initialized)
-    {
-      // delay_base being 0 suggests that we haven't initialized
-      // it or its history with any real measurements yet. Initialize
-      // everything with this sample.
-      for(size_t i = 0; i < DELAY_BASE_HISTORY; i++)
-      {
-        // if we don't have a value, set it to the current sample
-        delay_base_hist[i] = sample;
-        continue;
-      }
-      delay_base             = sample;
-      delay_base_initialized = true;
-    }
-
-    if(wrapping_compare_less(sample, delay_base_hist[delay_base_idx],
-                             TIMESTAMP_MASK))
-    {
-      // sample is smaller than the current delay_base_hist entry
-      // update it
-      delay_base_hist[delay_base_idx] = sample;
-    }
-
-    // is sample lower than delay_base? If so, update delay_base
-    if(wrapping_compare_less(sample, delay_base, TIMESTAMP_MASK))
-    {
-      // sample is smaller than the current delay_base
-      // update it
-      delay_base = sample;
-    }
-
-    // this operation may wrap, and is supposed to
-    const uint32 delay = sample - delay_base;
-    // sanity check. If this is triggered, something fishy is going on
-    // it means the measured sample was greater than 32 seconds!
-    // assert(delay < 0x2000000);
-
-    cur_delay_hist[cur_delay_idx] = delay;
-    cur_delay_idx                 = (cur_delay_idx + 1) % CUR_DELAY_SIZE;
-
-    // once every minute
-    if(current_ms - delay_base_time > 60 * 1000)
-    {
-      delay_base_time = current_ms;
-      delay_base_idx  = (delay_base_idx + 1) % DELAY_BASE_HISTORY;
-      // clear up the new delay base history spot by initializing
-      // it to the current sample, then update it
-      delay_base_hist[delay_base_idx] = sample;
-      delay_base                      = delay_base_hist[0];
-      // Assign the lowest delay in the last 2 minutes to delay_base
-      for(size_t i = 0; i < DELAY_BASE_HISTORY; i++)
-      {
-        if(wrapping_compare_less(delay_base_hist[i], delay_base,
-                                 TIMESTAMP_MASK))
-          delay_base = delay_base_hist[i];
-      }
-    }
-  }
-
-  uint32
-  get_value()
-  {
-    uint32 value = UINT_MAX;
-    for(size_t i = 0; i < CUR_DELAY_SIZE; i++)
-    {
-      value = min< uint32 >(cur_delay_hist[i], value);
-    }
-    // value could be UINT_MAX if we have no samples yet...
-    return value;
-  }
-};
-
-struct UTPSocket
-{
-  ~UTPSocket();
-
-  PackedSockAddr addr;
-  utp_context *ctx;
-
-  int ida;  // for ack socket list
-
-  uint16 retransmit_count;
-
-  uint16 reorder_count;
-  byte duplicate_ack;
-
-  // the number of packets in the send queue. Packets that haven't
-  // yet been sent count as well as packets marked as needing resend
-  // the oldest un-acked packet in the send queue is seq_nr - cur_window_packets
-  uint16 cur_window_packets;
-
-  // how much of the window is used, number of bytes in-flight
-  // packets that have not yet been sent do not count, packets
-  // that are marked as needing to be re-sent (due to a timeout)
-  // don't count either
-  size_t cur_window;
-  // maximum window size, in bytes
-  size_t max_window;
-  // UTP_SNDBUF setting, in bytes
-  size_t opt_sndbuf;
-  // UTP_RCVBUF setting, in bytes
-  size_t opt_rcvbuf;
-
-  // this is the target delay, in microseconds
-  // for this socket. defaults to 100000.
-  size_t target_delay;
-
-  // Is a FIN packet in the reassembly buffer?
-  bool got_fin : 1;
-  // Have we reached the FIN?
-  bool got_fin_reached : 1;
-
-  // Have we sent our FIN?
-  bool fin_sent : 1;
-  // Has our fin been ACKed?
-  bool fin_sent_acked : 1;
-
-  // Reading is disabled
-  bool read_shutdown : 1;
-  // User called utp_close()
-  bool close_requested : 1;
-
-  // Timeout procedure
-  bool fast_timeout : 1;
-
-  // max receive window for other end, in bytes
-  size_t max_window_user;
-  CONN_STATE state;
-  // TickCount when we last decayed window (wraps)
-  int64 last_rwin_decay;
-
-  // the sequence number of the FIN packet. This field is only set
-  // when we have received a FIN, and the flag field has the FIN flag set.
-  // it is used to know when it is safe to destroy the socket, we must have
-  // received all packets up to this sequence number first.
-  uint16 eof_pkt;
-
-  // All sequence numbers up to including this have been properly received
-  // by us
-  uint16 ack_nr;
-  // This is the sequence number for the next packet to be sent.
-  uint16 seq_nr;
-
-  uint16 timeout_seq_nr;
-
-  // This is the sequence number of the next packet we're allowed to
-  // do a fast resend with. This makes sure we only do a fast-resend
-  // once per packet. We can resend the packet with this sequence number
-  // or any later packet (with a higher sequence number).
-  uint16 fast_resend_seq_nr;
-
-  uint32 reply_micro;
-
-  uint64 last_got_packet;
-  uint64 last_sent_packet;
-  uint64 last_measured_delay;
-
-  // timestamp of the last time the cwnd was full
-  // this is used to prevent the congestion window
-  // from growing when we're not sending at capacity
-  mutable uint64 last_maxed_out_window;
-
-  void *userdata;
-
-  // Round trip time
-  uint rtt;
-  // Round trip time variance
-  uint rtt_var;
-  // Round trip timeout
-  uint rto;
-  DelayHist rtt_hist;
-  uint retransmit_timeout;
-  // The RTO timer will timeout here.
-  uint64 rto_timeout;
-  // When the window size is set to zero, start this timer. It will send a new
-  // packet every 30secs.
-  uint64 zerowindow_time;
-
-  uint32 conn_seed;
-  // Connection ID for packets I receive
-  uint32 conn_id_recv;
-  // Connection ID for packets I send
-  uint32 conn_id_send;
-  // Last rcv window we advertised, in bytes
-  size_t last_rcv_win;
-
-  DelayHist our_hist;
-  DelayHist their_hist;
-
-  // extension bytes from SYN packet
-  byte extensions[8];
-
-  // MTU Discovery
-  // time when we should restart the MTU discovery
-  uint64 mtu_discover_time;
-  // ceiling and floor of binary search. last is the mtu size
-  // we're currently using
-  uint32 mtu_ceiling, mtu_floor, mtu_last;
-  // we only ever have a single probe in flight at any given time.
-  // this is the sequence number of that probe, and the size of
-  // that packet
-  uint32 mtu_probe_seq, mtu_probe_size;
-
-  // this is the average delay samples, as compared to the initial
-  // sample. It's averaged over 5 seconds
-  int32 average_delay;
-  // this is the sum of all the delay samples
-  // we've made recently. The important distinction
-  // of these samples is that they are all made compared
-  // to the initial sample, this is to deal with
-  // wrapping in a simple way.
-  int64 current_delay_sum;
-  // number of sample ins current_delay_sum
-  int current_delay_samples;
-  // initialized to 0, set to the first raw delay sample
-  // each sample that's added to current_delay_sum
-  // is subtracted from the value first, to make it
-  // a delay relative to this sample
-  uint32 average_delay_base;
-  // the next time we should add an average delay
-  // sample into average_delay_hist
-  uint64 average_sample_time;
-  // the estimated clock drift between our computer
-  // and the endpoint computer. The unit is microseconds
-  // per 5 seconds
-  int32 clock_drift;
-  // just used for logging
-  int32 clock_drift_raw;
-
-  SizableCircularBuffer inbuf, outbuf;
-
-#ifdef _DEBUG
-  // Public per-socket statistics, returned by utp_get_stats()
-  utp_socket_stats _stats;
-#endif
-
-  // true if we're in slow-start (exponential growth) phase
-  bool slow_start;
-
-  // the slow-start threshold, in bytes
-  size_t ssthresh;
-
-  void
-  log(int level, char const *fmt, ...)
-  {
-    va_list va;
-    char buf[4096], buf2[4096];
-
-    // don't bother with vsnprintf() etc calls if we're not going to log.
-    if(!ctx->would_log(level))
-    {
-      return;
-    }
-
-    va_start(va, fmt);
-    vsnprintf(buf, 4096, fmt, va);
-    va_end(va);
-    buf[4095] = '\0';
-
-    snprintf(buf2, 4096, "%p %s %06u %s", this, addrfmt(addr, addrbuf),
-             conn_id_recv, buf);
-    buf2[4095] = '\0';
-
-    ctx->log_unchecked(this, buf2);
-  }
-
-  void
-  schedule_ack();
-
-  // called every time mtu_floor or mtu_ceiling are adjusted
-  void
-  mtu_search_update();
-  void
-  mtu_reset();
-
-  // Calculates the current receive window
-  size_t
-  get_rcv_window()
-  {
-    // Trim window down according to what's already in buffer.
-    const size_t numbuf = utp_call_get_read_buffer_size(this->ctx, this);
-    assert((int)numbuf >= 0);
-    return opt_rcvbuf > numbuf ? opt_rcvbuf - numbuf : 0;
-  }
-
-  // Test if we're ready to decay max_window
-  // XXX this breaks when spaced by > INT_MAX/2, which is 49
-  // days; the failure mode in that case is we do an extra decay
-  // or fail to do one when we really shouldn't.
-  bool
-  can_decay_win(int64 msec) const
-  {
-    return (msec - last_rwin_decay) >= MAX_WINDOW_DECAY;
-  }
-
-  // If we can, decay max window, returns true if we actually did so
-  void
-  maybe_decay_win(uint64 current_ms)
-  {
-    if(can_decay_win(current_ms))
-    {
-      // TCP uses 0.5
-      max_window      = (size_t)(max_window * .5);
-      last_rwin_decay = current_ms;
-      if(max_window < MIN_WINDOW_SIZE)
-        max_window = MIN_WINDOW_SIZE;
-      slow_start = false;
-      ssthresh   = max_window;
-    }
-  }
-
-  size_t
-  get_header_size() const
-  {
-    return sizeof(PacketFormatV1);
-  }
-
-  size_t
-  get_udp_mtu()
-  {
-    socklen_t len;
-    SOCKADDR_STORAGE sa = addr.get_sockaddr_storage(&len);
-    return utp_call_get_udp_mtu(this->ctx, this, (const struct sockaddr *)&sa,
-                                len);
-  }
-
-  size_t
-  get_udp_overhead()
-  {
-    socklen_t len;
-    SOCKADDR_STORAGE sa = addr.get_sockaddr_storage(&len);
-    return utp_call_get_udp_overhead(this->ctx, this,
-                                     (const struct sockaddr *)&sa, len);
-  }
-
-  size_t
-  get_overhead()
-  {
-    return get_udp_overhead() + get_header_size();
-  }
-
-  void
-  send_data(byte *b, size_t length, bandwidth_type_t type, uint32 flags = 0);
-
-  void
-  send_ack(bool synack = false);
-
-  void
-  send_keep_alive();
-
-  static void
-  send_rst(utp_context *ctx, const PackedSockAddr &addr, uint32 conn_id_send,
-           uint16 ack_nr, uint16 seq_nr);
-
-  void
-  send_packet(OutgoingPacket *pkt);
-
-  bool
-  is_full(int bytes = -1);
-  bool
-  flush_packets();
-  void
-  write_outgoing_packet(size_t payload, uint flags, struct utp_iovec *iovec,
-                        size_t num_iovecs);
-
-#ifdef _DEBUG
-  void
-  check_invariant();
-#endif
-
-  void
-  check_timeouts();
-  int
-  ack_packet(uint16 seq);
-  size_t
-  selective_ack_bytes(uint base, const byte *mask, byte len, int64 &min_rtt);
-  void
-  selective_ack(uint base, const byte *mask, byte len);
-  void
-  apply_ccontrol(size_t bytes_acked, uint32 actual_delay, int64 min_rtt);
-  size_t
-  get_packet_size() const;
-};
-
-void
-removeSocketFromAckList(UTPSocket *conn)
-{
-  if(conn->ida >= 0)
-  {
-    UTPSocket *last =
-        conn->ctx->ack_sockets[conn->ctx->ack_sockets.GetCount() - 1];
-
-    assert(last->ida < (int)(conn->ctx->ack_sockets.GetCount()));
-    assert(conn->ctx->ack_sockets[last->ida] == last);
-    last->ida                         = conn->ida;
-    conn->ctx->ack_sockets[conn->ida] = last;
-    conn->ida                         = -1;
-
-    // Decrease the count
-    conn->ctx->ack_sockets.SetCount(conn->ctx->ack_sockets.GetCount() - 1);
-  }
-}
-
-static void
-utp_register_sent_packet(utp_context *ctx, size_t length)
-{
-  if(length <= PACKET_SIZE_MID)
-  {
-    if(length <= PACKET_SIZE_EMPTY)
-    {
-      ctx->context_stats._nraw_send[PACKET_SIZE_EMPTY_BUCKET]++;
-    }
-    else if(length <= PACKET_SIZE_SMALL)
-    {
-      ctx->context_stats._nraw_send[PACKET_SIZE_SMALL_BUCKET]++;
-    }
-    else
-      ctx->context_stats._nraw_send[PACKET_SIZE_MID_BUCKET]++;
-  }
-  else
-  {
-    if(length <= PACKET_SIZE_BIG)
-    {
-      ctx->context_stats._nraw_send[PACKET_SIZE_BIG_BUCKET]++;
-    }
-    else
-      ctx->context_stats._nraw_send[PACKET_SIZE_HUGE_BUCKET]++;
-  }
-}
-
-void
-send_to_addr(utp_context *ctx, const byte *p, size_t len,
-             const PackedSockAddr &addr, int flags = 0)
-{
-  socklen_t tolen;
-  SOCKADDR_STORAGE to = addr.get_sockaddr_storage(&tolen);
-  utp_register_sent_packet(ctx, len);
-  utp_call_sendto(ctx, NULL, p, len, (const struct sockaddr *)&to, tolen,
-                  flags);
-}
-
-void
-UTPSocket::schedule_ack()
-{
-  if(ida == -1)
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "schedule_ack");
-#endif
-    ida = ctx->ack_sockets.Append(this);
-  }
-  else
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "schedule_ack: already in list");
-#endif
-  }
-}
-
-void
-UTPSocket::send_data(byte *b, size_t length, bandwidth_type_t type,
-                     uint32 flags)
-{
-  // time stamp this packet with local time, the stamp goes into
-  // the header of every packet at the 8th byte for 8 bytes :
-  // two integers, check packet.h for more
-  uint64 time = utp_call_get_microseconds(ctx, this);
-
-  PacketFormatV1 *b1 = (PacketFormatV1 *)b;
-  b1->tv_usec        = (uint32)time;
-  b1->reply_micro    = reply_micro;
-
-  last_sent_packet = ctx->current_ms;
-
-#ifdef _DEBUG
-  _stats.nbytes_xmit += length;
-  ++_stats.nxmit;
-#endif
-
-  if(ctx->callbacks[UTP_ON_OVERHEAD_STATISTICS])
-  {
-    size_t n;
-    if(type == payload_bandwidth)
-    {
-      // if this packet carries payload, just
-      // count the header as overhead
-      type = header_overhead;
-      n    = get_overhead();
-    }
-    else
-    {
-      n = length + get_udp_overhead();
-    }
-    utp_call_on_overhead_statistics(ctx, this, true, n, type);
-  }
-#if UTP_DEBUG_LOGGING
-  int flags2    = b1->type();
-  uint16 seq_nr = b1->seq_nr;
-  uint16 ack_nr = b1->ack_nr;
-  log(UTP_LOG_DEBUG,
-      "send %s len:%u id:%u timestamp:" I64u
-      " reply_micro:%u flags:%s seq_nr:%u ack_nr:%u",
-      addrfmt(addr, addrbuf), (uint)length, conn_id_send, time, reply_micro,
-      flagnames[flags2], seq_nr, ack_nr);
-#endif
-  send_to_addr(ctx, b, length, addr, flags);
-  removeSocketFromAckList(this);
-}
-
-void
-UTPSocket::send_ack(bool synack)
-{
-  PacketFormatAckV1 pfa;
-  zeromem(&pfa);
-
-  size_t len;
-  last_rcv_win = get_rcv_window();
-  pfa.pf.set_version(1);
-  pfa.pf.set_type(ST_STATE);
-  pfa.pf.ext        = 0;
-  pfa.pf.connid     = conn_id_send;
-  pfa.pf.ack_nr     = ack_nr;
-  pfa.pf.seq_nr     = seq_nr;
-  pfa.pf.windowsize = (uint32)last_rcv_win;
-  len               = sizeof(PacketFormatV1);
-
-  // we never need to send EACK for connections
-  // that are shutting down
-  if(reorder_count != 0 && !got_fin_reached)
-  {
-    // if reorder count > 0, send an EACK.
-    // reorder count should always be 0
-    // for synacks, so this should not be
-    // as synack
-    assert(!synack);
-    (void)synack;
-    pfa.pf.ext   = 1;
-    pfa.ext_next = 0;
-    pfa.ext_len  = 4;
-    uint m       = 0;
-
-    // reorder count should only be non-zero
-    // if the packet ack_nr + 1 has not yet
-    // been received
-    assert(inbuf.get(ack_nr + 1) == NULL);
-    size_t window = min< size_t >(14 + 16, inbuf.size());
-    // Generate bit mask of segments received.
-    for(size_t i = 0; i < window; i++)
-    {
-      if(inbuf.get(ack_nr + i + 2) != NULL)
-      {
-        m |= 1 << i;
-
-#if UTP_DEBUG_LOGGING
-        log(UTP_LOG_DEBUG, "EACK packet [%u]", ack_nr + i + 2);
-#endif
-      }
-    }
-    pfa.acks[0] = (byte)m;
-    pfa.acks[1] = (byte)(m >> 8);
-    pfa.acks[2] = (byte)(m >> 16);
-    pfa.acks[3] = (byte)(m >> 24);
-    len += 4 + 2;
-
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "Sending EACK %u [%u] bits:[%032b]", ack_nr,
-        conn_id_send, m);
-#endif
-  }
-  else
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "Sending ACK %u [%u]", ack_nr, conn_id_send);
-#endif
-  }
-
-  send_data((byte *)&pfa, len, ack_overhead);
-  removeSocketFromAckList(this);
-}
-
-void
-UTPSocket::send_keep_alive()
-{
-  ack_nr--;
-
-#if UTP_DEBUG_LOGGING
-  log(UTP_LOG_DEBUG, "Sending KeepAlive ACK %u [%u]", ack_nr, conn_id_send);
-#endif
-
-  send_ack();
-  ack_nr++;
-}
-
-void
-UTPSocket::send_rst(utp_context *ctx, const PackedSockAddr &addr,
-                    uint32 conn_id_send, uint16 ack_nr, uint16 seq_nr)
-{
-  PacketFormatV1 pf1;
-  zeromem(&pf1);
-
-  size_t len;
-  pf1.set_version(1);
-  pf1.set_type(ST_RESET);
-  pf1.ext        = 0;
-  pf1.connid     = conn_id_send;
-  pf1.ack_nr     = ack_nr;
-  pf1.seq_nr     = seq_nr;
-  pf1.windowsize = 0;
-  len            = sizeof(PacketFormatV1);
-
-  //	LOG_DEBUG("%s: Sending RST id:%u seq_nr:%u ack_nr:%u", addrfmt(addr,
-  // addrbuf), conn_id_send, seq_nr, ack_nr); 	LOG_DEBUG("send %s len:%u
-  // id:%u", addrfmt(addr, addrbuf), (uint)len, conn_id_send);
-  send_to_addr(ctx, (const byte *)&pf1, len, addr);
-}
-
-void
-UTPSocket::send_packet(OutgoingPacket *pkt)
-{
-  // only count against the quota the first time we
-  // send the packet. Don't enforce quota when closing
-  // a socket. Only enforce the quota when we're sending
-  // at slow rates (max window < packet size)
-
-  // size_t max_send = min(max_window, opt_sndbuf, max_window_user);
-  time_t cur_time = utp_call_get_milliseconds(this->ctx, this);
-
-  if(pkt->transmissions == 0 || pkt->need_resend)
-  {
-    cur_window += pkt->payload;
-  }
-
-  pkt->need_resend = false;
-
-  PacketFormatV1 *p1 = (PacketFormatV1 *)pkt->data;
-  p1->ack_nr         = ack_nr;
-  pkt->time_sent     = utp_call_get_microseconds(this->ctx, this);
-
-  // socklen_t salen;
-  // SOCKADDR_STORAGE sa = addr.get_sockaddr_storage(&salen);
-  bool use_as_mtu_probe = false;
-
-  // TODO: this is subject to nasty wrapping issues! Below as well
-  if(mtu_discover_time < (uint64)cur_time)
-  {
-    // it's time to reset our MTU assupmtions
-    // and trigger a new search
-    mtu_reset();
-  }
-
-  // don't use packets that are larger then mtu_ceiling
-  // as probes, since they were probably used as probes
-  // already and failed, now we need it to fragment
-  // just to get it through
-  // if seq_nr == 1, the probe would end up being 0
-  // which is a magic number representing no-probe
-  // that why we don't send a probe for a packet with
-  // sequence number 0
-  if(mtu_floor < mtu_ceiling && pkt->length > mtu_floor
-     && pkt->length <= mtu_ceiling && mtu_probe_seq == 0 && seq_nr != 1
-     && pkt->transmissions == 0)
-  {
-    // we've already incremented seq_nr
-    // for this packet
-    mtu_probe_seq  = (seq_nr - 1) & ACK_NR_MASK;
-    mtu_probe_size = pkt->length;
-    assert(pkt->length >= mtu_floor);
-    assert(pkt->length <= mtu_ceiling);
-    use_as_mtu_probe = true;
-    log(UTP_LOG_MTU, "MTU [PROBE] floor:%d ceiling:%d current:%d", mtu_floor,
-        mtu_ceiling, mtu_probe_size);
-  }
-
-  pkt->transmissions++;
-  send_data(
-      (byte *)pkt->data, pkt->length,
-      (state == CS_SYN_SENT)
-          ? connect_overhead
-          : (pkt->transmissions == 1) ? payload_bandwidth : retransmit_overhead,
-      use_as_mtu_probe ? UTP_UDP_DONTFRAG : 0);
-}
-
-bool
-UTPSocket::is_full(int bytes)
-{
-  size_t packet_size = get_packet_size();
-  if(bytes < 0)
-    bytes = packet_size;
-  else if(bytes > (int)packet_size)
-    bytes = (int)packet_size;
-  size_t max_send = min(max_window, opt_sndbuf, max_window_user);
-
-  // subtract one to save space for the FIN packet
-  if(cur_window_packets >= OUTGOING_BUFFER_MAX_SIZE - 1)
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "is_full:false cur_window_packets:%d MAX:%d",
-        cur_window_packets, OUTGOING_BUFFER_MAX_SIZE - 1);
-#endif
-
-    last_maxed_out_window = ctx->current_ms;
-    return true;
-  }
-
-#if UTP_DEBUG_LOGGING
-  log(UTP_LOG_DEBUG,
-      "is_full:%s. cur_window:%u pkt:%u max:%u cur_window_packets:%u "
-      "max_window:%u",
-      (cur_window + bytes > max_send) ? "true" : "false", cur_window, bytes,
-      max_send, cur_window_packets, max_window);
-#endif
-
-  if(cur_window + bytes > max_send)
-  {
-    last_maxed_out_window = ctx->current_ms;
-    return true;
-  }
-  return false;
-}
-
-bool
-UTPSocket::flush_packets()
-{
-  size_t packet_size = get_packet_size();
-
-  // send packets that are waiting on the pacer to be sent
-  // i has to be an unsigned 16 bit counter to wrap correctly
-  // signed types are not guaranteed to wrap the way you expect
-  for(uint16 i = seq_nr - cur_window_packets; i != seq_nr; ++i)
-  {
-    OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(i);
-    if(pkt == 0 || (pkt->transmissions > 0 && pkt->need_resend == false))
-      continue;
-    // have we run out of quota?
-    if(is_full())
-      return true;
-
-    // Nagle check
-    // don't send the last packet if we have one packet in-flight
-    // and the current packet is still smaller than packet_size.
-    if(i != ((seq_nr - 1) & ACK_NR_MASK) || cur_window_packets == 1
-       || pkt->payload >= packet_size)
-    {
-      send_packet(pkt);
-    }
-  }
-  return false;
-}
-
-// @payload: number of bytes to send
-// @flags: either ST_DATA, or ST_FIN
-// @iovec: base address of iovec array
-// @num_iovecs: number of iovecs in array
-void
-UTPSocket::write_outgoing_packet(size_t payload, uint flags,
-                                 struct utp_iovec *iovec, size_t num_iovecs)
-{
-  // Setup initial timeout timer
-  if(cur_window_packets == 0)
-  {
-    retransmit_timeout = rto;
-    rto_timeout        = ctx->current_ms + retransmit_timeout;
-    assert(cur_window == 0);
-  }
-
-  size_t packet_size = get_packet_size();
-  do
-  {
-    assert(cur_window_packets < OUTGOING_BUFFER_MAX_SIZE);
-    assert(flags == ST_DATA || flags == ST_FIN);
-
-    size_t added = 0;
-
-    OutgoingPacket *pkt = NULL;
-
-    if(cur_window_packets > 0)
-    {
-      pkt = (OutgoingPacket *)outbuf.get(seq_nr - 1);
-    }
-
-    const size_t header_size = get_header_size();
-    bool append              = true;
-
-    // if there's any room left in the last packet in the window
-    // and it hasn't been sent yet, fill that frame first
-    if(payload && pkt && !pkt->transmissions && pkt->payload < packet_size)
-    {
-      // Use the previous unsent packet
-      added =
-          min(payload + pkt->payload, max< size_t >(packet_size, pkt->payload))
-          - pkt->payload;
-      pkt = (OutgoingPacket *)realloc(
-          pkt,
-          (sizeof(OutgoingPacket) - 1) + header_size + pkt->payload + added);
-      outbuf.put(seq_nr - 1, pkt);
-      append = false;
-      assert(!pkt->need_resend);
-    }
-    else
-    {
-      // Create the packet to send.
-      added = payload;
-      pkt = (OutgoingPacket *)malloc((sizeof(OutgoingPacket) - 1) + header_size
-                                     + added);
-      pkt->payload       = 0;
-      pkt->transmissions = 0;
-      pkt->need_resend   = false;
-    }
-
-    if(added)
-    {
-      assert(flags == ST_DATA);
-
-      // Fill it with data from the upper layer.
-      unsigned char *p = pkt->data + header_size + pkt->payload;
-      size_t needed    = added;
-
-      /*
-      while (needed) {
-              *p = *(char*)iovec[0].iov_base;
-              p++;
-              iovec[0].iov_base = (char *)iovec[0].iov_base + 1;
-              needed--;
-      }
-      */
-
-      for(size_t i = 0; i < num_iovecs && needed; i++)
-      {
-        if(iovec[i].iov_len == 0)
-          continue;
-
-        size_t num = min< size_t >(needed, iovec[i].iov_len);
-        memcpy(p, iovec[i].iov_base, num);
-
-        p += num;
-
-        iovec[i].iov_len -= num;
-        iovec[i].iov_base = (byte *)iovec[i].iov_base
-            + num;  // iovec[i].iov_base += num, but without void* pointers
-        needed -= num;
-      }
-
-      assert(needed == 0);
-    }
-    pkt->payload += added;
-    pkt->length = header_size + pkt->payload;
-
-    last_rcv_win = get_rcv_window();
-
-    PacketFormatV1 *p1 = (PacketFormatV1 *)pkt->data;
-    // p1->ver_type; needs to be set!!
-    p1->set_version(1);
-    p1->set_type(flags);
-    p1->ext        = 0;
-    p1->connid     = conn_id_send;
-    p1->windowsize = (uint32)last_rcv_win;
-    p1->ack_nr     = ack_nr;
-
-    if(append)
-    {
-      // Remember the message in the outgoing queue.
-      outbuf.ensure_size(seq_nr, cur_window_packets);
-      outbuf.put(seq_nr, pkt);
-      p1->seq_nr = seq_nr;
-      seq_nr++;
-      cur_window_packets++;
-    }
-
-    payload -= added;
-
-  } while(payload);
-
-  flush_packets();
-}
-
-#ifdef _DEBUG
-void
-UTPSocket::check_invariant()
-{
-  if(reorder_count > 0)
-  {
-    assert(inbuf.get(ack_nr + 1) == NULL);
-  }
-
-  size_t outstanding_bytes = 0;
-  for(int i = 0; i < cur_window_packets; ++i)
-  {
-    OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(seq_nr - i - 1);
-    if(pkt == 0 || pkt->transmissions == 0 || pkt->need_resend)
-      continue;
-    outstanding_bytes += pkt->payload;
-  }
-  assert(outstanding_bytes == cur_window);
-}
-#endif
-
-void
-UTPSocket::check_timeouts()
-{
-#ifdef _DEBUG
-  check_invariant();
-#endif
-
-  // this invariant should always be true
-  assert(cur_window_packets == 0 || outbuf.get(seq_nr - cur_window_packets));
-
-#if UTP_DEBUG_LOGGING
-  log(UTP_LOG_DEBUG,
-      "CheckTimeouts timeout:%d max_window:%u cur_window:%u "
-      "state:%s cur_window_packets:%u",
-      (int)(rto_timeout - ctx->current_ms), (uint)max_window, (uint)cur_window,
-      statenames[state], cur_window_packets);
-#endif
-
-  if(state != CS_DESTROY)
-    flush_packets();
-
-  switch(state)
-  {
-    case CS_SYN_SENT:
-    case CS_SYN_RECV:
-    case CS_CONNECTED_FULL:
-    case CS_CONNECTED:
-    {
-      // Reset max window...
-      if((int)(ctx->current_ms - zerowindow_time) >= 0 && max_window_user == 0)
-      {
-        max_window_user = PACKET_SIZE;
-      }
-
-      if((int)(ctx->current_ms - rto_timeout) >= 0 && rto_timeout > 0)
-      {
-        bool ignore_loss = false;
-
-        if(cur_window_packets == 1
-           && ((seq_nr - 1) & ACK_NR_MASK) == mtu_probe_seq
-           && mtu_probe_seq != 0)
-        {
-          // we only had  a single outstanding packet that timed out, and it was
-          // the probe
-          mtu_ceiling = mtu_probe_size - 1;
-          mtu_search_update();
-          // this packet was most likely dropped because the packet size being
-          // too big and not because congestion. To accelerate the binary search
-          // for the MTU, resend immediately and don't reset the window size
-          ignore_loss = true;
-          log(UTP_LOG_MTU, "MTU [PROBE-TIMEOUT] floor:%d ceiling:%d current:%d",
-              mtu_floor, mtu_ceiling, mtu_last);
-        }
-        // we dropepd the probe, clear these fields to
-        // allow us to send a new one
-        mtu_probe_seq = mtu_probe_size = 0;
-        log(UTP_LOG_MTU, "MTU [TIMEOUT]");
-
-        /*
-        OutgoingPacket *pkt = (OutgoingPacket*)outbuf.get(seq_nr -
-        cur_window_packets);
-
-        // If there were a lot of retransmissions, force recomputation of round
-        trip time if (pkt->transmissions >= 4) rtt = 0;
-        */
-
-        // Increase RTO
-        const uint new_timeout =
-            ignore_loss ? retransmit_timeout : retransmit_timeout * 2;
-
-        // They initiated the connection but failed to respond before the rto.
-        // A malicious client can also spoof the destination address of a ST_SYN
-        // bringing us to this state. Kill the connection and do not notify the
-        // upper layer
-        if(state == CS_SYN_RECV)
-        {
-          state = CS_DESTROY;
-          utp_call_on_error(ctx, this, UTP_ETIMEDOUT);
-          return;
-        }
-
-        // We initiated the connection but the other side failed to respond
-        // before the rto
-        if(retransmit_count >= 4
-           || (state == CS_SYN_SENT && retransmit_count >= 2))
-        {
-          // 4 consecutive transmissions have timed out. Kill it. If we
-          // haven't even connected yet, give up after only 2 consecutive
-          // failed transmissions.
-          if(close_requested)
-            state = CS_DESTROY;
-          else
-            state = CS_RESET;
-          utp_call_on_error(ctx, this, UTP_ETIMEDOUT);
-          return;
-        }
-
-        retransmit_timeout = new_timeout;
-        rto_timeout        = ctx->current_ms + new_timeout;
-
-        if(!ignore_loss)
-        {
-          // On Timeout
-          duplicate_ack = 0;
-
-          int packet_size = get_packet_size();
-
-          if((cur_window_packets == 0) && ((int)max_window > packet_size))
-          {
-            // we don't have any packets in-flight, even though
-            // we could. This implies that the connection is just
-            // idling. No need to be aggressive about resetting the
-            // congestion window. Just let it decay by a 3:rd.
-            // don't set it any lower than the packet size though
-            max_window = max(max_window * 2 / 3, size_t(packet_size));
-          }
-          else
-          {
-            // our delay was so high that our congestion window
-            // was shrunk below one packet, preventing us from
-            // sending anything for one time-out period. Now, reset
-            // the congestion window to fit one packet, to start over
-            // again
-            max_window = packet_size;
-            slow_start = true;
-          }
-        }
-
-        // every packet should be considered lost
-        for(int i = 0; i < cur_window_packets; ++i)
-        {
-          OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(seq_nr - i - 1);
-          if(pkt == 0 || pkt->transmissions == 0 || pkt->need_resend)
-            continue;
-          pkt->need_resend = true;
-          assert(cur_window >= pkt->payload);
-          cur_window -= pkt->payload;
-        }
-
-        if(cur_window_packets > 0)
-        {
-          retransmit_count++;
-          // used in parse_log.py
-          log(UTP_LOG_NORMAL,
-              "Packet timeout. Resend. seq_nr:%u. timeout:%u "
-              "max_window:%u cur_window_packets:%d",
-              seq_nr - cur_window_packets, retransmit_timeout, (uint)max_window,
-              int(cur_window_packets));
-
-          fast_timeout   = true;
-          timeout_seq_nr = seq_nr;
-
-          OutgoingPacket *pkt =
-              (OutgoingPacket *)outbuf.get(seq_nr - cur_window_packets);
-          assert(pkt);
-
-          // Re-send the packet.
-          send_packet(pkt);
-        }
-      }
-
-      // Mark the socket as writable. If the cwnd has grown, or if the number of
-      // bytes in-flight is lower than cwnd, we need to make the socket writable
-      // again in case it isn't
-      if(state == CS_CONNECTED_FULL && !is_full())
-      {
-        state = CS_CONNECTED;
-
-#if UTP_DEBUG_LOGGING
-        log(UTP_LOG_DEBUG,
-            "Socket writable. max_window:%u cur_window:%u packet_size:%u",
-            (uint)max_window, (uint)cur_window, (uint)get_packet_size());
-#endif
-        utp_call_on_state_change(this->ctx, this, UTP_STATE_WRITABLE);
-      }
-
-      if(state >= CS_CONNECTED && !fin_sent)
-      {
-        if((int)(ctx->current_ms - last_sent_packet) >= KEEPALIVE_INTERVAL)
-        {
-          send_keep_alive();
-        }
-      }
-      break;
-    }
-
-    // prevent warning
-    case CS_UNINITIALIZED:
-    case CS_IDLE:
-    case CS_RESET:
-    case CS_DESTROY:
-      break;
-  }
-}
-
-// this should be called every time we change mtu_floor or mtu_ceiling
-void
-UTPSocket::mtu_search_update()
-{
-  assert(mtu_floor <= mtu_ceiling);
-
-  // binary search
-  mtu_last = (mtu_floor + mtu_ceiling) / 2;
-
-  // enable a new probe to be sent
-  mtu_probe_seq = mtu_probe_size = 0;
-
-  // if the floor and ceiling are close enough, consider the
-  // MTU binary search complete. We set the current value
-  // to floor since that's the only size we know can go through
-  // also set the ceiling to floor to terminate the searching
-  if(mtu_ceiling - mtu_floor <= 16)
-  {
-    mtu_last = mtu_floor;
-    log(UTP_LOG_MTU, "MTU [DONE] floor:%d ceiling:%d current:%d", mtu_floor,
-        mtu_ceiling, mtu_last);
-    mtu_ceiling = mtu_floor;
-    assert(mtu_floor <= mtu_ceiling);
-    // Do another search in 30 minutes
-    mtu_discover_time =
-        utp_call_get_milliseconds(this->ctx, this) + 30 * 60 * 1000;
-  }
-}
-
-void
-UTPSocket::mtu_reset()
-{
-  mtu_ceiling = get_udp_mtu();
-  // Less would not pass TCP...
-  mtu_floor = 576;
-  log(UTP_LOG_MTU, "MTU [RESET] floor:%d ceiling:%d current:%d", mtu_floor,
-      mtu_ceiling, mtu_last);
-  assert(mtu_floor <= mtu_ceiling);
-  mtu_discover_time =
-      utp_call_get_milliseconds(this->ctx, this) + 30 * 60 * 1000;
-}
-
-// returns:
-// 0: the packet was acked.
-// 1: it means that the packet had already been acked
-// 2: the packet has not been sent yet
-int
-UTPSocket::ack_packet(uint16 seq)
-{
-  OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(seq);
-
-  // the packet has already been acked (or not sent)
-  if(pkt == NULL)
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "got ack for:%u (already acked, or never sent)", seq);
-#endif
-
-    return 1;
-  }
-
-  // can't ack packets that haven't been sent yet!
-  if(pkt->transmissions == 0)
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG,
-        "got ack for:%u (never sent, pkt_size:%u need_resend:%u)", seq,
-        (uint)pkt->payload, pkt->need_resend);
-#endif
-
-    return 2;
-  }
-
-#if UTP_DEBUG_LOGGING
-  log(UTP_LOG_DEBUG, "got ack for:%u (pkt_size:%u need_resend:%u)", seq,
-      (uint)pkt->payload, pkt->need_resend);
-#endif
-
-  outbuf.put(seq, NULL);
-
-  // if we never re-sent the packet, update the RTT estimate
-  if(pkt->transmissions == 1)
-  {
-    // Estimate the round trip time.
-    const uint32 ertt = (uint32)(
-        (utp_call_get_microseconds(this->ctx, this) - pkt->time_sent) / 1000);
-    if(rtt == 0)
-    {
-      // First round trip time sample
-      rtt     = ertt;
-      rtt_var = ertt / 2;
-      // sanity check. rtt should never be more than 6 seconds
-      //			assert(rtt < 6000);
-    }
-    else
-    {
-      // Compute new round trip times
-      const int delta = (int)rtt - ertt;
-      rtt_var         = rtt_var + (int)(abs(delta) - rtt_var) / 4;
-      rtt             = rtt - rtt / 8 + ertt / 8;
-      // sanity check. rtt should never be more than 6 seconds
-      //			assert(rtt < 6000);
-      rtt_hist.add_sample(ertt, ctx->current_ms);
-    }
-    rto = max< uint >(rtt + rtt_var * 4, 1000);
-
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "rtt:%u avg:%u var:%u rto:%u", ertt, rtt, rtt_var, rto);
-#endif
-  }
-  retransmit_timeout = rto;
-  rto_timeout        = ctx->current_ms + rto;
-  // if need_resend is set, this packet has already
-  // been considered timed-out, and is not included in
-  // the cur_window anymore
-  if(!pkt->need_resend)
-  {
-    assert(cur_window >= pkt->payload);
-    cur_window -= pkt->payload;
-  }
-  free(pkt);
-  retransmit_count = 0;
-  return 0;
-}
-
-// count the number of bytes that were acked by the EACK header
-size_t
-UTPSocket::selective_ack_bytes(uint base, const byte *mask, byte len,
-                               int64 &min_rtt)
-{
-  if(cur_window_packets == 0)
-    return 0;
-
-  size_t acked_bytes = 0;
-  int bits           = len * 8;
-  uint64 now         = utp_call_get_microseconds(this->ctx, this);
-
-  do
-  {
-    uint v = base + bits;
-
-    // ignore bits that haven't been sent yet
-    // see comment in UTPSocket::selective_ack
-    if(((seq_nr - v - 1) & ACK_NR_MASK) >= (uint16)(cur_window_packets - 1))
-      continue;
-
-    // ignore bits that represents packets we haven't sent yet
-    // or packets that have already been acked
-    OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(v);
-    if(!pkt || pkt->transmissions == 0)
-      continue;
-
-    // Count the number of segments that were successfully received past it.
-    if(bits >= 0 && mask[bits >> 3] & (1 << (bits & 7)))
-    {
-      assert((int)(pkt->payload) >= 0);
-      acked_bytes += pkt->payload;
-      if(pkt->time_sent < now)
-        min_rtt = min< int64 >(min_rtt, now - pkt->time_sent);
-      else
-        min_rtt = min< int64 >(min_rtt, 50000);
-      continue;
-    }
-  } while(--bits >= -1);
-  return acked_bytes;
-}
-
-enum
-{
-  MAX_EACK = 128
-};
-
-void
-UTPSocket::selective_ack(uint base, const byte *mask, byte len)
-{
-  if(cur_window_packets == 0)
-    return;
-
-  // the range is inclusive [0, 31] bits
-  int bits = len * 8 - 1;
-
-  int count = 0;
-
-  // resends is a stack of sequence numbers we need to resend. Since we
-  // iterate in reverse over the acked packets, at the end, the top packets
-  // are the ones we want to resend
-  int resends[MAX_EACK];
-  int nr = 0;
-
-#if UTP_DEBUG_LOGGING
-  char bitmask[1024] = {0};
-  int counter        = bits;
-  for(int i = 0; i <= bits; ++i)
-  {
-    bool bit_set = counter >= 0 && mask[counter >> 3] & (1 << (counter & 7));
-    bitmask[i]   = bit_set ? '1' : '0';
-    --counter;
-  }
-
-  log(UTP_LOG_DEBUG, "Got EACK [%s] base:%u", bitmask, base);
-#endif
-
-  do
-  {
-    // we're iterating over the bits from higher sequence numbers
-    // to lower (kind of in reverse order, wich might not be very
-    // intuitive)
-    uint v = base + bits;
-
-    // ignore bits that haven't been sent yet
-    // and bits that fall below the ACKed sequence number
-    // this can happen if an EACK message gets
-    // reordered and arrives after a packet that ACKs up past
-    // the base for thie EACK message
-
-    // this is essentially the same as:
-    // if v >= seq_nr || v <= seq_nr - cur_window_packets
-    // but it takes wrapping into account
-
-    // if v == seq_nr the -1 will make it wrap. if v > seq_nr
-    // it will also wrap (since it will fall further below 0)
-    // and be > cur_window_packets.
-    // if v == seq_nr - cur_window_packets, the result will be
-    // seq_nr - (seq_nr - cur_window_packets) - 1
-    // == seq_nr - seq_nr + cur_window_packets - 1
-    // == cur_window_packets - 1 which will be caught by the
-    // test. If v < seq_nr - cur_window_packets the result will grow
-    // fall furhter outside of the cur_window_packets range.
-
-    // sequence number space:
-    //
-    //     rejected <   accepted   > rejected
-    // <============+--------------+============>
-    //              ^              ^
-    //              |              |
-    //        (seq_nr-wnd)         seq_nr
-
-    if(((seq_nr - v - 1) & ACK_NR_MASK) >= (uint16)(cur_window_packets - 1))
-      continue;
-
-    // this counts as a duplicate ack, even though we might have
-    // received an ack for this packet previously (in another EACK
-    // message for instance)
-    bool bit_set = bits >= 0 && mask[bits >> 3] & (1 << (bits & 7));
-
-    // if this packet is acked, it counts towards the duplicate ack counter
-    if(bit_set)
-      count++;
-
-    // ignore bits that represents packets we haven't sent yet
-    // or packets that have already been acked
-    OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(v);
-    if(!pkt || pkt->transmissions == 0)
-    {
-#if UTP_DEBUG_LOGGING
-      log(UTP_LOG_DEBUG, "skipping %u. pkt:%08x transmissions:%u %s", v, pkt,
-          pkt ? pkt->transmissions : 0,
-          pkt ? "(not sent yet?)" : "(already acked?)");
-#endif
-      continue;
-    }
-
-    // Count the number of segments that were successfully received past it.
-    if(bit_set)
-    {
-      // the selective ack should never ACK the packet we're waiting for to
-      // decrement cur_window_packets
-      assert((v & outbuf.mask)
-             != ((seq_nr - cur_window_packets) & outbuf.mask));
-      ack_packet(v);
-      continue;
-    }
-
-    // Resend segments
-    // if count is less than our re-send limit, we haven't seen enough
-    // acked packets in front of this one to warrant a re-send.
-    // if count == 0, we're still going through the tail of zeroes
-    if(((v - fast_resend_seq_nr) & ACK_NR_MASK) <= OUTGOING_BUFFER_MAX_SIZE
-       && count >= DUPLICATE_ACKS_BEFORE_RESEND)
-    {
-      // resends is a stack, and we're mostly interested in the top of it
-      // if we're full, just throw away the lower half
-      if(nr >= MAX_EACK - 2)
-      {
-        memmove(resends, &resends[MAX_EACK / 2],
-                MAX_EACK / 2 * sizeof(resends[0]));
-        nr -= MAX_EACK / 2;
-      }
-      resends[nr++] = v;
-
-#if UTP_DEBUG_LOGGING
-      log(UTP_LOG_DEBUG, "no ack for %u", v);
-#endif
-    }
-    else
-    {
-#if UTP_DEBUG_LOGGING
-      log(UTP_LOG_DEBUG,
-          "not resending %u count:%d dup_ack:%u fast_resend_seq_nr:%u", v,
-          count, duplicate_ack, fast_resend_seq_nr);
-#endif
-    }
-  } while(--bits >= -1);
-
-  if(((base - 1 - fast_resend_seq_nr) & ACK_NR_MASK) <= OUTGOING_BUFFER_MAX_SIZE
-     && count >= DUPLICATE_ACKS_BEFORE_RESEND)
-  {
-    // if we get enough duplicate acks to start
-    // resending, the first packet we should resend
-    // is base-1
-    resends[nr++] = (base - 1) & ACK_NR_MASK;
-
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG, "no ack for %u", (base - 1) & ACK_NR_MASK);
-#endif
-  }
-  else
-  {
-#if UTP_DEBUG_LOGGING
-    log(UTP_LOG_DEBUG,
-        "not resending %u count:%d dup_ack:%u fast_resend_seq_nr:%u", base - 1,
-        count, duplicate_ack, fast_resend_seq_nr);
-#endif
-  }
-
-  bool back_off = false;
-  int i         = 0;
-  while(nr > 0)
-  {
-    uint v = resends[--nr];
-    // don't consider the tail of 0:es to be lost packets
-    // only unacked packets with acked packets after should
-    // be considered lost
-    OutgoingPacket *pkt = (OutgoingPacket *)outbuf.get(v);
-
-    // this may be an old (re-ordered) packet, and some of the
-    // packets in here may have been acked already. In which
-    // case they will not be in the send queue anymore
-    if(!pkt)
-      continue;
-
-    // used in parse_log.py
-    log(UTP_LOG_NORMAL, "Packet %u lost. Resending", v);
-
-    // On Loss
-    back_off = true;
-
-#ifdef _DEBUG
-    ++_stats.rexmit;
-#endif
-
-    send_packet(pkt);
-    fast_resend_seq_nr = (v + 1) & ACK_NR_MASK;
-
-    // Re-send max 4 packets.
-    if(++i >= 4)
-      break;
-  }
-
-  if(back_off)
-    maybe_decay_win(ctx->current_ms);
-
-  duplicate_ack = count;
-}
-
-void
-UTPSocket::apply_ccontrol(size_t bytes_acked, uint32 actual_delay,
-                          int64 min_rtt)
-{
-  // the delay can never be greater than the rtt. The min_rtt
-  // variable is the RTT in microseconds
-
-  assert(min_rtt >= 0);
-  int32 our_delay = min< uint32 >(our_hist.get_value(), uint32(min_rtt));
-  assert(our_delay != INT_MAX);
-  assert(our_delay >= 0);
-
-  utp_call_on_delay_sample(this->ctx, this, our_delay / 1000);
-
-  // This test the connection under heavy load from foreground
-  // traffic. Pretend that our delays are very high to force the
-  // connection to use sub-packet size window sizes
-  // our_delay *= 4;
-
-  // target is microseconds
-  int target = target_delay;
-  if(target <= 0)
-    target = 100000;
-
-  // this is here to compensate for very large clock drift that affects
-  // the congestion controller into giving certain endpoints an unfair
-  // share of the bandwidth. We have an estimate of the clock drift
-  // (clock_drift). The unit of this is microseconds per 5 seconds.
-  // empirically, a reasonable cut-off appears to be about 200000
-  // (which is pretty high). The main purpose is to compensate for
-  // people trying to "cheat" uTP by making their clock run slower,
-  // and this definitely catches that without any risk of false positives
-  // if clock_drift < -200000 start applying a penalty delay proportional
-  // to how far beoynd -200000 the clock drift is
-  int32 penalty = 0;
-  if(clock_drift < -200000)
-  {
-    penalty = (-clock_drift - 200000) / 7;
-    our_delay += penalty;
-  }
-
-  double off_target = target - our_delay;
-
-  // this is the same as:
-  //
-  //    (min(off_target, target) / target) * (bytes_acked / max_window) *
-  //    MAX_CWND_INCREASE_BYTES_PER_RTT
-  //
-  // so, it's scaling the max increase by the fraction of the window this ack
-  // represents, and the fraction of the target delay the current delay
-  // represents. The min() around off_target protects against crazy values of
-  // our_delay, which may happen when th timestamps wraps, or by just having a
-  // malicious peer sending garbage. This caps the increase of the window size
-  // to MAX_CWND_INCREASE_BYTES_PER_RTT per rtt. as for large negative numbers,
-  // this direction is already capped at the min packet size further down the
-  // min around the bytes_acked protects against the case where the window size
-  // was recently shrunk and the number of acked bytes exceeds that. This is
-  // considered no more than one full window, in order to keep the gain within
-  // sane boundries.
-
-  assert(bytes_acked > 0);
-  double window_factor = (double)min(bytes_acked, max_window)
-      / (double)max(max_window, bytes_acked);
-
-  double delay_factor = off_target / target;
-  double scaled_gain =
-      MAX_CWND_INCREASE_BYTES_PER_RTT * window_factor * delay_factor;
-
-  // since MAX_CWND_INCREASE_BYTES_PER_RTT is a cap on how much the window size
-  // (max_window) may increase per RTT, we may not increase the window size more
-  // than that proportional to the number of bytes that were acked, so that once
-  // one window has been acked (one rtt) the increase limit is not exceeded the
-  // +1. is to allow for floating point imprecision
-  assert(scaled_gain <= 1.
-             + MAX_CWND_INCREASE_BYTES_PER_RTT
-                 * (double)min(bytes_acked, max_window)
-                 / (double)max(max_window, bytes_acked));
-
-  if(scaled_gain > 0 && ctx->current_ms - last_maxed_out_window > 1000)
-  {
-    // if it was more than 1 second since we tried to send a packet
-    // and stopped because we hit the max window, we're most likely rate
-    // limited (which prevents us from ever hitting the window size)
-    // if this is the case, we cannot let the max_window grow indefinitely
-    scaled_gain = 0;
-  }
-
-  size_t ledbat_cwnd = (max_window + scaled_gain < MIN_WINDOW_SIZE)
-      ? MIN_WINDOW_SIZE
-      : (size_t)(max_window + scaled_gain);
-
-  if(slow_start)
-  {
-    size_t ss_cwnd = (size_t)(max_window + window_factor * get_packet_size());
-    if(ss_cwnd > ssthresh)
-    {
-      slow_start = false;
-    }
-    else if(our_delay > target * 0.9)
-    {
-      // even if we're a little under the target delay, we conservatively
-      // discontinue the slow start phase
-      slow_start = false;
-      ssthresh   = max_window;
-    }
-    else
-    {
-      max_window = max(ss_cwnd, ledbat_cwnd);
-    }
-  }
-  else
-  {
-    max_window = ledbat_cwnd;
-  }
-
-  // make sure that the congestion window is below max
-  // make sure that we don't shrink our window too small
-  max_window = clamp< size_t >(max_window, MIN_WINDOW_SIZE, opt_sndbuf);
-
-  // used in parse_log.py
-  log(UTP_LOG_NORMAL,
-      "actual_delay:%u our_delay:%d their_delay:%u off_target:%d max_window:%u "
-      "delay_base:%u delay_sum:%d target_delay:%d acked_bytes:%u cur_window:%u "
-      "scaled_gain:%f rtt:%u rate:%u wnduser:%u rto:%u timeout:%d "
-      "get_microseconds:" I64u
-      " "
-      "cur_window_packets:%u packet_size:%u their_delay_base:%u "
-      "their_actual_delay:%u "
-      "average_delay:%d clock_drift:%d clock_drift_raw:%d delay_penalty:%d "
-      "current_delay_sum:" I64u
-      "current_delay_samples:%d average_delay_base:%d "
-      "last_maxed_out_window:" I64u
-      " opt_sndbuf:%d "
-      "current_ms:" I64u "",
-      actual_delay, our_delay / 1000, their_hist.get_value() / 1000,
-      int(off_target / 1000), uint(max_window), uint32(our_hist.delay_base),
-      int((our_delay + their_hist.get_value()) / 1000), int(target / 1000),
-      uint(bytes_acked), (uint)(cur_window - bytes_acked), (float)(scaled_gain),
-      rtt,
-      (uint)(max_window * 1000
-             / (rtt_hist.delay_base ? rtt_hist.delay_base : 50)),
-      (uint)max_window_user, rto, (int)(rto_timeout - ctx->current_ms),
-      utp_call_get_microseconds(this->ctx, this), cur_window_packets,
-      (uint)get_packet_size(), their_hist.delay_base,
-      their_hist.delay_base + their_hist.get_value(), average_delay,
-      clock_drift, clock_drift_raw, penalty / 1000, current_delay_sum,
-      current_delay_samples, average_delay_base, uint64(last_maxed_out_window),
-      int(opt_sndbuf), uint64(ctx->current_ms));
-}
-
-static void
-utp_register_recv_packet(UTPSocket *conn, size_t len)
-{
-#ifdef _DEBUG
-  ++conn->_stats.nrecv;
-  conn->_stats.nbytes_recv += len;
-#endif
-
-  if(len <= PACKET_SIZE_MID)
-  {
-    if(len <= PACKET_SIZE_EMPTY)
-    {
-      conn->ctx->context_stats._nraw_recv[PACKET_SIZE_EMPTY_BUCKET]++;
-    }
-    else if(len <= PACKET_SIZE_SMALL)
-    {
-      conn->ctx->context_stats._nraw_recv[PACKET_SIZE_SMALL_BUCKET]++;
-    }
-    else
-      conn->ctx->context_stats._nraw_recv[PACKET_SIZE_MID_BUCKET]++;
-  }
-  else
-  {
-    if(len <= PACKET_SIZE_BIG)
-    {
-      conn->ctx->context_stats._nraw_recv[PACKET_SIZE_BIG_BUCKET]++;
-    }
-    else
-      conn->ctx->context_stats._nraw_recv[PACKET_SIZE_HUGE_BUCKET]++;
-  }
-}
-
-// returns the max number of bytes of payload the uTP
-// connection is allowed to send
-size_t
-UTPSocket::get_packet_size() const
-{
-  int header_size = sizeof(PacketFormatV1);
-  size_t mtu      = mtu_last ? mtu_last : mtu_ceiling;
-  return mtu - header_size;
-}
-
-// Process an incoming packet
-// syn is true if this is the first packet received. It will cut off parsing
-// as soon as the header is done
-size_t
-utp_process_incoming(UTPSocket *conn, const byte *packet, size_t len,
-                     bool syn = false)
-{
-  utp_register_recv_packet(conn, len);
-
-  conn->ctx->current_ms = utp_call_get_milliseconds(conn->ctx, conn);
-
-  const PacketFormatV1 *pf1 = (PacketFormatV1 *)packet;
-  const byte *packet_end    = packet + len;
-
-  uint16 pk_seq_nr = pf1->seq_nr;
-  uint16 pk_ack_nr = pf1->ack_nr;
-  uint8 pk_flags   = pf1->type();
-
-  if(pk_flags >= ST_NUM_STATES)
-    return 0;
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG,
-            "Got %s. seq_nr:%u ack_nr:%u state:%s timestamp:" I64u
-            " reply_micro:%u",
-            flagnames[pk_flags], pk_seq_nr, pk_ack_nr, statenames[conn->state],
-            uint64(pf1->tv_usec), (uint32)(pf1->reply_micro));
-#endif
-
-  // mark receipt time
-  uint64 time = utp_call_get_microseconds(conn->ctx, conn);
-
-  // window packets size is used to calculate a minimum
-  // permissible range for received acks. connections with acks falling
-  // out of this range are dropped
-  const uint16 curr_window = max< uint16 >(
-      conn->cur_window_packets + ACK_NR_ALLOWED_WINDOW, ACK_NR_ALLOWED_WINDOW);
-
-  // ignore packets whose ack_nr is invalid. This would imply a spoofed address
-  // or a malicious attempt to attach the uTP implementation.
-  // acking a packet that hasn't been sent yet!
-  // SYN packets have an exception, since there are no previous packets
-  if((pk_flags != ST_SYN || conn->state != CS_SYN_RECV)
-     && (wrapping_compare_less(conn->seq_nr - 1, pk_ack_nr, ACK_NR_MASK)
-         || wrapping_compare_less(pk_ack_nr, conn->seq_nr - 1 - curr_window,
-                                  ACK_NR_MASK)))
-  {
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG,
-              "Invalid ack_nr: %u. our seq_nr: %u last unacked: %u", pk_ack_nr,
-              conn->seq_nr,
-              (conn->seq_nr - conn->cur_window_packets) & ACK_NR_MASK);
-#endif
-    return 0;
-  }
-
-  // RSTs are handled earlier, since the connid matches the send id not the recv
-  // id
-  assert(pk_flags != ST_RESET);
-
-  // TODO: maybe send a ST_RESET if we're in CS_RESET?
-
-  const byte *selack_ptr = NULL;
-
-  // Unpack UTP packet options
-  // Data pointer
-  const byte *data = (const byte *)pf1 + conn->get_header_size();
-  if(conn->get_header_size() > len)
-  {
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG, "Invalid packet size (less than header size)");
-#endif
-
-    return 0;
-  }
-  // Skip the extension headers
-  uint extension = pf1->ext;
-  if(extension != 0)
-  {
-    do
-    {
-      // Verify that the packet is valid.
-      data += 2;
-
-      if((int)(packet_end - data) < 0 || (int)(packet_end - data) < data[-1])
-      {
-#if UTP_DEBUG_LOGGING
-        conn->log(UTP_LOG_DEBUG, "Invalid len of extensions");
-#endif
-
-        return 0;
-      }
-
-      switch(extension)
-      {
-        case 1:  // Selective Acknowledgment
-          selack_ptr = data;
-          break;
-        case 2:  // extension bits
-          if(data[-1] != 8)
-          {
-#if UTP_DEBUG_LOGGING
-            conn->log(UTP_LOG_DEBUG, "Invalid len of extension bits header");
-#endif
-
-            return 0;
-          }
-          memcpy(conn->extensions, data, 8);
-
-#if UTP_DEBUG_LOGGING
-          conn->log(UTP_LOG_DEBUG,
-                    "got extension bits:%02x%02x%02x%02x%02x%02x%02x%02x",
-                    conn->extensions[0], conn->extensions[1],
-                    conn->extensions[2], conn->extensions[3],
-                    conn->extensions[4], conn->extensions[5],
-                    conn->extensions[6], conn->extensions[7]);
-#endif
-      }
-      extension = data[-2];
-      data += data[-1];
-    } while(extension);
-  }
-
-  if(conn->state == CS_SYN_SENT)
-  {
-    // if this is a syn-ack, initialize our ack_nr
-    // to match the sequence number we got from
-    // the other end
-    conn->ack_nr = (pk_seq_nr - 1) & SEQ_NR_MASK;
-  }
-
-  conn->last_got_packet = conn->ctx->current_ms;
-
-  if(syn)
-  {
-    return 0;
-  }
-
-  // seqnr is the number of packets past the expected
-  // packet this is. ack_nr is the last acked, seq_nr is the
-  // current. Subtracring 1 makes 0 mean "this is the next
-  // expected packet".
-  const uint seqnr = (pk_seq_nr - conn->ack_nr - 1) & SEQ_NR_MASK;
-
-  // Getting an invalid sequence number?
-  if(seqnr >= REORDER_BUFFER_MAX_SIZE)
-  {
-    if(seqnr >= (SEQ_NR_MASK + 1) - REORDER_BUFFER_MAX_SIZE
-       && pk_flags != ST_STATE)
-    {
-      conn->schedule_ack();
-    }
-
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG, "    Got old Packet/Ack (%u/%u)=%u", pk_seq_nr,
-              conn->ack_nr, seqnr);
-#endif
-    return 0;
-  }
-
-  // Process acknowledgment
-  // acks is the number of packets that was acked
-  int acks =
-      (pk_ack_nr - (conn->seq_nr - 1 - conn->cur_window_packets)) & ACK_NR_MASK;
-
-  // this happens when we receive an old ack nr
-  if(acks > conn->cur_window_packets)
-    acks = 0;
-
-  // if we get the same ack_nr as in the last packet
-  // increase the duplicate_ack counter, otherwise reset
-  // it to 0.
-  // It's important to only count ACKs in ST_STATE packets. Any other
-  // packet (primarily ST_DATA) is likely to have been sent because of the
-  // other end having new outgoing data, not in response to incoming data.
-  // For instance, if we're receiving a steady stream of payload with no
-  // outgoing data, and we suddently have a few bytes of payload to send (say,
-  // a bittorrent HAVE message), we're very likely to see 3 duplicate ACKs
-  // immediately after sending our payload packet. This effectively disables
-  // the fast-resend on duplicate-ack logic for bi-directional connections
-  // (except in the case of a selective ACK). This is in line with BSD4.4 TCP
-  // implementation.
-  if(conn->cur_window_packets > 0)
-  {
-    if(pk_ack_nr
-           == ((conn->seq_nr - conn->cur_window_packets - 1) & ACK_NR_MASK)
-       && conn->cur_window_packets > 0 && pk_flags == ST_STATE)
-    {
-      ++conn->duplicate_ack;
-      if(conn->duplicate_ack == DUPLICATE_ACKS_BEFORE_RESEND
-         && conn->mtu_probe_seq)
-      {
-        // It's likely that the probe was rejected due to its size, but we
-        // haven't got an ICMP report back yet
-        if(pk_ack_nr == ((conn->mtu_probe_seq - 1) & ACK_NR_MASK))
-        {
-          conn->mtu_ceiling = conn->mtu_probe_size - 1;
-          conn->mtu_search_update();
-          conn->log(UTP_LOG_MTU, "MTU [DUPACK] floor:%d ceiling:%d current:%d",
-                    conn->mtu_floor, conn->mtu_ceiling, conn->mtu_last);
-        }
-        else
-        {
-          // A non-probe was blocked before our probe.
-          // Can't conclude much, send a new probe
-          conn->mtu_probe_seq = conn->mtu_probe_size = 0;
-        }
-      }
-    }
-    else
-    {
-      conn->duplicate_ack = 0;
-    }
-
-    // TODO: if duplicate_ack == DUPLICATE_ACK_BEFORE_RESEND
-    // and fast_resend_seq_nr <= ack_nr + 1
-    //    resend ack_nr + 1
-    // also call maybe_decay_win()
-  }
-
-  // figure out how many bytes were acked
-  size_t acked_bytes = 0;
-
-  // the minimum rtt of all acks
-  // this is the upper limit on the delay we get back
-  // from the other peer. Our delay cannot exceed
-  // the rtt of the packet. If it does, clamp it.
-  // this is done in apply_ledbat_ccontrol()
-  int64 min_rtt = INT64_MAX;
-
-  uint64 now = utp_call_get_microseconds(conn->ctx, conn);
-
-  for(int i = 0; i < acks; ++i)
-  {
-    int seq = (conn->seq_nr - conn->cur_window_packets + i) & ACK_NR_MASK;
-    OutgoingPacket *pkt = (OutgoingPacket *)conn->outbuf.get(seq);
-    if(pkt == 0 || pkt->transmissions == 0)
-      continue;
-    assert((int)(pkt->payload) >= 0);
-    acked_bytes += pkt->payload;
-    if(conn->mtu_probe_seq && seq == static_cast< int >(conn->mtu_probe_seq))
-    {
-      conn->mtu_floor = conn->mtu_probe_size;
-      conn->mtu_search_update();
-      conn->log(UTP_LOG_MTU, "MTU [ACK] floor:%d ceiling:%d current:%d",
-                conn->mtu_floor, conn->mtu_ceiling, conn->mtu_last);
-    }
-
-    // in case our clock is not monotonic
-    if(pkt->time_sent < now)
-      min_rtt = min< int64 >(min_rtt, now - pkt->time_sent);
-    else
-      min_rtt = min< int64 >(min_rtt, 50000);
-  }
-
-  // count bytes acked by EACK
-  if(selack_ptr != NULL)
-  {
-    acked_bytes += conn->selective_ack_bytes(
-        (pk_ack_nr + 2) & ACK_NR_MASK, selack_ptr, selack_ptr[-1], min_rtt);
-  }
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG,
-            "acks:%d acked_bytes:%u seq_nr:%d cur_window:%u "
-            "cur_window_packets:%u relative_seqnr:%u max_window:%u min_rtt:%u "
-            "rtt:%u",
-            acks, (uint)acked_bytes, conn->seq_nr, (uint)conn->cur_window,
-            conn->cur_window_packets, seqnr, (uint)conn->max_window,
-            (uint)(min_rtt / 1000), conn->rtt);
-#endif
-
-  uint64 p = pf1->tv_usec;
-
-  conn->last_measured_delay = conn->ctx->current_ms;
-
-  // get delay in both directions
-  // record the delay to report back
-  const uint32 their_delay = (uint32)(p == 0 ? 0 : time - p);
-  conn->reply_micro        = their_delay;
-  uint32 prev_delay_base   = conn->their_hist.delay_base;
-  if(their_delay != 0)
-    conn->their_hist.add_sample(their_delay, conn->ctx->current_ms);
-
-  // if their new delay base is less than their previous one
-  // we should shift our delay base in the other direction in order
-  // to take the clock skew into account
-  if(prev_delay_base != 0
-     && wrapping_compare_less(conn->their_hist.delay_base, prev_delay_base,
-                              TIMESTAMP_MASK))
-  {
-    // never adjust more than 10 milliseconds
-    if(prev_delay_base - conn->their_hist.delay_base <= 10000)
-    {
-      conn->our_hist.shift(prev_delay_base - conn->their_hist.delay_base);
-    }
-  }
-
-  const uint32 actual_delay =
-      (uint32(pf1->reply_micro) == INT_MAX ? 0 : uint32(pf1->reply_micro));
-
-  // if the actual delay is 0, it means the other end
-  // hasn't received a sample from us yet, and doesn't
-  // know what it is. We can't update out history unless
-  // we have a true measured sample
-  if(actual_delay != 0)
-  {
-    conn->our_hist.add_sample(actual_delay, conn->ctx->current_ms);
-
-    // this is keeping an average of the delay samples
-    // we've recevied within the last 5 seconds. We sum
-    // all the samples and increase the count in order to
-    // calculate the average every 5 seconds. The samples
-    // are based off of the average_delay_base to deal with
-    // wrapping counters.
-    if(conn->average_delay_base == 0)
-      conn->average_delay_base = actual_delay;
-    int64 average_delay_sample = 0;
-    // distance walking from lhs to rhs, downwards
-    const uint32 dist_down = conn->average_delay_base - actual_delay;
-    // distance walking from lhs to rhs, upwards
-    const uint32 dist_up = actual_delay - conn->average_delay_base;
-
-    if(dist_down > dist_up)
-    {
-      //			assert(dist_up < INT_MAX / 4);
-      // average_delay_base < actual_delay, we should end up
-      // with a positive sample
-      average_delay_sample = dist_up;
-    }
-    else
-    {
-      //			assert(-int64(dist_down) < INT_MAX / 4);
-      // average_delay_base >= actual_delay, we should end up
-      // with a negative sample
-      average_delay_sample = -int64(dist_down);
-    }
-    conn->current_delay_sum += average_delay_sample;
-    ++conn->current_delay_samples;
-
-    if(conn->ctx->current_ms > conn->average_sample_time)
-    {
-      int32 prev_average_delay = conn->average_delay;
-
-      assert(conn->current_delay_sum / conn->current_delay_samples < INT_MAX);
-      assert(conn->current_delay_sum / conn->current_delay_samples > -INT_MAX);
-      // write the new average
-      conn->average_delay =
-          (int32)(conn->current_delay_sum / conn->current_delay_samples);
-      // each slot represents 5 seconds
-      conn->average_sample_time += 5000;
-
-      conn->current_delay_sum     = 0;
-      conn->current_delay_samples = 0;
-
-      // this makes things very confusing when logging the average delay
-      //#if !g_log_utp
-      // normalize the average samples
-      // since we're only interested in the slope
-      // of the curve formed by the average delay samples,
-      // we can cancel out the actual offset to make sure
-      // we won't have problems with wrapping.
-      int min_sample = min(prev_average_delay, conn->average_delay);
-      int max_sample = max(prev_average_delay, conn->average_delay);
-
-      // normalize around zero. Try to keep the min <= 0 and max >= 0
-      int adjust = 0;
-      if(min_sample > 0)
-      {
-        // adjust all samples (and the baseline) down by min_sample
-        adjust = -min_sample;
-      }
-      else if(max_sample < 0)
-      {
-        // adjust all samples (and the baseline) up by -max_sample
-        adjust = -max_sample;
-      }
-      if(adjust)
-      {
-        conn->average_delay_base -= adjust;
-        conn->average_delay += adjust;
-        prev_average_delay += adjust;
-      }
-      //#endif
-
-      // update the clock drift estimate
-      // the unit is microseconds per 5 seconds
-      // what we're doing is just calculating the average of the
-      // difference between each slot. Since each slot is 5 seconds
-      // and the timestamps unit are microseconds, we'll end up with
-      // the average slope across our history. If there is a consistent
-      // trend, it will show up in this value
-
-      // int64 slope = 0;
-      int32 drift = conn->average_delay - prev_average_delay;
-
-      // clock_drift is a rolling average
-      conn->clock_drift     = (int64(conn->clock_drift) * 7 + drift) / 8;
-      conn->clock_drift_raw = drift;
-    }
-  }
-
-  // if our new delay base is less than our previous one
-  // we should shift the other end's delay base in the other
-  // direction in order to take the clock skew into account
-  // This is commented out because it creates bad interactions
-  // with our adjustment in the other direction. We don't really
-  // need our estimates of the other peer to be very accurate
-  // anyway. The problem with shifting here is that we're more
-  // likely shift it back later because of a low latency. This
-  // second shift back would cause us to shift our delay base
-  // which then get's into a death spiral of shifting delay bases
-  /*	if (prev_delay_base != 0 &&
-                  wrapping_compare_less(conn->our_hist.delay_base,
-     prev_delay_base)) {
-                  // never adjust more than 10 milliseconds
-                  if (prev_delay_base - conn->our_hist.delay_base <= 10000) {
-                          conn->their_hist.Shift(prev_delay_base -
-     conn->our_hist.delay_base);
-                  }
-          }
-  */
-
-  // if the delay estimate exceeds the RTT, adjust the base_delay to
-  // compensate
-  assert(min_rtt >= 0);
-  if(int64(conn->our_hist.get_value()) > min_rtt)
-  {
-    conn->our_hist.shift((uint32)(conn->our_hist.get_value() - min_rtt));
-  }
-
-  // only apply the congestion controller on acks
-  // if we don't have a delay measurement, there's
-  // no point in invoking the congestion control
-  if(actual_delay != 0 && acked_bytes >= 1)
-    conn->apply_ccontrol(acked_bytes, actual_delay, min_rtt);
-
-  // sanity check, the other end should never ack packets
-  // past the point we've sent
-  if(acks <= conn->cur_window_packets)
-  {
-    conn->max_window_user = pf1->windowsize;
-
-    // If max user window is set to 0, then we startup a timer
-    // That will reset it to 1 after 15 seconds.
-    if(conn->max_window_user == 0)
-      // Reset max_window_user to 1 every 15 seconds.
-      conn->zerowindow_time = conn->ctx->current_ms + 15000;
-
-    // Respond to connect message
-    // Switch to CONNECTED state.
-    // If this is an ack and we're in still handshaking
-    // transition over to the connected state.
-
-    // Incoming connection completion
-    if(pk_flags == ST_DATA && conn->state == CS_SYN_RECV)
-    {
-      conn->state = CS_CONNECTED;
-    }
-
-    // Outgoing connection completion
-    if(pk_flags == ST_STATE && conn->state == CS_SYN_SENT)
-    {
-      conn->state = CS_CONNECTED;
-
-      // If the user has defined the ON_CONNECT callback, use that to
-      // notify the user that the socket is now connected.  If ON_CONNECT
-      // has not been defined, notify the user via ON_STATE_CHANGE.
-      if(conn->ctx->callbacks[UTP_ON_CONNECT])
-        utp_call_on_connect(conn->ctx, conn);
-      else
-        utp_call_on_state_change(conn->ctx, conn, UTP_STATE_CONNECT);
-
-      // We've sent a fin, and everything was ACKed (including the FIN).
-      // cur_window_packets == acks means that this packet acked all
-      // the remaining packets that were in-flight.
-    }
-    else if(conn->fin_sent && conn->cur_window_packets == acks)
-    {
-      conn->fin_sent_acked = true;
-      if(conn->close_requested)
-      {
-        conn->state = CS_DESTROY;
-      }
-    }
-
-    // Update fast resend counter
-    if(wrapping_compare_less(conn->fast_resend_seq_nr,
-                             (pk_ack_nr + 1) & ACK_NR_MASK, ACK_NR_MASK))
-      conn->fast_resend_seq_nr = (pk_ack_nr + 1) & ACK_NR_MASK;
-
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG, "fast_resend_seq_nr:%u", conn->fast_resend_seq_nr);
-#endif
-
-    for(int i = 0; i < acks; ++i)
-    {
-      int ack_status =
-          conn->ack_packet(conn->seq_nr - conn->cur_window_packets);
-      // if ack_status is 0, the packet was acked.
-      // if acl_stauts is 1, it means that the packet had already been acked
-      // if it's 2, the packet has not been sent yet
-      // We need to break this loop in the latter case. This could potentially
-      // happen if we get an ack_nr that does not exceed what we have stuffed
-      // into the outgoing buffer, but does exceed what we have sent
-      if(ack_status == 2)
-      {
-#ifdef _DEBUG
-        OutgoingPacket *pkt = (OutgoingPacket *)conn->outbuf.get(
-            conn->seq_nr - conn->cur_window_packets);
-        assert(pkt->transmissions == 0);
-#endif
-
-        break;
-      }
-      conn->cur_window_packets--;
-
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG, "decementing cur_window_packets:%u",
-                conn->cur_window_packets);
-#endif
-    }
-
-#ifdef _DEBUG
-    if(conn->cur_window_packets == 0)
-      assert(conn->cur_window == 0);
-#endif
-
-    // packets in front of this may have been acked by a
-    // selective ack (EACK). Keep decreasing the window packet size
-    // until we hit a packet that is still waiting to be acked
-    // in the send queue
-    // this is especially likely to happen when the other end
-    // has the EACK send bug older versions of uTP had
-    while(conn->cur_window_packets > 0
-          && !conn->outbuf.get(conn->seq_nr - conn->cur_window_packets))
-    {
-      conn->cur_window_packets--;
-
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG, "decementing cur_window_packets:%u",
-                conn->cur_window_packets);
-#endif
-    }
-
-#ifdef _DEBUG
-    if(conn->cur_window_packets == 0)
-      assert(conn->cur_window == 0);
-#endif
-
-    // this invariant should always be true
-    assert(conn->cur_window_packets == 0
-           || conn->outbuf.get(conn->seq_nr - conn->cur_window_packets));
-
-    // flush Nagle
-    if(conn->cur_window_packets == 1)
-    {
-      OutgoingPacket *pkt =
-          (OutgoingPacket *)conn->outbuf.get(conn->seq_nr - 1);
-      // do we still have quota?
-      if(pkt->transmissions == 0)
-      {
-        conn->send_packet(pkt);
-      }
-    }
-
-    // Fast timeout-retry
-    if(conn->fast_timeout)
-    {
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG, "Fast timeout %u,%u,%u?", (uint)conn->cur_window,
-                conn->seq_nr - conn->timeout_seq_nr, conn->timeout_seq_nr);
-#endif
-
-      // if the fast_resend_seq_nr is not pointing to the oldest outstanding
-      // packet, it suggests that we've already resent the packet that timed
-      // out, and we should leave the fast-timeout mode.
-      if(((conn->seq_nr - conn->cur_window_packets) & ACK_NR_MASK)
-         != conn->fast_resend_seq_nr)
-      {
-        conn->fast_timeout = false;
-      }
-      else
-      {
-        // resend the oldest packet and increment fast_resend_seq_nr
-        // to not allow another fast resend on it again
-        OutgoingPacket *pkt = (OutgoingPacket *)conn->outbuf.get(
-            conn->seq_nr - conn->cur_window_packets);
-        if(pkt && pkt->transmissions > 0)
-        {
-#if UTP_DEBUG_LOGGING
-          conn->log(UTP_LOG_DEBUG, "Packet %u fast timeout-retry.",
-                    conn->seq_nr - conn->cur_window_packets);
-#endif
-
-#ifdef _DEBUG
-          ++conn->_stats.fastrexmit;
-#endif
-
-          conn->fast_resend_seq_nr++;
-          conn->send_packet(pkt);
-        }
-      }
-    }
-  }
-
-  // Process selective acknowledgent
-  if(selack_ptr != NULL)
-  {
-    conn->selective_ack(pk_ack_nr + 2, selack_ptr, selack_ptr[-1]);
-  }
-
-  // this invariant should always be true
-  assert(conn->cur_window_packets == 0
-         || conn->outbuf.get(conn->seq_nr - conn->cur_window_packets));
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG,
-            "acks:%d acked_bytes:%u seq_nr:%u cur_window:%u "
-            "cur_window_packets:%u ",
-            acks, (uint)acked_bytes, conn->seq_nr, (uint)conn->cur_window,
-            conn->cur_window_packets);
-#endif
-
-  // In case the ack dropped the current window below
-  // the max_window size, Mark the socket as writable
-  if(conn->state == CS_CONNECTED_FULL && !conn->is_full())
-  {
-    conn->state = CS_CONNECTED;
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG,
-              "Socket writable. max_window:%u cur_window:%u packet_size:%u",
-              (uint)conn->max_window, (uint)conn->cur_window,
-              (uint)conn->get_packet_size());
-#endif
-    utp_call_on_state_change(conn->ctx, conn, UTP_STATE_WRITABLE);
-  }
-
-  if(pk_flags == ST_STATE)
-  {
-    // This is a state packet only.
-    return 0;
-  }
-
-  // The connection is not in a state that can accept data?
-  if(conn->state != CS_CONNECTED && conn->state != CS_CONNECTED_FULL)
-  {
-    return 0;
-  }
-
-  // Is this a finalize packet?
-  if(pk_flags == ST_FIN && !conn->got_fin)
-  {
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG, "Got FIN eof_pkt:%u", pk_seq_nr);
-#endif
-
-    conn->got_fin = true;
-    conn->eof_pkt = pk_seq_nr;
-    // at this point, it is possible for the
-    // other end to have sent packets with
-    // sequence numbers higher than seq_nr.
-    // if this is the case, our reorder_count
-    // is out of sync. This case is dealt with
-    // when we re-order and hit the eof_pkt.
-    // we'll just ignore any packets with
-    // sequence numbers past this
-  }
-
-  // Getting an in-order packet?
-  if(seqnr == 0)
-  {
-    size_t count = packet_end - data;
-    if(count > 0 && !conn->read_shutdown)
-    {
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG, "Got Data len:%u (rb:%u)", (uint)count,
-                (uint)utp_call_get_read_buffer_size(conn->ctx, conn));
-#endif
-
-      // Post bytes to the upper layer
-      utp_call_on_read(conn->ctx, conn, data, count);
-    }
-    conn->ack_nr++;
-
-    // Check if the next packet has been received too, but waiting
-    // in the reorder buffer.
-    for(;;)
-    {
-      if(!conn->got_fin_reached && conn->got_fin
-         && conn->eof_pkt == conn->ack_nr)
-      {
-        conn->got_fin_reached = true;
-        conn->rto_timeout =
-            conn->ctx->current_ms + min< uint >(conn->rto * 3, 60);
-
-#if UTP_DEBUG_LOGGING
-        conn->log(UTP_LOG_DEBUG, "Posting EOF");
-#endif
-
-        utp_call_on_state_change(conn->ctx, conn, UTP_STATE_EOF);
-
-        // if the other end wants to close, ack
-        conn->send_ack();
-
-        // reorder_count is not necessarily 0 at this point.
-        // even though it is most of the time, the other end
-        // may have sent packets with higher sequence numbers
-        // than what later end up being eof_pkt
-        // since we have received all packets up to eof_pkt
-        // just ignore the ones after it.
-        conn->reorder_count = 0;
-      }
-
-      // Quick get-out in case there is nothing to reorder
-      if(conn->reorder_count == 0)
-        break;
-
-      // Check if there are additional buffers in the reorder buffers
-      // that need delivery.
-      byte *p = (byte *)conn->inbuf.get(conn->ack_nr + 1);
-      if(p == NULL)
-        break;
-      conn->inbuf.put(conn->ack_nr + 1, NULL);
-      count = *(uint *)p;
-      if(count > 0 && !conn->read_shutdown)
-      {
-        // Pass the bytes to the upper layer
-        utp_call_on_read(conn->ctx, conn, p + sizeof(uint), count);
-      }
-      conn->ack_nr++;
-
-      // Free the element from the reorder buffer
-      free(p);
-      assert(conn->reorder_count > 0);
-      conn->reorder_count--;
-    }
-
-    conn->schedule_ack();
-  }
-  else
-  {
-    // Getting an out of order packet.
-    // The packet needs to be remembered and rearranged later.
-
-    // if we have received a FIN packet, and the EOF-sequence number
-    // is lower than the sequence number of the packet we just received
-    // something is wrong.
-    if(conn->got_fin && pk_seq_nr > conn->eof_pkt)
-    {
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG,
-                "Got an invalid packet sequence number, past EOF "
-                "reorder_count:%u len:%u (rb:%u)",
-                conn->reorder_count, (uint)(packet_end - data),
-                (uint)utp_call_get_read_buffer_size(conn->ctx, conn));
-#endif
-      return 0;
-    }
-
-    // if the sequence number is entirely off the expected
-    // one, just drop it. We can't allocate buffer space in
-    // the inbuf entirely based on untrusted input
-    if(seqnr > 0x3ff)
-    {
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG,
-                "0x%08x: Got an invalid packet sequence number, too far off "
-                "reorder_count:%u len:%u (rb:%u)",
-                conn->reorder_count, (uint)(packet_end - data),
-                (uint)utp_call_get_read_buffer_size(conn->ctx, conn));
-#endif
-      return 0;
-    }
-
-    // we need to grow the circle buffer before we
-    // check if the packet is already in here, so that
-    // we don't end up looking at an older packet (since
-    // the indices wraps around).
-    conn->inbuf.ensure_size(pk_seq_nr + 1, seqnr + 1);
-
-    // Has this packet already been received? (i.e. a duplicate)
-    // If that is the case, just discard it.
-    if(conn->inbuf.get(pk_seq_nr) != NULL)
-    {
-#ifdef _DEBUG
-      ++conn->_stats.nduprecv;
-#endif
-
-      return 0;
-    }
-
-    // Allocate memory to fit the packet that needs to re-ordered
-    byte *mem    = (byte *)malloc((packet_end - data) + sizeof(uint));
-    *(uint *)mem = (uint)(packet_end - data);
-    memcpy(mem + sizeof(uint), data, packet_end - data);
-
-    // Insert into reorder buffer and increment the count
-    // of # of packets to be reordered.
-    // we add one to seqnr in order to leave the last
-    // entry empty, that way the assert in send_ack
-    // is valid. we have to add one to seqnr too, in order
-    // to make the circular buffer grow around the correct
-    // point (which is conn->ack_nr + 1).
-    assert(conn->inbuf.get(pk_seq_nr) == NULL);
-    assert((pk_seq_nr & conn->inbuf.mask)
-           != ((conn->ack_nr + 1) & conn->inbuf.mask));
-    conn->inbuf.put(pk_seq_nr, mem);
-    conn->reorder_count++;
-
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG,
-              "0x%08x: Got out of order data reorder_count:%u len:%u (rb:%u)",
-              conn->reorder_count, (uint)(packet_end - data),
-              (uint)utp_call_get_read_buffer_size(conn->ctx, conn));
-#endif
-
-    conn->schedule_ack();
-  }
-
-  return (size_t)(packet_end - data);
-}
-
-inline byte
-UTP_Version(PacketFormatV1 const *pf)
-{
-  return (pf->type() < ST_NUM_STATES && pf->ext < 3 ? pf->version() : 0);
-}
-
-UTPSocket::~UTPSocket()
-{
-#if UTP_DEBUG_LOGGING
-  log(UTP_LOG_DEBUG, "Killing socket");
-#endif
-
-  utp_call_on_state_change(ctx, this, UTP_STATE_DESTROYING);
-
-  if(ctx->last_utp_socket == this)
-  {
-    ctx->last_utp_socket = NULL;
-  }
-
-  // Remove object from the global hash table
-  UTPSocketKeyData *kd =
-      ctx->utp_sockets->Delete(UTPSocketKey(addr, conn_id_recv));
-  assert(kd);
-  (void)kd;
-  // remove the socket from ack_sockets if it was there also
-  removeSocketFromAckList(this);
-
-  // Free all memory occupied by the socket object.
-  for(size_t i = 0; i <= inbuf.mask; i++)
-  {
-    free(inbuf.elements[i]);
-  }
-  for(size_t i = 0; i <= outbuf.mask; i++)
-  {
-    free(outbuf.elements[i]);
-  }
-  // TODO: The circular buffer should have a destructor
-  free(inbuf.elements);
-  free(outbuf.elements);
-}
-
-void
-UTP_FreeAll(struct UTPSocketHT *utp_sockets)
-{
-  utp_hash_iterator_t it;
-  UTPSocketKeyData *keyData;
-  while((keyData = utp_sockets->Iterate(it)))
-  {
-    delete keyData->socket;
-  }
-}
-
-void
-utp_initialize_socket(utp_socket *conn, const struct sockaddr *addr,
-                      socklen_t addrlen, bool need_seed_gen, uint32 conn_seed,
-                      uint32 conn_id_recv, uint32 conn_id_send)
-{
-  PackedSockAddr psaddr =
-      PackedSockAddr((const SOCKADDR_STORAGE *)addr, addrlen);
-
-  if(need_seed_gen)
-  {
-    do
-    {
-      conn_seed = utp_call_get_random(conn->ctx, conn);
-      // we identify v1 and higher by setting the first two bytes to 0x0001
-      conn_seed &= 0xffff;
-    } while(conn->ctx->utp_sockets->Lookup(UTPSocketKey(psaddr, conn_seed)));
-
-    conn_id_recv += conn_seed;
-    conn_id_send += conn_seed;
-  }
-
-  conn->state               = CS_IDLE;
-  conn->conn_seed           = conn_seed;
-  conn->conn_id_recv        = conn_id_recv;
-  conn->conn_id_send        = conn_id_send;
-  conn->addr                = psaddr;
-  conn->ctx->current_ms     = utp_call_get_milliseconds(conn->ctx, NULL);
-  conn->last_got_packet     = conn->ctx->current_ms;
-  conn->last_sent_packet    = conn->ctx->current_ms;
-  conn->last_measured_delay = conn->ctx->current_ms + 0x70000000;
-  conn->average_sample_time = conn->ctx->current_ms + 5000;
-  conn->last_rwin_decay     = conn->ctx->current_ms - MAX_WINDOW_DECAY;
-
-  conn->our_hist.clear(conn->ctx->current_ms);
-  conn->their_hist.clear(conn->ctx->current_ms);
-  conn->rtt_hist.clear(conn->ctx->current_ms);
-
-  // initialize MTU floor and ceiling
-  conn->mtu_reset();
-  conn->mtu_last = conn->mtu_ceiling;
-
-  conn->ctx->utp_sockets->Add(UTPSocketKey(conn->addr, conn->conn_id_recv))
-      ->socket = conn;
-
-  // we need to fit one packet in the window when we start the connection
-  conn->max_window = conn->get_packet_size();
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "UTP socket initialized");
-#endif
-}
-
-utp_socket *
-utp_create_socket(utp_context *ctx)
-{
-  assert(ctx);
-  if(!ctx)
-    return NULL;
-
-  UTPSocket *conn = new UTPSocket;  // TODO: UTPSocket should have a constructor
-
-  conn->state                 = CS_UNINITIALIZED;
-  conn->ctx                   = ctx;
-  conn->userdata              = NULL;
-  conn->reorder_count         = 0;
-  conn->duplicate_ack         = 0;
-  conn->timeout_seq_nr        = 0;
-  conn->last_rcv_win          = 0;
-  conn->got_fin               = false;
-  conn->got_fin_reached       = false;
-  conn->fin_sent              = false;
-  conn->fin_sent_acked        = false;
-  conn->read_shutdown         = false;
-  conn->close_requested       = false;
-  conn->fast_timeout          = false;
-  conn->rtt                   = 0;
-  conn->retransmit_timeout    = 0;
-  conn->rto_timeout           = 0;
-  conn->zerowindow_time       = 0;
-  conn->average_delay         = 0;
-  conn->current_delay_samples = 0;
-  conn->cur_window            = 0;
-  conn->eof_pkt               = 0;
-  conn->last_maxed_out_window = 0;
-  conn->mtu_probe_seq         = 0;
-  conn->mtu_probe_size        = 0;
-  conn->current_delay_sum     = 0;
-  conn->average_delay_base    = 0;
-  conn->retransmit_count      = 0;
-  conn->rto                   = 3000;
-  conn->rtt_var               = 800;
-  conn->seq_nr                = 1;
-  conn->ack_nr                = 0;
-  conn->max_window_user       = 255 * PACKET_SIZE;
-  conn->cur_window_packets    = 0;
-  conn->fast_resend_seq_nr    = conn->seq_nr;
-  conn->target_delay          = ctx->target_delay;
-  conn->reply_micro           = 0;
-  conn->opt_sndbuf            = ctx->opt_sndbuf;
-  conn->opt_rcvbuf            = ctx->opt_rcvbuf;
-  conn->slow_start            = true;
-  conn->ssthresh              = conn->opt_sndbuf;
-  conn->clock_drift           = 0;
-  conn->clock_drift_raw       = 0;
-  conn->outbuf.mask           = 15;
-  conn->inbuf.mask            = 15;
-  conn->outbuf.elements       = (void **)calloc(16, sizeof(void *));
-  conn->inbuf.elements        = (void **)calloc(16, sizeof(void *));
-  conn->ida = -1;  // set the index of every new socket in ack_sockets to
-                   // -1, which also means it is not in ack_sockets yet
-
-  memset(conn->extensions, 0, sizeof(conn->extensions));
-
-#ifdef _DEBUG
-  memset(&conn->_stats, 0, sizeof(utp_socket_stats));
-#endif
-
-  return conn;
-}
-
-int
-utp_context_set_option(utp_context *ctx, int opt, int val)
-{
-  assert(ctx);
-  if(!ctx)
-    return -1;
-
-  switch(opt)
-  {
-    case UTP_LOG_NORMAL:
-      ctx->log_normal = val ? true : false;
-      return 0;
-
-    case UTP_LOG_MTU:
-      ctx->log_mtu = val ? true : false;
-      return 0;
-
-    case UTP_LOG_DEBUG:
-      ctx->log_debug = val ? true : false;
-      return 0;
-
-    case UTP_TARGET_DELAY:
-      ctx->target_delay = val;
-      return 0;
-
-    case UTP_SNDBUF:
-      assert(val >= 1);
-      ctx->opt_sndbuf = val;
-      return 0;
-
-    case UTP_RCVBUF:
-      assert(val >= 1);
-      ctx->opt_rcvbuf = val;
-      return 0;
-  }
-  return -1;
-}
-
-int
-utp_context_get_option(utp_context *ctx, int opt)
-{
-  assert(ctx);
-  if(!ctx)
-    return -1;
-
-  switch(opt)
-  {
-    case UTP_LOG_NORMAL:
-      return ctx->log_normal ? 1 : 0;
-    case UTP_LOG_MTU:
-      return ctx->log_mtu ? 1 : 0;
-    case UTP_LOG_DEBUG:
-      return ctx->log_debug ? 1 : 0;
-    case UTP_TARGET_DELAY:
-      return ctx->target_delay;
-    case UTP_SNDBUF:
-      return ctx->opt_sndbuf;
-    case UTP_RCVBUF:
-      return ctx->opt_rcvbuf;
-  }
-  return -1;
-}
-
-int
-utp_setsockopt(UTPSocket *conn, int opt, int val)
-{
-  assert(conn);
-  if(!conn)
-    return -1;
-
-  switch(opt)
-  {
-    case UTP_SNDBUF:
-      assert(val >= 1);
-      conn->opt_sndbuf = val;
-      return 0;
-
-    case UTP_RCVBUF:
-      assert(val >= 1);
-      conn->opt_rcvbuf = val;
-      return 0;
-
-    case UTP_TARGET_DELAY:
-      conn->target_delay = val;
-      return 0;
-  }
-
-  return -1;
-}
-
-int
-utp_getsockopt(UTPSocket *conn, int opt)
-{
-  assert(conn);
-  if(!conn)
-    return -1;
-
-  switch(opt)
-  {
-    case UTP_SNDBUF:
-      return conn->opt_sndbuf;
-    case UTP_RCVBUF:
-      return conn->opt_rcvbuf;
-    case UTP_TARGET_DELAY:
-      return conn->target_delay;
-  }
-
-  return -1;
-}
-
-// Try to connect to a specified host.
-int
-utp_connect(utp_socket *conn, const struct sockaddr *to, socklen_t tolen)
-{
-  assert(conn);
-  if(!conn)
-    return -1;
-
-  assert(conn->state == CS_UNINITIALIZED);
-  if(conn->state != CS_UNINITIALIZED)
-  {
-    conn->state = CS_DESTROY;
-    return -1;
-  }
-
-  utp_initialize_socket(conn, to, tolen, true, 0, 0, 1);
-
-  assert(conn->cur_window_packets == 0);
-  assert(conn->outbuf.get(conn->seq_nr) == NULL);
-  assert(sizeof(PacketFormatV1) == 20);
-
-  conn->state           = CS_SYN_SENT;
-  conn->ctx->current_ms = utp_call_get_milliseconds(conn->ctx, conn);
-
-  // Create and send a connect message
-
-  // used in parse_log.py
-  conn->log(UTP_LOG_NORMAL,
-            "UTP_Connect conn_seed:%u packet_size:%u (B) "
-            "target_delay:%u (ms) delay_history:%u "
-            "delay_base_history:%u (minutes)",
-            conn->conn_seed, PACKET_SIZE,
-            static_cast< unsigned int >(conn->target_delay / 1000),
-            CUR_DELAY_SIZE, DELAY_BASE_HISTORY);
-
-  // Setup initial timeout timer.
-  conn->retransmit_timeout = 3000;
-  conn->rto_timeout        = conn->ctx->current_ms + conn->retransmit_timeout;
-  conn->last_rcv_win       = conn->get_rcv_window();
-
-  // if you need compatibiltiy with 1.8.1, use this. it increases attackability
-  // though.
-  // conn->seq_nr = 1;
-  conn->seq_nr = utp_call_get_random(conn->ctx, conn);
-
-  // Create the connect packet.
-  const size_t header_size = sizeof(PacketFormatV1);
-
-  OutgoingPacket *pkt =
-      (OutgoingPacket *)malloc(sizeof(OutgoingPacket) - 1 + header_size);
-  PacketFormatV1 *p1 = (PacketFormatV1 *)pkt->data;
-
-  memset(p1, 0, header_size);
-  // SYN packets are special, and have the receive ID in the connid field,
-  // instead of conn_id_send.
-  p1->set_version(1);
-  p1->set_type(ST_SYN);
-  p1->ext            = 0;
-  p1->connid         = conn->conn_id_recv;
-  p1->windowsize     = (uint32)conn->last_rcv_win;
-  p1->seq_nr         = conn->seq_nr;
-  pkt->transmissions = 0;
-  pkt->length        = header_size;
-  pkt->payload       = 0;
-
-  /*
-  #if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "Sending connect %s [%u].",
-                  addrfmt(conn->addr, addrbuf), conn_seed);
-  #endif
-  */
-
-  // Remember the message in the outgoing queue.
-  conn->outbuf.ensure_size(conn->seq_nr, conn->cur_window_packets);
-  conn->outbuf.put(conn->seq_nr, pkt);
-  conn->seq_nr++;
-  conn->cur_window_packets++;
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "incrementing cur_window_packets:%u",
-            conn->cur_window_packets);
-#endif
-
-  conn->send_packet(pkt);
-  return 0;
-}
-
-// Returns 1 if the UDP payload was recognized as a UTP packet, or 0 if it was
-// not
-int
-utp_process_udp(utp_context *ctx, const byte *buffer, size_t len,
-                const struct sockaddr *to, socklen_t tolen)
-{
-  assert(ctx);
-  if(!ctx)
-    return 0;
-
-  assert(buffer);
-  if(!buffer)
-    return 0;
-
-  assert(to);
-  if(!to)
-    return 0;
-
-  const PackedSockAddr addr((const SOCKADDR_STORAGE *)to, tolen);
-
-  if(len < sizeof(PacketFormatV1))
-  {
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL, "recv %s len:%u too small",
-             addrfmt(addr, addrbuf), (uint)len);
-#endif
-    return 0;
-  }
-
-  const PacketFormatV1 *pf1 = (PacketFormatV1 *)buffer;
-  const byte version        = UTP_Version(pf1);
-  const uint32 id           = uint32(pf1->connid);
-
-  if(version != 1)
-  {
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL,
-             "recv %s len:%u version:%u unsupported version",
-             addrfmt(addr, addrbuf), (uint)len, version);
-#endif
-
-    return 0;
-  }
-
-#if UTP_DEBUG_LOGGING
-  ctx->log(UTP_LOG_DEBUG, NULL, "recv %s len:%u id:%u", addrfmt(addr, addrbuf),
-           (uint)len, id);
-  ctx->log(UTP_LOG_DEBUG, NULL, "recv id:%u seq_nr:%u ack_nr:%u", id,
-           (uint)pf1->seq_nr, (uint)pf1->ack_nr);
-#endif
-
-  const byte flags = pf1->type();
-
-  if(flags == ST_RESET)
-  {
-    // id is either our recv id or our send id
-    // if it's our send id, and we initiated the connection, our recv id is id +
-    // 1 if it's our send id, and we did not initiate the connection, our recv
-    // id is id - 1 we have to check every case
-
-    UTPSocketKeyData *keyData;
-    if((keyData = ctx->utp_sockets->Lookup(UTPSocketKey(addr, id)))
-       || ((keyData = ctx->utp_sockets->Lookup(UTPSocketKey(addr, id + 1)))
-           && keyData->socket->conn_id_send == id)
-       || ((keyData = ctx->utp_sockets->Lookup(UTPSocketKey(addr, id - 1)))
-           && keyData->socket->conn_id_send == id))
-    {
-      UTPSocket *conn = keyData->socket;
-
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL, "recv RST for existing connection");
-#endif
-
-      if(conn->close_requested)
-        conn->state = CS_DESTROY;
-      else
-        conn->state = CS_RESET;
-
-      utp_call_on_overhead_statistics(conn->ctx, conn, false,
-                                      len + conn->get_udp_overhead(),
-                                      close_overhead);
-      const int err =
-          (conn->state == CS_SYN_SENT) ? UTP_ECONNREFUSED : UTP_ECONNRESET;
-      utp_call_on_error(conn->ctx, conn, err);
-    }
-    else
-    {
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL, "recv RST for unknown connection");
-#endif
-    }
-    return 1;
-  }
-  else if(flags != ST_SYN)
-  {
-    UTPSocket *conn = NULL;
-
-    if(ctx->last_utp_socket && ctx->last_utp_socket->addr == addr
-       && ctx->last_utp_socket->conn_id_recv == id)
-    {
-      conn = ctx->last_utp_socket;
-    }
-    else
-    {
-      UTPSocketKeyData *keyData =
-          ctx->utp_sockets->Lookup(UTPSocketKey(addr, id));
-      if(keyData)
-      {
-        conn                 = keyData->socket;
-        ctx->last_utp_socket = conn;
-      }
-    }
-
-    if(conn)
-    {
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL, "recv processing");
-#endif
-
-      const size_t read = utp_process_incoming(conn, buffer, len);
-      utp_call_on_overhead_statistics(conn->ctx, conn, false,
-                                      (len - read) + conn->get_udp_overhead(),
-                                      header_overhead);
-      return 1;
-    }
-  }
-
-  // We have not found a matching utp_socket, and this isn't a SYN.  Reject it.
-  const uint32 seq_nr = pf1->seq_nr;
-  if(flags != ST_SYN)
-  {
-    ctx->current_ms = utp_call_get_milliseconds(ctx, NULL);
-
-    for(size_t i = 0; i < ctx->rst_info.GetCount(); i++)
-    {
-      if((ctx->rst_info[i].connid == id) && (ctx->rst_info[i].addr == addr)
-         && (ctx->rst_info[i].ack_nr == seq_nr))
-      {
-        ctx->rst_info[i].timestamp = ctx->current_ms;
-
-#if UTP_DEBUG_LOGGING
-        ctx->log(UTP_LOG_DEBUG, NULL,
-                 "recv not sending RST to non-SYN (stored)");
-#endif
-
-        return 1;
-      }
-    }
-
-    if(ctx->rst_info.GetCount() > RST_INFO_LIMIT)
-    {
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL,
-               "recv not sending RST to non-SYN (limit at %u stored)",
-               (uint)ctx->rst_info.GetCount());
-#endif
-
-      return 1;
-    }
-
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL, "recv send RST to non-SYN (%u stored)",
-             (uint)ctx->rst_info.GetCount());
-#endif
-
-    RST_Info &r = ctx->rst_info.Append();
-    r.addr      = addr;
-    r.connid    = id;
-    r.ack_nr    = seq_nr;
-    r.timestamp = ctx->current_ms;
-
-    UTPSocket::send_rst(ctx, addr, id, seq_nr, utp_call_get_random(ctx, NULL));
-    return 1;
-  }
-
-  if(ctx->callbacks[UTP_ON_ACCEPT])
-  {
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL, "Incoming connection from %s",
-             addrfmt(addr, addrbuf));
-#endif
-
-    UTPSocketKeyData *keyData =
-        ctx->utp_sockets->Lookup(UTPSocketKey(addr, id + 1));
-    if(keyData)
-    {
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL,
-               "rejected incoming connection, connection already exists");
-#endif
-
-      return 1;
-    }
-    /*
-    if(ctx->utp_sockets->GetCount() > 3000)
-    {
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL,
-               "rejected incoming connection, too many uTP sockets %d",
-               ctx->utp_sockets->GetCount());
-#endif
-
-      return 1;
-    }
-    */
-    // true means yes, block connection.  false means no, don't block.
-    if(utp_call_on_firewall(ctx, to, tolen))
-    {
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL,
-               "rejected incoming connection, firewall callback returned true");
-#endif
-
-      return 1;
-    }
-
-    // Create a new UTP socket to handle this new connection
-    UTPSocket *conn = utp_create_socket(ctx);
-    utp_initialize_socket(conn, to, tolen, false, id, id + 1, id);
-    conn->ack_nr             = seq_nr;
-    conn->seq_nr             = utp_call_get_random(ctx, NULL);
-    conn->fast_resend_seq_nr = conn->seq_nr;
-    conn->state              = CS_SYN_RECV;
-
-    const size_t read = utp_process_incoming(conn, buffer, len, true);
-
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL, "recv send connect ACK");
-#endif
-
-    conn->send_ack(true);
-
-    utp_call_on_accept(ctx, conn, to, tolen);
-
-    // we report overhead after on_accept(), because the callbacks are setup now
-    utp_call_on_overhead_statistics(conn->ctx, conn, false,
-                                    (len - read) + conn->get_udp_overhead(),
-                                    header_overhead);  // SYN
-    utp_call_on_overhead_statistics(conn->ctx, conn, true, conn->get_overhead(),
-                                    ack_overhead);  // SYNACK
-  }
-  else
-  {
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL,
-             "rejected incoming connection, UTP_ON_ACCEPT callback not set");
-#endif
-  }
-
-  return 1;
-}
-
-// Called by utp_process_icmp_fragmentation() and utp_process_icmp_error() below
-static UTPSocket *
-parse_icmp_payload(utp_context *ctx, const byte *buffer, size_t len,
-                   const struct sockaddr *to, socklen_t tolen)
-{
-  assert(ctx);
-  if(!ctx)
-    return NULL;
-
-  assert(buffer);
-  if(!buffer)
-    return NULL;
-
-  assert(to);
-  if(!to)
-    return NULL;
-
-  const PackedSockAddr addr((const SOCKADDR_STORAGE *)to, tolen);
-
-  // ICMP packets are only required to quote the first 8 bytes of the layer4
-  // payload.  The UDP payload is 8 bytes, and the UTP header is another 20
-  // bytes.  So, in order to find the entire UTP header, we need the ICMP
-  // packet to quote 28 bytes.
-  if(len < sizeof(PacketFormatV1))
-  {
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL, "Ignoring ICMP from %s: runt length %d",
-             addrfmt(addr, addrbuf), len);
-#endif
-    return NULL;
-  }
-
-  const PacketFormatV1 *pf = (PacketFormatV1 *)buffer;
-  const byte version       = UTP_Version(pf);
-  const uint32 id          = uint32(pf->connid);
-
-  if(version != 1)
-  {
-#if UTP_DEBUG_LOGGING
-    ctx->log(UTP_LOG_DEBUG, NULL, "Ignoring ICMP from %s: not UTP version 1",
-             addrfmt(addr, addrbuf));
-#endif
-    return NULL;
-  }
-
-  UTPSocketKeyData *keyData;
-
-  if((keyData = ctx->utp_sockets->Lookup(UTPSocketKey(addr, id)))
-     || ((keyData = ctx->utp_sockets->Lookup(UTPSocketKey(addr, id + 1)))
-         && keyData->socket->conn_id_send == id)
-     || ((keyData = ctx->utp_sockets->Lookup(UTPSocketKey(addr, id - 1)))
-         && keyData->socket->conn_id_send == id))
-  {
-    return keyData->socket;
-  }
-
-#if UTP_DEBUG_LOGGING
-  ctx->log(UTP_LOG_DEBUG, NULL,
-           "Ignoring ICMP from %s: No matching connection found for id %u",
-           addrfmt(addr, addrbuf), id);
-#endif
-  return NULL;
-}
-
-// Should be called when an ICMP Type 3, Code 4 packet (fragmentation needed) is
-// received, to adjust the MTU
-//
-// Returns 1 if the UDP payload (delivered in the ICMP packet) was recognized as
-// a UTP packet, or 0 if it was not
-//
-// @ctx: utp_context
-// @buf: Contents of the original UDP payload, which the ICMP packet quoted.
-// *Not* the ICMP packet itself.
-// @len: buffer length
-// @to: destination address of the original UDP pakcet
-// @tolen: address length
-// @next_hop_mtu:
-int
-utp_process_icmp_fragmentation(utp_context *ctx, const byte *buffer, size_t len,
-                               const struct sockaddr *to, socklen_t tolen,
-                               uint16 next_hop_mtu)
-{
-  UTPSocket *conn = parse_icmp_payload(ctx, buffer, len, to, tolen);
-  if(!conn)
-    return 0;
-
-  // Constrain the next_hop_mtu to sane values.  It might not be initialized or
-  // sent properly
-  if(next_hop_mtu >= 576 && next_hop_mtu < 0x2000)
-  {
-    conn->mtu_ceiling = min< uint32 >(next_hop_mtu, conn->mtu_ceiling);
-    conn->mtu_search_update();
-    // this is something of a speecial case, where we don't set mtu_last
-    // to the value in between the floor and the ceiling. We can update the
-    // floor, because there might be more network segments after the one
-    // that sent this ICMP with smaller MTUs. But we want to test this
-    // MTU size first. If the next probe gets through, mtu_floor is updated
-    conn->mtu_last = conn->mtu_ceiling;
-  }
-  else
-  {
-    // Otherwise, binary search. At this point we don't actually know
-    // what size the packet that failed was, and apparently we can't
-    // trust the next hop mtu either. It seems reasonably conservative
-    // to just lower the ceiling. This should not happen on working networks
-    // anyway.
-    conn->mtu_ceiling = (conn->mtu_floor + conn->mtu_ceiling) / 2;
-    conn->mtu_search_update();
-  }
-
-  conn->log(UTP_LOG_MTU, "MTU [ICMP] floor:%d ceiling:%d current:%d",
-            conn->mtu_floor, conn->mtu_ceiling, conn->mtu_last);
-  return 1;
-}
-
-// Should be called when an ICMP message is received that should tear down the
-// connection.
-//
-// Returns 1 if the UDP payload (delivered in the ICMP packet) was recognized as
-// a UTP packet, or 0 if it was not
-//
-// @ctx: utp_context
-// @buf: Contents of the original UDP payload, which the ICMP packet quoted.
-// *Not* the ICMP packet itself.
-// @len: buffer length
-// @to: destination address of the original UDP pakcet
-// @tolen: address length
-int
-utp_process_icmp_error(utp_context *ctx, const byte *buffer, size_t len,
-                       const struct sockaddr *to, socklen_t tolen)
-{
-  UTPSocket *conn = parse_icmp_payload(ctx, buffer, len, to, tolen);
-  if(!conn)
-    return 0;
-
-  const int err =
-      (conn->state == CS_SYN_SENT) ? UTP_ECONNREFUSED : UTP_ECONNRESET;
-  const PackedSockAddr addr((const SOCKADDR_STORAGE *)to, tolen);
-
-  switch(conn->state)
-  {
-    // Don't pass on errors for idle/closed connections
-    case CS_IDLE:
-#if UTP_DEBUG_LOGGING
-      ctx->log(UTP_LOG_DEBUG, NULL, "ICMP from %s in state CS_IDLE, ignoring",
-               addrfmt(addr, addrbuf));
-#endif
-      return 1;
-
-    default:
-      if(conn->close_requested)
-      {
-#if UTP_DEBUG_LOGGING
-        ctx->log(UTP_LOG_DEBUG, NULL,
-                 "ICMP from %s after close, setting state to CS_DESTROY and "
-                 "causing error %d",
-                 addrfmt(addr, addrbuf), err);
-#endif
-        conn->state = CS_DESTROY;
-      }
-      else
-      {
-#if UTP_DEBUG_LOGGING
-        ctx->log(UTP_LOG_DEBUG, NULL,
-                 "ICMP from %s, setting state to CS_RESET and causing error %d",
-                 addrfmt(addr, addrbuf), err);
-#endif
-        conn->state = CS_RESET;
-      }
-      break;
-  }
-
-  utp_call_on_error(conn->ctx, conn, err);
-  return 1;
-}
-
-// Write bytes to the UTP socket.  Returns the number of bytes written.
-// 0 indicates the socket is no longer writable, -1 indicates an error
-ssize_t
-utp_writev(utp_socket *conn, struct utp_iovec *iovec_input, size_t num_iovecs)
-{
-  static utp_iovec iovec[UTP_IOV_MAX];
-
-  assert(conn);
-  if(!conn)
-    return -1;
-
-  assert(iovec_input);
-  if(!iovec_input)
-    return -1;
-
-  assert(num_iovecs);
-  if(!num_iovecs)
-    return -1;
-
-  if(num_iovecs > UTP_IOV_MAX)
-    num_iovecs = UTP_IOV_MAX;
-
-  memcpy(iovec, iovec_input, sizeof(struct utp_iovec) * num_iovecs);
-
-  size_t bytes = 0;
-  size_t sent  = 0;
-  for(size_t i = 0; i < num_iovecs; i++)
-    bytes += iovec[i].iov_len;
-
-#if UTP_DEBUG_LOGGING
-  size_t param = bytes;
-#endif
-
-  if(conn->state != CS_CONNECTED)
-  {
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG, "UTP_Write %u bytes = false (not CS_CONNECTED)",
-              (uint)bytes);
-#endif
-    return 0;
-  }
-
-  if(conn->fin_sent)
-  {
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG, "UTP_Write %u bytes = false (fin_sent already)",
-              (uint)bytes);
-#endif
-    return 0;
-  }
-
-  conn->ctx->current_ms = utp_call_get_milliseconds(conn->ctx, conn);
-
-  // don't send unless it will all fit in the window
-  size_t packet_size = conn->get_packet_size();
-  size_t num_to_send = min< size_t >(bytes, packet_size);
-  while(!conn->is_full(num_to_send))
-  {
-    // Send an outgoing packet.
-    // Also add it to the outgoing of packets that have been sent but not ACKed.
-
-    bytes -= num_to_send;
-    sent += num_to_send;
-
-#if UTP_DEBUG_LOGGING
-    conn->log(UTP_LOG_DEBUG,
-              "Sending packet. seq_nr:%u ack_nr:%u wnd:%u/%u/%u rcv_win:%u "
-              "size:%u cur_window_packets:%u",
-              conn->seq_nr, conn->ack_nr,
-              (uint)(conn->cur_window + num_to_send), (uint)conn->max_window,
-              (uint)conn->max_window_user, (uint)conn->last_rcv_win,
-              num_to_send, conn->cur_window_packets);
-#endif
-    conn->write_outgoing_packet(num_to_send, ST_DATA, iovec, num_iovecs);
-    num_to_send = min< size_t >(bytes, packet_size);
-
-    if(num_to_send == 0)
-    {
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG, "UTP_Write %u bytes = true", (uint)param);
-#endif
-      return sent;
-    }
-  }
-
-  bool full = conn->is_full();
-  if(full)
-  {
-    // mark the socket as not being writable.
-    conn->state = CS_CONNECTED_FULL;
-  }
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "UTP_Write %u bytes = %s", (uint)bytes,
-            full ? "false" : "true");
-#endif
-
-  // returns whether or not the socket is still writable
-  // if the congestion window is not full, we can still write to it
-  // return !full;
-  return sent;
-}
-
-void
-utp_read_drained(utp_socket *conn)
-{
-  assert(conn);
-  if(!conn)
-    return;
-
-  assert(conn->state != CS_UNINITIALIZED);
-  if(conn->state == CS_UNINITIALIZED)
-    return;
-
-  const size_t rcvwin = conn->get_rcv_window();
-
-  if(rcvwin > conn->last_rcv_win)
-  {
-    // If last window was 0 send ACK immediately, otherwise should set timer
-    if(conn->last_rcv_win == 0)
-    {
-      conn->send_ack();
-    }
-    else
-    {
-      conn->ctx->current_ms = utp_call_get_milliseconds(conn->ctx, conn);
-      conn->schedule_ack();
-    }
-  }
-}
-
-// Should be called each time the UDP socket is drained
-void
-utp_issue_deferred_acks(utp_context *ctx)
-{
-  assert(ctx);
-  if(!ctx)
-    return;
-
-  for(size_t i = 0; i < ctx->ack_sockets.GetCount(); i++)
-  {
-    UTPSocket *conn = ctx->ack_sockets[i];
-    conn->send_ack();
-    i--;
-  }
-}
-
-// Should be called every 500ms
-void
-utp_check_timeouts(utp_context *ctx)
-{
-  assert(ctx);
-  if(!ctx)
-    return;
-
-  ctx->current_ms = utp_call_get_milliseconds(ctx, NULL);
-
-  if(ctx->current_ms - ctx->last_check < TIMEOUT_CHECK_INTERVAL)
-    return;
-
-  ctx->last_check = ctx->current_ms;
-
-  for(size_t i = 0; i < ctx->rst_info.GetCount(); i++)
-  {
-    if((int)(ctx->current_ms - ctx->rst_info[i].timestamp) >= RST_INFO_TIMEOUT)
-    {
-      ctx->rst_info.MoveUpLast(i);
-      i--;
-    }
-  }
-  if(ctx->rst_info.GetCount() != ctx->rst_info.GetAlloc())
-  {
-    ctx->rst_info.Compact();
-  }
-
-  utp_hash_iterator_t it;
-  UTPSocketKeyData *keyData;
-  while((keyData = ctx->utp_sockets->Iterate(it)))
-  {
-    UTPSocket *conn = keyData->socket;
-    conn->check_timeouts();
-
-    // Check if the object was deleted
-    if(conn->state == CS_DESTROY)
-    {
-#if UTP_DEBUG_LOGGING
-      conn->log(UTP_LOG_DEBUG, "Destroying");
-#endif
-      delete conn;
-    }
-  }
-}
-
-int
-utp_getpeername(utp_socket *conn, struct sockaddr *addr, socklen_t *addrlen)
-{
-  assert(addr);
-  if(!addr)
-    return -1;
-
-  assert(addrlen);
-  if(!addrlen)
-    return -1;
-
-  assert(conn);
-  if(!conn)
-    return -1;
-
-  assert(conn->state != CS_UNINITIALIZED);
-  if(conn->state == CS_UNINITIALIZED)
-    return -1;
-
-  socklen_t len;
-  const SOCKADDR_STORAGE sa = conn->addr.get_sockaddr_storage(&len);
-  *addrlen                  = min(len, *addrlen);
-  memcpy(addr, &sa, *addrlen);
-  return 0;
-}
-
-int
-utp_get_delays(UTPSocket *conn, uint32 *ours, uint32 *theirs, uint32 *age)
-{
-  assert(conn);
-  if(!conn)
-    return -1;
-
-  assert(conn->state != CS_UNINITIALIZED);
-  if(conn->state == CS_UNINITIALIZED)
-  {
-    if(ours)
-      *ours = 0;
-    if(theirs)
-      *theirs = 0;
-    if(age)
-      *age = 0;
-    return -1;
-  }
-
-  if(ours)
-    *ours = conn->our_hist.get_value();
-  if(theirs)
-    *theirs = conn->their_hist.get_value();
-  if(age)
-    *age = (uint32)(conn->ctx->current_ms - conn->last_measured_delay);
-  return 0;
-}
-
-// Close the UTP socket.
-// It is not valid for the upper layer to refer to socket after it is closed.
-// Data will keep to try being delivered after the close.
-void
-utp_close(UTPSocket *conn)
-{
-  assert(conn);
-  if(!conn)
-    return;
-
-  assert(conn->state != CS_UNINITIALIZED && conn->state != CS_DESTROY);
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "UTP_Close in state:%s", statenames[conn->state]);
-#endif
-
-  switch(conn->state)
-  {
-    case CS_CONNECTED:
-    case CS_CONNECTED_FULL:
-      conn->read_shutdown   = true;
-      conn->close_requested = true;
-      if(!conn->fin_sent)
-      {
-        conn->fin_sent = true;
-        conn->write_outgoing_packet(0, ST_FIN, NULL, 0);
-      }
-      else if(conn->fin_sent_acked)
-      {
-        conn->state = CS_DESTROY;
-      }
-      break;
-
-    case CS_SYN_SENT:
-      conn->rto_timeout = utp_call_get_milliseconds(conn->ctx, conn)
-          + min< uint >(conn->rto * 2, 60);
-      // fall through
-    case CS_SYN_RECV:
-      // fall through
-    default:
-      conn->state = CS_DESTROY;
-      break;
-  }
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "UTP_Close end in state:%s",
-            statenames[conn->state]);
-#endif
-}
-
-void
-utp_shutdown(UTPSocket *conn, int how)
-{
-  assert(conn);
-  if(!conn)
-    return;
-
-  assert(conn->state != CS_UNINITIALIZED && conn->state != CS_DESTROY);
-
-#if UTP_DEBUG_LOGGING
-  conn->log(UTP_LOG_DEBUG, "UTP_shutdown(%d) in state:%s", how,
-            statenames[conn->state]);
-#endif
-
-  if(how != SHUT_WR)
-  {
-    conn->read_shutdown = true;
-  }
-  if(how != SHUT_RD)
-  {
-    switch(conn->state)
-    {
-      case CS_CONNECTED:
-      case CS_CONNECTED_FULL:
-        if(!conn->fin_sent)
-        {
-          conn->fin_sent = true;
-          conn->write_outgoing_packet(0, ST_FIN, NULL, 0);
-        }
-        break;
-      case CS_SYN_SENT:
-        conn->rto_timeout = utp_call_get_milliseconds(conn->ctx, conn)
-            + min< uint >(conn->rto * 2, 60);
-      default:
-        break;
-    }
-  }
-}
-
-utp_context *
-utp_get_context(utp_socket *socket)
-{
-  assert(socket);
-  return socket ? socket->ctx : NULL;
-}
-
-void *
-utp_set_userdata(utp_socket *socket, void *userdata)
-{
-  assert(socket);
-  if(socket)
-    socket->userdata = userdata;
-  return socket ? socket->userdata : NULL;
-}
-
-void *
-utp_get_userdata(utp_socket *socket)
-{
-  assert(socket);
-  return socket ? socket->userdata : NULL;
-}
-
-void
-struct_utp_context::log(int level, utp_socket *socket, char const *fmt, ...)
-{
-  if(!would_log(level))
-  {
-    return;
-  }
-
-  va_list va;
-  va_start(va, fmt);
-  log_unchecked(socket, fmt, va);
-  va_end(va);
-}
-
-void
-struct_utp_context::log_unchecked(utp_socket *socket, char const *fmt, ...)
-{
-  va_list va;
-  char buf[4096];
-
-  va_start(va, fmt);
-  vsnprintf(buf, 4096, fmt, va);
-  buf[4095] = '\0';
-  va_end(va);
-
-  utp_call_log(this, socket, (const byte *)buf);
-}
-
-inline bool
-struct_utp_context::would_log(int level)
-{
-  if(level == UTP_LOG_NORMAL)
-    return log_normal;
-  if(level == UTP_LOG_MTU)
-    return log_mtu;
-  if(level == UTP_LOG_DEBUG)
-    return log_debug;
-  return true;
-}
-
-utp_socket_stats *
-utp_get_stats(utp_socket *socket)
-{
-#ifdef _DEBUG
-  assert(socket);
-  if(!socket)
-    return NULL;
-  socket->_stats.mtu_guess =
-      socket->mtu_last ? socket->mtu_last : socket->mtu_ceiling;
-  return &socket->_stats;
-#else
-  (void)socket;
-  return NULL;
-#endif
-}
diff --git a/libutp/utp_internal.h b/libutp/utp_internal.h
deleted file mode 100644
index 01597f811..000000000
--- a/libutp/utp_internal.h
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef __UTP_INTERNAL_H__
-#define __UTP_INTERNAL_H__
-
-#include <stdarg.h>
-#include <string.h>
-#include <assert.h>
-#include <stdio.h>
-
-#include "utp.h"
-#include "utp_callbacks.h"
-#include "utp_templates.h"
-#include "utp_hash.h"
-#include "utp_hash.h"
-#include "utp_packedsockaddr.h"
-
-/* These originally lived in utp_config.h */
-#define CCONTROL_TARGET (100 * 1000)  // us
-
-enum bandwidth_type_t
-{
-  payload_bandwidth,
-  connect_overhead,
-  close_overhead,
-  ack_overhead,
-  header_overhead,
-  retransmit_overhead
-};
-
-#ifdef WIN32
-#ifdef _MSC_VER
-#include "libutp_inet_ntop.h"
-#endif
-
-// newer versions of MSVC define these in errno.h
-#ifndef ECONNRESET
-#define ECONNRESET WSAECONNRESET
-#define EMSGSIZE WSAEMSGSIZE
-#define ECONNREFUSED WSAECONNREFUSED
-#define ETIMEDOUT WSAETIMEDOUT
-#endif
-#endif
-
-struct PACKED_ATTRIBUTE RST_Info
-{
-  PackedSockAddr addr;
-  uint32 connid;
-  uint16 ack_nr;
-  uint64 timestamp;
-};
-
-// It's really important that we don't have duplicate keys in the hash table.
-// If we do, we'll eventually crash. if we try to remove the second instance
-// of the key, we'll accidentally remove the first instead. then later,
-// checkTimeouts will try to access the second one's already freed memory.
-void
-UTP_FreeAll(struct UTPSocketHT *utp_sockets);
-
-struct UTPSocketKey
-{
-  PackedSockAddr addr;
-  uint32 recv_id;  // "conn_seed", "conn_id"
-
-  UTPSocketKey(const PackedSockAddr &_addr, uint32 _recv_id)
-      : addr(_addr)
-      , recv_id(_recv_id)
-  {
-  }
-
-  bool
-  operator==(const UTPSocketKey &other) const
-  {
-    return recv_id == other.recv_id && addr == other.addr;
-  }
-
-  uint32
-  compute_hash() const
-  {
-    return recv_id ^ addr.compute_hash();
-  }
-};
-
-struct UTPSocketKeyData
-{
-  UTPSocketKey key;
-  UTPSocket *socket;
-  utp_link_t link;
-};
-
-#define UTP_SOCKET_BUCKETS 79
-#define UTP_SOCKET_INIT 15
-
-struct UTPSocketHT : utpHashTable< UTPSocketKey, UTPSocketKeyData >
-{
-  UTPSocketHT()
-  {
-    const int buckets = UTP_SOCKET_BUCKETS;
-    const int initial = UTP_SOCKET_INIT;
-    this->Create(buckets, initial);
-  }
-  ~UTPSocketHT()
-  {
-    UTP_FreeAll(this);
-    this->Free();
-  }
-};
-
-struct struct_utp_context
-{
-  void *userdata;
-  utp_callback_t *callbacks[UTP_ARRAY_SIZE];
-
-  uint64 current_ms;
-  utp_context_stats context_stats;
-  UTPSocket *last_utp_socket;
-  Array< UTPSocket * > ack_sockets;
-  Array< RST_Info > rst_info;
-  UTPSocketHT *utp_sockets;
-  size_t target_delay;
-  size_t opt_sndbuf;
-  size_t opt_rcvbuf;
-  uint64 last_check;
-
-  struct_utp_context();
-  ~struct_utp_context();
-
-  void
-  log(int level, utp_socket *socket, char const *fmt, ...);
-  void
-  log_unchecked(utp_socket *socket, char const *fmt, ...);
-  bool
-  would_log(int level);
-
-  bool log_normal : 1;  // log normal events?
-  bool log_mtu : 1;     // log MTU related events?
-  bool log_debug : 1;   // log debugging events? (Must also compile with
-                        // UTP_DEBUG_LOGGING defined)
-};
-
-#endif  //__UTP_INTERNAL_H__
diff --git a/libutp/utp_packedsockaddr.cpp b/libutp/utp_packedsockaddr.cpp
deleted file mode 100644
index 04aad3157..000000000
--- a/libutp/utp_packedsockaddr.cpp
+++ /dev/null
@@ -1,166 +0,0 @@
-// vim:set ts=4 sw=4 ai:
-
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <string.h>
-#include <assert.h>
-#include <stdio.h>
-
-#include "utp_types.h"
-#include "utp_hash.h"
-#include "utp_packedsockaddr.h"
-
-#include "libutp_inet_ntop.h"
-
-byte
-PackedSockAddr::get_family() const
-{
-#if defined(__sh__)
-  return ((_sin6d[0] == 0) && (_sin6d[1] == 0)
-          && (_sin6d[2] == htonl(0xffff)) != 0)
-      ? AF_INET
-      : AF_INET6;
-#else
-  return (IN6_IS_ADDR_V4MAPPED(&_in._in6addr) != 0) ? AF_INET : AF_INET6;
-#endif  // defined(__sh__)
-}
-
-bool
-PackedSockAddr::operator==(const PackedSockAddr &rhs) const
-{
-  if(&rhs == this)
-    return true;
-  if(_port != rhs._port)
-    return false;
-  return memcmp(_sin6, rhs._sin6, sizeof(_sin6)) == 0;
-}
-
-bool
-PackedSockAddr::operator!=(const PackedSockAddr &rhs) const
-{
-  return !(*this == rhs);
-}
-
-uint32
-PackedSockAddr::compute_hash() const
-{
-  return utp_hash_mem(&_in, sizeof(_in)) ^ _port;
-}
-
-void
-PackedSockAddr::set(const SOCKADDR_STORAGE *sa, socklen_t len)
-{
-  // on unix, the cast does nothing, socklen_t is _already_ unsigned
-  if(sa->ss_family == AF_INET)
-  {
-    assert((unsigned)len >= sizeof(sockaddr_in));
-    const sockaddr_in *sin = (sockaddr_in *)sa;
-    _sin6w[0]              = 0;
-    _sin6w[1]              = 0;
-    _sin6w[2]              = 0;
-    _sin6w[3]              = 0;
-    _sin6w[4]              = 0;
-    _sin6w[5]              = 0xffff;
-    _sin4                  = sin->sin_addr.s_addr;
-    _port                  = ntohs(sin->sin_port);
-  }
-  else
-  {
-    assert((unsigned)len >= sizeof(sockaddr_in6));
-    const sockaddr_in6 *sin6 = (sockaddr_in6 *)sa;
-    _in._in6addr             = sin6->sin6_addr;
-    _port                    = ntohs(sin6->sin6_port);
-  }
-  (void)len;
-}
-
-PackedSockAddr::PackedSockAddr(const SOCKADDR_STORAGE *sa, socklen_t len)
-{
-  set(sa, len);
-}
-
-PackedSockAddr::PackedSockAddr(void)
-{
-  SOCKADDR_STORAGE sa;
-  socklen_t len = sizeof(SOCKADDR_STORAGE);
-  memset(&sa, 0, len);
-  sa.ss_family = AF_INET;
-  set(&sa, len);
-}
-
-SOCKADDR_STORAGE
-PackedSockAddr::get_sockaddr_storage(socklen_t *len = NULL) const
-{
-  SOCKADDR_STORAGE sa;
-  const byte family = get_family();
-  if(family == AF_INET)
-  {
-    sockaddr_in *sin = (sockaddr_in *)&sa;
-    if(len)
-      *len = sizeof(sockaddr_in);
-    memset(sin, 0, sizeof(sockaddr_in));
-    sin->sin_family      = family;
-    sin->sin_port        = htons(_port);
-    sin->sin_addr.s_addr = _sin4;
-  }
-  else
-  {
-    sockaddr_in6 *sin6 = (sockaddr_in6 *)&sa;
-    memset(sin6, 0, sizeof(sockaddr_in6));
-    if(len)
-      *len = sizeof(sockaddr_in6);
-    sin6->sin6_family = family;
-    sin6->sin6_addr   = _in._in6addr;
-    sin6->sin6_port   = htons(_port);
-  }
-  return sa;
-}
-
-// #define addrfmt(x, s) x.fmt(s, sizeof(s))
-cstr
-PackedSockAddr::fmt(str s, size_t len) const
-{
-  memset(s, 0, len);
-  const byte family = get_family();
-  str i;
-  if(family == AF_INET)
-  {
-    INET_NTOP(family, (uint32 *)&_sin4, s, len);
-    i = s;
-    while(*++i)
-    {
-    }
-  }
-  else
-  {
-    i    = s;
-    *i++ = '[';
-    INET_NTOP(family, (in6_addr *)&_in._in6addr, i, len - 1);
-    while(*++i)
-    {
-    }
-    *i++ = ']';
-  }
-  snprintf(i, len - (i - s), ":%u", _port);
-  return s;
-}
diff --git a/libutp/utp_packedsockaddr.h b/libutp/utp_packedsockaddr.h
deleted file mode 100644
index 76e8accaa..000000000
--- a/libutp/utp_packedsockaddr.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef __UTP_PACKEDSOCKADDR_H__
-#define __UTP_PACKEDSOCKADDR_H__
-
-#include "utp_types.h"
-
-struct PACKED_ATTRIBUTE PackedSockAddr {
-	// The values are always stored here in network byte order
-	union {
-		byte _in6[16];		// IPv6
-		uint16 _in6w[8];	// IPv6, word based (for convenience)
-		uint32 _in6d[4];	// Dword access
-		in6_addr _in6addr;	// For convenience
-	} _in;
-
-	// Host byte order
-	uint16 _port;
-
-	#define _sin4 _in._in6d[3]	// IPv4 is stored where it goes if mapped
-
-	#define _sin6 _in._in6
-	#define _sin6w _in._in6w
-	#define _sin6d _in._in6d
-
-	byte get_family() const;
-	bool operator==(const PackedSockAddr& rhs) const;
-	bool operator!=(const PackedSockAddr& rhs) const;
-	void set(const SOCKADDR_STORAGE* sa, socklen_t len);
-
-	PackedSockAddr(const SOCKADDR_STORAGE* sa, socklen_t len);
-	PackedSockAddr(void);
-
-	SOCKADDR_STORAGE get_sockaddr_storage(socklen_t *len) const;
-	cstr fmt(str s, size_t len) const;
-
-	uint32 compute_hash() const;
-} ALIGNED_ATTRIBUTE(4);
-
-#endif //__UTP_PACKEDSOCKADDR_H__
diff --git a/libutp/utp_templates.h b/libutp/utp_templates.h
deleted file mode 100644
index 8f88f5c7c..000000000
--- a/libutp/utp_templates.h
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef __TEMPLATES_H__
-#define __TEMPLATES_H__
-
-#include "utp_types.h"
-#include <assert.h>
-
-#if defined(POSIX)
-/* Allow over-writing FORCEINLINE from makefile because gcc 3.4.4 for buffalo
-   doesn't seem to support __attribute__((always_inline)) in -O0 build
-   (strangely, it works in -Os build) */
-#ifndef FORCEINLINE
-// The always_inline attribute asks gcc to inline the function even if no optimization is being requested.
-// This macro should be used exclusive-or with the inline directive (use one or the other but not both)
-// since Microsoft uses __forceinline to also mean inline,
-// and this code is following a Microsoft compatibility model.
-// Just setting the attribute without also specifying the inline directive apparently won't inline the function,
-// as evidenced by multiply-defined symbols found at link time.
-#define FORCEINLINE inline __attribute__((always_inline))
-#endif
-#endif
-
-// Utility templates
-#undef min
-#undef max
-
-template <typename T> static inline T min(T a, T b) { if (a < b) return a; return b; }
-template <typename T> static inline T max(T a, T b) { if (a > b) return a; return b; }
-
-template <typename T> static inline T min(T a, T b, T c) { return min(min(a,b),c); }
-template <typename T> static inline T max(T a, T b, T c) { return max(max(a,b),c); }
-template <typename T> static inline T clamp(T v, T mi, T ma)
-{
-	if (v > ma) v = ma;
-	if (v < mi) v = mi;
-	return v;
-}
-
-#if (defined(__SVR4) && defined(__sun))
-	#pragma pack(1)
-#else
-	#pragma pack(push,1)
-#endif
-
-
-namespace aux
-{
-	FORCEINLINE uint16 host_to_network(uint16 i) { return htons(i); }
-	FORCEINLINE uint32 host_to_network(uint32 i) { return htonl(i); }
-	FORCEINLINE int32 host_to_network(int32 i) { return htonl(i); }
-	FORCEINLINE uint16 network_to_host(uint16 i) { return ntohs(i); }
-	FORCEINLINE uint32 network_to_host(uint32 i) { return ntohl(i); }
-	FORCEINLINE int32 network_to_host(int32 i) { return ntohl(i); }
-}
-
-template <class T>
-struct PACKED_ATTRIBUTE big_endian
-{
-	T operator=(T i) { m_integer = aux::host_to_network(i); return i; }
-	operator T() const { return aux::network_to_host(m_integer); }
-private:
-	T m_integer;
-};
-
-typedef big_endian<int32> int32_big;
-typedef big_endian<uint32> uint32_big;
-typedef big_endian<uint16> uint16_big;
-
-#if (defined(__SVR4) && defined(__sun))
-	#pragma pack(0)
-#else
-	#pragma pack(pop)
-#endif
-
-template<typename T> static inline void zeromem(T *a, size_t count = 1) { memset(a, 0, count * sizeof(T)); }
-
-typedef int SortCompareProc(const void *, const void *);
-
-template<typename T> static FORCEINLINE void QuickSortT(T *base, size_t num, int (*comp)(const T *, const T *)) { qsort(base, num, sizeof(T), (SortCompareProc*)comp); }
-
-
-// WARNING: The template parameter MUST be a POD type!
-template <typename T, size_t minsize = 16> class Array {
-protected:
-	T *mem;
-	size_t alloc,count;
-
-public:
-	Array(size_t init) { Init(init); }
-	Array() { Init(); }
-	~Array() { Free(); }
-
-	void inline Init() { mem = NULL; alloc = count = 0; }
-	void inline Init(size_t init) { Init(); if (init) Resize(init); }
-	size_t inline GetCount() const { return count; }
-	size_t inline GetAlloc() const { return alloc; }
-	void inline SetCount(size_t c) { count = c; }
-
-	inline T& operator[](size_t offset) { assert(offset ==0 || offset<alloc); return mem[offset]; }
-	inline const T& operator[](size_t offset) const { assert(offset ==0 || offset<alloc); return mem[offset]; }
-
-	void inline Resize(size_t a) {
-		if (a == 0) { free(mem); Init(); }
-		else { mem = (T*)realloc(mem, (alloc=a) * sizeof(T)); }
-	}
-
-	void Grow() { Resize(::max<size_t>(minsize, alloc * 2)); }
-
-	inline size_t Append(const T &t) {
-		if (count >= alloc) Grow();
-		size_t r=count++;
-		mem[r] = t;
-		return r;
-	}
-
-	T inline &Append() {
-		if (count >= alloc) Grow();
-		return mem[count++];
-	}
-
-	void inline Compact() {
-		Resize(count);
-	}
-
-	void inline Free() {
-		free(mem);
-		Init();
-	}
-
-	void inline Clear() {
-		count = 0;
-	}
-
-	bool inline MoveUpLast(size_t index) {
-		assert(index < count);
-		size_t c = --count;
-		if (index != c) {
-			mem[index] = mem[c];
-			return true;
-		}
-		return false;
-	}
-
-	bool inline MoveUpLastExist(const T &v) {
-		return MoveUpLast(LookupElementExist(v));
-	}
-
-	size_t inline LookupElement(const T &v) const {
-		for(size_t i = 0; i != count; i++)
-			if (mem[i] == v)
-				return i;
-		return (size_t) -1;
-	}
-
-	bool inline HasElement(const T &v) const {
-		return LookupElement(v) != -1;
-	}
-
-	typedef int SortCompareProc(const T *a, const T *b);
-
-	void Sort(SortCompareProc* proc, size_t start, size_t end) {
-		QuickSortT(&mem[start], end - start, proc);
-	}
-
-	void Sort(SortCompareProc* proc, size_t start) {
-		Sort(proc, start, count);
-	}
-
-	void Sort(SortCompareProc* proc) {
-		Sort(proc, 0, count);
-	}
-};
-
-#endif //__TEMPLATES_H__
diff --git a/libutp/utp_utils.cpp b/libutp/utp_utils.cpp
deleted file mode 100644
index c9c7261c5..000000000
--- a/libutp/utp_utils.cpp
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include <stdlib.h>
-#include <assert.h>
-#include "utp.h"
-#include "utp_types.h"
-
-#ifdef WIN32
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#endif
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#else  //! WIN32
-#include <time.h>
-#include <sys/time.h>  // Linux needs both time.h and sys/time.h
-#endif
-
-#if defined(__APPLE__)
-#include <mach/mach_time.h>
-#endif
-
-#include <absl/base/attributes.h>
-#include "utp_utils.h"
-
-#ifdef WIN32
-
-typedef ULONGLONG(WINAPI GetTickCount64Proc)(void);
-static GetTickCount64Proc *pt2GetTickCount64;
-static GetTickCount64Proc *pt2RealGetTickCount;
-
-static uint64 startPerformanceCounter;
-static uint64 startGetTickCount;
-// MSVC 6 standard doesn't like division with uint64s
-static double counterPerMicrosecond;
-
-static uint64
-UTGetTickCount64()
-{
-  if(pt2GetTickCount64)
-  {
-    return pt2GetTickCount64();
-  }
-  if(pt2RealGetTickCount)
-  {
-    uint64 v = pt2RealGetTickCount();
-    // fix return value from GetTickCount
-    return (DWORD)v | ((v >> 0x18) & 0xFFFFFFFF00000000);
-  }
-  return (uint64)GetTickCount();
-}
-
-static void
-Time_Initialize()
-{
-  HMODULE kernel32 = GetModuleHandleA("kernel32.dll");
-  pt2GetTickCount64 =
-      (GetTickCount64Proc *)GetProcAddress(kernel32, "GetTickCount64");
-  // not a typo. GetTickCount actually returns 64 bits
-  pt2RealGetTickCount =
-      (GetTickCount64Proc *)GetProcAddress(kernel32, "GetTickCount");
-
-  uint64 frequency;
-  QueryPerformanceCounter((LARGE_INTEGER *)&startPerformanceCounter);
-  QueryPerformanceFrequency((LARGE_INTEGER *)&frequency);
-  counterPerMicrosecond = (double)frequency / 1000000.0f;
-  startGetTickCount     = UTGetTickCount64();
-}
-
-static int64
-abs64(int64 x)
-{
-  return x < 0 ? -x : x;
-}
-
-static uint64
-__GetMicroseconds()
-{
-  static bool time_init = false;
-  if(!time_init)
-  {
-    time_init = true;
-    Time_Initialize();
-  }
-
-  uint64 counter;
-  uint64 tick;
-
-  QueryPerformanceCounter((LARGE_INTEGER *)&counter);
-  tick = UTGetTickCount64();
-
-  // unfortunately, QueryPerformanceCounter is not guaranteed
-  // to be monotonic. Make it so.
-  int64 ret = (int64)(((int64)counter - (int64)startPerformanceCounter)
-                      / counterPerMicrosecond);
-  // if the QPC clock leaps more than one second off GetTickCount64()
-  // something is seriously fishy. Adjust QPC to stay monotonic
-  int64 tick_diff = tick - startGetTickCount;
-  if(abs64(ret / 100000 - tick_diff / 100) > 10)
-  {
-    startPerformanceCounter -=
-        (uint64)((int64)(tick_diff * 1000 - ret) * counterPerMicrosecond);
-    ret = (int64)((counter - startPerformanceCounter) / counterPerMicrosecond);
-  }
-  return ret;
-}
-
-static inline uint64
-UTP_GetMilliseconds()
-{
-  return GetTickCount();
-}
-
-#else  //! WIN32
-
-static inline uint64
-UTP_GetMicroseconds(void);
-static inline uint64
-UTP_GetMilliseconds()
-{
-  return UTP_GetMicroseconds() / 1000;
-}
-
-#if defined(__APPLE__)
-
-static uint64
-__GetMicroseconds()
-{
-  // http://developer.apple.com/mac/library/qa/qa2004/qa1398.html
-  // http://www.macresearch.org/tutorial_performance_and_time
-  static mach_timebase_info_data_t sTimebaseInfo;
-  static uint64_t start_tick = 0;
-  uint64_t tick;
-  // Returns a counter in some fraction of a nanoseconds
-  tick = mach_absolute_time();
-  if(sTimebaseInfo.denom == 0)
-  {
-    // Get the timer ratio to convert mach_absolute_time to nanoseconds
-    mach_timebase_info(&sTimebaseInfo);
-    start_tick = tick;
-  }
-  // Calculate the elapsed time, convert it to microseconds and return it.
-  return ((tick - start_tick) * sTimebaseInfo.numer)
-      / (sTimebaseInfo.denom * 1000);
-}
-
-#else  // !__APPLE__
-
-// While _POSIX_TIMERS == -1 in openbsd, clock_gettime(2) _does_ support monotonic.
-// this is true for all the BSDs
-#if !(__OpenBSD__ || __NetBSD__ || __FreeBSD__)
-#if !(defined(_POSIX_TIMERS) && _POSIX_TIMERS > 0 && defined(CLOCK_MONOTONIC)) 
-#warning "Using non-monotonic function gettimeofday() in UTP_GetMicroseconds()"
-#endif
-#endif
-
-/* Unfortunately, #ifdef CLOCK_MONOTONIC is not enough to make sure that
-   POSIX clocks work -- we could be running a recent libc with an ancient
-   kernel (think OpenWRT). -- jch */
-
-static uint64_t
-__GetMicroseconds()
-{
-  struct timeval tv;
-
-#if defined(CLOCK_MONOTONIC)
-  static int have_posix_clocks = -1;
-  int rc;
-
-  if(have_posix_clocks < 0)
-  {
-    struct timespec ts;
-    rc = clock_gettime(CLOCK_MONOTONIC, &ts);
-    if(rc < 0)
-    {
-      have_posix_clocks = 0;
-    }
-    else
-    {
-      have_posix_clocks = 1;
-    }
-  }
-
-  if(have_posix_clocks)
-  {
-    struct timespec ts;
-    rc = clock_gettime(CLOCK_MONOTONIC, &ts);
-    return uint64(ts.tv_sec) * 1000000 + uint64(ts.tv_nsec) / 1000;
-  }
-#endif
-
-  gettimeofday(&tv, NULL);
-  return uint64(tv.tv_sec) * 1000000 + tv.tv_usec;
-}
-
-#endif  //!__APPLE__
-
-#endif  //! WIN32
-
-/*
- * Whew.  Okay.  After that #ifdef maze above, we now know we have a working
- * __GetMicroseconds() implementation on all platforms.
- *
- * Because there are a number of assertions in libutp that will cause a crash
- * if monotonic time isn't monotonic, now apply some safety checks.  While in
- * principle we're already protecting ourselves in cases where non-monotonic
- * time is likely to happen, this protects all versions.
- */
-
-static inline uint64
-UTP_GetMicroseconds()
-{
-  static uint64 offset = 0, previous = 0;
-
-  uint64 now = __GetMicroseconds() + offset;
-  if(previous > now)
-  {
-    /* Eek! */
-    offset += previous - now;
-    now = previous;
-  }
-  previous = now;
-  return now;
-}
-
-#define ETHERNET_MTU 1500
-#define IPV4_HEADER_SIZE 20
-#define IPV6_HEADER_SIZE 40
-#define UDP_HEADER_SIZE 8
-#define GRE_HEADER_SIZE 24
-#define PPPOE_HEADER_SIZE 8
-#define MPPE_HEADER_SIZE 2
-// packets have been observed in the wild that were fragmented
-// with a payload of 1416 for the first fragment
-// There are reports of routers that have MTU sizes as small as 1392
-#define FUDGE_HEADER_SIZE 36
-#define TEREDO_MTU 1280
-
-#define UDP_IPV4_OVERHEAD (IPV4_HEADER_SIZE + UDP_HEADER_SIZE)
-#define UDP_IPV6_OVERHEAD (IPV6_HEADER_SIZE + UDP_HEADER_SIZE)
-#define UDP_TEREDO_OVERHEAD (UDP_IPV4_OVERHEAD + UDP_IPV6_OVERHEAD)
-
-#define UDP_IPV4_MTU                                                   \
-  (ETHERNET_MTU - IPV4_HEADER_SIZE - UDP_HEADER_SIZE - GRE_HEADER_SIZE \
-   - PPPOE_HEADER_SIZE - MPPE_HEADER_SIZE - FUDGE_HEADER_SIZE)
-#define UDP_IPV6_MTU                                                   \
-  (ETHERNET_MTU - IPV6_HEADER_SIZE - UDP_HEADER_SIZE - GRE_HEADER_SIZE \
-   - PPPOE_HEADER_SIZE - MPPE_HEADER_SIZE - FUDGE_HEADER_SIZE)
-#define UDP_TEREDO_MTU (TEREDO_MTU - IPV6_HEADER_SIZE - UDP_HEADER_SIZE)
-
-uint64
-utp_default_get_udp_mtu(utp_callback_arguments *args)
-{
-  // Since we don't know the local address of the interface,
-  // be conservative and assume all IPv6 connections are Teredo.
-  return (args->address->sa_family == AF_INET6) ? UDP_TEREDO_MTU : UDP_IPV4_MTU;
-}
-
-uint64
-utp_default_get_udp_overhead(utp_callback_arguments *args)
-{
-  // Since we don't know the local address of the interface,
-  // be conservative and assume all IPv6 connections are Teredo.
-  return (args->address->sa_family == AF_INET6) ? UDP_TEREDO_OVERHEAD
-                                                : UDP_IPV4_OVERHEAD;
-}
-
-uint64
-utp_default_get_random(ABSL_ATTRIBUTE_UNUSED utp_callback_arguments *args)
-{
-  return rand();
-}
-
-uint64
-utp_default_get_milliseconds(ABSL_ATTRIBUTE_UNUSED
-                             utp_callback_arguments *args)
-{
-  return UTP_GetMilliseconds();
-}
-
-uint64
-utp_default_get_microseconds(ABSL_ATTRIBUTE_UNUSED
-                             utp_callback_arguments *args)
-{
-  return UTP_GetMicroseconds();
-}
diff --git a/libutp/utp_utils.h b/libutp/utp_utils.h
deleted file mode 100644
index 7eb0c5562..000000000
--- a/libutp/utp_utils.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2010-2013 BitTorrent, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-uint64 utp_default_get_udp_mtu(utp_callback_arguments *args);
-uint64 utp_default_get_udp_overhead(utp_callback_arguments *args);
-uint64 utp_default_get_random(utp_callback_arguments *args);
-uint64 utp_default_get_milliseconds(utp_callback_arguments *args);
-uint64 utp_default_get_microseconds(utp_callback_arguments *args);
diff --git a/llarp/CMakeLists.txt b/llarp/CMakeLists.txt
index b4157f283..264e90ed3 100644
--- a/llarp/CMakeLists.txt
+++ b/llarp/CMakeLists.txt
@@ -105,7 +105,7 @@ else()
 endif(WIN32)
 
 add_library(${PLATFORM_LIB} STATIC ${LIB_PLATFORM_SRC})
-target_link_libraries(${PLATFORM_LIB} PUBLIC ${CRYPTOGRAPHY_LIB} ${UTIL_LIB} libutp Threads::Threads ${LIBS})
+target_link_libraries(${PLATFORM_LIB} PUBLIC ${CRYPTOGRAPHY_LIB} ${UTIL_LIB} Threads::Threads ${LIBS})
 
 if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   if(NON_PC_TARGET)
@@ -247,17 +247,12 @@ set(LIB_SRC
   service/tag_lookup_job.cpp
   service/tag.cpp
   service/vanity.cpp
-  utp/inbound_message.cpp
-  utp/linklayer.cpp
-  utp/session.cpp
-  utp/utp.cpp
 )
 if(TESTNET)
   set(LIB_SRC ${LIB_SRC} testnet.c)
 endif()
 
 add_library(${STATIC_LIB} STATIC ${LIB_SRC})
-set(LIBS ${LIBS} libutp)
 target_link_libraries(${STATIC_LIB} PUBLIC cxxopts ${ABYSS_LIB} ${PLATFORM_LIB} ${UTIL_LIB} ${CRYPTOGRAPHY_LIB} ${FS_LIB})
 
 if(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index 1c9583a37..51bba1958 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -73,7 +73,7 @@ namespace llarp
     }
 
     void
-    LinkLayer::RecvFrom(const Addr& from, const void* pkt, size_t sz)
+    LinkLayer::RecvFrom(const Addr& from, ILinkSession::Packet_t pkt)
     {
       std::shared_ptr< ILinkSession > session;
       auto itr = m_AuthedAddrs.find(from);
@@ -95,8 +95,7 @@ namespace llarp
       }
       if(session)
       {
-        const llarp_buffer_t buf{pkt, sz};
-        session->Recv_LL(buf);
+        session->Recv_LL(std::move(pkt));
       }
     }
 
diff --git a/llarp/iwp/linklayer.hpp b/llarp/iwp/linklayer.hpp
index f481ddf61..d73b40e5e 100644
--- a/llarp/iwp/linklayer.hpp
+++ b/llarp/iwp/linklayer.hpp
@@ -39,7 +39,7 @@ namespace llarp
       Rank() const override;
 
       void
-      RecvFrom(const Addr &from, const void *buf, size_t sz) override;
+      RecvFrom(const Addr &from, ILinkSession::Packet_t pkt) override;
 
       bool
       MapAddr(const RouterID &pk, ILinkSession *s) override;
diff --git a/llarp/iwp/message_buffer.cpp b/llarp/iwp/message_buffer.cpp
index 33adcb624..33b3458c5 100644
--- a/llarp/iwp/message_buffer.cpp
+++ b/llarp/iwp/message_buffer.cpp
@@ -22,12 +22,11 @@ namespace llarp
     ILinkSession::Packet_t
     OutboundMessage::XMIT() const
     {
-      ILinkSession::Packet_t xmit(12 + 32);
-      xmit[0] = LLARP_PROTO_VERSION;
-      xmit[1] = Command::eXMIT;
-      htobe16buf(xmit.data() + 2, m_Data.size());
-      htobe64buf(xmit.data() + 4, m_MsgID);
-      std::copy_n(m_Digest.begin(), m_Digest.size(), xmit.begin() + 12);
+      auto xmit = CreatePacket(Command::eXMIT, 12 + 32, 0, 0);
+      htobe16buf(xmit.data() + 2 + PacketOverhead, m_Data.size());
+      htobe64buf(xmit.data() + 4 + PacketOverhead, m_MsgID);
+      std::copy_n(m_Digest.begin(), m_Digest.size(),
+                  xmit.begin() + 12 + PacketOverhead);
       return xmit;
     }
 
@@ -58,7 +57,7 @@ namespace llarp
         std::function< void(ILinkSession::Packet_t) > sendpkt, llarp_time_t now)
     {
       /// overhead for a data packet in plaintext
-      static constexpr size_t Overhead = 12;
+      static constexpr size_t Overhead = 10;
       uint16_t idx                     = 0;
       const auto datasz                = m_Data.size();
       while(idx < datasz)
@@ -67,14 +66,11 @@ namespace llarp
         {
           const size_t fragsz =
               idx + FragmentSize < datasz ? FragmentSize : datasz - idx;
-          ILinkSession::Packet_t frag(fragsz + Overhead);
-
-          frag[0] = LLARP_PROTO_VERSION;
-          frag[1] = Command::eDATA;
-          htobe16buf(frag.data() + 2, idx);
-          htobe64buf(frag.data() + 4, m_MsgID);
+          auto frag = CreatePacket(Command::eDATA, fragsz + Overhead, 0, 0);
+          htobe16buf(frag.data() + 2 + PacketOverhead, idx);
+          htobe64buf(frag.data() + 4 + PacketOverhead, m_MsgID);
           std::copy(m_Data.begin() + idx, m_Data.begin() + idx + fragsz,
-                    frag.begin() + Overhead);
+                    frag.begin() + PacketOverhead + Overhead + 2);
           sendpkt(std::move(frag));
         }
         idx += FragmentSize;
@@ -136,11 +132,9 @@ namespace llarp
     ILinkSession::Packet_t
     InboundMessage::ACKS() const
     {
-      ILinkSession::Packet_t acks(9);
-      acks[0] = LLARP_PROTO_VERSION;
-      acks[1] = Command::eACKS;
-      htobe64buf(acks.data() + 2, m_MsgID);
-      acks[8] = AcksBitmask();
+      auto acks = CreatePacket(Command::eACKS, 9, 0, 0);
+      htobe64buf(acks.data() + 2 + PacketOverhead, m_MsgID);
+      acks[PacketOverhead + 8] = AcksBitmask();
       return acks;
     }
 
@@ -178,9 +172,7 @@ namespace llarp
     InboundMessage::SendACKS(
         std::function< void(ILinkSession::Packet_t) > sendpkt, llarp_time_t now)
     {
-      auto acks = ACKS();
-      AddRandomPadding(acks);
-      sendpkt(std::move(acks));
+      sendpkt(ACKS());
       m_LastACKSent = now;
     }
 
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index c8c811f5d..967dc797d 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -11,7 +11,7 @@ namespace llarp
     ILinkSession::Packet_t
     CreatePacket(Command cmd, size_t plainsize, size_t minpad, size_t variance)
     {
-      const size_t pad = minpad > 0 ?: min + randint() % variance : 0;
+      const size_t pad = minpad > 0 ? minpad + randint() % variance : 0;
       ILinkSession::Packet_t pkt(PacketOverhead + plainsize + pad + 2);
       // randomize pad
       if(pad)
@@ -20,8 +20,7 @@ namespace llarp
             pkt.data() + PacketOverhead + 2 + plainsize, pad);
       }
       // randomize nounce
-      CryptoManager::instance()->randbytes(pkt.data() + HMACSIZE,
-                                           TUNNOUNCESIZE);
+      CryptoManager::instance()->randbytes(pkt.data() + HMACSIZE, TUNNONCESIZE);
       pkt[PacketOverhead]     = LLARP_PROTO_VERSION;
       pkt[PacketOverhead + 1] = cmd;
       return pkt;
@@ -40,6 +39,8 @@ namespace llarp
       GotLIM = util::memFn(&Session::GotOutboundLIM, this);
       CryptoManager::instance()->shorthash(m_SessionKey,
                                            llarp_buffer_t(rc.pubkey));
+      m_EncryptNext = std::make_shared< CryptoQueue_t >();
+      m_DecryptNext = std::make_shared< CryptoQueue_t >();
     }
 
     Session::Session(LinkLayer* p, Addr from)
@@ -53,6 +54,8 @@ namespace llarp
       GotLIM          = util::memFn(&Session::GotInboundLIM, this);
       const PubKey pk = m_Parent->GetOurRC().pubkey;
       CryptoManager::instance()->shorthash(m_SessionKey, llarp_buffer_t(pk));
+      m_EncryptNext = std::make_shared< CryptoQueue_t >();
+      m_DecryptNext = std::make_shared< CryptoQueue_t >();
     }
 
     void
@@ -113,15 +116,13 @@ namespace llarp
         LogError("failed to sign our RC for ", m_RemoteAddr);
         return;
       }
-      AlignedBuffer< LinkIntroMessage::MaxSize > data;
+      ILinkSession::Message_t data(LinkIntroMessage::MaxSize + PacketOverhead);
       llarp_buffer_t buf(data);
       if(not msg.BEncode(&buf))
       {
         LogError("failed to encode LIM for ", m_RemoteAddr);
       }
-      buf.sz  = buf.cur - buf.base;
-      buf.cur = buf.base;
-      if(!SendMessageBuffer(buf, h))
+      if(!SendMessageBuffer(std::move(data), h))
       {
         LogError("failed to send LIM to ", m_RemoteAddr);
       }
@@ -131,27 +132,27 @@ namespace llarp
     void
     Session::EncryptAndSend(ILinkSession::Packet_t data)
     {
-      m_EncryptNext.emplace_back(std::move(data));
+      m_EncryptNext->emplace_back(std::move(data));
       if(!IsEstablished())
+      {
         EncryptWorker(std::move(m_EncryptNext));
+        m_EncryptNext = std::make_shared< CryptoQueue_t >();
+      }
     }
 
     void
-    Session::EncryptWorker(CryptoQueue_t msgs)
+    Session::EncryptWorker(CryptoQueue_ptr msgs)
     {
-      LogDebug("encrypt worker ", msgs.size(), " messages");
-      for(auto& pkt : msgs)
+      LogDebug("encrypt worker ", msgs->size(), " messages");
+      for(auto& pkt : *msgs)
       {
         llarp_buffer_t pktbuf(pkt);
-        byte_t* nonce_ptr = pkt.data() + HMACSIZE;
-        CryptoManager::instance()->randbytes(nonce_ptr,
-                                             PacketOverhead - HMACSECSIZE);
+        const TunnelNonce nonce_ptr{pkt.data() + HMACSIZE};
         pktbuf.base += PacketOverhead;
         pktbuf.cur = pktbuf.base;
         pktbuf.sz -= PacketOverhead;
-        CryptoManager::instance()->xchacha20_alt(pktbuf, pktbuf, m_SessionKey,
-                                                 nonce_ptr);
-        pktbuf.base = nonce_ptr;
+        CryptoManager::instance()->xchacha20(pktbuf, m_SessionKey, nonce_ptr);
+        pktbuf.base = pkt.data() + HMACSIZE;
         pktbuf.sz   = pkt.size() - HMACSIZE;
         CryptoManager::instance()->hmac(pkt.data(), pktbuf, m_SessionKey);
         pktbuf.base = pkt.data();
@@ -187,8 +188,7 @@ namespace llarp
               .emplace(msgid,
                        OutboundMessage{msgid, std::move(buf), now, completed})
               .first->second;
-      auto xmit = msg.XMIT();
-      EncryptAndSend(pstd::move(xmit));
+      EncryptAndSend(msg.XMIT());
       msg.FlushUnAcked(util::memFn(&Session::EncryptAndSend, this), now);
       LogDebug("send message ", msgid);
       return true;
@@ -245,15 +245,21 @@ namespace llarp
         }
       }
       auto self = shared_from_this();
-      if(!m_EncryptNext.empty())
+      if(!m_EncryptNext->empty())
+      {
         m_Parent->QueueWork([self, data = std::move(m_EncryptNext)] {
           self->EncryptWorker(data);
         });
+        m_EncryptNext = std::make_shared< CryptoQueue_t >();
+      }
 
-      if(!m_DecryptNext.empty())
+      if(!m_DecryptNext->empty())
+      {
         m_Parent->QueueWork([self, data = std::move(m_DecryptNext)] {
           self->DecryptWorker(data);
         });
+        m_DecryptNext = std::make_shared< CryptoQueue_t >();
+      }
     }
 
     bool
@@ -353,22 +359,23 @@ namespace llarp
     {
       TunnelNonce N;
       N.Randomize();
-      std::vector< byte_t > req;
-      req.reserve(Introduction::SIZE - Signature::SIZE);
+      ILinkSession::Packet_t req(Introduction::SIZE + PacketOverhead);
       const auto pk   = m_Parent->GetOurRC().pubkey;
       const auto e_pk = m_Parent->RouterEncryptionSecret().toPublic();
-      auto itr        = req.begin();
+      auto itr        = req.begin() + PacketOverhead;
       std::copy_n(pk.begin(), pk.size(), itr);
       itr += pk.size();
       std::copy_n(e_pk.begin(), e_pk.size(), itr);
       itr += e_pk.size();
       std::copy(N.begin(), N.end(), itr);
       Signature Z;
-      llarp_buffer_t signbuf(req);
+      llarp_buffer_t signbuf(req.data() + PacketOverhead,
+                             Introduction::SIZE - Signature::SIZE);
       m_Parent->Sign(Z, signbuf);
-      req.reserve(Introduction::SIZE);
       std::copy_n(Z.begin(), Z.size(),
-                  req.begin() + (Introduction::SIZE - Signature::SIZE));
+                  req.begin() + PacketOverhead
+                      + (Introduction::SIZE - Signature::SIZE));
+      CryptoManager::instance()->randbytes(req.data() + HMACSIZE, TUNNONCESIZE);
       EncryptAndSend(std::move(req));
       m_State = State::Introduction;
       if(not CryptoManager::instance()->transport_dh_client(
@@ -392,7 +399,7 @@ namespace llarp
       }
       if(pkt.size() < token.size() + PacketOverhead)
       {
-        LogError("bad session request size, ", result.size(), " < ",
+        LogError("bad session request size, ", pkt.size(), " < ",
                  token.size() + PacketOverhead, " from ", m_RemoteAddr);
         return;
       }
@@ -415,7 +422,7 @@ namespace llarp
         LogWarn("intro too small from ", m_RemoteAddr);
         return;
       }
-      byte_t* ptr = pkt.begin() + PacketOverhead;
+      byte_t* ptr = pkt.data() + PacketOverhead;
       TunnelNonce N;
       std::copy_n(ptr, PubKey::SIZE, m_ExpectedIdent.begin());
       ptr += PubKey::SIZE;
@@ -425,7 +432,7 @@ namespace llarp
       ptr += TunnelNonce::SIZE;
       Signature Z;
       std::copy_n(ptr, Z.size(), Z.begin());
-      const llarp_buffer_t verifybuf(pkt.begin() + PacketOverhead,
+      const llarp_buffer_t verifybuf(pkt.data() + PacketOverhead,
                                      Introduction::SIZE - Signature::SIZE);
       if(!CryptoManager::instance()->verify(m_ExpectedIdent, verifybuf, Z))
       {
@@ -444,7 +451,7 @@ namespace llarp
       }
       std::vector< byte_t > reply(token.size() + PacketOverhead);
       // random nonce
-      CryptoManger::instance()->rand_bytes(reply.data() + HMACSIZE,
+      CryptoManager::instance()->randbytes(reply.data() + HMACSIZE,
                                            TUNNONCESIZE);
       // set token
       std::copy_n(token.begin(), token.size(), reply.begin() + PacketOverhead);
@@ -459,7 +466,7 @@ namespace llarp
     {
       if(pkt.size() < token.size() + PacketOverhead)
       {
-        LogError("bad intro ack size ", reply.size(), " < ",
+        LogError("bad intro ack size ", pkt.size(), " < ",
                  token.size() + PacketOverhead, " from ", m_RemoteAddr);
         return;
       }
@@ -473,8 +480,8 @@ namespace llarp
       std::copy_n(pkt.begin() + PacketOverhead, token.size(), token.begin());
       std::copy_n(token.begin(), token.size(), reply.begin() + PacketOverhead);
       // random nounce
-      CryptoManager::instance()->rand_bytes(reply.data() + HMACSIZE,
-                                            TUNNONCESIZE);
+      CryptoManager::instance()->randbytes(reply.data() + HMACSIZE,
+                                           TUNNONCESIZE);
       EncryptAndSend(std::move(reply));
       LogDebug("sent session request to ", m_RemoteAddr);
       m_State = State::LinkIntro;
@@ -505,10 +512,10 @@ namespace llarp
                  m_RemoteAddr, " state=", int(m_State), " size=", buf.sz);
         return false;
       }
-      const TunnelNounce N{curbuf.base};
+      const TunnelNonce N{curbuf.base};
       curbuf.base += 32;
       curbuf.sz -= 32;
-      LogDebug("decrypt: ", result.size(), " bytes from ", m_RemoteAddr);
+      LogDebug("decrypt: ", curbuf.sz, " bytes from ", m_RemoteAddr);
       return CryptoManager::instance()->xchacha20(curbuf, m_SessionKey, N);
     }
 
@@ -523,22 +530,14 @@ namespace llarp
     void
     Session::HandleSessionData(Packet_t pkt)
     {
-      m_DecryptNext.emplace_back(pkt);
-      // pump rx if we are big enough
-      if(m_DecryptNext.size() >= RXPumpSize)
-      {
-        auto self = shared_from_this();
-        m_Parent->QueueWork([self, data = std::move(m_DecryptNext)] {
-          self->DecryptWorker(data);
-        });
-      }
+      m_DecryptNext->emplace_back(pkt);
     }
 
     void
-    Session::DecryptWorker(CryptoQueue_t msgs)
+    Session::DecryptWorker(CryptoQueue_ptr msgs)
     {
       CryptoQueue_t recvMsgs;
-      for(auto& pkt : msgs)
+      for(auto& pkt : *msgs)
       {
         if(not DecryptMessageInPlace(pkt))
         {
@@ -547,11 +546,11 @@ namespace llarp
         }
         if(pkt[PacketOverhead] != LLARP_PROTO_VERSION)
         {
-          LogError("protocol version missmatch ", int(result[PacketOverhead]),
+          LogError("protocol version missmatch ", int(pkt[PacketOverhead]),
                    " != ", LLARP_PROTO_VERSION);
           continue;
         }
-        recvMsgs.emplace_back(std::move(result));
+        recvMsgs.emplace_back(std::move(pkt));
       }
       LogDebug("decrypted ", recvMsgs.size(), " packets from ", m_RemoteAddr);
       m_Parent->logic()->queue_func(std::bind(
@@ -596,18 +595,18 @@ namespace llarp
     void
     Session::HandleMACK(std::vector< byte_t > data)
     {
-      if(data.size() < 3)
+      if(data.size() < 3 + PacketOverhead)
       {
         LogError("impossibly short mack from ", m_RemoteAddr);
         return;
       }
-      byte_t numAcks = data[2];
+      byte_t numAcks = data[2 + PacketOverhead];
       if(data.size() < ((numAcks * sizeof(uint64_t)) + 3))
       {
         LogError("short mack from ", m_RemoteAddr);
         return;
       }
-      byte_t* ptr = data.data() + 3;
+      byte_t* ptr = data.data() + 3 + PacketOverhead;
       while(numAcks > 0)
       {
         uint64_t acked = bufbe64toh(ptr);
@@ -629,20 +628,18 @@ namespace llarp
     void
     Session::HandleNACK(std::vector< byte_t > data)
     {
-      if(data.size() < 10)
+      if(data.size() < 10 + PacketOverhead)
       {
         LogError("short nack from ", m_RemoteAddr);
         return;
       }
-      uint64_t txid = bufbe64toh(data.data() + 2);
+      uint64_t txid = bufbe64toh(data.data() + 2 + PacketOverhead);
       LogDebug("got nack on ", txid, " from ", m_RemoteAddr);
       auto itr = m_TXMsgs.find(txid);
       if(itr != m_TXMsgs.end())
       {
         auto xmit = itr->second.XMIT();
-        AddRandomPadding(xmit);
-        const llarp_buffer_t pkt(xmit);
-        EncryptAndSend(pkt);
+        EncryptAndSend(std::move(xmit));
       }
       m_LastRX = m_Parent->Now();
     }
@@ -650,14 +647,14 @@ namespace llarp
     void
     Session::HandleXMIT(std::vector< byte_t > data)
     {
-      if(data.size() < 44)
+      if(data.size() < 44 + PacketOverhead)
       {
-        LogError("short XMIT from ", m_RemoteAddr, " ", data.size(), " < 44");
+        LogError("short XMIT from ", m_RemoteAddr);
         return;
       }
-      uint16_t sz   = bufbe16toh(data.data() + 2);
-      uint64_t rxid = bufbe64toh(data.data() + 4);
-      ShortHash h{data.data() + 12};
+      uint16_t sz   = bufbe16toh(data.data() + 2 + PacketOverhead);
+      uint64_t rxid = bufbe64toh(data.data() + 4 + PacketOverhead);
+      ShortHash h{data.data() + 12 + PacketOverhead};
       LogDebug("rxid=", rxid, " sz=", sz, " h=", h.ToHex());
       m_LastRX = m_Parent->Now();
       {
@@ -682,32 +679,30 @@ namespace llarp
     void
     Session::HandleDATA(std::vector< byte_t > data)
     {
-      if(data.size() <= 12)
+      if(data.size() <= 12 + PacketOverhead)
       {
         LogError("short DATA from ", m_RemoteAddr, " ", data.size());
         return;
       }
       m_LastRX      = m_Parent->Now();
-      uint16_t sz   = bufbe16toh(data.data() + 2);
-      uint64_t rxid = bufbe64toh(data.data() + 4);
+      uint16_t sz   = bufbe16toh(data.data() + 2 + PacketOverhead);
+      uint64_t rxid = bufbe64toh(data.data() + 4 + PacketOverhead);
       auto itr      = m_RXMsgs.find(rxid);
       if(itr == m_RXMsgs.end())
       {
         if(m_ReplayFilter.find(rxid) == m_ReplayFilter.end())
         {
           LogDebug("no rxid=", rxid, " for ", m_RemoteAddr);
-          std::vector< byte_t > nack = {
-              LLARP_PROTO_VERSION, Command::eNACK, 0, 0, 0, 0, 0, 0, 0, 0};
-          htobe64buf(nack.data() + 2, rxid);
-          AddRandomPadding(nack);
-          const llarp_buffer_t nackbuf(nack);
-          EncryptAndSend(nackbuf);
+          auto nack = CreatePacket(Command::eNACK, 8);
+          htobe64buf(nack.data() + PacketOverhead + 2, rxid);
+          EncryptAndSend(std::move(nack));
         }
         return;
       }
 
       {
-        const llarp_buffer_t buf(data.data() + 12, data.size() - 12);
+        const llarp_buffer_t buf(data.data() + PacketOverhead + 12,
+                                 data.size() - (PacketOverhead + 12));
         itr->second.HandleData(sz, buf, m_Parent->Now());
       }
 
@@ -732,21 +727,21 @@ namespace llarp
     void
     Session::HandleACKS(std::vector< byte_t > data)
     {
-      if(data.size() < 11)
+      if(data.size() < 11 + PacketOverhead)
       {
-        LogError("short ACKS from ", m_RemoteAddr, " ", data.size(), " < 11");
+        LogError("short ACKS from ", m_RemoteAddr);
         return;
       }
       const auto now = m_Parent->Now();
       m_LastRX       = now;
-      uint64_t txid  = bufbe64toh(data.data() + 2);
+      uint64_t txid  = bufbe64toh(data.data() + 2 + PacketOverhead);
       auto itr       = m_TXMsgs.find(txid);
       if(itr == m_TXMsgs.end())
       {
         LogDebug("no txid=", txid, " for ", m_RemoteAddr);
         return;
       }
-      itr->second.Ack(data[10]);
+      itr->second.Ack(data[10 + PacketOverhead]);
 
       if(itr->second.IsTransmitted())
       {
@@ -777,9 +772,8 @@ namespace llarp
     {
       if(m_State == State::Ready)
       {
-        std::vector< byte_t > ping{LLARP_PROTO_VERSION, Command::ePING};
-        const llarp_buffer_t buf(ping);
-        EncryptAndSend(buf);
+        auto pkt = CreatePacket(Command::ePING, 0);
+        EncryptAndSend(std::move(pkt));
         return true;
       }
       return false;
@@ -792,9 +786,8 @@ namespace llarp
     }
 
     void
-    Session::Recv_LL(const llarp_buffer_t& buf)
+    Session::Recv_LL(ILinkSession::Packet_t data)
     {
-      std::vector< byte_t > data;
       switch(m_State)
       {
         case State::Initial:
@@ -802,8 +795,8 @@ namespace llarp
           {
             // initial data
             // enter introduction phase
-            if(DecryptMessage(buf, data))
-              HandleGotIntro(llarp_buffer_t(data));
+            if(DecryptMessageInPlace(data))
+              HandleGotIntro(std::move(data));
             else
               LogError("bad intro from ", m_RemoteAddr);
           }
@@ -817,18 +810,18 @@ namespace llarp
           if(m_Inbound)
           {
             // we are replying to an intro ack
-            HandleCreateSessionRequest(buf);
+            HandleCreateSessionRequest(std::move(data));
           }
           else
           {
             // we got an intro ack
             // send a session request
-            HandleGotIntroAck(buf);
+            HandleGotIntroAck(std::move(data));
           }
           break;
         case State::LinkIntro:
         default:
-          HandleSessionData(buf);
+          HandleSessionData(std::move(data));
           break;
       }
     }
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index 6d235276b..1ed39eb30 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -167,16 +167,16 @@ namespace llarp
       /// list of rx messages to send in next set of multiacks
       std::vector< uint64_t > m_SendMACKS;
 
-      using CryptoQueue_t = std::vector< Packet_t >;
-
-      CryptoQueue_t m_EncryptNext;
-      CryptoQueue_t m_DecryptNext;
+      using CryptoQueue_t   = std::vector< Packet_t >;
+      using CryptoQueue_ptr = std::shared_ptr< CryptoQueue_t >;
+      CryptoQueue_ptr m_EncryptNext;
+      CryptoQueue_ptr m_DecryptNext;
 
       void
-      EncryptWorker(CryptoQueue_t msgs);
+      EncryptWorker(CryptoQueue_ptr msgs);
 
       void
-      DecryptWorker(CryptoQueue_t msgs);
+      DecryptWorker(CryptoQueue_ptr msgs);
 
       void
       HandlePlaintext(CryptoQueue_t msgs);
diff --git a/llarp/link/factory.cpp b/llarp/link/factory.cpp
index 734e7b1ef..41785e2ac 100644
--- a/llarp/link/factory.cpp
+++ b/llarp/link/factory.cpp
@@ -1,14 +1,11 @@
 #include <link/factory.hpp>
 #include <iwp/iwp.hpp>
-#include <utp/utp.hpp>
 
 namespace llarp
 {
   LinkFactory::LinkType
   LinkFactory::TypeFromName(string_view str)
   {
-    if(str == "utp")
-      return LinkType::eLinkUTP;
     if(str == "iwp")
       return LinkType::eLinkIWP;
     if(str == "mempipe")
@@ -21,8 +18,6 @@ namespace llarp
   {
     switch(tp)
     {
-      case LinkType::eLinkUTP:
-        return "utp";
       case LinkType::eLinkIWP:
         return "iwp";
       case LinkType::eLinkMempipe:
@@ -37,10 +32,6 @@ namespace llarp
   {
     switch(tp)
     {
-      case LinkType::eLinkUTP:
-        if(permitInbound)
-          return llarp::utp::NewInboundLink;
-        return llarp::utp::NewOutboundLink;
       case LinkType::eLinkIWP:
         if(permitInbound)
           return llarp::iwp::NewInboundLink;
diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp
index be1e5d8a9..09d37dbc6 100644
--- a/llarp/link/server.cpp
+++ b/llarp/link/server.cpp
@@ -267,6 +267,7 @@ namespace llarp
   ILinkLayer::Start(std::shared_ptr< Logic > l,
                     std::shared_ptr< thread::ThreadPool > worker)
   {
+    m_Recv   = std::make_shared< TrafficQueue_t >();
     m_Worker = worker;
     m_Logic  = l;
     ScheduleTick(100);
@@ -320,6 +321,7 @@ namespace llarp
         ++itr;
       }
     }
+    m_Recv.reset();
   }
 
   void
@@ -374,7 +376,9 @@ namespace llarp
         ++itr;
       }
     }
-    return s && s->SendMessageBuffer(buf, completed);
+    ILinkSession::Message_t pkt(buf.sz);
+    std::copy_n(buf.base, buf.sz, pkt.begin());
+    return s && s->SendMessageBuffer(std::move(pkt), completed);
   }
 
   bool
@@ -455,4 +459,34 @@ namespace llarp
     tick_id = m_Logic->call_later({interval, this, &ILinkLayer::on_timer_tick});
   }
 
+  void
+  ILinkLayer::udp_tick(llarp_udp_io* udp)
+  {
+    ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
+    if(link->m_Recv == nullptr)
+      return;
+    link->logic()->queue_func([traffic = std::move(link->m_Recv), l = link]() {
+      auto itr = traffic->begin();
+      while(itr != traffic->end())
+      {
+        l->RecvFrom(itr->first, std::move(itr->second));
+        ++itr;
+      }
+      l->Pump();
+    });
+    link->m_Recv.reset(new TrafficQueue_t());
+  }
+
+  void
+  ILinkLayer::udp_recv_from(llarp_udp_io* udp, const sockaddr* from,
+                            ManagedBuffer buf)
+  {
+    ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
+    if(link->m_Recv == nullptr)
+      return;
+    link->m_Recv->emplace_back(std::make_pair(*from, buf.underlying.sz));
+    std::copy_n(buf.underlying.base, buf.underlying.sz,
+                link->m_Recv->back().second.begin());
+  }
+
 }  // namespace llarp
diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp
index 3ef387719..b4fe17f1b 100644
--- a/llarp/link/server.hpp
+++ b/llarp/link/server.hpp
@@ -77,30 +77,10 @@ namespace llarp
         LOCKS_EXCLUDED(m_AuthedLinksMutex);
 
     static void
-    udp_tick(llarp_udp_io* udp)
-    {
-      ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
-      link->logic()->queue_func([link]() { link->Pump(); });
-    }
+    udp_tick(llarp_udp_io* udp);
 
     static void
-    udp_recv_from(llarp_udp_io* udp, const sockaddr* from, ManagedBuffer buf)
-    {
-      if(!udp)
-      {
-        llarp::LogWarn("no udp set");
-        return;
-      }
-      std::vector< byte_t > pkt(buf.underlying.sz);
-      std::copy_n(buf.underlying.base, buf.underlying.sz, pkt.begin());
-      const llarp::Addr srcaddr(*from);
-      // maybe check from too?
-      // no it's never null
-      ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
-      link->logic()->queue_func([link, srcaddr, pkt]() {
-        link->RecvFrom(srcaddr, pkt.data(), pkt.size());
-      });
-    }
+    udp_recv_from(llarp_udp_io* udp, const sockaddr* from, ManagedBuffer buf);
 
     void
     SendTo_LL(const llarp::Addr& to, const llarp_buffer_t& pkt)
@@ -119,7 +99,7 @@ namespace llarp
     Pump();
 
     virtual void
-    RecvFrom(const Addr& from, const void* buf, size_t sz) = 0;
+    RecvFrom(const Addr& from, ILinkSession::Packet_t pkt) = 0;
 
     bool
     PickAddress(const RouterContact& rc, AddressInfo& picked) const;
@@ -272,6 +252,11 @@ namespace llarp
     mutable DECLARE_LOCK(Mutex_t, m_PendingMutex,
                          ACQUIRED_AFTER(m_AuthedLinksMutex));
     Pending m_Pending GUARDED_BY(m_PendingMutex);
+
+    using TrafficEvent_t = std::pair< Addr, ILinkSession::Packet_t >;
+    using TrafficQueue_t = std::vector< TrafficEvent_t >;
+
+    std::shared_ptr< TrafficQueue_t > m_Recv;
   };
 
   using LinkLayer_ptr = std::shared_ptr< ILinkLayer >;
diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp
index ea0fd9a2c..6c4b51fbf 100644
--- a/llarp/nodedb.cpp
+++ b/llarp/nodedb.cpp
@@ -113,31 +113,18 @@ llarp_nodedb::getRCFilePath(const llarp::RouterID &pubkey) const
   return filepath.string();
 }
 
-static void
-handle_async_insert_rc(llarp_nodedb *nodedb, const llarp::RouterContact &rc,
-                       std::shared_ptr< llarp::Logic > logic,
-                       const std::function< void(void) > &completedHook)
-{
-  nodedb->Insert(rc);
-  if(logic && completedHook)
-  {
-    logic->queue_func(completedHook);
-  }
-}
-
 void
 llarp_nodedb::InsertAsync(llarp::RouterContact rc,
                           std::shared_ptr< llarp::Logic > logic,
                           std::function< void(void) > completionHandler)
 {
-  const auto job =
-      std::bind(&handle_async_insert_rc, this, rc, logic, completionHandler);
-  size_t tries = 10;
-  while((!disk->addJob(job)) && tries-- > 0)
-    std::this_thread::sleep_for(std::chrono::milliseconds(1));
-  // on fail do synchronous write
-  if(tries == 0)
-    Insert(rc);
+  disk->addJob([this, rc, logic, completionHandler]() {
+    this->Insert(rc);
+    if(logic && completionHandler)
+    {
+      logic->queue_func([completionHandler] { completionHandler(); });
+    }
+  });
 }
 
 bool
@@ -197,8 +184,8 @@ llarp_nodedb::Insert(const llarp::RouterContact &rc)
     if(itr != entries.end())
       entries.erase(itr);
     entries.emplace(rc.pubkey.as_array(), rc);
-    LogInfo("Added or updated RC for ", llarp::RouterID(rc.pubkey),
-            " to nodedb.  Current nodedb count is: ", entries.size());
+    LogDebug("Added or updated RC for ", llarp::RouterID(rc.pubkey),
+             " to nodedb.  Current nodedb count is: ", entries.size());
   }
   return true;
 }
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 2f36aab94..99e713cdd 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -21,7 +21,6 @@
 #include <util/meta/memfn.hpp>
 #include <util/metrics/metrics.hpp>
 #include <util/str.hpp>
-#include <utp/utp.hpp>
 #include <ev/ev.hpp>
 
 #include <fstream>
diff --git a/llarp/util/thread/logic.cpp b/llarp/util/thread/logic.cpp
index ed4fd0532..432b73b53 100644
--- a/llarp/util/thread/logic.cpp
+++ b/llarp/util/thread/logic.cpp
@@ -62,14 +62,9 @@ namespace llarp
   }
 
   bool
-  Logic::queue_func(std::function< void(void) > f)
+  Logic::queue_func(std::function< void(void) >&& f)
   {
-    if(!this->thread->QueueFunc(f))
-    {
-      // try calling it later if the job queue overflows
-      this->call_later(1, f);
-    }
-    return true;
+    return this->thread->impl->addJob(f);
   }
 
   void
diff --git a/llarp/util/thread/logic.hpp b/llarp/util/thread/logic.hpp
index 6e36898ed..ec3b2fae2 100644
--- a/llarp/util/thread/logic.hpp
+++ b/llarp/util/thread/logic.hpp
@@ -43,7 +43,7 @@ namespace llarp
     queue_job(struct llarp_thread_job job);
 
     bool
-    queue_func(std::function< void(void) > func);
+    queue_func(std::function< void(void) >&& func);
 
     uint32_t
     call_later(const llarp_timeout_job& job);
diff --git a/llarp/util/thread/threading.cpp b/llarp/util/thread/threading.cpp
index 285e3cd0b..359417a03 100644
--- a/llarp/util/thread/threading.cpp
+++ b/llarp/util/thread/threading.cpp
@@ -1,6 +1,7 @@
 #include <util/thread/threading.hpp>
 
 #include <util/logging/logger.hpp>
+#include <cstring>
 
 #ifdef POSIX
 #include <pthread.h>
@@ -40,7 +41,7 @@ namespace llarp
       if(rc)
       {
         LogError("Failed to set thread name to ", name, " errno = ", rc,
-                 " errstr = ", strerror(rc));
+                 " errstr = ", ::strerror(rc));
       }
 #endif
 #elif _MSC_VER
diff --git a/llarp/util/thread/threadpool.h b/llarp/util/thread/threadpool.h
index c8f6e9cee..33ce00df4 100644
--- a/llarp/util/thread/threadpool.h
+++ b/llarp/util/thread/threadpool.h
@@ -38,15 +38,6 @@ struct llarp_threadpool
       return jobs->size();
     return 0;
   }
-
-  bool
-  QueueFunc(std::function< void(void) > f)
-  {
-    if(impl)
-      return impl->tryAddJob(f);
-
-    return jobs->tryPushBack(f) == llarp::thread::QueueReturn::Success;
-  }
 };
 
 struct llarp_threadpool *
diff --git a/llarp/utp/inbound_message.cpp b/llarp/utp/inbound_message.cpp
deleted file mode 100644
index a86eb9096..000000000
--- a/llarp/utp/inbound_message.cpp
+++ /dev/null
@@ -1,27 +0,0 @@
-#include <utp/inbound_message.hpp>
-
-#include <cstring>
-
-namespace llarp
-{
-  namespace utp
-  {
-    bool
-    InboundMessage::IsExpired(llarp_time_t now) const
-    {
-      return now > lastActive && now - lastActive >= 2000;
-    }
-
-    bool
-    InboundMessage::AppendData(const byte_t* ptr, uint16_t sz)
-    {
-      if(buffer.size_left() < sz)
-        return false;
-      std::copy_n(ptr, sz, buffer.cur);
-      buffer.cur += sz;
-      return true;
-    }
-
-  }  // namespace utp
-
-}  // namespace llarp
diff --git a/llarp/utp/inbound_message.hpp b/llarp/utp/inbound_message.hpp
deleted file mode 100644
index 0bcf69757..000000000
--- a/llarp/utp/inbound_message.hpp
+++ /dev/null
@@ -1,86 +0,0 @@
-#ifndef LLARP_UTP_INBOUND_MESSAGE_HPP
-#define LLARP_UTP_INBOUND_MESSAGE_HPP
-
-#include <constants/link_layer.hpp>
-#include <util/aligned.hpp>
-#include <util/types.hpp>
-
-#include <utp_types.h>  // for uint32
-#include <cstring>
-
-namespace llarp
-{
-  namespace utp
-  {
-    /// size of keyed hash
-    constexpr size_t FragmentHashSize = 32;
-    /// size of outer nonce
-    constexpr size_t FragmentNonceSize = 32;
-    /// size of outer overhead
-    constexpr size_t FragmentOverheadSize =
-        FragmentHashSize + FragmentNonceSize;
-    /// max fragment payload size
-    constexpr size_t FragmentBodyPayloadSize = 512;
-    /// size of inner nonce
-    constexpr size_t FragmentBodyNonceSize = 24;
-    /// size of fragment body overhead
-    constexpr size_t FragmentBodyOverhead = FragmentBodyNonceSize
-        + sizeof(uint32) + sizeof(uint16_t) + sizeof(uint16_t);
-    /// size of fragment body
-    constexpr size_t FragmentBodySize =
-        FragmentBodyOverhead + FragmentBodyPayloadSize;
-    /// size of fragment
-    constexpr size_t FragmentBufferSize =
-        FragmentOverheadSize + FragmentBodySize;
-
-    static_assert(FragmentBufferSize == 608, "Fragment Buffer Size is not 608");
-
-    /// buffer for a single utp fragment
-    using FragmentBuffer = AlignedBuffer< FragmentBufferSize >;
-
-    /// buffer for a link layer message
-    using MessageBuffer = AlignedBuffer< MAX_LINK_MSG_SIZE >;
-
-    /// pending inbound message being received
-    struct InboundMessage
-    {
-      /// timestamp of last activity
-      llarp_time_t lastActive{0};
-      /// the underlying message buffer
-      MessageBuffer _msg;
-
-      /// for accessing message buffer
-      llarp_buffer_t buffer;
-
-      /// return true if this inbound message can be removed due to expiration
-      bool
-      IsExpired(llarp_time_t now) const;
-
-      /// append data at ptr of size sz bytes to message buffer
-      /// increment current position
-      /// return false if we don't have enough room
-      /// return true on success
-      bool
-      AppendData(const byte_t* ptr, uint16_t sz);
-
-      InboundMessage() : _msg(), buffer(_msg)
-      {
-      }
-
-      InboundMessage(const InboundMessage& other)
-          : lastActive(other.lastActive), _msg(other._msg), buffer(_msg)
-      {
-      }
-    };
-
-    inline bool
-    operator==(const InboundMessage& lhs, const InboundMessage& rhs)
-    {
-      return lhs.buffer.base == rhs.buffer.base;
-    }
-
-  }  // namespace utp
-
-}  // namespace llarp
-
-#endif
diff --git a/llarp/utp/linklayer.cpp b/llarp/utp/linklayer.cpp
deleted file mode 100644
index 41a968255..000000000
--- a/llarp/utp/linklayer.cpp
+++ /dev/null
@@ -1,341 +0,0 @@
-#include <utp/linklayer.hpp>
-
-#include <utp/session.hpp>
-
-#ifdef __linux__
-#include <linux/errqueue.h>
-#include <netinet/ip_icmp.h>
-#endif
-
-#ifdef _WIN32
-#include <winsock2.h>
-#include <ws2tcpip.h>
-#include <wspiapi.h>
-#endif
-
-#ifndef IP_DONTFRAGMENT
-#define IP_DONTFRAGMENT IP_DONTFRAG
-#endif
-
-#include <functional>
-#include <cstring>
-
-namespace llarp
-{
-  namespace utp
-  {
-    uint64
-    LinkLayer::OnConnect(utp_callback_arguments* arg)
-    {
-      auto* l =
-          static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
-      auto* session = static_cast< Session* >(utp_get_userdata(arg->socket));
-      if(session && l)
-        session->OutboundLinkEstablished(l);
-      return 0;
-    }
-
-    uint64
-    LinkLayer::SendTo(utp_callback_arguments* arg)
-    {
-      auto* l =
-          static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
-      if(l == nullptr)
-        return 0;
-      LogDebug("utp_sendto ", Addr(*arg->address), " ", arg->len, " bytes");
-      return l->m_udp.sendto(&l->m_udp, arg->address, arg->buf, arg->len);
-    }
-
-    uint64
-    LinkLayer::OnError(utp_callback_arguments* arg)
-    {
-      auto* session = static_cast< Session* >(utp_get_userdata(arg->socket));
-
-      auto* link =
-          static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
-
-      if(session && link)
-      {
-        link->HandleTimeout(session);
-        session->Close();
-      }
-      return 0;
-    }
-
-    uint64
-    LinkLayer::OnLog(utp_callback_arguments* arg)
-    {
-      LogDebug(arg->buf);
-      return 0;
-    }
-
-    LinkLayer::LinkLayer(const SecretKey& routerEncSecret, GetRCFunc getrc,
-                         LinkMessageHandler h, SignBufferFunc sign,
-                         SessionEstablishedHandler established,
-                         SessionRenegotiateHandler reneg,
-                         TimeoutHandler timeout, SessionClosedHandler closed,
-                         bool permitInbound)
-        : ILinkLayer(routerEncSecret, getrc, h, sign, established, reneg,
-                     timeout, closed)
-    {
-      _utp_ctx = utp_init(2);
-      utp_context_set_userdata(_utp_ctx, this);
-      utp_set_callback(_utp_ctx, UTP_SENDTO, &LinkLayer::SendTo);
-      if(permitInbound)
-        utp_set_callback(_utp_ctx, UTP_ON_ACCEPT, &LinkLayer::OnAccept);
-      utp_set_callback(_utp_ctx, UTP_ON_CONNECT, &LinkLayer::OnConnect);
-      utp_set_callback(_utp_ctx, UTP_ON_STATE_CHANGE,
-                       &LinkLayer::OnStateChange);
-      utp_set_callback(_utp_ctx, UTP_ON_READ, &LinkLayer::OnRead);
-      utp_set_callback(_utp_ctx, UTP_ON_ERROR, &LinkLayer::OnError);
-      utp_set_callback(_utp_ctx, UTP_LOG, &LinkLayer::OnLog);
-      utp_context_set_option(_utp_ctx, UTP_LOG_NORMAL, 1);
-      utp_context_set_option(_utp_ctx, UTP_LOG_MTU, 1);
-      utp_context_set_option(_utp_ctx, UTP_LOG_DEBUG, 1);
-      utp_context_set_option(
-          _utp_ctx, UTP_SNDBUF,
-          (MAX_LINK_MSG_SIZE * MaxSendQueueSize * size_t{3}) / size_t{2});
-      utp_context_set_option(
-          _utp_ctx, UTP_RCVBUF,
-          (MAX_LINK_MSG_SIZE * MaxSendQueueSize * size_t{3}) / size_t{2});
-    }
-
-    LinkLayer::~LinkLayer()
-    {
-      m_Pending.clear();
-      m_AuthedLinks.clear();
-      utp_destroy(_utp_ctx);
-    }
-
-    uint16_t
-    LinkLayer::Rank() const
-    {
-      return 1;
-    }
-
-    void
-    LinkLayer::RecvFrom(const Addr& from, const void* buf, size_t sz)
-    {
-      utp_process_udp(_utp_ctx, (const byte_t*)buf, sz, from, from.SockLen());
-    }
-
-#ifdef __linux__
-
-    void
-    LinkLayer::ProcessICMP()
-    {
-#ifndef TESTNET
-      do
-      {
-        byte_t vec_buf[4096], ancillary_buf[4096];
-        struct iovec iov = {vec_buf, sizeof(vec_buf)};
-        struct sockaddr_in remote;
-        struct msghdr msg;
-        ssize_t len;
-        struct cmsghdr* cmsg;
-        struct sock_extended_err* e;
-        struct sockaddr* icmp_addr;
-        struct sockaddr_in* icmp_sin;
-
-        memset(&msg, 0, sizeof(msg));
-
-        msg.msg_name       = &remote;
-        msg.msg_namelen    = sizeof(remote);
-        msg.msg_iov        = &iov;
-        msg.msg_iovlen     = 1;
-        msg.msg_flags      = 0;
-        msg.msg_control    = ancillary_buf;
-        msg.msg_controllen = sizeof(ancillary_buf);
-
-        len = recvmsg(m_udp.fd, &msg, MSG_ERRQUEUE | MSG_DONTWAIT);
-        if(len < 0)
-        {
-          if(errno == EAGAIN || errno == EWOULDBLOCK)
-            errno = 0;
-          else
-            LogError("failed to read icmp for utp ", strerror(errno));
-          return;
-        }
-
-        for(cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg))
-        {
-          if(cmsg->cmsg_type != IP_RECVERR)
-          {
-            continue;
-          }
-          if(cmsg->cmsg_level != SOL_IP)
-          {
-            continue;
-          }
-          e = (struct sock_extended_err*)CMSG_DATA(cmsg);
-          if(!e)
-            continue;
-          if(e->ee_origin != SO_EE_ORIGIN_ICMP)
-          {
-            continue;
-          }
-          icmp_addr = (struct sockaddr*)SO_EE_OFFENDER(e);
-          icmp_sin  = (struct sockaddr_in*)icmp_addr;
-          if(icmp_sin->sin_port != 0)
-          {
-            continue;
-          }
-          if(e->ee_type == 3 && e->ee_code == 4)
-          {
-            utp_process_icmp_fragmentation(_utp_ctx, vec_buf, len,
-                                           (struct sockaddr*)&remote,
-                                           sizeof(remote), e->ee_info);
-          }
-          else
-          {
-            utp_process_icmp_error(_utp_ctx, vec_buf, len,
-                                   (struct sockaddr*)&remote, sizeof(remote));
-          }
-        }
-      } while(true);
-#endif
-    }
-#endif
-
-    void
-    LinkLayer::Pump()
-    {
-#ifdef __linux__
-      ProcessICMP();
-#endif
-      utp_issue_deferred_acks(_utp_ctx);
-      std::set< RouterID > sessions;
-      {
-        ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
-        auto itr = m_AuthedLinks.begin();
-        while(itr != m_AuthedLinks.end())
-        {
-          sessions.insert(itr->first);
-          ++itr;
-        }
-      }
-      ILinkLayer::Pump();
-      {
-        ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
-        for(const auto& pk : sessions)
-        {
-          if(m_AuthedLinks.count(pk) == 0)
-          {
-            // all sessions were removed
-            SessionClosed(pk);
-          }
-        }
-      }
-    }
-
-    void
-    LinkLayer::Stop()
-    {
-      ForEachSession([](ILinkSession* s) { s->Close(); });
-    }
-
-    bool
-    LinkLayer::KeyGen(SecretKey& k)
-    {
-      CryptoManager::instance()->encryption_keygen(k);
-      return true;
-    }
-
-    void
-    LinkLayer::Tick(llarp_time_t now)
-    {
-      utp_check_timeouts(_utp_ctx);
-      ILinkLayer::Tick(now);
-    }
-
-    utp_socket*
-    LinkLayer::NewSocket()
-    {
-      return utp_create_socket(_utp_ctx);
-    }
-
-    const char*
-    LinkLayer::Name() const
-    {
-      return "utp";
-    }
-
-    std::shared_ptr< ILinkSession >
-    LinkLayer::NewOutboundSession(const RouterContact& rc,
-                                  const AddressInfo& addr)
-    {
-      return std::make_shared< OutboundSession >(this, NewSocket(), rc, addr);
-    }
-
-    uint64
-    LinkLayer::OnRead(utp_callback_arguments* arg)
-    {
-      auto* self = static_cast< Session* >(utp_get_userdata(arg->socket));
-
-      if(self)
-      {
-        if(self->state == Session::eClose)
-        {
-          return 0;
-        }
-        if(!self->Recv(arg->buf, arg->len))
-        {
-          LogDebug("recv fail for ", self->remoteAddr);
-          self->Close();
-          return 0;
-        }
-        utp_read_drained(arg->socket);
-      }
-      else
-      {
-        LogWarn("utp_socket got data with no underlying session");
-        utp_close(arg->socket);
-      }
-      return 0;
-    }
-
-    uint64
-    LinkLayer::OnStateChange(utp_callback_arguments* arg)
-    {
-      auto* session = static_cast< Session* >(utp_get_userdata(arg->socket));
-      if(session)
-      {
-        if(arg->state == UTP_STATE_WRITABLE)
-        {
-          session->Pump();
-        }
-        else if(arg->state == UTP_STATE_EOF)
-        {
-          LogDebug("got eof from ", session->remoteAddr);
-          session->Close();
-        }
-      }
-      return 0;
-    }
-
-    uint64
-    LinkLayer::OnAccept(utp_callback_arguments* arg)
-    {
-      auto* self =
-          static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
-      Addr remote(*arg->address);
-
-      std::shared_ptr< ILinkSession > session =
-          std::make_shared< InboundSession >(self, arg->socket, remote);
-      if(!self->PutSession(session))
-      {
-        LogWarn("dropping inbound utp session from ", remote);
-        // close later
-        self->m_Logic->call_later(50, [=]() { session->Close(); });
-      }
-      else
-      {
-        LogDebug("utp accepted from ", remote);
-        session->OnLinkEstablished(self);
-      }
-      return 0;
-    }
-
-  }  // namespace utp
-
-}  // namespace llarp
diff --git a/llarp/utp/linklayer.hpp b/llarp/utp/linklayer.hpp
deleted file mode 100644
index cf74df8a5..000000000
--- a/llarp/utp/linklayer.hpp
+++ /dev/null
@@ -1,105 +0,0 @@
-#ifndef LLARP_UTP_LINKLAYER_HPP
-#define LLARP_UTP_LINKLAYER_HPP
-
-#include <utp/inbound_message.hpp>
-
-#include <crypto/crypto.hpp>
-#include <crypto/types.hpp>
-#include <link/server.hpp>
-#include <utp.h>
-
-#include <deque>
-
-namespace llarp
-{
-  namespace utp
-  {
-    struct LinkLayer final : public ILinkLayer
-    {
-      utp_context* _utp_ctx = nullptr;
-
-      // low level read callback
-      static uint64
-      OnRead(utp_callback_arguments* arg);
-
-      // low level sendto callback
-      static uint64
-      SendTo(utp_callback_arguments* arg);
-
-      /// error callback
-      static uint64
-      OnError(utp_callback_arguments* arg);
-
-      /// state change callback
-      static uint64
-      OnStateChange(utp_callback_arguments*);
-
-      static uint64
-      OnConnect(utp_callback_arguments*);
-
-      /// accept callback
-      static uint64
-      OnAccept(utp_callback_arguments*);
-
-      /// logger callback
-      static uint64
-      OnLog(utp_callback_arguments* arg);
-
-      /// construct
-      LinkLayer(const SecretKey& routerEncSecret, GetRCFunc getrc,
-                LinkMessageHandler h, SignBufferFunc sign,
-                SessionEstablishedHandler established,
-                SessionRenegotiateHandler reneg, TimeoutHandler timeout,
-                SessionClosedHandler closed, bool acceptInbound);
-
-      /// destruct
-      ~LinkLayer() override;
-
-      /// get AI rank
-      uint16_t
-      Rank() const override;
-
-      /// handle low level recv
-      void
-      RecvFrom(const Addr& from, const void* buf, size_t sz) override;
-
-#ifdef __linux__
-      /// process ICMP stuff on linux
-      void
-      ProcessICMP();
-#endif
-
-      /// pump sessions
-      void
-      Pump() override;
-
-      /// stop link layer
-      void
-      Stop() override;
-
-      /// regenerate transport keypair
-      bool
-      KeyGen(SecretKey& k) override;
-
-      /// do tick
-      void
-      Tick(llarp_time_t now);
-
-      /// create new outbound session
-      std::shared_ptr< ILinkSession >
-      NewOutboundSession(const RouterContact& rc,
-                         const AddressInfo& addr) override;
-
-      /// create new socket
-      utp_socket*
-      NewSocket();
-
-      /// get ai name
-      const char*
-      Name() const override;
-    };
-
-  }  // namespace utp
-}  // namespace llarp
-
-#endif
diff --git a/llarp/utp/session.cpp b/llarp/utp/session.cpp
deleted file mode 100644
index 8c48383af..000000000
--- a/llarp/utp/session.cpp
+++ /dev/null
@@ -1,810 +0,0 @@
-#include <utp/session.hpp>
-
-#include <utp/linklayer.hpp>
-#include <messages/discard.hpp>
-#include <messages/link_intro.hpp>
-#include <util/metrics/metrics.hpp>
-#include <util/meta/memfn.hpp>
-
-namespace llarp
-{
-  namespace utp
-  {
-    void
-    Session::OnLinkEstablished(ILinkLayer* p)
-    {
-      parent = p;
-      EnterState(eLinkEstablished);
-      LogDebug("link established with ", remoteAddr);
-    }
-
-    /// pump tx queue
-    void
-    Session::PumpWrite(size_t numSend)
-    {
-      if(!sock)
-        return;
-      ssize_t expect = 0;
-      std::vector< utp_iovec > send;
-
-      for(const auto& msg : sendq)
-      {
-        for(const auto& vec : msg.vecs)
-        {
-          if(vec.iov_len > 0)
-          {
-            expect += vec.iov_len;
-            send.emplace_back(vec);
-            numSend--;
-          }
-        }
-      }
-      if(expect)
-      {
-        ssize_t s = utp_writev(sock, send.data(), send.size());
-        if(s <= 0)
-          return;
-        lastSend = parent->Now();
-
-        metrics::integerTick("utp.session.tx", "writes", s, "id",
-                             RouterID(remoteRC.pubkey).ToString());
-        m_TXRate += s;
-        size_t sz = s;
-        do
-        {
-          auto& msg = sendq.front();
-          while(msg.vecs.size() > 0 && sz >= msg.vecs.front().iov_len)
-          {
-            sz -= msg.vecs.front().iov_len;
-            msg.vecs.pop_front();
-          }
-          if(msg.vecs.size() == 0)
-          {
-            msg.Delivered();
-            sendq.pop_front();
-          }
-          else if(sz)
-          {
-            auto& front = msg.vecs.front();
-            front.iov_len -= sz;
-            front.iov_base = ((byte_t*)front.iov_base) + sz;
-            return;
-          }
-          else
-            return;
-        } while(sendq.size());
-      }
-    }
-
-    /// prune expired inbound messages
-    void
-    Session::PruneInboundMessages(llarp_time_t now)
-    {
-      auto itr = m_RecvMsgs.begin();
-      while(itr != m_RecvMsgs.end())
-      {
-        if(itr->second.IsExpired(now))
-        {
-          itr = m_RecvMsgs.erase(itr);
-        }
-        else
-          ++itr;
-      }
-    }
-
-    void
-    Session::OutboundLinkEstablished(LinkLayer* p)
-    {
-      OnLinkEstablished(p);
-      metrics::integerTick("utp.session.open", "to", 1, "id",
-                           RouterID(remoteRC.pubkey).ToString());
-      OutboundHandshake();
-    }
-
-    template < bool (Crypto::*dh_func)(SharedSecret&, const PubKey&,
-                                       const SecretKey&, const TunnelNonce&) >
-    bool
-    Session::DoKeyExchange(SharedSecret& K, const KeyExchangeNonce& n,
-                           const PubKey& other, const SecretKey& secret)
-    {
-      ShortHash t_h;
-      static constexpr size_t TMP_SIZE = 64;
-      static_assert(SharedSecret::SIZE + KeyExchangeNonce::SIZE == TMP_SIZE,
-                    "Invalid sizes");
-
-      AlignedBuffer< TMP_SIZE > tmp;
-      std::copy(K.begin(), K.end(), tmp.begin());
-      std::copy(n.begin(), n.end(), tmp.begin() + K.size());
-      // t_h = HS(K + L.n)
-      if(!CryptoManager::instance()->shorthash(t_h, llarp_buffer_t(tmp)))
-      {
-        LogError("failed to mix key to ", remoteAddr);
-        return false;
-      }
-
-      // K = TKE(a.p, B_a.e, sk, t_h)
-      if(!(CryptoManager::instance()->*dh_func)(K, other, secret, t_h))
-      {
-        LogError("key exchange with ", other, " failed");
-        return false;
-      }
-      LogDebug("keys mixed with session to ", remoteAddr);
-      return true;
-    }
-
-    bool
-    Session::MutateKey(SharedSecret& K, const AlignedBuffer< 24 >& A)
-    {
-      AlignedBuffer< 56 > tmp;
-      llarp_buffer_t buf{tmp};
-      std::copy(K.begin(), K.end(), buf.cur);
-      buf.cur += K.size();
-      std::copy(A.begin(), A.end(), buf.cur);
-      buf.cur = buf.base;
-      return CryptoManager::instance()->shorthash(K, buf);
-    }
-
-    void
-    Session::Tick(llarp_time_t now)
-    {
-      PruneInboundMessages(now);
-      // ensure that this section is called every 1s or so
-      if(now > m_LastTick && now - m_LastTick >= 1000)
-      {
-        m_TXRate = 0;
-        m_RXRate = 0;
-        metrics::integerTick("utp.session.sendq", "size", sendq.size(), "id",
-                             RouterID(remoteRC.pubkey).ToString());
-        m_LastTick = now;
-      }
-      else
-      {
-        // try sending 1 segment
-        PumpWrite(1);
-      }
-    }
-
-    /// low level read
-    bool
-    Session::Recv(const byte_t* buf, size_t sz)
-    {
-      // mark we are alive
-      Alive();
-      m_RXRate += sz;
-      size_t s = sz;
-      metrics::integerTick("utp.session.rx", "size", s, "id",
-                           RouterID(remoteRC.pubkey).ToString());
-      // process leftovers
-      if(recvBufOffset)
-      {
-        auto left = FragmentBufferSize - recvBufOffset;
-        if(s >= left)
-        {
-          // yes it fills it
-          LogDebug("process leftovers, offset=", recvBufOffset, " sz=", s,
-                   " left=", left);
-          std::copy(buf, buf + left, recvBuf.begin() + recvBufOffset);
-          s -= left;
-          recvBufOffset = 0;
-          buf += left;
-          if(!VerifyThenDecrypt(recvBuf.data()))
-            return false;
-        }
-      }
-      // process full fragments
-      while(s >= FragmentBufferSize)
-      {
-        recvBufOffset = 0;
-        LogDebug("process full sz=", s);
-        if(!VerifyThenDecrypt(buf))
-          return false;
-        buf += FragmentBufferSize;
-        s -= FragmentBufferSize;
-      }
-      if(s)
-      {
-        // hold onto leftovers
-        LogDebug("leftovers sz=", s);
-        std::copy(buf, buf + s, recvBuf.begin() + recvBufOffset);
-        recvBufOffset += s;
-      }
-      return true;
-    }
-
-    bool
-    Session::TimedOut(llarp_time_t now) const
-    {
-      if(state == eClose)
-        return true;
-      if(state == eConnecting)
-        return now - lastActive > 5000;
-      if(state == eSessionReady)
-      {
-        const bool remoteIsSNode = remoteRC.IsPublicRouter();
-        const bool weAreSnode    = parent->GetOurRC().IsPublicRouter();
-        const bool recvTimeout =
-            (now > lastActive) && now - lastActive > sessionTimeout;
-        const bool sendTimeout =
-            (now > lastSend) && now - lastSend > sessionTimeout;
-        if(remoteIsSNode && weAreSnode)
-        {
-          /// for s2s connections only check write direction
-          return sendTimeout;
-        }
-        else if(weAreSnode)
-        {
-          // for edge connection as service node check either direction for
-          // timeout
-          return recvTimeout || sendTimeout;
-        }
-        else
-        {
-          /// for edge connections as client we check if both directions have
-          /// been silent for 60s
-          return recvTimeout && sendTimeout;
-        }
-      }
-      if(state == eLinkEstablished)
-        return now - lastActive
-            > 10000;  /// 10 second timeout for the whole handshake
-      return true;
-    }
-
-    PubKey
-    Session::GetPubKey() const
-    {
-      return remoteRC.pubkey;
-    }
-
-    Addr
-    Session::GetRemoteEndpoint() const
-    {
-      return remoteAddr;
-    }
-
-    /// base constructor
-    Session::Session(LinkLayer* p)
-    {
-      state         = eInitial;
-      m_NextTXMsgID = 0;
-      m_NextRXMsgID = 0;
-      parent        = p;
-      remoteTransportPubKey.Zero();
-
-      gotLIM        = false;
-      recvBufOffset = 0;
-
-      lastActive = parent->Now();
-    }
-
-    bool
-    Session::ShouldPing() const
-    {
-      if(state != eSessionReady)
-        return false;
-      const auto dlt = parent->Now() - lastSend;
-      return dlt >= 10000;
-    }
-
-    ILinkLayer*
-    Session::GetLinkLayer() const
-    {
-      return parent;
-    }
-
-    void
-    Session::Pump()
-    {
-      if(sendq.size() > (MaxSendQueueSize / 4))
-        PumpWrite(sendq.size() / 2);
-      else
-        PumpWrite(sendq.size());
-      // prune inbound messages
-      PruneInboundMessages(parent->Now());
-    }
-
-    bool
-    Session::SendMessageBuffer(
-        const llarp_buffer_t& buf,
-        ILinkSession::CompletionHandler completionHandler)
-    {
-      if(SendQueueBacklog() >= MaxSendQueueSize)
-      {
-        // pump write queue if we seem to be full
-        PumpWrite(MaxSendQueueSize / 2);
-      }
-      if(SendQueueBacklog() >= MaxSendQueueSize)
-      {
-        // we didn't pump anything wtf
-        // this means we're stalled
-        return false;
-      }
-      size_t sz      = buf.sz;
-      byte_t* ptr    = buf.base;
-      uint32_t msgid = m_NextTXMsgID++;
-      sendq.emplace_back(msgid, completionHandler);
-      while(sz)
-      {
-        uint32_t s = std::min(FragmentBodyPayloadSize, sz);
-        if(!EncryptThenHash(ptr, msgid, s, sz - s))
-        {
-          LogError("EncryptThenHash failed?!");
-          Close();
-          return false;
-        }
-        LogDebug("encrypted ", s, " bytes");
-        ptr += s;
-        sz -= s;
-      }
-      if(state != eSessionReady)
-        PumpWrite(sendq.size());
-      return true;
-    }
-
-    bool
-    Session::SendKeepAlive()
-    {
-      if(ShouldPing())
-      {
-        DiscardMessage msg;
-        std::array< byte_t, 128 > tmp;
-        llarp_buffer_t buf(tmp);
-        if(!msg.BEncode(&buf))
-          return false;
-        buf.sz  = buf.cur - buf.base;
-        buf.cur = buf.base;
-        return this->SendMessageBuffer(buf, nullptr);
-      }
-      return true;
-    }
-
-    void
-    Session::OutboundHandshake()
-    {
-      std::array< byte_t, LinkIntroMessage::MaxSize > tmp;
-      llarp_buffer_t buf(tmp);
-      // build our RC
-      LinkIntroMessage msg;
-      msg.rc = parent->GetOurRC();
-      if(!msg.rc.Verify(parent->Now()))
-      {
-        LogError("our RC is invalid? closing session to", remoteAddr);
-        Close();
-        return;
-      }
-      msg.N.Randomize();
-      msg.P = DefaultLinkSessionLifetime;
-      if(!msg.Sign(parent->Sign))
-      {
-        LogError("failed to sign LIM for outbound handshake to ", remoteAddr);
-        Close();
-        return;
-      }
-      // encode
-      if(!msg.BEncode(&buf))
-      {
-        LogError("failed to encode LIM for handshake to ", remoteAddr);
-        Close();
-        return;
-      }
-      // rewind
-      buf.sz  = buf.cur - buf.base;
-      buf.cur = buf.base;
-      // send
-      if(!SendMessageBuffer(buf, nullptr))
-      {
-        LogError("failed to send handshake to ", remoteAddr);
-        Close();
-        return;
-      }
-
-      if(!DoClientKeyExchange(txKey, msg.N, remoteTransportPubKey,
-                              parent->RouterEncryptionSecret()))
-      {
-        LogError("failed to mix keys for outbound session to ", remoteAddr);
-        Close();
-        return;
-      }
-    }
-
-    Session::~Session()
-    {
-      if(sock)
-      {
-        utp_set_userdata(sock, nullptr);
-        sock = nullptr;
-      }
-    }
-
-    bool
-    Session::EncryptThenHash(const byte_t* ptr, uint32_t msgid, uint16_t length,
-                             uint16_t remaining)
-
-    {
-      auto& msg = sendq.back();
-      msg.vecs.emplace_back();
-      auto& vec = msg.vecs.back();
-      msg.fragments.emplace_back();
-      auto& buf    = msg.fragments.back();
-      vec.iov_base = buf.data();
-      vec.iov_len  = FragmentBufferSize;
-      buf.Randomize();
-      byte_t* noncePtr = buf.data() + FragmentHashSize;
-      byte_t* body     = noncePtr + FragmentNonceSize;
-      byte_t* base     = body;
-      AlignedBuffer< 24 > A(base);
-      // skip inner nonce
-      body += A.size();
-      // put msgid
-      htobe32buf(body, msgid);
-      body += sizeof(uint32_t);
-      // put length
-      htobe16buf(body, length);
-      body += sizeof(uint16_t);
-      // put remaining
-      htobe16buf(body, remaining);
-      body += sizeof(uint16_t);
-      // put body
-      memcpy(body, ptr, length);
-
-      llarp_buffer_t payload(base, base,
-                             FragmentBufferSize - FragmentOverheadSize);
-
-      TunnelNonce nonce(noncePtr);
-
-      // encrypt
-      if(!CryptoManager::instance()->xchacha20(payload, txKey, nonce))
-        return false;
-
-      payload.base = noncePtr;
-      payload.cur  = payload.base;
-      payload.sz   = FragmentBufferSize - FragmentHashSize;
-      // key'd hash
-      if(!CryptoManager::instance()->hmac(buf.data(), payload, txKey))
-        return false;
-      return MutateKey(txKey, A);
-    }
-
-    void
-    Session::EnterState(State st)
-    {
-      state = st;
-      if(st != eClose)
-      {
-        Alive();
-      }
-      if(st == eSessionReady)
-      {
-        parent->MapAddr(remoteRC.pubkey.as_array(), this);
-        if(!parent->SessionEstablished(this))
-          Close();
-      }
-    }
-
-    util::StatusObject
-    Session::ExtractStatus() const
-    {
-      return {{"client", !remoteRC.IsPublicRouter()},
-              {"sendBacklog", uint64_t(SendQueueBacklog())},
-              {"tx", m_TXRate},
-              {"rx", m_RXRate},
-              {"remoteAddr", remoteAddr.ToString()},
-              {"pubkey", remoteRC.pubkey.ToHex()}};
-    }
-
-    bool
-    Session::GotSessionRenegotiate(const LinkIntroMessage* msg)
-    {
-      // check with parent and possibly process and store new rc
-      if(!parent->SessionRenegotiate(msg->rc, remoteRC))
-      {
-        // failed to renegotiate
-        Close();
-        return false;
-      }
-      // set remote rc
-      remoteRC = msg->rc;
-      // recalculate rx key
-      return DoServerKeyExchange(rxKey, msg->N, remoteRC.enckey,
-                                 parent->RouterEncryptionSecret());
-    }
-
-    bool
-    Session::RenegotiateSession()
-    {
-      LinkIntroMessage lim;
-      lim.rc = parent->GetOurRC();
-      lim.N.Randomize();
-      lim.P = 60 * 1000 * 10;
-      if(!lim.Sign(parent->Sign))
-        return false;
-
-      std::array< byte_t, LinkIntroMessage::MaxSize > tmp;
-      llarp_buffer_t buf(tmp);
-      if(!lim.BEncode(&buf))
-        return false;
-      // rewind and resize buffer
-      buf.sz  = buf.cur - buf.base;
-      buf.cur = buf.base;
-      // send message
-      if(!SendMessageBuffer(buf, nullptr))
-        return false;
-      // regen our tx Key
-      return DoClientKeyExchange(txKey, lim.N, remoteRC.enckey,
-                                 parent->RouterEncryptionSecret());
-    }
-
-    bool
-    Session::VerifyThenDecrypt(const byte_t* ptr)
-    {
-      LogDebug("verify then decrypt ", remoteAddr);
-      ShortHash digest;
-
-      llarp_buffer_t hbuf(ptr + FragmentHashSize,
-                          FragmentBufferSize - FragmentHashSize);
-      if(!CryptoManager::instance()->hmac(digest.data(), hbuf, rxKey))
-      {
-        LogError("keyed hash failed");
-        return false;
-      }
-      const ShortHash expected(ptr);
-      if(expected != digest)
-      {
-        LogError("Message Integrity Failed: got ", digest, " from ", remoteAddr,
-                 " instead of ", expected);
-        Close();
-        return false;
-      }
-
-      llarp_buffer_t in(ptr + FragmentOverheadSize,
-                        FragmentBufferSize - FragmentOverheadSize);
-
-      llarp_buffer_t out(rxFragBody);
-
-      // decrypt
-      if(!CryptoManager::instance()->xchacha20_alt(out, in, rxKey,
-                                                   ptr + FragmentHashSize))
-      {
-        LogError("failed to decrypt message from ", remoteAddr);
-        return false;
-      }
-      // get inner nonce
-      AlignedBuffer< 24 > A(out.base);
-      // advance buffer
-      out.cur += A.size();
-      // read msgid
-      uint32_t msgid;
-      if(!out.read_uint32(msgid))
-      {
-        LogError("failed to read msgid");
-        return false;
-      }
-      // read length and remaining
-      uint16_t length, remaining;
-      if(!(out.read_uint16(length) && out.read_uint16(remaining)))
-      {
-        LogError("failed to read the rest of the header");
-        return false;
-      }
-      if(length > (out.sz - (out.cur - out.base)))
-      {
-        // too big length
-        LogError("fragment body too big");
-        return false;
-      }
-      if(msgid < m_NextRXMsgID)
-        return false;
-      m_NextRXMsgID = msgid;
-
-      // get message
-      if(m_RecvMsgs.find(msgid) == m_RecvMsgs.end())
-      {
-        m_RecvMsgs.emplace(msgid, InboundMessage());
-      }
-
-      auto itr = m_RecvMsgs.find(msgid);
-      // add message activity
-      itr->second.lastActive = parent->Now();
-      // append data
-      if(!itr->second.AppendData(out.cur, length))
-      {
-        LogError("inbound buffer is full");
-        m_RecvMsgs.erase(itr);
-        return false;  // not enough room
-      }
-      // mutate key
-      if(!MutateKey(rxKey, A))
-      {
-        LogError("failed to mutate rx key");
-        return false;
-      }
-
-      if(remaining == 0)
-      {
-        ManagedBuffer buf{itr->second.buffer};
-        // resize
-        buf.underlying.sz = buf.underlying.cur - buf.underlying.base;
-        // rewind
-        buf.underlying.cur = buf.underlying.base;
-        // process buffer
-        LogDebug("got message ", msgid, " from ", remoteAddr);
-        parent->HandleMessage(this, buf.underlying);
-        m_RecvMsgs.erase(itr);
-      }
-      return true;
-    }
-
-    void
-    Session::Close()
-    {
-      if(state != eClose)
-      {
-        if(sock)
-        {
-          if(state == eLinkEstablished || state == eSessionReady)
-          {
-            // only call shutdown when we are actually connected
-            utp_shutdown(sock, SHUT_RDWR);
-          }
-          utp_close(sock);
-          utp_set_userdata(sock, nullptr);
-          sock = nullptr;
-          LogDebug("utp_close ", remoteAddr);
-          if(remoteRC.IsPublicRouter())
-          {
-            metrics::integerTick("utp.session.close", "to", 1, "id",
-                                 RouterID(remoteRC.pubkey).ToString());
-          }
-          // discard sendq
-          // TODO: retry on another session ?
-          while(sendq.size())
-          {
-            sendq.front().Dropped();
-            sendq.pop_front();
-          }
-        }
-      }
-      EnterState(eClose);
-    }
-
-    void
-    Session::Alive()
-    {
-      lastActive = parent->Now();
-    }
-
-    InboundSession::InboundSession(LinkLayer* p, utp_socket* s,
-                                   const Addr& addr)
-        : Session(p)
-    {
-      sock         = s;
-      remoteAddr   = addr;
-      RouterID rid = p->GetOurRC().pubkey;
-      CryptoManager::instance()->shorthash(rxKey, llarp_buffer_t(rid));
-      remoteRC.Clear();
-
-      ABSL_ATTRIBUTE_UNUSED void* res = utp_set_userdata(sock, this);
-      assert(res == this);
-      assert(s == sock);
-      GotLIM = util::memFn(&InboundSession::InboundLIM, this);
-    }
-
-    bool
-    InboundSession::InboundLIM(const LinkIntroMessage* msg)
-    {
-      if(gotLIM && remoteRC.pubkey != msg->rc.pubkey)
-      {
-        Close();
-        return false;
-      }
-      if(!gotLIM)
-      {
-        remoteRC = msg->rc;
-        CryptoManager::instance()->shorthash(txKey,
-                                             llarp_buffer_t(remoteRC.pubkey));
-
-        if(!DoServerKeyExchange(rxKey, msg->N, remoteRC.enckey,
-                                parent->TransportSecretKey()))
-          return false;
-
-        std::array< byte_t, LinkIntroMessage::MaxSize > tmp;
-        llarp_buffer_t buf(tmp);
-        LinkIntroMessage replymsg;
-        replymsg.rc = parent->GetOurRC();
-        if(!replymsg.rc.Verify(parent->Now()))
-        {
-          LogError("our RC is invalid? closing session to", remoteAddr);
-          Close();
-          return false;
-        }
-        replymsg.N.Randomize();
-        replymsg.P = DefaultLinkSessionLifetime;
-        if(!replymsg.Sign(parent->Sign))
-        {
-          LogError("failed to sign LIM for inbound handshake from ",
-                   remoteAddr);
-          Close();
-          return false;
-        }
-        // encode
-        if(!replymsg.BEncode(&buf))
-        {
-          LogError("failed to encode LIM for handshake from ", remoteAddr);
-          Close();
-          return false;
-        }
-        // rewind
-        buf.sz  = buf.cur - buf.base;
-        buf.cur = buf.base;
-        // send
-        if(!SendMessageBuffer(buf, nullptr))
-        {
-          LogError("failed to repl to handshake from ", remoteAddr);
-          Close();
-          return false;
-        }
-        if(!DoClientKeyExchange(txKey, replymsg.N, remoteRC.enckey,
-                                parent->RouterEncryptionSecret()))
-
-          return false;
-        LogDebug("Sent reply LIM");
-        gotLIM = true;
-        EnterState(eSessionReady);
-        /// future LIM are used for session renegotiation
-        GotLIM = util::memFn(&Session::GotSessionRenegotiate, this);
-      }
-      return true;
-    }
-
-    OutboundSession::OutboundSession(LinkLayer* p, utp_socket* s,
-                                     const RouterContact& rc,
-                                     const AddressInfo& addr)
-        : Session(p)
-    {
-      remoteTransportPubKey = addr.pubkey;
-      remoteRC              = rc;
-      sock                  = s;
-      remoteAddr            = addr;
-
-      RouterID rid = remoteRC.pubkey;
-      CryptoManager::instance()->shorthash(txKey, llarp_buffer_t(rid));
-      rid = p->GetOurRC().pubkey;
-      CryptoManager::instance()->shorthash(rxKey, llarp_buffer_t(rid));
-
-      ABSL_ATTRIBUTE_UNUSED void* res = utp_set_userdata(sock, this);
-      assert(res == this);
-      assert(s == sock);
-
-      GotLIM = util::memFn(&OutboundSession::OutboundLIM, this);
-    }
-
-    void
-    OutboundSession::Start()
-    {
-      utp_connect(sock, remoteAddr, remoteAddr.SockLen());
-      EnterState(eConnecting);
-    }
-
-    bool
-    OutboundSession::OutboundLIM(const LinkIntroMessage* msg)
-    {
-      if(gotLIM && remoteRC.pubkey != msg->rc.pubkey)
-      {
-        return false;
-      }
-      remoteRC = msg->rc;
-      gotLIM   = true;
-
-      if(!DoServerKeyExchange(rxKey, msg->N, remoteRC.enckey,
-                              parent->RouterEncryptionSecret()))
-      {
-        Close();
-        return false;
-      }
-      /// future LIM are used for session renegotiation
-      GotLIM = util::memFn(&Session::GotSessionRenegotiate, this);
-      EnterState(eSessionReady);
-      return true;
-    }
-  }  // namespace utp
-}  // namespace llarp
diff --git a/llarp/utp/session.hpp b/llarp/utp/session.hpp
deleted file mode 100644
index a275a64db..000000000
--- a/llarp/utp/session.hpp
+++ /dev/null
@@ -1,301 +0,0 @@
-#ifndef LLARP_UTP_SESSION_HPP
-#define LLARP_UTP_SESSION_HPP
-
-#include <crypto/crypto.hpp>
-#include <link/session.hpp>
-#include <utility>
-#include <utp/inbound_message.hpp>
-#include <deque>
-
-#include <utp.h>
-
-namespace llarp
-{
-  namespace utp
-  {
-    struct LinkLayer;
-
-    struct Session : public ILinkSession,
-                     public std::enable_shared_from_this< Session >
-    {
-      /// remote router's rc
-      RouterContact remoteRC;
-      /// underlying socket
-      utp_socket* sock;
-      /// link layer parent
-      ILinkLayer* parent;
-      /// did we get a LIM from the remote yet?
-      bool gotLIM;
-      /// remote router's transport pubkey
-      PubKey remoteTransportPubKey;
-      /// remote router's transport ip
-      Addr remoteAddr;
-      /// rx session key
-      SharedSecret rxKey;
-      /// tx session key
-      SharedSecret txKey;
-      /// timestamp last active
-      llarp_time_t lastActive = 0;
-      /// timestamp last send success
-      llarp_time_t lastSend = 0;
-      /// session timeout (60s)
-      const static llarp_time_t sessionTimeout = DefaultLinkSessionLifetime;
-
-      struct OutboundMessage
-      {
-        OutboundMessage(uint32_t id, CompletionHandler func)
-            : msgid{id}, completed{std::move(func)}
-        {
-        }
-
-        const uint32_t msgid;
-        std::deque< utp_iovec > vecs;
-        std::deque< FragmentBuffer > fragments;
-        CompletionHandler completed;
-
-        void
-        Dropped()
-        {
-          if(completed)
-          {
-            completed(DeliveryStatus::eDeliveryDropped);
-            completed = nullptr;
-          }
-        }
-
-        void
-        Delivered()
-        {
-          if(completed)
-          {
-            completed(DeliveryStatus::eDeliverySuccess);
-            completed = nullptr;
-          }
-        }
-
-        bool
-        operator<(const OutboundMessage& other) const
-        {
-          return msgid < other.msgid;
-        }
-      };
-
-      /// current rx fragment buffer
-      FragmentBuffer recvBuf;
-      /// current offset in current rx fragment buffer
-      size_t recvBufOffset;
-      /// rx fragment message body
-      AlignedBuffer< FragmentBodySize > rxFragBody;
-
-      /// the next message id for tx
-      uint32_t m_NextTXMsgID;
-      /// the next message id for rx
-      uint32_t m_NextRXMsgID;
-
-      using SendQueue_t = std::deque< OutboundMessage >;
-      /// messages we are currently sending
-      SendQueue_t sendq;
-      /// messages we are recving right now
-      std::unordered_map< uint32_t, InboundMessage > m_RecvMsgs;
-      /// are we stalled or nah?
-      bool stalled = false;
-
-      uint64_t m_RXRate = 0;
-      uint64_t m_TXRate = 0;
-
-      llarp_time_t m_LastTick = 0;
-
-      /// mark session as alive
-      void
-      Alive();
-
-      util::StatusObject
-      ExtractStatus() const override;
-
-      ~Session() override = 0;
-
-      /// base
-      explicit Session(LinkLayer* p);
-
-      enum State
-      {
-        eInitial,          // initial state
-        eConnecting,       // we are connecting
-        eLinkEstablished,  // when utp connection is established
-        eCryptoHandshake,  // crypto handshake initiated
-        eSessionReady,     // session is ready
-        eClose             // utp connection is closed
-      };
-
-      /// session state, call EnterState(State) to set
-      State state;
-
-      /// hook for utp for when we have established a connection
-      void
-      OnLinkEstablished(ILinkLayer* p) override;
-
-      /// switch states
-      void
-      EnterState(State st);
-
-      /// handle LIM after handshake
-      bool
-      GotSessionRenegotiate(const LinkIntroMessage* msg);
-
-      /// re negotiate session with our new local RC
-      bool
-      RenegotiateSession() override;
-
-      bool
-      ShouldPing() const override;
-
-      /// pump tx queue
-      void
-      PumpWrite(size_t numMessages);
-
-      void
-      Pump() override;
-
-      std::shared_ptr< ILinkSession >
-      BorrowSelf() override
-      {
-        return shared_from_this();
-      }
-
-      bool
-      SendKeepAlive() override;
-
-      bool
-      IsEstablished() const override
-      {
-        return state == eSessionReady;
-      }
-
-      bool
-      TimedOut(llarp_time_t now) const override;
-
-      /// verify a fragment buffer and the decrypt it
-      /// buf is assumed to be FragmentBufferSize bytes long
-      bool
-      VerifyThenDecrypt(const byte_t* buf);
-
-      /// encrypt a fragment then hash the ciphertext
-      bool
-      EncryptThenHash(const byte_t* ptr, uint32_t msgid, uint16_t sz,
-                      uint16_t remain);
-
-      /// queue a fully formed message
-      bool
-      SendMessageBuffer(const llarp_buffer_t& buf,
-                        ILinkSession::CompletionHandler) override;
-
-      /// prune expired inbound messages
-      void
-      PruneInboundMessages(llarp_time_t now);
-
-      /// do low level connect
-      void
-      Connect();
-
-      /// handle outbound connection made
-      void
-      OutboundLinkEstablished(LinkLayer* p);
-
-      // send first message
-      void
-      OutboundHandshake();
-
-      // do key exchange for handshake
-      template < bool (Crypto::*dh_func)(SharedSecret&, const PubKey&,
-                                         const SecretKey&, const TunnelNonce&) >
-      bool
-      DoKeyExchange(SharedSecret& K, const KeyExchangeNonce& n,
-                    const PubKey& other, const SecretKey& secret);
-
-      bool
-      DoClientKeyExchange(SharedSecret& K, const KeyExchangeNonce& n,
-                          const PubKey& other, const SecretKey& secret)
-      {
-        return DoKeyExchange< &Crypto::transport_dh_client >(K, n, other,
-                                                             secret);
-      }
-
-      bool
-      DoServerKeyExchange(SharedSecret& K, const KeyExchangeNonce& n,
-                          const PubKey& other, const SecretKey& secret)
-      {
-        return DoKeyExchange< &Crypto::transport_dh_server >(K, n, other,
-                                                             secret);
-      }
-
-      /// does K = HS(K + A)
-      bool
-      MutateKey(SharedSecret& K, const AlignedBuffer< 24 >& A);
-
-      void
-      Tick(llarp_time_t now) override;
-
-      /// close session
-      void
-      Close() override;
-
-      /// low level read
-      bool
-      Recv(const byte_t* buf, size_t sz);
-
-      /// get remote identity pubkey
-      PubKey
-      GetPubKey() const override;
-
-      /// get remote address
-      Addr
-      GetRemoteEndpoint() const override;
-
-      RouterContact
-      GetRemoteRC() const override
-      {
-        return remoteRC;
-      }
-
-      /// get parent link
-      ILinkLayer*
-      GetLinkLayer() const override;
-
-      void
-      MarkEstablished();
-
-      size_t
-      SendQueueBacklog() const override
-      {
-        return sendq.size();
-      }
-    };
-
-    struct InboundSession final : public Session
-    {
-      InboundSession(LinkLayer* p, utp_socket* s, const Addr& addr);
-
-      bool
-      InboundLIM(const LinkIntroMessage* msg);
-
-      void
-      Start() override
-      {
-      }
-    };
-
-    struct OutboundSession final : public Session
-    {
-      OutboundSession(LinkLayer* p, utp_socket* s, const RouterContact& rc,
-                      const AddressInfo& addr);
-
-      bool
-      OutboundLIM(const LinkIntroMessage* msg);
-
-      void
-      Start() override;
-    };
-  }  // namespace utp
-}  // namespace llarp
-
-#endif
diff --git a/llarp/utp/utp.cpp b/llarp/utp/utp.cpp
deleted file mode 100644
index ab11c79e8..000000000
--- a/llarp/utp/utp.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-#include <utp/utp.hpp>
-
-#include <router/abstractrouter.hpp>
-#include <util/meta/memfn.hpp>
-#include <utp/linklayer.hpp>
-
-namespace llarp
-{
-  namespace utp
-  {
-    LinkLayer_ptr
-    NewOutboundLink(const SecretKey& routerEncSecret, GetRCFunc getrc,
-                    LinkMessageHandler h, SignBufferFunc sign,
-                    SessionEstablishedHandler est,
-                    SessionRenegotiateHandler reneg, TimeoutHandler timeout,
-                    SessionClosedHandler closed)
-    {
-      return std::make_shared< LinkLayer >(routerEncSecret, getrc, h, sign, est,
-                                           reneg, timeout, closed, false);
-    }
-
-    LinkLayer_ptr
-    NewInboundLink(const SecretKey& routerEncSecret, GetRCFunc getrc,
-                   LinkMessageHandler h, SignBufferFunc sign,
-                   SessionEstablishedHandler est,
-                   SessionRenegotiateHandler reneg, TimeoutHandler timeout,
-                   SessionClosedHandler closed)
-    {
-      return std::make_shared< LinkLayer >(routerEncSecret, getrc, h, sign, est,
-                                           reneg, timeout, closed, true);
-    }
-
-  }  // namespace utp
-
-}  // namespace llarp
diff --git a/llarp/utp/utp.hpp b/llarp/utp/utp.hpp
deleted file mode 100644
index d368065e2..000000000
--- a/llarp/utp/utp.hpp
+++ /dev/null
@@ -1,28 +0,0 @@
-#ifndef LLARP_UTP_UTP_HPP
-#define LLARP_UTP_UTP_HPP
-
-#include <memory>
-#include <link/server.hpp>
-
-namespace llarp
-{
-  namespace utp
-  {
-    LinkLayer_ptr
-    NewInboundLink(const SecretKey& routerEncSecret, GetRCFunc getrc,
-                   LinkMessageHandler h, SignBufferFunc sign,
-                   SessionEstablishedHandler est,
-                   SessionRenegotiateHandler reneg, TimeoutHandler timeout,
-                   SessionClosedHandler closed);
-    LinkLayer_ptr
-    NewOutboundLink(const SecretKey& routerEncSecret, GetRCFunc getrc,
-                    LinkMessageHandler h, SignBufferFunc sign,
-                    SessionEstablishedHandler est,
-                    SessionRenegotiateHandler reneg, TimeoutHandler timeout,
-                    SessionClosedHandler closed);
-    /// shim
-    const auto NewServer = NewInboundLink;
-  }  // namespace utp
-}  // namespace llarp
-
-#endif
diff --git a/test/link/test_llarp_link.cpp b/test/link/test_llarp_link.cpp
index c8ac1955f..d93ce3dbe 100644
--- a/test/link/test_llarp_link.cpp
+++ b/test/link/test_llarp_link.cpp
@@ -5,7 +5,7 @@
 #include <iwp/iwp.hpp>
 #include <messages/link_intro.hpp>
 #include <messages/discard.hpp>
-#include <utp/utp.hpp>
+
 
 
 #include <test_util.hpp>
@@ -102,13 +102,15 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
       if(link)
         link->Stop();
       if(worker)
+      {
+        worker->drain();
         worker->stop();
+      }
     }
 
     void
     TearDown()
     {
-      Stop();
       link.reset();
       worker.reset();
     }
@@ -131,6 +133,7 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
   void
   SetUp()
   {
+    llarp::SetLogLevel(eLogDebug);
     oldRCLifetime              = RouterContact::Lifetime;
     RouterContact::BlockBogons = false;
     RouterContact::Lifetime    = 500;
@@ -170,7 +173,11 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
   void
   Stop()
   {
-    llarp_ev_loop_stop(netLoop);
+    m_logic->queue_func([&]() {
+      Alice.Stop();
+      Bob.Stop();
+      llarp_ev_loop_stop(netLoop);
+    });
   }
 
   bool
@@ -225,14 +232,12 @@ TEST_F(LinkLayerTest, TestIWP)
 
   auto sendDiscardMessage = [](ILinkSession* s) -> bool {
     // send discard message in reply to complete unit test
-    std::array< byte_t, 32 > tmp;
+    std::vector< byte_t> tmp(32);
     llarp_buffer_t otherBuf(tmp);
     DiscardMessage discard;
     if(!discard.BEncode(&otherBuf))
       return false;
-    otherBuf.sz = otherBuf.cur - otherBuf.base;
-    otherBuf.cur = otherBuf.base;
-    return s->SendMessageBuffer(otherBuf, nullptr);
+    return s->SendMessageBuffer(std::move(tmp), nullptr);
   };
 
   Bob.link = iwp::NewInboundLink(
@@ -268,194 +273,9 @@ TEST_F(LinkLayerTest, TestIWP)
   ASSERT_TRUE(Alice.Start(m_logic, netLoop, AlicePort));
   ASSERT_TRUE(Bob.Start(m_logic, netLoop, BobPort));
 
-  ASSERT_TRUE(Alice.link->TryEstablishTo(Bob.GetRC()));
+  m_logic->queue_func([&]() { ASSERT_TRUE(Alice.link->TryEstablishTo(Bob.GetRC())); });
 
   RunMainloop();
   ASSERT_TRUE(Bob.gotLIM);
 #endif
 };
-
-TEST_F(LinkLayerTest, TestUTPAliceRenegWithBob)
-{
-#ifdef WIN32
-  GTEST_SKIP();
-#else
-  Alice.link = utp::NewServer(
-      Alice.encryptionKey,
-      [&]() -> const RouterContact& { return Alice.GetRC(); },
-      [&](ILinkSession* s, const llarp_buffer_t& buf) -> bool {
-        if(Alice.gotLIM)
-        {
-          Alice.Regen();
-          return s->RenegotiateSession();
-        }
-        else
-        {
-          LinkIntroMessage msg;
-          ManagedBuffer copy{buf};
-          if(!msg.BDecode(&copy.underlying))
-            return false;
-          if(!s->GotLIM(&msg))
-            return false;
-          Alice.gotLIM = true;
-          return true;
-        }
-      },
-      [&](Signature& sig, const llarp_buffer_t& buf) -> bool {
-        return m_crypto.sign(sig, Alice.signingKey, buf);
-      },
-      [&](ILinkSession* s) -> bool {
-        const auto rc = s->GetRemoteRC();
-        return rc.pubkey == Bob.GetRC().pubkey;
-      },
-      [&](RouterContact, RouterContact) -> bool { return true; },
-     
-      [&](ILinkSession* session) {
-        ASSERT_FALSE(session->IsEstablished());
-        Stop();
-      },
-      [&](RouterID router) { ASSERT_EQ(router, Bob.GetRouterID()); });
-
-  auto sendDiscardMessage = [](ILinkSession* s) -> bool {
-    // send discard message in reply to complete unit test
-    std::array< byte_t, 32 > tmp;
-    llarp_buffer_t otherBuf(tmp);
-    DiscardMessage discard;
-    if(!discard.BEncode(&otherBuf))
-      return false;
-    otherBuf.sz = otherBuf.cur - otherBuf.base;
-    otherBuf.cur = otherBuf.base;
-    return s->SendMessageBuffer(otherBuf, nullptr);
-  };
-
-  Bob.link = utp::NewServer(
-      Bob.encryptionKey, [&]() -> const RouterContact& { return Bob.GetRC(); },
-      [&](ILinkSession* s, const llarp_buffer_t& buf) -> bool {
-        LinkIntroMessage msg;
-        ManagedBuffer copy{buf};
-        if(!msg.BDecode(&copy.underlying))
-          return false;
-        if(!s->GotLIM(&msg))
-          return false;
-        Bob.gotLIM = true;
-        return sendDiscardMessage(s);
-      },
-
-      [&](Signature& sig, const llarp_buffer_t& buf) -> bool {
-        return m_crypto.sign(sig, Bob.signingKey, buf);
-      },
-      [&](ILinkSession* s) -> bool {
-        if(s->GetRemoteRC().pubkey != Alice.GetRC().pubkey)
-          return false;
-        LogInfo("bob established with alice");
-        return Bob.link->VisitSessionByPubkey(Alice.GetRC().pubkey.as_array(),
-                                              sendDiscardMessage);
-      },
-      [&](RouterContact newrc, RouterContact oldrc) -> bool {
-        success = newrc.pubkey == oldrc.pubkey;
-        return true;
-      },
-      [&](ILinkSession* session) { ASSERT_FALSE(session->IsEstablished()); },
-      [&](RouterID router) { ASSERT_EQ(router, Alice.GetRouterID()); });
-
-  ASSERT_TRUE(Alice.Start(m_logic, netLoop, AlicePort));
-  ASSERT_TRUE(Bob.Start(m_logic, netLoop, BobPort));
-
-  ASSERT_TRUE(Alice.link->TryEstablishTo(Bob.GetRC()));
-
-  RunMainloop();
-  ASSERT_TRUE(Bob.gotLIM);
-  ASSERT_TRUE(success);
-#endif
-}
-
-TEST_F(LinkLayerTest, TestUTPAliceConnectToBob)
-{
-#ifdef WIN32
-  GTEST_SKIP();
-#else
-  Alice.link = utp::NewServer(
-      Alice.encryptionKey,
-      [&]() -> const RouterContact& { return Alice.GetRC(); },
-      [&](ILinkSession* s, const llarp_buffer_t& buf) -> bool {
-        LinkIntroMessage lim;
-        llarp_buffer_t copy(buf.base, buf.sz);
-        if(lim.BDecode(&copy))
-        {
-          if(s->GotLIM(&lim))
-          {
-            Alice.gotLIM = true;
-            return true;
-          }
-          return false;
-        }
-        return AliceGotMessage(buf);
-      },
-            [&](Signature& sig, const llarp_buffer_t& buf) -> bool {
-        return m_crypto.sign(sig, Alice.signingKey, buf);
-      },
-      [&](ILinkSession* s) -> bool {
-        if(s->GetRemoteRC().pubkey != Bob.GetRC().pubkey)
-          return false;
-        LogInfo("alice established with bob");
-        return true;
-      },
-      [&](RouterContact, RouterContact) -> bool { return true; },
-
-      [&](ILinkSession* session) {
-        ASSERT_FALSE(session->IsEstablished());
-        Stop();
-      },
-      [&](RouterID router) { ASSERT_EQ(router, Bob.GetRouterID()); });
-
-  Bob.link = utp::NewServer(
-      Bob.encryptionKey, [&]() -> const RouterContact& { return Bob.GetRC(); },
-      [&](ILinkSession* s, const llarp_buffer_t& buf) -> bool {
-        LinkIntroMessage lim;
-        llarp_buffer_t copy(buf.base, buf.sz);
-        if(lim.BDecode(&copy))
-        {
-          if(s->GotLIM(&lim))
-          {
-            Bob.gotLIM = true;
-            return true;
-          }
-          return false;
-        }
-        return true;
-      },   
-      [&](Signature& sig, const llarp_buffer_t& buf) -> bool {
-        return m_crypto.sign(sig, Bob.signingKey, buf);
-      },
-      [&](ILinkSession* s) -> bool {
-        if(s->GetRemoteRC().pubkey != Alice.GetRC().pubkey)
-          return false;
-        LogInfo("bob established with alice");
-        m_logic->queue_job({s, [](void* u) {
-                              ILinkSession* self =
-                                  static_cast< ILinkSession* >(u);
-                              std::array< byte_t, 32 > tmp;
-                              llarp_buffer_t otherBuf(tmp);
-                              DiscardMessage discard;
-                              if(!discard.BEncode(&otherBuf))
-                                return;
-                              otherBuf.sz = otherBuf.cur - otherBuf.base;
-                              otherBuf.cur = otherBuf.base;
-                              self->SendMessageBuffer(otherBuf, nullptr);
-                            }});
-        return true;
-      },
-      [&](RouterContact, RouterContact) -> bool { return true; },
-      [&](ILinkSession* session) { ASSERT_FALSE(session->IsEstablished()); },
-      [&](RouterID router) { ASSERT_EQ(router, Alice.GetRouterID()); });
-
-  ASSERT_TRUE(Alice.Start(m_logic, netLoop, AlicePort));
-  ASSERT_TRUE(Bob.Start(m_logic, netLoop, BobPort));
-
-  ASSERT_TRUE(Alice.link->TryEstablishTo(Bob.GetRC()));
-
-  RunMainloop();
-  ASSERT_TRUE(Bob.gotLIM);
-  ASSERT_TRUE(success);
-#endif
-}
diff --git a/test/test_libabyss.cpp b/test/test_libabyss.cpp
index c67400e7b..d023a6f03 100644
--- a/test/test_libabyss.cpp
+++ b/test/test_libabyss.cpp
@@ -11,7 +11,6 @@
 struct AbyssTestBase : public ::testing::Test
 {
   llarp::sodium::CryptoLibSodium crypto;
-  llarp_threadpool* threadpool = nullptr;
   llarp_ev_loop_ptr loop       = nullptr;
   std::shared_ptr< llarp::Logic > logic;
   abyss::httpd::BaseReqHandler* server = nullptr;
@@ -49,7 +48,6 @@ struct AbyssTestBase : public ::testing::Test
   {
     loop       = llarp_make_ev_loop();
     logic      = std::make_shared< llarp::Logic >();
-    threadpool = logic->thread;
     sockaddr_in addr;
     addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
     addr.sin_port        = htons((llarp::randint() % 2000) + 2000);
@@ -83,7 +81,6 @@ struct AbyssTestBase : public ::testing::Test
   ~AbyssTestBase()
   {
     logic.reset();
-    llarp_free_threadpool(&threadpool);
     llarp::SetLogLevel(llarp::eLogInfo);
   }
 };

From 14c9ef15edef1607ec93f985736f22cd5b2e82cb Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Mon, 16 Sep 2019 06:21:12 -0400
Subject: [PATCH 17/27] try calling stuff in logic thread from event loop

---
 llarp/ev/ev.cpp              |  1 +
 llarp/ev/ev.hpp              |  3 ++
 llarp/ev/ev_libuv.cpp        | 59 ++++++++++++++++++++++++------------
 llarp/ev/ev_libuv.hpp        | 16 ++++++++++
 llarp/handlers/tun.cpp       |  5 +--
 llarp/iwp/linklayer.cpp      |  2 +-
 llarp/iwp/message_buffer.cpp |  6 +++-
 llarp/iwp/session.cpp        | 34 +++++++++++++--------
 llarp/iwp/session.hpp        |  5 ++-
 llarp/link/server.cpp        |  3 +-
 llarp/path/ihophandler.hpp   | 30 +++++++++++-------
 llarp/path/path.cpp          | 29 ++++++++++++------
 llarp/path/path.hpp          |  9 ++++--
 llarp/path/path_context.cpp  | 14 +++++++--
 llarp/path/path_context.hpp  |  5 ++-
 llarp/path/pathbuilder.cpp   |  4 +--
 llarp/path/pathset.cpp       |  9 ++++--
 llarp/path/pathset.hpp       |  5 ++-
 llarp/path/transit_hop.cpp   | 32 ++++++++++++-------
 llarp/path/transit_hop.hpp   |  9 ++++--
 llarp/router/router.cpp      |  7 ++---
 llarp/service/endpoint.cpp   |  1 +
 llarp/util/thread/logic.cpp  | 17 +++++++++--
 llarp/util/thread/logic.hpp  |  8 ++---
 24 files changed, 213 insertions(+), 100 deletions(-)

diff --git a/llarp/ev/ev.cpp b/llarp/ev/ev.cpp
index f2e979a64..da98c5f32 100644
--- a/llarp/ev/ev.cpp
+++ b/llarp/ev/ev.cpp
@@ -36,6 +36,7 @@ void
 llarp_ev_loop_run_single_process(llarp_ev_loop_ptr ev,
                                  std::shared_ptr< llarp::Logic > logic)
 {
+  ev->give_logic(logic);
   while(ev->running())
   {
     ev->update_time();
diff --git a/llarp/ev/ev.hpp b/llarp/ev/ev.hpp
index dbbd16440..5e18fe699 100644
--- a/llarp/ev/ev.hpp
+++ b/llarp/ev/ev.hpp
@@ -783,6 +783,9 @@ struct llarp_ev_loop
     return false;
   }
 
+  /// give this event loop a logic thread for calling
+  virtual void give_logic(std::shared_ptr< llarp::Logic >) = 0;
+
   /// register event listener
   virtual bool
   add_ev(llarp::ev_io* ev, bool write) = 0;
diff --git a/llarp/ev/ev_libuv.cpp b/llarp/ev/ev_libuv.cpp
index a4e0b4735..a2f35200b 100644
--- a/llarp/ev/ev_libuv.cpp
+++ b/llarp/ev/ev_libuv.cpp
@@ -1,10 +1,19 @@
 #include <ev/ev_libuv.hpp>
 #include <net/net_addr.hpp>
+#include <util/thread/logic.hpp>
 
 #include <cstring>
 
 namespace libuv
 {
+  /// call a function in logic thread via a handle
+  template < typename Handle, typename Func >
+  void
+  Call(Handle* h, Func&& f)
+  {
+    static_cast< Loop* >(h->loop->data)->Call(f);
+  }
+
   struct glue
   {
     virtual ~glue() = default;
@@ -65,8 +74,9 @@ namespace libuv
     static void
     OnOutboundConnect(uv_connect_t* c, int status)
     {
-      auto* self = static_cast< conn_glue* >(c->data);
-      self->HandleConnectResult(status);
+      conn_glue* self = static_cast< conn_glue* >(c->data);
+      Call(self->Stream(),
+           std::bind(&conn_glue::HandleConnectResult, self, status));
       c->data = nullptr;
     }
 
@@ -165,13 +175,14 @@ namespace libuv
     static void
     OnWritten(uv_write_t* req, int status)
     {
+      conn_glue* conn = static_cast< conn_glue* >(req->data);
       if(status)
       {
         llarp::LogError("write failed on tcp: ", uv_strerror(status));
-        static_cast< conn_glue* >(req->data)->Close();
+        conn->Close();
       }
       else
-        static_cast< conn_glue* >(req->data)->DrainOne();
+        Call(conn->Stream(), std::bind(&conn_glue::DrainOne, conn));
       delete req;
     }
 
@@ -195,7 +206,8 @@ namespace libuv
     static void
     OnClosed(uv_handle_t* h)
     {
-      static_cast< conn_glue* >(h->data)->HandleClosed();
+      conn_glue* conn = static_cast< conn_glue* >(h->data);
+      Call(h, std::bind(&conn_glue::HandleClosed, conn));
     }
 
     static void
@@ -251,7 +263,8 @@ namespace libuv
     {
       if(status == 0)
       {
-        static_cast< conn_glue* >(stream->data)->Accept();
+        conn_glue* conn = static_cast< conn_glue* >(stream->data);
+        Call(stream, std::bind(&conn_glue::Accept, conn));
       }
       else
       {
@@ -262,8 +275,8 @@ namespace libuv
     static void
     OnTick(uv_check_t* t)
     {
-      auto* conn = static_cast< conn_glue* >(t->data);
-      conn->Tick();
+      conn_glue* conn = static_cast< conn_glue* >(t->data);
+      Call(t, std::bind(&conn_glue::Tick, conn));
     }
 
     void
@@ -331,7 +344,8 @@ namespace libuv
     static void
     OnTick(uv_check_t* t)
     {
-      static_cast< ticker_glue* >(t->data)->func();
+      ticker_glue* ticker = static_cast< ticker_glue* >(t->data);
+      Call(&ticker->m_Ticker, [ticker]() { ticker->func(); });
     }
 
     bool
@@ -393,14 +407,14 @@ namespace libuv
         const size_t pktsz = sz;
         const llarp_buffer_t pkt{(const byte_t*)buf->base, pktsz};
         m_UDP->recvfrom(m_UDP, fromaddr, ManagedBuffer{pkt});
-        gotpkts = true;
       }
     }
 
     static void
     OnTick(uv_check_t* t)
     {
-      static_cast< udp_glue* >(t->data)->Tick();
+      udp_glue* udp = static_cast< udp_glue* >(t->data);
+      udp->Tick();
     }
 
     void
@@ -408,7 +422,6 @@ namespace libuv
     {
       if(m_UDP && m_UDP->tick)
         m_UDP->tick(m_UDP);
-      gotpkts = false;
     }
 
     static int
@@ -443,6 +456,7 @@ namespace libuv
       if(uv_fileno((const uv_handle_t*)&m_Handle, &m_UDP->fd))
         return false;
       m_UDP->sendto = &SendTo;
+      m_UDP->impl   = this;
       return true;
     }
 
@@ -452,8 +466,7 @@ namespace libuv
       auto* glue = static_cast< udp_glue* >(h->data);
       if(glue)
       {
-        h->data           = nullptr;
-        glue->m_UDP->impl = nullptr;
+        h->data = nullptr;
         delete glue;
       }
     }
@@ -461,6 +474,7 @@ namespace libuv
     void
     Close() override
     {
+      m_UDP->impl = nullptr;
       uv_check_stop(&m_Ticker);
       uv_close((uv_handle_t*)&m_Handle, &OnClosed);
     }
@@ -481,7 +495,7 @@ namespace libuv
     void
     Tick()
     {
-      m_Pipe->tick();
+      Call(&m_Handle, std::bind(&llarp_ev_pkt_pipe::tick, m_Pipe));
     }
 
     static void
@@ -520,7 +534,8 @@ namespace libuv
     static void
     OnTick(uv_check_t* h)
     {
-      static_cast< pipe_glue* >(h->data)->Tick();
+      pipe_glue* pipe = static_cast< pipe_glue* >(h->data);
+      Call(h, std::bind(&pipe_glue::Tick, pipe));
     }
 
     bool
@@ -561,7 +576,8 @@ namespace libuv
     static void
     OnTick(uv_check_t* timer)
     {
-      static_cast< tun_glue* >(timer->data)->Tick();
+      tun_glue* tun = static_cast< tun_glue* >(timer->data);
+      Call(timer, std::bind(&tun_glue::Tick, tun));
     }
 
     static void
@@ -601,8 +617,7 @@ namespace libuv
       auto* self = static_cast< tun_glue* >(h->data);
       if(self)
       {
-        self->m_Tun->impl = nullptr;
-        h->data           = nullptr;
+        h->data = nullptr;
         delete self;
       }
     }
@@ -610,6 +625,7 @@ namespace libuv
     void
     Close() override
     {
+      m_Tun->impl = nullptr;
       uv_check_stop(&m_Ticker);
       uv_close((uv_handle_t*)&m_Handle, &OnClosed);
     }
@@ -623,7 +639,8 @@ namespace libuv
     static bool
     WritePkt(llarp_tun_io* tun, const byte_t* pkt, size_t sz)
     {
-      return static_cast< tun_glue* >(tun->impl)->Write(pkt, sz);
+      tun_glue* glue = static_cast< tun_glue* >(tun->impl);
+      return glue && glue->Write(pkt, sz);
     }
 
     bool
@@ -670,6 +687,7 @@ namespace libuv
         return false;
       }
       m_Tun->writepkt = &WritePkt;
+      m_Tun->impl     = this;
       return true;
     }
   };
@@ -680,6 +698,7 @@ namespace libuv
     m_Impl.reset(uv_loop_new());
     if(uv_loop_init(m_Impl.get()) == -1)
       return false;
+    m_Impl->data = this;
     uv_loop_configure(m_Impl.get(), UV_LOOP_BLOCK_SIGNAL, SIGPIPE);
     m_TickTimer.data = this;
     m_Run.store(true);
diff --git a/llarp/ev/ev_libuv.hpp b/llarp/ev/ev_libuv.hpp
index ab19fec73..c04e6a370 100644
--- a/llarp/ev/ev_libuv.hpp
+++ b/llarp/ev/ev_libuv.hpp
@@ -5,6 +5,7 @@
 #include <uv.h>
 #include <vector>
 #include <functional>
+#include <util/thread/logic.hpp>
 
 namespace libuv
 {
@@ -88,6 +89,20 @@ namespace libuv
       return false;
     }
 
+    void
+    give_logic(std::shared_ptr< llarp::Logic > l) override
+    {
+      m_Logic = l;
+    }
+
+    /// call function in logic thread
+    template < typename F >
+    void
+    Call(F f)
+    {
+      m_Logic->queue_func(f);
+    }
+
    private:
     struct DestructLoop
     {
@@ -101,6 +116,7 @@ namespace libuv
     std::unique_ptr< uv_loop_t, DestructLoop > m_Impl;
     uv_timer_t m_TickTimer;
     std::atomic< bool > m_Run;
+    std::shared_ptr< llarp::Logic > m_Logic;
   };
 
 }  // namespace libuv
diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp
index 27fa27c53..57588e343 100644
--- a/llarp/handlers/tun.cpp
+++ b/llarp/handlers/tun.cpp
@@ -314,11 +314,7 @@ namespace llarp
       RouterLogic()->queue_func([=] {
         self->m_ExitMap.ForEachValue(
             [](const auto &exit) { exit->FlushUpstream(); });
-        self->Router()->PumpLL();
-      });
-      RouterLogic()->queue_func([=]() {
         self->Pump(self->Now());
-        self->Router()->PumpLL();
       });
     }
 
@@ -788,6 +784,7 @@ namespace llarp
         }
         llarp::LogWarn(Name(), " did not flush packets");
       });
+      SendAllDownstream(Router());
     }
 
     bool
diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index 51bba1958..8cee5e636 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -28,7 +28,7 @@ namespace llarp
         auto itr = m_AuthedLinks.begin();
         while(itr != m_AuthedLinks.end())
         {
-          sessions.insert(itr->first);
+          sessions.emplace(itr->first);
           ++itr;
         }
       }
diff --git a/llarp/iwp/message_buffer.cpp b/llarp/iwp/message_buffer.cpp
index 33b3458c5..03b59a240 100644
--- a/llarp/iwp/message_buffer.cpp
+++ b/llarp/iwp/message_buffer.cpp
@@ -121,7 +121,11 @@ namespace llarp
                                llarp_time_t now)
     {
       if(idx + buf.sz > m_Data.size())
+      {
+        LogWarn("invalid fragment offset ", idx);
         return;
+      }
+
       auto *dst = m_Data.data() + idx;
       std::copy_n(buf.base, buf.sz, dst);
       m_Acks.set(idx / FragmentSize);
@@ -134,7 +138,7 @@ namespace llarp
     {
       auto acks = CreatePacket(Command::eACKS, 9, 0, 0);
       htobe64buf(acks.data() + 2 + PacketOverhead, m_MsgID);
-      acks[PacketOverhead + 8] = AcksBitmask();
+      acks[PacketOverhead + 10] = AcksBitmask();
       return acks;
     }
 
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index 967dc797d..5f612e3e9 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -198,19 +198,21 @@ namespace llarp
     Session::SendMACK()
     {
       // send multi acks
-      while(!m_SendMACKS.empty())
+      while(m_SendMACKS.size() > 0)
       {
-        const auto sz  = m_SendMACKS.size();
-        const auto max = Session::MaxACKSInMACK;
-        auto numAcks   = std::min(sz, max);
-        auto mack =
-            CreatePacket(Command::eMACK, numAcks * sizeof(uint64_t), 0, 0);
+        const auto sz            = m_SendMACKS.size();
+        const auto max           = Session::MaxACKSInMACK;
+        auto numAcks             = std::min(sz, max);
+        auto mack                = CreatePacket(Command::eMACK,
+                                 1 + (numAcks * sizeof(uint64_t)), 0, 0);
         mack[PacketOverhead + 2] = byte_t{numAcks};
         byte_t* ptr              = mack.data() + 3 + PacketOverhead;
+        LogDebug("send ", numAcks, " macks to ", m_RemoteAddr);
+        auto itr = m_SendMACKS.begin();
         while(numAcks > 0)
         {
-          htobe64buf(ptr, m_SendMACKS.back());
-          m_SendMACKS.pop_back();
+          htobe64buf(ptr, *itr);
+          itr = m_SendMACKS.erase(itr);
           numAcks--;
           ptr += sizeof(uint64_t);
         }
@@ -226,7 +228,6 @@ namespace llarp
       {
         if(ShouldPing())
           SendKeepAlive();
-        SendMACK();
         for(auto& item : m_RXMsgs)
         {
           if(item.second.ShouldSendACKS(now))
@@ -562,6 +563,7 @@ namespace llarp
     {
       for(auto& result : msgs)
       {
+        LogDebug("Command ", int(result[PacketOverhead + 1]));
         switch(result[PacketOverhead + 1])
         {
           case Command::eXMIT:
@@ -590,6 +592,7 @@ namespace llarp
                      " from ", m_RemoteAddr);
         }
       }
+      SendMACK();
     }
 
     void
@@ -601,16 +604,18 @@ namespace llarp
         return;
       }
       byte_t numAcks = data[2 + PacketOverhead];
-      if(data.size() < ((numAcks * sizeof(uint64_t)) + 3))
+      if(data.size() < 3 + PacketOverhead + (numAcks * sizeof(uint64_t)))
       {
         LogError("short mack from ", m_RemoteAddr);
         return;
       }
+      LogDebug("got ", int(numAcks), " mack from ", m_RemoteAddr);
       byte_t* ptr = data.data() + 3 + PacketOverhead;
       while(numAcks > 0)
       {
         uint64_t acked = bufbe64toh(ptr);
-        auto itr       = m_TXMsgs.find(acked);
+        LogDebug("mack containing txid=", acked, " from ", m_RemoteAddr);
+        auto itr = m_TXMsgs.find(acked);
         if(itr != m_TXMsgs.end())
         {
           itr->second.Completed();
@@ -697,6 +702,11 @@ namespace llarp
           htobe64buf(nack.data() + PacketOverhead + 2, rxid);
           EncryptAndSend(std::move(nack));
         }
+        else
+        {
+          LogDebug("replay hit for rxid=", rxid, " for ", m_RemoteAddr);
+          m_SendMACKS.emplace(rxid);
+        }
         return;
       }
 
@@ -714,7 +724,7 @@ namespace llarp
           const llarp_buffer_t buf(msg.m_Data.data(), msg.m_Size);
           m_Parent->HandleMessage(this, buf);
           m_ReplayFilter.emplace(itr->first, m_Parent->Now());
-          m_SendMACKS.emplace_back(itr->first);
+          m_SendMACKS.emplace(itr->first);
         }
         else
         {
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index 1ed39eb30..4f466ca5e 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -27,8 +27,7 @@ namespace llarp
       /// How often to acks RX messages
       static constexpr llarp_time_t ACKResendInterval = 250;
       /// How often to retransmit TX fragments
-      static constexpr llarp_time_t TXFlushInterval =
-          (ACKResendInterval * 3) / 2;
+      static constexpr llarp_time_t TXFlushInterval = ACKResendInterval * 2;
       /// How often we send a keepalive
       static constexpr llarp_time_t PingInterval = 2000;
       /// How long we wait for a session to die with no tx from them
@@ -165,7 +164,7 @@ namespace llarp
       /// maps rxid to time recieved
       std::unordered_map< uint64_t, llarp_time_t > m_ReplayFilter;
       /// list of rx messages to send in next set of multiacks
-      std::vector< uint64_t > m_SendMACKS;
+      std::set< uint64_t > m_SendMACKS;
 
       using CryptoQueue_t   = std::vector< Packet_t >;
       using CryptoQueue_ptr = std::shared_ptr< CryptoQueue_t >;
diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp
index 09d37dbc6..a8075894b 100644
--- a/llarp/link/server.cpp
+++ b/llarp/link/server.cpp
@@ -484,7 +484,8 @@ namespace llarp
     ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
     if(link->m_Recv == nullptr)
       return;
-    link->m_Recv->emplace_back(std::make_pair(*from, buf.underlying.sz));
+    link->m_Recv->emplace_back(
+        std::make_pair(Addr(*from), ILinkSession::Packet_t(buf.underlying.sz)));
     std::copy_n(buf.underlying.base, buf.underlying.sz,
                 link->m_Recv->back().second.begin());
   }
diff --git a/llarp/path/ihophandler.hpp b/llarp/path/ihophandler.hpp
index c0495e57c..35e4958e4 100644
--- a/llarp/path/ihophandler.hpp
+++ b/llarp/path/ihophandler.hpp
@@ -24,8 +24,9 @@ namespace llarp
   {
     struct IHopHandler
     {
-      using TrafficEvent_t = std::pair< std::vector< byte_t >, TunnelNonce >;
-      using TrafficQueue_t = std::vector< TrafficEvent_t >;
+      using TrafficEvent_t   = std::pair< std::vector< byte_t >, TunnelNonce >;
+      using TrafficQueue_t   = std::vector< TrafficEvent_t >;
+      using TrafficQueue_ptr = std::shared_ptr< TrafficQueue_t >;
 
       virtual ~IHopHandler() = default;
 
@@ -44,8 +45,10 @@ namespace llarp
       HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y,
                      AbstractRouter*)
       {
-        m_UpstreamQueue.emplace_back();
-        auto& pkt = m_UpstreamQueue.back();
+        if(m_UpstreamQueue == nullptr)
+          m_UpstreamQueue = std::make_shared< TrafficQueue_t >();
+        m_UpstreamQueue->emplace_back();
+        auto& pkt = m_UpstreamQueue->back();
         pkt.first.resize(X.sz);
         std::copy_n(X.base, X.sz, pkt.first.begin());
         pkt.second = Y;
@@ -57,8 +60,10 @@ namespace llarp
       HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y,
                        AbstractRouter*)
       {
-        m_DownstreamQueue.emplace_back();
-        auto& pkt = m_DownstreamQueue.back();
+        if(m_DownstreamQueue == nullptr)
+          m_DownstreamQueue = std::make_shared< TrafficQueue_t >();
+        m_DownstreamQueue->emplace_back();
+        auto& pkt = m_DownstreamQueue->back();
         pkt.first.resize(X.sz);
         std::copy_n(X.base, X.sz, pkt.first.begin());
         pkt.second = Y;
@@ -79,18 +84,21 @@ namespace llarp
         return m_SequenceNum++;
       }
       virtual void
-      FlushQueues(AbstractRouter* r) = 0;
+      FlushUpstream(AbstractRouter* r) = 0;
+
+      virtual void
+      FlushDownstream(AbstractRouter* r) = 0;
 
      protected:
       uint64_t m_SequenceNum = 0;
-      TrafficQueue_t m_UpstreamQueue;
-      TrafficQueue_t m_DownstreamQueue;
+      TrafficQueue_ptr m_UpstreamQueue;
+      TrafficQueue_ptr m_DownstreamQueue;
 
       virtual void
-      UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) = 0;
+      UpstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) = 0;
 
       virtual void
-      DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) = 0;
+      DownstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) = 0;
 
       virtual void
       HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp
index d24ca336b..930d6813b 100644
--- a/llarp/path/path.cpp
+++ b/llarp/path/path.cpp
@@ -377,11 +377,11 @@ namespace llarp
     }
 
     void
-    Path::UpstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
+    Path::UpstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r)
     {
-      std::vector< RelayUpstreamMessage > sendmsgs(msgs.size());
+      std::vector< RelayUpstreamMessage > sendmsgs(msgs->size());
       size_t idx = 0;
-      for(const auto& ev : msgs)
+      for(auto& ev : *msgs)
       {
         const llarp_buffer_t buf(ev.first);
         TunnelNonce n = ev.second;
@@ -402,16 +402,27 @@ namespace llarp
     }
 
     void
-    Path::FlushQueues(AbstractRouter* r)
+    Path::FlushUpstream(AbstractRouter* r)
     {
-      if(!m_UpstreamQueue.empty())
+      if(m_UpstreamQueue && !m_UpstreamQueue->empty())
+      {
         r->threadpool()->addJob(std::bind(&Path::UpstreamWork,
                                           shared_from_this(),
                                           std::move(m_UpstreamQueue), r));
-      if(!m_DownstreamQueue.empty())
+      }
+      m_UpstreamQueue = nullptr;
+    }
+
+    void
+    Path::FlushDownstream(AbstractRouter* r)
+    {
+      if(m_DownstreamQueue && !m_DownstreamQueue->empty())
+      {
         r->threadpool()->addJob(std::bind(&Path::DownstreamWork,
                                           shared_from_this(),
                                           std::move(m_DownstreamQueue), r));
+      }
+      m_DownstreamQueue = nullptr;
     }
 
     bool
@@ -438,11 +449,11 @@ namespace llarp
     }
 
     void
-    Path::DownstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
+    Path::DownstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r)
     {
-      std::vector< RelayDownstreamMessage > sendMsgs(msgs.size());
+      std::vector< RelayDownstreamMessage > sendMsgs(msgs->size());
       size_t idx = 0;
-      for(auto& ev : msgs)
+      for(auto& ev : *msgs)
       {
         const llarp_buffer_t buf(ev.first);
         sendMsgs[idx].Y = ev.second;
diff --git a/llarp/path/path.hpp b/llarp/path/path.hpp
index e1a9b45c7..72a1c7918 100644
--- a/llarp/path/path.hpp
+++ b/llarp/path/path.hpp
@@ -325,14 +325,17 @@ namespace llarp
       SendExitClose(const routing::CloseExitMessage& msg, AbstractRouter* r);
 
       void
-      FlushQueues(AbstractRouter* r) override;
+      FlushUpstream(AbstractRouter* r) override;
+
+      void
+      FlushDownstream(AbstractRouter* r) override;
 
      protected:
       void
-      UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+      UpstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override;
 
       void
-      DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+      DownstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override;
 
       void
       HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp
index c4608bbc7..f67ecce40 100644
--- a/llarp/path/path_context.cpp
+++ b/llarp/path/path_context.cpp
@@ -257,10 +257,18 @@ namespace llarp
     }
 
     void
-    PathContext::Pump()
+    PathContext::PumpUpstream()
     {
-      m_TransitPaths.ForEach([&](auto& ptr) { ptr->FlushQueues(m_Router); });
-      m_OurPaths.ForEach([&](auto& ptr) { ptr->FlushQueues(m_Router); });
+      m_TransitPaths.ForEach([&](auto& ptr) { ptr->FlushUpstream(m_Router); });
+      m_OurPaths.ForEach([&](auto& ptr) { ptr->SendAllUpstream(m_Router); });
+    }
+
+    void
+    PathContext::PumpDownstream()
+    {
+      m_TransitPaths.ForEach(
+          [&](auto& ptr) { ptr->FlushDownstream(m_Router); });
+      m_OurPaths.ForEach([&](auto& ptr) { ptr->SendAllDownstream(m_Router); });
     }
 
     void
diff --git a/llarp/path/path_context.hpp b/llarp/path/path_context.hpp
index 412080a15..8b7c48179 100644
--- a/llarp/path/path_context.hpp
+++ b/llarp/path/path_context.hpp
@@ -38,7 +38,10 @@ namespace llarp
       ExpirePaths(llarp_time_t now);
 
       void
-      Pump();
+      PumpUpstream();
+
+      void
+      PumpDownstream();
 
       void
       AllowTransit();
diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp
index 4ef91fb9b..b67b018d4 100644
--- a/llarp/path/pathbuilder.cpp
+++ b/llarp/path/pathbuilder.cpp
@@ -396,7 +396,7 @@ namespace llarp
       for(size_t idx = 0; idx < hops.size(); ++idx)
       {
         hops[idx].Clear();
-        size_t tries = 4;
+        size_t tries = 32;
         while(tries > 0 && !SelectHop(nodedb, exclude, hops[idx], idx, roles))
         {
           --tries;
@@ -406,7 +406,7 @@ namespace llarp
           LogWarn(Name(), " failed to select hop ", idx);
           return false;
         }
-        exclude.insert(hops[idx].pubkey);
+        exclude.emplace(hops[idx].pubkey);
       }
       return true;
     }
diff --git a/llarp/path/pathset.cpp b/llarp/path/pathset.cpp
index 79b3af3b4..2db9377cb 100644
--- a/llarp/path/pathset.cpp
+++ b/llarp/path/pathset.cpp
@@ -376,10 +376,15 @@ namespace llarp
     }
 
     void
-    PathSet::FlushQueues(AbstractRouter* r)
+    PathSet::SendAllUpstream(AbstractRouter* r)
+    {
+      ForEachPath([r](const Path_ptr& p) { p->FlushUpstream(r); });
+    }
 
+    void
+    PathSet::SendAllDownstream(AbstractRouter* r)
     {
-      ForEachPath([r](const Path_ptr& ptr) { ptr->FlushQueues(r); });
+      ForEachPath([r](const Path_ptr& p) { p->FlushDownstream(r); });
     }
 
   }  // namespace path
diff --git a/llarp/path/pathset.hpp b/llarp/path/pathset.hpp
index ed59cb532..b5beb5242 100644
--- a/llarp/path/pathset.hpp
+++ b/llarp/path/pathset.hpp
@@ -276,7 +276,10 @@ namespace llarp
       }
 
       void
-      FlushQueues(AbstractRouter* r);
+      SendAllUpstream(AbstractRouter* r);
+
+      void
+      SendAllDownstream(AbstractRouter* r);
 
       size_t numPaths;
 
diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp
index 858b8f1f1..4ccaf4d9b 100644
--- a/llarp/path/transit_hop.cpp
+++ b/llarp/path/transit_hop.cpp
@@ -115,11 +115,11 @@ namespace llarp
     }
 
     void
-    TransitHop::DownstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
+    TransitHop::DownstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r)
     {
-      std::vector< RelayDownstreamMessage > sendmsgs(msgs.size());
+      std::vector< RelayDownstreamMessage > sendmsgs(msgs->size());
       size_t idx = 0;
-      for(auto& ev : msgs)
+      for(auto& ev : *msgs)
       {
         const llarp_buffer_t buf(ev.first);
         auto& msg  = sendmsgs[idx];
@@ -137,11 +137,11 @@ namespace llarp
     }
 
     void
-    TransitHop::UpstreamWork(TrafficQueue_t msgs, AbstractRouter* r)
+    TransitHop::UpstreamWork(TrafficQueue_ptr msgs, AbstractRouter* r)
     {
-      std::vector< RelayUpstreamMessage > sendmsgs(msgs.size());
+      std::vector< RelayUpstreamMessage > sendmsgs(msgs->size());
       size_t idx = 0;
-      for(auto& ev : msgs)
+      for(auto& ev : *msgs)
       {
         const llarp_buffer_t buf(ev.first);
         auto& msg = sendmsgs[idx];
@@ -166,7 +166,9 @@ namespace llarp
         {
           const llarp_buffer_t buf(msg.X);
           if(!r->ParseRoutingMessageBuffer(buf, this, info.rxID))
-            continue;
+          {
+            LogWarn("invalid upstream data on endpoint ", info);
+          }
           m_LastActivity = r->Now();
         }
       }
@@ -188,22 +190,30 @@ namespace llarp
       for(const auto& msg : msgs)
       {
         llarp::LogDebug("relay ", msg.X.size(), " bytes downstream from ",
-                        info.downstream, " to ", info.upstream);
+                        info.upstream, " to ", info.downstream);
         r->SendToOrQueue(info.downstream, &msg);
       }
     }
 
     void
-    TransitHop::FlushQueues(AbstractRouter* r)
+    TransitHop::FlushUpstream(AbstractRouter* r)
     {
-      if(!m_UpstreamQueue.empty())
+      if(m_UpstreamQueue && !m_UpstreamQueue->empty())
         r->threadpool()->addJob(std::bind(&TransitHop::UpstreamWork,
                                           shared_from_this(),
                                           std::move(m_UpstreamQueue), r));
-      if(!m_DownstreamQueue.empty())
+
+      m_UpstreamQueue = nullptr;
+    }
+
+    void
+    TransitHop::FlushDownstream(AbstractRouter* r)
+    {
+      if(m_DownstreamQueue && !m_DownstreamQueue->empty())
         r->threadpool()->addJob(std::bind(&TransitHop::DownstreamWork,
                                           shared_from_this(),
                                           std::move(m_DownstreamQueue), r));
+      m_DownstreamQueue = nullptr;
     }
 
     bool
diff --git a/llarp/path/transit_hop.hpp b/llarp/path/transit_hop.hpp
index c29a5b6b4..008ec7841 100644
--- a/llarp/path/transit_hop.hpp
+++ b/llarp/path/transit_hop.hpp
@@ -196,14 +196,17 @@ namespace llarp
       HandleDHTMessage(const dht::IMessage& msg, AbstractRouter* r) override;
 
       void
-      FlushQueues(AbstractRouter* r) override;
+      FlushUpstream(AbstractRouter* r) override;
+
+      void
+      FlushDownstream(AbstractRouter* r) override;
 
      protected:
       void
-      UpstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+      UpstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override;
 
       void
-      DownstreamWork(TrafficQueue_t queue, AbstractRouter* r) override;
+      DownstreamWork(TrafficQueue_ptr queue, AbstractRouter* r) override;
 
       void
       HandleAllUpstream(std::vector< RelayUpstreamMessage > msgs,
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 99e713cdd..53af76f9a 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -164,10 +164,9 @@ namespace llarp
   void
   Router::PumpLL()
   {
-    _logic->tick(time_now_ms());
-    paths.Pump();
-    _logic->tick(time_now_ms());
+    paths.PumpDownstream();
     _linkManager.PumpLinks();
+    paths.PumpUpstream();
   }
 
   bool
@@ -1063,7 +1062,7 @@ namespace llarp
     _exitContext.Stop();
     if(rpcServer)
       rpcServer->Stop();
-    paths.Pump();
+    paths.PumpUpstream();
     _linkManager.PumpLinks();
     _logic->call_later({200, this, &RouterAfterStopIssued});
   }
diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp
index c0e94bc58..2de65698e 100644
--- a/llarp/service/endpoint.cpp
+++ b/llarp/service/endpoint.cpp
@@ -1065,6 +1065,7 @@ namespace llarp
       for(const auto& item : m_state->m_SendQueue)
         item.second->SendRoutingMessage(*item.first, router);
       m_state->m_SendQueue.clear();
+      SendAllUpstream(Router());
     }
 
     bool
diff --git a/llarp/util/thread/logic.cpp b/llarp/util/thread/logic.cpp
index 432b73b53..22b01b2b6 100644
--- a/llarp/util/thread/logic.cpp
+++ b/llarp/util/thread/logic.cpp
@@ -13,6 +13,15 @@ namespace llarp
     llarp_threadpool_tick(this->thread);
   }
 
+  Logic::Logic()
+      : thread(llarp_init_threadpool(1, "llarp-logic"))
+      , timer(llarp_init_timer())
+  {
+    llarp_threadpool_start(thread);
+    /// set thread id
+    thread->impl->addJob([&]() { id.emplace(std::this_thread::get_id()); });
+  }
+
   Logic::~Logic()
   {
     llarp_threadpool_stop(this->thread);
@@ -64,7 +73,11 @@ namespace llarp
   bool
   Logic::queue_func(std::function< void(void) >&& f)
   {
-    return this->thread->impl->addJob(f);
+    if(!this->thread->impl->tryAddJob(f))
+    {
+      call_later(0, f);
+    }
+    return true;
   }
 
   void
@@ -98,7 +111,7 @@ namespace llarp
   bool
   Logic::can_flush() const
   {
-    return false;
+    return id.has_value() && id.value() == std::this_thread::get_id();
   }
 
 }  // namespace llarp
diff --git a/llarp/util/thread/logic.hpp b/llarp/util/thread/logic.hpp
index ec3b2fae2..4373ba0d2 100644
--- a/llarp/util/thread/logic.hpp
+++ b/llarp/util/thread/logic.hpp
@@ -12,13 +12,9 @@ namespace llarp
    public:
     struct llarp_threadpool* thread;
     struct llarp_timer_context* timer;
+    std::optional< std::thread::id > id;
 
-    Logic()
-        : thread(llarp_init_threadpool(1, "llarp-logic"))
-        , timer(llarp_init_timer())
-    {
-      llarp_threadpool_start(thread);
-    }
+    Logic();
 
     ~Logic();
 

From 327c545530241bd40c0c94aa08a5b1933bd1b180 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Mon, 16 Sep 2019 12:12:05 -0400
Subject: [PATCH 18/27] finish multithread cryptography first pass

---
 llarp/handlers/tun.cpp       |  2 +-
 llarp/iwp/linklayer.cpp      |  6 ++-
 llarp/iwp/message_buffer.cpp | 31 ++++++------
 llarp/iwp/message_buffer.hpp |  8 ++--
 llarp/iwp/session.cpp        | 93 ++++++++++++++++++++----------------
 llarp/iwp/session.hpp        | 19 ++++----
 llarp/path/path.cpp          |  9 +++-
 llarp/path/path_context.cpp  |  4 +-
 llarp/path/pathset.cpp       |  4 +-
 llarp/path/pathset.hpp       |  4 +-
 llarp/path/transit_hop.cpp   |  6 ++-
 llarp/router/router.cpp      |  5 +-
 llarp/service/endpoint.cpp   |  2 +-
 llarp/util/thread/logic.cpp  |  2 -
 14 files changed, 111 insertions(+), 84 deletions(-)

diff --git a/llarp/handlers/tun.cpp b/llarp/handlers/tun.cpp
index 57588e343..475b4ba67 100644
--- a/llarp/handlers/tun.cpp
+++ b/llarp/handlers/tun.cpp
@@ -784,7 +784,7 @@ namespace llarp
         }
         llarp::LogWarn(Name(), " did not flush packets");
       });
-      SendAllDownstream(Router());
+      Router()->PumpLL();
     }
 
     bool
diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index 8cee5e636..d24d2f50b 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -1,5 +1,6 @@
 #include <iwp/linklayer.hpp>
 #include <iwp/session.hpp>
+#include <unordered_set>
 
 namespace llarp
 {
@@ -22,13 +23,14 @@ namespace llarp
     void
     LinkLayer::Pump()
     {
-      std::set< RouterID > sessions;
+      std::unordered_set< RouterID, RouterID::Hash > sessions;
       {
         ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
         auto itr = m_AuthedLinks.begin();
         while(itr != m_AuthedLinks.end())
         {
-          sessions.emplace(itr->first);
+          const RouterID r{itr->first};
+          sessions.emplace(r);
           ++itr;
         }
       }
diff --git a/llarp/iwp/message_buffer.cpp b/llarp/iwp/message_buffer.cpp
index 03b59a240..d63f2a129 100644
--- a/llarp/iwp/message_buffer.cpp
+++ b/llarp/iwp/message_buffer.cpp
@@ -22,11 +22,11 @@ namespace llarp
     ILinkSession::Packet_t
     OutboundMessage::XMIT() const
     {
-      auto xmit = CreatePacket(Command::eXMIT, 12 + 32, 0, 0);
-      htobe16buf(xmit.data() + 2 + PacketOverhead, m_Data.size());
-      htobe64buf(xmit.data() + 4 + PacketOverhead, m_MsgID);
+      auto xmit = CreatePacket(Command::eXMIT, 10 + 32);
+      htobe16buf(xmit.data() + CommandOverhead + PacketOverhead, m_Data.size());
+      htobe64buf(xmit.data() + 2 + CommandOverhead + PacketOverhead, m_MsgID);
       std::copy_n(m_Digest.begin(), m_Digest.size(),
-                  xmit.begin() + 12 + PacketOverhead);
+                  xmit.begin() + 10 + CommandOverhead + PacketOverhead);
       return xmit;
     }
 
@@ -84,7 +84,7 @@ namespace llarp
       const auto sz = m_Data.size();
       for(uint16_t idx = 0; idx < sz; idx += FragmentSize)
       {
-        if(!m_Acks.test(idx / FragmentSize))
+        if(not m_Acks.test(idx / FragmentSize))
           return false;
       }
       return true;
@@ -109,8 +109,8 @@ namespace llarp
 
     InboundMessage::InboundMessage(uint64_t msgid, uint16_t sz, ShortHash h,
                                    llarp_time_t now)
-        : m_Digset{std::move(h)}
-        , m_Size{sz}
+        : m_Data(size_t{sz})
+        , m_Digset{std::move(h)}
         , m_MsgID{msgid}
         , m_LastActiveAt{now}
     {
@@ -126,18 +126,18 @@ namespace llarp
         return;
       }
 
-      auto *dst = m_Data.data() + idx;
+      byte_t *dst = m_Data.data() + idx;
       std::copy_n(buf.base, buf.sz, dst);
       m_Acks.set(idx / FragmentSize);
-      LogDebug("got fragment ", idx / FragmentSize, " of ", m_Size);
+      LogDebug("got fragment ", idx / FragmentSize);
       m_LastActiveAt = now;
     }
 
     ILinkSession::Packet_t
     InboundMessage::ACKS() const
     {
-      auto acks = CreatePacket(Command::eACKS, 9, 0, 0);
-      htobe64buf(acks.data() + 2 + PacketOverhead, m_MsgID);
+      auto acks = CreatePacket(Command::eACKS, 9);
+      htobe64buf(acks.data() + CommandOverhead + PacketOverhead, m_MsgID);
       acks[PacketOverhead + 10] = AcksBitmask();
       return acks;
     }
@@ -151,9 +151,10 @@ namespace llarp
     bool
     InboundMessage::IsCompleted() const
     {
-      for(uint16_t idx = 0; idx < m_Size; idx += FragmentSize)
+      const auto sz = m_Data.size();
+      for(size_t idx = 0; idx < sz; idx += FragmentSize)
       {
-        if(!m_Acks.test(idx / FragmentSize))
+        if(not m_Acks.test(idx / FragmentSize))
           return false;
       }
       return true;
@@ -162,7 +163,7 @@ namespace llarp
     bool
     InboundMessage::ShouldSendACKS(llarp_time_t now) const
     {
-      return now - m_LastACKSent > 1000 || IsCompleted();
+      return (now > m_LastACKSent && now - m_LastACKSent > 1000);
     }
 
     bool
@@ -184,7 +185,7 @@ namespace llarp
     InboundMessage::Verify() const
     {
       ShortHash gotten;
-      const llarp_buffer_t buf(m_Data.data(), m_Size);
+      const llarp_buffer_t buf(m_Data);
       CryptoManager::instance()->shorthash(gotten, buf);
       return gotten == m_Digset;
     }
diff --git a/llarp/iwp/message_buffer.hpp b/llarp/iwp/message_buffer.hpp
index e09883094..c15e484d3 100644
--- a/llarp/iwp/message_buffer.hpp
+++ b/llarp/iwp/message_buffer.hpp
@@ -26,10 +26,13 @@ namespace llarp
       /// multiack
       eMACK = 5,
       /// close session
-      eCLOS = 6,
+      eCLOS = 0xff,
     };
 
+    /// max size of data fragments
     static constexpr size_t FragmentSize = 1024;
+    /// plaintext header overhead size
+    static constexpr size_t CommandOverhead = 2;
 
     struct OutboundMessage
     {
@@ -78,9 +81,8 @@ namespace llarp
       InboundMessage(uint64_t msgid, uint16_t sz, ShortHash h,
                      llarp_time_t now);
 
-      AlignedBuffer< MAX_LINK_MSG_SIZE > m_Data;
+      ILinkSession::Message_t m_Data;
       ShortHash m_Digset;
-      uint16_t m_Size             = 0;
       uint64_t m_MsgID            = 0;
       llarp_time_t m_LastACKSent  = 0;
       llarp_time_t m_LastActiveAt = 0;
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index 5f612e3e9..ad76513ed 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -11,13 +11,15 @@ namespace llarp
     ILinkSession::Packet_t
     CreatePacket(Command cmd, size_t plainsize, size_t minpad, size_t variance)
     {
-      const size_t pad = minpad > 0 ? minpad + randint() % variance : 0;
-      ILinkSession::Packet_t pkt(PacketOverhead + plainsize + pad + 2);
+      const size_t pad =
+          minpad > 0 ? minpad + (variance > 0 ? randint() % variance : 0) : 0;
+      ILinkSession::Packet_t pkt(PacketOverhead + plainsize + pad
+                                 + CommandOverhead);
       // randomize pad
       if(pad)
       {
         CryptoManager::instance()->randbytes(
-            pkt.data() + PacketOverhead + 2 + plainsize, pad);
+            pkt.data() + PacketOverhead + CommandOverhead + plainsize, pad);
       }
       // randomize nounce
       CryptoManager::instance()->randbytes(pkt.data() + HMACSIZE, TUNNONCESIZE);
@@ -39,8 +41,6 @@ namespace llarp
       GotLIM = util::memFn(&Session::GotOutboundLIM, this);
       CryptoManager::instance()->shorthash(m_SessionKey,
                                            llarp_buffer_t(rc.pubkey));
-      m_EncryptNext = std::make_shared< CryptoQueue_t >();
-      m_DecryptNext = std::make_shared< CryptoQueue_t >();
     }
 
     Session::Session(LinkLayer* p, Addr from)
@@ -54,8 +54,6 @@ namespace llarp
       GotLIM          = util::memFn(&Session::GotInboundLIM, this);
       const PubKey pk = m_Parent->GetOurRC().pubkey;
       CryptoManager::instance()->shorthash(m_SessionKey, llarp_buffer_t(pk));
-      m_EncryptNext = std::make_shared< CryptoQueue_t >();
-      m_DecryptNext = std::make_shared< CryptoQueue_t >();
     }
 
     void
@@ -132,11 +130,13 @@ namespace llarp
     void
     Session::EncryptAndSend(ILinkSession::Packet_t data)
     {
+      if(m_EncryptNext == nullptr)
+        m_EncryptNext = std::make_shared< CryptoQueue_t >();
       m_EncryptNext->emplace_back(std::move(data));
       if(!IsEstablished())
       {
         EncryptWorker(std::move(m_EncryptNext));
-        m_EncryptNext = std::make_shared< CryptoQueue_t >();
+        m_EncryptNext = nullptr;
       }
     }
 
@@ -198,21 +198,21 @@ namespace llarp
     Session::SendMACK()
     {
       // send multi acks
-      while(m_SendMACKS.size() > 0)
-      {
-        const auto sz            = m_SendMACKS.size();
-        const auto max           = Session::MaxACKSInMACK;
-        auto numAcks             = std::min(sz, max);
-        auto mack                = CreatePacket(Command::eMACK,
-                                 1 + (numAcks * sizeof(uint64_t)), 0, 0);
-        mack[PacketOverhead + 2] = byte_t{numAcks};
-        byte_t* ptr              = mack.data() + 3 + PacketOverhead;
+      while(m_SendMACKs.size() > 0)
+      {
+        const auto sz  = m_SendMACKs.size();
+        const auto max = Session::MaxACKSInMACK;
+        auto numAcks   = std::min(sz, max);
+        auto mack =
+            CreatePacket(Command::eMACK, 1 + (numAcks * sizeof(uint64_t)));
+        mack[PacketOverhead + CommandOverhead] = byte_t{numAcks};
+        byte_t* ptr = mack.data() + 3 + PacketOverhead;
         LogDebug("send ", numAcks, " macks to ", m_RemoteAddr);
-        auto itr = m_SendMACKS.begin();
+        auto itr = m_SendMACKs.begin();
         while(numAcks > 0)
         {
           htobe64buf(ptr, *itr);
-          itr = m_SendMACKS.erase(itr);
+          itr = m_SendMACKs.erase(itr);
           numAcks--;
           ptr += sizeof(uint64_t);
         }
@@ -246,20 +246,20 @@ namespace llarp
         }
       }
       auto self = shared_from_this();
-      if(!m_EncryptNext->empty())
+      if(m_EncryptNext && !m_EncryptNext->empty())
       {
         m_Parent->QueueWork([self, data = std::move(m_EncryptNext)] {
           self->EncryptWorker(data);
         });
-        m_EncryptNext = std::make_shared< CryptoQueue_t >();
+        m_EncryptNext = nullptr;
       }
 
-      if(!m_DecryptNext->empty())
+      if(m_DecryptNext && !m_DecryptNext->empty())
       {
         m_Parent->QueueWork([self, data = std::move(m_DecryptNext)] {
           self->DecryptWorker(data);
         });
-        m_DecryptNext = std::make_shared< CryptoQueue_t >();
+        m_DecryptNext = nullptr;
       }
     }
 
@@ -531,6 +531,8 @@ namespace llarp
     void
     Session::HandleSessionData(Packet_t pkt)
     {
+      if(m_DecryptNext == nullptr)
+        m_DecryptNext = std::make_shared< CryptoQueue_t >();
       m_DecryptNext->emplace_back(pkt);
     }
 
@@ -603,14 +605,15 @@ namespace llarp
         LogError("impossibly short mack from ", m_RemoteAddr);
         return;
       }
-      byte_t numAcks = data[2 + PacketOverhead];
-      if(data.size() < 3 + PacketOverhead + (numAcks * sizeof(uint64_t)))
+      byte_t numAcks = data[CommandOverhead + PacketOverhead];
+      if(data.size()
+         < 1 + CommandOverhead + PacketOverhead + (numAcks * sizeof(uint64_t)))
       {
         LogError("short mack from ", m_RemoteAddr);
         return;
       }
       LogDebug("got ", int(numAcks), " mack from ", m_RemoteAddr);
-      byte_t* ptr = data.data() + 3 + PacketOverhead;
+      byte_t* ptr = data.data() + CommandOverhead + PacketOverhead + 1;
       while(numAcks > 0)
       {
         uint64_t acked = bufbe64toh(ptr);
@@ -633,18 +636,18 @@ namespace llarp
     void
     Session::HandleNACK(std::vector< byte_t > data)
     {
-      if(data.size() < 10 + PacketOverhead)
+      if(data.size() < CommandOverhead + sizeof(uint64_t) + PacketOverhead)
       {
         LogError("short nack from ", m_RemoteAddr);
         return;
       }
-      uint64_t txid = bufbe64toh(data.data() + 2 + PacketOverhead);
+      uint64_t txid =
+          bufbe64toh(data.data() + CommandOverhead + PacketOverhead);
       LogDebug("got nack on ", txid, " from ", m_RemoteAddr);
       auto itr = m_TXMsgs.find(txid);
       if(itr != m_TXMsgs.end())
       {
-        auto xmit = itr->second.XMIT();
-        EncryptAndSend(std::move(xmit));
+        EncryptAndSend(itr->second.XMIT());
       }
       m_LastRX = m_Parent->Now();
     }
@@ -652,14 +655,17 @@ namespace llarp
     void
     Session::HandleXMIT(std::vector< byte_t > data)
     {
-      if(data.size() < 44 + PacketOverhead)
+      if(data.size() < CommandOverhead + PacketOverhead + sizeof(uint16_t)
+             + sizeof(uint64_t) + ShortHash::SIZE)
       {
         LogError("short XMIT from ", m_RemoteAddr);
         return;
       }
-      uint16_t sz   = bufbe16toh(data.data() + 2 + PacketOverhead);
-      uint64_t rxid = bufbe64toh(data.data() + 4 + PacketOverhead);
-      ShortHash h{data.data() + 12 + PacketOverhead};
+      uint16_t sz = bufbe16toh(data.data() + CommandOverhead + PacketOverhead);
+      uint64_t rxid = bufbe64toh(data.data() + CommandOverhead
+                                 + sizeof(uint16_t) + PacketOverhead);
+      ShortHash h{data.data() + CommandOverhead + sizeof(uint16_t)
+                  + sizeof(uint64_t) + PacketOverhead};
       LogDebug("rxid=", rxid, " sz=", sz, " h=", h.ToHex());
       m_LastRX = m_Parent->Now();
       {
@@ -667,6 +673,7 @@ namespace llarp
         auto itr = m_ReplayFilter.find(rxid);
         if(itr != m_ReplayFilter.end())
         {
+          m_SendMACKs.emplace(rxid);
           LogDebug("duplicate rxid=", rxid, " from ", m_RemoteAddr);
           return;
         }
@@ -684,14 +691,16 @@ namespace llarp
     void
     Session::HandleDATA(std::vector< byte_t > data)
     {
-      if(data.size() <= 12 + PacketOverhead)
+      if(data.size() <= CommandOverhead + sizeof(uint16_t) + sizeof(uint64_t)
+             + PacketOverhead)
       {
         LogError("short DATA from ", m_RemoteAddr, " ", data.size());
         return;
       }
-      m_LastRX      = m_Parent->Now();
-      uint16_t sz   = bufbe16toh(data.data() + 2 + PacketOverhead);
-      uint64_t rxid = bufbe64toh(data.data() + 4 + PacketOverhead);
+      m_LastRX    = m_Parent->Now();
+      uint16_t sz = bufbe16toh(data.data() + CommandOverhead + PacketOverhead);
+      uint64_t rxid = bufbe64toh(data.data() + CommandOverhead
+                                 + sizeof(uint16_t) + PacketOverhead);
       auto itr      = m_RXMsgs.find(rxid);
       if(itr == m_RXMsgs.end())
       {
@@ -699,13 +708,13 @@ namespace llarp
         {
           LogDebug("no rxid=", rxid, " for ", m_RemoteAddr);
           auto nack = CreatePacket(Command::eNACK, 8);
-          htobe64buf(nack.data() + PacketOverhead + 2, rxid);
+          htobe64buf(nack.data() + PacketOverhead + CommandOverhead, rxid);
           EncryptAndSend(std::move(nack));
         }
         else
         {
           LogDebug("replay hit for rxid=", rxid, " for ", m_RemoteAddr);
-          m_SendMACKS.emplace(rxid);
+          m_SendMACKs.emplace(rxid);
         }
         return;
       }
@@ -721,10 +730,10 @@ namespace llarp
         if(itr->second.Verify())
         {
           auto msg = std::move(itr->second);
-          const llarp_buffer_t buf(msg.m_Data.data(), msg.m_Size);
+          const llarp_buffer_t buf(msg.m_Data);
           m_Parent->HandleMessage(this, buf);
           m_ReplayFilter.emplace(itr->first, m_Parent->Now());
-          m_SendMACKS.emplace(itr->first);
+          m_SendMACKs.emplace(itr->first);
         }
         else
         {
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index 4f466ca5e..7d121fe5d 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -4,11 +4,13 @@
 #include <link/session.hpp>
 #include <iwp/linklayer.hpp>
 #include <iwp/message_buffer.hpp>
+#include <unordered_set>
 
 namespace llarp
 {
   namespace iwp
   {
+    /// packet crypto overhead size
     static constexpr size_t PacketOverhead = HMACSIZE + TUNNONCESIZE;
     /// creates a packet with plaintext size + wire overhead + random pad
     ILinkSession::Packet_t
@@ -19,20 +21,19 @@ namespace llarp
                      public std::enable_shared_from_this< Session >
     {
       /// Time how long we try delivery for
-      static constexpr llarp_time_t DeliveryTimeout = 2000;
+      static constexpr llarp_time_t DeliveryTimeout = 1000;
       /// Time how long we wait to recieve a message
       static constexpr llarp_time_t RecievalTimeout = (DeliveryTimeout * 8) / 5;
       /// How long to keep a replay window for
       static constexpr llarp_time_t ReplayWindow = (RecievalTimeout * 3) / 2;
       /// How often to acks RX messages
-      static constexpr llarp_time_t ACKResendInterval = 250;
+      static constexpr llarp_time_t ACKResendInterval = DeliveryTimeout / 4;
       /// How often to retransmit TX fragments
-      static constexpr llarp_time_t TXFlushInterval = ACKResendInterval * 2;
+      static constexpr llarp_time_t TXFlushInterval = (DeliveryTimeout / 5) * 2;
       /// How often we send a keepalive
-      static constexpr llarp_time_t PingInterval = 2000;
+      static constexpr llarp_time_t PingInterval = 5000;
       /// How long we wait for a session to die with no tx from them
-      static constexpr llarp_time_t SessionAliveTimeout =
-          (PingInterval * 13) / 3;
+      static constexpr llarp_time_t SessionAliveTimeout = PingInterval * 5;
       /// maximum number of messages we can ack in a multiack
       static constexpr std::size_t MaxACKSInMACK = 1024 / sizeof(uint64_t);
 
@@ -163,10 +164,10 @@ namespace llarp
 
       /// maps rxid to time recieved
       std::unordered_map< uint64_t, llarp_time_t > m_ReplayFilter;
-      /// list of rx messages to send in next set of multiacks
-      std::set< uint64_t > m_SendMACKS;
+      /// set of rx messages to send in next round of multiacks
+      std::unordered_set< uint64_t > m_SendMACKs;
 
-      using CryptoQueue_t   = std::vector< Packet_t >;
+      using CryptoQueue_t   = std::list< Packet_t >;
       using CryptoQueue_ptr = std::shared_ptr< CryptoQueue_t >;
       CryptoQueue_ptr m_EncryptNext;
       CryptoQueue_ptr m_DecryptNext;
diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp
index 930d6813b..fd3014831 100644
--- a/llarp/path/path.cpp
+++ b/llarp/path/path.cpp
@@ -1,6 +1,7 @@
 #include <path/path.hpp>
 
 #include <exit/exit_messages.hpp>
+#include <link/i_link_manager.hpp>
 #include <messages/discard.hpp>
 #include <messages/relay_commit.hpp>
 #include <messages/relay_status.hpp>
@@ -341,6 +342,7 @@ namespace llarp
           m_LastLatencyTestID   = latency.T;
           m_LastLatencyTestTime = now;
           SendRoutingMessage(latency, r);
+          FlushUpstream(r);
           return;
         }
         if(m_LastRecvMessage && now > m_LastRecvMessage)
@@ -374,6 +376,7 @@ namespace llarp
           LogDebug("failed to send upstream to ", Upstream());
         }
       }
+      r->linkManager().PumpLinks();
     }
 
     void
@@ -485,6 +488,7 @@ namespace llarp
         }
         m_LastRecvMessage = r->Now();
       }
+      FlushUpstream(r);
     }
 
     bool
@@ -586,7 +590,10 @@ namespace llarp
         latency.T             = randint();
         m_LastLatencyTestID   = latency.T;
         m_LastLatencyTestTime = now;
-        return SendRoutingMessage(latency, r);
+        if(!SendRoutingMessage(latency, r))
+          return false;
+        FlushUpstream(r);
+        return true;
       }
       LogWarn("got unwarranted path confirm message on tx=", RXID(),
               " rx=", RXID());
diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp
index f67ecce40..ff2c02622 100644
--- a/llarp/path/path_context.cpp
+++ b/llarp/path/path_context.cpp
@@ -260,7 +260,7 @@ namespace llarp
     PathContext::PumpUpstream()
     {
       m_TransitPaths.ForEach([&](auto& ptr) { ptr->FlushUpstream(m_Router); });
-      m_OurPaths.ForEach([&](auto& ptr) { ptr->SendAllUpstream(m_Router); });
+      m_OurPaths.ForEach([&](auto& ptr) { ptr->UpstreamFlush(m_Router); });
     }
 
     void
@@ -268,7 +268,7 @@ namespace llarp
     {
       m_TransitPaths.ForEach(
           [&](auto& ptr) { ptr->FlushDownstream(m_Router); });
-      m_OurPaths.ForEach([&](auto& ptr) { ptr->SendAllDownstream(m_Router); });
+      m_OurPaths.ForEach([&](auto& ptr) { ptr->DownstreamFlush(m_Router); });
     }
 
     void
diff --git a/llarp/path/pathset.cpp b/llarp/path/pathset.cpp
index 2db9377cb..b7555e895 100644
--- a/llarp/path/pathset.cpp
+++ b/llarp/path/pathset.cpp
@@ -376,13 +376,13 @@ namespace llarp
     }
 
     void
-    PathSet::SendAllUpstream(AbstractRouter* r)
+    PathSet::UpstreamFlush(AbstractRouter* r)
     {
       ForEachPath([r](const Path_ptr& p) { p->FlushUpstream(r); });
     }
 
     void
-    PathSet::SendAllDownstream(AbstractRouter* r)
+    PathSet::DownstreamFlush(AbstractRouter* r)
     {
       ForEachPath([r](const Path_ptr& p) { p->FlushDownstream(r); });
     }
diff --git a/llarp/path/pathset.hpp b/llarp/path/pathset.hpp
index b5beb5242..13bb13aa2 100644
--- a/llarp/path/pathset.hpp
+++ b/llarp/path/pathset.hpp
@@ -276,10 +276,10 @@ namespace llarp
       }
 
       void
-      SendAllUpstream(AbstractRouter* r);
+      UpstreamFlush(AbstractRouter* r);
 
       void
-      SendAllDownstream(AbstractRouter* r);
+      DownstreamFlush(AbstractRouter* r);
 
       size_t numPaths;
 
diff --git a/llarp/path/transit_hop.cpp b/llarp/path/transit_hop.cpp
index 4ccaf4d9b..c3c2e9d03 100644
--- a/llarp/path/transit_hop.cpp
+++ b/llarp/path/transit_hop.cpp
@@ -3,6 +3,7 @@
 #include <dht/context.hpp>
 #include <exit/context.hpp>
 #include <exit/exit_messages.hpp>
+#include <link/i_link_manager.hpp>
 #include <messages/discard.hpp>
 #include <messages/relay_commit.hpp>
 #include <messages/relay_status.hpp>
@@ -171,6 +172,7 @@ namespace llarp
           }
           m_LastActivity = r->Now();
         }
+        FlushDownstream(r);
       }
       else
       {
@@ -181,6 +183,7 @@ namespace llarp
           r->SendToOrQueue(info.upstream, &msg);
         }
       }
+      r->linkManager().PumpLinks();
     }
 
     void
@@ -193,6 +196,7 @@ namespace llarp
                         info.upstream, " to ", info.downstream);
         r->SendToOrQueue(info.downstream, &msg);
       }
+      r->linkManager().PumpLinks();
     }
 
     void
@@ -430,7 +434,7 @@ namespace llarp
     void
     TransitHop::QueueDestroySelf(AbstractRouter* r)
     {
-      auto func = std::bind(&TransitHop::SetSelfDestruct, this);
+      auto func = std::bind(&TransitHop::SetSelfDestruct, shared_from_this());
       r->logic()->queue_func(func);
     }
   }  // namespace path
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 53af76f9a..cef9d608e 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -164,9 +164,11 @@ namespace llarp
   void
   Router::PumpLL()
   {
+    if(_stopping.load())
+      return;
+    paths.PumpUpstream();
     paths.PumpDownstream();
     _linkManager.PumpLinks();
-    paths.PumpUpstream();
   }
 
   bool
@@ -1011,6 +1013,7 @@ namespace llarp
     }
 
     LogInfo("have ", nodedb->num_loaded(), " routers");
+    _netloop->add_ticker(std::bind(&Router::PumpLL, this));
     ScheduleTicker(1000);
     _running.store(true);
     _startedAt = Now();
diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp
index 2de65698e..212d5bde9 100644
--- a/llarp/service/endpoint.cpp
+++ b/llarp/service/endpoint.cpp
@@ -1065,7 +1065,7 @@ namespace llarp
       for(const auto& item : m_state->m_SendQueue)
         item.second->SendRoutingMessage(*item.first, router);
       m_state->m_SendQueue.clear();
-      SendAllUpstream(Router());
+      router->PumpLL();
     }
 
     bool
diff --git a/llarp/util/thread/logic.cpp b/llarp/util/thread/logic.cpp
index 22b01b2b6..8a6c0d803 100644
--- a/llarp/util/thread/logic.cpp
+++ b/llarp/util/thread/logic.cpp
@@ -55,9 +55,7 @@ namespace llarp
     if(this->thread)
     {
       llarp_threadpool_stop(this->thread);
-      llarp_threadpool_join(this->thread);
     }
-    llarp_free_threadpool(&this->thread);
 
     llarp::LogDebug("logic timer stop");
     if(this->timer)

From 5863e3382513d18915b44aabf6ce3edeed88ded4 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Mon, 16 Sep 2019 12:21:21 -0400
Subject: [PATCH 19/27] move it arround

---
 llarp/path/ihophandler.cpp | 36 ++++++++++++++++++++++++++++++++++++
 llarp/path/ihophandler.hpp | 25 ++-----------------------
 2 files changed, 38 insertions(+), 23 deletions(-)

diff --git a/llarp/path/ihophandler.cpp b/llarp/path/ihophandler.cpp
index 02c202d5f..e0a0cc649 100644
--- a/llarp/path/ihophandler.cpp
+++ b/llarp/path/ihophandler.cpp
@@ -1 +1,37 @@
 #include <path/ihophandler.hpp>
+
+namespace llarp
+{
+  namespace path
+  {
+    // handle data in upstream direction
+    bool
+    IHopHandler::HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y,
+                                AbstractRouter*)
+    {
+      if(m_UpstreamQueue == nullptr)
+        m_UpstreamQueue = std::make_shared< TrafficQueue_t >();
+      m_UpstreamQueue->emplace_back();
+      auto& pkt = m_UpstreamQueue->back();
+      pkt.first.resize(X.sz);
+      std::copy_n(X.base, X.sz, pkt.first.begin());
+      pkt.second = Y;
+      return true;
+    }
+
+    // handle data in downstream direction
+    bool
+    IHopHandler::HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y,
+                                  AbstractRouter*)
+    {
+      if(m_DownstreamQueue == nullptr)
+        m_DownstreamQueue = std::make_shared< TrafficQueue_t >();
+      m_DownstreamQueue->emplace_back();
+      auto& pkt = m_DownstreamQueue->back();
+      pkt.first.resize(X.sz);
+      std::copy_n(X.base, X.sz, pkt.first.begin());
+      pkt.second = Y;
+      return true;
+    }
+  }  // namespace path
+}  // namespace llarp
\ No newline at end of file
diff --git a/llarp/path/ihophandler.hpp b/llarp/path/ihophandler.hpp
index 35e4958e4..02ab0c08a 100644
--- a/llarp/path/ihophandler.hpp
+++ b/llarp/path/ihophandler.hpp
@@ -43,32 +43,11 @@ namespace llarp
       // handle data in upstream direction
       bool
       HandleUpstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                     AbstractRouter*)
-      {
-        if(m_UpstreamQueue == nullptr)
-          m_UpstreamQueue = std::make_shared< TrafficQueue_t >();
-        m_UpstreamQueue->emplace_back();
-        auto& pkt = m_UpstreamQueue->back();
-        pkt.first.resize(X.sz);
-        std::copy_n(X.base, X.sz, pkt.first.begin());
-        pkt.second = Y;
-        return true;
-      }
-
+                     AbstractRouter*);
       // handle data in downstream direction
       bool
       HandleDownstream(const llarp_buffer_t& X, const TunnelNonce& Y,
-                       AbstractRouter*)
-      {
-        if(m_DownstreamQueue == nullptr)
-          m_DownstreamQueue = std::make_shared< TrafficQueue_t >();
-        m_DownstreamQueue->emplace_back();
-        auto& pkt = m_DownstreamQueue->back();
-        pkt.first.resize(X.sz);
-        std::copy_n(X.base, X.sz, pkt.first.begin());
-        pkt.second = Y;
-        return true;
-      }
+                       AbstractRouter*);
 
       /// return timestamp last remote activity happened at
       virtual llarp_time_t

From 6c2ebbb925caebafba7c725892bc28dc58f73d9a Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Thu, 19 Sep 2019 10:41:31 -0400
Subject: [PATCH 20/27] try fixing handover and add snapp traffic to
 multithreaded crypto workers

---
 llarp/path/path.hpp                  | 28 ++++++++++
 llarp/service/async_key_exchange.cpp | 19 +++----
 llarp/service/async_key_exchange.hpp |  9 ++--
 llarp/service/endpoint.cpp           | 40 +++++++-------
 llarp/service/endpoint_util.cpp      |  2 +-
 llarp/service/lookup.hpp             |  2 +-
 llarp/service/outbound_context.cpp   | 13 +++--
 llarp/service/sendcontext.cpp        | 79 +++++++++++++++++-----------
 llarp/service/sendcontext.hpp        |  2 +-
 9 files changed, 118 insertions(+), 76 deletions(-)

diff --git a/llarp/path/path.hpp b/llarp/path/path.hpp
index 72a1c7918..a2705e0fe 100644
--- a/llarp/path/path.hpp
+++ b/llarp/path/path.hpp
@@ -108,6 +108,34 @@ namespace llarp
         return _role;
       }
 
+      struct Hash
+      {
+        size_t
+        operator()(const Path& p) const
+        {
+          const auto& tx = p.hops[0].txID;
+          const auto& rx = p.hops[0].rxID;
+          const auto& r  = p.hops[0].upstream;
+          const size_t rhash =
+              std::accumulate(r.begin(), r.end(), 0, std::bit_xor< size_t >());
+          return std::accumulate(rx.begin(), rx.begin(),
+                                 std::accumulate(tx.begin(), tx.end(), rhash,
+                                                 std::bit_xor< size_t >()),
+                                 std::bit_xor< size_t >());
+        }
+      };
+
+      struct Ptr_Hash
+      {
+        size_t
+        operator()(const std::shared_ptr< Path >& p) const
+        {
+          if(p == nullptr)
+            return 0;
+          return Hash{}(*p);
+        }
+      };
+
       bool
       operator<(const Path& other) const
       {
diff --git a/llarp/service/async_key_exchange.cpp b/llarp/service/async_key_exchange.cpp
index cbf06e1b3..b948520bf 100644
--- a/llarp/service/async_key_exchange.cpp
+++ b/llarp/service/async_key_exchange.cpp
@@ -17,6 +17,7 @@ namespace llarp
         : logic(std::move(l))
         , m_remote(std::move(r))
         , m_LocalIdentity(localident)
+        , frame(std::make_shared< ProtocolFrame >())
         , introPubKey(introsetPubKey)
         , remoteIntro(remote)
         , handler(h)
@@ -26,34 +27,31 @@ namespace llarp
     }
 
     void
-    AsyncKeyExchange::Result(void* user)
+    AsyncKeyExchange::Result(std::shared_ptr< AsyncKeyExchange > self)
     {
-      auto* self = static_cast< AsyncKeyExchange* >(user);
       // put values
       self->handler->PutSenderFor(self->msg.tag, self->m_remote, false);
       self->handler->PutCachedSessionKeyFor(self->msg.tag, self->sharedKey);
       self->handler->PutIntroFor(self->msg.tag, self->remoteIntro);
       self->handler->PutReplyIntroFor(self->msg.tag, self->msg.introReply);
       self->hook(self->frame);
-      delete self;
     }
 
     void
-    AsyncKeyExchange::Encrypt(void* user)
+    AsyncKeyExchange::Encrypt(std::shared_ptr< AsyncKeyExchange > self)
     {
-      auto* self = static_cast< AsyncKeyExchange* >(user);
       // derive ntru session key component
       SharedSecret K;
       auto crypto = CryptoManager::instance();
-      crypto->pqe_encrypt(self->frame.C, K, self->introPubKey);
+      crypto->pqe_encrypt(self->frame->C, K, self->introPubKey);
       // randomize Nonce
-      self->frame.N.Randomize();
+      self->frame->N.Randomize();
       // compure post handshake session key
       // PKE (A, B, N)
       SharedSecret sharedSecret;
       path_dh_func dh_client = util::memFn(&Crypto::dh_client, crypto);
       if(!self->m_LocalIdentity.KeyExchange(dh_client, sharedSecret,
-                                            self->m_remote, self->frame.N))
+                                            self->m_remote, self->frame->N))
       {
         LogError("failed to derive x25519 shared key component");
       }
@@ -70,12 +68,11 @@ namespace llarp
       // set version
       self->msg.version = LLARP_PROTO_VERSION;
       // encrypt and sign
-      if(self->frame.EncryptAndSign(self->msg, K, self->m_LocalIdentity))
-        self->logic->queue_job({self, &Result});
+      if(self->frame->EncryptAndSign(self->msg, K, self->m_LocalIdentity))
+        self->logic->queue_func(std::bind(&AsyncKeyExchange::Result, self));
       else
       {
         LogError("failed to encrypt and sign");
-        delete self;
       }
     }
   }  // namespace service
diff --git a/llarp/service/async_key_exchange.hpp b/llarp/service/async_key_exchange.hpp
index 358481b23..b43ffd279 100644
--- a/llarp/service/async_key_exchange.hpp
+++ b/llarp/service/async_key_exchange.hpp
@@ -12,17 +12,18 @@ namespace llarp
   namespace service
   {
     struct AsyncKeyExchange
+        : public std::enable_shared_from_this< AsyncKeyExchange >
     {
       std::shared_ptr< Logic > logic;
       SharedSecret sharedKey;
       ServiceInfo m_remote;
       const Identity& m_LocalIdentity;
       ProtocolMessage msg;
-      ProtocolFrame frame;
+      std::shared_ptr< ProtocolFrame > frame;
       Introduction intro;
       const PQPubKey introPubKey;
       Introduction remoteIntro;
-      std::function< void(ProtocolFrame&) > hook;
+      std::function< void(std::shared_ptr< ProtocolFrame >) > hook;
       IDataHandler* handler;
       ConvoTag tag;
 
@@ -33,11 +34,11 @@ namespace llarp
                        const ConvoTag& t, ProtocolType proto);
 
       static void
-      Result(void* user);
+      Result(std::shared_ptr< AsyncKeyExchange > user);
 
       /// given protocol message make protocol frame
       static void
-      Encrypt(void* user);
+      Encrypt(std::shared_ptr< AsyncKeyExchange > user);
     };
 
   }  // namespace service
diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp
index 212d5bde9..132f101be 100644
--- a/llarp/service/endpoint.cpp
+++ b/llarp/service/endpoint.cpp
@@ -1132,30 +1132,30 @@ namespace llarp
           if(p)
           {
             // TODO: check expiration of our end
-            ProtocolMessage m(f.T);
-            m.PutBuffer(data);
+            auto m = std::make_shared< ProtocolMessage >(f.T);
+            m->PutBuffer(data);
             f.N.Randomize();
             f.C.Zero();
             transfer->Y.Randomize();
-            m.proto      = t;
-            m.introReply = p->intro;
-            PutReplyIntroFor(f.T, m.introReply);
-            m.sender    = m_Identity.pub;
-            m.seqno     = GetSeqNoForConvo(f.T);
+            m->proto      = t;
+            m->introReply = p->intro;
+            PutReplyIntroFor(f.T, m->introReply);
+            m->sender   = m_Identity.pub;
+            m->seqno    = GetSeqNoForConvo(f.T);
             f.S         = 1;
-            f.F         = m.introReply.pathID;
+            f.F         = m->introReply.pathID;
             transfer->P = remoteIntro.pathID;
-            if(!f.EncryptAndSign(m, K, m_Identity))
-            {
-              LogError("failed to encrypt and sign");
-              return false;
-            }
-            LogDebug(Name(), " send ", data.sz, " via ", remoteIntro.router);
-            {
-              util::Lock lock(&m_state->m_SendQueueMutex);
-              m_state->m_SendQueue.emplace_back(transfer, p);
-            }
-            return true;
+            auto self   = this;
+            return CryptoWorker()->addJob([transfer, p, m, K, self]() {
+              if(not transfer->T.EncryptAndSign(*m, K, self->m_Identity))
+              {
+                LogError("failed to encrypt and sign");
+                return;
+              }
+
+              util::Lock lock(&self->m_state->m_SendQueueMutex);
+              self->m_state->m_SendQueue.emplace_back(transfer, p);
+            });
           }
         }
       }
@@ -1193,7 +1193,7 @@ namespace llarp
             }
             m_state->m_PendingTraffic.erase(r);
           },
-          5000, true);
+          5000);
     }
 
     bool
diff --git a/llarp/service/endpoint_util.cpp b/llarp/service/endpoint_util.cpp
index ecf41f45b..95c9af7d0 100644
--- a/llarp/service/endpoint_util.cpp
+++ b/llarp/service/endpoint_util.cpp
@@ -166,7 +166,7 @@ namespace llarp
       {
         if(itr->second.remote.Addr() == info)
         {
-          if(tags.insert(itr->first).second)
+          if(tags.emplace(itr->first).second)
           {
             inserted = true;
           }
diff --git a/llarp/service/lookup.hpp b/llarp/service/lookup.hpp
index 03ffb09e4..5177fb0a6 100644
--- a/llarp/service/lookup.hpp
+++ b/llarp/service/lookup.hpp
@@ -19,7 +19,7 @@ namespace llarp
   {
     struct ILookupHolder;
 
-    constexpr size_t MaxConcurrentLookups = size_t(4);
+    constexpr size_t MaxConcurrentLookups = size_t(16);
 
     struct IServiceLookup
     {
diff --git a/llarp/service/outbound_context.cpp b/llarp/service/outbound_context.cpp
index 185c6d034..9a65c7085 100644
--- a/llarp/service/outbound_context.cpp
+++ b/llarp/service/outbound_context.cpp
@@ -64,6 +64,7 @@ namespace llarp
         if(intro.expiresAt > m_NextIntro.expiresAt)
           m_NextIntro = intro;
       }
+      currentConvoTag.Randomize();
     }
 
     OutboundContext::~OutboundContext() = default;
@@ -160,8 +161,7 @@ namespace llarp
           return;
         }
       }
-      currentConvoTag.Randomize();
-      AsyncKeyExchange* ex = new AsyncKeyExchange(
+      auto ex = std::make_shared< AsyncKeyExchange >(
           m_Endpoint->RouterLogic(), remoteIdent, m_Endpoint->GetIdentity(),
           currentIntroSet.K, remoteIntro, m_DataHandler, currentConvoTag, t);
 
@@ -170,7 +170,7 @@ namespace llarp
 
       ex->msg.PutBuffer(payload);
       ex->msg.introReply = path->intro;
-      ex->frame.F        = ex->msg.introReply.pathID;
+      ex->frame->F       = ex->msg.introReply.pathID;
       m_Endpoint->CryptoWorker()->addJob(
           std::bind(&AsyncKeyExchange::Encrypt, ex));
     }
@@ -245,7 +245,8 @@ namespace llarp
       if(remoteIntro.ExpiresSoon(now))
       {
         // shift intro if it expires "soon"
-        ShiftIntroduction();
+        if(ShiftIntroduction())
+          SwapIntros();  // swap intros if we shifted
       }
       // lookup router in intro if set and unknown
       m_Endpoint->EnsureRouterIsKnown(remoteIntro.router);
@@ -253,7 +254,7 @@ namespace llarp
       auto itr = m_BadIntros.begin();
       while(itr != m_BadIntros.end())
       {
-        if(now - itr->second > path::default_lifetime)
+        if(now > itr->second && now - itr->second > path::default_lifetime)
           itr = m_BadIntros.erase(itr);
         else
           ++itr;
@@ -274,8 +275,6 @@ namespace llarp
             tmp.Randomize();
             llarp_buffer_t buf(tmp.data(), tmp.size());
             AsyncEncryptAndSendTo(buf, eProtocolControl);
-            if(currentConvoTag.IsZero())
-              return false;
             return !m_DataHandler->HasConvoTag(currentConvoTag);
           }
         }
diff --git a/llarp/service/sendcontext.cpp b/llarp/service/sendcontext.cpp
index 9571caffa..509a95a3e 100644
--- a/llarp/service/sendcontext.cpp
+++ b/llarp/service/sendcontext.cpp
@@ -5,6 +5,7 @@
 #include <service/endpoint.hpp>
 #include <util/thread/logic.hpp>
 #include <utility>
+#include <unordered_set>
 
 namespace llarp
 {
@@ -23,12 +24,12 @@ namespace llarp
     }
 
     bool
-    SendContext::Send(const ProtocolFrame& msg, path::Path_ptr path)
+    SendContext::Send(std::shared_ptr< ProtocolFrame > msg, path::Path_ptr path)
     {
       util::Lock lock(&m_SendQueueMutex);
       m_SendQueue.emplace_back(
           std::make_shared< const routing::PathTransferMessage >(
-              msg, remoteIntro.pathID),
+              *msg, remoteIntro.pathID),
           path);
       return true;
     }
@@ -37,17 +38,27 @@ namespace llarp
     SendContext::FlushUpstream()
     {
       auto r = m_Endpoint->Router();
-      util::Lock lock(&m_SendQueueMutex);
-      for(const auto& item : m_SendQueue)
+      std::unordered_set< path::Path_ptr, path::Path::Ptr_Hash > flushpaths;
       {
-        if(item.second->SendRoutingMessage(*item.first, r))
+        util::Lock lock(&m_SendQueueMutex);
+        for(const auto& item : m_SendQueue)
         {
-          lastGoodSend = r->Now();
+          if(item.second->SendRoutingMessage(*item.first, r))
+          {
+            lastGoodSend = r->Now();
+            flushpaths.emplace(item.second);
+          }
+          else
+            LogError(m_Endpoint->Name(), " failed to send frame on path");
         }
-        else
-          LogError(m_Endpoint->Name(), " failed to send frame on path");
+
+        m_SendQueue.clear();
+      }
+      // flush the select path's upstream
+      for(const auto& path : flushpaths)
+      {
+        path->FlushUpstream(r);
       }
-      m_SendQueue.clear();
     }
 
     /// send on an established convo tag
@@ -55,10 +66,10 @@ namespace llarp
     SendContext::EncryptAndSendTo(const llarp_buffer_t& payload, ProtocolType t)
     {
       SharedSecret shared;
-      ProtocolFrame f;
-      f.N.Randomize();
-      f.T = currentConvoTag;
-      f.S = ++sequenceNo;
+      auto f = std::make_shared< ProtocolFrame >();
+      f->N.Randomize();
+      f->T = currentConvoTag;
+      f->S = ++sequenceNo;
 
       auto path = m_PathSet->GetNewestPathByRouter(remoteIntro.router);
       if(!path)
@@ -68,29 +79,35 @@ namespace llarp
         return;
       }
 
-      if(!m_DataHandler->GetCachedSessionKeyFor(f.T, shared))
+      if(!m_DataHandler->GetCachedSessionKeyFor(f->T, shared))
       {
         LogError(m_Endpoint->Name(),
-                 " has no cached session key on session T=", f.T);
+                 " has no cached session key on session T=", f->T);
         return;
       }
 
-      ProtocolMessage m;
-      m_DataHandler->PutIntroFor(f.T, remoteIntro);
-      m_DataHandler->PutReplyIntroFor(f.T, path->intro);
-      m.proto      = t;
-      m.seqno      = m_Endpoint->GetSeqNoForConvo(f.T);
-      m.introReply = path->intro;
-      f.F          = m.introReply.pathID;
-      m.sender     = m_Endpoint->GetIdentity().pub;
-      m.tag        = f.T;
-      m.PutBuffer(payload);
-      if(!f.EncryptAndSign(m, shared, m_Endpoint->GetIdentity()))
-      {
-        LogError(m_Endpoint->Name(), " failed to sign message");
-        return;
-      }
-      Send(f, path);
+      auto m = std::make_shared< ProtocolMessage >();
+      m_DataHandler->PutIntroFor(f->T, remoteIntro);
+      m_DataHandler->PutReplyIntroFor(f->T, path->intro);
+      m->proto      = t;
+      m->seqno      = m_Endpoint->GetSeqNoForConvo(f->T);
+      m->introReply = path->intro;
+      f->F          = m->introReply.pathID;
+      m->sender     = m_Endpoint->GetIdentity().pub;
+      m->tag        = f->T;
+      m->PutBuffer(payload);
+      auto self = this;
+      m_Endpoint->CryptoWorker()->addJob([f, m, shared, path, self]() {
+        if(!f->EncryptAndSign(*m, shared, self->m_Endpoint->GetIdentity()))
+        {
+          LogError(self->m_Endpoint->Name(), " failed to sign message");
+          return;
+        }
+        self->m_Endpoint->RouterLogic()->queue_func([self, f, path]() {
+          self->Send(f, path);
+          self->FlushUpstream();
+        });
+      });
     }
 
     void
diff --git a/llarp/service/sendcontext.hpp b/llarp/service/sendcontext.hpp
index 9ca0cb8c0..8d72e3905 100644
--- a/llarp/service/sendcontext.hpp
+++ b/llarp/service/sendcontext.hpp
@@ -29,7 +29,7 @@ namespace llarp
       /// queue send a fully encrypted hidden service frame
       /// via a path
       bool
-      Send(const ProtocolFrame& f, path::Path_ptr path)
+      Send(std::shared_ptr< ProtocolFrame > f, path::Path_ptr path)
           LOCKS_EXCLUDED(m_SendQueueMutex);
 
       /// flush upstream traffic when in router thread

From da9437d0cf6ff7bf1757679213532e459792383c Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Tue, 1 Oct 2019 11:05:37 -0400
Subject: [PATCH 21/27] don't need that

---
 llarp/util/thread/threading.hpp | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/llarp/util/thread/threading.hpp b/llarp/util/thread/threading.hpp
index ece993535..8ce07db51 100644
--- a/llarp/util/thread/threading.hpp
+++ b/llarp/util/thread/threading.hpp
@@ -26,13 +26,6 @@ using pid_t = int;
 #define ACQUIRE_LOCK(lock, mtx) lock(&mtx)
 #endif
 
-#ifdef YOLO_DISABLE_LOCKING
-#warning \
-    "!!! locking disabled !!! This may cause hella crashes, please mind the gap."
-#undef ACQUIRE_LOCK
-#define ACQUIRE_LOCK(lock, mtx)
-#endif
-
 namespace llarp
 {
   namespace util

From c3451fc77a0c8ad8349374283c3f2e1141d61546 Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Tue, 1 Oct 2019 11:06:56 -0400
Subject: [PATCH 22/27] remove log level change

---
 test/link/test_llarp_link.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/link/test_llarp_link.cpp b/test/link/test_llarp_link.cpp
index d93ce3dbe..22dce59ad 100644
--- a/test/link/test_llarp_link.cpp
+++ b/test/link/test_llarp_link.cpp
@@ -133,7 +133,6 @@ struct LinkLayerTest : public test::LlarpTest< llarp::sodium::CryptoLibSodium >
   void
   SetUp()
   {
-    llarp::SetLogLevel(eLogDebug);
     oldRCLifetime              = RouterContact::Lifetime;
     RouterContact::BlockBogons = false;
     RouterContact::Lifetime    = 500;

From 3c1d5518d8ecbc64decc758504af39d1df7c6a54 Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Wed, 2 Oct 2019 09:06:14 -0400
Subject: [PATCH 23/27] fix windows port and make it compile

---
 llarp/ev/ev.cpp                        |  2 +-
 llarp/ev/ev.h                          |  9 ++++
 llarp/ev/ev.hpp                        | 64 +++++++++++++++++++++++++-
 llarp/ev/ev_libuv.cpp                  | 39 ++++++++++++----
 llarp/ev/ev_libuv.hpp                  |  2 +-
 llarp/ev/ev_win32.cpp                  | 31 ++++++++++++-
 llarp/ev/ev_win32.hpp                  | 23 +++++++++
 llarp/iwp/linklayer.cpp                |  4 +-
 llarp/iwp/message_buffer.cpp           |  4 +-
 llarp/iwp/session.cpp                  | 53 +++++++++++----------
 llarp/iwp/session.hpp                  | 19 ++++----
 llarp/link/server.cpp                  | 39 ++++++----------
 llarp/link/server.hpp                  |  3 --
 llarp/link/session.hpp                 |  3 +-
 llarp/util/logging/logger_internal.hpp |  2 +-
 15 files changed, 213 insertions(+), 84 deletions(-)

diff --git a/llarp/ev/ev.cpp b/llarp/ev/ev.cpp
index da98c5f32..bab081302 100644
--- a/llarp/ev/ev.cpp
+++ b/llarp/ev/ev.cpp
@@ -36,7 +36,7 @@ void
 llarp_ev_loop_run_single_process(llarp_ev_loop_ptr ev,
                                  std::shared_ptr< llarp::Logic > logic)
 {
-  ev->give_logic(logic);
+  ev->set_logic(logic);
   while(ev->running())
   {
     ev->update_time();
diff --git a/llarp/ev/ev.h b/llarp/ev/ev.h
index 961dcbcc9..c3960fa84 100644
--- a/llarp/ev/ev.h
+++ b/llarp/ev/ev.h
@@ -69,6 +69,10 @@ llarp_ev_loop_time_now_ms(const llarp_ev_loop_ptr &ev);
 void
 llarp_ev_loop_stop(const llarp_ev_loop_ptr &ev);
 
+/// list of packets we recv'd
+/// forward declared
+struct llarp_pkt_list;
+
 /// UDP handling configuration
 struct llarp_udp_io
 {
@@ -88,6 +92,11 @@ struct llarp_udp_io
                 size_t);
 };
 
+/// get all packets recvieved last tick
+/// return true if we got packets return false if we didn't
+bool
+llarp_ev_udp_recvmany(struct llarp_udp_io *udp, struct llarp_pkt_list *pkts);
+
 /// add UDP handler
 int
 llarp_ev_add_udp(struct llarp_ev_loop *ev, struct llarp_udp_io *udp,
diff --git a/llarp/ev/ev.hpp b/llarp/ev/ev.hpp
index 5e18fe699..658ae056d 100644
--- a/llarp/ev/ev.hpp
+++ b/llarp/ev/ev.hpp
@@ -1,6 +1,7 @@
 #ifndef LLARP_EV_HPP
 #define LLARP_EV_HPP
 
+#include <net/net_addr.hpp>
 #include <ev/ev.h>
 #include <util/buffer.hpp>
 #include <util/codel.hpp>
@@ -784,7 +785,7 @@ struct llarp_ev_loop
   }
 
   /// give this event loop a logic thread for calling
-  virtual void give_logic(std::shared_ptr< llarp::Logic >) = 0;
+  virtual void set_logic(std::shared_ptr< llarp::Logic >) = 0;
 
   /// register event listener
   virtual bool
@@ -801,7 +802,7 @@ struct llarp_ev_loop
 
   std::list< std::unique_ptr< llarp::ev_io > > handlers;
 
-  void
+  virtual void
   tick_listeners()
   {
     auto itr = handlers.begin();
@@ -818,4 +819,63 @@ struct llarp_ev_loop
   }
 };
 
+struct PacketBuffer
+{
+  PacketBuffer(PacketBuffer&& other)
+  {
+    _ptr       = other._ptr;
+    _sz        = other._sz;
+    other._ptr = nullptr;
+    other._sz  = 0;
+  }
+
+  PacketBuffer() : PacketBuffer(nullptr, 0){};
+  explicit PacketBuffer(size_t sz) : PacketBuffer(new char[sz], sz)
+  {
+  }
+  PacketBuffer(char* buf, size_t sz) : _ptr{buf}, _sz{sz}
+  {
+  }
+  ~PacketBuffer()
+  {
+    if(_ptr)
+      delete[] _ptr;
+  }
+  byte_t*
+  data()
+  {
+    return (byte_t*)_ptr;
+  }
+  size_t
+  size()
+  {
+    return _sz;
+  }
+  byte_t& operator[](size_t sz)
+  {
+    return data()[sz];
+  }
+  void
+  reserve(size_t sz)
+  {
+    if(_ptr)
+      delete[] _ptr;
+    _ptr = new char[sz];
+  }
+
+ private:
+  char* _ptr = nullptr;
+  size_t _sz = 0;
+};
+
+struct PacketEvent
+{
+  llarp::Addr remote = {};
+  PacketBuffer pkt   = {};
+};
+
+struct llarp_pkt_list : public std::vector< PacketEvent >
+{
+};
+
 #endif
diff --git a/llarp/ev/ev_libuv.cpp b/llarp/ev/ev_libuv.cpp
index a2f35200b..dcfe6b5cd 100644
--- a/llarp/ev/ev_libuv.cpp
+++ b/llarp/ev/ev_libuv.cpp
@@ -371,14 +371,14 @@ namespace libuv
     uv_check_t m_Ticker;
     llarp_udp_io* const m_UDP;
     llarp::Addr m_Addr;
-    bool gotpkts;
+    llarp_pkt_list m_LastPackets;
+    std::array< char, 1500 > m_Buffer;
 
     udp_glue(uv_loop_t* loop, llarp_udp_io* udp, const sockaddr* src)
         : m_UDP(udp), m_Addr(*src)
     {
       m_Handle.data = this;
       m_Ticker.data = this;
-      gotpkts       = false;
       uv_udp_init(loop, &m_Handle);
       uv_check_init(loop, &m_Ticker);
     }
@@ -386,27 +386,44 @@ namespace libuv
     static void
     Alloc(uv_handle_t*, size_t suggested_size, uv_buf_t* buf)
     {
-      buf->base = new char[suggested_size];
-      buf->len  = suggested_size;
+      const size_t sz = std::min(suggested_size, size_t{1500});
+      buf->base       = new char[sz];
+      buf->len        = sz;
     }
 
     static void
     OnRecv(uv_udp_t* handle, ssize_t nread, const uv_buf_t* buf,
            const struct sockaddr* addr, unsigned)
     {
+      udp_glue* glue = static_cast< udp_glue* >(handle->data);
       if(addr)
-        static_cast< udp_glue* >(handle->data)->RecvFrom(nread, buf, addr);
-      delete[] buf->base;
+        glue->RecvFrom(nread, buf, addr);
+      if(glue->m_UDP == nullptr || glue->m_UDP->recvfrom != nullptr)
+        delete[] buf->base;
+    }
+
+    bool
+    RecvMany(llarp_pkt_list* pkts)
+    {
+      *pkts         = std::move(m_LastPackets);
+      m_LastPackets = llarp_pkt_list();
+      return pkts->size() > 0;
     }
 
     void
     RecvFrom(ssize_t sz, const uv_buf_t* buf, const struct sockaddr* fromaddr)
     {
-      if(sz >= 0 && m_UDP && m_UDP->recvfrom)
+      if(sz > 0 && m_UDP)
       {
         const size_t pktsz = sz;
         const llarp_buffer_t pkt{(const byte_t*)buf->base, pktsz};
-        m_UDP->recvfrom(m_UDP, fromaddr, ManagedBuffer{pkt});
+        if(m_UDP->recvfrom)
+          m_UDP->recvfrom(m_UDP, fromaddr, ManagedBuffer{pkt});
+        else
+        {
+          m_LastPackets.emplace_back(
+              PacketEvent{llarp::Addr(*fromaddr), PacketBuffer(buf->base, sz)});
+        }
       }
     }
 
@@ -855,3 +872,9 @@ namespace libuv
   }
 
 }  // namespace libuv
+
+bool
+llarp_ev_udp_recvmany(struct llarp_udp_io* u, struct llarp_pkt_list* pkts)
+{
+  return static_cast< libuv::udp_glue* >(u->impl)->RecvMany(pkts);
+}
\ No newline at end of file
diff --git a/llarp/ev/ev_libuv.hpp b/llarp/ev/ev_libuv.hpp
index c04e6a370..a3f37fbc7 100644
--- a/llarp/ev/ev_libuv.hpp
+++ b/llarp/ev/ev_libuv.hpp
@@ -90,7 +90,7 @@ namespace libuv
     }
 
     void
-    give_logic(std::shared_ptr< llarp::Logic > l) override
+    set_logic(std::shared_ptr< llarp::Logic > l) override
     {
       m_Logic = l;
     }
diff --git a/llarp/ev/ev_win32.cpp b/llarp/ev/ev_win32.cpp
index cff227486..d7a8f19aa 100644
--- a/llarp/ev/ev_win32.cpp
+++ b/llarp/ev/ev_win32.cpp
@@ -366,10 +366,25 @@ namespace llarp
     if(static_cast< size_t >(ret) > sz)
       return -1;
     b.sz = ret;
-    udp->recvfrom(udp, addr, ManagedBuffer{b});
+    if(udp->recvfrom)
+      udp->recvfrom(udp, addr, ManagedBuffer{b});
+    else
+    {
+      m_RecvPackets.emplace_back(
+          PacketEvent{llarp::Addr(*addr), PacketBuffer(ret)});
+      std::copy_n(buf, ret, m_RecvPackets.back().pkt.data());
+    }
     return 0;
   }
 
+  bool
+  udp_listener::RecvMany(llarp_pkt_list* pkts)
+  {
+    *pkts         = std::move(m_RecvPackets);
+    m_RecvPackets = llarp_pkt_list();
+    return pkts->size() > 0;
+  }
+
   static int
   UDPSendTo(llarp_udp_io* udp, const sockaddr* to, const byte_t* ptr, size_t sz)
   {
@@ -696,4 +711,18 @@ llarp_win32_loop::stop()
   llarp::LogDebug("destroy upoll");
 }
 
+void
+llarp_win32_loop::tick_listeners()
+{
+  llarp_ev_loop::tick_listeners();
+  for(auto& func : m_Tickers)
+    m_Logic->queue_func([func]() { func(); });
+}
+
+bool
+llarp_ev_udp_recvmany(struct llarp_udp_io* u, struct llarp_pkt_list* pkts)
+{
+  return static_cast< llarp::udp_listener* >(u->impl)->RecvMany(pkts);
+}
+
 #endif
diff --git a/llarp/ev/ev_win32.hpp b/llarp/ev/ev_win32.hpp
index a111627be..e84dffec4 100644
--- a/llarp/ev/ev_win32.hpp
+++ b/llarp/ev/ev_win32.hpp
@@ -5,6 +5,7 @@
 #include <net/net.h>
 #include <net/net.hpp>
 #include <util/buffer.hpp>
+#include <util/thread/logic.hpp>
 
 #include <windows.h>
 #include <process.h>
@@ -36,6 +37,7 @@ namespace llarp
   struct udp_listener : public ev_io
   {
     llarp_udp_io* udp;
+    llarp_pkt_list m_RecvPackets;
 
     udp_listener(int fd, llarp_udp_io* u) : ev_io(fd), udp(u){};
 
@@ -43,6 +45,9 @@ namespace llarp
     {
     }
 
+    bool
+    RecvMany(llarp_pkt_list*);
+
     bool
     tick();
 
@@ -99,6 +104,8 @@ struct win32_tun_io
 struct llarp_win32_loop : public llarp_ev_loop
 {
   upoll_t* upollfd;
+  std::shared_ptr< llarp::Logic > m_Logic;
+  std::vector< std::function< void(void) > > m_Tickers;
 
   llarp_win32_loop() : upollfd(nullptr)
   {
@@ -148,6 +155,22 @@ struct llarp_win32_loop : public llarp_ev_loop
 
   void
   stop();
+
+  bool
+  add_ticker(std::function< void(void) > func) override
+  {
+    m_Tickers.emplace_back(func);
+    return true;
+  }
+
+  void
+  set_logic(std::shared_ptr< llarp::Logic > l) override
+  {
+    m_Logic = l;
+  }
+
+  void
+  tick_listeners() override;
 };
 
 #endif
diff --git a/llarp/iwp/linklayer.cpp b/llarp/iwp/linklayer.cpp
index aea1e223e..44d664fef 100644
--- a/llarp/iwp/linklayer.cpp
+++ b/llarp/iwp/linklayer.cpp
@@ -81,7 +81,7 @@ namespace llarp
       auto itr = m_AuthedAddrs.find(from);
       if(itr == m_AuthedAddrs.end())
       {
-        // ACQUIRE_LOCK(Lock_t lock , m_PendingMutex);
+        ACQUIRE_LOCK(Lock_t lock, m_PendingMutex);
         if(m_Pending.count(from) == 0)
         {
           if(not permitInbound)
@@ -92,7 +92,7 @@ namespace llarp
       }
       else
       {
-        Lock lock(&m_AuthedLinksMutex);
+        ACQUIRE_LOCK(Lock_t lock, m_AuthedLinksMutex);
         auto range = m_AuthedLinks.equal_range(itr->second);
         session    = range.first->second;
       }
diff --git a/llarp/iwp/message_buffer.cpp b/llarp/iwp/message_buffer.cpp
index d63f2a129..8cd20edf2 100644
--- a/llarp/iwp/message_buffer.cpp
+++ b/llarp/iwp/message_buffer.cpp
@@ -26,7 +26,7 @@ namespace llarp
       htobe16buf(xmit.data() + CommandOverhead + PacketOverhead, m_Data.size());
       htobe64buf(xmit.data() + 2 + CommandOverhead + PacketOverhead, m_MsgID);
       std::copy_n(m_Digest.begin(), m_Digest.size(),
-                  xmit.begin() + 10 + CommandOverhead + PacketOverhead);
+                  xmit.data() + 10 + CommandOverhead + PacketOverhead);
       return xmit;
     }
 
@@ -70,7 +70,7 @@ namespace llarp
           htobe16buf(frag.data() + 2 + PacketOverhead, idx);
           htobe64buf(frag.data() + 4 + PacketOverhead, m_MsgID);
           std::copy(m_Data.begin() + idx, m_Data.begin() + idx + fragsz,
-                    frag.begin() + PacketOverhead + Overhead + 2);
+                    frag.data() + PacketOverhead + Overhead + 2);
           sendpkt(std::move(frag));
         }
         idx += FragmentSize;
diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index ad76513ed..c5a36c340 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -363,7 +363,7 @@ namespace llarp
       ILinkSession::Packet_t req(Introduction::SIZE + PacketOverhead);
       const auto pk   = m_Parent->GetOurRC().pubkey;
       const auto e_pk = m_Parent->RouterEncryptionSecret().toPublic();
-      auto itr        = req.begin() + PacketOverhead;
+      auto itr        = req.data() + PacketOverhead;
       std::copy_n(pk.begin(), pk.size(), itr);
       itr += pk.size();
       std::copy_n(e_pk.begin(), e_pk.size(), itr);
@@ -373,9 +373,9 @@ namespace llarp
       llarp_buffer_t signbuf(req.data() + PacketOverhead,
                              Introduction::SIZE - Signature::SIZE);
       m_Parent->Sign(Z, signbuf);
-      std::copy_n(Z.begin(), Z.size(),
-                  req.begin() + PacketOverhead
-                      + (Introduction::SIZE - Signature::SIZE));
+      std::copy_n(
+          Z.begin(), Z.size(),
+          req.data() + PacketOverhead + (Introduction::SIZE - Signature::SIZE));
       CryptoManager::instance()->randbytes(req.data() + HMACSIZE, TUNNONCESIZE);
       EncryptAndSend(std::move(req));
       m_State = State::Introduction;
@@ -404,7 +404,7 @@ namespace llarp
                  token.size() + PacketOverhead, " from ", m_RemoteAddr);
         return;
       }
-      const auto begin = pkt.begin() + PacketOverhead;
+      const auto begin = pkt.data() + PacketOverhead;
       if(not std::equal(begin, begin + token.size(), token.begin()))
       {
         LogError("token missmatch from ", m_RemoteAddr);
@@ -450,12 +450,12 @@ namespace llarp
                  m_RemoteAddr);
         return;
       }
-      std::vector< byte_t > reply(token.size() + PacketOverhead);
+      Packet_t reply(token.size() + PacketOverhead);
       // random nonce
       CryptoManager::instance()->randbytes(reply.data() + HMACSIZE,
                                            TUNNONCESIZE);
       // set token
-      std::copy_n(token.begin(), token.size(), reply.begin() + PacketOverhead);
+      std::copy_n(token.begin(), token.size(), reply.data() + PacketOverhead);
       m_LastRX = m_Parent->Now();
       EncryptAndSend(std::move(reply));
       LogDebug("sent intro ack to ", m_RemoteAddr);
@@ -471,15 +471,15 @@ namespace llarp
                  token.size() + PacketOverhead, " from ", m_RemoteAddr);
         return;
       }
-      std::vector< byte_t > reply(token.size() + PacketOverhead);
+      Packet_t reply(token.size() + PacketOverhead);
       if(not DecryptMessageInPlace(pkt))
       {
         LogError("intro ack decrypt failed from ", m_RemoteAddr);
         return;
       }
       m_LastRX = m_Parent->Now();
-      std::copy_n(pkt.begin() + PacketOverhead, token.size(), token.begin());
-      std::copy_n(token.begin(), token.size(), reply.begin() + PacketOverhead);
+      std::copy_n(pkt.data() + PacketOverhead, token.size(), token.begin());
+      std::copy_n(token.begin(), token.size(), reply.data() + PacketOverhead);
       // random nounce
       CryptoManager::instance()->randbytes(reply.data() + HMACSIZE,
                                            TUNNONCESIZE);
@@ -533,13 +533,13 @@ namespace llarp
     {
       if(m_DecryptNext == nullptr)
         m_DecryptNext = std::make_shared< CryptoQueue_t >();
-      m_DecryptNext->emplace_back(pkt);
+      m_DecryptNext->emplace_back(std::move(pkt));
     }
 
     void
     Session::DecryptWorker(CryptoQueue_ptr msgs)
     {
-      CryptoQueue_t recvMsgs;
+      CryptoQueue_ptr recvMsgs = std::make_shared< CryptoQueue_t >();
       for(auto& pkt : *msgs)
       {
         if(not DecryptMessageInPlace(pkt))
@@ -553,17 +553,17 @@ namespace llarp
                    " != ", LLARP_PROTO_VERSION);
           continue;
         }
-        recvMsgs.emplace_back(std::move(pkt));
+        recvMsgs->emplace_back(std::move(pkt));
       }
-      LogDebug("decrypted ", recvMsgs.size(), " packets from ", m_RemoteAddr);
-      m_Parent->logic()->queue_func(std::bind(
-          &Session::HandlePlaintext, shared_from_this(), std::move(recvMsgs)));
+      LogDebug("decrypted ", recvMsgs->size(), " packets from ", m_RemoteAddr);
+      m_Parent->logic()->queue_func(
+          std::bind(&Session::HandlePlaintext, shared_from_this(), recvMsgs));
     }
 
     void
-    Session::HandlePlaintext(CryptoQueue_t msgs)
+    Session::HandlePlaintext(CryptoQueue_ptr msgs)
     {
-      for(auto& result : msgs)
+      for(auto& result : *msgs)
       {
         LogDebug("Command ", int(result[PacketOverhead + 1]));
         switch(result[PacketOverhead + 1])
@@ -598,7 +598,7 @@ namespace llarp
     }
 
     void
-    Session::HandleMACK(std::vector< byte_t > data)
+    Session::HandleMACK(Packet_t data)
     {
       if(data.size() < 3 + PacketOverhead)
       {
@@ -634,7 +634,7 @@ namespace llarp
     }
 
     void
-    Session::HandleNACK(std::vector< byte_t > data)
+    Session::HandleNACK(Packet_t data)
     {
       if(data.size() < CommandOverhead + sizeof(uint64_t) + PacketOverhead)
       {
@@ -653,7 +653,7 @@ namespace llarp
     }
 
     void
-    Session::HandleXMIT(std::vector< byte_t > data)
+    Session::HandleXMIT(Packet_t data)
     {
       if(data.size() < CommandOverhead + PacketOverhead + sizeof(uint16_t)
              + sizeof(uint64_t) + ShortHash::SIZE)
@@ -689,7 +689,7 @@ namespace llarp
     }
 
     void
-    Session::HandleDATA(std::vector< byte_t > data)
+    Session::HandleDATA(Packet_t data)
     {
       if(data.size() <= CommandOverhead + sizeof(uint16_t) + sizeof(uint64_t)
              + PacketOverhead)
@@ -744,7 +744,7 @@ namespace llarp
     }
 
     void
-    Session::HandleACKS(std::vector< byte_t > data)
+    Session::HandleACKS(Packet_t data)
     {
       if(data.size() < 11 + PacketOverhead)
       {
@@ -775,13 +775,13 @@ namespace llarp
       }
     }
 
-    void Session::HandleCLOS(std::vector< byte_t >)
+    void Session::HandleCLOS(Packet_t)
     {
       LogInfo("remote closed by ", m_RemoteAddr);
       Close();
     }
 
-    void Session::HandlePING(std::vector< byte_t >)
+    void Session::HandlePING(Packet_t)
     {
       m_LastRX = m_Parent->Now();
     }
@@ -791,8 +791,7 @@ namespace llarp
     {
       if(m_State == State::Ready)
       {
-        auto pkt = CreatePacket(Command::ePING, 0);
-        EncryptAndSend(std::move(pkt));
+        EncryptAndSend(CreatePacket(Command::ePING, 0));
         return true;
       }
       return false;
diff --git a/llarp/iwp/session.hpp b/llarp/iwp/session.hpp
index 7d121fe5d..657b153b9 100644
--- a/llarp/iwp/session.hpp
+++ b/llarp/iwp/session.hpp
@@ -5,6 +5,7 @@
 #include <iwp/linklayer.hpp>
 #include <iwp/message_buffer.hpp>
 #include <unordered_set>
+#include <deque>
 
 namespace llarp
 {
@@ -167,7 +168,7 @@ namespace llarp
       /// set of rx messages to send in next round of multiacks
       std::unordered_set< uint64_t > m_SendMACKs;
 
-      using CryptoQueue_t   = std::list< Packet_t >;
+      using CryptoQueue_t   = std::vector< Packet_t >;
       using CryptoQueue_ptr = std::shared_ptr< CryptoQueue_t >;
       CryptoQueue_ptr m_EncryptNext;
       CryptoQueue_ptr m_DecryptNext;
@@ -179,7 +180,7 @@ namespace llarp
       DecryptWorker(CryptoQueue_ptr msgs);
 
       void
-      HandlePlaintext(CryptoQueue_t msgs);
+      HandlePlaintext(CryptoQueue_ptr msgs);
 
       void
       HandleGotIntro(Packet_t pkt);
@@ -218,25 +219,25 @@ namespace llarp
       SendOurLIM(ILinkSession::CompletionHandler h = nullptr);
 
       void
-      HandleXMIT(std::vector< byte_t > msg);
+      HandleXMIT(Packet_t msg);
 
       void
-      HandleDATA(std::vector< byte_t > msg);
+      HandleDATA(Packet_t msg);
 
       void
-      HandleACKS(std::vector< byte_t > msg);
+      HandleACKS(Packet_t msg);
 
       void
-      HandleNACK(std::vector< byte_t > msg);
+      HandleNACK(Packet_t msg);
 
       void
-      HandlePING(std::vector< byte_t > msg);
+      HandlePING(Packet_t msg);
 
       void
-      HandleCLOS(std::vector< byte_t > msg);
+      HandleCLOS(Packet_t msg);
 
       void
-      HandleMACK(std::vector< byte_t > msg);
+      HandleMACK(Packet_t msg);
     };
   }  // namespace iwp
 }  // namespace llarp
diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp
index a8075894b..ccdf87dbf 100644
--- a/llarp/link/server.cpp
+++ b/llarp/link/server.cpp
@@ -1,5 +1,5 @@
 #include <link/server.hpp>
-
+#include <ev/ev.hpp>
 #include <crypto/crypto.hpp>
 #include <util/fs.hpp>
 #include <utility>
@@ -107,7 +107,7 @@ namespace llarp
   {
     m_Loop         = loop;
     m_udp.user     = this;
-    m_udp.recvfrom = &ILinkLayer::udp_recv_from;
+    m_udp.recvfrom = nullptr;
     m_udp.tick     = &ILinkLayer::udp_tick;
     if(ifname == "*")
     {
@@ -357,7 +357,7 @@ namespace llarp
   ILinkLayer::SendTo(const RouterID& remote, const llarp_buffer_t& buf,
                      ILinkSession::CompletionHandler completed)
   {
-    ILinkSession* s = nullptr;
+    std::shared_ptr< ILinkSession > s;
     {
       ACQUIRE_LOCK(Lock_t l, m_AuthedLinksMutex);
       auto range = m_AuthedLinks.equal_range(remote);
@@ -367,10 +367,10 @@ namespace llarp
 
       while(itr != range.second)
       {
-        auto backlog = itr->second->SendQueueBacklog();
+        const auto backlog = itr->second->SendQueueBacklog();
         if(backlog < min)
         {
-          s   = itr->second.get();
+          s   = itr->second;
           min = backlog;
         }
         ++itr;
@@ -463,31 +463,18 @@ namespace llarp
   ILinkLayer::udp_tick(llarp_udp_io* udp)
   {
     ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
-    if(link->m_Recv == nullptr)
-      return;
-    link->logic()->queue_func([traffic = std::move(link->m_Recv), l = link]() {
-      auto itr = traffic->begin();
-      while(itr != traffic->end())
+    auto pkts        = std::make_shared< llarp_pkt_list >();
+    llarp_ev_udp_recvmany(&link->m_udp, pkts.get());
+
+    link->logic()->queue_func([pkts, link]() {
+      auto itr = pkts->begin();
+      while(itr != pkts->end())
       {
-        l->RecvFrom(itr->first, std::move(itr->second));
+        link->RecvFrom(itr->remote, std::move(itr->pkt));
         ++itr;
       }
-      l->Pump();
+      link->Pump();
     });
-    link->m_Recv.reset(new TrafficQueue_t());
-  }
-
-  void
-  ILinkLayer::udp_recv_from(llarp_udp_io* udp, const sockaddr* from,
-                            ManagedBuffer buf)
-  {
-    ILinkLayer* link = static_cast< ILinkLayer* >(udp->user);
-    if(link->m_Recv == nullptr)
-      return;
-    link->m_Recv->emplace_back(
-        std::make_pair(Addr(*from), ILinkSession::Packet_t(buf.underlying.sz)));
-    std::copy_n(buf.underlying.base, buf.underlying.sz,
-                link->m_Recv->back().second.begin());
   }
 
 }  // namespace llarp
diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp
index b4fe17f1b..aa05d5e98 100644
--- a/llarp/link/server.hpp
+++ b/llarp/link/server.hpp
@@ -79,9 +79,6 @@ namespace llarp
     static void
     udp_tick(llarp_udp_io* udp);
 
-    static void
-    udp_recv_from(llarp_udp_io* udp, const sockaddr* from, ManagedBuffer buf);
-
     void
     SendTo_LL(const llarp::Addr& to, const llarp_buffer_t& pkt)
     {
diff --git a/llarp/link/session.hpp b/llarp/link/session.hpp
index 9ff8f0380..572ac9b0b 100644
--- a/llarp/link/session.hpp
+++ b/llarp/link/session.hpp
@@ -3,6 +3,7 @@
 
 #include <crypto/types.hpp>
 #include <net/net.hpp>
+#include <ev/ev.hpp>
 #include <router_contact.hpp>
 #include <util/types.hpp>
 
@@ -44,7 +45,7 @@ namespace llarp
     /// message delivery result hook function
     using CompletionHandler = std::function< void(DeliveryStatus) >;
 
-    using Packet_t  = std::vector< byte_t >;
+    using Packet_t  = PacketBuffer;
     using Message_t = std::vector< byte_t >;
 
     /// send a message buffer to the remote endpoint
diff --git a/llarp/util/logging/logger_internal.hpp b/llarp/util/logging/logger_internal.hpp
index 0a6090aa5..293569a99 100644
--- a/llarp/util/logging/logger_internal.hpp
+++ b/llarp/util/logging/logger_internal.hpp
@@ -8,7 +8,7 @@
 #include <ctime>
 #include <iomanip>
 #include <sstream>
-#include <thread>
+#include <util/thread/threading.hpp>
 
 namespace llarp
 {

From 605da68e155564fefd9c1425fe8000d61724448f Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Wed, 2 Oct 2019 09:17:12 -0400
Subject: [PATCH 24/27] use absl optional

---
 llarp/util/thread/logic.cpp | 2 +-
 llarp/util/thread/logic.hpp | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/llarp/util/thread/logic.cpp b/llarp/util/thread/logic.cpp
index 8a6c0d803..b27aa4d87 100644
--- a/llarp/util/thread/logic.cpp
+++ b/llarp/util/thread/logic.cpp
@@ -109,7 +109,7 @@ namespace llarp
   bool
   Logic::can_flush() const
   {
-    return id.has_value() && id.value() == std::this_thread::get_id();
+    return id.value() == std::this_thread::get_id();
   }
 
 }  // namespace llarp
diff --git a/llarp/util/thread/logic.hpp b/llarp/util/thread/logic.hpp
index 4373ba0d2..3a7ac25b3 100644
--- a/llarp/util/thread/logic.hpp
+++ b/llarp/util/thread/logic.hpp
@@ -4,6 +4,7 @@
 #include <util/mem.h>
 #include <util/thread/threadpool.h>
 #include <util/thread/timer.hpp>
+#include <absl/types/optional.h>
 
 namespace llarp
 {
@@ -12,7 +13,7 @@ namespace llarp
    public:
     struct llarp_threadpool* thread;
     struct llarp_timer_context* timer;
-    std::optional< std::thread::id > id;
+    absl::optional< std::thread::id > id;
 
     Logic();
 

From 7d949ebf5d9513594002cfc8a53c3424f15f25f5 Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Wed, 2 Oct 2019 11:35:20 -0400
Subject: [PATCH 25/27] always use absl for string_view

---
 llarp/util/string_view.hpp | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/llarp/util/string_view.hpp b/llarp/util/string_view.hpp
index e47b80906..8478ef927 100644
--- a/llarp/util/string_view.hpp
+++ b/llarp/util/string_view.hpp
@@ -1,21 +1,6 @@
 #ifndef LLARP_STRING_VIEW_HPP
 #define LLARP_STRING_VIEW_HPP
 
-#if __cplusplus >= 201703L
-#include <string_view>
-#include <string>
-namespace llarp
-{
-  using string_view      = std::string_view;
-  using string_view_hash = std::hash< string_view >;
-
-  static std::string
-  string_view_string(const string_view& v)
-  {
-    return std::string(v.data(), v.size());
-  }
-}  // namespace llarp
-#else
 #include <absl/hash/hash.h>
 #include <absl/strings/string_view.h>
 namespace llarp
@@ -30,4 +15,3 @@ namespace llarp
   }
 }  // namespace llarp
 #endif
-#endif

From 4af6dca24688bf291785cdbb02c3a2f1318bf104 Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Wed, 2 Oct 2019 11:35:33 -0400
Subject: [PATCH 26/27] use static_cast

---
 llarp/iwp/session.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llarp/iwp/session.cpp b/llarp/iwp/session.cpp
index c5a36c340..e928c5de9 100644
--- a/llarp/iwp/session.cpp
+++ b/llarp/iwp/session.cpp
@@ -205,7 +205,8 @@ namespace llarp
         auto numAcks   = std::min(sz, max);
         auto mack =
             CreatePacket(Command::eMACK, 1 + (numAcks * sizeof(uint64_t)));
-        mack[PacketOverhead + CommandOverhead] = byte_t{numAcks};
+        mack[PacketOverhead + CommandOverhead] =
+            byte_t{static_cast< byte_t >(numAcks)};
         byte_t* ptr = mack.data() + 3 + PacketOverhead;
         LogDebug("send ", numAcks, " macks to ", m_RemoteAddr);
         auto itr = m_SendMACKs.begin();

From 1e57397c9984bccade324cb5cb752f69635a26f1 Mon Sep 17 00:00:00 2001
From: jeff <jeff@i2p.rocks>
Date: Wed, 2 Oct 2019 11:40:26 -0400
Subject: [PATCH 27/27] revert to c++14 for non win32

---
 CMakeLists.txt | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 34c697536..71882d928 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -71,7 +71,12 @@ include(cmake/solaris.cmake)
 include(cmake/unix.cmake)
 include(cmake/win32.cmake)
 
-set(CMAKE_CXX_STANDARD 17)
+if(WIN32)
+  set(CMAKE_CXX_STANDARD 17)
+else()
+  set(CMAKE_CXX_STANDARD 14)
+endif(WIN32)
+
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_C_STANDARD 99)