more tun stuff

include/llarp/codel.hpp

@@ -74,15 +74,18 @@ namespace llarp
       bool
       EmplaceIf(std::function< bool(T*) > pred, Args&&... args)
       {
-        std::unique_ptr< T > ptr = std::make_unique< T >(args...);
-        if(!pred(ptr.get()))
+        T* ptr = new T(std::forward< Args >(args)...);
+        if(!pred(ptr))
+        {
+          delete ptr;
           return false;
-        PutTime()(ptr.get());
+        }
+        PutTime()(ptr);
         {
           Lock_t lock(m_QueueMutex);
           if(firstPut == 0)
-            firstPut = GetTime()(ptr.get());
-          m_Queue.push(std::move(ptr));
+            firstPut = GetTime()(ptr);
+          m_Queue.push(ptr);
         }
         return true;
       }
@@ -91,25 +94,25 @@ namespace llarp
       void
       Emplace(Args&&... args)
       {
-        std::unique_ptr< T > ptr = std::make_unique< T >(args...);
-        PutTime()(ptr.get());
+        T* ptr = new T(std::forward< Args >(args)...);
+        PutTime()(ptr);
         {
           Lock_t lock(m_QueueMutex);
           if(firstPut == 0)
-            firstPut = GetTime()(ptr.get());
-          m_Queue.push(std::move(ptr));
+            firstPut = GetTime()(ptr);
+          m_Queue.push(ptr);
         }
       }
 
       void
-      Put(std::unique_ptr< T >& ptr)
+      Put(T* ptr)
       {
-        PutTime()(ptr.get());
+        PutTime()(ptr);
         {
           Lock_t lock(m_QueueMutex);
           if(firstPut == 0)
-            firstPut = GetTime()(ptr.get());
-          m_Queue.push(std::move(ptr));
+            firstPut = GetTime()(ptr);
+          m_Queue.push(ptr);
         }
       }
 
@@ -117,10 +120,11 @@ namespace llarp
       void
       _sort(Q& queue)
       {
+        /*
         std::vector< std::unique_ptr< T > > q;
         while(queue.size())
         {
-          q.push_back(std::move(queue.front()));
+          q.emplace_back(std::move(queue.front()));
           queue.pop();
         }
         std::sort(q.begin(), q.end(), Compare());
@@ -130,6 +134,7 @@ namespace llarp
           queue.push(std::move(*itr));
           ++itr;
         }
+        */
       }
 
       /// visit returns true to discard entry otherwise the entry is
@@ -144,12 +149,12 @@ namespace llarp
         Lock_t lock(m_QueueMutex);
         _sort(m_Queue);
         auto start = firstPut;
-        std::queue< std::unique_ptr< T > > requeue;
+        std::queue< T* > requeue;
         while(m_Queue.size())
         {
           llarp::LogDebug("CoDelQueue::Process - queue has ", m_Queue.size());
-          auto& item = m_Queue.front();
-          auto dlt   = start - GetTime()(item.get());
+          T* item  = m_Queue.front();
+          auto dlt = start - GetTime()(item);
           // llarp::LogInfo("CoDelQueue::Process - dlt ", dlt);
           lowest = std::min(dlt, lowest);
           if(m_Queue.size() == 1)
@@ -160,6 +165,7 @@ namespace llarp
             {
               nextTickInterval += initialIntervalMs / std::sqrt(++dropNum);
               m_Queue.pop();
+              delete item;
               break;
             }
             else
@@ -172,7 +178,11 @@ namespace llarp
           if(!visitor(item))
           {
             // requeue item as we are not done
-            requeue.push(std::move(item));
+            requeue.push(item);
+          }
+          else
+          {
+            delete item;
           }
           m_Queue.pop();
         }
@@ -184,7 +194,7 @@ namespace llarp
       void
       Process(Func visitor)
       {
-        ProcessIf([visitor](const std::unique_ptr< T >& t) -> bool {
+        ProcessIf([visitor](T* t) -> bool {
           visitor(t);
           return true;
         });
@@ -194,7 +204,7 @@ namespace llarp
       size_t dropNum                = 0;
       llarp_time_t nextTickInterval = initialIntervalMs;
       Mutex_t m_QueueMutex;
-      std::queue< std::unique_ptr< T > > m_Queue;
+      std::queue< T* > m_Queue;
       std::string m_name;
     };
   }  // namespace util

include/llarp/handlers/tun.hpp

@@ -29,6 +29,9 @@ namespace llarp
       void
       TickTun(llarp_time_t now);
 
+      bool
+      MapAddress(const service::Address& remote, uint32_t ip);
+
       bool
       Start();
 
@@ -61,6 +64,11 @@ namespace llarp
       static void
       tunifBeforeWrite(llarp_tun_io* t);
 
+      /// handle user to network send buffer flush
+      /// called in router logic thread
+      static void
+      handleNetSend(void*);
+
       /// called every time we wish to read a packet from the tun interface
       static void
       tunifRecvPkt(llarp_tun_io* t, const void* pkt, ssize_t sz);
@@ -89,6 +97,9 @@ namespace llarp
       void
       MarkIPActive(uint32_t ip);
 
+      void
+      FlushSend();
+
      private:
 #ifndef _WIN32
       /// handles setup, given value true on success and false on failure to set

include/llarp/ip.hpp

@@ -53,9 +53,12 @@ namespace llarp
     struct IPv4Packet
     {
       static constexpr size_t MaxSize = 1500;
-      llarp_time_t timestamp          = 0;
-      size_t sz                       = 0;
-      byte_t buf[MaxSize]             = {0};
+      llarp_time_t timestamp;
+      size_t sz;
+      byte_t buf[MaxSize];
+
+      llarp_buffer_t
+      Buffer();
 
       bool
       Load(llarp_buffer_t buf);
@@ -91,25 +94,25 @@ namespace llarp
       iphdr*
       Header()
       {
-        return (iphdr*)buf;
+        return (iphdr*)&buf[0];
       }
 
       const iphdr*
       Header() const
       {
-        return (iphdr*)buf;
+        return (iphdr*)&buf[0];
       }
 
       uint32_t
       src()
       {
-        return ntohs(Header()->saddr);
+        return Header()->saddr;
       }
 
       uint32_t
       dst()
       {
-        return ntohs(Header()->daddr);
+        return Header()->daddr;
       }
 
       void

include/llarp/iwp/session.hpp

@@ -167,7 +167,7 @@ struct llarp_link_session
   void
   EncryptOutboundFrames();
 
-  std::unique_ptr< iwp_async_frame >
+  iwp_async_frame *
   alloc_frame(const void *buf, size_t sz);
   void
   decrypt_frame(const void *buf, size_t sz);

include/llarp/mem.hpp

@@ -36,13 +36,34 @@ namespace llarp
       {
         printf("%c[1;31m", 27);
       }
-      if(std::isprint(buff.base[idx]))
+      printf("%.2x", buff.base[idx]);
+      if(buff.base + idx == buff.cur)
       {
-        printf("%c", buff.base[idx]);
+        printf("%c[0;0m", 27);
       }
-      else
+      ++idx;
+      if(idx % align == 0)
+        printf("\n");
+    }
+    printf("\n");
+    fflush(stdout);
+  }
+
+  template < typename T, size_t align = 128 >
+  void
+  DumpBufferHex(const T &buff)
+  {
+    size_t idx = 0;
+    printf("buffer of size %zu\n", buff.sz);
+    while(idx < buff.sz)
+    {
+      if(buff.base + idx == buff.cur)
+      {
+        printf("%c[1;31m", 27);
+      }
+      if(std::isprint(buff.base[idx]))
       {
-        printf("X");
+        printf("%c", buff.base[idx]);
       }
       if(buff.base + idx == buff.cur)
       {

include/llarp/service/endpoint.hpp

@@ -120,6 +120,30 @@ namespace llarp
       void
       HandlePathBuilt(path::Path* path);
 
+      bool
+      SendToOrQueue(const Address& remote, llarp_buffer_t payload,
+                    ProtocolType t);
+
+      struct PendingBuffer
+      {
+        std::vector< byte_t > payload;
+        ProtocolType protocol;
+
+        PendingBuffer(llarp_buffer_t buf, ProtocolType t)
+            : payload(buf.sz), protocol(t)
+        {
+          memcpy(payload.data(), buf.base, buf.sz);
+        }
+
+        llarp_buffer_t
+        Buffer()
+        {
+          return llarp::InitBuffer(payload.data(), payload.size());
+        }
+      };
+
+      typedef std::queue< PendingBuffer > PendingBufferQueue;
+
       /// context needed to initiate an outbound hidden service session
       struct OutboundContext : public llarp_pathbuilder_context
       {
@@ -182,7 +206,7 @@ namespace llarp
 
       // passed a sendto context when we have a path established otherwise
       // nullptr if the path was not made before the timeout
-      typedef std::function< void(OutboundContext*) > PathEnsureHook;
+      typedef std::function< void(Address, OutboundContext*) > PathEnsureHook;
 
       /// return false if we have already called this function before for this
       /// address
@@ -287,7 +311,11 @@ namespace llarp
       std::string m_Name;
       std::string m_NetNS;
 
-      std::unordered_map< Address, OutboundContext*, Address::Hash >
+      std::unordered_map< Address, PendingBufferQueue, Address::Hash >
+          m_PendingTraffic;
+
+      std::unordered_map< Address, std::unique_ptr< OutboundContext >,
+                          Address::Hash >
           m_RemoteSessions;
       std::unordered_map< Address, PathEnsureHook, Address::Hash >
           m_PendingServiceLookups;

llarp/ev.hpp

@@ -53,21 +53,20 @@ namespace llarp
     virtual void
     flush_write()
     {
-      m_writeq.ProcessIf(
-          [&](const std::unique_ptr< WriteBuffer >& buffer) -> bool {
+      m_writeq.ProcessIf([&](WriteBuffer* buffer) -> bool {
       // todo: wtf???
 #ifndef _WIN32
-            if(write(fd, buffer->buf, buffer->bufsz) == -1)
-            {
-              // if we would block we save the entries for later
-              return errno == EWOULDBLOCK || errno == EAGAIN;
-            }
-            // discard entry
-            return true;
+        if(write(fd, buffer->buf, buffer->bufsz) == -1)
+        {
+          // if we would block we save the entries for later
+          return errno == EWOULDBLOCK || errno == EAGAIN;
+        }
+        // discard entry
+        return true;
 #else
       // writefile
 #endif
-          });
+      });
       /// reset errno
       errno = 0;
     }

llarp/ev_epoll.hpp

@@ -8,9 +8,11 @@
 #include <tuntap.h>
 #include <unistd.h>
 #include <cstdio>
+#include "buffer.hpp"
 #include "ev.hpp"
 #include "llarp/net.hpp"
 #include "logger.hpp"
+#include "mem.hpp"
 
 namespace llarp
 {
@@ -95,23 +97,33 @@ namespace llarp
     {
       ssize_t ret = tuntap_read(tunif, buf, sz);
       if(ret > 0 && t->recvpkt)
+      {
         t->recvpkt(t, buf, ret);
+      }
       return ret;
     }
 
     bool
     setup()
     {
-      llarp::LogDebug("set up tunif");
+      llarp::LogDebug("set ifname to ", t->ifname);
+      strncpy(tunif->if_name, t->ifname, sizeof(tunif->if_name));
+
       if(tuntap_start(tunif, TUNTAP_MODE_TUNNEL, 0) == -1)
+      {
+        llarp::LogWarn("failed to start interface");
         return false;
-      llarp::LogDebug("set ifname to ", t->ifname);
-      if(tuntap_set_ifname(tunif, t->ifname) == -1)
+      }
+      if(tuntap_up(tunif) == -1)
+      {
+        llarp::LogWarn("failed to put interface up: ", strerror(errno));
         return false;
+      }
       if(tuntap_set_ip(tunif, t->ifaddr, t->netmask) == -1)
+      {
+        llarp::LogWarn("failed to set ip");
         return false;
-      if(tuntap_up(tunif) == -1)
-        return false;
+      }
       fd = tunif->tun_fd;
       if(fd == -1)
         return false;

llarp/handlers/tun.cpp

@@ -22,6 +22,32 @@ namespace llarp
     bool
     TunEndpoint::SetOption(const std::string &k, const std::string &v)
     {
+      if(k == "mapaddr")
+      {
+        auto pos = v.find(":");
+        if(pos == std::string::npos)
+        {
+          llarp::LogError("Cannot map address ", v,
+                          " invalid format, expects "
+                          "address.loki:ip.address.goes.here");
+          return false;
+        }
+        service::Address addr;
+        auto addr_str = v.substr(0, pos);
+        if(!addr.FromString(addr_str))
+        {
+          llarp::LogError("cannot map invalid address ", addr_str);
+          return false;
+        }
+        auto ip_str = v.substr(pos + 1);
+        uint32_t ip;
+        if(inet_pton(AF_INET, ip_str.c_str(), &ip) != 1)
+        {
+          llarp::LogError("cannot map to invalid ip ", ip_str);
+          return false;
+        }
+        return MapAddress(addr, ip);
+      }
       if(k == "ifname")
       {
         strncpy(tunif.ifname, v.c_str(), sizeof(tunif.ifname) - 1);
@@ -50,12 +76,32 @@ namespace llarp
           tunif.netmask = 32;
           addr          = v;
         }
+        llarp::LogInfo("set ifaddr to ", addr, " with netmask ", tunif.netmask);
         strncpy(tunif.ifaddr, addr.c_str(), sizeof(tunif.ifaddr) - 1);
         return true;
       }
       return Endpoint::SetOption(k, v);
     }
 
+    bool
+    TunEndpoint::MapAddress(const service::Address &addr, uint32_t ip)
+    {
+      char buf[32] = {0};
+      inet_ntop(AF_INET, &ip, buf, sizeof(buf));
+      auto itr = m_IPToAddr.find(ip);
+      if(itr != m_IPToAddr.end())
+      {
+        llarp::LogWarn(buf, " already mapped to ", itr->second.ToString());
+        return false;
+      }
+      llarp::LogInfo("map ", addr.ToString(), " to ", buf);
+      m_IPToAddr.insert(std::make_pair(ip, addr));
+      m_AddrToIP.insert(std::make_pair(addr, ip));
+      // TODO: make ip mapping persist forever
+      MarkIPActive(ip);
+      return true;
+    }
+
     bool
     TunEndpoint::Start()
     {
@@ -124,9 +170,28 @@ namespace llarp
     {
       // call tun code in endpoint logic in case of network isolation
       llarp_logic_queue_job(EndpointLogic(), {this, handleTickTun});
+      FlushSend();
       Endpoint::Tick(now);
     }
 
+    void
+    TunEndpoint::FlushSend()
+    {
+      m_UserToNetworkPktQueue.Process([&](net::IPv4Packet *pkt) {
+        auto itr = m_IPToAddr.find(pkt->dst());
+        if(itr == m_IPToAddr.end())
+        {
+          in_addr a;
+          a.s_addr = pkt->dst();
+          llarp::LogWarn("drop packet to ", inet_ntoa(a));
+          llarp::DumpBuffer(pkt->Buffer());
+          return true;
+        }
+        return SendToOrQueue(itr->second, pkt->Buffer(),
+                             service::eProtocolTraffic);
+      });
+    }
+
     void
     TunEndpoint::HandleDataMessage(service::ProtocolMessage *msg)
     {
@@ -158,14 +223,18 @@ namespace llarp
     uint32_t
     TunEndpoint::ObtainIPForAddr(const service::Address &addr)
     {
+      llarp_time_t now = llarp_time_now_ms();
+      uint32_t nextIP;
       {
         // previously allocated address
         auto itr = m_AddrToIP.find(addr);
         if(itr != m_AddrToIP.end())
+        {
+          // mark ip active
+          m_IPActivity[itr->second] = now;
           return itr->second;
+        }
       }
-      llarp_time_t now = llarp_time_now_ms();
-      uint32_t nextIP;
       if(m_NextIP < m_MaxIP)
       {
         nextIP = ++m_NextIP;
@@ -236,11 +305,19 @@ namespace llarp
     {
       // called in the isolated network thread
       TunEndpoint *self = static_cast< TunEndpoint * >(tun->user);
-      self->m_NetworkToUserPktQueue.Process(
-          [tun](const std::unique_ptr< net::IPv4Packet > &pkt) {
-            if(!llarp_ev_tun_async_write(tun, pkt->buf, pkt->sz))
-              llarp::LogWarn("packet dropped");
-          });
+      self->m_NetworkToUserPktQueue.Process([self, tun](net::IPv4Packet *pkt) {
+        if(!llarp_ev_tun_async_write(tun, pkt->buf, pkt->sz))
+          llarp::LogWarn("packet dropped");
+      });
+      if(self->m_UserToNetworkPktQueue.Size())
+        llarp_logic_queue_job(self->RouterLogic(), {self, &handleNetSend});
+    }
+
+    void
+    TunEndpoint::handleNetSend(void *user)
+    {
+      TunEndpoint *self = static_cast< TunEndpoint * >(user);
+      self->FlushSend();
     }
 
     void
@@ -250,8 +327,9 @@ namespace llarp
       TunEndpoint *self = static_cast< TunEndpoint * >(tun->user);
       llarp::LogDebug("got pkt ", sz, " bytes");
       if(!self->m_UserToNetworkPktQueue.EmplaceIf(
-             [buf, sz](net::IPv4Packet *pkt) -> bool {
-               return pkt->Load(llarp::InitBuffer(buf, sz));
+             [self, buf, sz](net::IPv4Packet *pkt) -> bool {
+               return pkt->Load(llarp::InitBuffer(buf, sz))
+                   && pkt->Header()->version == 4;
              }))
         llarp::LogError("Failed to parse ipv4 packet");
     }

llarp/ip.cpp

@@ -1,5 +1,7 @@
 #include <llarp/endian.h>
 #include <llarp/ip.hpp>
+#include "buffer.hpp"
+#include "mem.hpp"
 
 namespace llarp
 {
@@ -8,10 +10,18 @@ namespace llarp
     bool
     IPv4Packet::Load(llarp_buffer_t pkt)
     {
-      memcpy(buf, pkt.base, std::min(pkt.sz, sizeof(buf)));
+      sz = std::min(pkt.sz, sizeof(buf));
+      memcpy(buf, pkt.base, sz);
+      llarp::DumpBufferHex(pkt);
       return true;
     }
 
+    llarp_buffer_t
+    IPv4Packet::Buffer()
+    {
+      return llarp::InitBuffer(buf, sz);
+    }
+
     void
     IPv4Packet::UpdateChecksum()
     {

llarp/iwp/frame_state.cpp

@@ -23,7 +23,7 @@ bool
 frame_state::process_inbound_queue()
 {
   uint64_t last = 0;
-  recvqueue.Process([&](const std::unique_ptr< InboundMessage > &msg) {
+  recvqueue.Process([&](InboundMessage *msg) {
     if(last != msg->msgid)
     {
       auto buffer = msg->Buffer();
@@ -181,8 +181,8 @@ void
 frame_state::push_ackfor(uint64_t id, uint32_t bitmask)
 {
   llarp::LogDebug("ACK for msgid=", id, " mask=", bitmask);
-  auto pkt      = std::unique_ptr< sendbuf_t >(new sendbuf_t(12 + 6));
-  auto body_ptr = init_sendbuf(pkt.get(), eACKS, 12, txflags);
+  auto pkt      = new sendbuf_t(12 + 6);
+  auto body_ptr = init_sendbuf(pkt, eACKS, 12, txflags);
   htobe64buf(body_ptr, id);
   htobe32buf(body_ptr + 8, bitmask);
   sendqueue.Put(pkt);
@@ -232,9 +232,7 @@ frame_state::inbound_frame_complete(uint64_t id)
     }
     else
     {
-      std::unique_ptr< InboundMessage > m =
-          std::unique_ptr< InboundMessage >(new InboundMessage(id, msg));
-      recvqueue.Put(m);
+      recvqueue.Emplace(id, msg);
       success = true;
     }
   }

llarp/iwp/session.cpp

@@ -400,9 +400,8 @@ llarp_link_session::get_parent()
 void
 llarp_link_session::TickLogic(llarp_time_t now)
 {
-  decryptedFrames.Process([&](const std::unique_ptr< iwp_async_frame > &msg) {
-    handle_frame_decrypt(msg.get());
-  });
+  decryptedFrames.Process(
+      [&](iwp_async_frame *msg) { handle_frame_decrypt(msg); });
   frame.process_inbound_queue();
   frame.retransmit(now);
   pump();
@@ -446,9 +445,9 @@ llarp_link_session::keepalive()
 void
 llarp_link_session::EncryptOutboundFrames()
 {
-  outboundFrames.Process([&](const std::unique_ptr< iwp_async_frame > &frame) {
-    if(iwp_encrypt_frame(frame.get()))
-      handle_frame_encrypt(frame.get());
+  outboundFrames.Process([&](iwp_async_frame *frame) {
+    if(iwp_encrypt_frame(frame))
+      handle_frame_encrypt(frame);
   });
 }
 
@@ -595,7 +594,7 @@ llarp_link_session::decrypt_frame(const void *buf, size_t sz)
     // inboundFrames.Put(frame);
     auto f = alloc_frame(buf, sz);
 
-    if(iwp_decrypt_frame(f.get()))
+    if(iwp_decrypt_frame(f))
     {
       decryptedFrames.Put(f);
     }
@@ -741,7 +740,7 @@ llarp_link_session::recv(const void *buf, size_t sz)
   }
 }
 
-std::unique_ptr< iwp_async_frame >
+iwp_async_frame *
 llarp_link_session::alloc_frame(const void *buf, size_t sz)
 {
   // TODO don't hard code 1500
@@ -762,7 +761,7 @@ llarp_link_session::alloc_frame(const void *buf, size_t sz)
   // frame->created = now;
   // llarp::LogInfo("alloc_frame putting into q");
   // q.Put(frame);
-  return std::unique_ptr< iwp_async_frame >(frame);
+  return frame;
 }
 
 void
@@ -784,7 +783,7 @@ void
 llarp_link_session::pump()
 {
   bool flush = false;
-  frame.sendqueue.Process([&](const std::unique_ptr< sendbuf_t > &msg) {
+  frame.sendqueue.Process([&](sendbuf_t *msg) {
     llarp_buffer_t buf = msg->Buffer();
     encrypt_frame_async_send(buf.base, buf.sz);
     flush = true;

llarp/iwp/transit_message.cpp

@@ -109,8 +109,8 @@ void
 transit_message::generate_xmit(sendqueue_t &queue, byte_t flags)
 {
   uint16_t sz   = lastfrag.size() + sizeof(msginfo.buffer);
-  auto pkt      = std::unique_ptr< sendbuf_t >(new sendbuf_t(sz + 6));
-  auto body_ptr = init_sendbuf(pkt.get(), eXMIT, sz, flags);
+  auto pkt      = new sendbuf_t(sz + 6);
+  auto body_ptr = init_sendbuf(pkt, eXMIT, sz, flags);
   memcpy(body_ptr, msginfo.buffer, sizeof(msginfo.buffer));
   body_ptr += sizeof(msginfo.buffer);
   memcpy(body_ptr, lastfrag.data(), lastfrag.size());
@@ -128,8 +128,8 @@ transit_message::retransmit_frags(sendqueue_t &queue, byte_t flags)
     if(status.test(frag.first))
       continue;
     uint16_t sz   = 9 + fragsize;
-    auto pkt      = std::unique_ptr< sendbuf_t >(new sendbuf_t(sz + 6));
-    auto body_ptr = init_sendbuf(pkt.get(), eFRAG, sz, flags);
+    auto pkt      = new sendbuf_t(sz + 6);
+    auto body_ptr = init_sendbuf(pkt, eFRAG, sz, flags);
     htobe64buf(body_ptr, msgid);
     body_ptr[8] = frag.first;
     memcpy(body_ptr + 9, frag.second.data(), fragsize);

llarp/service/endpoint.cpp

@@ -71,27 +71,6 @@ namespace llarp
       return m_IsolatedLogic && m_IsolatedWorker;
     }
 
-    struct PathAlignJob
-    {
-      void
-      HandleResult(Endpoint::OutboundContext* context)
-      {
-        if(context)
-        {
-          byte_t tmp[128] = {0};
-          memcpy(tmp, "BEEP", 4);
-          auto buf = llarp::StackBuffer< decltype(tmp) >(tmp);
-          buf.sz   = 4;
-          context->AsyncEncryptAndSendTo(buf, eProtocolText);
-        }
-        else
-        {
-          llarp::LogWarn("PathAlignJob timed out");
-        }
-        delete this;
-      }
-    };
-
     bool
     Endpoint::SetupIsolatedNetwork(void* user, bool failed)
     {
@@ -180,14 +159,10 @@ namespace llarp
       {
         if(!HasPathToService(addr))
         {
-          PathAlignJob* j = new PathAlignJob();
-          if(!EnsurePathToService(addr,
-                                  std::bind(&PathAlignJob::HandleResult, j,
-                                            std::placeholders::_1),
-                                  10000))
+          if(!EnsurePathToService(
+                 addr, [](Address addr, OutboundContext* ctx) {}, 10000))
           {
             llarp::LogWarn("failed to ensure path to ", addr);
-            delete j;
           }
         }
       }
@@ -206,15 +181,12 @@ namespace llarp
         {
           if(HasPendingPathToService(introset.A.Addr()))
             continue;
-          PathAlignJob* j = new PathAlignJob();
           if(!EnsurePathToService(introset.A.Addr(),
-                                  std::bind(&PathAlignJob::HandleResult, j,
-                                            std::placeholders::_1),
+                                  [](Address addr, OutboundContext* ctx) {},
                                   10000))
           {
             llarp::LogWarn("failed to ensure path to ", introset.A.Addr(),
                            " for tag ", tag.ToString());
-            delete j;
           }
         }
         itr->second.Expire(now);
@@ -236,7 +208,6 @@ namespace llarp
         {
           if(itr->second->Tick(now))
           {
-            delete itr->second;
             itr = m_RemoteSessions.erase(itr);
           }
           else
@@ -605,7 +576,7 @@ namespace llarp
       {
         auto f = itr->second;
         m_PendingServiceLookups.erase(itr);
-        f(m_RemoteSessions.at(addr));
+        f(itr->first, m_RemoteSessions.at(addr).get());
       }
     }
 
@@ -714,7 +685,7 @@ namespace llarp
         auto itr = m_RemoteSessions.find(remote);
         if(itr != m_RemoteSessions.end())
         {
-          hook(itr->second);
+          hook(itr->first, itr->second.get());
           return true;
         }
       }
@@ -766,6 +737,44 @@ namespace llarp
       return true;
     }
 
+    bool
+    Endpoint::SendToOrQueue(const Address& remote, llarp_buffer_t data,
+                            ProtocolType t)
+    {
+      if(HasPathToService(remote))
+      {
+        m_RemoteSessions[remote]->AsyncEncryptAndSendTo(data, t);
+        return true;
+      }
+
+      auto itr = m_PendingTraffic.find(remote);
+      if(itr == m_PendingTraffic.end())
+      {
+        m_PendingTraffic.insert(std::make_pair(remote, PendingBufferQueue()));
+        EnsurePathToService(remote,
+                            [&](Address addr, OutboundContext* ctx) {
+                              if(ctx)
+                              {
+                                auto itr = m_PendingTraffic.find(addr);
+                                if(itr != m_PendingTraffic.end())
+                                {
+                                  while(itr->second.size())
+                                  {
+                                    auto& front = itr->second.front();
+                                    ctx->AsyncEncryptAndSendTo(front.Buffer(),
+                                                               front.protocol);
+                                    itr->second.pop();
+                                  }
+                                }
+                              }
+                              m_PendingTraffic.erase(addr);
+                            },
+                            10000);
+      }
+      m_PendingTraffic[remote].emplace(data, t);
+      return true;
+    }
+
     void
     Endpoint::OutboundContext::ShiftIntroduction()
     {

llarp/win32_inet.c

@@ -10,9 +10,9 @@
 
 // these need to be in a specific order
 #include <assert.h>
+#include <iphlpapi.h>
 #include <llarp/net.h>
 #include <windows.h>
-#include <iphlpapi.h>
 #include "win32_intrnl.h"
 
 const char *

llarp/win32_intrnl.c

@@ -16,9 +16,9 @@
 #endif
 
 // these need to be in a specific order
+#include <tdi.h>
 #include <windows.h>
 #include <winternl.h>
-#include <tdi.h>
 #include "win32_intrnl.h"
 
 const PWCHAR TcpFileName = L"\\Device\\Tcp";