Merge pull request #384 from majestrate/staging

Staging

CMakeLists.txt

@@ -347,8 +347,9 @@ add_log_tag(${ABYSS_EXE})
 add_log_tag(${ABYSS_LIB})
 
 if(SHADOW)
-  add_shadow_plugin(shadow-plugin-${SHARED_LIB} ${EXE_SRC} ${LIB_SRC} ${UTP_SRC} ${LIB_PLATFORM_SRC} ${CPP_BACKPORT_SRC} ${ABYSS_SRC} ${CRYPTOGRAPHY_SRC})
+  add_shadow_plugin(shadow-plugin-${SHARED_LIB} ${EXE_SRC})
-  target_link_libraries(shadow-plugin-${SHARED_LIB} ${LIBS})
+  target_include_directories(shadow-plugin-${SHARED_LIB} PUBLIC ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/llarp ${PROJECT_SOURCE_DIR}/crypto/include ${PROJECT_SOURCE_DIR}/vendor/cppbackport-master/lib)
+  target_link_libraries(shadow-plugin-${SHARED_LIB} ${STATIC_LIB} ${LIBS})
   install(TARGETS shadow-plugin-${SHARED_LIB} DESTINATION plugins)
 else()
   if(NOT WIN32)

Makefile

@@ -16,12 +16,12 @@ SETCAP ?= which setcap && setcap cap_net_admin,cap_net_bind_service=+eip
 SHADOW_ROOT ?= $(HOME)/.shadow
 SHADOW_BIN=$(SHADOW_ROOT)/bin/shadow
 SHADOW_CONFIG=$(REPO)/shadow.config.xml
-SHADOW_PLUGIN=$(REPO)/libshadow-plugin-llarp.so
+SHADOW_PLUGIN=$(BUILD_ROOT)/libshadow-plugin-lokinet-shared.so
 SHADOW_LOG=$(REPO)/shadow.log.txt
 
 SHADOW_SRC ?= $(HOME)/local/shadow
-SHADOW_PARSE ?= python $(SHADOW_SRC)/src/tools/parse-shadow.py - -m 0 --packet-data
+SHADOW_PARSE ?= $(PYTHON) $(SHADOW_SRC)/src/tools/parse-shadow.py - -m 0 --packet-data
-SHADOW_PLOT ?= python $(SHADOW_SRC)/src/tools/plot-shadow.py -d $(REPO) LokiNET -c $(SHADOW_CONFIG) -r 10000 -e '.*'
+SHADOW_PLOT ?= $(PYTHON) $(SHADOW_SRC)/src/tools/plot-shadow.py -d $(REPO) LokiNET -c $(SHADOW_CONFIG) -r 10000 -e '.*'
 
 TESTNET_ROOT=/tmp/lokinet_testnet_tmp
 TESTNET_CONF=$(TESTNET_ROOT)/supervisor.conf
@@ -135,11 +135,11 @@ shadow-configure: clean
 	$(CONFIG_CMD) -DCMAKE_BUILD_TYPE=Debug -DSHADOW=ON -DCMAKE_C_COMPILER=$(CC) -DCMAKE_CXX_COMPILER=$(CXX)
 
 shadow-build: shadow-configure
-	$(MAKE) -C $(BUILD_ROOT) clean
 	$(MAKE) -C $(BUILD_ROOT)
 
 shadow-run: shadow-build
 	$(PYTHON) $(REPO)/contrib/shadow/genconf.py $(SHADOW_CONFIG)
+	cp $(SHADOW_PLUGIN) $(REPO)
 	bash -c "$(SHADOW_BIN) -w $$(cat /proc/cpuinfo | grep processor | wc -l) $(SHADOW_CONFIG) | $(SHADOW_PARSE)"
 
 shadow-plot: shadow-run

contrib/shadow/genconf.py

@@ -13,7 +13,7 @@ def getSetting(s, name, fallback): return name in s and s[name] or fallback
 shadowRoot = getSetting(os.environ, "SHADOW_ROOT",
                         os.path.join(os.environ['HOME'], '.shadow'))
 
-libpath = 'libshadow-plugin-lokinet.so'
+libpath = 'libshadow-plugin-lokinet-shared.so'
 
 
 def nodeconf(conf, baseDir, name, ifname=None, port=None):
@@ -59,11 +59,11 @@ def makeClient(settings, name, id):
     basedir = getSetting(settings, 'baseDir', 'tmp')
     nodeconf(peer['config'], basedir, name)
     fname = os.path.join(basedir, "test-service.ini")
-    peer['config']['services'] = {
+    peer['config']['network'] = {
-        'test-service': fname
+        'type': 'null',
+        'tag':'test',
+        'prefetch-tag':'test'
     }
-    with open(fname, 'w') as f:
-        f.write("[test-service]")
     return peer
 
 
@@ -71,6 +71,9 @@ def makeSVCNode(settings, name, id, port):
     peer = makeBase(settings, name, id)
     nodeconf(peer['config'], getSetting(
         settings, 'baseDir', 'tmp'), name, 'eth0', port)
+    peer['config']['network'] = {
+        'type': 'null'
+    }
     return peer
 
 
@@ -81,7 +84,7 @@ def genconf(settings, outf):
     topology.attrib['path'] = getSetting(settings, 'topology', os.path.join(
         shadowRoot, 'share', 'topology.graphml.xml'))
 
-    pluginName = getSetting(settings, 'name', 'llarpd')
+    pluginName = getSetting(settings, 'name', 'lokinet-shared')
 
     kill = etree.SubElement(root, 'kill')
     kill.attrib['time'] = getSetting(settings, 'runFor', '600')
@@ -96,7 +99,7 @@ def genconf(settings, outf):
     plugin.attrib['id'] = pluginName
     plugin.attrib['path'] = libpath
     basePort = getSetting(settings, 'svc-base-port', 19000)
-    svcNodeCount = getSetting(settings, 'service-nodes', 20)
+    svcNodeCount = getSetting(settings, 'service-nodes', 80)
     peers = list()
     for nodeid in range(svcNodeCount):
         peers.append(makeSVCNode(

llarp/ev/ev_kqueue.cpp

@@ -373,12 +373,6 @@ llarp_kqueue_loop::tick(int ms)
     {
       llarp::ev_io* ev = static_cast< llarp::ev_io* >(events[idx].udata);
       if(ev)
-      {
-        if(events[idx].filter & EV_ERROR)
-        {
-          ev->error();
-        }
-        else if((events[idx].filter & EV_EOF) == 0)
       {
         if(events[idx].filter & EVFILT_WRITE)
         {
@@ -390,7 +384,6 @@ llarp_kqueue_loop::tick(int ms)
                    std::min(sizeof(readbuf), size_t(events[idx].data)));
         }
       }
-      }
       ++idx;
     }
   }

llarp/exit/session.cpp

@@ -110,25 +110,34 @@ namespace llarp
     bool
     BaseSession::HandleGotExit(llarp::path::Path* p, llarp_time_t b)
     {
-      BaseSession* self = this;
       m_LastUse = router->Now();
       if(b == 0)
         llarp::LogInfo("obtained an exit via ", p->Endpoint());
-      else
+      if(IsReady())
-        self = nullptr;
+        CallPendingCallbacks(true);
-
-      for(auto& f : m_PendingCallbacks)
-        f(self);
-      m_PendingCallbacks.clear();
       return true;
     }
 
+    void
+    BaseSession::CallPendingCallbacks(bool success)
+    {
+      if(success)
+      {
+        for(auto& f : m_PendingCallbacks)
+          f(this);
+      }
+      else
+      {
+        for(auto& f : m_PendingCallbacks)
+          f(nullptr);
+      }
+      m_PendingCallbacks.clear();
+    }
+
     bool
     BaseSession::Stop()
     {
-      for(auto& f : m_PendingCallbacks)
+      CallPendingCallbacks(false);
-        f(nullptr);
-      m_PendingCallbacks.clear();
       auto sendExitClose = [&](llarp::path::Path* p) {
         if(p->SupportsAnyRoles(llarp::path::ePathRoleExit))
         {
@@ -199,7 +208,8 @@ namespace llarp
     bool
     BaseSession::IsReady() const
     {
-      return AvailablePaths(llarp::path::ePathRoleExit) > 0;
+      const size_t expect = (1 + (m_NumPaths / 2));
+      return AvailablePaths(llarp::path::ePathRoleExit) >= expect;
     }
 
     bool

llarp/exit/session.hpp

@@ -49,6 +49,12 @@ namespace llarp
       bool
       Flush();
 
+      path::PathRole
+      GetRoles() const override
+      {
+        return path::ePathRoleExit;
+      }
+
       /// send close and stop session
       bool
       Stop() override;
@@ -116,6 +122,9 @@ namespace llarp
       llarp_time_t m_LastUse;
 
       std::vector< SessionReadyFunc > m_PendingCallbacks;
+
+      void
+      CallPendingCallbacks(bool success);
     };
 
     struct ExitSession final : public BaseSession

llarp/handlers/tun.cpp

@@ -343,12 +343,6 @@ namespace llarp
         }
         else if(addr.FromString(qname, ".snode"))
         {
-          if(HasPathToSNode(addr.as_array()))
-          {
-            msg.AddINReply(ObtainIPForAddr(addr, true));
-            reply(msg);
-            return true;
-          }
           dns::Message *replyMsg = new dns::Message(std::move(msg));
           EnsurePathToSNode(addr.as_array(),
                             [=](const RouterID &remote, exit::BaseSession *s) {

llarp/link/iwp.cpp

@@ -6,62 +6,149 @@ namespace llarp
 {
   namespace iwp
   {
+    std::array< byte_t, 6 > OuterMessage::obtain_flow_id_magic =
+        std::array< byte_t, 6 >{'n', 'e', 't', 'i', 'd', '?'};
+
+    std::array< byte_t, 6 > OuterMessage::give_flow_id_magic =
+        std::array< byte_t, 6 >{'n', 'e', 't', 'i', 'd', '!'};
+
+    OuterMessage::OuterMessage()
+    {
+      Clear();
+    }
+
+    OuterMessage::~OuterMessage()
+    {
+    }
+
     void
     OuterMessage::Clear()
     {
       command = 0;
       flow.Zero();
       netid.Zero();
-      nextFlowID.Zero();
+      reject.fill(0);
-      rejectReason.clear();
       N.Zero();
       X.Zero();
       Xsize = 0;
-      Z.Zero();
+      Zsig.Zero();
+      Zhash.Zero();
+      pubkey.Zero();
+      magic.fill(0);
+      uinteger = 0;
+      A.reset();
+    }
+
+    void
+    OuterMessage::CreateReject(const char* msg, llarp_time_t now,
+                               const PubKey& pk)
+    {
+      Clear();
+      std::copy_n(msg, std::min(strlen(msg), reject.size()), reject.begin());
+      uinteger = now;
+      pubkey   = pk;
+    }
+
+    bool
+    OuterMessage::Encode(llarp_buffer_t* buf) const
+    {
+      if(buf->size_left() < 2)
+        return false;
+      *buf->cur = command;
+      buf->cur++;
+      *buf->cur = '=';
+      buf->cur++;
+      switch(command)
+      {
+        case eOCMD_ObtainFlowID:
+
+        case eOCMD_GiveFlowID:
+          if(!buf->write(reject.begin(), reject.end()))
+            return false;
+          if(!buf->write(give_flow_id_magic.begin(), give_flow_id_magic.end()))
+            return false;
+          if(!buf->write(flow.begin(), flow.end()))
+            return false;
+          if(!buf->write(pubkey.begin(), pubkey.end()))
+            return false;
+          return buf->write(Zsig.begin(), Zsig.end());
+        default:
+          return false;
+      }
     }
 
     bool
     OuterMessage::Decode(llarp_buffer_t* buf)
     {
-      if(buf->size_left() < 34)
+      static constexpr size_t header_size = 2;
+
+      if(buf->size_left() < header_size)
         return false;
       command = *buf->cur;
       ++buf->cur;
       if(*buf->cur != '=')
         return false;
-      std::copy_n(flow.begin(), 32, buf->cur);
+      ++buf->cur;
-      buf->cur += 32;
       switch(command)
       {
         case eOCMD_ObtainFlowID:
-          if(buf->size_left() < 40)
+          if(!buf->read_into(magic.begin(), magic.end()))
             return false;
-          buf->cur += 32;
+          if(!buf->read_into(netid.begin(), netid.end()))
-          std::copy_n(netid.begin(), 8, buf->cur);
+            return false;
-          return true;
+          if(!buf->read_uint64(uinteger))
+            return false;
+          if(!buf->read_into(pubkey.begin(), pubkey.end()))
+            return false;
+          if(buf->size_left() <= Zsig.size())
+            return false;
+          Xsize = buf->size_left() - Zsig.size();
+          if(!buf->read_into(X.begin(), X.begin() + Xsize))
+            return false;
+          return buf->read_into(Zsig.begin(), Zsig.end());
         case eOCMD_GiveFlowID:
-          if(buf->size_left() < 32)
+          if(!buf->read_into(magic.begin(), magic.end()))
             return false;
-          std::copy_n(nextFlowID.begin(), 32, buf->cur);
+          if(!buf->read_into(flow.begin(), flow.end()))
-          return true;
+            return false;
+          if(!buf->read_into(pubkey.begin(), pubkey.end()))
+            return false;
+          buf->cur += buf->size_left() - Zsig.size();
+          return buf->read_into(Zsig.begin(), Zsig.end());
         case eOCMD_Reject:
-          rejectReason = std::string(buf->cur, buf->base + buf->sz);
+          if(!buf->read_into(reject.begin(), reject.end()))
-          return true;
+            return false;
+          if(!buf->read_uint64(uinteger))
+            return false;
+          if(!buf->read_into(pubkey.begin(), pubkey.end()))
+            return false;
+          buf->cur += buf->size_left() - Zsig.size();
+          return buf->read_into(Zsig.begin(), Zsig.end());
         case eOCMD_SessionNegotiate:
-          // explicit fallthrough
+          if(!buf->read_into(flow.begin(), flow.end()))
+            return false;
+          if(!buf->read_into(pubkey.begin(), pubkey.end()))
+            return false;
+          if(!buf->read_uint64(uinteger))
+            return false;
+          if(buf->size_left() == Zsig.size() + 32)
+          {
+            A.reset(new AlignedBuffer< 32 >());
+            if(!buf->read_into(A->begin(), A->end()))
+              return false;
+          }
+          return buf->read_into(Zsig.begin(), Zsig.end());
         case eOCMD_TransmitData:
-          if(buf->size_left() <= 56)
+          if(!buf->read_into(flow.begin(), flow.end()))
             return false;
-          std::copy_n(N.begin(), N.size(), buf->cur);
+          if(!buf->read_into(N.begin(), N.end()))
-          buf->cur += N.size();
-          Xsize = buf->size_left() - Z.size();
-          if(Xsize > X.size())
             return false;
-          std::copy_n(X.begin(), Xsize, buf->cur);
+          if(buf->size_left() <= Zhash.size())
-          buf->cur += Xsize;
+            return false;
-          std::copy_n(Z.begin(), Z.size(), buf->cur);
+          Xsize = buf->size_left() - Zhash.size();
-          return true;
+          if(!buf->read_into(X.begin(), X.begin() + Xsize))
-
+            return false;
+          return buf->read_into(Zhash.begin(), Zhash.end());
         default:
           return false;
       }
@@ -73,6 +160,7 @@ namespace llarp
                          TimeoutHandler t, SessionClosedHandler closed)
         : ILinkLayer(enckey, getrc, h, sign, est, reneg, t, closed), crypto(c)
     {
+      m_FlowCookie.Randomize();
     }
 
     LinkLayer::~LinkLayer()
@@ -118,8 +206,9 @@ namespace llarp
     LinkLayer::RecvFrom(const Addr& from, const void* pkt, size_t sz)
     {
       m_OuterMsg.Clear();
-      llarp_buffer_t buf(pkt, sz);
+      llarp_buffer_t sigbuf(pkt, sz);
-      if(!m_OuterMsg.Decode(&buf))
+      llarp_buffer_t decodebuf(pkt, sz);
+      if(!m_OuterMsg.Decode(&decodebuf))
       {
         LogError("failed to decode outer message");
         return;
@@ -128,13 +217,27 @@ namespace llarp
       switch(m_OuterMsg.command)
       {
         case eOCMD_ObtainFlowID:
+          sigbuf.sz -= m_OuterMsg.Zsig.size();
+          if(!crypto->verify(m_OuterMsg.pubkey, sigbuf, m_OuterMsg.Zsig))
+          {
+            LogError("failed to verify signature on '",
+                     (char)m_OuterMsg.command, "' message from ", from);
+            return;
+          }
           if(!ShouldSendFlowID(from))
-            return;  // drop
+          {
-
+            SendReject(from, "no flo 4u :^)");
+            return;
+          }
           if(m_OuterMsg.netid == ourNetID)
+          {
+            if(GenFlowIDFor(m_OuterMsg.pubkey, from, m_OuterMsg.flow))
               SendFlowID(from, m_OuterMsg.flow);
             else
-            SendReject(from, m_OuterMsg.flow, "bad net id");
+              SendReject(from, "genflow fail");
+          }
+          else
+            SendReject(from, "bad netid");
       }
     }
 
@@ -156,6 +259,35 @@ namespace llarp
       (void)flow;
     }
 
+    bool
+    LinkLayer::VerifyFlowID(const PubKey& pk, const Addr& from,
+                            const FlowID_t& flow) const
+    {
+      FlowID_t expected;
+      if(!GenFlowIDFor(pk, from, expected))
+        return false;
+      return expected == flow;
+    }
+
+    bool
+    LinkLayer::GenFlowIDFor(const PubKey& pk, const Addr& from,
+                            FlowID_t& flow) const
+    {
+      std::array< byte_t, 128 > tmp = {0};
+      if(inet_ntop(AF_INET6, from.addr6(), (char*)tmp.data(), tmp.size())
+         == nullptr)
+        return false;
+      std::copy_n(pk.begin(), pk.size(), tmp.begin() + 64);
+      std::copy_n(m_FlowCookie.begin(), m_FlowCookie.size(),
+                  tmp.begin() + 64 + pk.size());
+      llarp_buffer_t buf(tmp);
+      ShortHash h;
+      if(!crypto->shorthash(h, buf))
+        return false;
+      std::copy_n(h.begin(), flow.size(), flow.begin());
+      return true;
+    }
+
     bool
     LinkLayer::ShouldSendFlowID(const Addr& to) const
     {
@@ -165,12 +297,33 @@ namespace llarp
     }
 
     void
-    LinkLayer::SendReject(const Addr& to, const FlowID_t& flow, const char* msg)
+    LinkLayer::SendReject(const Addr& to, const char* msg)
     {
-      // TODO: implement me
+      if(strlen(msg) > 14)
-      (void)to;
+      {
-      (void)flow;
+        throw std::logic_error("reject message too big");
-      (void)msg;
+      }
+      std::array< byte_t, 120 > pkt;
+      auto now  = Now();
+      PubKey pk = GetOurRC().pubkey;
+      OuterMessage m;
+      m.CreateReject(msg, now, pk);
+      llarp_buffer_t encodebuf(pkt);
+      if(!m.Encode(&encodebuf))
+      {
+        LogError("failed to encode reject message to ", to);
+        return;
+      }
+      llarp_buffer_t signbuf(pkt.data(), pkt.size() - m.Zsig.size());
+      if(!Sign(m.Zsig, signbuf))
+      {
+        LogError("failed to sign reject messsage to ", to);
+        return;
+      }
+      std::copy_n(m.Zsig.begin(), m.Zsig.size(),
+                  pkt.begin() + (pkt.size() - m.Zsig.size()));
+      llarp_buffer_t pktbuf(pkt);
+      SendTo_LL(to, pktbuf);
     }
 
     std::unique_ptr< ILinkLayer >

llarp/link/iwp_internal.hpp

@@ -8,6 +8,7 @@
 #include <link/server.hpp>
 #include <link/session.hpp>
 
+#include <array>
 #include <bitset>
 #include <deque>
 
@@ -46,15 +47,35 @@ namespace llarp
       byte_t command;
       FlowID_t flow;
 
+      OuterMessage();
+      ~OuterMessage();
+
+      // static members
+      static std::array< byte_t, 6 > obtain_flow_id_magic;
+      static std::array< byte_t, 6 > give_flow_id_magic;
+
+      void
+      CreateReject(const char *msg, llarp_time_t now, const PubKey &pk);
+
       // optional memebers follow
+      std::array< byte_t, 6 > magic;
       NetID netid;
-      FlowID_t nextFlowID;
+      // either timestamp or counter
-      std::string rejectReason;
+      uint64_t uinteger;
+      std::array< byte_t, 14 > reject;
       AlignedBuffer< 24 > N;
-      // TODO: compute optimal size
+      PubKey pubkey;
-      AlignedBuffer< 1440 > X;
+
+      std::unique_ptr< AlignedBuffer< 32 > > A;
+
+      static constexpr size_t ipv6_mtu      = 1280;
+      static constexpr size_t overhead_size = 16 + 24 + 32;
+      static constexpr size_t payload_size  = ipv6_mtu - overhead_size;
+
+      AlignedBuffer< payload_size > X;
       size_t Xsize;
-      ShortHash Z;
+      ShortHash Zhash;
+      Signature Zsig;
 
       /// encode to buffer
       bool
@@ -64,10 +85,6 @@ namespace llarp
       bool
       Decode(llarp_buffer_t *buf);
 
-      /// verify signature if needed
-      bool
-      Verify(const SharedSecret &K) const;
-
       /// clear members
       void
       Clear();
@@ -264,20 +281,23 @@ namespace llarp
       uint16_t
       Rank() const override;
 
-      /// verify that a new flow id matches addresses and old flow id
+      /// verify that a new flow id matches addresses and pubkey
       bool
-      VerifyFlowID(const FlowID_t &newID, const Addr &from,
+      VerifyFlowID(const PubKey &pk, const Addr &from,
-                   const FlowID_t &oldID) const;
+                   const FlowID_t &flow) const;
 
       void
       RecvFrom(const llarp::Addr &from, const void *buf, size_t sz) override;
 
      private:
+      bool
+      GenFlowIDFor(const PubKey &pk, const Addr &from, FlowID_t &flow) const;
+
       bool
       ShouldSendFlowID(const Addr &from) const;
 
       void
-      SendReject(const Addr &to, const FlowID_t &flow, const char *msg);
+      SendReject(const Addr &to, const char *msg);
 
       void
       SendFlowID(const Addr &to, const FlowID_t &flow);

llarp/link/server.cpp

@@ -232,6 +232,7 @@ namespace llarp
 
   void
   ILinkLayer::Tick(llarp_time_t now)
+  {
     {
       Lock l(&m_AuthedLinksMutex);
       auto itr = m_AuthedLinks.begin();
@@ -242,6 +243,17 @@ namespace llarp
       }
     }
 
+    {
+      Lock l(&m_PendingMutex);
+      auto itr = m_Pending.begin();
+      while(itr != m_Pending.end())
+      {
+        itr->second->Tick(now);
+        ++itr;
+      }
+    }
+  }
+
   void
   ILinkLayer::Stop()
   {

llarp/link/server.hpp

@@ -95,7 +95,7 @@ namespace llarp
     }
 
     void
-    SendTo_LL(const llarp::Addr& to, llarp_buffer_t pkt)
+    SendTo_LL(const llarp::Addr& to, const llarp_buffer_t& pkt)
     {
       llarp_ev_udp_sendto(&m_udp, to, pkt);
     }

llarp/link/utp.cpp

@@ -236,7 +236,7 @@ namespace llarp
       auto dlt = now - lastActive;
       if(dlt >= sessionTimeout)
       {
-        LogDebug("session timeout reached for ", remoteAddr);
+        LogInfo("session timeout reached for ", remoteAddr);
         return true;
       }
       return false;
@@ -254,6 +254,17 @@ namespace llarp
       return remoteAddr;
     }
 
+    uint64
+    LinkLayer::OnConnect(utp_callback_arguments* arg)
+    {
+      LinkLayer* l =
+          static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
+      Session* session = static_cast< Session* >(utp_get_userdata(arg->socket));
+      if(session && l)
+        session->OutboundLinkEstablished(l);
+      return 0;
+    }
+
     uint64
     LinkLayer::SendTo(utp_callback_arguments* arg)
     {
@@ -374,6 +385,7 @@ namespace llarp
       utp_context_set_userdata(_utp_ctx, this);
       utp_set_callback(_utp_ctx, UTP_SENDTO, &LinkLayer::SendTo);
       utp_set_callback(_utp_ctx, UTP_ON_ACCEPT, &LinkLayer::OnAccept);
+      utp_set_callback(_utp_ctx, UTP_ON_CONNECT, &LinkLayer::OnConnect);
       utp_set_callback(_utp_ctx, UTP_ON_STATE_CHANGE,
                        &LinkLayer::OnStateChange);
       utp_set_callback(_utp_ctx, UTP_ON_READ, &LinkLayer::OnRead);
@@ -404,9 +416,11 @@ namespace llarp
     }
 
 #ifdef __linux__
+
     void
     LinkLayer::ProcessICMP()
     {
+#ifndef TESTNET
       do
       {
         byte_t vec_buf[4096], ancillary_buf[4096];
@@ -475,13 +489,13 @@ namespace llarp
           }
         }
       } while(true);
+#endif
     }
 #endif
 
     void
     LinkLayer::Pump()
     {
-      utp_issue_deferred_acks(_utp_ctx);
 #ifdef __linux__
       ProcessICMP();
 #endif
@@ -507,6 +521,7 @@ namespace llarp
           }
         }
       }
+      utp_issue_deferred_acks(_utp_ctx);
     }
 
     void
@@ -577,9 +592,7 @@ namespace llarp
       SendQueueBacklog = [&]() -> size_t { return sendq.size(); };
 
       SendKeepAlive = [&]() -> bool {
-        auto now = parent->Now();
+        if(state == eSessionReady)
-        if(sendq.size() == 0 && state == eSessionReady && now > lastActive
-           && now - lastActive > 5000)
         {
           DiscardMessage msg;
           std::array< byte_t, 128 > tmp;
@@ -590,6 +603,7 @@ namespace llarp
           buf.cur = buf.base;
           if(!this->QueueWriteBuffers(buf))
             return false;
+          PumpWrite();
         }
         return true;
       };
@@ -840,6 +854,7 @@ namespace llarp
       if(sock)
       {
         utp_set_userdata(sock, nullptr);
+        sock = nullptr;
       }
     }
 
@@ -872,7 +887,6 @@ namespace llarp
       else
       {
         LogWarn("utp_socket got data with no underlying session");
-        utp_shutdown(arg->socket, SHUT_RDWR);
         utp_close(arg->socket);
       }
       return 0;
@@ -881,20 +895,10 @@ namespace llarp
     uint64
     LinkLayer::OnStateChange(utp_callback_arguments* arg)
     {
-      LinkLayer* l =
-          static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
       Session* session = static_cast< Session* >(utp_get_userdata(arg->socket));
       if(session)
       {
-        if(arg->state == UTP_STATE_CONNECT)
+        if(arg->state == UTP_STATE_WRITABLE)
-        {
-          if(session->state == Session::eClose)
-          {
-            return 0;
-          }
-          session->OutboundLinkEstablished(l);
-        }
-        else if(arg->state == UTP_STATE_WRITABLE)
         {
           session->PumpWrite();
         }
@@ -1155,11 +1159,9 @@ namespace llarp
             utp_close(sock);
           }
           LogDebug("utp_close ", remoteAddr);
-          utp_set_userdata(sock, nullptr);
         }
       }
       EnterState(eClose);
-      sock = nullptr;
     }
 
     void

llarp/link/utp_internal.hpp

@@ -109,8 +109,8 @@ namespace llarp
       SharedSecret txKey;
       /// timestamp last active
       llarp_time_t lastActive;
-      /// session timeout (30s)
+      /// session timeout (60s)
-      const static llarp_time_t sessionTimeout = 30 * 1000;
+      const static llarp_time_t sessionTimeout = DefaultLinkSessionLifetime;
 
       /// send queue for utp
       std::deque< utp_iovec > vecq;
@@ -298,6 +298,9 @@ namespace llarp
       static uint64
       OnStateChange(utp_callback_arguments*);
 
+      static uint64
+      OnConnect(utp_callback_arguments*);
+
       /// accept callback
       static uint64
       OnAccept(utp_callback_arguments*);

llarp/path/path.cpp

@@ -767,7 +767,7 @@ namespace llarp
         if(m_BuiltHook)
           m_BuiltHook(this);
         m_BuiltHook = nullptr;
-
+        LogDebug("path latency is now ", intro.latency, " for ", Name());
         return true;
       }
       else

llarp/path/pathbuilder.cpp

@@ -256,7 +256,9 @@ namespace llarp
     bool
     Builder::ShouldBuildMore(llarp_time_t now) const
     {
+      if(llarp::randint() % 3 >= 1)
         return PathSet::ShouldBuildMore(now) && !BuildCooldownHit(now);
+      return false;
     }
 
     void

llarp/path/pathbuilder.hpp

@@ -52,6 +52,13 @@ namespace llarp
       bool
       BuildCooldownHit(llarp_time_t now) const;
 
+      /// get roles for this path builder
+      virtual PathRole
+      GetRoles() const
+      {
+        return ePathRoleAny;
+      }
+
       virtual bool
       Stop() override;
 

llarp/profiling.cpp

@@ -70,9 +70,11 @@ namespace llarp
   bool
   RouterProfile::IsGood(uint64_t chances) const
   {
+    if(connectTimeoutCount > 3)
       return connectTimeoutCount <= connectGoodCount
-        /// N chances
           && (pathSuccessCount * chances) >= pathFailCount;
+    else
+      return (pathSuccessCount * chances) >= pathFailCount;
   }
 
   bool

llarp/routing/message_parser.cpp

@@ -67,6 +67,7 @@ namespace llarp
         if(strbuf.sz != 1)
           return false;
         self->key = *strbuf.cur;
+        LogDebug("routing message '", self->key, "'");
         switch(self->key)
         {
           case 'D':

llarp/service/endpoint.cpp

@@ -21,7 +21,7 @@ namespace llarp
   {
     Endpoint::Endpoint(const std::string& name, AbstractRouter* r,
                        Context* parent)
-        : path::Builder(r, r->dht(), 6, DEFAULT_HOP_LENGTH)
+        : path::Builder(r, r->dht(), 3, DEFAULT_HOP_LENGTH)
         , context(parent)
         , m_Router(r)
         , m_Name(name)
@@ -1223,6 +1223,9 @@ namespace llarp
       auto itr   = range.first;
       while(itr != range.second)
       {
+        if(itr->second->IsReady())
+          h(snode, itr->second.get());
+        else
           itr->second->AddReadyHook(std::bind(h, snode, std::placeholders::_1));
         ++itr;
       }
@@ -1269,11 +1272,8 @@ namespace llarp
           ProtocolFrame& f = transfer.T;
           path::Path* p    = nullptr;
           std::set< ConvoTag > tags;
-          if(!GetConvoTagsForService(itr->second, tags))
+          if(GetConvoTagsForService(itr->second, tags))
           {
-            llarp::LogError("no convo tag");
-            return false;
-          }
             Introduction remoteIntro;
             SharedSecret K;
             // pick tag
@@ -1319,6 +1319,7 @@ namespace llarp
             }
           }
         }
+      }
       // outbound converstation
       if(HasPathToService(remote))
       {
@@ -1749,7 +1750,7 @@ namespace llarp
         {
           // we can safely set remoteIntro to the next one
           SwapIntros();
-          llarp::LogInfo(Name(), "swapped intro");
+          llarp::LogInfo(Name(), " swapped intro");
         }
       }
       // lookup router in intro if set and unknown
@@ -1767,6 +1768,10 @@ namespace llarp
       // send control message if we look too quiet
       if(now - lastGoodSend > (sendTimeout / 2))
       {
+        if(!GetNewestPathByRouter(remoteIntro.router))
+        {
+          BuildOneAlignedTo(remoteIntro.router);
+        }
         Encrypted< 64 > tmp;
         tmp.Randomize();
         llarp_buffer_t buf(tmp.data(), tmp.size());
@@ -1864,7 +1869,6 @@ namespace llarp
       {
         llarp::LogError("cannot encrypt and send: no path for intro ",
                         remoteIntro);
-
         return;
       }
 

llarp/util/buffer.cpp

@@ -43,6 +43,16 @@ llarp_buffer_t::put_uint16(uint16_t i)
   return true;
 }
 
+bool
+llarp_buffer_t::put_uint64(uint64_t i)
+{
+  if(size_left() < sizeof(uint64_t))
+    return false;
+  htobe64buf(cur, i);
+  cur += sizeof(uint64_t);
+  return true;
+}
+
 bool
 llarp_buffer_t::put_uint32(uint32_t i)
 {
@@ -73,6 +83,16 @@ llarp_buffer_t::read_uint32(uint32_t& i)
   return true;
 }
 
+bool
+llarp_buffer_t::read_uint64(uint64_t& i)
+{
+  if(size_left() < sizeof(uint64_t))
+    return false;
+  i = bufbe64toh(cur);
+  cur += sizeof(uint64_t);
+  return true;
+}
+
 size_t
 llarp_buffer_t::read_until(char delim, byte_t* result, size_t resultsize)
 {

llarp/util/buffer.hpp

@@ -12,6 +12,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <utility>
+#include <algorithm>
 
 /**
  * buffer.h
@@ -116,6 +117,10 @@ struct llarp_buffer_t
   size_t
   size_left() const;
 
+  template < typename OutputIt >
+  bool
+  read_into(OutputIt begin, OutputIt end);
+
   template < typename InputIt >
   bool
   write(InputIt begin, InputIt end);
@@ -136,11 +141,17 @@ struct llarp_buffer_t
   bool
   put_uint32(uint32_t i);
 
+  bool
+  put_uint64(uint64_t i);
+
   bool
   read_uint16(uint16_t &i);
   bool
   read_uint32(uint32_t &i);
 
+  bool
+  read_uint64(uint64_t &i);
+
   size_t
   read_until(char delim, byte_t *result, size_t resultlen);
 
@@ -153,6 +164,20 @@ struct llarp_buffer_t
 bool
 operator==(const llarp_buffer_t &buff, const char *data);
 
+template < typename OutputIt >
+bool
+llarp_buffer_t::read_into(OutputIt begin, OutputIt end)
+{
+  auto dist = std::distance(begin, end);
+  if(static_cast< decltype(dist) >(size_left()) >= dist)
+  {
+    std::copy_n(cur, dist, begin);
+    cur += dist;
+    return true;
+  }
+  return false;
+}
+
 template < typename InputIt >
 bool
 llarp_buffer_t::write(InputIt begin, InputIt end)

llarp/util/logger.hpp

@@ -239,6 +239,9 @@ namespace llarp
 #endif
       ss << (char)27 << "[0;0m";
       _glog.out << ss.str() << std::endl;
+#ifdef TESTNET
+      _glog.out << std::flush;
+#endif
 #ifdef _WIN32
     }
     else