make the lokinet endpoint code work

still needs the quic sides to work

llarp/endpoint_base.hpp

@@ -26,6 +26,9 @@ namespace llarp
 
     using AddressVariant_t = std::variant<service::Address, RouterID>;
 
+    virtual std::string
+    LocalAddress() const = 0;
+
     virtual quic::TunnelManager*
     GetQUICTunnel() = 0;
 

llarp/handlers/exit.cpp

@@ -557,6 +557,13 @@ namespace llarp
       return found;
     }
 
+    std::string
+    ExitEndpoint::LocalAddress() const
+    {
+      const RouterID r{m_Router->pubkey()};
+      return r.ToString();
+    }
+
     bool
     ExitEndpoint::QueueOutboundTraffic(net::IPPacket pkt)
     {

llarp/handlers/exit.hpp

@@ -109,6 +109,9 @@ namespace llarp
       bool
       QueueOutboundTraffic(net::IPPacket pkt);
 
+      std::string
+      LocalAddress() const override;
+
       /// sets up networking and starts traffic
       bool
       Start();

llarp/handlers/null.hpp

@@ -1,7 +1,10 @@
 #pragma once
 
 #include <llarp/service/endpoint.hpp>
-#include "service/protocol_type.hpp"
+#include <llarp/service/protocol_type.hpp>
+#include <llarp/quic/tunnel.hpp>
+#include <llarp/router/abstractrouter.hpp>
+#include <llarp/ev/ev.hpp>
 
 namespace llarp
 {
@@ -12,12 +15,35 @@ namespace llarp
     {
       NullEndpoint(AbstractRouter* r, llarp::service::Context* parent)
           : llarp::service::Endpoint(r, parent)
-      {}
+      {
+        r->loop()->add_ticker([this] { Pump(Now()); });
+      }
 
       virtual bool
       HandleInboundPacket(
-          const service::ConvoTag, const llarp_buffer_t&, service::ProtocolType, uint64_t) override
+          const service::ConvoTag tag,
+          const llarp_buffer_t& buf,
+          service::ProtocolType t,
+          uint64_t) override
       {
+        if (t != service::ProtocolType::QUIC and t != service::ProtocolType::Control)
+          return false;
+
+        auto* quic = GetQUICTunnel();
+        if (!quic)
+        {
+          LogWarn("incoming quic packet but this endpoint is not quic capable; dropping");
+          return false;
+        }
+        if (buf.sz < 4)
+        {
+          LogWarn("invalid incoming quic packet, dropping");
+          return false;
+        }
+        LogInfo("tag active T=", tag);
+        MarkConvoTagActive(tag);
+        quic->receive_packet(tag, buf);
+        m_router->loop()->wakeup();
         return true;
       }
 

llarp/quic/address.hpp

@@ -21,7 +21,7 @@ namespace llarp::quic
   class Address
   {
     sockaddr_in6 saddr{};
-    ngtcp2_addr a{0, reinterpret_cast<sockaddr*>(&saddr)};
+    ngtcp2_addr a{sizeof(saddr), reinterpret_cast<sockaddr*>(&saddr)};
 
    public:
     Address() = default;

llarp/quic/connection.cpp

@@ -331,7 +331,6 @@ namespace llarp::quic
 #pragma GCC diagnostic pop
   }  // namespace
 
-#if 0
 #ifndef NDEBUG
   extern "C" inline void
   ngtcp_trace_logger([[maybe_unused]] void* user_data, const char* fmt, ...)
@@ -345,7 +344,6 @@ namespace llarp::quic
     }
     va_end(ap);
   }
-#endif
 #endif
 
   io_result
@@ -411,11 +409,10 @@ namespace llarp::quic
     cb.update_key = update_key;
 
     ngtcp2_settings_default(&settings);
-#if 0
 #ifndef NDEBUG
     settings.log_printf = ngtcp_trace_logger;
 #endif
-#endif
+
     settings.initial_ts = get_timestamp();
     // FIXME: IPv6
     settings.max_udp_payload_size = NGTCP2_MAX_PKTLEN_IPV4;

llarp/quic/tunnel.hpp

@@ -47,7 +47,7 @@ namespace llarp::quic
 
     // Timeout for the next `open()`.  Note that when `open()` is given a ONS name to resolve this
     // includes the resolution time.
-    std::chrono::milliseconds open_timeout = 10s;
+    std::chrono::milliseconds open_timeout = 4s;
 
     TunnelManager(EndpointBase& endpoint);
 

llarp/rpc/rpc_server.cpp

@@ -141,6 +141,10 @@ namespace llarp::rpc
                 if (auto itr = obj.find("endpoint"); itr != obj.end())
                   endpoint = itr->get<std::string>();
 
+                std::string bindAddr = "127.0.0.1:0";
+                if (auto itr = obj.find("bind"); itr != obj.end())
+                  bindAddr = itr->get<std::string>();
+
                 std::string remoteHost;
                 if (auto itr = obj.find("host"); itr != obj.end())
                   remoteHost = itr->get<std::string>();
@@ -163,6 +167,8 @@ namespace llarp::rpc
                   reply(CreateJSONError("host not provided"));
                   return;
                 }
+                SockAddr laddr{};
+                laddr.fromString(bindAddr);
 
                 r->loop()->call([reply, endpoint, r, remoteHost, port, closeID]() {
                   auto ep = GetEndpointByName(r, endpoint);
@@ -184,21 +190,11 @@ namespace llarp::rpc
                     return;
                   }
 
-                  auto status = std::make_shared<util::StatusObject>();
-
-                  auto hook = [status, reply](bool success) {
-                    if (success)
-                    {
-                      reply(CreateJSONResponse(*status));
-                    }
-                    else
-                    {
-                      reply(CreateJSONError("failed"));
-                    }
-                  };
-                  auto [addr, id] = quic->open(remoteHost, port, hook);
-                  status->operator[]("addr") = addr.toString();
-                  status->operator[]("id") = id;
+                  auto [addr, id] = quic->open(remoteHost, port);
+                  util::StatusObject status;
+                  status["addr"] = addr.toString();
+                  status["id"] = id;
+                  reply(CreateJSONResponse(status));
                 });
               });
             })
@@ -248,7 +244,9 @@ namespace llarp::rpc
                       reply(CreateJSONError(ex.what()));
                       return;
                     }
-                    util::StatusObject result{{"id", id}};
+                    util::StatusObject result;
+                    result["id"] = id;
+                    result["addr"] = ep->LocalAddress() + ":" + std::to_string(port);
                     reply(CreateJSONResponse(result));
                   }
                   else if (closeID)

llarp/service/convotag.cpp

@@ -14,10 +14,9 @@ namespace llarp::service
   sockaddr_in6
   ConvoTag::ToV6() const
   {
-    const auto* ptr = reinterpret_cast<const uint64_t*>(data());
     sockaddr_in6 saddr{};
     saddr.sin6_family = AF_INET6;
-    saddr.sin6_addr = net::HUIntToIn6(huint128_t{uint128_t{ptr[0], ptr[1]}});
+    std::copy_n(data(), size(), saddr.sin6_addr.s6_addr);
     return saddr;
   }
 

llarp/service/endpoint.cpp

@@ -370,13 +370,9 @@ namespace llarp
     void
     Endpoint::PutIntroFor(const ConvoTag& tag, const Introduction& intro)
     {
-      auto itr = Sessions().find(tag);
-      if (itr == Sessions().end())
-      {
-        return;
-      }
-      itr->second.intro = intro;
-      itr->second.lastUsed = Now();
+      auto& s = Sessions()[tag];
+      s.intro = intro;
+      s.lastUsed = Now();
     }
 
     bool
@@ -670,10 +666,9 @@ namespace llarp
     }
 
     void
-    Endpoint::PutNewOutboundContext(const service::IntroSet& introset)
+    Endpoint::PutNewOutboundContext(const service::IntroSet& introset, llarp_time_t left)
     {
-      Address addr;
-      introset.A.CalculateAddress(addr.as_array());
+      Address addr{introset.A.Addr()};
 
       auto& remoteSessions = m_state->m_RemoteSessions;
       auto& serviceLookups = m_state->m_PendingServiceLookups;
@@ -686,14 +681,20 @@ namespace llarp
         auto i = range.first;
         while (i != range.second)
         {
-          i->second(addr, itr->second.get());
+          itr->second->SetReadyHook(
+              [callback = i->second, addr](auto session) {
+                LogInfo(addr, " is ready to send");
+                callback(addr, session);
+              },
+              left);
           ++i;
         }
         serviceLookups.erase(addr);
         return;
       }
 
-      auto it = remoteSessions.emplace(addr, std::make_shared<OutboundContext>(introset, this));
+      auto session = std::make_shared<OutboundContext>(introset, this);
+      remoteSessions.emplace(addr, session);
       LogInfo("Created New outbound context for ", addr.ToString());
 
       // inform pending
@@ -701,7 +702,12 @@ namespace llarp
       auto itr = range.first;
       if (itr != range.second)
       {
-        itr->second(addr, it->second.get());
+        session->SetReadyHook(
+            [callback = itr->second, addr](auto session) {
+              LogInfo(addr, " is ready to send");
+              callback(addr, session);
+            },
+            left);
         ++itr;
       }
       serviceLookups.erase(addr);
@@ -1001,12 +1007,19 @@ namespace llarp
       return false;
     }
 
+    std::string
+    Endpoint::LocalAddress() const
+    {
+      return m_Identity.pub.Addr().ToString();
+    }
+
     bool
     Endpoint::ProcessDataMessage(std::shared_ptr<ProtocolMessage> msg)
     {
       if ((msg->proto == ProtocolType::Exit
            && (m_state->m_ExitEnabled || m_ExitMap.ContainsValue(msg->sender.Addr())))
-          || msg->proto == ProtocolType::TrafficV4 || msg->proto == ProtocolType::TrafficV6)
+          || msg->proto == ProtocolType::TrafficV4 || msg->proto == ProtocolType::TrafficV6
+          || (msg->proto == ProtocolType::QUIC and m_quic))
       {
         m_InboundTrafficQueue.tryPushBack(std::move(msg));
         return true;
@@ -1042,9 +1055,6 @@ namespace llarp
     Endpoint::SendAuthResult(
         path::Path_ptr path, PathID_t replyPath, ConvoTag tag, AuthResult result)
     {
-      // this should not run if we have no auth policy
-      if (m_AuthPolicy == nullptr)
-        return;
       ProtocolFrame f;
       f.R = AuthResultCodeAsInt(result.code);
       f.T = tag;
@@ -1060,7 +1070,11 @@ namespace llarp
         msg.PutBuffer(reason);
         f.N.Randomize();
         f.C.Zero();
-        msg.proto = ProtocolType::Auth;
+        if (m_AuthPolicy)
+          msg.proto = ProtocolType::Auth;
+        else
+          msg.proto = ProtocolType::Control;
+
         if (not GetReplyIntroFor(tag, msg.introReply))
         {
           LogError("Failed to send auth reply: no reply intro");
@@ -1110,24 +1124,28 @@ namespace llarp
         if (!frame.Verify(si))
           return false;
         // remove convotag it doesn't exist
-        LogWarn("remove convotag T=", frame.T);
+        LogWarn("remove convotag T=", frame.T, " R=", frame.R);
         RemoveConvoTag(frame.T);
         return true;
       }
       if (not frame.AsyncDecryptAndVerify(Router()->loop(), p, m_Identity, this))
       {
-        // send reset convo tag message
-        ProtocolFrame f;
-        f.R = 1;
-        f.T = frame.T;
-        f.F = p->intro.pathID;
-
-        f.Sign(m_Identity);
+        LogError("Failed to decrypt protocol frame");
+        if (not frame.C.IsZero())
         {
-          LogWarn("invalidating convotag T=", frame.T);
-          RemoveConvoTag(frame.T);
-          m_SendQueue.tryPushBack(
-              SendEvent_t{std::make_shared<const routing::PathTransferMessage>(f, frame.F), p});
+          // send reset convo tag message
+          ProtocolFrame f;
+          f.R = 1;
+          f.T = frame.T;
+          f.F = p->intro.pathID;
+
+          f.Sign(m_Identity);
+          {
+            LogWarn("invalidating convotag T=", frame.T);
+            RemoveConvoTag(frame.T);
+            m_SendQueue.tryPushBack(
+                SendEvent_t{std::make_shared<const routing::PathTransferMessage>(f, frame.F), p});
+          }
         }
       }
       return true;
@@ -1148,7 +1166,10 @@ namespace llarp
 
     bool
     Endpoint::OnLookup(
-        const Address& addr, std::optional<IntroSet> introset, const RouterID& endpoint)
+        const Address& addr,
+        std::optional<IntroSet> introset,
+        const RouterID& endpoint,
+        llarp_time_t timeLeft)
     {
       const auto now = Router()->Now();
       auto& fails = m_state->m_ServiceLookupFails;
@@ -1172,7 +1193,7 @@ namespace llarp
       if (m_state->m_RemoteSessions.count(addr) > 0)
         return true;
 
-      PutNewOutboundContext(*introset);
+      PutNewOutboundContext(*introset, timeLeft);
       return true;
     }
 
@@ -1189,8 +1210,7 @@ namespace llarp
     }
 
     bool
-    Endpoint::EnsurePathToService(
-        const Address remote, PathEnsureHook hook, llarp_time_t /*timeoutMS*/)
+    Endpoint::EnsurePathToService(const Address remote, PathEnsureHook hook, llarp_time_t timeout)
     {
       /// how many routers to use for lookups
       static constexpr size_t NumParallelLookups = 2;
@@ -1201,15 +1221,6 @@ namespace llarp
 
       auto& sessions = m_state->m_RemoteSessions;
 
-      {
-        auto itr = sessions.find(remote);
-        if (itr != sessions.end())
-        {
-          hook(itr->first, itr->second.get());
-          return true;
-        }
-      }
-
       // add response hook to list for address.
       m_state->m_PendingServiceLookups.emplace(remote, hook);
 
@@ -1238,11 +1249,14 @@ namespace llarp
         {
           HiddenServiceAddressLookup* job = new HiddenServiceAddressLookup(
               this,
-              util::memFn(&Endpoint::OnLookup, this),
+              [this](auto addr, auto result, auto from, auto left) {
+                return OnLookup(addr, result, from, left);
+              },
               location,
               PubKey{remote.as_array()},
               order,
-              GenTXID());
+              GenTXID(),
+              timeout);
           LogInfo(
               "doing lookup for ",
               remote,
@@ -1328,6 +1342,9 @@ namespace llarp
     bool
     Endpoint::SendToOrQueue(ConvoTag tag, const llarp_buffer_t& pkt, ProtocolType t)
     {
+      if (tag.IsZero())
+        return false;
+      LogWarn("sent to tag T=", tag);
       if (auto maybe = GetEndpointWithConvoTag(tag))
         return SendToOrQueue(*maybe, pkt, t);
       return false;
@@ -1357,8 +1374,10 @@ namespace llarp
       while (not m_InboundTrafficQueue.empty())
       {
         auto msg = m_InboundTrafficQueue.popFront();
-        const llarp_buffer_t buf(msg->payload);
-        HandleInboundPacket(msg->tag, buf, msg->proto, msg->seqno);
+        if (not HandleInboundPacket(msg->tag, msg->payload, msg->proto, msg->seqno))
+        {
+          LogWarn("Failed to handle inbound message");
+        }
       }
 
       auto router = Router();
@@ -1391,7 +1410,7 @@ namespace llarp
         std::optional<ConvoTag> ret = std::nullopt;
         for (const auto& [tag, session] : Sessions())
         {
-          if (session.remote.Addr() == *ptr and session.lastUsed > time)
+          if (session.remote.Addr() == *ptr and session.lastUsed >= time)
           {
             time = session.lastUsed;
             ret = tag;
@@ -1471,24 +1490,33 @@ namespace llarp
           const auto tag = *maybe;
 
           if (!GetCachedSessionKeyFor(tag, K))
+          {
+            LogError("no cached key for T=", tag);
             return false;
-          if (!GetReplyIntroFor(tag, replyPath))
-            return false;
+          }
           if (!GetIntroFor(tag, remoteIntro))
+          {
+            LogError("no intro for T=", tag);
             return false;
-          // get path for intro
-          ForEachPath([&](path::Path_ptr path) {
-            if (path->intro == replyPath)
-            {
-              p = path;
-              return;
-            }
-            if (p && p->ExpiresSoon(now) && path->IsReady()
-                && path->intro.router == replyPath.router)
-            {
-              p = path;
-            }
-          });
+          }
+          if (GetReplyIntroFor(tag, replyPath))
+          {
+            // get path for intro
+            ForEachPath([&](path::Path_ptr path) {
+              if (path->intro == replyPath)
+              {
+                p = path;
+                return;
+              }
+              if (p && p->ExpiresSoon(now) && path->IsReady()
+                  && path->intro.router == replyPath.router)
+              {
+                p = path;
+              }
+            });
+          }
+          else
+            p = GetPathByRouter(remoteIntro.router);
 
           if (p)
           {
@@ -1498,6 +1526,7 @@ namespace llarp
             m->PutBuffer(data);
             f.N.Randomize();
             f.C.Zero();
+            f.R = 0;
             transfer->Y.Randomize();
             m->proto = t;
             m->introReply = p->intro;
@@ -1541,20 +1570,24 @@ namespace llarp
           // add pending traffic
           auto& traffic = m_state->m_PendingTraffic;
           traffic[remote].emplace_back(data, t);
-          return EnsurePathToService(
+          EnsurePathToService(
               remote,
               [self = this](Address addr, OutboundContext* ctx) {
                 if (ctx)
                 {
-                  ctx->UpdateIntroSet();
                   for (auto& pending : self->m_state->m_PendingTraffic[addr])
                   {
                     ctx->AsyncEncryptAndSendTo(pending.Buffer(), pending.protocol);
                   }
                 }
+                else
+                {
+                  LogWarn("no path made to ", addr);
+                }
                 self->m_state->m_PendingTraffic.erase(addr);
               },
               1500ms);
+          return true;
         }
       }
       return false;

llarp/service/endpoint.hpp

@@ -136,6 +136,9 @@ namespace llarp
       std::string
       Name() const override;
 
+      std::string
+      LocalAddress() const override;
+
       bool
       ShouldPublishDescriptors(llarp_time_t now) const override;
 
@@ -338,7 +341,7 @@ namespace llarp
       GetConvoTagsForService(const Address& si, std::set<ConvoTag>& tag) const override;
 
       void
-      PutNewOutboundContext(const IntroSet& introset);
+      PutNewOutboundContext(const IntroSet& introset, llarp_time_t timeLeftToAlign);
 
       uint64_t
       GetSeqNoForConvo(const ConvoTag& tag);
@@ -422,7 +425,11 @@ namespace llarp
       HandleVerifyGotRouter(dht::GotRouterMessage_constptr msg, RouterID id, bool valid);
 
       bool
-      OnLookup(const service::Address& addr, std::optional<IntroSet> i, const RouterID& endpoint);
+      OnLookup(
+          const service::Address& addr,
+          std::optional<IntroSet> i,
+          const RouterID& endpoint,
+          llarp_time_t timeLeft);
 
       bool
       DoNetworkIsolation(bool failed);

llarp/service/hidden_service_address_lookup.cpp

@@ -14,8 +14,9 @@ namespace llarp
         const dht::Key_t& l,
         const PubKey& k,
         uint64_t order,
-        uint64_t tx)
-        : IServiceLookup(p, tx, "HSLookup")
+        uint64_t tx,
+        llarp_time_t timeout)
+        : IServiceLookup(p, tx, "HSLookup", timeout)
         , rootkey(k)
         , relayOrder(order)
         , location(l)
@@ -42,7 +43,7 @@ namespace llarp
           found = *maybe;
         }
       }
-      return handle(remote, found, endpoint);
+      return handle(remote, found, endpoint, TimeLeft(time_now_ms()));
     }
 
     std::shared_ptr<routing::IMessage>

llarp/service/hidden_service_address_lookup.hpp

@@ -14,8 +14,8 @@ namespace llarp
       const PubKey rootkey;
       uint64_t relayOrder;
       const dht::Key_t location;
-      using HandlerFunc =
-          std::function<bool(const Address&, std::optional<IntroSet>, const RouterID&)>;
+      using HandlerFunc = std::function<bool(
+          const Address&, std::optional<IntroSet>, const RouterID&, llarp_time_t)>;
       HandlerFunc handle;
 
       HiddenServiceAddressLookup(
@@ -24,7 +24,8 @@ namespace llarp
           const dht::Key_t& location,
           const PubKey& rootkey,
           uint64_t relayOrder,
-          uint64_t tx);
+          uint64_t tx,
+          llarp_time_t timeout);
 
       ~HiddenServiceAddressLookup() override = default;
 

llarp/service/lookup.cpp

@@ -11,10 +11,10 @@ namespace llarp
 
   namespace service
   {
-    IServiceLookup::IServiceLookup(ILookupHolder* p, uint64_t tx, std::string n)
-        : m_parent(p), txid(tx), name(std::move(n))
+    IServiceLookup::IServiceLookup(
+        ILookupHolder* p, uint64_t tx, std::string n, llarp_time_t timeout)
+        : m_parent(p), txid(tx), name(std::move(n)), m_created{time_now_ms()}, m_timeout{timeout}
     {
-      m_created = time_now_ms();
       p->PutLookup(this, tx);
     }
 

llarp/service/lookup.hpp

@@ -46,11 +46,18 @@ namespace llarp
 
       /// determine if this request has timed out
       bool
-      IsTimedOut(llarp_time_t now, llarp_time_t timeout = 20s) const
+      IsTimedOut(llarp_time_t now) const
       {
-        if (now <= m_created)
-          return false;
-        return now - m_created > timeout;
+        return TimeLeft(now) == 0ms;
+      }
+
+      /// return how long this request has left to be fufilled
+      llarp_time_t
+      TimeLeft(llarp_time_t now) const
+      {
+        if (now > (m_created + m_timeout))
+          return 0s;
+        return now - (m_created + m_timeout);
       }
 
       /// build request message for service lookup
@@ -80,9 +87,10 @@ namespace llarp
       }
 
      protected:
-      IServiceLookup(ILookupHolder* parent, uint64_t tx, std::string name);
+      IServiceLookup(
+          ILookupHolder* parent, uint64_t tx, std::string name, llarp_time_t timeout = 10s);
 
-      llarp_time_t m_created;
+      const llarp_time_t m_created, m_timeout;
     };
 
     struct ILookupHolder

llarp/service/outbound_context.cpp

@@ -79,8 +79,9 @@ namespace llarp
     {
       if (remoteIntro != m_NextIntro)
       {
-        LogInfo(Name(), " swap intro to use ", RouterID(m_NextIntro.router));
+        LogInfo(Name(), " swap intro to use ", RouterID{m_NextIntro.router});
         remoteIntro = m_NextIntro;
+        m_DataHandler->PutSenderFor(currentConvoTag, currentIntroSet.A, false);
         m_DataHandler->PutIntroFor(currentConvoTag, remoteIntro);
         ShiftIntroduction(false);
       }
@@ -88,7 +89,7 @@ namespace llarp
 
     bool
     OutboundContext::OnIntroSetUpdate(
-        const Address&, std::optional<IntroSet> foundIntro, const RouterID& endpoint)
+        const Address&, std::optional<IntroSet> foundIntro, const RouterID& endpoint, llarp_time_t)
     {
       if (markedBad)
         return true;
@@ -113,7 +114,6 @@ namespace llarp
           return true;
         }
         currentIntroSet = *foundIntro;
-        SwapIntros();
       }
       else
       {
@@ -173,7 +173,7 @@ namespace llarp
       p->SetDataHandler(util::memFn(&OutboundContext::HandleHiddenServiceFrame, this));
       p->SetDropHandler(util::memFn(&OutboundContext::HandleDataDrop, this));
       // we now have a path to the next intro, swap intros
-      if (p->Endpoint() == m_NextIntro.router)
+      if (p->Endpoint() == m_NextIntro.router or p->Endpoint() == remoteIntro.router)
         SwapIntros();
       else
       {
@@ -184,6 +184,9 @@ namespace llarp
     void
     OutboundContext::AsyncGenIntro(const llarp_buffer_t& payload, ProtocolType t)
     {
+      if (sentIntro)
+        return;
+
       if (remoteIntro.router.IsZero())
         SwapIntros();
 
@@ -200,6 +203,7 @@ namespace llarp
           return;
         }
       }
+      sentIntro = true;
       auto frame = std::make_shared<ProtocolFrame>();
       auto ex = std::make_shared<AsyncKeyExchange>(
           m_Endpoint->Loop(),
@@ -218,7 +222,9 @@ namespace llarp
       ex->msg.PutBuffer(payload);
       ex->msg.introReply = path->intro;
       frame->F = ex->msg.introReply.pathID;
+      frame->R = 0;
       m_Endpoint->Router()->QueueWork([ex, frame] { return AsyncKeyExchange::Encrypt(ex, frame); });
+      LogInfo("send intro frame");
     }
 
     std::string
@@ -245,7 +251,8 @@ namespace llarp
             location,
             PubKey{addr.as_array()},
             relayOrder,
-            m_Endpoint->GenTXID());
+            m_Endpoint->GenTXID(),
+            5s);
         relayOrder++;
         if (job->SendRequestViaPath(path, m_Endpoint->Router()))
           updatingIntroSet = true;
@@ -275,6 +282,14 @@ namespace llarp
       return obj;
     }
 
+    void
+    OutboundContext::KeepAlive()
+    {
+      Encrypted<64> tmp;
+      tmp.Randomize();
+      SendPacketToRemote(tmp, ProtocolType::Control);
+    }
+
     bool
     OutboundContext::Pump(llarp_time_t now)
     {
@@ -284,6 +299,11 @@ namespace llarp
 
       constexpr auto InboundTrafficTimeout = 5s;
 
+      if (ReadyToSend() and remoteIntro.router.IsZero())
+      {
+        SwapIntros();
+      }
+
       if (m_GotInboundTraffic and m_LastInboundTraffic + InboundTrafficTimeout <= now)
       {
         if (std::chrono::abs(now - lastGoodSend) < InboundTrafficTimeout)
@@ -312,30 +332,31 @@ namespace llarp
         else
           ++itr;
       }
-      // send control message if we look too quiet
-      if (lastGoodSend > 0s)
+
+      if (ReadyToSend() and m_ReadyHook)
       {
-        if (now - lastGoodSend > (sendTimeout / 2))
-        {
-          if (!GetNewestPathByRouter(remoteIntro.router))
-          {
-            if (!BuildCooldownHit(now))
-              BuildOneAlignedTo(remoteIntro.router);
-          }
-          else
-          {
-            Encrypted<64> tmp;
-            tmp.Randomize();
-            llarp_buffer_t buf(tmp.data(), tmp.size());
-            AsyncEncryptAndSendTo(buf, ProtocolType::Control);
-          }
-        }
+        m_ReadyHook(this);
+        m_ReadyHook = nullptr;
       }
+
       // if we are dead return true so we are removed
       return lastGoodSend > 0s ? (now >= lastGoodSend && now - lastGoodSend > sendTimeout)
                                : (now >= createdAt && now - createdAt > connectTimeout);
     }
 
+    void
+    OutboundContext::SetReadyHook(std::function<void(OutboundContext*)> hook, llarp_time_t timeout)
+    {
+      if (m_ReadyHook)
+        return;
+      m_ReadyHook = hook;
+      m_router->loop()->call_later(timeout, [this]() {
+        if (m_ReadyHook)
+          m_ReadyHook(nullptr);
+        m_ReadyHook = nullptr;
+      });
+    }
+
     std::optional<std::vector<RouterContact>>
     OutboundContext::GetHopsForBuild()
     {
@@ -533,12 +554,18 @@ namespace llarp
         // handle discard
         ServiceInfo si;
         if (!m_Endpoint->GetSenderFor(frame.T, si))
+        {
+          LogWarn("no sender for T=", frame.T);
           return false;
+        }
         // verify source
         if (!frame.Verify(si))
+        {
+          LogWarn("signature failed");
           return false;
+        }
         // remove convotag it doesn't exist
-        LogWarn("remove convotag T=", frame.T);
+        LogWarn("remove convotag T=", frame.T, " R=", frame.R);
 
         AuthResult result{AuthResultCode::eAuthFailed, "unknown reason"};
         if (const auto maybe = AuthResultCodeFromInt(frame.R))
@@ -584,6 +611,7 @@ namespace llarp
       if (not frame.AsyncDecryptAndVerify(m_Endpoint->Loop(), p, ident, m_Endpoint, hook))
       {
         // send reset convo tag message
+        LogError("failed to decrypt and verify frame");
         ProtocolFrame f;
         f.R = 1;
         f.T = frame.T;

llarp/service/outbound_context.hpp

@@ -70,6 +70,9 @@ namespace llarp
       bool
       ReadyToSend() const;
 
+      void
+      SetReadyHook(std::function<void(OutboundContext*)> readyHook, llarp_time_t timeout);
+
       /// for exits
       void
       SendPacketToRemote(const llarp_buffer_t&, ProtocolType t) override;
@@ -114,6 +117,9 @@ namespace llarp
       std::string
       Name() const override;
 
+      void
+      KeepAlive();
+
       const IntroSet&
       GetCurrentIntroSet() const
       {
@@ -129,7 +135,8 @@ namespace llarp
       OnGeneratedIntroFrame(AsyncKeyExchange* k, PathID_t p);
 
       bool
-      OnIntroSetUpdate(const Address& addr, std::optional<IntroSet> i, const RouterID& endpoint);
+      OnIntroSetUpdate(
+          const Address& addr, std::optional<IntroSet> i, const RouterID& endpoint, llarp_time_t);
 
       const dht::Key_t location;
       uint64_t m_UpdateIntrosetTX = 0;
@@ -141,6 +148,8 @@ namespace llarp
       uint16_t m_BuildFails = 0;
       llarp_time_t m_LastInboundTraffic = 0s;
       bool m_GotInboundTraffic = false;
+      bool sentIntro = false;
+      std::function<void(OutboundContext*)> m_ReadyHook;
     };
   }  // namespace service
 

llarp/service/protocol.cpp

@@ -367,13 +367,19 @@ namespace llarp
                 AuthResult result) {
               if (result.code == AuthResultCode::eAuthAccepted)
               {
+                handler->PutSenderFor(msg->tag, msg->sender, true);
                 handler->PutIntroFor(msg->tag, msg->introReply);
                 handler->PutReplyIntroFor(msg->tag, fromIntro);
-                handler->PutSenderFor(msg->tag, msg->sender, true);
                 handler->PutCachedSessionKeyFor(msg->tag, sharedKey);
+                handler->SendAuthResult(path, from, msg->tag, result);
+                LogInfo("auth okay for T=", msg->tag, " from ", msg->sender.Addr());
                 ProtocolMessage::ProcessAsync(path, from, msg);
               }
-              handler->SendAuthResult(path, from, msg->tag, result);
+              else
+              {
+                LogWarn("auth not okay for T=", msg->tag, ": ", result.reason);
+              }
+              handler->Pump(time_now_ms());
             });
       }
     };
@@ -427,12 +433,23 @@ namespace llarp
         LogError("No cached session for T=", T);
         return false;
       }
+      if (v->shared.IsZero())
+      {
+        LogError("bad cached session key for T=", T);
+        return false;
+      }
 
       if (!handler->GetSenderFor(T, v->si))
       {
         LogError("No sender for T=", T);
         return false;
       }
+      if (v->si.Addr().IsZero())
+      {
+        LogError("Bad sender for T=", T);
+        return false;
+      }
+
       v->frame = *this;
       auto callback = [loop, hook](std::shared_ptr<ProtocolMessage> msg) {
         if (hook)