more hidden service code

doc/proto_v0.txt

@@ -557,6 +557,7 @@ ordered data message (variant 2)
   D: "<N bytes encrypted HSD>",
   N: "<32 bytes nonce for symettric cipher>",
   S: sequence_number_uint64,
+  T: "<16 bytes converstation tag>",
   V: 0,
   Z: "<64 bytes signature using sender's signing key>"
 }
@@ -567,11 +568,12 @@ data sent anonymously over the network to a recipiant from a sender.
 sent inside a HSFM encrypted with a shared secret.
 
 {
-  A: protocol_number_uint,
-  D: "<N bytes payload>",
-  I: Introduction for reply,
-  S: SI of sender,
-  V: 0
+  a: protocol_number_uint,
+  d: "<N bytes payload>",
+  i: Introduction for reply,
+  s: SI of sender,
+  t: "<16 bytes converstation tag present only in message 0>",
+  v: 0
 }
 
 transfer data fragment message (TDFM)

include/llarp/dns.h

@@ -10,16 +10,6 @@
 #define uint UINT
 #endif
 
-#ifndef DEFAULT_RESOLVER_US
-#define DEFAULT_RESOLVER_US "128.52.130.209"
-#endif
-#ifndef DEFAULT_RESOLVER_EU
-#define DEFAULT_RESOLVER_EU "85.208.208.141"
-#endif
-#ifndef DEFAULT_RESOLVER_AU
-#define DEFAULT_RESOLVER_AU "103.236.162.119"
-#endif
-
 #ifdef __cplusplus
 extern "C"
 {

include/llarp/encrypted.hpp

@@ -12,6 +12,7 @@ namespace llarp
   struct Encrypted
   {
     Encrypted(Encrypted&&) = delete;
+    Encrypted(const Encrypted& other);
     Encrypted();
     Encrypted(const byte_t* buf, size_t sz);
     Encrypted(size_t sz);

include/llarp/logic.h

@@ -21,6 +21,10 @@ llarp_init_single_process_logic(struct llarp_threadpool* tp);
 void
 llarp_logic_tick(struct llarp_logic* logic);
 
+/// isolated tick
+void
+llarp_logic_tick_async(struct llarp_logic* logic);
+
 void
 llarp_free_logic(struct llarp_logic** logic);
 
@@ -29,6 +33,7 @@ llarp_logic_queue_job(struct llarp_logic* logic, struct llarp_thread_job job);
 
 uint32_t
 llarp_logic_call_later(struct llarp_logic* logic, struct llarp_timeout_job job);
+
 void
 llarp_logic_cancel_call(struct llarp_logic* logic, uint32_t id);
 

include/llarp/service/endpoint.hpp

@@ -10,7 +10,9 @@ namespace llarp
 {
   namespace service
   {
-    struct Endpoint : public llarp_pathbuilder_context, public ILookupHolder
+    struct Endpoint : public llarp_pathbuilder_context,
+                      public ILookupHolder,
+                      public IDataHandler
     {
       /// minimum interval for publishing introsets
       static const llarp_time_t INTROSET_PUBLISH_INTERVAL =
@@ -30,8 +32,13 @@ namespace llarp
       void
       Tick(llarp_time_t now);
 
+      /// router's logic
       llarp_logic*
-      Logic();
+      RouterLogic();
+
+      /// endpoint's logic
+      llarp_logic*
+      EndpointLogic();
 
       llarp_crypto*
       Crypto();
@@ -72,6 +79,12 @@ namespace llarp
       bool
       ForgetPathToService(const Address& remote);
 
+      virtual void
+      HandleDataMessage(ProtocolMessage* msg)
+      {
+        // override me in subclass
+      }
+
       Identity*
       GetIdentity()
       {
@@ -132,7 +145,7 @@ namespace llarp
 
        private:
         void
-        AsyncEncrypt(llarp_buffer_t payload);
+        EncryptAndSendTo(llarp_buffer_t payload);
 
         void
         AsyncGenIntro(llarp_buffer_t payload);
@@ -164,6 +177,29 @@ namespace llarp
         return true;
       }
 
+      void
+      PutSenderFor(const ConvoTag& tag, const ServiceInfo& info);
+
+      bool
+      GetCachedSessionKeyFor(const ConvoTag& remote,
+                             SharedSecret& secret) const;
+      void
+      PutCachedSessionKeyFor(const ConvoTag& remote,
+                             const SharedSecret& secret);
+
+      bool
+      GetSenderFor(const ConvoTag& remote, ServiceInfo& si) const;
+
+      void
+      PutIntroFor(const ConvoTag& remote, const Introduction& intro);
+
+      bool
+      GetIntroFor(const ConvoTag& remote, Introduction& intro) const;
+
+      bool
+      GetConvoTagsForService(const ServiceInfo& si,
+                             std::set< ConvoTag >& tag) const;
+
       void
       PutNewOutboundContext(const IntroSet& introset);
 
@@ -179,18 +215,34 @@ namespace llarp
       void
       PrefetchServicesByTag(const Tag& tag);
 
+      uint64_t
+      GetSeqNoForConvo(const ConvoTag& tag);
+
+      bool
+      IsolateNetwork();
+
      private:
+      static bool
+      SetupIsolatedNetwork(void* user);
+
+      bool
+      DoNetworkIsolation();
+
       uint64_t
       GenTXID();
 
      protected:
       IDataHandler* m_DataHandler = nullptr;
+      Identity m_Identity;
 
      private:
       llarp_router* m_Router;
+      llarp_threadpool* m_IsolatedWorker = nullptr;
+      llarp_logic* m_IsolatedLogic       = nullptr;
       std::string m_Keyfile;
       std::string m_Name;
-      Identity m_Identity;
+      std::string m_NetNS;
+
       std::unordered_map< Address, OutboundContext*, Address::Hash >
           m_RemoteSessions;
       std::unordered_map< Address, PathEnsureHook, Address::Hash >
@@ -208,6 +260,20 @@ namespace llarp
       Tag m_Tag;
       /// prefetch descriptors for these hidden service tags
       std::set< Tag > m_PrefetchTags;
+      /// on initialize functions
+      std::list< std::function< bool(void) > > m_OnInit;
+
+      struct Session
+      {
+        SharedSecret sharedKey;
+        ServiceInfo remote;
+        Introduction intro;
+        llarp_time_t lastUsed = 0;
+        uint64_t seqno        = 0;
+      };
+
+      /// sessions
+      std::unordered_map< ConvoTag, Session, ConvoTag::Hash > m_Sessions;
 
       struct CachedTagResult : public IServiceLookup
       {

include/llarp/service/handler.hpp

@@ -1,15 +1,42 @@
 #ifndef LLARP_SERVICE_HANDLER_HPP
 #define LLARP_SERVICE_HANDLER_HPP
-#include <llarp/service/protocol.hpp>
-
+#include <llarp/aligned.hpp>
+#include <llarp/crypto.hpp>
+#include <llarp/service/IntroSet.hpp>
 namespace llarp
 {
   namespace service
   {
+    typedef llarp::AlignedBuffer< 16 > ConvoTag;
+
+    struct ProtocolMessage;
     struct IDataHandler
     {
       virtual void
       HandleDataMessage(ProtocolMessage* msg) = 0;
+
+      virtual bool
+      GetCachedSessionKeyFor(const ConvoTag& remote,
+                             SharedSecret& secret) const = 0;
+      virtual void
+      PutCachedSessionKeyFor(const ConvoTag& remote,
+                             const SharedSecret& secret) = 0;
+
+      virtual void
+      PutSenderFor(const ConvoTag& remote, const ServiceInfo& si) = 0;
+
+      virtual bool
+      GetSenderFor(const ConvoTag& remote, ServiceInfo& si) const = 0;
+
+      virtual void
+      PutIntroFor(const ConvoTag& remote, const Introduction& intro) = 0;
+
+      virtual bool
+      GetIntroFor(const ConvoTag& remote, Introduction& intro) const = 0;
+
+      virtual bool
+      GetConvoTagsForService(const ServiceInfo& si,
+                             std::set< ConvoTag >& tag) const = 0;
     };
   }  // namespace service
 }  // namespace llarp

include/llarp/service/protocol.hpp

@@ -7,6 +7,7 @@
 #include <llarp/routing/message.hpp>
 #include <llarp/service/Info.hpp>
 #include <llarp/service/Intro.hpp>
+#include <llarp/service/handler.hpp>
 #include <vector>
 
 namespace llarp
@@ -15,30 +16,36 @@ namespace llarp
   {
     constexpr std::size_t MAX_PROTOCOL_MESSAGE_SIZE = 2048;
 
-    enum ProtocolType
-    {
-      eProtocolText    = 0,
-      eProtocolTraffic = 1
-    };
+    typedef uint64_t ProtocolType;
+
+    constexpr ProtocolType eProtocolText    = 0UL;
+    constexpr ProtocolType eProtocolTraffic = 1UL;
 
     /// inner message
-    struct ProtocolMessage : public llarp::IBEncodeMessage
+    struct ProtocolMessage : public IBEncodeMessage
     {
+      ProtocolMessage(const ConvoTag& tag);
       ProtocolMessage();
       ~ProtocolMessage();
-      ProtocolType proto;
+      ProtocolType proto  = eProtocolText;
       llarp_time_t queued = 0;
       std::vector< byte_t > payload;
       Introduction introReply;
       ServiceInfo sender;
+      IDataHandler* handler = nullptr;
+      ConvoTag tag;
 
       bool
       DecodeKey(llarp_buffer_t key, llarp_buffer_t* val);
+
       bool
       BEncode(llarp_buffer_t* buf) const;
 
       void
       PutBuffer(llarp_buffer_t payload);
+
+      static void
+      ProcessAsync(void* user);
     };
 
     /// outer message
@@ -48,6 +55,10 @@ namespace llarp
       llarp::PubKey H;
       llarp::KeyExchangeNonce N;
       llarp::Signature Z;
+      llarp::service::ConvoTag T;
+
+      ProtocolFrame();
+      ProtocolFrame(const ProtocolFrame& other);
 
       ~ProtocolFrame();
 
@@ -55,6 +66,11 @@ namespace llarp
       EncryptAndSign(llarp_crypto* c, const ProtocolMessage* msg,
                      byte_t* sharedkey, byte_t* signingkey);
 
+      bool
+      AsyncDecryptAndVerify(llarp_logic* logic, llarp_crypto* c,
+                            llarp_threadpool* worker, byte_t* localSecret,
+                            IDataHandler* handler) const;
+
       bool
       DecryptPayloadInto(llarp_crypto* c, byte_t* sharedkey,
                          ProtocolMessage* into) const;
@@ -66,7 +82,7 @@ namespace llarp
       BEncode(llarp_buffer_t* buf) const;
 
       bool
-      Verify(llarp_crypto* c, byte_t* signingkey);
+      Verify(llarp_crypto* c, byte_t* signingkey) const;
 
       bool
       HandleMessage(llarp::routing::IMessageHandler* h, llarp_router* r) const;

include/llarp/threadpool.h

@@ -10,11 +10,12 @@ llarp_init_threadpool(int workers, const char *name);
 struct llarp_threadpool *
 llarp_init_same_process_threadpool();
 
-typedef bool (*setup_net_func)(void);
+typedef bool (*setup_net_func)(void *);
 
 /// for network isolation
 struct llarp_threadpool *
-llarp_init_isolated_net_threadpool(const char *name, setup_net_func setupNet);
+llarp_init_isolated_net_threadpool(const char *name, setup_net_func setupNet,
+                                   void *context);
 
 void
 llarp_free_threadpool(struct llarp_threadpool **tp);

include/llarp/timer.h

@@ -38,8 +38,12 @@ llarp_timer_run(struct llarp_timer_context *t, struct llarp_threadpool *pool);
 
 /// single threaded run timer, tick all timers
 void
-llarp_timer_tick_all(struct llarp_timer_context *t,
-                     struct llarp_threadpool *pool);
+llarp_timer_tick_all(struct llarp_timer_context *t);
+
+/// tick all timers into a threadpool asynchronously
+void
+llarp_timer_tick_all_async(struct llarp_timer_context *t,
+                           struct llarp_threadpool *pool);
 
 void
 llarp_free_timer(struct llarp_timer_context **t);

include/tuntap.h

@@ -27,7 +27,6 @@
 
 #include <sys/types.h>
 #if defined Windows
-#include <In6addr.h>
 #else /* Unix */
 #include <sys/socket.h>
 #endif

llarp/encrypted_frame.cpp

@@ -10,6 +10,11 @@ namespace llarp
     UpdateBuffer();
   }
 
+  Encrypted::Encrypted(const Encrypted& other)
+      : Encrypted(other._data, other._sz)
+  {
+  }
+
   Encrypted::Encrypted(const byte_t* buf, size_t sz) : _sz(sz)
   {
     _data = new byte_t[sz];

llarp/logic.cpp

@@ -29,10 +29,16 @@ llarp_init_single_process_logic(struct llarp_threadpool* tp)
 void
 llarp_logic_tick(struct llarp_logic* logic)
 {
-  llarp_timer_tick_all(logic->timer, logic->thread);
+  llarp_timer_tick_all(logic->timer);
   llarp_threadpool_tick(logic->thread);
 }
 
+void
+llarp_logic_tick_async(struct llarp_logic* logic)
+{
+  llarp_timer_tick_all_async(logic->timer, logic->thread);
+}
+
 void
 llarp_free_logic(struct llarp_logic** logic)
 {

llarp/service/endpoint.cpp

@@ -38,6 +38,20 @@ namespace llarp
         if(addr.FromString(v))
           m_PrefetchAddrs.insert(addr);
       }
+      if(k == "netns")
+      {
+        m_NetNS = v;
+        m_OnInit.push_back(std::bind(&Endpoint::IsolateNetwork, this));
+      }
+      return true;
+    }
+
+    bool
+    Endpoint::IsolateNetwork()
+    {
+      m_IsolatedWorker = llarp_init_isolated_net_threadpool(
+          m_Name.c_str(), &SetupIsolatedNetwork, this);
+      m_IsolatedLogic = llarp_init_single_process_logic(m_IsolatedWorker);
       return true;
     }
 
@@ -68,6 +82,12 @@ namespace llarp
       }
     };
 
+    bool
+    Endpoint::SetupIsolatedNetwork(void* user)
+    {
+      return static_cast< Endpoint* >(user)->DoNetworkIsolation();
+    }
+
     void
     Endpoint::Tick(llarp_time_t now)
     {
@@ -254,6 +274,89 @@ namespace llarp
       return result;
     }
 
+    void
+    Endpoint::PutSenderFor(const ConvoTag& tag, const ServiceInfo& info)
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+      {
+        itr = m_Sessions.insert(std::make_pair(tag, Session{})).first;
+      }
+      itr->second.remote   = info;
+      itr->second.lastUsed = llarp_time_now_ms();
+    }
+
+    bool
+    Endpoint::GetSenderFor(const ConvoTag& tag, ServiceInfo& si) const
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+        return false;
+      si = itr->second.remote;
+      return true;
+    }
+
+    void
+    Endpoint::PutIntroFor(const ConvoTag& tag, const Introduction& intro)
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+      {
+        itr = m_Sessions.insert(std::make_pair(tag, Session{})).first;
+      }
+      itr->second.intro    = intro;
+      itr->second.lastUsed = llarp_time_now_ms();
+    }
+
+    bool
+    Endpoint::GetIntroFor(const ConvoTag& tag, Introduction& intro) const
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+        return false;
+      intro = itr->second.intro;
+      return true;
+    }
+
+    bool
+    Endpoint::GetConvoTagsForService(const ServiceInfo& info,
+                                     std::set< ConvoTag >& tags) const
+    {
+      bool inserted = false;
+      auto itr      = m_Sessions.begin();
+      while(itr != m_Sessions.end())
+      {
+        if(itr->second.remote == info)
+        {
+          inserted |= tags.insert(itr->first).second;
+        }
+      }
+      return inserted;
+    }
+
+    bool
+    Endpoint::GetCachedSessionKeyFor(const ConvoTag& tag,
+                                     SharedSecret& secret) const
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+        return false;
+      secret = itr->second.sharedKey;
+      return true;
+    }
+
+    void
+    Endpoint::PutCachedSessionKeyFor(const ConvoTag& tag, const SharedSecret& k)
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+      {
+        itr = m_Sessions.insert(std::make_pair(tag, Session{})).first;
+      }
+      itr->second.sharedKey = k;
+      itr->second.lastUsed  = llarp_time_now_ms();
+    }
+
     bool
     Endpoint::Start()
     {
@@ -267,7 +370,17 @@ namespace llarp
       {
         m_Identity.RegenerateKeys(crypto);
       }
-
+      if(!m_DataHandler)
+      {
+        m_DataHandler = this;
+      }
+      while(m_OnInit.size())
+      {
+        if(m_OnInit.front()())
+          m_OnInit.pop_front();
+        else
+          return false;
+      }
       return true;
     }
 
@@ -409,6 +522,13 @@ namespace llarp
       }
     };
 
+    bool
+    Endpoint::DoNetworkIsolation()
+    {
+      /// TODO: implement me
+      return false;
+    }
+
     void
     Endpoint::PutNewOutboundContext(const llarp::service::IntroSet& introset)
     {
@@ -442,8 +562,8 @@ namespace llarp
     bool
     Endpoint::HandleHiddenServiceFrame(const ProtocolFrame* frame)
     {
-      llarp::LogInfo("handle hidden service frame");
-      return true;
+      return frame->AsyncDecryptAndVerify(EndpointLogic(), Crypto(), Worker(),
+                                          m_Identity.enckey, m_DataHandler);
     }
 
     void
@@ -572,7 +692,7 @@ namespace llarp
     {
       if(sequenceNo)
       {
-        AsyncEncrypt(data);
+        EncryptAndSendTo(data);
       }
       else
       {
@@ -585,19 +705,24 @@ namespace llarp
       llarp_logic* logic;
       llarp_crypto* crypto;
       byte_t* sharedKey;
-      byte_t* remotePubkey;
+      ServiceInfo remote;
       Identity* m_LocalIdentity;
       ProtocolMessage msg;
       ProtocolFrame frame;
+      Introduction intro;
       std::function< void(ProtocolFrame&) > hook;
+      IDataHandler* handler;
 
       AsyncIntroGen(llarp_logic* l, llarp_crypto* c, byte_t* key,
-                    byte_t* remote, Identity* localident)
+                    const ServiceInfo& r, Identity* localident,
+                    const Introduction& us, IDataHandler* h)
           : logic(l)
           , crypto(c)
           , sharedKey(key)
-          , remotePubkey(remote)
+          , remote(r)
           , m_LocalIdentity(localident)
+          , intro(us)
+          , handler(h)
       {
       }
 
@@ -605,6 +730,10 @@ namespace llarp
       Result(void* user)
       {
         AsyncIntroGen* self = static_cast< AsyncIntroGen* >(user);
+        // put values
+        self->handler->PutCachedSessionKeyFor(self->msg.tag, self->sharedKey);
+        self->handler->PutIntroFor(self->msg.tag, self->msg.introReply);
+        self->handler->PutSenderFor(self->msg.tag, self->remote);
         self->hook(self->frame);
         delete self;
       }
@@ -615,9 +744,16 @@ namespace llarp
         AsyncIntroGen* self = static_cast< AsyncIntroGen* >(user);
         // randomize Nounce
         self->frame.N.Randomize();
+        // randomize tag
+        self->msg.tag.Randomize();
+        // set sender
+        self->msg.sender = self->m_LocalIdentity->pub;
+        // set our introduction
+        self->msg.introReply = self->intro;
         // derive session key
-        self->crypto->dh_server(self->sharedKey, self->remotePubkey,
+        self->crypto->dh_server(self->sharedKey, self->remote.enckey,
                                 self->m_LocalIdentity->enckey, self->frame.N);
+
         // encrypt and sign
         self->frame.EncryptAndSign(self->crypto, &self->msg, self->sharedKey,
                                    self->m_LocalIdentity->signkey);
@@ -629,9 +765,10 @@ namespace llarp
     void
     Endpoint::OutboundContext::AsyncGenIntro(llarp_buffer_t payload)
     {
-      AsyncIntroGen* ex =
-          new AsyncIntroGen(m_Parent->Logic(), m_Parent->Crypto(), sharedKey,
-                            currentIntroSet.A.enckey, m_Parent->GetIdentity());
+      AsyncIntroGen* ex = new AsyncIntroGen(
+          m_Parent->RouterLogic(), m_Parent->Crypto(), sharedKey,
+          currentIntroSet.A, m_Parent->GetIdentity(), selectedIntro,
+          m_Parent->m_DataHandler);
       ex->hook = std::bind(&Endpoint::OutboundContext::Send, this,
                            std::placeholders::_1);
 
@@ -703,7 +840,8 @@ namespace llarp
     bool
     Endpoint::OutboundContext::Tick(llarp_time_t now)
     {
-      if(selectedIntro.expiresAt >= now || selectedIntro.expiresAt - now < 5000)
+      if(selectedIntro.expiresAt >= now
+         || selectedIntro.expiresAt - now < 30000)
       {
         UpdateIntroSet();
       }
@@ -729,7 +867,8 @@ namespace llarp
         {
           // we don't have it?
           llarp::LogError(
-              "cannot build aligned path, don't have router for introduction ",
+              "cannot build aligned path, don't have router for "
+              "introduction ",
               selectedIntro);
           return false;
         }
@@ -738,18 +877,82 @@ namespace llarp
         return llarp_pathbuilder_context::SelectHop(db, prev, cur, hop);
     }
 
+    uint64_t
+    Endpoint::GetSeqNoForConvo(const ConvoTag& tag)
+    {
+      auto itr = m_Sessions.find(tag);
+      if(itr == m_Sessions.end())
+        return 0;
+      return ++(itr->second.seqno);
+    }
+
     void
-    Endpoint::OutboundContext::AsyncEncrypt(llarp_buffer_t payload)
+    Endpoint::OutboundContext::EncryptAndSendTo(llarp_buffer_t payload)
     {
-      // TODO: implement me
+      auto path = GetPathByRouter(selectedIntro.router);
+      if(path)
+      {
+        std::set< ConvoTag > tags;
+        if(!m_Parent->m_DataHandler->GetConvoTagsForService(currentIntroSet.A,
+                                                            tags))
+        {
+          llarp::LogError("no open converstations with remote endpoint?");
+          return;
+        }
+        auto crypto = m_Parent->Crypto();
+        SharedSecret shared;
+
+        ProtocolFrame f;
+        f.N.Randomize();
+        f.T = *tags.begin();
+        f.S = m_Parent->GetSeqNoForConvo(f.T);
+
+        if(m_Parent->m_DataHandler->GetCachedSessionKeyFor(f.T, shared))
+        {
+          ProtocolMessage msg;
+          msg.introReply = selectedIntro;
+          msg.sender     = m_Parent->m_Identity.pub;
+          msg.PutBuffer(payload);
+
+          if(!f.EncryptAndSign(crypto, &msg, shared, currentIntroSet.A.signkey))
+          {
+            llarp::LogError("failed to sign");
+            return;
+          }
+        }
+        else
+        {
+          llarp::LogError("No cached session key");
+          return;
+        }
+
+        routing::PathTransferMessage msg;
+        msg.P = selectedIntro.pathID;
+        msg.Y.Randomize();
+        msg.T = &f;
+        if(!path->SendRoutingMessage(&msg, m_Parent->Router()))
+        {
+          llarp::LogWarn("Failed to send routing message for data");
+        }
+      }
+      else
+      {
+        llarp::LogError("no outbound path for sending message");
+      }
     }
 
     llarp_logic*
-    Endpoint::Logic()
+    Endpoint::RouterLogic()
     {
       return m_Router->logic;
     }
 
+    llarp_logic*
+    Endpoint::EndpointLogic()
+    {
+      return m_IsolatedLogic ? m_IsolatedLogic : m_Router->logic;
+    }
+
     llarp_crypto*
     Endpoint::Crypto()
     {

llarp/service/protocol.cpp

@@ -8,25 +8,15 @@ namespace llarp
   {
     ProtocolMessage::ProtocolMessage()
     {
+      tag.Zero();
     }
 
-    ProtocolMessage::~ProtocolMessage()
-    {
-    }
-
-    bool
-    ProtocolMessage::BEncode(llarp_buffer_t* buf) const
+    ProtocolMessage::ProtocolMessage(const ConvoTag& t) : tag(t)
     {
-      if(!bencode_start_dict(buf))
-        return false;
-      return bencode_end(buf);
     }
 
-    bool
-    ProtocolMessage::DecodeKey(llarp_buffer_t key, llarp_buffer_t* val)
+    ProtocolMessage::~ProtocolMessage()
     {
-      // TODO: implement me
-      return false;
     }
 
     void
@@ -37,6 +27,64 @@ namespace llarp
       payload.shrink_to_fit();
     }
 
+    void
+    ProtocolMessage::ProcessAsync(void* user)
+    {
+      ProtocolMessage* self = static_cast< ProtocolMessage* >(user);
+      self->handler->HandleDataMessage(self);
+      delete self;
+    }
+
+    bool
+    ProtocolMessage::DecodeKey(llarp_buffer_t k, llarp_buffer_t* buf)
+    {
+      bool read = false;
+      if(!BEncodeMaybeReadDictInt("a", proto, read, k, buf))
+        return false;
+      if(llarp_buffer_eq(k, "d"))
+      {
+        llarp_buffer_t strbuf;
+        if(!bencode_read_string(buf, &strbuf))
+          return false;
+        PutBuffer(strbuf);
+        return true;
+      }
+      if(!BEncodeMaybeReadDictEntry("i", introReply, read, k, buf))
+        return false;
+      if(!BEncodeMaybeReadDictEntry("s", sender, read, k, buf))
+        return false;
+      if(!BEncodeMaybeReadDictEntry("t", tag, read, k, buf))
+        return false;
+      if(!BEncodeMaybeReadDictInt("v", version, read, k, buf))
+        return false;
+      return read;
+    }
+
+    bool
+    ProtocolMessage::BEncode(llarp_buffer_t* buf) const
+    {
+      if(!bencode_start_dict(buf))
+        return false;
+      if(!BEncodeWriteDictInt("a", proto, buf))
+        return false;
+      if(!bencode_write_bytestring(buf, "d", 1))
+        return false;
+      if(!bencode_write_bytestring(buf, payload.data(), payload.size()))
+        return false;
+      if(!BEncodeWriteDictEntry("i", introReply, buf))
+        return false;
+      if(!BEncodeWriteDictEntry("s", sender, buf))
+        return false;
+      if(!tag.IsZero())
+      {
+        if(!BEncodeWriteDictEntry("t", tag, buf))
+          return false;
+      }
+      if(!bencode_write_version_entry(buf))
+        return false;
+      return bencode_end(buf);
+    }
+
     ProtocolFrame::~ProtocolFrame()
     {
     }
@@ -59,6 +107,11 @@ namespace llarp
         return false;
       if(!BEncodeWriteDictInt("S", S, buf))
         return false;
+      if(S == 0)
+      {
+        if(!BEncodeWriteDictEntry("T", T, buf))
+          return false;
+      }
       if(!BEncodeWriteDictInt("V", version, buf))
         return false;
       if(!BEncodeWriteDictEntry("Z", Z, buf))
@@ -78,6 +131,8 @@ namespace llarp
         return false;
       if(!BEncodeMaybeReadDictInt("S", S, read, key, val))
         return false;
+      if(!BEncodeMaybeReadDictEntry("T", T, read, key, val))
+        return false;
       if(!BEncodeMaybeReadVersion("V", version, LLARP_PROTO_VERSION, read, key,
                                   val))
         return false;
@@ -86,6 +141,16 @@ namespace llarp
       return read;
     }
 
+    bool
+    ProtocolFrame::DecryptPayloadInto(llarp_crypto* crypto, byte_t* sharedkey,
+                                      ProtocolMessage* msg) const
+    {
+      auto buf = D.Buffer();
+      crypto->xchacha20(buf, sharedkey, N);
+      msg->PutBuffer(buf);
+      return true;
+    }
+
     bool
     ProtocolFrame::EncryptAndSign(llarp_crypto* crypto,
                                   const ProtocolMessage* msg,
@@ -110,27 +175,139 @@ namespace llarp
       return crypto->sign(Z, signingkey, buf);
     }
 
+    struct AsyncFrameDH
+    {
+      llarp_crypto* crypto;
+      llarp_logic* logic;
+      ProtocolMessage* msg;
+      byte_t* localSecret;
+      PubKey H;
+      KeyExchangeNonce N;
+      IDataHandler* handler;
+      Address remote;
+      Encrypted D;
+
+      AsyncFrameDH(llarp_logic* l, llarp_crypto* c, byte_t* sec,
+                   IDataHandler* h, ProtocolMessage* m,
+                   const ProtocolFrame* frame)
+          : crypto(c)
+          , logic(l)
+          , msg(m)
+          , localSecret(sec)
+          , H(frame->H)
+          , N(frame->N)
+          , handler(h)
+          , D(frame->D)
+      {
+      }
+
+      static void
+      Work(void* user)
+      {
+        AsyncFrameDH* self = static_cast< AsyncFrameDH* >(user);
+        auto crypto        = self->crypto;
+        SharedSecret shared;
+        if(!crypto->dh_client(shared, self->H, self->localSecret, self->N))
+        {
+          llarp::LogError("Failed to derive shared secret for initial message");
+          delete self->msg;
+          delete self;
+          return;
+        }
+        auto buf = self->D.Buffer();
+        crypto->xchacha20(*buf, shared, self->N);
+        if(!self->msg->BDecode(buf))
+        {
+          llarp::LogError("failed to decode inner protocol message");
+          delete self->msg;
+          delete self;
+          return;
+        }
+        self->handler->PutIntroFor(self->msg->tag, self->msg->introReply);
+        self->handler->PutSenderFor(self->msg->tag, self->msg->sender);
+        self->handler->PutCachedSessionKeyFor(self->msg->tag, shared);
+        self->msg->handler = self->handler;
+        llarp_logic_queue_job(self->logic,
+                              {self->msg, &ProtocolMessage::ProcessAsync});
+        delete self;
+      }
+    };
+
+    bool
+    ProtocolFrame::AsyncDecryptAndVerify(llarp_logic* logic,
+                                         llarp_crypto* crypto,
+                                         llarp_threadpool* worker,
+                                         byte_t* localSecret,
+                                         IDataHandler* handler) const
+    {
+      if(S == 0)
+      {
+        ProtocolMessage* msg = new ProtocolMessage();
+        // we need to dh
+        auto dh =
+            new AsyncFrameDH(logic, crypto, localSecret, handler, msg, this);
+        llarp_threadpool_queue_job(worker, {dh, &AsyncFrameDH::Work});
+        return true;
+      }
+      SharedSecret shared;
+      if(!handler->GetCachedSessionKeyFor(T, shared))
+      {
+        llarp::LogError("No cached session for T=", T);
+        return false;
+      }
+      ServiceInfo si;
+      if(!handler->GetSenderFor(T, si))
+      {
+        llarp::LogError("No sender for T=", T);
+        return false;
+      }
+      if(!Verify(crypto, si.signkey))
+      {
+        llarp::LogError("Signature failure");
+        return false;
+      }
+      ProtocolMessage* msg = new ProtocolMessage();
+      if(!DecryptPayloadInto(crypto, shared, msg))
+      {
+        llarp::LogError("failed to decrypt message");
+        delete msg;
+        return false;
+      }
+      msg->handler = handler;
+      llarp_logic_queue_job(logic, {msg, &ProtocolMessage::ProcessAsync});
+      return true;
+    }
+
+    ProtocolFrame::ProtocolFrame()
+    {
+      T.Zero();
+    }
+
+    ProtocolFrame::ProtocolFrame(const ProtocolFrame& other)
+        : D(other.D), H(other.H), N(other.N), Z(other.Z), T(other.T)
+    {
+    }
+
     bool
-    ProtocolFrame::Verify(llarp_crypto* crypto, byte_t* signkey)
+    ProtocolFrame::Verify(llarp_crypto* crypto, byte_t* signkey) const
     {
+      ProtocolFrame copy(*this);
       // save signature
-      llarp::Signature sig = Z;
       // zero out signature for verify
-      Z.Zero();
+      copy.Z.Zero();
       bool result = false;
       // serialize
       byte_t tmp[MAX_PROTOCOL_MESSAGE_SIZE];
       auto buf = llarp::StackBuffer< decltype(tmp) >(tmp);
-      if(BEncode(&buf))
+      if(copy.BEncode(&buf))
       {
         // rewind buffer
         buf.sz  = buf.cur - buf.base;
         buf.cur = buf.base;
         // verify
-        result = crypto->verify(sig, buf, signkey);
+        result = crypto->verify(Z, buf, signkey);
       }
       // restore signature
-      Z = sig;
       return result;
     }
 

llarp/threadpool.cpp

@@ -163,16 +163,20 @@ namespace llarp
     }
 
 #ifdef __linux__
-    NetIsolatedPool::NetIsolatedPool(std::function< bool(void) > setupNet)
+    NetIsolatedPool::NetIsolatedPool(std::function< bool(void *) > setupNet,
+                                     void *user)
         : IsolatedPool(CLONE_NEWNET)
     {
       m_NetSetup = setupNet;
+      m_user     = user;
     }
 #else
-    NetIsolatedPool::NetIsolatedPool(std::function< bool(void) > setupNet)
+    NetIsolatedPool::NetIsolatedPool(std::function< bool(void *) > setupNet,
+                                     void *user)
         : IsolatedPool(0)
     {
       m_NetSetup = setupNet;
+      m_user = user;
     }
 #endif
   }  // namespace thread
@@ -186,10 +190,10 @@ struct llarp_threadpool
   std::queue< llarp_thread_job * > jobs;
 
   llarp_threadpool(int workers, const char *name, bool isolate,
-                   setup_net_func setup = nullptr)
+                   setup_net_func setup = nullptr, void *user = nullptr)
   {
     if(isolate)
-      impl = new llarp::thread::NetIsolatedPool(setup);
+      impl = new llarp::thread::NetIsolatedPool(setup, user);
     else
       impl = new llarp::thread::Pool();
     impl->Spawn(workers, name);
@@ -216,9 +220,10 @@ llarp_init_same_process_threadpool()
 }
 
 struct llarp_threadpool *
-llarp_init_isolated_net_threadpool(const char *name, setup_net_func setup)
+llarp_init_isolated_net_threadpool(const char *name, setup_net_func setup,
+                                   void *context)
 {
-  return new llarp_threadpool(1, name, true, setup);
+  return new llarp_threadpool(1, name, true, setup, context);
 }
 
 void

llarp/threadpool.hpp

@@ -83,15 +83,16 @@ namespace llarp
 
     struct NetIsolatedPool : public IsolatedPool
     {
-      NetIsolatedPool(std::function< bool(void) > setupNet);
+      NetIsolatedPool(std::function< bool(void*) > setupNet, void* user);
 
       bool
       Isolated()
       {
-        return m_NetSetup();
+        return m_NetSetup(m_user);
       }
 
-      std::function< bool(void) > m_NetSetup;
+      std::function< bool(void*) > m_NetSetup;
+      void* m_user;
     };
 
   }  // namespace thread

llarp/timer.cpp

@@ -179,8 +179,7 @@ llarp_timer_cancel_job(struct llarp_timer_context* t, uint32_t id)
 }
 
 void
-llarp_timer_tick_all(struct llarp_timer_context* t,
-                     struct llarp_threadpool* pool)
+llarp_timer_tick_all(struct llarp_timer_context* t)
 {
   if(!t->run())
     return;
@@ -206,6 +205,19 @@ llarp_timer_tick_all(struct llarp_timer_context* t,
   }
 }
 
+static void
+llarp_timer_tick_all_job(void* user)
+{
+  llarp_timer_tick_all(static_cast< llarp_timer_context* >(user));
+}
+
+void
+llarp_timer_tick_all_async(struct llarp_timer_context* t,
+                           struct llarp_threadpool* pool)
+{
+  llarp_threadpool_queue_job(pool, {t, llarp_timer_tick_all_job});
+}
+
 void
 llarp_timer_run(struct llarp_timer_context* t, struct llarp_threadpool* pool)
 {
@@ -223,7 +235,7 @@ llarp_timer_run(struct llarp_timer_context* t, struct llarp_threadpool* pool)
     {
       std::unique_lock< std::mutex > lock(t->timersMutex);
       // we woke up
-      llarp_timer_tick_all(t, pool);
+      llarp_timer_tick_all_async(t, pool);
     }
   }
 }