* logging updates

* nicknames in RC (yw kee)

* spec update

* more hidden service code

Makefile

@@ -77,7 +77,10 @@ shadow-plot: shadow-run
 
 shadow: shadow-plot
 
-testnet-configure: clean
+testnet-clean: clean
+	rm -rf $(TESTNET_ROOT)
+
+testnet-configure: testnet-clean
 	cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=$(CC) -DCMAKE_CXX_COMPILER=$(CXX)
 
 testnet-build: testnet-configure

doc/proto_v0.txt

@@ -121,6 +121,7 @@ router's full identity
 {
   a: [ one, or, many, AI, here ... ],
   k: "<32 bytes public long term identity signing key>",
+  n: "<optional max 32 bytes router nickname>",
   p: "<32 bytes public path encryption key>",   
   u: last_updated_seconds_since_epoch_uint64,
   v: 0,

include/llarp/logger.h

@@ -1,15 +1,21 @@
 #ifndef LLARP_LOGGER_H
 #define LLARP_LOGGER_H
 
-enum LogLevel
+extern "C"
 {
-  eLogDebug,
-  eLogInfo,
-  eLogWarn,
-  eLogError
-};
-
-void
-cSetLogLevel(enum LogLevel lvl);
+  enum LogLevel
+  {
+    eLogDebug,
+    eLogInfo,
+    eLogWarn,
+    eLogError
+  };
+
+  void
+  cSetLogLevel(enum LogLevel lvl);
+
+  void
+  cSetLogNodeName(const char* name);
+}
 
 #endif

include/llarp/logger.hpp

@@ -22,13 +22,15 @@ namespace llarp
 
   struct Logger
   {
+    std::string nodeName;
     LogLevel minlevel = eLogInfo;
     std::ostream& out;
     std::mutex access;
-    Logger() : Logger(std::cout)
+    Logger() : Logger(std::cout, "unnamed")
     {
     }
-    Logger(std::ostream& o) : out(o)
+
+    Logger(std::ostream& o, const std::string& name) : nodeName(name), out(o)
     {
     }
   };
@@ -57,7 +59,7 @@ namespace llarp
   /** internal */
   template < typename... TArgs >
   void
-  _Log(LogLevel lvl, const char* fname, TArgs&&... args) noexcept
+  _Log(LogLevel lvl, const char* fname, int lineno, TArgs&&... args) noexcept
   {
     if(_glog.minlevel > lvl)
       return;
@@ -83,7 +85,8 @@ namespace llarp
         break;
     }
     std::string tag = fname;
-    ss << llarp_time_now_ms() << " " << tag;
+    ss << _glog.nodeName << " " << llarp_time_now_ms() << " " << tag << ":"
+       << lineno;
     ss << "\t";
     LogAppend(ss, std::forward< TArgs >(args)...);
     ss << (char)27 << "[0;0m";
@@ -97,15 +100,23 @@ namespace llarp
   }
 }  // namespace llarp
 
-#define LogDebug(x, ...) _Log(llarp::eLogDebug, LOG_TAG, x, ##__VA_ARGS__)
-#define LogInfo(x, ...) _Log(llarp::eLogInfo, LOG_TAG, x, ##__VA_ARGS__)
-#define LogWarn(x, ...) _Log(llarp::eLogWarn, LOG_TAG, x, ##__VA_ARGS__)
-#define LogError(x, ...) _Log(llarp::eLogError, LOG_TAG, x, ##__VA_ARGS__)
+#define LogDebug(x, ...) \
+  _Log(llarp::eLogDebug, LOG_TAG, __LINE__, x, ##__VA_ARGS__)
+#define LogInfo(x, ...) \
+  _Log(llarp::eLogInfo, LOG_TAG, __LINE__, x, ##__VA_ARGS__)
+#define LogWarn(x, ...) \
+  _Log(llarp::eLogWarn, LOG_TAG, __LINE__, x, ##__VA_ARGS__)
+#define LogError(x, ...) \
+  _Log(llarp::eLogError, LOG_TAG, __LINE__, x, ##__VA_ARGS__)
 
-#define LogDebugTag(tag, x, ...) _Log(llarp::eLogDebug, tag, x, ##__VA_ARGS__)
-#define LogInfoTag(tag, x, ...) _Log(llarp::eLogInfo, tag, x, ##__VA_ARGS__)
-#define LogWarnTag(tag, x, ...) _Log(llarp::eLogWarn, tag, x, ##__VA_ARGS__)
-#define LogErrorTag(tag, x, ...) _Log(llarp::eLogError, tag, x, ##__VA_ARGS__)
+#define LogDebugTag(tag, x, ...) \
+  _Log(llarp::eLogDebug, tag, __LINE__, x, ##__VA_ARGS__)
+#define LogInfoTag(tag, x, ...) \
+  _Log(llarp::eLogInfo, tag, __LINE__, x, ##__VA_ARGS__)
+#define LogWarnTag(tag, x, ...) \
+  _Log(llarp::eLogWarn, tag, __LINE__, x, ##__VA_ARGS__)
+#define LogErrorTag(tag, x, ...) \
+  _Log(llarp::eLogError, tag, __LINE__, x, ##__VA_ARGS__)
 
 #ifndef LOG_TAG
 #define LOG_TAG "default"

include/llarp/path.hpp

@@ -163,6 +163,14 @@ namespace llarp
       HandlePathLatencyMessage(const llarp::routing::PathLatencyMessage* msg,
                                llarp_router* r);
 
+      bool
+      HandleHiddenServiceFrame(const llarp::service::ProtocolFrame* frame)
+      {
+        /// TODO: implement me
+        llarp::LogWarn("Got hidden service data on transit hop");
+        return false;
+      }
+
       bool
       HandleGotIntroMessage(const llarp::dht::GotIntroMessage* msg);
 
@@ -207,6 +215,9 @@ namespace llarp
     {
       typedef std::function< void(Path*) > BuildResultHookFunc;
       typedef std::vector< PathHopConfig > HopList;
+      typedef std::function< bool(const service::ProtocolFrame*) >
+          DataHandlerFunc;
+
       HopList hops;
 
       llarp::service::Introduction intro;
@@ -219,6 +230,12 @@ namespace llarp
       void
       SetBuildResultHook(BuildResultHookFunc func);
 
+      void
+      SetDataHandler(DataHandlerFunc func)
+      {
+        m_DataHandler = func;
+      }
+
       bool
       Expired(llarp_time_t now) const;
 
@@ -240,6 +257,9 @@ namespace llarp
       HandlePathTransferMessage(const llarp::routing::PathTransferMessage* msg,
                                 llarp_router* r);
 
+      bool
+      HandleHiddenServiceFrame(const llarp::service::ProtocolFrame* frame);
+
       bool
       HandleGotIntroMessage(const llarp::dht::GotIntroMessage* msg);
 
@@ -279,6 +299,7 @@ namespace llarp
 
      private:
       BuildResultHookFunc m_BuiltHook;
+      DataHandlerFunc m_DataHandler;
       llarp_time_t m_LastLatencyTestTime = 0;
       uint64_t m_LastLatencyTestID       = 0;
     };

include/llarp/pathbuilder.hpp

@@ -2,12 +2,14 @@
 #define LLARP_PATHBUILDER_HPP_
 #include <llarp/pathbuilder.h>
 #include <llarp/router.h>
+#include <llarp/path.hpp>
 #include <llarp/pathset.hpp>
 
 struct llarp_pathbuilder_context : public llarp::path::PathSet
 {
   struct llarp_router* router;
   struct llarp_dht_context* dht;
+  llarp::SecretKey enckey;
   size_t numHops;
   /// construct
   llarp_pathbuilder_context(llarp_router* p_router,
@@ -21,6 +23,9 @@ struct llarp_pathbuilder_context : public llarp::path::PathSet
 
   void
   BuildOne();
+
+  virtual byte_t*
+  GetTunnelEncryptionSecretKey();
 };
 
 #endif

include/llarp/pathset.hpp

@@ -44,7 +44,7 @@ namespace llarp
       void
       RemovePath(Path* path);
 
-      void
+      virtual void
       HandlePathBuilt(Path* path);
 
       void

include/llarp/router_contact.h

@@ -4,11 +4,16 @@
 #include <llarp/crypto.h>
 #include <llarp/exit_info.h>
 
+#ifdef __cplusplus
+#include <string>
+#endif
+
 // forward declare
 struct llarp_alloc;
 struct llarp_rc;
 
 #define MAX_RC_SIZE (1024)
+#define NICKLEN (32)
 
 bool
 llarp_rc_bdecode(struct llarp_rc *rc, llarp_buffer_t *buf);
@@ -24,6 +29,9 @@ struct llarp_rc
   byte_t pubkey[PUBKEYSIZE];
   struct llarp_xi_list *exits;
   byte_t signature[SIGSIZE];
+  /// node nickname, yw kee
+  byte_t nickname[NICKLEN];
+
   uint64_t last_updated;
 
 #ifdef __cplusplus
@@ -38,6 +46,14 @@ struct llarp_rc
   {
     return llarp_rc_bdecode(this, buf);
   }
+
+  std::string
+  Nick() const
+  {
+    const char *nick = (const char *)nickname;
+    return std::string(nick, strnlen(nick, sizeof(nickname)));
+  }
+
 #endif
 };
 
@@ -60,6 +76,9 @@ void
 llarp_rc_set_addrs(struct llarp_rc *rc, struct llarp_alloc *mem,
                    struct llarp_ai_list *addr);
 
+bool
+llarp_rc_set_nickname(struct llarp_rc *rc, const char *name);
+
 void
 llarp_rc_set_pubenckey(struct llarp_rc *rc, const uint8_t *pubenckey);
 

include/llarp/routing/handler.hpp

@@ -21,10 +21,7 @@ namespace llarp
                                 llarp_router *r) = 0;
 
       virtual bool
-      HandleHiddenServiceFrame(const service::ProtocolFrame *msg)
-      {
-        return false;
-      }
+      HandleHiddenServiceFrame(const service::ProtocolFrame *msg) = 0;
 
       virtual bool
       HandlePathConfirmMessage(const PathConfirmMessage *msg,

include/llarp/service/address.hpp

@@ -13,6 +13,9 @@ namespace llarp
       std::string
       ToString() const;
 
+      bool
+      FromString(const std::string& str);
+
       Address() : llarp::AlignedBuffer< 32 >()
       {
       }

include/llarp/service/endpoint.hpp

@@ -74,6 +74,9 @@ namespace llarp
         return &m_Identity;
       }
 
+      void
+      HandlePathBuilt(path::Path* path);
+
       /// context needed to initiate an outbound hidden service session
       struct OutboundContext : public llarp_pathbuilder_context
       {
@@ -102,12 +105,18 @@ namespace llarp
         void
         UpdateIntroSet();
 
+        void
+        HandlePathBuilt(path::Path* path);
+
         bool
         SelectHop(llarp_nodedb* db, llarp_rc* prev, llarp_rc* cur, size_t hop);
 
         bool
         HandleGotIntroMessage(const llarp::dht::GotIntroMessage* msg);
 
+        bool
+        HandleHiddenServiceFrame(const ProtocolFrame* frame);
+
        private:
         void
         AsyncEncrypt(llarp_buffer_t payload);
@@ -176,6 +185,8 @@ namespace llarp
       service::IntroSet m_IntroSet;
       /// pending remote service lookups by id
       std::unordered_map< uint64_t, service::IServiceLookup* > m_PendingLookups;
+      /// prefetch remote address list
+      std::set< Address > m_PrefetchAddrs;
       /// hidden service tag
       Tag m_Tag;
       /// prefetch descriptors for these hidden service tags

include/llarp/service/protocol.hpp

@@ -45,7 +45,6 @@ namespace llarp
     struct ProtocolFrame : public llarp::routing::IMessage
     {
       llarp::Encrypted D;
-      uint64_t S = 0;
       llarp::PubKey H;
       llarp::KeyExchangeNonce N;
       llarp::Signature Z;

llarp/logger.cpp

@@ -12,8 +12,17 @@ namespace llarp
   }
 }  // namespace llarp
 
-void
-cSetLogLevel(LogLevel lvl)
+extern "C"
 {
-  llarp::SetLogLevel((llarp::LogLevel)lvl);
-}
+  void
+  cSetLogLevel(LogLevel lvl)
+  {
+    llarp::SetLogLevel((llarp::LogLevel)lvl);
+  }
+
+  void
+  cSetLogNodeName(const char* name)
+  {
+    llarp::_glog.nodeName = name;
+  }
+}

llarp/path.cpp

@@ -5,7 +5,6 @@
 #include "buffer.hpp"
 #include "router.hpp"
 
-
 namespace llarp
 {
   namespace path
@@ -463,6 +462,14 @@ namespace llarp
       return false;
     }
 
+    bool
+    Path::HandleHiddenServiceFrame(const llarp::service::ProtocolFrame* frame)
+    {
+      if(m_DataHandler)
+        return m_DataHandler(frame);
+      return false;
+    }
+
     bool
     Path::HandlePathLatencyMessage(
         const llarp::routing::PathLatencyMessage* msg, llarp_router* r)

llarp/pathbuilder.cpp

@@ -119,7 +119,7 @@ namespace llarp
       user   = u;
       result = func;
       worker = pool;
-      LRCM   = new LR_CommitMessage;
+      LRCM   = new LR_CommitMessage();
 
       for(size_t idx = 0; idx < MAXHOPS; ++idx)
       {
@@ -187,6 +187,7 @@ llarp_pathbuilder_context::llarp_pathbuilder_context(
     : llarp::path::PathSet(pathNum), router(p_router), dht(p_dht), numHops(hops)
 {
   p_router->paths.AddPathBuilder(this);
+  p_router->crypto.encryption_keygen(enckey);
 }
 
 bool
@@ -200,6 +201,12 @@ llarp_pathbuilder_context::SelectHop(llarp_nodedb* db, llarp_rc* prev,
   return true;
 }
 
+byte_t*
+llarp_pathbuilder_context::GetTunnelEncryptionSecretKey()
+{
+  return enckey;
+}
+
 void
 llarp_pathbuilder_context::BuildOne()
 {

llarp/pathset.cpp

@@ -46,14 +46,16 @@ namespace llarp
     }
 
     Path*
-    PathSet::GetEstablishedPathClosestTo(const RouterID& id) const
+    PathSet::GetEstablishedPathClosestTo(const AlignedBuffer< 32 >& id) const
     {
       Path* path = nullptr;
-      RouterID dist;
+      AlignedBuffer< 32 > dist;
       dist.Fill(0xff);
       for(const auto& item : m_Paths)
       {
-        RouterID localDist = item.second->Endpoint() ^ id;
+        if(!item.second->IsReady())
+          continue;
+        AlignedBuffer< 32 > localDist = item.second->Endpoint() ^ id;
         if(localDist < dist)
         {
           dist = localDist;

llarp/router.cpp

@@ -384,7 +384,6 @@ llarp_router::Tick()
   paths.TickPaths();
 }
 
-
 void
 llarp_router::SendTo(llarp::RouterID remote, const llarp::ILinkMessage *msg,
                      llarp_link *link)
@@ -1190,6 +1189,16 @@ namespace llarp
     }
     else if(StrEq(section, "router"))
     {
+      if(StrEq(key, "nickname"))
+      {
+        if(llarp_rc_set_nickname(&self->rc, val))
+        {
+          // set logger name here
+          _glog.nodeName = self->rc.Nick();
+        }
+        else
+          llarp::LogWarn("failed to set nickname to ", val);
+      }
       if(StrEq(key, "encryption-privkey"))
       {
         self->encryption_keyfile = val;

llarp/router_contact.cpp

@@ -4,6 +4,7 @@
 #include <llarp/crypto.hpp>
 #include "buffer.hpp"
 #include "logger.hpp"
+#include "mem.hpp"
 
 bool
 llarp_rc_new(struct llarp_rc *rc)
@@ -11,6 +12,7 @@ llarp_rc_new(struct llarp_rc *rc)
   rc->addrs        = llarp_ai_list_new();
   rc->exits        = llarp_xi_list_new();
   rc->last_updated = 0;
+  llarp::Zero(rc->nickname, sizeof(rc->nickname));
   return true;
 }
 
@@ -62,6 +64,17 @@ llarp_rc_decode_dict(struct dict_reader *r, llarp_buffer_t *key)
     return true;
   }
 
+  if(llarp_buffer_eq(*key, "n"))
+  {
+    if(!bencode_read_string(r->buffer, &strbuf))
+      return false;
+    if(strbuf.sz < sizeof(rc->nickname))
+      return false;
+    llarp::Zero(rc->nickname, sizeof(rc->nickname));
+    memcpy(rc->nickname, strbuf.base, strbuf.sz);
+    return true;
+  }
+
   if(llarp_buffer_eq(*key, "p"))
   {
     if(!bencode_read_string(r->buffer, &strbuf))
@@ -115,6 +128,14 @@ llarp_rc_is_public_router(const struct llarp_rc *const rc)
   return rc->addrs && llarp_ai_list_size(rc->addrs) > 0;
 }
 
+bool
+llarp_rc_set_nickname(struct llarp_rc *rc, const char *nick)
+{
+  strncpy((char *)rc->nickname, nick, sizeof(rc->nickname));
+  /// TODO: report nickname truncation
+  return true;
+}
+
 void
 llarp_rc_copy(struct llarp_rc *dst, const struct llarp_rc *src)
 {
@@ -135,6 +156,7 @@ llarp_rc_copy(struct llarp_rc *dst, const struct llarp_rc *src)
     dst->exits = llarp_xi_list_new();
     llarp_xi_list_copy(dst->exits, src->exits);
   }
+  memcpy(dst->nickname, src->nickname, sizeof(dst->nickname));
 }
 
 bool
@@ -198,6 +220,14 @@ llarp_rc_bencode(const struct llarp_rc *rc, llarp_buffer_t *buff)
   if(!bencode_write_bytestring(buff, rc->pubkey, PUBKEYSIZE))
     return false;
 
+  /* write nickname */
+  if(!bencode_write_bytestring(buff, "n", 1))
+    return false;
+  if(!bencode_write_bytestring(
+         buff, rc->nickname,
+         strnlen((char *)rc->nickname, sizeof(rc->nickname))))
+    return false;
+
   /* write encryption pubkey */
   if(!bencode_write_bytestring(buff, "p", 1))
     return false;

llarp/service/address.cpp

@@ -11,5 +11,15 @@ namespace llarp
       std::string str        = Base32Encode(*this, tmp);
       return str + ".loki";
     }
+
+    bool
+    Address::FromString(const std::string& str)
+    {
+      auto pos = str.find(".loki");
+      if(pos == std::string::npos)
+        return false;
+      auto sub = str.substr(0, pos);
+      return Base32Decode(sub, *this);
+    }
   }  // namespace service
 }  // namespace llarp

llarp/service/endpoint.cpp

@@ -31,6 +31,12 @@ namespace llarp
       {
         m_PrefetchTags.insert(v);
       }
+      if(k == "prefetch-addr")
+      {
+        Address addr;
+        if(addr.FromString(v))
+          m_PrefetchAddrs.insert(addr);
+      }
       return true;
     }
 
@@ -47,6 +53,7 @@ namespace llarp
       {
         if(context)
         {
+          llarp::LogInfo("BEEP");
           byte_t tmp[128] = {0};
           memcpy(tmp, "BEEP", 4);
           auto buf = llarp::StackBuffer< decltype(tmp) >(tmp);
@@ -109,6 +116,22 @@ namespace llarp
         }
       }
 
+      for(const auto& addr : m_PrefetchAddrs)
+      {
+        if(!HasPathToService(addr))
+        {
+          PathAlignJob* j = new PathAlignJob(addr);
+          if(!EnsurePathToService(j->remote,
+                                  std::bind(&PathAlignJob::HandleResult, j,
+                                            std::placeholders::_1),
+                                  10000))
+          {
+            llarp::LogWarn("failed to ensure path to ", addr);
+            delete j;
+          }
+        }
+      }
+
       // prefetch tags
       for(const auto& tag : m_PrefetchTags)
       {
@@ -126,7 +149,11 @@ namespace llarp
                                   std::bind(&PathAlignJob::HandleResult, j,
                                             std::placeholders::_1),
                                   10000))
+          {
+            llarp::LogWarn("failed to ensure path to ", introset.A.Addr(),
+                           " for tag");
             delete j;
+          }
         }
         itr->second.Expire(now);
         if(itr->second.ShouldRefresh(now))
@@ -172,6 +199,12 @@ namespace llarp
       return m_Name + ":" + m_Identity.pub.Name();
     }
 
+    bool
+    Endpoint::HasPathToService(const Address& addr) const
+    {
+      return m_RemoteSessions.find(addr) != m_RemoteSessions.end();
+    }
+
     bool
     Endpoint::HandleGotIntroMessage(const llarp::dht::GotIntroMessage* msg)
     {
@@ -287,7 +320,7 @@ namespace llarp
     bool
     Endpoint::PublishIntroSet(llarp_router* r)
     {
-      auto path = PickRandomEstablishedPath();
+      auto path = GetEstablishedPathClosestTo(m_Identity.pub.Addr());
       if(path)
       {
         m_CurrentPublishTX = llarp_randint();
@@ -392,16 +425,46 @@ namespace llarp
       }
     }
 
+    void
+    Endpoint::HandlePathBuilt(path::Path* p)
+    {
+      p->SetDataHandler(std::bind(&Endpoint::HandleHiddenServiceFrame, this,
+                                  std::placeholders::_1));
+    }
+
+    bool
+    Endpoint::HandleHiddenServiceFrame(const ProtocolFrame* frame)
+    {
+      llarp::LogInfo("handle hidden service frame");
+      return true;
+    }
+
+    void
+    Endpoint::OutboundContext::HandlePathBuilt(path::Path* p)
+    {
+      p->SetDataHandler(
+          std::bind(&Endpoint::OutboundContext::HandleHiddenServiceFrame, this,
+                    std::placeholders::_1));
+    }
+
+    bool
+    Endpoint::OutboundContext::HandleHiddenServiceFrame(
+        const ProtocolFrame* frame)
+    {
+      return m_Parent->HandleHiddenServiceFrame(frame);
+    }
+
     bool
     Endpoint::EnsurePathToService(const Address& remote, PathEnsureHook hook,
                                   llarp_time_t timeoutMS)
     {
-      auto path = PickRandomEstablishedPath();
+      auto path = GetEstablishedPathClosestTo(remote);
       if(!path)
       {
         llarp::LogWarn("No outbound path for lookup yet");
         return false;
       }
+      llarp::LogInfo(Name(), " Ensure Path to ", remote.ToString());
       {
         auto itr = m_RemoteSessions.find(remote);
         if(itr != m_RemoteSessions.end())
@@ -414,9 +477,10 @@ namespace llarp
       if(itr != m_PendingServiceLookups.end())
       {
         // duplicate
+        llarp::LogWarn("duplicate pending service lookup to ",
+                       remote.ToString());
         return false;
       }
-      llarp::LogInfo(Name(), " Ensure Path to ", remote.ToString());
 
       m_PendingServiceLookups.insert(std::make_pair(remote, hook));
 

llarp/service/protocol.cpp

@@ -17,8 +17,9 @@ namespace llarp
     bool
     ProtocolMessage::BEncode(llarp_buffer_t* buf) const
     {
-      // TODO: implement me
-      return false;
+      if(!bencode_start_dict(buf))
+        return false;
+      return bencode_end(buf);
     }
 
     bool

test/hiddenservice_unittest.cpp

@@ -22,6 +22,7 @@ struct HiddenServiceTest : public ::testing::Test
   {
     ident.RegenerateKeys(Crypto());
     ident.pub.vanity.Randomize();
+    ident.pub.UpdateAddr();
   }
 };
 
@@ -40,4 +41,12 @@ TEST_F(HiddenServiceTest, TestGenerateIntroSet)
   }
   ASSERT_TRUE(ident.SignIntroSet(I, Crypto()));
   ASSERT_TRUE(I.VerifySignature(Crypto()));
-};
+};
+
+TEST_F(HiddenServiceTest, TestAddressToFromString)
+{
+  auto str = ident.pub.Addr().ToString();
+  llarp::service::Address addr;
+  ASSERT_TRUE(addr.FromString(str));
+  ASSERT_TRUE(addr == ident.pub.Addr());
+}