From 800f5001c200c15465027a0ed41d7b9604923082 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 09:51:57 -0400
Subject: [PATCH 1/9] allow up to 5 sessions per endpoint

---
 llarp/link/server.cpp |  4 ++--
 llarp/link/server.hpp |  4 ++--
 llarp/link/utp.cpp    | 12 +++++++++---
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/llarp/link/server.cpp b/llarp/link/server.cpp
index 2e5f901f6..93359a20f 100644
--- a/llarp/link/server.cpp
+++ b/llarp/link/server.cpp
@@ -382,10 +382,10 @@ namespace llarp
   bool
   ILinkLayer::PutSession(ILinkSession* s)
   {
+    static constexpr size_t MaxSessionsPerEndpoint = 5;
     Lock lock(&m_PendingMutex);
     llarp::Addr addr = s->GetRemoteEndpoint();
-    auto itr         = m_Pending.find(addr);
-    if(itr != m_Pending.end())
+    if(m_Pending.count(addr) >= MaxSessionsPerEndpoint)
       return false;
     m_Pending.emplace(addr, std::unique_ptr< ILinkSession >(s));
     return true;
diff --git a/llarp/link/server.hpp b/llarp/link/server.hpp
index b968e1e3a..134a2d6bc 100644
--- a/llarp/link/server.hpp
+++ b/llarp/link/server.hpp
@@ -240,8 +240,8 @@ namespace llarp
         m_AuthedLinks GUARDED_BY(m_AuthedLinksMutex);
     Mutex m_PendingMutex
         ACQUIRED_AFTER(m_AuthedLinksMutex);  // protects m_Pending
-    std::unordered_map< llarp::Addr, std::unique_ptr< ILinkSession >,
-                        llarp::Addr::Hash >
+    std::unordered_multimap< llarp::Addr, std::unique_ptr< ILinkSession >,
+                             llarp::Addr::Hash >
         m_Pending GUARDED_BY(m_PendingMutex);
   };
 }  // namespace llarp
diff --git a/llarp/link/utp.cpp b/llarp/link/utp.cpp
index 3f42ba3a4..eb93743fb 100644
--- a/llarp/link/utp.cpp
+++ b/llarp/link/utp.cpp
@@ -228,6 +228,8 @@ namespace llarp
     bool
     Session::IsTimedOut(llarp_time_t now) const
     {
+      if(state == eInitial)
+        return false;
       if(sendq.size() >= MaxSendQueueSize)
       {
         return now - lastActive > 5000;
@@ -344,13 +346,16 @@ namespace llarp
       LinkLayer* link =
           static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
 
+      const llarp::Addr remoteAddr(*arg->address);
+      llarp::LogError(utp_error_code_names[arg->error_code], " via ",
+                      remoteAddr);
       if(session && link)
       {
-        link->HandleTimeout(session);
-        llarp::LogError(utp_error_code_names[arg->error_code], " via ",
-                        session->remoteAddr);
         if(arg->error_code == UTP_ETIMEDOUT)
+        {
+          link->HandleTimeout(session);
           utp_close(arg->socket);
+        }
         else
           session->Close();
       }
@@ -577,6 +582,7 @@ namespace llarp
     /// base constructor
     Session::Session(LinkLayer* p)
     {
+      state         = eInitial;
       m_NextTXMsgID = 0;
       m_NextRXMsgID = 0;
       parent        = p;

From 74534c22b8addf56f6ff04e4176b550a267a6b3d Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 11:27:14 -0400
Subject: [PATCH 2/9] mark profile success on outbound connection success, add
 default case in metrics

---
 llarp/router/router.cpp      | 1 +
 llarp/util/metrics_types.cpp | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 5ae844f4f..0a0f4fc5a 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -63,6 +63,7 @@ struct TryConnectJob
   void
   Success()
   {
+    router->routerProfiling().MarkSuccess(rc.pubkey);
     router->FlushOutboundFor(rc.pubkey, link);
   }
 
diff --git a/llarp/util/metrics_types.cpp b/llarp/util/metrics_types.cpp
index 6d8f0a743..6c97be19b 100644
--- a/llarp/util/metrics_types.cpp
+++ b/llarp/util/metrics_types.cpp
@@ -64,6 +64,8 @@ namespace llarp
           return "Rate";
         case Type::RateCount:
           return "RateCount";
+        default:
+          return "???";
       }
     }
 

From 6a544d3f099527c62f25eb09b01b9bcbf0769431 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 11:40:21 -0400
Subject: [PATCH 3/9] remove to prevent crash

---
 llarp/link/utp.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/llarp/link/utp.cpp b/llarp/link/utp.cpp
index eb93743fb..f6cbc5f87 100644
--- a/llarp/link/utp.cpp
+++ b/llarp/link/utp.cpp
@@ -346,9 +346,6 @@ namespace llarp
       LinkLayer* link =
           static_cast< LinkLayer* >(utp_context_get_userdata(arg->context));
 
-      const llarp::Addr remoteAddr(*arg->address);
-      llarp::LogError(utp_error_code_names[arg->error_code], " via ",
-                      remoteAddr);
       if(session && link)
       {
         if(arg->error_code == UTP_ETIMEDOUT)

From 0258eb9cb8abed26605f9acbc40ac9263530eaa1 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 15:30:10 -0400
Subject: [PATCH 4/9] add some metrics

---
 llarp/link/utp.cpp              |  8 +++++++-
 llarp/messages/link_message.cpp | 11 +++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/llarp/link/utp.cpp b/llarp/link/utp.cpp
index f6cbc5f87..ce9f7a0e6 100644
--- a/llarp/link/utp.cpp
+++ b/llarp/link/utp.cpp
@@ -25,6 +25,8 @@
 
 #include <link/utp_internal.hpp>
 
+#include <util/metrics.hpp>
+
 namespace llarp
 {
   namespace utp
@@ -85,6 +87,8 @@ namespace llarp
         ssize_t s = utp_writev(sock, vecs.data(), vecs.size());
         if(s < 0)
           return;
+        METRICS_DYNAMIC_INT_UPDATE(
+            "utp.session.tx", RouterID(remoteRC.pubkey).ToString().c_str(), s);
         m_TXRate += s;
         size_t sz = s;
         while(vecq.size() && sz >= vecq.front().iov_len)
@@ -188,6 +192,8 @@ namespace llarp
       Alive();
       m_RXRate += sz;
       size_t s = sz;
+      METRICS_DYNAMIC_INT_UPDATE(
+          "utp.session.rx", RouterID(remoteRC.pubkey).ToString().c_str(), s);
       // process leftovers
       if(recvBufOffset)
       {
@@ -351,11 +357,11 @@ namespace llarp
         if(arg->error_code == UTP_ETIMEDOUT)
         {
           link->HandleTimeout(session);
-          utp_close(arg->socket);
         }
         else
           session->Close();
       }
+      utp_close(arg->socket);
       return 0;
     }
 
diff --git a/llarp/messages/link_message.cpp b/llarp/messages/link_message.cpp
index d9a49d7b8..c034b9548 100644
--- a/llarp/messages/link_message.cpp
+++ b/llarp/messages/link_message.cpp
@@ -9,6 +9,7 @@
 #include <router_contact.hpp>
 #include <util/buffer.hpp>
 #include <util/logger.hpp>
+#include <util/metrics.hpp>
 
 namespace llarp
 {
@@ -90,6 +91,14 @@ namespace llarp
         default:
           return false;
       }
+      {
+        const std::string host =
+            handler->from->GetRemoteEndpoint().xtohl().ToString();
+        char buf[16] = "LinkLayerRecv_";
+        buf[14]      = *strbuf.cur;
+        buf[15]      = 0;
+        METRICS_DYNAMIC_INCREMENT(buf, host.c_str());
+      }
       handler->msg->session = handler->from;
       handler->firstkey     = false;
       return true;
@@ -107,6 +116,8 @@ namespace llarp
     bool result = false;
     if(msg)
     {
+      const std::string host = from->GetRemoteEndpoint().xtohl().ToString();
+      METRICS_TIME_BLOCK("HandleMessage", host.c_str());
       result = msg->HandleMessage(router);
     }
     Reset();

From b254c19c5ff2a864b21f632963b4dab42b20b210 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 16:04:41 -0400
Subject: [PATCH 5/9] more metrics

---
 llarp/link/utp.cpp               |  3 +++
 llarp/messages/dht_immediate.hpp |  6 ++++++
 llarp/messages/discard.hpp       |  6 ++++++
 llarp/messages/link_intro.hpp    |  6 ++++++
 llarp/messages/link_message.cpp  | 14 +++++++-------
 llarp/messages/link_message.hpp  |  4 ++++
 llarp/messages/relay.hpp         | 12 ++++++++++++
 llarp/messages/relay_commit.hpp  |  6 ++++++
 llarp/router/router.cpp          |  3 ++-
 9 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/llarp/link/utp.cpp b/llarp/link/utp.cpp
index ce9f7a0e6..2d7d21396 100644
--- a/llarp/link/utp.cpp
+++ b/llarp/link/utp.cpp
@@ -182,6 +182,9 @@ namespace llarp
       PruneInboundMessages(now);
       m_TXRate = 0;
       m_RXRate = 0;
+      METRICS_DYNAMIC_UPDATE("utp.session.sendq",
+                             RouterID(remoteRC.pubkey).ToString().c_str(),
+                             sendq.size());
     }
 
     /// low level read
diff --git a/llarp/messages/dht_immediate.hpp b/llarp/messages/dht_immediate.hpp
index 9802eecb2..c6142a0b0 100644
--- a/llarp/messages/dht_immediate.hpp
+++ b/llarp/messages/dht_immediate.hpp
@@ -29,6 +29,12 @@ namespace llarp
 
     void
     Clear() override;
+
+    const char*
+    Name() const override
+    {
+      return "DHTImmediate";
+    }
   };
 }  // namespace llarp
 
diff --git a/llarp/messages/discard.hpp b/llarp/messages/discard.hpp
index eeac2e862..fd755ca85 100644
--- a/llarp/messages/discard.hpp
+++ b/llarp/messages/discard.hpp
@@ -31,6 +31,12 @@ namespace llarp
     {
     }
 
+    const char*
+    Name() const override
+    {
+      return "Discard";
+    }
+
     bool
     DecodeKey(__attribute__((unused)) const llarp_buffer_t& key,
               __attribute__((unused)) llarp_buffer_t* buf) override
diff --git a/llarp/messages/link_intro.hpp b/llarp/messages/link_intro.hpp
index 75c63c43e..83538761f 100644
--- a/llarp/messages/link_intro.hpp
+++ b/llarp/messages/link_intro.hpp
@@ -44,6 +44,12 @@ namespace llarp
 
     void
     Clear() override;
+
+    const char*
+    Name() const override
+    {
+      return "LinkIntro";
+    }
   };
 }  // namespace llarp
 
diff --git a/llarp/messages/link_message.cpp b/llarp/messages/link_message.cpp
index c034b9548..c132dcb33 100644
--- a/llarp/messages/link_message.cpp
+++ b/llarp/messages/link_message.cpp
@@ -68,10 +68,12 @@ namespace llarp
       }
       // create the message to parse based off message type
       llarp::LogDebug("inbound message ", *strbuf.cur);
+      bool isLIM = false;
       switch(*strbuf.cur)
       {
         case 'i':
           handler->msg = &handler->holder->i;
+          isLIM        = true;
           break;
         case 'd':
           handler->msg = &handler->holder->d;
@@ -91,14 +93,14 @@ namespace llarp
         default:
           return false;
       }
+
+      if(!isLIM)
       {
         const std::string host =
-            handler->from->GetRemoteEndpoint().xtohl().ToString();
-        char buf[16] = "LinkLayerRecv_";
-        buf[14]      = *strbuf.cur;
-        buf[15]      = 0;
-        METRICS_DYNAMIC_INCREMENT(buf, host.c_str());
+            "RX_" + RouterID(handler->from->GetPubKey()).ToString();
+        METRICS_DYNAMIC_INCREMENT(handler->msg->Name(), host.c_str());
       }
+
       handler->msg->session = handler->from;
       handler->firstkey     = false;
       return true;
@@ -116,8 +118,6 @@ namespace llarp
     bool result = false;
     if(msg)
     {
-      const std::string host = from->GetRemoteEndpoint().xtohl().ToString();
-      METRICS_TIME_BLOCK("HandleMessage", host.c_str());
       result = msg->HandleMessage(router);
     }
     Reset();
diff --git a/llarp/messages/link_message.hpp b/llarp/messages/link_message.hpp
index 6edd91b6c..256a59d82 100644
--- a/llarp/messages/link_message.hpp
+++ b/llarp/messages/link_message.hpp
@@ -30,6 +30,10 @@ namespace llarp
 
     virtual void
     Clear() = 0;
+
+    // the name of this kind of message
+    virtual const char*
+    Name() const = 0;
   };
 
 }  // namespace llarp
diff --git a/llarp/messages/relay.hpp b/llarp/messages/relay.hpp
index 7b4b10be3..e64975c34 100644
--- a/llarp/messages/relay.hpp
+++ b/llarp/messages/relay.hpp
@@ -31,6 +31,12 @@ namespace llarp
 
     void
     Clear() override;
+    const char*
+
+    Name() const override
+    {
+      return "RelayUpstream";
+    }
   };
 
   struct RelayDownstreamMessage : public ILinkMessage
@@ -53,6 +59,12 @@ namespace llarp
 
     void
     Clear() override;
+
+    const char*
+    Name() const override
+    {
+      return "RelayDownstream";
+    }
   };
 }  // namespace llarp
 
diff --git a/llarp/messages/relay_commit.hpp b/llarp/messages/relay_commit.hpp
index d68fce51e..72180b1b1 100644
--- a/llarp/messages/relay_commit.hpp
+++ b/llarp/messages/relay_commit.hpp
@@ -68,6 +68,12 @@ namespace llarp
 
     bool
     AsyncDecrypt(llarp::path::PathContext *context) const;
+
+    const char *
+    Name() const override
+    {
+      return "RelayCommit";
+    }
   };
 }  // namespace llarp
 
diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index 0a0f4fc5a..c04c429f0 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -1145,7 +1145,8 @@ namespace llarp
   void
   Router::SendTo(RouterID remote, const ILinkMessage *msg, ILinkLayer *selected)
   {
-    METRICS_TIME_BLOCK("RouterSendTo", remote.ToString().c_str());
+    const std::string remoteName = "TX_" + remote.ToString();
+    METRICS_DYNAMIC_INCREMENT(msg->Name(), remoteName.c_str());
     llarp_buffer_t buf(linkmsg_buffer);
 
     if(!msg->BEncode(&buf))

From 127cfbbfbc336a43df412e292ed20b31f6c5062f Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 16:20:46 -0400
Subject: [PATCH 6/9] lower a way too big value

---
 llarp/link/utp_internal.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llarp/link/utp_internal.hpp b/llarp/link/utp_internal.hpp
index 9662d74ee..8671dac59 100644
--- a/llarp/link/utp_internal.hpp
+++ b/llarp/link/utp_internal.hpp
@@ -42,7 +42,7 @@ namespace llarp
     using FragmentBuffer = llarp::AlignedBuffer< FragmentBufferSize >;
 
     /// maximum size for send queue for a session before we drop
-    constexpr size_t MaxSendQueueSize = 1024;
+    constexpr size_t MaxSendQueueSize = 64;
 
     /// buffer for a link layer message
     using MessageBuffer = llarp::AlignedBuffer< MAX_LINK_MSG_SIZE >;

From 9e531c026c9b19c8e451cdc9fae043575229f11d Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 16:41:41 -0400
Subject: [PATCH 7/9] fix warning on hop selection

---
 llarp/service/endpoint.cpp | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp
index d603a2c97..efcd1217c 100644
--- a/llarp/service/endpoint.cpp
+++ b/llarp/service/endpoint.cpp
@@ -1175,12 +1175,10 @@ namespace llarp
         , currentIntroSet(introset)
 
     {
-      auto& profiling  = parent->m_Router->routerProfiling();
       updatingIntroSet = false;
       for(const auto intro : introset.I)
       {
-        if(intro.expiresAt > m_NextIntro.expiresAt
-           && !profiling.IsBad(intro.router))
+        if(intro.expiresAt > m_NextIntro.expiresAt)
         {
           m_NextIntro = intro;
           remoteIntro = intro;
@@ -1848,13 +1846,21 @@ namespace llarp
                                          llarp::path::PathRole roles)
     {
       if(m_NextIntro.router.IsZero())
+      {
+        llarp::LogError("intro is not set, cannot select hops");
         return false;
+      }
       if(hop == numHops - 1)
       {
         if(db->Get(m_NextIntro.router, cur))
         {
           return true;
         }
+        else if(router->routerProfiling().IsBad(m_NextIntro.router))
+        {
+          llarp::LogError("bad intro chosen, not selecting hop");
+          return false;
+        }
         else
         {
           // we don't have it?

From 714f5c6b5eded7769f194118e358605cceebc730 Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 17:01:20 -0400
Subject: [PATCH 8/9] more

---
 llarp/link/utp.cpp         |  2 +-
 llarp/service/endpoint.cpp | 12 ++++++++----
 llarp/service/endpoint.hpp |  5 +++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/llarp/link/utp.cpp b/llarp/link/utp.cpp
index 2d7d21396..3ff337c63 100644
--- a/llarp/link/utp.cpp
+++ b/llarp/link/utp.cpp
@@ -360,11 +360,11 @@ namespace llarp
         if(arg->error_code == UTP_ETIMEDOUT)
         {
           link->HandleTimeout(session);
+          utp_close(arg->socket);
         }
         else
           session->Close();
       }
-      utp_close(arg->socket);
       return 0;
     }
 
diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp
index efcd1217c..806b4e7f6 100644
--- a/llarp/service/endpoint.cpp
+++ b/llarp/service/endpoint.cpp
@@ -1487,7 +1487,7 @@ namespace llarp
     }
 
     bool
-    Endpoint::OutboundContext::ShiftIntroduction()
+    Endpoint::OutboundContext::ShiftIntroduction(bool rebuild)
     {
       bool success = false;
       auto now     = Now();
@@ -1522,7 +1522,7 @@ namespace llarp
           break;
         }
       }
-      if(shifted)
+      if(shifted && rebuild)
       {
         lastShift = now;
         BuildOneAlignedTo(m_NextIntro.router);
@@ -1858,8 +1858,12 @@ namespace llarp
         }
         else if(router->routerProfiling().IsBad(m_NextIntro.router))
         {
-          llarp::LogError("bad intro chosen, not selecting hop");
-          return false;
+          if(!ShiftIntroduction(false))
+          {
+            llarp::LogError("bad intro chosen, not selecting hop");
+            return false;
+          }
+          return db->Get(m_NextIntro.router, cur);
         }
         else
         {
diff --git a/llarp/service/endpoint.hpp b/llarp/service/endpoint.hpp
index b7afeb853..3eef971a4 100644
--- a/llarp/service/endpoint.hpp
+++ b/llarp/service/endpoint.hpp
@@ -239,8 +239,9 @@ namespace llarp
         bool markedBad              = false;
 
         virtual bool
-        ShiftIntroduction()
+        ShiftIntroduction(bool rebuild = true)
         {
+          (void)rebuild;
           return true;
         };
 
@@ -288,7 +289,7 @@ namespace llarp
         /// update the current selected intro to be a new best introduction
         /// return true if we have changed intros
         bool
-        ShiftIntroduction() override;
+        ShiftIntroduction(bool rebuild = true) override;
 
         /// mark the current remote intro as bad
         bool

From 52d9cef79e53f84ba501e0098e4d17e9d01643ad Mon Sep 17 00:00:00 2001
From: Jeff Becker <jeff@i2p.rocks>
Date: Tue, 26 Mar 2019 17:38:53 -0400
Subject: [PATCH 9/9] check for service node whitelist

---
 llarp/router/router.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp
index c04c429f0..4f11f40ba 100644
--- a/llarp/router/router.cpp
+++ b/llarp/router/router.cpp
@@ -166,6 +166,8 @@ namespace llarp
     const RouterID us = pubkey();
     if(remote.pubkey == us)
       return false;
+    if(!ConnectionToRouterAllowed(remote.pubkey))
+      return false;
     // do we already have a pending job for this remote?
     if(HasPendingConnectJob(remote.pubkey))
     {
@@ -628,6 +630,7 @@ namespace llarp
     const RouterID us = pubkey();
     if(us == remote)
       return;
+
     if(!ConnectionToRouterAllowed(remote))
     {
       LogWarn("not connecting to ", remote, " as it's not permitted by config");
@@ -1265,6 +1268,11 @@ namespace llarp
   {
     if(rc.IsPublicRouter() && whitelistRouters && IsServiceNode())
     {
+      if(lokinetRouters.size() == 0)
+      {
+        LogError("we have no service nodes in whitelist");
+        return false;
+      }
       if(lokinetRouters.find(rc.pubkey) == lokinetRouters.end())
       {
         RouterID sn(rc.pubkey);