Merge pull request #477 from majestrate/master

handle path death better

llarp/exit/session.cpp

@@ -25,6 +25,11 @@ namespace llarp
     {
     }
 
+    void
+    BaseSession::HandlePathDied(path::Path*)
+    {
+    }
+
     util::StatusObject
     BaseSession::ExtractStatus() const
     {

llarp/exit/session.hpp

@@ -32,6 +32,9 @@ namespace llarp
       util::StatusObject
       ExtractStatus() const override;
 
+      void
+      HandlePathDied(llarp::path::Path* p) override;
+
       bool
       SelectHop(llarp_nodedb* db, const RouterContact& prev, RouterContact& cur,
                 size_t hop, llarp::path::PathRole roles) override;

llarp/path/path.cpp

@@ -462,7 +462,7 @@ namespace llarp
     void
     Path::EnterState(PathStatus st, llarp_time_t now)
     {
-      if(st == ePathTimeout)
+      if(st == ePathTimeout && _status == ePathBuilding)
       {
         m_PathSet->HandlePathBuildTimeout(this);
       }
@@ -475,6 +475,12 @@ namespace llarp
       {
         LogInfo("path ", Name(), " is built, took ", now - buildStarted, " ms");
       }
+      else if(st == ePathTimeout && _status == ePathEstablished)
+      {
+        LogInfo("path ", Name(), " died");
+        _status = st;
+        m_PathSet->HandlePathDied(this);
+      }
       _status = st;
     }
 

llarp/path/pathset.hpp

@@ -81,6 +81,10 @@ namespace llarp
       virtual void
       HandlePathBuildTimeout(__attribute__((unused)) Path* path);
 
+      /// a path died now what?
+      virtual void
+      HandlePathDied(Path* path) = 0;
+
       bool
       GetNewestIntro(service::Introduction& intro) const;
 

llarp/profiling.cpp

@@ -111,6 +111,13 @@ namespace llarp
     m_Profiles[r].lastUpdated = llarp::time_now_ms();
   }
 
+  void
+  Profiling::ClearProfile(const RouterID& r)
+  {
+    lock_t lock(&m_ProfilesMutex);
+    m_Profiles.erase(r);
+  }
+
   void
   Profiling::MarkPathFail(path::Path* p)
   {

llarp/profiling.hpp

@@ -68,6 +68,9 @@ namespace llarp
     void
     MarkPathSuccess(path::Path* p) LOCKS_EXCLUDED(m_ProfilesMutex);
 
+    void
+    ClearProfile(const RouterID& r) LOCKS_EXCLUDED(m_ProfilesMutex);
+
     void
     Tick() LOCKS_EXCLUDED(m_ProfilesMutex);
 

llarp/router/abstractrouter.hpp

@@ -165,6 +165,12 @@ namespace llarp
     HandleDHTLookupForExplore(RouterID remote,
                               const std::vector< RouterContact > &results) = 0;
 
+    /// lookup router by pubkey
+    /// if we are a service node this is done direct otherwise it's done via
+    /// path
+    virtual void
+    LookupRouter(RouterID remote) = 0;
+
     /// check if newRc matches oldRC and update local rc for this remote contact
     /// if valid
     /// returns true on valid and updated

llarp/router/router.cpp

@@ -1027,6 +1027,23 @@ namespace llarp
                                         this, router, std::placeholders::_1));
   }
 
+  void
+  Router::LookupRouter(RouterID remote)
+  {
+    if(IsServiceNode())
+    {
+      ServiceNodeLookupRouterWhenExpired(remote);
+      return;
+    }
+    auto ep = hiddenServiceContext().getFirstEndpoint();
+    if(ep == nullptr)
+    {
+      LogError("cannot lookup ", remote, " no service endpoints available");
+      return;
+    }
+    ep->LookupRouterAnon(remote);
+  }
+
   void
   Router::Tick()
   {
@@ -1035,15 +1052,16 @@ namespace llarp
 
     routerProfiling().Tick();
 
-    if(_rc.ExpiresSoon(now, randint() % 10000))
-    {
-      LogInfo("regenerating RC");
-      if(!UpdateOurRC(false))
-        LogError("Failed to update our RC");
-    }
-
     if(IsServiceNode())
     {
+      if(_rc.ExpiresSoon(now, randint() % 10000)
+         || (now - _rc.last_updated) > rcRegenInterval)
+      {
+        LogInfo("regenerating RC");
+        if(!UpdateOurRC(false))
+          LogError("Failed to update our RC");
+      }
+
       // only do this as service node
       // client endpoints do this on their own
       nodedb()->visit([&](const RouterContact &rc) -> bool {
@@ -1053,9 +1071,19 @@ namespace llarp
       });
     }
     // kill dead nodes
+    std::set< RouterID > removed;
     nodedb()->RemoveIf([&](const RouterContact &rc) -> bool {
-      return routerProfiling().IsBad(rc.pubkey);
+      if(!routerProfiling().IsBad(rc.pubkey))
+        return false;
+      routerProfiling().ClearProfile(rc.pubkey);
+      removed.insert(rc.pubkey);
+      return true;
     });
+
+    // request killed nodes 1 time
+    for(const auto &pk : removed)
+      LookupRouter(pk);
+
     paths.TickPaths(now);
     paths.ExpirePaths(now);
 

llarp/router/router.hpp

@@ -211,11 +211,14 @@ namespace llarp
     uint16_t m_OutboundPort = 0;
 
     /// always maintain this many connections to other routers
-    size_t minConnectedRouters = 3;
+    size_t minConnectedRouters = 2;
     /// hard upperbound limit on the number of router to router connections
     size_t maxConnectedRouters = 2000;
 
     size_t minRequiredRouters = 4;
+    /// how often do we resign our RC? milliseconds.
+    // TODO: make configurable
+    llarp_time_t rcRegenInterval = 60 * 60 * 1000;
 
     // should we be sending padded messages every interval?
     bool sendPadding = false;
@@ -428,6 +431,9 @@ namespace llarp
     void
     FlushOutboundFor(RouterID remote, ILinkLayer *chosen = nullptr);
 
+    void
+    LookupRouter(RouterID remote) override;
+
     /// manually discard all pending messages to remote router
     void
     DiscardOutboundFor(const RouterID &remote);

llarp/service/endpoint.cpp

@@ -853,6 +853,8 @@ namespace llarp
         job->hook                  = nullptr;
         job->rc                    = msg->R[0];
         llarp_nodedb_async_verify(job);
+        router->routerProfiling().MarkSuccess(msg->R[0].pubkey);
+        m_PendingRouters.erase(itr);
         return true;
       }
       return success;
@@ -1050,25 +1052,79 @@ namespace llarp
     }
 
     void
-    Endpoint::HandlePathDead(void* user)
+    Endpoint::HandlePathDied(path::Path*)
     {
-      Endpoint* self = static_cast< Endpoint* >(user);
-      self->RegenAndPublishIntroSet(self->Now(), true);
+      RegenAndPublishIntroSet(Now(), true);
+    }
+
+    void
+    Endpoint::OutboundContext::HandlePathDied(path::Path* path)
+    {
+      // unconditionally update introset
+      UpdateIntroSet(true);
+      const RouterID endpoint(path->Endpoint());
+      // if a path to our current intro died...
+      if(endpoint == remoteIntro.router)
+      {
+        // figure out how many paths to this router we have
+        size_t num = 0;
+        ForEachPath([&](path::Path* p) {
+          if(p->Endpoint() == endpoint && p->IsReady())
+            ++num;
+        });
+        // if we have more than two then we are probably fine
+        if(num > 2)
+          return;
+        // if we have one working one ...
+        if(num == 1)
+        {
+          num = 0;
+          ForEachPath([&](path::Path* p) {
+            if(p->Endpoint() == endpoint)
+              ++num;
+          });
+          // if we have 2 or more established or pending don't do anything
+          if(num > 2)
+            return;
+          BuildOneAlignedTo(endpoint);
+        }
+        else if(num == 0)
+        {
+          // we have no paths to this router right now
+          // hop off it
+          Introduction picked;
+          // get the latest intro that isn't on that endpoint
+          for(const auto& intro : currentIntroSet.I)
+          {
+            if(intro.router == endpoint)
+              continue;
+            if(intro.expiresAt > picked.expiresAt)
+              picked = intro;
+          }
+          // we got nothing
+          if(picked.router.IsZero())
+          {
+            return;
+          }
+          m_NextIntro = picked;
+          // check if we have a path to this router
+          num = 0;
+          ForEachPath([&](path::Path* p) {
+            if(p->Endpoint() == m_NextIntro.router)
+              ++num;
+          });
+          // build a path if one isn't already pending build or established
+          if(num == 0)
+            BuildOneAlignedTo(m_NextIntro.router);
+          SwapIntros();
+        }
+      }
     }
 
     bool
     Endpoint::CheckPathIsDead(path::Path*, llarp_time_t dlt)
     {
-      if(dlt <= 30000)
-        return false;
-
-      RouterLogic()->call_later(
-          {100, this, [](void* u, uint64_t, uint64_t left) {
-             if(left)
-               return;
-             HandlePathDead(u);
-           }});
-      return true;
+      return dlt > 20000;
     }
 
     bool

llarp/service/endpoint.hpp

@@ -102,6 +102,9 @@ namespace llarp
       bool
       ShouldPublishDescriptors(llarp_time_t now) const override;
 
+      void
+      HandlePathDied(path::Path* p) override;
+
       void
       EnsureReplyPath(const ServiceInfo& addr);
 
@@ -283,6 +286,9 @@ namespace llarp
         bool
         HandleDataDrop(path::Path* p, const PathID_t& dst, uint64_t s);
 
+        void
+        HandlePathDied(path::Path* p) override;
+
         /// set to true if we are updating the remote introset right now
         bool updatingIntroSet;