hive pytest framework in place (and path build test works)!

.gitignore

@@ -7,6 +7,7 @@
 *.so
 
 build/
+**/__pycache__/**
 
 llarpd
 *.test

CMakeLists.txt

@@ -30,6 +30,10 @@ option(TRACY_ROOT "include tracy profiler source" OFF)
 option(WITH_TESTS "build unit tests" ON)
 option(WITH_HIVE "build simulation stubs" OFF)
 
+if(WITH_HIVE)
+  set(WITH_SHARED ON)
+endif()
+
 include(cmake/target_link_libraries_system.cmake)
 include(cmake/add_import_library.cmake)
 include(cmake/add_log_tag.cmake)
@@ -304,7 +308,7 @@ add_subdirectory(pybind)
 
 
 if (NOT SHADOW)
-  if(WITH_TESTS)
+  if(WITH_TESTS OR WITH_HIVE)
     add_subdirectory(test)
   endif()
   if(ANDROID)

llarp/messages/relay_commit.cpp

@@ -262,6 +262,7 @@ namespace llarp
       if(self->fromAddr.has_value())
       {
         // only do ip limiting from non service nodes
+#ifndef LOKINET_HIVE
         if(self->context->CheckPathLimitHitByIP(self->fromAddr.value()))
         {
           // we hit a limit so tell it to slow tf down
@@ -273,6 +274,7 @@ namespace llarp
           self->hop = nullptr;
           return;
         }
+#endif
       }
 
       if(!self->context->Router()->ConnectionToRouterAllowed(

llarp/tooling/router_event.cpp

@@ -55,6 +55,8 @@ namespace tooling
     : RouterEvent("PathRequestReceivedEvent", routerID, true)
     , prevHop(hop->info.downstream)
     , nextHop(hop->info.upstream)
+    , txid(hop->info.txID)
+    , rxid(hop->info.rxID)
   {
     isEndpoint = false;
     if (routerID == nextHop)

llarp/tooling/router_event.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <router_id.hpp>
+#include <path/path_types.hpp>
 
 #include <string>
 #include <vector>
@@ -9,6 +10,8 @@
 namespace llarp
 {
 
+  struct PathID_t;
+
   namespace path
   {
     struct Path;
@@ -62,6 +65,9 @@ namespace tooling
     llarp::RouterID prevHop;
     llarp::RouterID nextHop;
 
+    llarp::PathID_t txid;
+    llarp::PathID_t rxid;
+
     bool isEndpoint = false;
   };
 

pybind/CMakeLists.txt

@@ -8,6 +8,7 @@ set(LLARP_PYBIND_SRC
   llarp/router_contact.cpp
   llarp/crypto/types.cpp
   llarp/config.cpp
+  llarp/path/path_types.cpp
   llarp/path/path_hop_config.cpp
   llarp/handlers/pyhandler.cpp
   llarp/tooling/router_hive.cpp

pybind/common.hpp

@@ -49,6 +49,9 @@ namespace llarp
   void
   Config_Init(py::module & mod);
 
+  void
+  PathTypes_Init(py::module & mod);
+
   namespace path
   {
     void

pybind/llarp/path/path_hop_config.cpp

@@ -6,7 +6,7 @@ namespace llarp
   namespace path
   {
     void
-    PathHopConfig_Init(py::module& mod)
+    PathHopConfig_Init(py::module & mod)
     {
       auto str_func = [](PathHopConfig *hop) {
           std::string s = "Hop: [";
@@ -19,6 +19,8 @@ namespace llarp
       py::class_< PathHopConfig >(mod, "PathHopConfig")
       .def_readonly("rc", &PathHopConfig::rc)
       .def_readonly("upstreamRouter", &PathHopConfig::upstream)
+      .def_readonly("txid", &PathHopConfig::txID)
+      .def_readonly("rxid", &PathHopConfig::rxID)
       .def("ToString", str_func)
       .def("__str__", str_func)
       .def("__repr__", str_func);

pybind/llarp/path/path_types.cpp

@@ -0,0 +1,18 @@
+#include <path/path.hpp>
+#include "common.hpp"
+
+namespace llarp
+{
+  void
+  PathTypes_Init(py::module & mod)
+  {
+    py::class_< PathID_t >(mod, "PathID")
+    .def("__eq__", [](const PathID_t* const lhs, const PathID_t* const rhs) {
+        return *lhs == *rhs;
+    })
+    .def("ShortHex", &PathID_t::ShortHex)
+    .def("__str__", &PathID_t::ShortHex)
+    .def("__repr__", &PathID_t::ShortHex);
+  }
+
+}  // namespace llarp

pybind/llarp/router_contact.cpp

@@ -8,14 +8,17 @@ namespace llarp
   {
     py::class_< RouterContact >(mod, "RouterContact")
         .def(py::init<>())
+        .def_property_readonly("routerID", [](const RouterContact* const rc) -> llarp::RouterID {
+            return llarp::RouterID(rc->pubkey);
+        })
         .def("ReadFile", &RouterContact::Read)
         .def("WriteFile", &RouterContact::Write)
-        .def("ToString", [](const RouterContact rc) -> std::string {
-          return rc.ToJson().dump();
+        .def("ToString", [](const RouterContact* const rc) -> std::string {
+          return rc->ToJson().dump();
         })
-        .def("Verify", [](const RouterContact rc) -> bool {
+        .def("Verify", [](const RouterContact* const rc) -> bool {
           const llarp_time_t now = llarp::time_now_ms();
-          return rc.Verify(now);
+          return rc->Verify(now);
         });
   }
 }  // namespace llarp

pybind/llarp/router_id.cpp

@@ -15,6 +15,8 @@ namespace llarp
         .def("__repr__", &RouterID::ToString)
         .def("__str__", &RouterID::ToString)
         .def("ShortString", &RouterID::ShortString)
-        .def("__eq__", &RouterID::operator==);
+        .def("__eq__", [](const RouterID* const lhs, const RouterID* const rhs) {
+            return *lhs == *rhs;
+        });
   }
 }

pybind/llarp/tooling/router_event.cpp

@@ -23,12 +23,16 @@ namespace tooling
     py::class_<PathRequestReceivedEvent, RouterEvent>(mod, "PathRequestReceivedEvent")
     .def_readonly("prevHop", &PathRequestReceivedEvent::prevHop)
     .def_readonly("nextHop", &PathRequestReceivedEvent::nextHop)
+    .def_readonly("txid", &PathRequestReceivedEvent::txid)
+    .def_readonly("rxid", &PathRequestReceivedEvent::rxid)
     .def_readonly("isEndpoint", &PathRequestReceivedEvent::isEndpoint);
+
     py::class_<PubIntroReceivedEvent, RouterEvent>(mod, "DhtPubIntroReceivedEvent")
     .def_readonly("from", &PubIntroReceivedEvent::From)
     .def_readonly("location", &PubIntroReceivedEvent::IntrosetLocation)
       .def_readonly("relayOrder", &PubIntroReceivedEvent::RelayOrder)
       .def_readonly("txid", &PubIntroReceivedEvent::TxID);
+
     py::class_<GotIntroReceivedEvent, RouterEvent>(mod, "DhtGotIntroReceivedEvent")
     .def_readonly("from", &GotIntroReceivedEvent::From)
     .def_readonly("location", &GotIntroReceivedEvent::Introset)

pybind/module.cpp

@@ -10,6 +10,7 @@ PYBIND11_MODULE(pyllarp, m)
   llarp::CryptoTypes_Init(m);
   llarp::Context_Init(m);
   llarp::Config_Init(m);
+  llarp::PathTypes_Init(m);
   llarp::path::PathHopConfig_Init(m);
   llarp::handlers::PyHandler_Init(m);
   llarp::service::Address_Init(m);

test/CMakeLists.txt

@@ -1,3 +1,12 @@
+if (WITH_HIVE)
+  add_custom_target(hive ${CMAKE_COMMAND} -E
+      env PYTHONPATH="$ENV{PYTHONPATH}:${CMAKE_BINARY_DIR}/pyllarp"
+      ${PYTHON_EXECUTABLE} -m pytest
+      ${CMAKE_CURRENT_SOURCE_DIR}/hive
+      DEPENDS
+      ${STATIC_LIB} pyllarp)
+endif()
+
 set(GTEST_EXE testAll)
 set(CATCH_EXE catchAll)
 

test/hive/conftest.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+import hive
+import pytest
+
+@pytest.fixture(scope="session")
+def HiveTenTen():
+  router_hive = hive.RouterHive(n_relays=10, n_clients=10, netid="hive")
+  router_hive.Start()
+
+  yield router_hive
+
+  router_hive.Stop()

test/hive/hive.py

@@ -34,9 +34,8 @@ class RouterHive(object):
   def RemoveTmpDir(self):
     if self.tmpdir.startswith("/tmp/") and len(self.tmpdir) > 5:
       print("calling rmdir -r %s" % self.tmpdir)
-      if (input("Is this ok? (y/n): ").lower().strip()[:1] == "y"):
-        rmtree(self.tmpdir, ignore_errors=True)
-        return True
+      rmtree(self.tmpdir, ignore_errors=True)
+      return True
     else:
       print("not removing dir %s because it doesn't start with /tmp/" % self.tmpdir)
 

test/hive/test_path_builds.py

@@ -0,0 +1,71 @@
+from time import time
+
+def test_path_builds(HiveTenTen):
+  h = HiveTenTen
+
+  start_time = time()
+  cur_time = start_time
+  test_duration = 10 #seconds
+
+  log_attempts = True
+
+  paths = []
+
+  while cur_time < start_time + test_duration:
+
+    h.CollectAllEvents()
+
+    for event in h.events:
+      event_name = event.__class__.__name__
+
+      if log_attempts and event_name == "PathAttemptEvent":
+        path = dict()
+        path["hops"] = event.hops
+        path["received"] = [False] * len(event.hops)
+        path["prev"] = [None] * len(event.hops)
+        for i in range(1, len(event.hops)):
+          path["prev"][i] = event.hops[i-1].rc.routerID
+        path["prev"][0] = event.routerID
+        paths.append(path)
+
+      elif event_name == "PathRequestReceivedEvent":
+        for path in paths:
+          for i in range(len(path["hops"])):
+            assert type(path["hops"][i].upstreamRouter) == type(event.nextHop)
+            assert type(path["prev"][i]) == type(event.prevHop)
+            assert type(path["hops"][i].txid) == type(event.txid)
+            assert type(path["hops"][i].rxid) == type(event.rxid)
+            if (path["hops"][i].upstreamRouter == event.nextHop and
+                path["prev"][i] == event.prevHop and
+                path["hops"][i].txid == event.txid and
+                path["hops"][i].rxid == event.rxid):
+              path["received"][i] = True
+
+    h.events = []
+    cur_time = time()
+
+    # only collect path attempts for 3 seconds
+    if cur_time > start_time + 3:
+      log_attempts = False
+
+  assert len(paths) > 0
+
+  fail_count = 0
+  expected_count = 0
+
+  paths_ok = []
+  for path in paths:
+    path_ok = True
+    for rcv in path["received"]:
+      expected_count = expected_count + 1
+      if not rcv:
+        path_ok = False
+        fail_count = fail_count + 1
+
+    paths_ok.append(path_ok)
+
+  print("Path count: {}, Expected rcv: {}, Failed rcv: {}".format(len(paths), expected_count, fail_count))
+
+  for path_ok in paths_ok:
+    assert path_ok
+