Merge pull request #1167 from tewinget/tooling

RouterHive initial PR
pull/1219/head
Jeff 4 years ago committed by GitHub
commit d3091cf9fc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

1
.gitignore vendored

@ -7,6 +7,7 @@
*.so
build/
**/__pycache__/**
llarpd
*.test

4
.gitmodules vendored

@ -19,3 +19,7 @@
[submodule "external/date"]
path = external/date
url = https://github.com/HowardHinnant/date.git
[submodule "external/pybind11"]
path = external/pybind11
url = https://github.com/pybind/pybind11
branch = stable

@ -38,7 +38,11 @@ option(USE_SHELLHOOKS "enable shell hooks on compile time (dangerous)" OFF)
option(WARNINGS_AS_ERRORS "treat all warnings as errors. turn off for development, on for release" OFF)
option(TRACY_ROOT "include tracy profiler source" OFF)
option(WITH_TESTS "build unit tests" ON)
#option(WITH_SYSTEMD ...) defined below
option(WITH_HIVE "build simulation stubs" OFF)
if(WITH_HIVE)
set(WITH_SHARED ON)
endif()
include(cmake/target_link_libraries_system.cmake)
include(cmake/add_import_library.cmake)
@ -270,9 +274,12 @@ if(SUBMODULE_CHECK)
check_submodule(external/ghc-filesystem)
check_submodule(external/optional-lite)
check_submodule(external/date)
check_submodule(external/pybind11)
endif()
endif()
add_subdirectory(external/pybind11 EXCLUDE_FROM_ALL)
if(WITH_TESTS)
add_subdirectory(external/googletest EXCLUDE_FROM_ALL)
endif()
@ -295,13 +302,20 @@ if(TRACY_ROOT)
list(APPEND LIBS -ldl)
endif()
if(WITH_HIVE)
add_definitions(-DLOKINET_HIVE=1)
endif()
add_subdirectory(crypto)
add_subdirectory(llarp)
add_subdirectory(libabyss)
add_subdirectory(daemon)
add_subdirectory(pybind)
if (NOT SHADOW)
if(WITH_TESTS)
if(WITH_TESTS OR WITH_HIVE)
add_subdirectory(test)
endif()
if(ANDROID)

@ -109,6 +109,11 @@ COVERAGE_OUTDIR ?= "$(TMPDIR)/lokinet-coverage"
TRACY_ROOT ?=
# enable sanitizer
XSAN ?= False
# lokinet hive build
HIVE ?= OFF
# compile unittests
TESTS ?= ON
# cmake generator type
CMAKE_GEN ?= Unix Makefiles
@ -125,7 +130,7 @@ SCAN_BUILD ?= scan-build
UNAME = $(shell which uname)
COMMON_CMAKE_OPTIONS = -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DWITH_SHARED=$(SHARED_LIB) -DDOWNLOAD_SODIUM=$(DOWNLOAD_SODIUM) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DXSAN=$(XSAN)
COMMON_CMAKE_OPTIONS = -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DWITH_SHARED=$(SHARED_LIB) -DDOWNLOAD_SODIUM=$(DOWNLOAD_SODIUM) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DXSAN=$(XSAN) -DWITH_HIVE=$(HIVE) -DWITH_TESTS=$(TESTS)
ifeq ($(shell $(UNAME)),SunOS)
CONFIG_CMD = $(shell gecho -n "cd '$(BUILD_ROOT)' && " ; gecho -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DUSE_SHELLHOOKS=$(SHELL_HOOKS) $(COMMON_CMAKE_OPTIONS) '$(REPO)'")
@ -305,7 +310,7 @@ abyss: debug
$(ABYSS_EXE)
format:
$(FORMAT) -i $$(find jni daemon llarp include libabyss | grep -E '\.[h,c](pp)?$$')
$(FORMAT) -i $$(find jni daemon llarp include libabyss pybind | grep -E '\.[h,c](pp)?$$')
format-verify: format
(type $(FORMAT))

@ -0,0 +1,38 @@
What is a RouterEvent?
A RouterEvent is a way of representing a conceptual event that took place in a "router" (relay or client).
RouterEvents are used in order to collect information about a network all in one place and preserve causality.
How do I make a new RouterEvent?
Add your event following the structure in llarp/tooling/router_event.{hpp,cpp}
Add your event to pybind in pybind/llarp/tooling/router_event.cpp
What if a class my event uses is missing members in pybind?
Find the relevant file pybind/whatever/class.cpp and remedy that!
What if I need to refer to classes which aren't available already in pybind?
Add pybind/namespace/namespace/etc/class.cpp and pybind it!
You will need to edit the following files accordingly:
pybind/common.hpp
pybind/module.cpp
pybind/CMakeLists.txt
How do I use a RouterEvent?
From the cpp side, find the place in the code where it makes the most logical sense
that the conceptual event has taken place (and you have the relevant information to
create the "event" instance) and create it there as follows:
#include <llarp/tooling/relevant_event_header.hpp>
where the event takes place, do the following:
auto event = std::make_unique<event_type_here>(constructor_args...);
somehow_get_a_router->NotifyRouterEvent(std::move(event));
From the Python side...it's a python object!

1
external/pybind11 vendored

@ -0,0 +1 @@
Subproject commit fe755dce12766820a99eefbde32d6ceb0a828ca8

@ -16,6 +16,13 @@ struct llarp_nodedb;
struct llarp_nodedb_iter;
struct llarp_main;
#ifdef LOKINET_HIVE
namespace tooling
{
struct RouterHive;
} // namespace tooling
#endif
namespace llarp
{
class Logic;
@ -32,7 +39,7 @@ namespace llarp
struct Context
{
/// get context from main pointer
static Context *
static std::shared_ptr< Context >
Get(llarp_main *);
Context() = default;
@ -88,6 +95,11 @@ namespace llarp
bool
CallSafe(std::function< void(void) > f);
#ifdef LOKINET_HIVE
void
InjectHive(tooling::RouterHive *hive);
#endif
private:
void
SetPIDFile(const std::string &fname);

@ -199,6 +199,7 @@ set(LIB_SRC
service/tag_lookup_job.cpp
service/tag.cpp
)
if(TRACY_ROOT)
set(LIB_SRC ${LIB_SRC} ${TRACY_ROOT}/TracyClient.cpp)
endif()
@ -207,7 +208,12 @@ if(TESTNET)
set(LIB_SRC ${LIB_SRC} testnet.c)
endif()
if(WITH_HIVE)
set(LIB_SRC ${LIB_SRC} tooling/router_hive.cpp)
endif()
add_library(${STATIC_LIB} STATIC ${LIB_SRC})
target_include_directories(${STATIC_LIB} PUBLIC ${CURL_INCLUDE_DIRS})
target_link_libraries(${STATIC_LIB} PUBLIC cxxopts ${ABYSS_LIB} ${PLATFORM_LIB} ${UTIL_LIB} ${CRYPTOGRAPHY_LIB})
@ -219,7 +225,7 @@ endif()
if(WITH_SHARED)
add_library(${SHARED_LIB} SHARED ${LIB_SRC})
set(LIBS ${LIBS} Threads::Threads)
target_link_libraries(${SHARED_LIB} PUBLIC ${ABYSS_LIB} ${CRYPTOGRAPHY_LIB} ${UTIL_LIB} ${PLATFORM_LIB} ${LIBS})
target_link_libraries(${SHARED_LIB} PUBLIC cxxopts ${ABYSS_LIB} ${CRYPTOGRAPHY_LIB} ${UTIL_LIB} ${PLATFORM_LIB} ${LIBS})
if (WIN32)
target_link_libraries(${SHARED_LIB} PUBLIC ws2_32 iphlpapi)
else()

@ -13,6 +13,7 @@
#include <vector>
#include <unordered_set>
struct llarp_config;
namespace llarp
{
struct ConfigParser;
@ -33,7 +34,7 @@ namespace llarp
class RouterConfig
{
private:
public:
/// always maintain this many connections to other routers
size_t m_minConnectedRouters = 2;
@ -96,7 +97,7 @@ namespace llarp
public:
using NetConfig = std::unordered_multimap< std::string, std::string >;
private:
public:
nonstd::optional< bool > m_enableProfiling;
std::string m_routerProfilesFile = "profiles.dat";
std::string m_strictConnect;
@ -116,7 +117,7 @@ namespace llarp
class NetdbConfig
{
private:
public:
std::string m_nodedbDir;
public:
@ -148,7 +149,7 @@ namespace llarp
using LinkInfo = std::tuple< std::string, int, uint16_t, ServerOptions >;
using Links = std::vector< LinkInfo >;
private:
public:
LinkInfo m_OutboundLink;
Links m_InboundLinks;
@ -188,7 +189,7 @@ namespace llarp
class ApiConfig
{
private:
public:
bool m_enableRPCServer = false;
std::string m_rpcBindAddr = "127.0.0.1:1190";
@ -233,7 +234,7 @@ namespace llarp
struct Config
{
private:
public:
bool
parse(const ConfigParser& parser);
@ -256,6 +257,9 @@ namespace llarp
bool
LoadFromStr(string_view str);
llarp_config*
Copy() const;
};
fs::path

@ -95,7 +95,8 @@ namespace llarp
{
llarp::LogInfo(llarp::VERSION_FULL, " ", llarp::RELEASE_MOTTO);
llarp::LogInfo("starting up");
mainloop = llarp_make_ev_loop();
if(mainloop == nullptr)
mainloop = llarp_make_ev_loop();
logic->set_event_loop(mainloop.get());
mainloop->set_logic(logic);
@ -145,6 +146,7 @@ namespace llarp
// run net io thread
llarp::LogInfo("running mainloop");
llarp_ev_loop_run_single_process(mainloop, logic);
if(closeWaiter)
{
@ -298,13 +300,21 @@ namespace llarp
configfile = fname;
return Configure();
}
#ifdef LOKINET_HIVE
void
Context::InjectHive(tooling::RouterHive *hive)
{
router->hive = hive;
}
#endif
} // namespace llarp
struct llarp_main
{
llarp_main(llarp_config *conf);
~llarp_main() = default;
std::unique_ptr< llarp::Context > ctx;
std::shared_ptr< llarp::Context > ctx;
};
struct llarp_config
@ -317,6 +327,17 @@ struct llarp_config
}
};
namespace llarp
{
llarp_config *
Config::Copy() const
{
llarp_config *ptr = new llarp_config();
ptr->impl = *this;
return ptr;
}
} // namespace llarp
extern "C"
{
size_t
@ -539,11 +560,11 @@ llarp_main::llarp_main(llarp_config *conf)
namespace llarp
{
Context *
std::shared_ptr< Context >
Context::Get(llarp_main *m)
{
if(m == nullptr || m->ctx == nullptr)
return nullptr;
return m->ctx.get();
return m->ctx;
}
} // namespace llarp

@ -6,6 +6,8 @@
#include <nodedb.hpp>
#include <tooling/dht_event.hpp>
namespace llarp
{
namespace dht
@ -13,7 +15,14 @@ namespace llarp
void
ExploreNetworkJob::Start(const TXOwner &peer)
{
parent->DHTSendTo(peer.node.as_array(), new FindRouterMessage(peer.txid));
auto msg = new FindRouterMessage(peer.txid);
auto router = parent->GetRouter();
router->NotifyRouterEvent< tooling::FindRouterSentEvent >(
router->pubkey(),
msg);
parent->DHTSendTo(peer.node.as_array(), msg);
}
void

@ -7,6 +7,8 @@
#include <router/abstractrouter.hpp>
#include <routing/dht_message.hpp>
#include <tooling/dht_event.hpp>
namespace llarp
{
namespace dht
@ -152,6 +154,12 @@ namespace llarp
std::vector< std::unique_ptr< IMessage > > &replies) const
{
auto &dht = *ctx->impl;
auto router = dht.GetRouter();
router->NotifyRouterEvent< tooling::FindRouterReceivedEvent >(
router->pubkey(),
this);
if(!dht.AllowTransit())
{
llarp::LogWarn("Got DHT lookup from ", From,

@ -1,10 +1,12 @@
#include <dht/messages/gotintro.hpp>
#include <service/intro.hpp>
#include <dht/context.hpp>
#include <memory>
#include <path/path_context.hpp>
#include <router/abstractrouter.hpp>
#include <routing/dht_message.hpp>
#include <tooling/dht_event.hpp>
#include <utility>
namespace llarp
@ -22,7 +24,13 @@ namespace llarp
llarp_dht_context *ctx,
std::vector< std::unique_ptr< IMessage > > & /*replies*/) const
{
auto &dht = *ctx->impl;
auto &dht = *ctx->impl;
auto *router = dht.GetRouter();
router->NotifyRouterEvent< tooling::GotIntroReceivedEvent >(
router->pubkey(), Key_t(From.data()),
(found.size() > 0 ? found[0] : llarp::service::EncryptedIntroSet{}),
txid);
for(const auto &introset : found)
{

@ -5,6 +5,7 @@
#include <path/path_context.hpp>
#include <router/abstractrouter.hpp>
#include <router/i_rc_lookup_handler.hpp>
#include <tooling/rc_event.hpp>
namespace llarp
{
@ -124,9 +125,13 @@ namespace llarp
{
if(not dht.GetRouter()->rcLookupHandler().CheckRC(rc))
return false;
if(txid == 0)
if(txid == 0) // txid == 0 on gossip
{
dht.GetRouter()->GossipRCIfNeeded(rc);
LogWarn("Received Gossiped RC, generating RCGossipReceivedEvent");
auto *router = dht.GetRouter();
router->NotifyRouterEvent< tooling::RCGossipReceivedEvent >(
router->pubkey(), rc);
router->GossipRCIfNeeded(rc);
}
}
return true;

@ -7,6 +7,8 @@
#include <routing/dht_message.hpp>
#include <nodedb.hpp>
#include <tooling/dht_event.hpp>
namespace llarp
{
namespace dht
@ -56,8 +58,14 @@ namespace llarp
llarp_dht_context *ctx,
std::vector< std::unique_ptr< IMessage > > &replies) const
{
const auto now = ctx->impl->Now();
const auto keyStr = introset.derivedSigningKey.ToHex();
const auto now = ctx->impl->Now();
const llarp::dht::Key_t addr(introset.derivedSigningKey);
const auto keyStr = addr.ToHex();
auto router = ctx->impl->GetRouter();
router->NotifyRouterEvent< tooling::PubIntroReceivedEvent >(
router->pubkey(), Key_t(relayed ? router->pubkey() : From.data()),
addr, txID, relayOrder);
auto &dht = *ctx->impl;
if(!introset.Verify(now))
@ -78,8 +86,6 @@ namespace llarp
return true;
}
const llarp::dht::Key_t addr(introset.derivedSigningKey);
// identify closest 4 routers
auto closestRCs = dht.GetRouter()->nodedb()->FindClosestTo(
addr, IntroSetStorageRedundancy);

@ -788,14 +788,22 @@ namespace libuv
uv_loop_configure(&m_Impl, UV_LOOP_BLOCK_SIGNAL, SIGPIPE);
#endif
m_LogicCaller.data = this;
uv_async_init(&m_Impl, &m_LogicCaller, [](uv_async_t* h) {
Loop* l = static_cast< Loop* >(h->data);
while(not l->m_LogicCalls.empty())
{
auto f = l->m_LogicCalls.popFront();
f();
}
});
int err;
if((err = uv_async_init(&m_Impl, &m_LogicCaller,
[](uv_async_t* h) {
Loop* l = static_cast< Loop* >(h->data);
while(not l->m_LogicCalls.empty())
{
auto f = l->m_LogicCalls.popFront();
f();
}
}))
!= 0)
{
llarp::LogError("Libuv uv_async_init returned error: ", uv_strerror(err));
return false;
}
m_TickTimer = new uv_timer_t;
m_TickTimer->data = this;
m_Run.store(true);

@ -14,7 +14,7 @@ llarp_vpn_io_impl::AsyncClose()
void
llarp_vpn_io_impl::CallSafe(std::function< void(void) > f)
{
llarp::Context* ctx = llarp::Context::Get(ptr);
auto ctx = llarp::Context::Get(ptr);
if(ctx && ctx->CallSafe(f))
return;
else if(ctx == nullptr || ctx->logic == nullptr)

@ -17,7 +17,7 @@ namespace llarp
{
}
bool
virtual bool
HandleInboundPacket(const service::ConvoTag, const llarp_buffer_t &,
service::ProtocolType) override
{

@ -13,6 +13,7 @@
#include <util/logging/logger.hpp>
#include <util/meta/memfn.hpp>
#include <util/thread/logic.hpp>
#include <tooling/path_event.hpp>
#include <functional>
#include <nonstd/optional.hpp>
@ -261,6 +262,7 @@ namespace llarp
if(self->fromAddr.has_value())
{
// only do ip limiting from non service nodes
#ifndef LOKINET_HIVE
if(self->context->CheckPathLimitHitByIP(self->fromAddr.value()))
{
// we hit a limit so tell it to slow tf down
@ -272,6 +274,7 @@ namespace llarp
self->hop = nullptr;
return;
}
#endif
}
if(!self->context->Router()->ConnectionToRouterAllowed(
@ -416,6 +419,9 @@ namespace llarp
// TODO: check if we really want to accept it
self->hop->started = now;
self->context->Router()->NotifyRouterEvent< tooling::PathRequestReceivedEvent >(
self->context->Router()->pubkey(), self->hop);
size_t sz = self->frames[0].size();
// shift
std::array< EncryptedFrame, 8 > frames;

@ -10,6 +10,7 @@
#include <util/logging/logger.hpp>
#include <util/meta/memfn.hpp>
#include <util/thread/logic.hpp>
#include <tooling/path_event.hpp>
#include <functional>
#include <utility>
@ -25,13 +26,16 @@ namespace llarp
uint64_t status = 0;
HopHandler_ptr path;
AbstractRouter* router;
PathID_t pathid;
LRSM_AsyncHandler(std::array< EncryptedFrame, 8 > _frames, uint64_t _status,
HopHandler_ptr _path, AbstractRouter* _router)
HopHandler_ptr _path, AbstractRouter* _router,
const PathID_t& pathid)
: frames(std::move(_frames))
, status(_status)
, path(std::move(_path))
, router(_router)
, pathid(pathid)
{
}
@ -40,6 +44,9 @@ namespace llarp
void
handle()
{
router->NotifyRouterEvent< tooling::PathStatusReceivedEvent >(
router->pubkey(), pathid, status);
path->HandleLRSM(status, frames, router);
}
@ -140,8 +147,8 @@ namespace llarp
return false;
}
auto handler =
std::make_shared< LRSM_AsyncHandler >(frames, status, path, router);
auto handler = std::make_shared< LRSM_AsyncHandler >(frames, status, path,
router, pathid);
handler->queue_handle();

@ -15,6 +15,7 @@
#include <util/buffer.hpp>
#include <util/endian.hpp>
#include <util/thread/logic.hpp>
#include <tooling/path_event.hpp>
#include <deque>
@ -201,6 +202,10 @@ namespace llarp
{
if(failedAt.has_value())
{
r->NotifyRouterEvent< tooling::PathBuildRejectedEvent >(
Endpoint(),
RXID(),
failedAt.value());
LogWarn(Name(), " build failed at ", failedAt.value());
r->routerProfiling().MarkHopFail(failedAt.value());
}

@ -8,6 +8,7 @@
#include <router/abstractrouter.hpp>
#include <util/buffer.hpp>
#include <util/thread/logic.hpp>
#include <tooling/path_event.hpp>
#include <functional>
@ -131,6 +132,10 @@ namespace llarp
{
if(!ctx->pathset->IsStopped())
{
ctx->router->NotifyRouterEvent< tooling::PathAttemptEvent >(
ctx->router->pubkey(),
ctx->path);
const RouterID remote = ctx->path->Upstream();
const ILinkMessage* msg = &ctx->LRCM;
auto sentHandler = [ctx](auto status) {
@ -445,6 +450,7 @@ namespace llarp
auto path = std::make_shared< path::Path >(hops, self.get(), roles,
std::move(path_shortName));
LogInfo(Name(), " build ", path->ShortName(), ": ", path->HopsString());
path->SetBuildResultHook(
[self](Path_ptr p) { self->HandlePathBuilt(p); });
ctx->AsyncGenerateKeys(path, m_router->logic(), m_router->threadpool(),
@ -456,6 +462,7 @@ namespace llarp
{
buildIntervalLimit = MIN_PATH_BUILD_INTERVAL;
m_router->routerProfiling().MarkPathSuccess(p.get());
LogInfo(p->Name(), " built latency=", p->intro.latency);
m_BuildStats.success++;
}

@ -10,6 +10,11 @@
#include <ev/ev.h>
#include <functional>
#include <router_contact.hpp>
#include <tooling/router_event.hpp>
#ifdef LOKINET_HIVE
#include "tooling/router_hive.hpp"
#endif
struct llarp_buffer_t;
struct llarp_dht_context;
@ -59,6 +64,10 @@ namespace llarp
struct AbstractRouter
{
#ifdef LOKINET_HIVE
tooling::RouterHive *hive;
#endif
virtual ~AbstractRouter() = default;
virtual bool
@ -260,6 +269,19 @@ namespace llarp
/// gossip an rc if required
virtual void
GossipRCIfNeeded(const RouterContact rc) = 0;
template<class EventType, class... Params>
void
NotifyRouterEvent(Params&&... args) const
{
// TODO: no-op when appropriate
auto event = std::make_unique< EventType >(args...);
#ifdef LOKINET_HIVE
hive->NotifyEvent(std::move(event));
#elif LOKINET_DEBUG
LogDebug(event->ToString());
#endif
}
};
} // namespace llarp

@ -3,6 +3,7 @@
#include <dht/messages/gotrouter.hpp>
#include <util/time.hpp>
#include <constants/link_layer.hpp>
#include <tooling/rc_event.hpp>
namespace llarp
{
@ -20,16 +21,20 @@ namespace llarp
}
void
RCGossiper::Init(ILinkManager* l, const RouterID& ourID)
RCGossiper::Init(ILinkManager* l, const RouterID& ourID,
AbstractRouter* router)
{
m_OurRouterID = ourID;
m_LinkManager = l;
m_router = router;
}
bool
RCGossiper::ShouldGossipOurRC(Time_t now) const
{
return now >= (m_LastGossipedOurRC + GossipOurRCInterval);
bool should = now >= (m_LastGossipedOurRC + GossipOurRCInterval);
LogWarn("ShouldGossipOurRC: ", should);
return should;
}
bool
@ -93,6 +98,11 @@ namespace llarp
if(not gossip.BEncode(&buf))
return;
msg.resize(buf.cur - buf.base);
m_router->NotifyRouterEvent< tooling::RCGossipSentEvent >(
m_router->pubkey(),
rc);
// send message
peerSession->SendMessageBuffer(std::move(msg), nullptr);
});

@ -5,6 +5,7 @@
#include <router/i_gossiper.hpp>
#include <router/i_outbound_message_handler.hpp>
#include <link/i_link_manager.hpp>
#include <router/abstractrouter.hpp>
namespace llarp
{
@ -27,13 +28,15 @@ namespace llarp
IsOurRC(const RouterContact &rc) const override;
void
Init(ILinkManager *, const RouterID &);
Init(ILinkManager *, const RouterID &, AbstractRouter *);
private:
RouterID m_OurRouterID;
Time_t m_LastGossipedOurRC = 0s;
ILinkManager *m_LinkManager = nullptr;
util::DecayingHashSet< RouterID > m_Filter;
AbstractRouter *m_router;
};
} // namespace llarp

@ -23,6 +23,8 @@
#include <util/str.hpp>
#include <ev/ev.hpp>
#include "tooling/router_event.hpp"
#include <fstream>
#include <cstdlib>
#include <iterator>
@ -53,7 +55,12 @@ namespace llarp
, _dht(llarp_dht_context_new(this))
, inbound_link_msg_parser(this)
, _hiddenServiceContext(this)
#ifdef LOKINET_HIVE
, _randomStartDelay(
std::chrono::milliseconds((llarp::randint() % 1250) + 2000))
#else
, _randomStartDelay(std::chrono::seconds((llarp::randint() % 30) + 10))
#endif
{
m_keyManager = std::make_shared< KeyManager >();
@ -298,6 +305,7 @@ namespace llarp
llarp_ev_loop_stop(_netloop);
disk->stop();
disk->shutdown();
_running.store(false);
}
void
@ -462,6 +470,8 @@ namespace llarp
llarp::LogError("invalid key for strict-connect: ", val);
}
llarp::LogWarn("Bootstrap routers list size: ",
conf->bootstrap.routers.size());
std::vector< std::string > configRouters = conf->connect.routers;
configRouters.insert(configRouters.end(), conf->bootstrap.routers.begin(),
conf->bootstrap.routers.end());
@ -562,6 +572,7 @@ namespace llarp
const auto &key = std::get< LinksConfig::Interface >(serverConfig);
int af = std::get< LinksConfig::AddressFamily >(serverConfig);
uint16_t port = std::get< LinksConfig::Port >(serverConfig);
llarp::LogWarn("tun: ", key, " -- af: ", af, " -- port: ", port);
if(!server->Configure(netloop(), key, af, port))
{
LogError("failed to bind inbound link on ", key, " port ", port);
@ -1050,7 +1061,7 @@ namespace llarp
const RouterID us = pubkey();
LogInfo("initalized service node: ", us);
// init gossiper here
_rcGossiper.Init(&_linkManager, us);
_rcGossiper.Init(&_linkManager, us, this);
// relays do not use profiling
routerProfiling().Disable();
}

@ -116,6 +116,12 @@ namespace llarp
return ExtractStatus();
}
std::string
ToString() const
{
return ToJson().dump();
}
bool
BEncode(llarp_buffer_t *buf) const;

@ -9,6 +9,12 @@ namespace llarp
return std::string(llarp::Base32Encode(*this, stack)) + ".snode";
}
std::string
RouterID::ShortString() const
{
return ToString().substr(0, 8);
}
util::StatusObject
RouterID::ExtractStatus() const
{

@ -30,6 +30,9 @@ namespace llarp
std::string
ToString() const;
std::string
ShortString() const;
bool
FromString(const std::string& str);

@ -39,6 +39,12 @@ namespace llarp
{
}
explicit Address(const std::string& str) : AlignedBuffer< 32 >()
{
if(not FromString(str))
throw std::runtime_error("invalid address");
}
explicit Address(const Data& buf) : AlignedBuffer< 32 >(buf)
{
}

@ -192,6 +192,16 @@ namespace llarp
return nullptr;
}
void
Context::InjectEndpoint(std::string name, std::shared_ptr< Endpoint > ep)
{
ep->LoadKeyFile();
if(ep->Start())
{
m_Endpoints.emplace(std::move(name), std::move(ep));
}
}
bool
Context::AddEndpoint(const Config::section_t &conf, bool autostart)
{

@ -46,6 +46,10 @@ namespace llarp
bool
AddEndpoint(const Config::section_t &conf, bool autostart = false);
/// inject endpoint instance
void
InjectEndpoint(std::string name, std::shared_ptr< Endpoint > ep);
/// stop and remove an endpoint by name
/// return false if we don't have the hidden service with that name
bool
@ -54,6 +58,12 @@ namespace llarp
Endpoint_ptr
GetEndpointByName(const std::string &name);
Endpoint_ptr
GetDefault()
{
return GetEndpointByName("default");
}
bool
StartAll();

@ -1,7 +1,9 @@
#include <chrono>
#include <memory>
#include <service/endpoint.hpp>
#include <dht/context.hpp>
#include <dht/key.hpp>
#include <dht/messages/findintro.hpp>
#include <dht/messages/findrouter.hpp>
#include <dht/messages/gotintro.hpp>
@ -23,6 +25,7 @@
#include <util/meta/memfn.hpp>
#include <hook/shell.hpp>
#include <link/link_manager.hpp>
#include <tooling/dht_event.hpp>
#include <utility>
@ -197,14 +200,19 @@ namespace llarp
// prefetch addrs
for(const auto& addr : m_state->m_PrefetchAddrs)
{
if(!EndpointUtil::HasPathToService(addr, m_state->m_RemoteSessions))
{
if(!EnsurePathToService(
addr, [](Address, OutboundContext*) {}, 10s))
{
LogWarn("failed to ensure path to ", addr);
}
}
EnsurePathToService(
addr,
[](Address, OutboundContext* ctx) {
#ifdef LOKINET_HIVE
std::vector< byte_t > discard;
discard.resize(128);
ctx->AsyncEncryptAndSendTo(llarp_buffer_t(discard),
eProtocolControl);
#else
(void)ctx;
#endif
},
10s);
}
// deregister dead sessions
@ -467,10 +475,15 @@ namespace llarp
// do publishing for each path selected
size_t published = 0;
for(const auto& path : paths)
{
for(size_t i = 0; i < llarp::dht::IntroSetRequestsPerRelay; ++i)
{
r->NotifyRouterEvent< tooling::PubIntroSentEvent >(
r->pubkey(),
llarp::dht::Key_t{introset.derivedSigningKey.as_array()},
RouterID(path->hops[path->hops.size() - 1].rc.pubkey), published);
if(PublishIntroSetVia(introset, r, path, published))
published++;
}
@ -758,6 +771,8 @@ namespace llarp
bool
Endpoint::LookupRouterAnon(RouterID router, RouterLookupHandler handler)
{
using llarp::dht::FindRouterMessage;
auto& routers = m_state->m_PendingRouters;
if(routers.find(router) == routers.end())
{
@ -765,10 +780,20 @@ namespace llarp
routing::DHTMessage msg;
auto txid = GenTXID();
msg.M.emplace_back(
std::make_unique< dht::FindRouterMessage >(txid, router));
std::make_unique< FindRouterMessage >(txid, router));
if(path && path->SendRoutingMessage(msg, Router()))
{
RouterLookupJob job(this, handler);
assert(msg.M.size() == 1);
auto dhtMsg = dynamic_cast< FindRouterMessage* >(msg.M[0].get());
m_router->NotifyRouterEvent< tooling::FindRouterSentEvent >(
m_router->pubkey(),
dhtMsg);
routers.emplace(router, RouterLookupJob(this, handler));
return true;
}

@ -369,13 +369,13 @@ namespace llarp
const std::set< RouterID >&
SnodeBlacklist() const;
protected:
bool
SendToServiceOrQueue(const service::Address& addr,
const llarp_buffer_t& payload, ProtocolType t);
bool
SendToSNodeOrQueue(const RouterID& addr, const llarp_buffer_t& payload);
protected:
/// parent context that owns this endpoint
Context* const context;

@ -0,0 +1,38 @@
#include <simulation/sim_context.hpp>
#include <llarp.hpp>
namespace llarp
{
namespace simulate
{
Simulation::Simulation() : m_CryptoManager(new sodium::CryptoLibSodium())
{
}
void
Simulation::NodeUp(llarp::Context *)
{
}
Node_ptr
Simulation::AddNode(const std::string &name)
{
auto itr = m_Nodes.find(name);
if(itr == m_Nodes.end())
{
itr =
m_Nodes
.emplace(name,
std::make_shared< llarp::Context >(shared_from_this()))
.first;
}
return itr->second;
}
void
Simulation::DelNode(const std::string &name)
{
m_Nodes.erase(name);
}
} // namespace simulate
} // namespace llarp

@ -0,0 +1,34 @@
#pragma once
#include <crypto/crypto_libsodium.hpp>
#include <ev/ev.h>
namespace llarp
{
// forward declair
struct Context;
using Node_ptr = std::shared_ptr< llarp::Context >;
namespace simulate
{
struct Simulation : public std::enable_shared_from_this< Simulation >
{
Simulation();
llarp::CryptoManager m_CryptoManager;
llarp_ev_loop_ptr m_NetLoop;
std::unordered_map< std::string, Node_ptr > m_Nodes;
void
NodeUp(llarp::Context* node);
Node_ptr
AddNode(const std::string& name);
void
DelNode(const std::string& name);
};
using Sim_ptr = std::shared_ptr< Simulation >;
} // namespace simulate
} // namespace llarp

@ -0,0 +1,140 @@
#pragma once
#include "router_event.hpp"
#include "dht/key.hpp"
#include "service/intro_set.hpp"
#include "dht/messages/findrouter.hpp"
namespace tooling
{
struct PubIntroSentEvent : public RouterEvent
{
llarp::dht::Key_t introsetPubkey;
llarp::RouterID relay;
uint64_t relayIndex;
PubIntroSentEvent(
const llarp::RouterID& ourRouter,
const llarp::dht::Key_t& introsetPubkey_,
const llarp::RouterID& relay_,
uint64_t relayIndex_)
: RouterEvent("DHT: PubIntroSentEvent", ourRouter, false)
, introsetPubkey(introsetPubkey_)
, relay(relay_)
, relayIndex(relayIndex_)
{
}
std::string
ToString() const
{
return RouterEvent::ToString() + " ---- introset pubkey: "
+ introsetPubkey.ShortHex() + ", relay: " + relay.ShortString()
+ ", relayIndex: " + std::to_string(relayIndex);
}
};
struct PubIntroReceivedEvent : public RouterEvent
{
llarp::dht::Key_t from;
llarp::dht::Key_t location;
uint64_t txid;
uint64_t relayOrder;
PubIntroReceivedEvent(
const llarp::RouterID& ourRouter,
const llarp::dht::Key_t& from_,
const llarp::dht::Key_t& location_,
uint64_t txid_,
uint64_t relayOrder_)
: RouterEvent("DHT: PubIntroReceivedEvent", ourRouter, true)
, from(from_)
, location(location_)
, txid(txid_)
, relayOrder(relayOrder_)
{
}
std::string
ToString() const override
{
return RouterEvent::ToString() + "from " + from.ShortHex() + " location="
+ location.ShortHex() + " order=" + std::to_string(relayOrder)
+ " txid=" + std::to_string(txid);
}
};
struct GotIntroReceivedEvent : public RouterEvent
{
llarp::dht::Key_t From;
llarp::service::EncryptedIntroSet Introset;
uint64_t RelayOrder;
uint64_t TxID;
GotIntroReceivedEvent(
const llarp::RouterID& ourRouter_,
const llarp::dht::Key_t& from_,
const llarp::service::EncryptedIntroSet& introset_,
uint64_t txid_)
: RouterEvent("DHT:: GotIntroReceivedEvent", ourRouter_, true)
, From(from_)
, Introset(introset_)
, TxID(txid_)
{
}
std::string
ToString() const override
{
return RouterEvent::ToString() + "from " + From.ShortHex()
+ " location=" + Introset.derivedSigningKey.ShortHex() + " order="
+ std::to_string(RelayOrder) + " txid=" + std::to_string(TxID);
}
};
struct FindRouterEvent : public RouterEvent
{
llarp::dht::Key_t from;
llarp::RouterID targetKey;
bool iterative;
bool exploritory;
uint64_t txid;
uint64_t version;
FindRouterEvent(
const llarp::RouterID& ourRouter,
const llarp::dht::FindRouterMessage& msg)
: RouterEvent("DHT: FindRouterEvent", ourRouter, true)
, from(msg.From)
, targetKey(msg.targetKey)
, iterative(msg.iterative)
, exploritory(msg.exploritory)
, txid(msg.txid)
, version(msg.version)
{
}
std::string
ToString() const override
{
return RouterEvent::ToString()
+" from "+ from.ShortHex()
+", targetKey: "+ targetKey.ToString()
+", iterative: "+ std::to_string(iterative)
+", exploritory "+ std::to_string(exploritory)
+", txid "+ std::to_string(txid)
+", version "+ std::to_string(version);
}
};
struct FindRouterReceivedEvent : public FindRouterEvent
{
using FindRouterEvent::FindRouterEvent;
};
struct FindRouterSentEvent : public FindRouterEvent
{
using FindRouterEvent::FindRouterEvent;
};
} // namespace tooling

@ -0,0 +1,147 @@
#include "router_event.hpp"
#include <path/path_types.hpp>
#include <path/path.hpp>
#include <path/transit_hop.hpp>
namespace tooling
{
struct PathAttemptEvent : public RouterEvent
{
std::vector< llarp::path::PathHopConfig > hops;
llarp::PathID_t pathid;
PathAttemptEvent(
const llarp::RouterID& routerID,
std::shared_ptr< const llarp::path::Path > path)
: RouterEvent("PathAttemptEvent", routerID, false)
, hops(path->hops)
, pathid(path->hops[0].rxID)
{
}
std::string
ToString() const
{
std::string result = RouterEvent::ToString();
result += "---- [";
size_t i = 0;
for(const auto& hop : hops)
{
i++;
result += llarp::RouterID(hop.rc.pubkey).ShortString();
result += "]";
if(i != hops.size())
{
result += " -> [";
}
}
return result;
}
};
struct PathRequestReceivedEvent : public RouterEvent
{
llarp::RouterID prevHop;
llarp::RouterID nextHop;
llarp::PathID_t txid;
llarp::PathID_t rxid;
bool isEndpoint = false;
PathRequestReceivedEvent(
const llarp::RouterID& routerID,
std::shared_ptr< const llarp::path::TransitHop > hop)
: RouterEvent("PathRequestReceivedEvent", routerID, true)
, prevHop(hop->info.downstream)
, nextHop(hop->info.upstream)
, txid(hop->info.txID)
, rxid(hop->info.rxID)
, isEndpoint(routerID == nextHop ? true : false)
{
}
std::string
ToString() const
{
std::string result = RouterEvent::ToString();
result += "---- [";
result += prevHop.ShortString();
result += "] -> [*";
result += routerID.ShortString();
result += "] -> [";
if(isEndpoint)
{
result += "nowhere]";
}
else
{
result += nextHop.ShortString();
result += "]";
}
return result;
}
};
struct PathStatusReceivedEvent : public RouterEvent
{
llarp::PathID_t rxid;
uint64_t status;
PathStatusReceivedEvent(
const llarp::RouterID& routerID,
const llarp::PathID_t rxid_,
uint64_t status_)
: RouterEvent("PathStatusReceivedEvent", routerID, true)
, rxid(rxid_)
, status(status_)
{
}
std::string
ToString() const
{
std::string result = RouterEvent::ToString();
result += "---- path rxid: " + rxid.ShortHex();
result += ", status: " + std::to_string(status);
return result;
}
};
struct PathBuildRejectedEvent : public RouterEvent
{
llarp::PathID_t rxid;
llarp::RouterID rejectedBy;
PathBuildRejectedEvent(
const llarp::RouterID& routerID,
const llarp::PathID_t rxid_,
const llarp::RouterID& rejectedBy_)
: RouterEvent("PathBuildRejectedEvent", routerID, false)
, rxid(rxid_)
, rejectedBy(rejectedBy_)
{
}
std::string
ToString() const
{
std::string result = RouterEvent::ToString();
result += "---- path rxid: " + rxid.ShortHex();
result += ", rejectedBy: " + rejectedBy.ShortString();
return result;
}
};
} // namespace tooling

@ -0,0 +1,57 @@
#pragma once
#include <tooling/router_event.hpp>
#include <router_contact.hpp>
namespace tooling
{
struct RCGossipReceivedEvent : public RouterEvent
{
RCGossipReceivedEvent(const llarp::RouterID& routerID,
const llarp::RouterContact& rc_)
: RouterEvent("RCGossipReceivedEvent", routerID, true), rc(rc_)
{
}
std::string
ToString() const override
{
return RouterEvent::ToString()
+ " ---- other RouterID: " + llarp::RouterID(rc.pubkey).ShortString();
}
std::string
LongString() const
{
return RouterEvent::ToString() + " ---- RC: " + rc.ToString();
}
llarp::RouterContact rc;
};
struct RCGossipSentEvent : public RouterEvent
{
RCGossipSentEvent(const llarp::RouterID& routerID,
const llarp::RouterContact& rc_)
: RouterEvent("RCGossipSentEvent", routerID, true), rc(rc_)
{
}
std::string
ToString() const override
{
return RouterEvent::ToString() + " ---- sending RC for RouterID: "
+ llarp::RouterID(rc.pubkey).ShortString();
}
std::string
LongString() const
{
return RouterEvent::ToString() + " ---- RC: " + rc.ToString();
}
llarp::RouterContact rc;
};
} // namespace tooling

@ -0,0 +1,58 @@
#pragma once
#include <router_id.hpp>
#include <string>
#include <vector>
#include <memory>
namespace llarp
{
struct PathID_t;
namespace path
{
struct Path;
struct PathHopConfig;
struct TransitHop;
} // namespace path
} // namespace llarp
namespace tooling
{
struct RouterHive;
struct RouterEvent
{
RouterEvent(std::string eventType, llarp::RouterID routerID,
bool triggered)
: eventType(eventType), routerID(routerID), triggered(triggered)
{
}
virtual ~RouterEvent() = default;
virtual std::string
ToString() const
{
std::string result;
result += eventType;
result += " [";
result += routerID.ShortString();
result += "] -- ";
return result;
}
const std::string eventType;
llarp::RouterID routerID;
bool triggered = false;
};
using RouterEventPtr = std::unique_ptr< RouterEvent >;
} // namespace tooling

@ -0,0 +1,243 @@
#include <tooling/router_hive.hpp>
#include "llarp.h"
#include "llarp.hpp"
#include "util/thread/logic.hpp"
#include "router/abstractrouter.hpp"
#include <chrono>
#include <algorithm>
using namespace std::chrono_literals;
namespace tooling
{
void
RouterHive::AddRouter(const std::shared_ptr< llarp::Config >& config,
std::vector< llarp_main* >* routers)
{
llarp_main* ctx = llarp_main_init_from_config(config->Copy());
if(llarp_main_setup(ctx) == 0)
{
llarp::Context::Get(ctx)->InjectHive(this);
routers->push_back(ctx);
}
}
void
RouterHive::AddRelay(const std::shared_ptr< llarp::Config >& config)
{
AddRouter(config, &relays);
}
void
RouterHive::AddClient(const std::shared_ptr< llarp::Config >& config)
{
AddRouter(config, &clients);
}
void
RouterHive::StartRouters(std::vector< llarp_main* >* routers)
{
for(llarp_main* ctx : *routers)
{
routerMainThreads.emplace_back([ctx]() {
llarp_main_run(ctx, llarp_main_runtime_opts{false, false, false});
});
std::this_thread::sleep_for(2ms);
}
}
void
RouterHive::StartRelays()
{
StartRouters(&relays);
}
void
RouterHive::StartClients()
{
StartRouters(&clients);
}
void
RouterHive::StopRouters()
{
llarp::LogInfo("Signalling all routers to stop");
for(llarp_main* ctx : relays)
{
llarp_main_signal(ctx, 2 /* SIGINT */);
}
for(llarp_main* ctx : clients)
{
llarp_main_signal(ctx, 2 /* SIGINT */);
}
llarp::LogInfo("Waiting on routers to be stopped");
for(llarp_main* ctx : relays)
{
while(llarp_main_is_running(ctx))
{
std::this_thread::sleep_for(10ms);
}
}
for(llarp_main* ctx : clients)
{
while(llarp_main_is_running(ctx))
{
std::this_thread::sleep_for(10ms);
}
}
llarp::LogInfo("Joining all router threads");
for(auto& thread : routerMainThreads)
{
while(not thread.joinable())
{
std::this_thread::sleep_for(500ms);
}
thread.join();
}
llarp::LogInfo("RouterHive::StopRouters finished");
}
void
RouterHive::NotifyEvent(RouterEventPtr event)
{
std::lock_guard< std::mutex > guard{eventQueueMutex};
eventQueue.push_back(std::move(event));
}
RouterEventPtr
RouterHive::GetNextEvent()
{
std::lock_guard< std::mutex > guard{eventQueueMutex};
if(not eventQueue.empty())
{
auto ptr = std::move(eventQueue.front());
eventQueue.pop_front();
return ptr;
}
return nullptr;
}
std::deque< RouterEventPtr >
RouterHive::GetAllEvents()
{
std::lock_guard< std::mutex > guard{eventQueueMutex};
std::deque< RouterEventPtr > events;
if(not eventQueue.empty())
{
eventQueue.swap(events);
}
return events;
}
void
RouterHive::VisitRouter(llarp_main* router,
std::function< void(Context_ptr) > visit)
{
auto ctx = llarp::Context::Get(router);
LogicCall(ctx->logic, [visit, ctx]() { visit(ctx); });
}
void
RouterHive::VisitRelay(size_t index, std::function< void(Context_ptr) > visit)
{
if(index >= relays.size())
{
visit(nullptr);
return;
}
VisitRouter(relays[index], visit);
}
void
RouterHive::VisitClient(size_t index,
std::function< void(Context_ptr) > visit)
{
if(index >= clients.size())
{
visit(nullptr);
return;
}
VisitRouter(clients[index], visit);
}
std::vector< size_t >
RouterHive::RelayConnectedRelays()
{
std::vector< size_t > results;
results.resize(relays.size());
std::mutex results_lock;
size_t i = 0;
size_t done_count = 0;
for(auto relay : relays)
{
auto ctx = llarp::Context::Get(relay);
LogicCall(ctx->logic, [&, i, ctx]() {
size_t count = ctx->router->NumberOfConnectedRouters();
std::lock_guard< std::mutex > guard{results_lock};
results[i] = count;
done_count++;
});
i++;
}
while(true)
{
size_t read_done_count = 0;
{
std::lock_guard< std::mutex > guard{results_lock};
read_done_count = done_count;
}
if(read_done_count == relays.size())
break;
std::this_thread::sleep_for(100ms);
}
return results;
}
std::vector< llarp::RouterContact >
RouterHive::GetRelayRCs()
{
std::vector< llarp::RouterContact > results;
results.resize(relays.size());
std::mutex results_lock;
size_t i = 0;
size_t done_count = 0;
for(auto relay : relays)
{
auto ctx = llarp::Context::Get(relay);
LogicCall(ctx->logic, [&, i, ctx]() {
llarp::RouterContact rc = ctx->router->rc();
std::lock_guard< std::mutex > guard{results_lock};
results[i] = std::move(rc);
done_count++;
});
i++;
}
while(true)
{
size_t read_done_count = 0;
{
std::lock_guard< std::mutex > guard{results_lock};
read_done_count = done_count;
}
if(read_done_count == relays.size())
break;
std::this_thread::sleep_for(100ms);
}
return results;
}
} // namespace tooling

@ -0,0 +1,115 @@
#pragma once
#include <tooling/router_event.hpp>
#include <llarp.h>
#include <config/config.hpp>
#include <vector>
#include <deque>
#include <thread>
#include <mutex>
struct llarp_config;
struct llarp_main;
namespace llarp
{
struct Context;
}
namespace tooling
{
struct RouterHive
{
using Context_ptr = std::shared_ptr< llarp::Context >;
private:
void
StartRouters(std::vector< llarp_main * > *routers);
void
AddRouter(const std::shared_ptr< llarp::Config > &config,
std::vector< llarp_main * > *routers);
/// safely visit router
void
VisitRouter(llarp_main *router, std::function< void(Context_ptr) > visit);
/// safely visit relay at index N
void
VisitRelay(size_t index, std::function< void(Context_ptr) > visit);
/// safely visit client at index N
void
VisitClient(size_t index, std::function< void(Context_ptr) > visit);
public:
RouterHive() = default;
void
AddRelay(const std::shared_ptr< llarp::Config > &conf);
void
AddClient(const std::shared_ptr< llarp::Config > &conf);
void
StartRelays();
void
StartClients();
void
StopRouters();
void
NotifyEvent(RouterEventPtr event);
RouterEventPtr
GetNextEvent();
std::deque< RouterEventPtr >
GetAllEvents();
void
ForEachRelay(std::function< void(Context_ptr) > visit)
{
for(size_t idx = 0; idx < relays.size(); ++idx)
{
VisitRelay(idx, visit);
}
}
void
ForEachClient(std::function< void(Context_ptr) > visit)
{
for(size_t idx = 0; idx < clients.size(); ++idx)
{
VisitClient(idx, visit);
}
}
/// safely visit every router context
void
ForEachRouter(std::function< void(Context_ptr) > visit)
{
ForEachRelay(visit);
ForEachClient(visit);
}
std::vector< size_t >
RelayConnectedRelays();
std::vector< llarp::RouterContact >
GetRelayRCs();
std::vector< llarp_main * > relays;
std::vector< llarp_main * > clients;
std::vector< std::thread > routerMainThreads;
std::mutex eventQueueMutex;
std::deque< RouterEventPtr > eventQueue;
};
} // namespace tooling

@ -259,6 +259,12 @@ namespace llarp
return std::string(HexEncode(*this, strbuf));
}
std::string
ShortHex() const
{
return ToHex().substr(0, 8);
}
std::ostream&
print(std::ostream& stream, int level, int spaces) const
{

@ -107,14 +107,21 @@ namespace llarp
/** internal */
template < typename... TArgs >
inline static void
#ifndef LOKINET_HIVE
_Log(LogLevel lvl, const char* fname, int lineno, TArgs&&... args) noexcept
#else
_Log(LogLevel, const char*, int, TArgs&&...) noexcept
#endif
{
/* nop out logging for hive mode for now */
#ifndef LOKINET_HIVE
auto& log = LogContext::Instance();
if(log.curLevel > lvl)
return;
std::stringstream ss;
LogAppend(ss, std::forward< TArgs >(args)...);
log.logStream->AppendLog(lvl, fname, lineno, log.nodeName, ss.str());
#endif
}
/*
std::stringstream ss;

@ -16,7 +16,7 @@ namespace llarp
PreLog(std::stringstream& s, LogLevel lvl, const char* fname, int lineno,
const std::string& nodename) const override;
void
virtual void
Print(LogLevel lvl, const char* tag, const std::string& msg) override;
void

@ -73,6 +73,11 @@ namespace llarp
Type
popFront();
// Remove an element from the queue. Block until an element is available
// or until <timeout> microseconds have elapsed
nonstd::optional< Type >
popFrontWithTimeout(std::chrono::microseconds timeout);
nonstd::optional< Type >
tryPopFront();
@ -385,6 +390,42 @@ namespace llarp
return Type(std::move(m_data[index]));
}
template < typename Type >
nonstd::optional< Type >
Queue< Type >::popFrontWithTimeout(std::chrono::microseconds timeout)
{
uint32_t generation = 0;
uint32_t index = 0;
bool secondTry = false;
bool success = false;
for(;;)
{
success = m_manager.reservePopIndex(generation, index)
== QueueReturn::Success;
if(secondTry || success)
break;
m_waitingPoppers.fetch_add(1, std::memory_order_relaxed);
if(empty())
{
m_popSemaphore.waitFor(timeout);
secondTry = true;
}
m_waitingPoppers.fetch_sub(1, std::memory_order_relaxed);
}
if(success)
{
QueuePopGuard< Type > popGuard(*this, generation, index);
return Type(std::move(m_data[index]));
}
return {};
}
template < typename Type >
void
Queue< Type >::removeAll()

@ -0,0 +1,20 @@
set(LLARP_PYBIND_SRC
module.cpp
llarp/context.cpp
llarp/router_id.cpp
llarp/router_contact.cpp
llarp/crypto/types.cpp
llarp/config.cpp
llarp/dht/dht_types.cpp
llarp/path/path_types.cpp
llarp/path/path_hop_config.cpp
llarp/handlers/pyhandler.cpp
llarp/tooling/router_hive.cpp
llarp/tooling/router_event.cpp
llarp/service/address.cpp
)
pybind11_add_module(pyllarp MODULE ${LLARP_PYBIND_SRC})
target_include_directories(pyllarp PRIVATE ${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/crypto/libntrup/include ${CMAKE_SOURCE_DIR}/llarp ${CMAKE_CURRENT_SOURCE_DIR})
target_link_libraries(pyllarp PUBLIC ${EXE_LIBS})

@ -0,0 +1,93 @@
#pragma once
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/functional.h>
#include <nonstd/optional.hpp>
#include <unordered_map>
namespace py = pybind11;
namespace pybind11
{
namespace detail
{
template < typename Key, typename Value, typename Hash, typename Equal,
typename Alloc >
struct type_caster<
std::unordered_multimap< Key, Value, Hash, Equal, Alloc > >
: map_caster< std::unordered_multimap< Key, Value, Hash, Equal, Alloc >,
Key, Value >
{
};
template < typename T >
struct type_caster< nonstd::optional< T > >
: public optional_caster< nonstd::optional< T > >
{
};
/*
template <typename CharT, class Traits>
struct type_caster<simple_string_view>
: string_caster<simple_string_view, true> {};
*/
} // namespace detail
} // namespace pybind11
namespace llarp
{
void
Context_Init(py::module &mod);
void
CryptoTypes_Init(py::module &mod);
void
RouterID_Init(py::module &mod);
void
RouterContact_Init(py::module &mod);
void
Config_Init(py::module &mod);
void
PathTypes_Init(py::module &mod);
namespace dht
{
void
DHTTypes_Init(py::module &mod);
}
namespace path
{
void
PathHopConfig_Init(py::module &mod);
}
namespace handlers
{
void
PyHandler_Init(py::module &mod);
}
namespace service
{
void
Address_Init(py::module &mod);
}
} // namespace llarp
namespace tooling
{
void
RouterHive_Init(py::module &mod);
void
RouterEvent_Init(py::module &mod);
} // namespace tooling

@ -0,0 +1,122 @@
#include "common.hpp"
#include "config/config.hpp"
#include <netinet/in.h>
namespace llarp
{
void
in_addr_set(in_addr *addr, const char *str)
{
inet_aton(str, addr);
}
void
Config_Init(py::module &mod)
{
using Config_ptr = std::shared_ptr< Config >;
py::class_< Config, Config_ptr >(mod, "Config")
.def(py::init<>())
.def_readwrite("router", &Config::router)
.def_readwrite("network", &Config::network)
.def_readwrite("connect", &Config::connect)
.def_readwrite("netdb", &Config::netdb)
.def_readwrite("dns", &Config::dns)
.def_readwrite("links", &Config::links)
.def_readwrite("services", &Config::services)
.def_readwrite("system", &Config::system)
.def_readwrite("api", &Config::api)
.def_readwrite("lokid", &Config::lokid)
.def_readwrite("bootstrap", &Config::bootstrap)
.def_readwrite("logging", &Config::logging)
.def("LoadFile", &Config::Load);
py::class_< RouterConfig >(mod, "RouterConfig")
.def(py::init<>())
.def_readwrite("minConnectedRouters",
&RouterConfig::m_minConnectedRouters)
.def_readwrite("maxConnectedRouters",
&RouterConfig::m_maxConnectedRouters)
.def_readwrite("netid", &RouterConfig::m_netId)
.def_readwrite("nickname", &RouterConfig::m_nickname)
.def_readwrite("encryptionKeyfile", &RouterConfig::m_encryptionKeyfile)
.def_readwrite("ourRcFile", &RouterConfig::m_ourRcFile)
.def_readwrite("transportKeyfile", &RouterConfig::m_transportKeyfile)
.def_readwrite("identKeyfile", &RouterConfig::m_identKeyfile)
.def_readwrite("blockBogons", &RouterConfig::m_blockBogons)
.def_readwrite("publicOverride", &RouterConfig::m_publicOverride)
.def_readwrite("ip4addr", &RouterConfig::m_ip4addr)
.def("overrideAddress",
[](RouterConfig &self, std::string ip, std::string port) {
self.fromSection("public-ip", ip);
self.fromSection("public-port", port);
})
.def_readwrite("workerThreads", &RouterConfig::m_workerThreads)
.def_readwrite("numNetThreads", &RouterConfig::m_numNetThreads)
.def_readwrite("JobQueueSize", &RouterConfig::m_JobQueueSize)
.def_readwrite("DefaultLinkProto", &RouterConfig::m_DefaultLinkProto);
py::class_< NetworkConfig >(mod, "NetworkConfig")
.def(py::init<>())
.def_readwrite("enableProfiling", &NetworkConfig::m_enableProfiling)
.def_readwrite("routerProfilesFile",
&NetworkConfig::m_routerProfilesFile)
.def_readwrite("strictConnect", &NetworkConfig::m_strictConnect)
.def_readwrite("netConfig", &NetworkConfig::m_netConfig);
py::class_< ConnectConfig >(mod, "ConnectConfig")
.def(py::init<>())
.def_readwrite("routers", &ConnectConfig::routers);
py::class_< NetdbConfig >(mod, "NetdbConfig")
.def(py::init<>())
.def_readwrite("nodedbDir", &NetdbConfig::m_nodedbDir);
py::class_< DnsConfig >(mod, "DnsConfig")
.def(py::init<>())
.def_readwrite("netConfig", &DnsConfig::netConfig);
py::class_< LinksConfig >(mod, "LinksConfig")
.def(py::init<>())
.def_readwrite("OutboundLink", &LinksConfig::m_OutboundLink)
.def_readwrite("InboundLinks", &LinksConfig::m_InboundLinks);
py::class_< ServicesConfig >(mod, "ServicesConfig")
.def(py::init<>())
.def_readwrite("services", &ServicesConfig::services);
py::class_< SystemConfig >(mod, "SystemConfig")
.def(py::init<>())
.def_readwrite("pidfile", &SystemConfig::pidfile);
py::class_< ApiConfig >(mod, "ApiConfig")
.def(py::init<>())
.def_readwrite("enableRPCServer", &ApiConfig::m_enableRPCServer)
.def_readwrite("rpcBindAddr", &ApiConfig::m_rpcBindAddr);
py::class_< LokidConfig >(mod, "LokidConfig")
.def(py::init<>())
.def_readwrite("usingSNSeed", &LokidConfig::usingSNSeed)
.def_readwrite("whitelistRouters", &LokidConfig::whitelistRouters)
.def_readwrite("ident_keyfile", &LokidConfig::ident_keyfile)
.def_readwrite("lokidRPCAddr", &LokidConfig::lokidRPCAddr)
.def_readwrite("lokidRPCUser", &LokidConfig::lokidRPCUser)
.def_readwrite("lokidRPCPassword", &LokidConfig::lokidRPCPassword);
py::class_< BootstrapConfig >(mod, "BootstrapConfig")
.def(py::init<>())
.def_readwrite("routers", &BootstrapConfig::routers);
py::class_< LoggingConfig >(mod, "LoggingConfig")
.def(py::init<>())
.def_readwrite("LogJSON", &LoggingConfig::m_LogJSON)
.def_readwrite("LogFile", &LoggingConfig::m_LogFile);
py::class_< sockaddr_in >(mod, "sockaddr_in")
.def_readwrite("sin_family", &sockaddr_in::sin_family)
.def_readwrite("sin_port", &sockaddr_in::sin_port)
.def_readwrite("sin_addr", &sockaddr_in::sin_addr);
py::class_< in_addr >(mod, "in_addr").def("set", &in_addr_set);
}
} // namespace llarp

@ -0,0 +1,45 @@
#include "common.hpp"
#include <llarp.hpp>
#include <router/router.cpp>
#include "llarp/handlers/pyhandler.hpp"
namespace llarp
{
void
Context_Init(py::module& mod)
{
using Context_ptr = std::shared_ptr< Context >;
py::class_< Context, Context_ptr >(mod, "Context")
.def("Setup",
[](Context_ptr self) -> bool { return self->Setup() == 0; })
.def("Run",
[](Context_ptr self) -> int {
return self->Run(llarp_main_runtime_opts{});
})
.def("Stop", [](Context_ptr self) { self->CloseAsync(); })
.def("IsUp", &Context::IsUp)
.def("IsRelay",
[](Context_ptr self) -> bool {
return self->router->IsServiceNode();
})
.def("LooksAlive", &Context::LooksAlive)
.def("Configure", &Context::Configure)
.def("TrySendPacket",
[](Context_ptr self, std::string from, service::Address to,
std::string pkt) {
auto ep =
self->router->hiddenServiceContext().GetEndpointByName(from);
std::vector< byte_t > buf;
buf.resize(pkt.size());
std::copy_n(pkt.c_str(), pkt.size(), buf.data());
return ep
and ep->SendToServiceOrQueue(to, std::move(buf),
service::eProtocolControl);
})
.def("AddEndpoint",
[](Context_ptr self, handlers::PythonEndpoint_ptr ep) {
self->router->hiddenServiceContext().InjectEndpoint(ep->OurName,
ep);
})
.def("CallSafe", &Context::CallSafe);
}
} // namespace llarp

@ -0,0 +1,27 @@
#include <crypto/types.hpp>
#include "common.hpp"
namespace llarp
{
void
CryptoTypes_Init(py::module& mod)
{
py::class_< PubKey >(mod, "PubKey")
.def(py::init<>())
.def("FromHex", &PubKey::FromString)
.def("__repr__", &PubKey::ToString);
py::class_< SecretKey >(mod, "SecretKey")
.def(py::init<>())
.def("LoadFile", &SecretKey::LoadFromFile)
.def("SaveFile", &SecretKey::SaveToFile)
.def("ToPublic", &SecretKey::toPublic);
py::class_< Signature >(mod, "Signature")
.def(py::init<>())
.def("__repr__", [](const Signature sig) -> std::string {
std::stringstream ss;
ss << sig;
return ss.str();
});
}
} // namespace llarp

@ -0,0 +1,27 @@
#include <dht/key.hpp>
#include "common.hpp"
#include <pybind11/operators.h>
namespace llarp
{
namespace dht
{
void
DHTTypes_Init(py::module& mod)
{
py::class_< Key_t >(mod, "DHTKey")
.def(py::self == py::self)
.def(py::self < py::self)
.def(py::self ^ py::self)
.def("distance",
[](const Key_t* const lhs, const Key_t* const rhs) {
return *lhs ^ *rhs;
})
.def("ShortString", [](const Key_t* const key) {
return llarp::RouterID(key->as_array()).ShortString();
});
}
} // namespace dht
} // namespace llarp

@ -0,0 +1,18 @@
#include "llarp/handlers/pyhandler.hpp"
namespace llarp
{
namespace handlers
{
void
PyHandler_Init(py::module& mod)
{
py::class_< PythonEndpoint, PythonEndpoint_ptr >(mod, "Endpoint")
.def(py::init< std::string, Context_ptr >())
.def("SendTo", &PythonEndpoint::SendPacket)
.def("OurAddress", &PythonEndpoint::GetOurAddress)
.def_readwrite("GotPacket", &PythonEndpoint::handlePacket);
}
} // namespace handlers
} // namespace llarp

@ -0,0 +1,85 @@
#pragma once
#include "common.hpp"
#include "llarp.hpp"
#include "service/context.hpp"
#include "service/endpoint.hpp"
#include "router/abstractrouter.hpp"
namespace llarp
{
namespace handlers
{
using Context_ptr = std::shared_ptr< llarp::Context >;
struct PythonEndpoint final
: public llarp::service::Endpoint,
public std::enable_shared_from_this< PythonEndpoint >
{
PythonEndpoint(std::string name, Context_ptr routerContext)
: llarp::service::Endpoint(
name, routerContext->router.get(),
&routerContext->router->hiddenServiceContext())
, OurName(name)
{
}
const std::string OurName;
bool
HandleInboundPacket(const service::ConvoTag tag,
const llarp_buffer_t& pktbuf,
service::ProtocolType proto) override
{
if(handlePacket)
{
AlignedBuffer< 32 > addr;
bool isSnode = false;
if(not GetEndpointWithConvoTag(tag, addr, isSnode))
return false;
if(isSnode)
return true;
std::vector< byte_t > pkt;
pkt.resize(pktbuf.sz);
std::copy_n(pktbuf.base, pktbuf.sz, pkt.data());
handlePacket(service::Address(addr), std::move(pkt), proto);
}
return true;
}
path::PathSet_ptr
GetSelf() override
{
return shared_from_this();
}
bool
SupportsV6() const override
{
return false;
}
using PacketHandler_t = std::function< void(
service::Address, std::vector< byte_t >, service::ProtocolType) >;
PacketHandler_t handlePacket;
void
SendPacket(service::Address remote, std::vector< byte_t > pkt,
service::ProtocolType proto)
{
LogicCall(m_router->logic(),
[remote, pkt, proto, self = shared_from_this()]() {
self->SendToServiceOrQueue(remote, llarp_buffer_t(pkt),
proto);
});
}
std::string
GetOurAddress() const
{
return m_Identity.pub.Addr().ToString();
}
};
using PythonEndpoint_ptr = std::shared_ptr< PythonEndpoint >;
} // namespace handlers
} // namespace llarp

@ -0,0 +1,29 @@
#include <path/path.hpp>
#include "common.hpp"
namespace llarp
{
namespace path
{
void
PathHopConfig_Init(py::module &mod)
{
auto str_func = [](PathHopConfig *hop) {
std::string s = "Hop: [";
s += RouterID(hop->rc.pubkey).ShortString();
s += "] -> [";
s += hop->upstream.ShortString();
s += "]";
return s;
};
py::class_< PathHopConfig >(mod, "PathHopConfig")
.def_readonly("rc", &PathHopConfig::rc)
.def_readonly("upstreamRouter", &PathHopConfig::upstream)
.def_readonly("txid", &PathHopConfig::txID)
.def_readonly("rxid", &PathHopConfig::rxID)
.def("ToString", str_func)
.def("__str__", str_func)
.def("__repr__", str_func);
}
} // namespace path
} // namespace llarp

@ -0,0 +1,18 @@
#include <path/path.hpp>
#include "common.hpp"
#include <pybind11/operators.h>
namespace llarp
{
void
PathTypes_Init(py::module& mod)
{
py::class_< PathID_t >(mod, "PathID")
.def(py::self == py::self)
.def("ShortHex", &PathID_t::ShortHex)
.def("__str__", &PathID_t::ShortHex)
.def("__repr__", &PathID_t::ShortHex);
}
} // namespace llarp

@ -0,0 +1,32 @@
#include <router_contact.hpp>
#include <dht/key.hpp>
#include "common.hpp"
namespace llarp
{
void
RouterContact_Init(py::module& mod)
{
py::class_< RouterContact >(mod, "RouterContact")
.def(py::init<>())
.def_property_readonly(
"routerID",
[](const RouterContact* const rc) -> llarp::RouterID {
return llarp::RouterID(rc->pubkey);
})
.def_property_readonly(
"AsDHTKey",
[](const RouterContact* const rc) -> llarp::dht::Key_t {
return llarp::dht::Key_t{rc->pubkey.as_array()};
})
.def("ReadFile", &RouterContact::Read)
.def("WriteFile", &RouterContact::Write)
.def("ToString", &RouterContact::ToString)
.def("__str__", &RouterContact::ToString)
.def("__repr__", &RouterContact::ToString)
.def("Verify", [](const RouterContact* const rc) -> bool {
const llarp_time_t now = llarp::time_now_ms();
return rc->Verify(now);
});
}
} // namespace llarp

@ -0,0 +1,23 @@
#include "common.hpp"
#include "router_id.hpp"
namespace llarp
{
void
RouterID_Init(py::module& mod)
{
py::class_< RouterID >(mod, "RouterID")
.def("FromHex",
[](RouterID* r, const std::string& hex) -> bool {
return HexDecode(hex.c_str(), r->data(), r->size());
})
.def("__repr__", &RouterID::ToString)
.def("__str__", &RouterID::ToString)
.def("ShortString", &RouterID::ShortString)
.def("__eq__",
[](const RouterID* const lhs, const RouterID* const rhs) {
return *lhs == *rhs;
});
}
} // namespace llarp

@ -0,0 +1,18 @@
#include "common.hpp"
#include "service/address.hpp"
namespace llarp
{
namespace service
{
void
Address_Init(py::module& mod)
{
py::class_< Address >(mod, "ServiceAddress")
.def(py::init< std::string >())
.def("__str__", [](const Address& addr) -> std::string {
return addr.ToString();
});
}
} // namespace service
} // namespace llarp

@ -0,0 +1,87 @@
#include "common.hpp"
#include "pybind11/stl.h"
#include "tooling/router_event.hpp"
#include "tooling/dht_event.hpp"
#include "tooling/path_event.hpp"
#include "tooling/rc_event.hpp"
#include <messages/relay_status.hpp>
#include <path/path.hpp>
namespace tooling
{
void
RouterEvent_Init(py::module& mod)
{
py::class_< RouterEvent >(mod, "RouterEvent")
.def("__repr__", &RouterEvent::ToString)
.def("__str__", &RouterEvent::ToString)
.def_readonly("routerID", &RouterEvent::routerID)
.def_readonly("triggered", &RouterEvent::triggered);
py::class_< PathAttemptEvent, RouterEvent >(mod, "PathAttemptEvent")
.def_readonly("hops", &PathAttemptEvent::hops);
py::class_< PathRequestReceivedEvent, RouterEvent >(
mod, "PathRequestReceivedEvent")
.def_readonly("prevHop", &PathRequestReceivedEvent::prevHop)
.def_readonly("nextHop", &PathRequestReceivedEvent::nextHop)
.def_readonly("txid", &PathRequestReceivedEvent::txid)
.def_readonly("rxid", &PathRequestReceivedEvent::rxid)
.def_readonly("isEndpoint", &PathRequestReceivedEvent::isEndpoint);
py::class_< PathStatusReceivedEvent, RouterEvent >(
mod, "PathStatusReceivedEvent")
.def_readonly("rxid", &PathStatusReceivedEvent::rxid)
.def_readonly("status", &PathStatusReceivedEvent::rxid)
.def_property_readonly(
"Successful", [](const PathStatusReceivedEvent* const ev) {
return ev->status == llarp::LR_StatusRecord::SUCCESS;
});
py::class_< PubIntroSentEvent, RouterEvent >(mod, "DhtPubIntroSentEvent")
.def_readonly("introsetPubkey", &PubIntroSentEvent::introsetPubkey)
.def_readonly("relay", &PubIntroSentEvent::relay)
.def_readonly("relayIndex", &PubIntroSentEvent::relayIndex);
py::class_< PubIntroReceivedEvent, RouterEvent >(mod,
"DhtPubIntroReceivedEvent")
.def_readonly("from", &PubIntroReceivedEvent::from)
.def_readonly("location", &PubIntroReceivedEvent::location)
.def_readonly("relayOrder", &PubIntroReceivedEvent::relayOrder)
.def_readonly("txid", &PubIntroReceivedEvent::txid);
py::class_< GotIntroReceivedEvent, RouterEvent >(mod,
"DhtGotIntroReceivedEvent")
.def_readonly("from", &GotIntroReceivedEvent::From)
.def_readonly("location", &GotIntroReceivedEvent::Introset)
.def_readonly("relayOrder", &GotIntroReceivedEvent::RelayOrder)
.def_readonly("txid", &GotIntroReceivedEvent::TxID);
py::class_< RCGossipReceivedEvent, RouterEvent >(mod,
"RCGossipReceivedEvent")
.def_readonly("rc", &RCGossipReceivedEvent::rc)
.def("LongString", &RCGossipReceivedEvent::LongString);
py::class_< RCGossipSentEvent, RouterEvent >(mod, "RCGossipSentEvent")
.def_readonly("rc", &RCGossipSentEvent::rc)
.def("LongString", &RCGossipSentEvent::LongString);
py::class_< FindRouterEvent, RouterEvent >(mod, "FindRouterEvent")
.def_readonly("from", &FindRouterEvent::from)
.def_readonly("iterative", &FindRouterEvent::iterative)
.def_readonly("exploritory", &FindRouterEvent::exploritory)
.def_readonly("txid", &FindRouterEvent::txid)
.def_readonly("version", &FindRouterEvent::version);
py::class_< FindRouterReceivedEvent,
FindRouterEvent,
RouterEvent >(mod, "FindRouterReceivedEvent");
py::class_< FindRouterSentEvent,
FindRouterEvent,
RouterEvent >(mod, "FindRouterSentEvent");
}
} // namespace tooling

@ -0,0 +1,28 @@
#include "common.hpp"
#include "pybind11/stl.h"
#include "pybind11/iostream.h"
#include <tooling/router_hive.hpp>
#include "llarp.hpp"
namespace tooling
{
void
RouterHive_Init(py::module& mod)
{
using RouterHive_ptr = std::shared_ptr< RouterHive >;
py::class_< RouterHive, RouterHive_ptr >(mod, "RouterHive")
.def(py::init<>())
.def("AddRelay", &RouterHive::AddRelay)
.def("AddClient", &RouterHive::AddClient)
.def("StartRelays", &RouterHive::StartRelays)
.def("StartClients", &RouterHive::StartClients)
.def("StopAll", &RouterHive::StopRouters)
.def("ForEachRelay", &RouterHive::ForEachRelay)
.def("ForEachClient", &RouterHive::ForEachClient)
.def("ForEachRouter", &RouterHive::ForEachRouter)
.def("GetNextEvent", &RouterHive::GetNextEvent)
.def("GetAllEvents", &RouterHive::GetAllEvents)
.def("RelayConnectedRelays", &RouterHive::RelayConnectedRelays)
.def("GetRelayRCs", &RouterHive::GetRelayRCs);
}
} // namespace tooling

@ -0,0 +1,19 @@
#include "common.hpp"
#include "util/logging/logger.hpp"
PYBIND11_MODULE(pyllarp, m)
{
tooling::RouterHive_Init(m);
tooling::RouterEvent_Init(m);
llarp::RouterID_Init(m);
llarp::RouterContact_Init(m);
llarp::CryptoTypes_Init(m);
llarp::Context_Init(m);
llarp::Config_Init(m);
llarp::dht::DHTTypes_Init(m);
llarp::PathTypes_Init(m);
llarp::path::PathHopConfig_Init(m);
llarp::handlers::PyHandler_Init(m);
llarp::service::Address_Init(m);
m.def("EnableDebug", []() { llarp::SetLogLevel(llarp::eLogDebug); });
}

@ -1,3 +1,13 @@
if (WITH_HIVE)
add_custom_target(hive_build DEPENDS ${STATIC_LIB} pyllarp)
add_custom_target(hive ${CMAKE_COMMAND} -E
env PYTHONPATH="$ENV{PYTHONPATH}:${CMAKE_BINARY_DIR}/pybind"
${PYTHON_EXECUTABLE} -m pytest
${CMAKE_CURRENT_SOURCE_DIR}/hive
DEPENDS
hive_build)
endif()
set(GTEST_EXE testAll)
set(CATCH_EXE catchAll)

@ -0,0 +1,35 @@
#!/usr/bin/env python3
import hive
import pytest
@pytest.fixture(scope="session")
def HiveTenRTenC():
router_hive = hive.RouterHive(n_relays=10, n_clients=10, netid="hive")
router_hive.Start()
yield router_hive
router_hive.Stop()
@pytest.fixture(scope="session")
def HiveThirtyRTenC():
router_hive = hive.RouterHive(n_relays=30, n_clients=10, netid="hive")
router_hive.Start()
yield router_hive
router_hive.Stop()
@pytest.fixture()
def HiveArbitrary():
router_hive = None
def _make(n_relays=10, n_clients=10, netid="hive"):
nonlocal router_hive
router_hive = hive.RouterHive(n_relays=30, n_clients=10, netid="hive")
router_hive.Start()
return router_hive
yield _make
router_hive.Stop()

@ -0,0 +1,283 @@
#!/usr/bin/env python3
import pyllarp
from time import sleep
from signal import signal, SIGINT
from shutil import rmtree
from os import makedirs
from socket import AF_INET, htons, inet_aton
from pprint import pprint
import sys
from argparse import ArgumentParser as ap
import threading
from collections import deque
class RouterHive(object):
def __init__(self, n_relays=10, n_clients=10, netid="hive"):
self.endpointName = "pyllarp"
self.tmpdir = "/tmp/lokinet_hive"
self.netid = netid
self.n_relays = n_relays
self.n_clients = n_clients
self.addrs = []
self.events = deque()
self.hive = None
self.RCs = []
pyllarp.EnableDebug()
if not self.RemoveTmpDir():
raise RuntimeError("Failed to initialize Router Hive")
def RemoveTmpDir(self):
if self.tmpdir.startswith("/tmp/") and len(self.tmpdir) > 5:
print("calling rmdir -r %s" % self.tmpdir)
rmtree(self.tmpdir, ignore_errors=True)
return True
else:
print("not removing dir %s because it doesn't start with /tmp/" % self.tmpdir)
return False
def MakeEndpoint(self, router, after):
if router.IsRelay():
return
ep = pyllarp.Endpoint(self.endpointName, router)
router.AddEndpoint(ep)
if after is not None:
router.CallSafe(lambda : after(ep))
def AddRelay(self, index):
dirname = "%s/relays/%d" % (self.tmpdir, index)
makedirs("%s/netdb" % dirname, exist_ok=True)
config = pyllarp.Config()
port = index + 30000
tunname = "lokihive%d" % index
config.router.encryptionKeyfile = "%s/encryption.key" % dirname
config.router.transportKeyfile = "%s/transport.key" % dirname
config.router.identKeyfile = "%s/identity.key" % dirname
config.router.ourRcFile = "%s/rc.signed" % dirname
config.router.netid = self.netid
config.router.nickname = "Router%d" % index
config.router.publicOverride = True
config.router.overrideAddress("127.0.0.1", '{}'.format(port))
"""
config.router.ip4addr.sin_family = AF_INET
config.router.ip4addr.sin_port = htons(port)
config.router.ip4addr.sin_addr.set("127.0.0.1")
"""
config.router.blockBogons = False
config.network.enableProfiling = False
config.network.routerProfilesFile = "%s/profiles.dat" % dirname
config.network.netConfig = {"type": "null"}
config.netdb.nodedbDir = "%s/netdb" % dirname
config.links.InboundLinks = [("lo", AF_INET, port, set())]
config.system.pidfile = "%s/lokinet.pid" % dirname
config.dns.netConfig = {"local-dns": ("127.3.2.1:%d" % port)}
if index != 1:
config.bootstrap.routers = ["%s/relays/1/rc.signed" % self.tmpdir]
self.hive.AddRelay(config)
def AddClient(self, index):
dirname = "%s/clients/%d" % (self.tmpdir, index)
makedirs("%s/netdb" % dirname, exist_ok=True)
config = pyllarp.Config()
port = index + 40000
tunname = "lokihive%d" % index
config.router.encryptionKeyfile = "%s/encryption.key" % dirname
config.router.transportKeyfile = "%s/transport.key" % dirname
config.router.identKeyfile = "%s/identity.key" % dirname
config.router.ourRcFile = "%s/rc.signed" % dirname
config.router.netid = self.netid
config.router.blockBogons = False
config.network.enableProfiling = False
config.network.routerProfilesFile = "%s/profiles.dat" % dirname
config.network.netConfig = {"type": "null"}
config.netdb.nodedbDir = "%s/netdb" % dirname
config.system.pidfile = "%s/lokinet.pid" % dirname
config.dns.netConfig = {"local-dns": ("127.3.2.1:%d" % port)}
config.bootstrap.routers = ["%s/relays/1/rc.signed" % self.tmpdir]
self.hive.AddClient(config)
def onGotEndpoint(self, ep):
addr = ep.OurAddress()
self.addrs.append(pyllarp.ServiceAddress(addr))
def sendToAddress(self, router, toaddr, pkt):
if router.IsRelay():
return
if router.TrySendPacket("default", toaddr, pkt):
print("sending {} bytes to {}".format(len(pkt), toaddr))
def broadcastTo(self, addr, pkt):
self.hive.ForEachRouter(lambda r : sendToAddress(r, addr, pkt))
def InitFirstRC(self):
print("Starting first router to init its RC for bootstrap")
self.hive = pyllarp.RouterHive()
self.AddRelay(1)
self.hive.StartRelays()
print("sleeping 2 sec to give plenty of time to save bootstrap rc")
sleep(2)
self.hive.StopAll()
def Start(self):
self.InitFirstRC()
print("Resetting hive. Creating %d relays and %d clients" % (self.n_relays, self.n_clients))
self.hive = pyllarp.RouterHive()
for i in range(1, self.n_relays + 1):
self.AddRelay(i)
for i in range(1, self.n_clients + 1):
self.AddClient(i)
print("Starting relays")
self.hive.StartRelays()
sleep(0.2)
self.hive.ForEachRelay(lambda r: self.MakeEndpoint(r, self.onGotEndpoint))
print("Sleeping 2 seconds before starting clients")
sleep(2)
self.RCs = self.hive.GetRelayRCs()
self.hive.StartClients()
sleep(0.2)
self.hive.ForEachClient(lambda r: self.MakeEndpoint(r, self.onGotEndpoint))
def Stop(self):
self.hive.StopAll()
def CollectNextEvent(self):
self.events.append(self.hive.GetNextEvent())
def CollectAllEvents(self):
self.events.extend(self.hive.GetAllEvents())
def PopEvent(self):
self.CollectAllEvents()
if len(self.events):
return self.events.popleft()
return None
def DistanceSortedRCs(self, dht_key):
rcs = []
distances = []
for rc in self.RCs:
distances.append(rc.AsDHTKey ^ dht_key)
rcs.append(rc)
distances, rcs = (list(t) for t in zip(*sorted(zip(distances, rcs))))
return rcs
def main(n_relays=10, n_clients=10, print_each_event=True):
running = True
def handle_sigint(sig, frame):
nonlocal running
running = False
print("SIGINT received, attempting to stop all routers")
signal(SIGINT, handle_sigint)
try:
hive = RouterHive(n_relays, n_clients)
hive.Start()
except Exception as err:
print(err)
return 1
first_dht_pub = False
dht_pub_sorted = None
dht_pub_location = None
total_events = 0
event_counts = dict()
while running:
hive.CollectAllEvents()
print("Hive collected {} events this second.".format(len(hive.events)))
for event in hive.events:
event_name = event.__class__.__name__
if event:
if print_each_event:
print("Event: %s -- Triggered: %s" % (event_name, event.triggered))
print(event)
hops = getattr(event, "hops", None)
if hops:
for hop in hops:
print(hop)
total_events = total_events + 1
if event_name in event_counts:
event_counts[event_name] = event_counts[event_name] + 1
else:
event_counts[event_name] = 1
if total_events % 10 == 0:
pprint(event_counts)
if event_name == "DhtPubIntroReceivedEvent":
if not first_dht_pub:
dht_pub_sorted = hive.DistanceSortedRCs(event.location)
dht_pub_location = event.location
print("location: {} landed at: {}, sorted distance list:".format(dht_pub_location.ShortString(), event.routerID.ShortString()))
print([x.routerID.ShortString() for x in dht_pub_sorted[:4]])
first_dht_pub = True
else:
if event.location == dht_pub_location:
print("location: {}, landed at: {}".format(dht_pub_location.ShortString(), event.routerID.ShortString()))
# won't have printed event count above in this case.
if len(hive.events) == 0:
pprint(event_counts)
hive.events = []
sleep(1)
print('stopping')
hive.Stop()
print('stopped')
del hive
if __name__ == '__main__':
parser = ap()
print_events = False
relay_count = 10
client_count = 10
parser.add_argument('--print-events', dest="print_events", action='store_true')
parser.add_argument('--relay-count', dest="relay_count", type=int, default=10)
parser.add_argument('--client-count', dest="client_count", type=int, default=10)
args = parser.parse_args()
main(n_relays=args.relay_count, n_clients=args.client_count, print_each_event = args.print_events)

@ -0,0 +1,84 @@
from time import time
def test_path_builds(HiveArbitrary):
h = HiveArbitrary(n_relays=30, n_clients=10)
start_time = time()
cur_time = start_time
test_duration = 5 #seconds
log_attempts = True
paths = []
while cur_time < start_time + test_duration:
h.CollectAllEvents()
for event in h.events:
event_name = event.__class__.__name__
if log_attempts and event_name == "PathAttemptEvent":
path = dict()
path["hops"] = event.hops
path["received"] = [False] * len(event.hops)
path["prev"] = [None] * len(event.hops)
for i in range(1, len(event.hops)):
path["prev"][i] = event.hops[i-1].rc.routerID
path["prev"][0] = event.routerID
path["rxid"] = event.hops[0].rxid
path["status"] = None
paths.append(path)
elif event_name == "PathRequestReceivedEvent":
for path in paths:
for i in range(len(path["hops"])):
assert type(path["hops"][i].upstreamRouter) == type(event.nextHop)
assert type(path["prev"][i]) == type(event.prevHop)
assert type(path["hops"][i].txid) == type(event.txid)
assert type(path["hops"][i].rxid) == type(event.rxid)
if (path["hops"][i].upstreamRouter == event.nextHop and
path["prev"][i] == event.prevHop and
path["hops"][i].txid == event.txid and
path["hops"][i].rxid == event.rxid):
path["received"][i] = True
elif event_name == "PathStatusReceivedEvent":
for path in paths:
if event.rxid == path["rxid"]:
path["status"] = event
h.events = []
cur_time = time()
# only collect path attempts for 3 seconds
if cur_time > start_time + 3:
log_attempts = False
assert len(paths) > 0
fail_status_count = 0
missing_status_count = 0
missing_rcv_count = 0
expected_count = 0
for path in paths:
if path["status"]:
if not path["status"].Successful:
print(path["status"])
fail_status_count = fail_status_count + 1
else:
missing_status_count = missing_status_count + 1
for rcv in path["received"]:
expected_count = expected_count + 1
if not rcv:
missing_rcv_count = missing_rcv_count + 1
print("Path count: {}, Expected rcv: {}, missing rcv: {}, fail_status_count: {}, missing_status_count: {}".format(len(paths), expected_count, missing_rcv_count, fail_status_count, missing_status_count))
assert fail_status_count == 0
assert missing_rcv_count == 0
assert missing_status_count == 0
Loading…
Cancel
Save