From 6fd714d19344ae507a304aef21e6996c9ae1f770 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 15:12:47 -0500 Subject: [PATCH] contrib/testnet: fix up testnet config generator to make super centralized topology llarp/context.cpp, llarp/nodedb.{h,c}pp: load netdb AFTER whitelist llarp/router/router.cpp: explore always llarp/router/{i,}rc_lookup_handler.{h,c}pp explore with whitelist, update routers with lookup before stale --- contrib/testnet/genconf.py | 3 +- llarp/context.cpp | 12 +------- llarp/dht/recursiverouterlookup.cpp | 2 ++ llarp/nodedb.cpp | 16 ++--------- llarp/nodedb.hpp | 9 ++++-- llarp/router/i_rc_lookup_handler.hpp | 3 +- llarp/router/rc_lookup_handler.cpp | 41 ++++++++++++++++++++++------ llarp/router/rc_lookup_handler.hpp | 4 +-- llarp/router/router.cpp | 23 ++++++++-------- 9 files changed, 60 insertions(+), 53 deletions(-) diff --git a/contrib/testnet/genconf.py b/contrib/testnet/genconf.py index 13300dece..19b62204f 100644 --- a/contrib/testnet/genconf.py +++ b/contrib/testnet/genconf.py @@ -37,7 +37,6 @@ def main(): exe = 'valgrind {}'.format(args.bin) else: exe = args.bin - basedir = os.path.abspath(args.dir) for nodeid in range(args.svc): @@ -108,7 +107,7 @@ def main(): fp = os.path.join(d, 'client.ini') with open(fp, 'w') as f: config.write(f) - for n in range(args.connect): + for n in [0]: otherID = (n + nodeid) % args.svc f.write("[bootstrap]\nadd-node={}\n".format(os.path.join(basedir,svcNodeName(otherID), 'rc.signed'))) with open(hiddenservice, 'w') as f: diff --git a/llarp/context.cpp b/llarp/context.cpp index 86a6ba5d0..b57ee4877 100644 --- a/llarp/context.cpp +++ b/llarp/context.cpp @@ -183,16 +183,6 @@ __ ___ ____ _ _ ___ _ _ ____ llarp::LogError("nodedb_dir is incorrect"); return 0; } - // llarp::LogInfo("nodedb_dir [", nodedb_dir, "] configured!"); - ssize_t loaded = nodedb->load_dir(nodedb_dir.c_str()); - llarp::LogInfo("nodedb_dir loaded ", loaded, " RCs from [", nodedb_dir, - "]"); - if(loaded < 0) - { - // shouldn't be possible - llarp::LogError("nodedb_dir directory doesn't exist"); - return 0; - } return 1; } @@ -211,7 +201,7 @@ __ ___ ____ _ _ ___ _ _ ____ router = std::make_unique< Router >(worker, mainloop, logic); - nodedb = std::make_unique< llarp_nodedb >(router->diskworker()); + nodedb = std::make_unique< llarp_nodedb >(router->diskworker(), nodedb_dir); if(!router->Configure(config.get(), nodedb.get())) { diff --git a/llarp/dht/recursiverouterlookup.cpp b/llarp/dht/recursiverouterlookup.cpp index f0bcb1b84..726963317 100644 --- a/llarp/dht/recursiverouterlookup.cpp +++ b/llarp/dht/recursiverouterlookup.cpp @@ -42,6 +42,7 @@ namespace llarp void RecursiveRouterLookup::DoNextRequest(const Key_t &peer) { + peersAsked.emplace(peer); parent->LookupRouterRecursive(target, whoasked.node, whoasked.txid, peer, resultHandler); } @@ -49,6 +50,7 @@ namespace llarp void RecursiveRouterLookup::Start(const TXOwner &peer) { + peersAsked.emplace(peer.node); parent->DHTSendTo(peer.node.as_array(), new FindRouterMessage(peer.txid, target)); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index c9b7d7f4e..3384c0ed3 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -440,22 +440,10 @@ llarp_nodedb::ensure_dir(const char *dir) return true; } -void -llarp_nodedb::set_dir(const char *dir) -{ - nodePath = dir; -} - ssize_t -llarp_nodedb::load_dir(const char *dir) +llarp_nodedb::LoadAll() { - std::error_code ec; - if(!fs::exists(dir, ec)) - { - return -1; - } - set_dir(dir); - return Load(dir); + return Load(nodePath.c_str()); } /// maybe rename to verify_and_set diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index acc63495f..c2a82cab5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -40,8 +40,10 @@ struct llarp_nodedb_iter struct llarp_nodedb { - explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker) - : disk(std::move(diskworker)) + explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker, + const std::string rootdir) + : disk(std::move(diskworker)), nodePath(rootdir) + { } @@ -127,7 +129,8 @@ struct llarp_nodedb set_dir(const char *dir); ssize_t - load_dir(const char *dir); + LoadAll(); + ssize_t store_dir(const char *dir); diff --git a/llarp/router/i_rc_lookup_handler.hpp b/llarp/router/i_rc_lookup_handler.hpp index 15920df87..24318def1 100644 --- a/llarp/router/i_rc_lookup_handler.hpp +++ b/llarp/router/i_rc_lookup_handler.hpp @@ -37,7 +37,8 @@ namespace llarp SetRouterWhitelist(const std::vector< RouterID > &routers) = 0; virtual void - GetRC(const RouterID &router, RCRequestCallback callback) = 0; + GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup = false) = 0; virtual bool RemoteIsAllowed(const RouterID &remote) const = 0; diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index 35d9e2cb6..77bb94e76 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -48,20 +49,22 @@ namespace llarp } void - RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback) + RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup) { RouterContact remoteRC; - - if(_nodedb->Get(router, remoteRC)) + if(not forceLookup) { - if(callback) + if(_nodedb->Get(router, remoteRC)) { - callback(router, &remoteRC, RCRequestResult::Success); + if(callback) + { + callback(router, &remoteRC, RCRequestResult::Success); + } + FinalizeRequest(router, &remoteRC, RCRequestResult::Success); + return; } - FinalizeRequest(router, &remoteRC, RCRequestResult::Success); - return; } - bool shouldDoLookup = false; { @@ -212,7 +215,7 @@ namespace llarp for(const auto &router : routersToLookUp) { - GetRC(router, nullptr); + GetRC(router, nullptr, true); } _nodedb->RemoveStaleRCs(_bootstrapRouterIDList, @@ -235,6 +238,26 @@ namespace llarp LogError("we have no bootstrap nodes specified"); } + if(useWhitelist) + { + std::set< RouterID > lookupRouters; + { + static constexpr size_t LookupPerTick = 25; + // if we are using a whitelist look up a few routers we don't have + util::Lock l(&_mutex); + for(const auto &r : whitelistRouters) + { + if(_nodedb->Has(r)) + continue; + lookupRouters.emplace(r); + if(lookupRouters.size() >= LookupPerTick) + break; + } + } + for(const auto &r : lookupRouters) + GetRC(r, nullptr, true); + return; + } // TODO: only explore via random subset // explore via every connected peer _linkManager->ForEachPeer([&](ILinkSession *s) { diff --git a/llarp/router/rc_lookup_handler.hpp b/llarp/router/rc_lookup_handler.hpp index 87e0dd39e..3ec2bbd93 100644 --- a/llarp/router/rc_lookup_handler.hpp +++ b/llarp/router/rc_lookup_handler.hpp @@ -41,8 +41,8 @@ namespace llarp LOCKS_EXCLUDED(_mutex); void - GetRC(const RouterID &router, RCRequestCallback callback) override - LOCKS_EXCLUDED(_mutex); + GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup = false) override LOCKS_EXCLUDED(_mutex); bool RemoteIsAllowed(const RouterID &remote) const override diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2014bd1d4..ad88f4408 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -696,18 +696,9 @@ namespace llarp { connected += _linkManager.NumberOfPendingConnections(); } - const size_t N = nodedb()->num_loaded(); - if(N < llarp::path::default_len) - { - LogInfo("We need at least ", llarp::path::default_len, - " service nodes to build paths but we have ", N, " in nodedb"); - _rcLookupHandler.ExploreNetwork(); - } - else if(isSvcNode) - { - _rcLookupHandler.ExploreNetwork(); - } + _rcLookupHandler.ExploreNetwork(); + size_t connectToNum = _outboundSessionMaker.minConnectedRouters; const auto strictConnect = _rcLookupHandler.NumberOfStrictConnectRouters(); if(strictConnect > 0 && connectToNum > strictConnect) @@ -1014,6 +1005,16 @@ namespace llarp return false; } + { + ssize_t loaded = _nodedb->LoadAll(); + llarp::LogInfo("loaded ", loaded, " RCs"); + if(loaded < 0) + { + // shouldn't be possible + return false; + } + } + llarp_dht_context_start(dht(), pubkey()); for(const auto &rc : bootstrapRCList)