From 09f5f6dfa70d1a522d7b2c907a7fd9dd5aa362f9 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 13 Jan 2020 19:49:09 -0500 Subject: [PATCH 01/10] disable iterative lookup --- llarp/dht/txholder.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llarp/dht/txholder.hpp b/llarp/dht/txholder.hpp index e31da5fc3..280821499 100644 --- a/llarp/dht/txholder.hpp +++ b/llarp/dht/txholder.hpp @@ -138,10 +138,7 @@ namespace llarp { return; } - - // ask for next peer - if(!txitr->second->AskNextPeer(from.node, next)) - Inform(from, txitr->second->target, {}, true, true); + Inform(from, txitr->second->target, {}, true, true); } template < typename K, typename V, typename K_Hash > From 495ad6d5666fadb6cd7e11cde5a6363f8b700d6c Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 13 Jan 2020 20:01:33 -0500 Subject: [PATCH 02/10] testnet patch --- llarp/path/path_context.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp index 789f318c5..3769becc3 100644 --- a/llarp/path/path_context.cpp +++ b/llarp/path/path_context.cpp @@ -39,12 +39,16 @@ namespace llarp bool PathContext::CheckPathLimitHitByIP(const llarp::Addr& ip) { +#ifdef TESTNET + return false; +#else llarp::Addr remote = ip; // set port to zero remote.port(0); // try inserting remote address by ip into decaying hash set // if it cannot insert it has hit a limit return not m_PathLimits.Insert(remote); +#endif } std::shared_ptr< Logic > From 0526fef37b8c15144c903c482487c4a499fedf4d Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 13:08:27 -0500 Subject: [PATCH 03/10] store RCs if valid --- llarp/dht/messages/gotrouter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/llarp/dht/messages/gotrouter.cpp b/llarp/dht/messages/gotrouter.cpp index e66b59907..eab4cc7f7 100644 --- a/llarp/dht/messages/gotrouter.cpp +++ b/llarp/dht/messages/gotrouter.cpp @@ -116,8 +116,13 @@ namespace llarp dht.pendingRouterLookups().Found(owner, R[0].pubkey, R); return true; } - llarp::LogWarn("Unwarranted GRM from ", From, " txid=", txid); - return false; + // store if valid + for(const auto &rc : R) + { + if(not dht.GetRouter().rcLookupHandler().CheckRC(rc)) + return false; + } + return true; } } // namespace dht } // namespace llarp From 2af64671bffbb23071f467e70493dbcf9bfeb0b3 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 13:10:10 -0500 Subject: [PATCH 04/10] make it compile --- llarp/dht/messages/gotrouter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llarp/dht/messages/gotrouter.cpp b/llarp/dht/messages/gotrouter.cpp index eab4cc7f7..933df1928 100644 --- a/llarp/dht/messages/gotrouter.cpp +++ b/llarp/dht/messages/gotrouter.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace llarp { @@ -119,7 +120,7 @@ namespace llarp // store if valid for(const auto &rc : R) { - if(not dht.GetRouter().rcLookupHandler().CheckRC(rc)) + if(not dht.GetRouter()->rcLookupHandler().CheckRC(rc)) return false; } return true; From 9322df013b1b5b3b48be7bdeb5d0660cad8d40b6 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 13:39:08 -0500 Subject: [PATCH 05/10] check with rc handler before insert --- llarp/dht/context.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llarp/dht/context.cpp b/llarp/dht/context.cpp index bf4c3a62d..23ed48183 100644 --- a/llarp/dht/context.cpp +++ b/llarp/dht/context.cpp @@ -20,7 +20,7 @@ #include #include #include - +#include #include namespace llarp @@ -41,7 +41,7 @@ namespace llarp void StoreRC(const RouterContact rc) const override { - GetRouter()->nodedb()->InsertAsync(rc); + GetRouter()->rcLookupHandler().CheckRC(rc); } /// on behalf of whoasked request introset for target from dht router with From 6fd714d19344ae507a304aef21e6996c9ae1f770 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 15:12:47 -0500 Subject: [PATCH 06/10] contrib/testnet: fix up testnet config generator to make super centralized topology llarp/context.cpp, llarp/nodedb.{h,c}pp: load netdb AFTER whitelist llarp/router/router.cpp: explore always llarp/router/{i,}rc_lookup_handler.{h,c}pp explore with whitelist, update routers with lookup before stale --- contrib/testnet/genconf.py | 3 +- llarp/context.cpp | 12 +------- llarp/dht/recursiverouterlookup.cpp | 2 ++ llarp/nodedb.cpp | 16 ++--------- llarp/nodedb.hpp | 9 ++++-- llarp/router/i_rc_lookup_handler.hpp | 3 +- llarp/router/rc_lookup_handler.cpp | 41 ++++++++++++++++++++++------ llarp/router/rc_lookup_handler.hpp | 4 +-- llarp/router/router.cpp | 23 ++++++++-------- 9 files changed, 60 insertions(+), 53 deletions(-) diff --git a/contrib/testnet/genconf.py b/contrib/testnet/genconf.py index 13300dece..19b62204f 100644 --- a/contrib/testnet/genconf.py +++ b/contrib/testnet/genconf.py @@ -37,7 +37,6 @@ def main(): exe = 'valgrind {}'.format(args.bin) else: exe = args.bin - basedir = os.path.abspath(args.dir) for nodeid in range(args.svc): @@ -108,7 +107,7 @@ def main(): fp = os.path.join(d, 'client.ini') with open(fp, 'w') as f: config.write(f) - for n in range(args.connect): + for n in [0]: otherID = (n + nodeid) % args.svc f.write("[bootstrap]\nadd-node={}\n".format(os.path.join(basedir,svcNodeName(otherID), 'rc.signed'))) with open(hiddenservice, 'w') as f: diff --git a/llarp/context.cpp b/llarp/context.cpp index 86a6ba5d0..b57ee4877 100644 --- a/llarp/context.cpp +++ b/llarp/context.cpp @@ -183,16 +183,6 @@ __ ___ ____ _ _ ___ _ _ ____ llarp::LogError("nodedb_dir is incorrect"); return 0; } - // llarp::LogInfo("nodedb_dir [", nodedb_dir, "] configured!"); - ssize_t loaded = nodedb->load_dir(nodedb_dir.c_str()); - llarp::LogInfo("nodedb_dir loaded ", loaded, " RCs from [", nodedb_dir, - "]"); - if(loaded < 0) - { - // shouldn't be possible - llarp::LogError("nodedb_dir directory doesn't exist"); - return 0; - } return 1; } @@ -211,7 +201,7 @@ __ ___ ____ _ _ ___ _ _ ____ router = std::make_unique< Router >(worker, mainloop, logic); - nodedb = std::make_unique< llarp_nodedb >(router->diskworker()); + nodedb = std::make_unique< llarp_nodedb >(router->diskworker(), nodedb_dir); if(!router->Configure(config.get(), nodedb.get())) { diff --git a/llarp/dht/recursiverouterlookup.cpp b/llarp/dht/recursiverouterlookup.cpp index f0bcb1b84..726963317 100644 --- a/llarp/dht/recursiverouterlookup.cpp +++ b/llarp/dht/recursiverouterlookup.cpp @@ -42,6 +42,7 @@ namespace llarp void RecursiveRouterLookup::DoNextRequest(const Key_t &peer) { + peersAsked.emplace(peer); parent->LookupRouterRecursive(target, whoasked.node, whoasked.txid, peer, resultHandler); } @@ -49,6 +50,7 @@ namespace llarp void RecursiveRouterLookup::Start(const TXOwner &peer) { + peersAsked.emplace(peer.node); parent->DHTSendTo(peer.node.as_array(), new FindRouterMessage(peer.txid, target)); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index c9b7d7f4e..3384c0ed3 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -440,22 +440,10 @@ llarp_nodedb::ensure_dir(const char *dir) return true; } -void -llarp_nodedb::set_dir(const char *dir) -{ - nodePath = dir; -} - ssize_t -llarp_nodedb::load_dir(const char *dir) +llarp_nodedb::LoadAll() { - std::error_code ec; - if(!fs::exists(dir, ec)) - { - return -1; - } - set_dir(dir); - return Load(dir); + return Load(nodePath.c_str()); } /// maybe rename to verify_and_set diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index acc63495f..c2a82cab5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -40,8 +40,10 @@ struct llarp_nodedb_iter struct llarp_nodedb { - explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker) - : disk(std::move(diskworker)) + explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker, + const std::string rootdir) + : disk(std::move(diskworker)), nodePath(rootdir) + { } @@ -127,7 +129,8 @@ struct llarp_nodedb set_dir(const char *dir); ssize_t - load_dir(const char *dir); + LoadAll(); + ssize_t store_dir(const char *dir); diff --git a/llarp/router/i_rc_lookup_handler.hpp b/llarp/router/i_rc_lookup_handler.hpp index 15920df87..24318def1 100644 --- a/llarp/router/i_rc_lookup_handler.hpp +++ b/llarp/router/i_rc_lookup_handler.hpp @@ -37,7 +37,8 @@ namespace llarp SetRouterWhitelist(const std::vector< RouterID > &routers) = 0; virtual void - GetRC(const RouterID &router, RCRequestCallback callback) = 0; + GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup = false) = 0; virtual bool RemoteIsAllowed(const RouterID &remote) const = 0; diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index 35d9e2cb6..77bb94e76 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -48,20 +49,22 @@ namespace llarp } void - RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback) + RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup) { RouterContact remoteRC; - - if(_nodedb->Get(router, remoteRC)) + if(not forceLookup) { - if(callback) + if(_nodedb->Get(router, remoteRC)) { - callback(router, &remoteRC, RCRequestResult::Success); + if(callback) + { + callback(router, &remoteRC, RCRequestResult::Success); + } + FinalizeRequest(router, &remoteRC, RCRequestResult::Success); + return; } - FinalizeRequest(router, &remoteRC, RCRequestResult::Success); - return; } - bool shouldDoLookup = false; { @@ -212,7 +215,7 @@ namespace llarp for(const auto &router : routersToLookUp) { - GetRC(router, nullptr); + GetRC(router, nullptr, true); } _nodedb->RemoveStaleRCs(_bootstrapRouterIDList, @@ -235,6 +238,26 @@ namespace llarp LogError("we have no bootstrap nodes specified"); } + if(useWhitelist) + { + std::set< RouterID > lookupRouters; + { + static constexpr size_t LookupPerTick = 25; + // if we are using a whitelist look up a few routers we don't have + util::Lock l(&_mutex); + for(const auto &r : whitelistRouters) + { + if(_nodedb->Has(r)) + continue; + lookupRouters.emplace(r); + if(lookupRouters.size() >= LookupPerTick) + break; + } + } + for(const auto &r : lookupRouters) + GetRC(r, nullptr, true); + return; + } // TODO: only explore via random subset // explore via every connected peer _linkManager->ForEachPeer([&](ILinkSession *s) { diff --git a/llarp/router/rc_lookup_handler.hpp b/llarp/router/rc_lookup_handler.hpp index 87e0dd39e..3ec2bbd93 100644 --- a/llarp/router/rc_lookup_handler.hpp +++ b/llarp/router/rc_lookup_handler.hpp @@ -41,8 +41,8 @@ namespace llarp LOCKS_EXCLUDED(_mutex); void - GetRC(const RouterID &router, RCRequestCallback callback) override - LOCKS_EXCLUDED(_mutex); + GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup = false) override LOCKS_EXCLUDED(_mutex); bool RemoteIsAllowed(const RouterID &remote) const override diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2014bd1d4..ad88f4408 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -696,18 +696,9 @@ namespace llarp { connected += _linkManager.NumberOfPendingConnections(); } - const size_t N = nodedb()->num_loaded(); - if(N < llarp::path::default_len) - { - LogInfo("We need at least ", llarp::path::default_len, - " service nodes to build paths but we have ", N, " in nodedb"); - _rcLookupHandler.ExploreNetwork(); - } - else if(isSvcNode) - { - _rcLookupHandler.ExploreNetwork(); - } + _rcLookupHandler.ExploreNetwork(); + size_t connectToNum = _outboundSessionMaker.minConnectedRouters; const auto strictConnect = _rcLookupHandler.NumberOfStrictConnectRouters(); if(strictConnect > 0 && connectToNum > strictConnect) @@ -1014,6 +1005,16 @@ namespace llarp return false; } + { + ssize_t loaded = _nodedb->LoadAll(); + llarp::LogInfo("loaded ", loaded, " RCs"); + if(loaded < 0) + { + // shouldn't be possible + return false; + } + } + llarp_dht_context_start(dht(), pubkey()); for(const auto &rc : bootstrapRCList) From c5feb07fb9e1da2c1bf155560afd526fcb5befce Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 15:48:39 -0500 Subject: [PATCH 07/10] address feedback from tom --- llarp/dht/recursiverouterlookup.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llarp/dht/recursiverouterlookup.cpp b/llarp/dht/recursiverouterlookup.cpp index 726963317..c5820312a 100644 --- a/llarp/dht/recursiverouterlookup.cpp +++ b/llarp/dht/recursiverouterlookup.cpp @@ -3,6 +3,10 @@ #include #include #include + +#include +#include + #include namespace llarp @@ -63,7 +67,8 @@ namespace llarp RouterContact found; for(const auto &rc : valuesFound) { - if(found.OtherIsNewer(rc)) + if(found.OtherIsNewer(rc) + && parent->GetRouter()->rcLookupHandler().CheckRC(rc)) found = rc; } valuesFound.clear(); @@ -79,9 +84,6 @@ namespace llarp whoasked.node.as_array(), new GotRouterMessage({}, whoasked.txid, valuesFound, false), false); } - // store this in our nodedb for caching - if(valuesFound.size() > 0) - parent->StoreRC(valuesFound[0]); } } // namespace dht } // namespace llarp From 4cb6301e2872621429e6e998b9419a612596317d Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 15 Jan 2020 10:41:43 -0500 Subject: [PATCH 08/10] use copy not move --- llarp/dht/messages/gotrouter.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llarp/dht/messages/gotrouter.hpp b/llarp/dht/messages/gotrouter.hpp index 4bb174410..9b01f217e 100644 --- a/llarp/dht/messages/gotrouter.hpp +++ b/llarp/dht/messages/gotrouter.hpp @@ -18,8 +18,9 @@ namespace llarp { } GotRouterMessage(const Key_t& from, uint64_t id, - std::vector< RouterContact > results, bool tunneled) - : IMessage(from), R(std::move(results)), txid(id), relayed(tunneled) + const std::vector< RouterContact >& results, + bool tunneled) + : IMessage(from), R(results), txid(id), relayed(tunneled) { } From d9297a4907cf164f7ae6b9a9a59f4087868de151 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 15 Jan 2020 10:42:28 -0500 Subject: [PATCH 09/10] log invalid signatures --- llarp/router_contact.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index b4f68da8a..a9b51d8dc 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -334,7 +334,7 @@ namespace llarp } if(!VerifySignature()) { - llarp::LogError("invalid signature"); + llarp::LogError("invalid signature: ", *this); return false; } return true; From a17b5f25b520f821467e94599fc472b13f0672ed Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 15 Jan 2020 10:43:21 -0500 Subject: [PATCH 10/10] check for zero'd rc --- llarp/dht/localrouterlookup.cpp | 6 +++++- llarp/dht/messages/findrouter.cpp | 7 ++++++- llarp/dht/messages/gotrouter.cpp | 2 ++ llarp/service/endpoint.cpp | 10 +++++----- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/llarp/dht/localrouterlookup.cpp b/llarp/dht/localrouterlookup.cpp index 7ab3955df..a60194a61 100644 --- a/llarp/dht/localrouterlookup.cpp +++ b/llarp/dht/localrouterlookup.cpp @@ -43,7 +43,11 @@ namespace llarp found = rc; } valuesFound.clear(); - valuesFound.emplace_back(found); + if(not found.pubkey.IsZero()) + { + valuesFound.resize(1); + valuesFound[0] = found; + } } routing::DHTMessage msg; msg.M.emplace_back(new GotRouterMessage(parent->OurKey(), whoasked.txid, diff --git a/llarp/dht/messages/findrouter.cpp b/llarp/dht/messages/findrouter.cpp index d2cef6c0d..d173f0475 100644 --- a/llarp/dht/messages/findrouter.cpp +++ b/llarp/dht/messages/findrouter.cpp @@ -167,7 +167,12 @@ namespace llarp return false; } RouterContact found; - const Key_t k{K}; + if(K.IsZero()) + { + llarp::LogError("invalid FRM from ", From, "K is zero"); + return false; + } + const Key_t k(K); if(exploritory) return dht.HandleExploritoryRouterLookup(From, txid, K, replies); if(!dht.GetRouter()->ConnectionToRouterAllowed(K)) diff --git a/llarp/dht/messages/gotrouter.cpp b/llarp/dht/messages/gotrouter.cpp index 933df1928..701ac8bbe 100644 --- a/llarp/dht/messages/gotrouter.cpp +++ b/llarp/dht/messages/gotrouter.cpp @@ -113,6 +113,8 @@ namespace llarp LogDebug("got ", R.size(), " results in GRM for lookup"); if(R.size() == 0) dht.pendingRouterLookups().NotFound(owner, K); + else if(R[0].pubkey.IsZero()) + return false; else dht.pendingRouterLookups().Found(owner, R[0].pubkey, R); return true; diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 534bd185c..18f842eef 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -733,11 +733,11 @@ namespace llarp { if(msg->R.size()) { - auto* job = new llarp_async_verify_rc; - job->nodedb = Router()->nodedb(); - job->cryptoworker = Router()->threadpool(); - job->diskworker = Router()->diskworker(); - job->logic = Router()->logic(); + llarp_async_verify_rc* job = new llarp_async_verify_rc(); + job->nodedb = Router()->nodedb(); + job->cryptoworker = Router()->threadpool(); + job->diskworker = Router()->diskworker(); + job->logic = Router()->logic(); job->hook = std::bind(&Endpoint::HandleVerifyGotRouter, this, msg, std::placeholders::_1); job->rc = msg->R[0];