From 955c475dbe415c5c00c68ed9130326fa132e871c Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Wed, 8 Jan 2020 20:44:37 -0400 Subject: [PATCH 01/26] Enable cpu detection on amd64 The optional runtime detection shouldn't rely on being compiled in avx2 mode. --- crypto/libntrup/src/ntru.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/libntrup/src/ntru.cpp b/crypto/libntrup/src/ntru.cpp index c86451b56..fd8f62037 100644 --- a/crypto/libntrup/src/ntru.cpp +++ b/crypto/libntrup/src/ntru.cpp @@ -1,6 +1,6 @@ #include -#if __AVX2__ +#ifdef __x86_64__ #include #include From b2784dd5ec87b744737507e1dfcbef45d9e9d1d2 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 9 Jan 2020 06:14:04 -0500 Subject: [PATCH 02/26] explore faster --- llarp/dht/dht.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/dht/dht.cpp b/llarp/dht/dht.cpp index 9569c6cba..472f7c3f7 100644 --- a/llarp/dht/dht.cpp +++ b/llarp/dht/dht.cpp @@ -35,7 +35,7 @@ llarp_dht_allow_transit(llarp_dht_context *ctx) void llarp_dht_context_start(struct llarp_dht_context *ctx, const byte_t *key) { - ctx->impl->Init(llarp::dht::Key_t(key), ctx->parent, 20000); + ctx->impl->Init(llarp::dht::Key_t(key), ctx->parent, 2000); } void From 2d82e931dae5d76fe80b163d934f0f592b3fdf65 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 9 Jan 2020 06:26:41 -0500 Subject: [PATCH 03/26] try doing lookup from close router, randomize when above lookup threshold --- llarp/service/endpoint.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index b90556e8e..534bd185c 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -1009,9 +1009,7 @@ namespace llarp if(lookups.count(remote) >= MaxConcurrentLookups) { - LogWarn(Name(), " has too many pending service lookups for ", - remote.ToString()); - return false; + path = PickRandomEstablishedPath(); } using namespace std::placeholders; @@ -1263,7 +1261,7 @@ namespace llarp } m_state->m_PendingTraffic.erase(r); }, - 5000); + 5000, false); } bool From 78e325c4c547def782f986e9e98e9e2d4d2c6ee3 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Thu, 9 Jan 2020 07:21:38 -0500 Subject: [PATCH 04/26] always lookup introset from nearest router --- llarp/service/outbound_context.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llarp/service/outbound_context.cpp b/llarp/service/outbound_context.cpp index abd97c217..b62254636 100644 --- a/llarp/service/outbound_context.cpp +++ b/llarp/service/outbound_context.cpp @@ -48,7 +48,7 @@ namespace llarp { SwapIntros(); } - UpdateIntroSet(true); + UpdateIntroSet(false); } return true; } @@ -300,7 +300,7 @@ namespace llarp } if(currentIntroSet.HasExpiredIntros(now)) { - UpdateIntroSet(true); + UpdateIntroSet(false); } // send control message if we look too quiet if(lastGoodSend) @@ -412,7 +412,7 @@ namespace llarp { // update introset LogInfo(Name(), " updating introset"); - UpdateIntroSet(true); + UpdateIntroSet(false); return true; } return false; @@ -487,7 +487,7 @@ namespace llarp OutboundContext::HandlePathDied(path::Path_ptr path) { // unconditionally update introset - UpdateIntroSet(true); + UpdateIntroSet(false); const RouterID endpoint(path->Endpoint()); // if a path to our current intro died... if(endpoint == remoteIntro.router) From 77f0944d6d692536a1f7cf015bbbdeac0f55363f Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Fri, 10 Jan 2020 07:23:54 -0500 Subject: [PATCH 05/26] use shared_from_this --- llarp/service/outbound_context.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llarp/service/outbound_context.cpp b/llarp/service/outbound_context.cpp index abd97c217..d378bbd9b 100644 --- a/llarp/service/outbound_context.cpp +++ b/llarp/service/outbound_context.cpp @@ -196,8 +196,8 @@ namespace llarp m_Endpoint->RouterLogic(), remoteIdent, m_Endpoint->GetIdentity(), currentIntroSet.K, remoteIntro, m_DataHandler, currentConvoTag, t); - ex->hook = - std::bind(&OutboundContext::Send, this, std::placeholders::_1, path); + ex->hook = std::bind(&OutboundContext::Send, shared_from_this(), + std::placeholders::_1, path); ex->msg.PutBuffer(payload); ex->msg.introReply = path->intro; @@ -236,7 +236,8 @@ namespace llarp if(path) { HiddenServiceAddressLookup* job = new HiddenServiceAddressLookup( - m_Endpoint, util::memFn(&OutboundContext::OnIntroSetUpdate, this), + m_Endpoint, + util::memFn(&OutboundContext::OnIntroSetUpdate, shared_from_this()), addr, m_Endpoint->GenTXID()); updatingIntroSet = job->SendRequestViaPath(path, m_Endpoint->Router()); From 8bb492da6e54e73664350471dfb1da736b385fc0 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Fri, 10 Jan 2020 10:45:25 -0500 Subject: [PATCH 06/26] dht timeout tweak --- llarp/dht/txholder.hpp | 45 +++++++++++++++++----------------------- llarp/service/lookup.hpp | 2 +- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/llarp/dht/txholder.hpp b/llarp/dht/txholder.hpp index d2904a783..e31da5fc3 100644 --- a/llarp/dht/txholder.hpp +++ b/llarp/dht/txholder.hpp @@ -13,8 +13,7 @@ namespace llarp { namespace dht { - template < typename K, typename V, typename K_Hash, - llarp_time_t requestTimeoutMS = 30000UL > + template < typename K, typename V, typename K_Hash > struct TXHolder { using TXPtr = std::unique_ptr< TX< K, V > >; @@ -72,7 +71,7 @@ namespace llarp void NewTX(const TXOwner& askpeer, const TXOwner& whoasked, const K& k, - TX< K, V >* t); + TX< K, V >* t, llarp_time_t requestTimeoutMS = 15000); /// mark tx as not fond void @@ -93,11 +92,9 @@ namespace llarp Expire(llarp_time_t now); }; - template < typename K, typename V, typename K_Hash, - llarp_time_t requestTimeoutMS > + template < typename K, typename V, typename K_Hash > const TX< K, V >* - TXHolder< K, V, K_Hash, requestTimeoutMS >::GetPendingLookupFrom( - const TXOwner& owner) const + TXHolder< K, V, K_Hash >::GetPendingLookupFrom(const TXOwner& owner) const { auto itr = tx.find(owner); if(itr == tx.end()) @@ -108,12 +105,12 @@ namespace llarp return itr->second.get(); } - template < typename K, typename V, typename K_Hash, - llarp_time_t requestTimeoutMS > + template < typename K, typename V, typename K_Hash > void - TXHolder< K, V, K_Hash, requestTimeoutMS >::NewTX(const TXOwner& askpeer, - const TXOwner& whoasked, - const K& k, TX< K, V >* t) + TXHolder< K, V, K_Hash >::NewTX(const TXOwner& askpeer, + const TXOwner& whoasked, const K& k, + TX< K, V >* t, + llarp_time_t requestTimeoutMS) { (void)whoasked; tx.emplace(askpeer, std::unique_ptr< TX< K, V > >(t)); @@ -131,11 +128,10 @@ namespace llarp } } - template < typename K, typename V, typename K_Hash, - llarp_time_t requestTimeoutMS > + template < typename K, typename V, typename K_Hash > void - TXHolder< K, V, K_Hash, requestTimeoutMS >::NotFound( - const TXOwner& from, const std::unique_ptr< Key_t >& next) + TXHolder< K, V, K_Hash >::NotFound(const TXOwner& from, + const std::unique_ptr< Key_t >& next) { auto txitr = tx.find(from); if(txitr == tx.end()) @@ -148,13 +144,11 @@ namespace llarp Inform(from, txitr->second->target, {}, true, true); } - template < typename K, typename V, typename K_Hash, - llarp_time_t requestTimeoutMS > + template < typename K, typename V, typename K_Hash > void - TXHolder< K, V, K_Hash, requestTimeoutMS >::Inform(TXOwner from, K key, - std::vector< V > values, - bool sendreply, - bool removeTimeouts) + TXHolder< K, V, K_Hash >::Inform(TXOwner from, K key, + std::vector< V > values, bool sendreply, + bool removeTimeouts) { auto range = waiting.equal_range(key); auto itr = range.first; @@ -187,15 +181,14 @@ namespace llarp } } - template < typename K, typename V, typename K_Hash, - llarp_time_t requestTimeoutMS > + template < typename K, typename V, typename K_Hash > void - TXHolder< K, V, K_Hash, requestTimeoutMS >::Expire(llarp_time_t now) + TXHolder< K, V, K_Hash >::Expire(llarp_time_t now) { auto itr = timeouts.begin(); while(itr != timeouts.end()) { - if(now > itr->second && now - itr->second >= requestTimeoutMS) + if(now >= itr->second) { Inform(TXOwner{}, itr->first, {}, true, false); itr = timeouts.erase(itr); diff --git a/llarp/service/lookup.hpp b/llarp/service/lookup.hpp index 5177fb0a6..a95c86ff9 100644 --- a/llarp/service/lookup.hpp +++ b/llarp/service/lookup.hpp @@ -36,7 +36,7 @@ namespace llarp /// determine if this request has timed out bool - IsTimedOut(llarp_time_t now, llarp_time_t timeout = 60000) const + IsTimedOut(llarp_time_t now, llarp_time_t timeout = 20000) const { if(now <= m_created) return false; From 78cdf8f96b756c01d1685e6dc2332b723d6a4025 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Fri, 10 Jan 2020 11:23:30 -0500 Subject: [PATCH 07/26] variable lookup timeout on participants --- llarp/dht/context.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llarp/dht/context.cpp b/llarp/dht/context.cpp index 5691a48f8..bf4c3a62d 100644 --- a/llarp/dht/context.cpp +++ b/llarp/dht/context.cpp @@ -598,7 +598,7 @@ namespace llarp TXOwner peer(askpeer, ++ids); _pendingIntrosetLookups.NewTX( peer, asker, addr, - new ServiceAddressLookup(asker, addr, this, R, handler)); + new ServiceAddressLookup(asker, addr, this, R, handler), (R * 2000)); } void @@ -611,7 +611,7 @@ namespace llarp TXOwner peer(askpeer, ++ids); _pendingIntrosetLookups.NewTX( peer, asker, addr, - new ServiceAddressLookup(asker, addr, this, 0, handler)); + new ServiceAddressLookup(asker, addr, this, 0, handler), 1000); } void From c7b94f32be264a4b70da6ab3646853a8d7429576 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Sun, 12 Jan 2020 07:29:36 -0500 Subject: [PATCH 08/26] reduce disk io --- llarp/nodedb.cpp | 92 +++++++++++++++++++++--------------------------- llarp/nodedb.hpp | 5 +-- 2 files changed, 44 insertions(+), 53 deletions(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index c9b7d7f4e..53459eee0 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -151,33 +151,35 @@ llarp_nodedb::UpdateAsyncIfNewer(llarp::RouterContact rc, /// insert and write to disk bool -llarp_nodedb::Insert(const llarp::RouterContact &rc) +llarp_nodedb::Insert(const llarp::RouterContact &rc, bool writeToDisk) { - std::array< byte_t, MAX_RC_SIZE > tmp; - llarp_buffer_t buf(tmp); + if(writeToDisk) + { + std::array< byte_t, MAX_RC_SIZE > tmp; + llarp_buffer_t buf(tmp); - if(!rc.BEncode(&buf)) - return false; + if(!rc.BEncode(&buf)) + return false; - buf.sz = buf.cur - buf.base; - auto filepath = getRCFilePath(rc.pubkey); - llarp::LogDebug("saving RC.pubkey ", filepath); - auto optional_ofs = llarp::util::OpenFileStream< std::ofstream >( - filepath, - std::ofstream::out | std::ofstream::binary | std::ofstream::trunc); - if(!optional_ofs) - return false; - auto &ofs = optional_ofs.value(); - ofs.write((char *)buf.base, buf.sz); - ofs.flush(); - ofs.close(); - if(!ofs) - { - llarp::LogError("Failed to write: ", filepath); - return false; + buf.sz = buf.cur - buf.base; + auto filepath = getRCFilePath(rc.pubkey); + llarp::LogDebug("saving RC.pubkey ", filepath); + auto optional_ofs = llarp::util::OpenFileStream< std::ofstream >( + filepath, + std::ofstream::out | std::ofstream::binary | std::ofstream::trunc); + if(!optional_ofs) + return false; + auto &ofs = optional_ofs.value(); + ofs.write((char *)buf.base, buf.sz); + ofs.flush(); + ofs.close(); + if(!ofs) + { + llarp::LogError("Failed to write: ", filepath); + return false; + } + llarp::LogDebug("saved RC.pubkey: ", filepath); } - llarp::LogDebug("saved RC.pubkey: ", filepath); - // save rc after writing to disk { llarp::util::Lock lock(&access); auto itr = entries.find(rc.pubkey.as_array()); @@ -375,16 +377,17 @@ crypto_threadworker_verifyrc(void *user) // if it's valid we need to set it if(verify_request->valid && rc.IsPublicRouter()) { - llarp::LogDebug("RC is valid, saving to disk"); - verify_request->diskworker->addJob( - std::bind(&disk_threadworker_setRC, verify_request)); - } - else - { - // callback to logic thread - verify_request->logic->queue_job( - {verify_request, &logic_threadworker_callback}); + if(verify_request->diskworker) + { + llarp::LogDebug("RC is valid, saving to disk"); + verify_request->diskworker->addJob( + std::bind(&disk_threadworker_setRC, verify_request)); + return; + } } + // callback to logic thread + verify_request->logic->queue_job( + {verify_request, &logic_threadworker_callback}); } void @@ -394,6 +397,12 @@ nodedb_inform_load_rc(void *user) job->hook(job); } +void +llarp_nodedb_async_verify(struct llarp_async_verify_rc *job) +{ + job->cryptoworker->addJob(std::bind(&crypto_threadworker_verifyrc, job)); +} + void nodedb_async_load_rc(void *user) { @@ -458,25 +467,6 @@ llarp_nodedb::load_dir(const char *dir) return Load(dir); } -/// maybe rename to verify_and_set -void -llarp_nodedb_async_verify(struct llarp_async_verify_rc *job) -{ - // switch to crypto threadpool and continue with - // crypto_threadworker_verifyrc - job->cryptoworker->addJob(std::bind(&crypto_threadworker_verifyrc, job)); -} - -// disabled for now -/* -void -llarp_nodedb_async_load_rc(struct llarp_async_load_rc *job) -{ - // call in the disk io thread so we don't bog down the others - llarp_threadpool_queue_job(job->diskworker, {job, &nodedb_async_load_rc}); -} -*/ - size_t llarp_nodedb::num_loaded() const { diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index acc63495f..8676f5d2e 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -87,9 +87,10 @@ struct llarp_nodedb std::string getRCFilePath(const llarp::RouterID &pubkey) const; - /// insert and write to disk + /// insert and optionally write to disk bool - Insert(const llarp::RouterContact &rc) LOCKS_EXCLUDED(access); + Insert(const llarp::RouterContact &rc, bool writeToDisk = false) + LOCKS_EXCLUDED(access); /// unconditional insert and write to disk in background /// updates the inserted time of the entry From 09f5f6dfa70d1a522d7b2c907a7fd9dd5aa362f9 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 13 Jan 2020 19:49:09 -0500 Subject: [PATCH 09/26] disable iterative lookup --- llarp/dht/txholder.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llarp/dht/txholder.hpp b/llarp/dht/txholder.hpp index e31da5fc3..280821499 100644 --- a/llarp/dht/txholder.hpp +++ b/llarp/dht/txholder.hpp @@ -138,10 +138,7 @@ namespace llarp { return; } - - // ask for next peer - if(!txitr->second->AskNextPeer(from.node, next)) - Inform(from, txitr->second->target, {}, true, true); + Inform(from, txitr->second->target, {}, true, true); } template < typename K, typename V, typename K_Hash > From 495ad6d5666fadb6cd7e11cde5a6363f8b700d6c Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 13 Jan 2020 20:01:33 -0500 Subject: [PATCH 10/26] testnet patch --- llarp/path/path_context.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llarp/path/path_context.cpp b/llarp/path/path_context.cpp index 789f318c5..3769becc3 100644 --- a/llarp/path/path_context.cpp +++ b/llarp/path/path_context.cpp @@ -39,12 +39,16 @@ namespace llarp bool PathContext::CheckPathLimitHitByIP(const llarp::Addr& ip) { +#ifdef TESTNET + return false; +#else llarp::Addr remote = ip; // set port to zero remote.port(0); // try inserting remote address by ip into decaying hash set // if it cannot insert it has hit a limit return not m_PathLimits.Insert(remote); +#endif } std::shared_ptr< Logic > From b56a3528db83b29a5d84a059d6d0c3f75d356403 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 12:01:41 -0500 Subject: [PATCH 11/26] periodic nodedb flush --- llarp/nodedb.cpp | 10 ++++++++++ llarp/nodedb.hpp | 8 ++++++++ llarp/router/router.cpp | 7 ++++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 53459eee0..f55d834e3 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -214,6 +214,7 @@ llarp_nodedb::Load(const fs::path &path) if(l > 0) loaded += l; } + m_NextSaveToDisk = llarp::time_now_ms() + m_SaveInterval; return loaded; } @@ -243,10 +244,19 @@ llarp_nodedb::SaveAll() } } +bool +llarp_nodedb::ShouldSaveToDisk(llarp_time_t now) const +{ + if(now == 0) + now = llarp::time_now_ms(); + return m_NextSaveToDisk > 0 && m_NextSaveToDisk <= now; +} + void llarp_nodedb::AsyncFlushToDisk() { disk->addJob(std::bind(&llarp_nodedb::SaveAll, this)); + m_NextSaveToDisk = llarp::time_now_ms() + m_SaveInterval; } ssize_t diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index 8676f5d2e..a023c825f 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -52,6 +52,10 @@ struct llarp_nodedb std::shared_ptr< llarp::thread::ThreadPool > disk; mutable llarp::util::Mutex access; // protects entries + /// time for next save to disk event, 0 if never happened + llarp_time_t m_NextSaveToDisk = 0; + /// how often to save to disk + const llarp_time_t m_SaveInterval = 60 * 5 * 1000; struct NetDBEntry { @@ -67,6 +71,10 @@ struct llarp_nodedb NetDBMap_t entries GUARDED_BY(access); fs::path nodePath; + /// return true if we should save our nodedb to disk + bool + ShouldSaveToDisk(llarp_time_t now = 0) const; + bool Remove(const llarp::RouterID &pk) LOCKS_EXCLUDED(access); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2014bd1d4..e7f88e60f 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -727,12 +727,17 @@ namespace llarp if(rpcCaller) rpcCaller->Tick(now); - // save profiles async + // save profiles if(routerProfiling().ShouldSave(now)) { diskworker()->addJob( [&]() { routerProfiling().Save(routerProfilesFile.c_str()); }); } + // save nodedb + if(nodedb()->ShouldSaveToDisk(now)) + { + nodedb()->AsyncFlushToDisk(); + } // get connected peers std::set< dht::Key_t > peersWeHave; From 0526fef37b8c15144c903c482487c4a499fedf4d Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 13:08:27 -0500 Subject: [PATCH 12/26] store RCs if valid --- llarp/dht/messages/gotrouter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/llarp/dht/messages/gotrouter.cpp b/llarp/dht/messages/gotrouter.cpp index e66b59907..eab4cc7f7 100644 --- a/llarp/dht/messages/gotrouter.cpp +++ b/llarp/dht/messages/gotrouter.cpp @@ -116,8 +116,13 @@ namespace llarp dht.pendingRouterLookups().Found(owner, R[0].pubkey, R); return true; } - llarp::LogWarn("Unwarranted GRM from ", From, " txid=", txid); - return false; + // store if valid + for(const auto &rc : R) + { + if(not dht.GetRouter().rcLookupHandler().CheckRC(rc)) + return false; + } + return true; } } // namespace dht } // namespace llarp From 2af64671bffbb23071f467e70493dbcf9bfeb0b3 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 13:10:10 -0500 Subject: [PATCH 13/26] make it compile --- llarp/dht/messages/gotrouter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llarp/dht/messages/gotrouter.cpp b/llarp/dht/messages/gotrouter.cpp index eab4cc7f7..933df1928 100644 --- a/llarp/dht/messages/gotrouter.cpp +++ b/llarp/dht/messages/gotrouter.cpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace llarp { @@ -119,7 +120,7 @@ namespace llarp // store if valid for(const auto &rc : R) { - if(not dht.GetRouter().rcLookupHandler().CheckRC(rc)) + if(not dht.GetRouter()->rcLookupHandler().CheckRC(rc)) return false; } return true; From 9322df013b1b5b3b48be7bdeb5d0660cad8d40b6 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 13:39:08 -0500 Subject: [PATCH 14/26] check with rc handler before insert --- llarp/dht/context.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llarp/dht/context.cpp b/llarp/dht/context.cpp index bf4c3a62d..23ed48183 100644 --- a/llarp/dht/context.cpp +++ b/llarp/dht/context.cpp @@ -20,7 +20,7 @@ #include #include #include - +#include #include namespace llarp @@ -41,7 +41,7 @@ namespace llarp void StoreRC(const RouterContact rc) const override { - GetRouter()->nodedb()->InsertAsync(rc); + GetRouter()->rcLookupHandler().CheckRC(rc); } /// on behalf of whoasked request introset for target from dht router with From 6fd714d19344ae507a304aef21e6996c9ae1f770 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 15:12:47 -0500 Subject: [PATCH 15/26] contrib/testnet: fix up testnet config generator to make super centralized topology llarp/context.cpp, llarp/nodedb.{h,c}pp: load netdb AFTER whitelist llarp/router/router.cpp: explore always llarp/router/{i,}rc_lookup_handler.{h,c}pp explore with whitelist, update routers with lookup before stale --- contrib/testnet/genconf.py | 3 +- llarp/context.cpp | 12 +------- llarp/dht/recursiverouterlookup.cpp | 2 ++ llarp/nodedb.cpp | 16 ++--------- llarp/nodedb.hpp | 9 ++++-- llarp/router/i_rc_lookup_handler.hpp | 3 +- llarp/router/rc_lookup_handler.cpp | 41 ++++++++++++++++++++++------ llarp/router/rc_lookup_handler.hpp | 4 +-- llarp/router/router.cpp | 23 ++++++++-------- 9 files changed, 60 insertions(+), 53 deletions(-) diff --git a/contrib/testnet/genconf.py b/contrib/testnet/genconf.py index 13300dece..19b62204f 100644 --- a/contrib/testnet/genconf.py +++ b/contrib/testnet/genconf.py @@ -37,7 +37,6 @@ def main(): exe = 'valgrind {}'.format(args.bin) else: exe = args.bin - basedir = os.path.abspath(args.dir) for nodeid in range(args.svc): @@ -108,7 +107,7 @@ def main(): fp = os.path.join(d, 'client.ini') with open(fp, 'w') as f: config.write(f) - for n in range(args.connect): + for n in [0]: otherID = (n + nodeid) % args.svc f.write("[bootstrap]\nadd-node={}\n".format(os.path.join(basedir,svcNodeName(otherID), 'rc.signed'))) with open(hiddenservice, 'w') as f: diff --git a/llarp/context.cpp b/llarp/context.cpp index 86a6ba5d0..b57ee4877 100644 --- a/llarp/context.cpp +++ b/llarp/context.cpp @@ -183,16 +183,6 @@ __ ___ ____ _ _ ___ _ _ ____ llarp::LogError("nodedb_dir is incorrect"); return 0; } - // llarp::LogInfo("nodedb_dir [", nodedb_dir, "] configured!"); - ssize_t loaded = nodedb->load_dir(nodedb_dir.c_str()); - llarp::LogInfo("nodedb_dir loaded ", loaded, " RCs from [", nodedb_dir, - "]"); - if(loaded < 0) - { - // shouldn't be possible - llarp::LogError("nodedb_dir directory doesn't exist"); - return 0; - } return 1; } @@ -211,7 +201,7 @@ __ ___ ____ _ _ ___ _ _ ____ router = std::make_unique< Router >(worker, mainloop, logic); - nodedb = std::make_unique< llarp_nodedb >(router->diskworker()); + nodedb = std::make_unique< llarp_nodedb >(router->diskworker(), nodedb_dir); if(!router->Configure(config.get(), nodedb.get())) { diff --git a/llarp/dht/recursiverouterlookup.cpp b/llarp/dht/recursiverouterlookup.cpp index f0bcb1b84..726963317 100644 --- a/llarp/dht/recursiverouterlookup.cpp +++ b/llarp/dht/recursiverouterlookup.cpp @@ -42,6 +42,7 @@ namespace llarp void RecursiveRouterLookup::DoNextRequest(const Key_t &peer) { + peersAsked.emplace(peer); parent->LookupRouterRecursive(target, whoasked.node, whoasked.txid, peer, resultHandler); } @@ -49,6 +50,7 @@ namespace llarp void RecursiveRouterLookup::Start(const TXOwner &peer) { + peersAsked.emplace(peer.node); parent->DHTSendTo(peer.node.as_array(), new FindRouterMessage(peer.txid, target)); } diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index c9b7d7f4e..3384c0ed3 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -440,22 +440,10 @@ llarp_nodedb::ensure_dir(const char *dir) return true; } -void -llarp_nodedb::set_dir(const char *dir) -{ - nodePath = dir; -} - ssize_t -llarp_nodedb::load_dir(const char *dir) +llarp_nodedb::LoadAll() { - std::error_code ec; - if(!fs::exists(dir, ec)) - { - return -1; - } - set_dir(dir); - return Load(dir); + return Load(nodePath.c_str()); } /// maybe rename to verify_and_set diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index acc63495f..c2a82cab5 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -40,8 +40,10 @@ struct llarp_nodedb_iter struct llarp_nodedb { - explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker) - : disk(std::move(diskworker)) + explicit llarp_nodedb(std::shared_ptr< llarp::thread::ThreadPool > diskworker, + const std::string rootdir) + : disk(std::move(diskworker)), nodePath(rootdir) + { } @@ -127,7 +129,8 @@ struct llarp_nodedb set_dir(const char *dir); ssize_t - load_dir(const char *dir); + LoadAll(); + ssize_t store_dir(const char *dir); diff --git a/llarp/router/i_rc_lookup_handler.hpp b/llarp/router/i_rc_lookup_handler.hpp index 15920df87..24318def1 100644 --- a/llarp/router/i_rc_lookup_handler.hpp +++ b/llarp/router/i_rc_lookup_handler.hpp @@ -37,7 +37,8 @@ namespace llarp SetRouterWhitelist(const std::vector< RouterID > &routers) = 0; virtual void - GetRC(const RouterID &router, RCRequestCallback callback) = 0; + GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup = false) = 0; virtual bool RemoteIsAllowed(const RouterID &remote) const = 0; diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index 35d9e2cb6..77bb94e76 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -48,20 +49,22 @@ namespace llarp } void - RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback) + RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup) { RouterContact remoteRC; - - if(_nodedb->Get(router, remoteRC)) + if(not forceLookup) { - if(callback) + if(_nodedb->Get(router, remoteRC)) { - callback(router, &remoteRC, RCRequestResult::Success); + if(callback) + { + callback(router, &remoteRC, RCRequestResult::Success); + } + FinalizeRequest(router, &remoteRC, RCRequestResult::Success); + return; } - FinalizeRequest(router, &remoteRC, RCRequestResult::Success); - return; } - bool shouldDoLookup = false; { @@ -212,7 +215,7 @@ namespace llarp for(const auto &router : routersToLookUp) { - GetRC(router, nullptr); + GetRC(router, nullptr, true); } _nodedb->RemoveStaleRCs(_bootstrapRouterIDList, @@ -235,6 +238,26 @@ namespace llarp LogError("we have no bootstrap nodes specified"); } + if(useWhitelist) + { + std::set< RouterID > lookupRouters; + { + static constexpr size_t LookupPerTick = 25; + // if we are using a whitelist look up a few routers we don't have + util::Lock l(&_mutex); + for(const auto &r : whitelistRouters) + { + if(_nodedb->Has(r)) + continue; + lookupRouters.emplace(r); + if(lookupRouters.size() >= LookupPerTick) + break; + } + } + for(const auto &r : lookupRouters) + GetRC(r, nullptr, true); + return; + } // TODO: only explore via random subset // explore via every connected peer _linkManager->ForEachPeer([&](ILinkSession *s) { diff --git a/llarp/router/rc_lookup_handler.hpp b/llarp/router/rc_lookup_handler.hpp index 87e0dd39e..3ec2bbd93 100644 --- a/llarp/router/rc_lookup_handler.hpp +++ b/llarp/router/rc_lookup_handler.hpp @@ -41,8 +41,8 @@ namespace llarp LOCKS_EXCLUDED(_mutex); void - GetRC(const RouterID &router, RCRequestCallback callback) override - LOCKS_EXCLUDED(_mutex); + GetRC(const RouterID &router, RCRequestCallback callback, + bool forceLookup = false) override LOCKS_EXCLUDED(_mutex); bool RemoteIsAllowed(const RouterID &remote) const override diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 2014bd1d4..ad88f4408 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -696,18 +696,9 @@ namespace llarp { connected += _linkManager.NumberOfPendingConnections(); } - const size_t N = nodedb()->num_loaded(); - if(N < llarp::path::default_len) - { - LogInfo("We need at least ", llarp::path::default_len, - " service nodes to build paths but we have ", N, " in nodedb"); - _rcLookupHandler.ExploreNetwork(); - } - else if(isSvcNode) - { - _rcLookupHandler.ExploreNetwork(); - } + _rcLookupHandler.ExploreNetwork(); + size_t connectToNum = _outboundSessionMaker.minConnectedRouters; const auto strictConnect = _rcLookupHandler.NumberOfStrictConnectRouters(); if(strictConnect > 0 && connectToNum > strictConnect) @@ -1014,6 +1005,16 @@ namespace llarp return false; } + { + ssize_t loaded = _nodedb->LoadAll(); + llarp::LogInfo("loaded ", loaded, " RCs"); + if(loaded < 0) + { + // shouldn't be possible + return false; + } + } + llarp_dht_context_start(dht(), pubkey()); for(const auto &rc : bootstrapRCList) From c5feb07fb9e1da2c1bf155560afd526fcb5befce Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Tue, 14 Jan 2020 15:48:39 -0500 Subject: [PATCH 16/26] address feedback from tom --- llarp/dht/recursiverouterlookup.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llarp/dht/recursiverouterlookup.cpp b/llarp/dht/recursiverouterlookup.cpp index 726963317..c5820312a 100644 --- a/llarp/dht/recursiverouterlookup.cpp +++ b/llarp/dht/recursiverouterlookup.cpp @@ -3,6 +3,10 @@ #include #include #include + +#include +#include + #include namespace llarp @@ -63,7 +67,8 @@ namespace llarp RouterContact found; for(const auto &rc : valuesFound) { - if(found.OtherIsNewer(rc)) + if(found.OtherIsNewer(rc) + && parent->GetRouter()->rcLookupHandler().CheckRC(rc)) found = rc; } valuesFound.clear(); @@ -79,9 +84,6 @@ namespace llarp whoasked.node.as_array(), new GotRouterMessage({}, whoasked.txid, valuesFound, false), false); } - // store this in our nodedb for caching - if(valuesFound.size() > 0) - parent->StoreRC(valuesFound[0]); } } // namespace dht } // namespace llarp From 4cb6301e2872621429e6e998b9419a612596317d Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 15 Jan 2020 10:41:43 -0500 Subject: [PATCH 17/26] use copy not move --- llarp/dht/messages/gotrouter.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llarp/dht/messages/gotrouter.hpp b/llarp/dht/messages/gotrouter.hpp index 4bb174410..9b01f217e 100644 --- a/llarp/dht/messages/gotrouter.hpp +++ b/llarp/dht/messages/gotrouter.hpp @@ -18,8 +18,9 @@ namespace llarp { } GotRouterMessage(const Key_t& from, uint64_t id, - std::vector< RouterContact > results, bool tunneled) - : IMessage(from), R(std::move(results)), txid(id), relayed(tunneled) + const std::vector< RouterContact >& results, + bool tunneled) + : IMessage(from), R(results), txid(id), relayed(tunneled) { } From d9297a4907cf164f7ae6b9a9a59f4087868de151 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 15 Jan 2020 10:42:28 -0500 Subject: [PATCH 18/26] log invalid signatures --- llarp/router_contact.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router_contact.cpp b/llarp/router_contact.cpp index b4f68da8a..a9b51d8dc 100644 --- a/llarp/router_contact.cpp +++ b/llarp/router_contact.cpp @@ -334,7 +334,7 @@ namespace llarp } if(!VerifySignature()) { - llarp::LogError("invalid signature"); + llarp::LogError("invalid signature: ", *this); return false; } return true; From a17b5f25b520f821467e94599fc472b13f0672ed Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Wed, 15 Jan 2020 10:43:21 -0500 Subject: [PATCH 19/26] check for zero'd rc --- llarp/dht/localrouterlookup.cpp | 6 +++++- llarp/dht/messages/findrouter.cpp | 7 ++++++- llarp/dht/messages/gotrouter.cpp | 2 ++ llarp/service/endpoint.cpp | 10 +++++----- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/llarp/dht/localrouterlookup.cpp b/llarp/dht/localrouterlookup.cpp index 7ab3955df..a60194a61 100644 --- a/llarp/dht/localrouterlookup.cpp +++ b/llarp/dht/localrouterlookup.cpp @@ -43,7 +43,11 @@ namespace llarp found = rc; } valuesFound.clear(); - valuesFound.emplace_back(found); + if(not found.pubkey.IsZero()) + { + valuesFound.resize(1); + valuesFound[0] = found; + } } routing::DHTMessage msg; msg.M.emplace_back(new GotRouterMessage(parent->OurKey(), whoasked.txid, diff --git a/llarp/dht/messages/findrouter.cpp b/llarp/dht/messages/findrouter.cpp index d2cef6c0d..d173f0475 100644 --- a/llarp/dht/messages/findrouter.cpp +++ b/llarp/dht/messages/findrouter.cpp @@ -167,7 +167,12 @@ namespace llarp return false; } RouterContact found; - const Key_t k{K}; + if(K.IsZero()) + { + llarp::LogError("invalid FRM from ", From, "K is zero"); + return false; + } + const Key_t k(K); if(exploritory) return dht.HandleExploritoryRouterLookup(From, txid, K, replies); if(!dht.GetRouter()->ConnectionToRouterAllowed(K)) diff --git a/llarp/dht/messages/gotrouter.cpp b/llarp/dht/messages/gotrouter.cpp index 933df1928..701ac8bbe 100644 --- a/llarp/dht/messages/gotrouter.cpp +++ b/llarp/dht/messages/gotrouter.cpp @@ -113,6 +113,8 @@ namespace llarp LogDebug("got ", R.size(), " results in GRM for lookup"); if(R.size() == 0) dht.pendingRouterLookups().NotFound(owner, K); + else if(R[0].pubkey.IsZero()) + return false; else dht.pendingRouterLookups().Found(owner, R[0].pubkey, R); return true; diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 534bd185c..18f842eef 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -733,11 +733,11 @@ namespace llarp { if(msg->R.size()) { - auto* job = new llarp_async_verify_rc; - job->nodedb = Router()->nodedb(); - job->cryptoworker = Router()->threadpool(); - job->diskworker = Router()->diskworker(); - job->logic = Router()->logic(); + llarp_async_verify_rc* job = new llarp_async_verify_rc(); + job->nodedb = Router()->nodedb(); + job->cryptoworker = Router()->threadpool(); + job->diskworker = Router()->diskworker(); + job->logic = Router()->logic(); job->hook = std::bind(&Endpoint::HandleVerifyGotRouter, this, msg, std::placeholders::_1); job->rc = msg->R[0]; From 08149112b2c859d4c04e33b35882275c22de1991 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 15 Jan 2020 19:16:18 -0700 Subject: [PATCH 20/26] Randomize routers to explore in ExploreNetwork() --- llarp/router/rc_lookup_handler.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index 77bb94e76..b4b529b8c 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -14,6 +14,7 @@ #include #include +#include namespace llarp { @@ -240,20 +241,29 @@ namespace llarp if(useWhitelist) { - std::set< RouterID > lookupRouters; + static constexpr size_t LookupPerTick = 25; + + std::vector< RouterID > lookupRouters; + lookupRouters.reserve(LookupPerTick); + { - static constexpr size_t LookupPerTick = 25; // if we are using a whitelist look up a few routers we don't have util::Lock l(&_mutex); for(const auto &r : whitelistRouters) { if(_nodedb->Has(r)) continue; - lookupRouters.emplace(r); - if(lookupRouters.size() >= LookupPerTick) - break; + lookupRouters.emplace_back(r); } } + + if (lookupRouters.size() > LookupPerTick) + { + static std::mt19937_64 rng{std::random_device{}()}; + std::shuffle(lookupRouters.begin(), lookupRouters.end(), rng); + lookupRouters.resize(LookupPerTick); + } + for(const auto &r : lookupRouters) GetRC(r, nullptr, true); return; From c92e7ae5c7b43fc133de8948af14457b79651521 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 15 Jan 2020 19:37:12 -0700 Subject: [PATCH 21/26] Add warning when we found a null RC for a DHT lookup --- llarp/dht/localrouterlookup.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llarp/dht/localrouterlookup.cpp b/llarp/dht/localrouterlookup.cpp index a60194a61..7e953f9e7 100644 --- a/llarp/dht/localrouterlookup.cpp +++ b/llarp/dht/localrouterlookup.cpp @@ -48,6 +48,10 @@ namespace llarp valuesFound.resize(1); valuesFound[0] = found; } + else + { + llarp::LogWarn("We found a null RC for dht request, dropping it"); + } } routing::DHTMessage msg; msg.M.emplace_back(new GotRouterMessage(parent->OurKey(), whoasked.txid, From 8206557ac70d9b48850b1cf32c0d0456c13bb05e Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 15 Jan 2020 21:12:38 -0700 Subject: [PATCH 22/26] Don't respect whitelist when we haven't received it yet --- llarp/router/rc_lookup_handler.cpp | 7 +++++++ llarp/router/rc_lookup_handler.hpp | 3 +++ llarp/router/router.cpp | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index b4b529b8c..f5a6d56ea 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -49,6 +49,13 @@ namespace llarp " routers"); } + bool + RCLookupHandler::HaveReceivedWhitelist() + { + util::Lock l(&_mutex); + return whitelistRouters.empty(); + } + void RCLookupHandler::GetRC(const RouterID &router, RCRequestCallback callback, bool forceLookup) diff --git a/llarp/router/rc_lookup_handler.hpp b/llarp/router/rc_lookup_handler.hpp index 3ec2bbd93..afae472e0 100644 --- a/llarp/router/rc_lookup_handler.hpp +++ b/llarp/router/rc_lookup_handler.hpp @@ -40,6 +40,9 @@ namespace llarp SetRouterWhitelist(const std::vector< RouterID > &routers) override LOCKS_EXCLUDED(_mutex); + bool + HaveReceivedWhitelist(); + void GetRC(const RouterID &router, RCRequestCallback callback, bool forceLookup = false) override LOCKS_EXCLUDED(_mutex); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index ad88f4408..d5ca74a97 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -679,7 +679,7 @@ namespace llarp LogError("Failed to update our RC"); } - if(isSvcNode) + if(isSvcNode && _rcLookupHandler.HaveReceivedWhitelist()) { // remove RCs for nodes that are no longer allowed by network policy nodedb()->RemoveIf([&](const RouterContact &rc) -> bool { From fbb274a72451407609af93ac34346498774ab380 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 15 Jan 2020 21:15:37 -0700 Subject: [PATCH 23/26] Make format (mutter mutter) --- llarp/router/rc_lookup_handler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/router/rc_lookup_handler.cpp b/llarp/router/rc_lookup_handler.cpp index f5a6d56ea..3b9b7c0d1 100644 --- a/llarp/router/rc_lookup_handler.cpp +++ b/llarp/router/rc_lookup_handler.cpp @@ -264,7 +264,7 @@ namespace llarp } } - if (lookupRouters.size() > LookupPerTick) + if(lookupRouters.size() > LookupPerTick) { static std::mt19937_64 rng{std::random_device{}()}; std::shuffle(lookupRouters.begin(), lookupRouters.end(), rng); From 416c71e5541e04f7134f5f79b9c1ee42e0355100 Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Wed, 15 Jan 2020 22:56:16 -0400 Subject: [PATCH 24/26] Fix json api bind IP Only the port was being used, the IP was being ignored and it was always binding to localhost. --- llarp/rpc/rpc.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/llarp/rpc/rpc.cpp b/llarp/rpc/rpc.cpp index 91f558e82..c218f4f1d 100644 --- a/llarp/rpc/rpc.cpp +++ b/llarp/rpc/rpc.cpp @@ -423,18 +423,18 @@ namespace llarp bool Start(const std::string& addr) { - uint16_t port = 0; - auto idx = addr.find_first_of(':'); - Addr netaddr; - if(idx != std::string::npos) - { - port = std::stoi(addr.substr(1 + idx)); - netaddr = Addr(addr.substr(0, idx)); - } sockaddr_in saddr; saddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); saddr.sin_family = AF_INET; - saddr.sin_port = htons(port); + saddr.sin_port = 0; + + auto idx = addr.find_first_of(':'); + if(idx != std::string::npos) + { + Addr netaddr{addr.substr(0, idx), addr.substr(1 + idx)}; + saddr.sin_addr.s_addr = netaddr.ton(); + saddr.sin_port = htons(netaddr.port()); + } return _handler.ServeAsync(router->netloop(), router->logic(), (const sockaddr*)&saddr); } From 702c1300209f5208f5c799ecfc8d7f12ac0ac089 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 15 Jan 2020 21:44:25 -0700 Subject: [PATCH 25/26] Remove dead code in llarp_nodedb::Insert() --- llarp/nodedb.cpp | 47 +++++++++-------------------------------------- llarp/nodedb.hpp | 9 ++++----- 2 files changed, 13 insertions(+), 43 deletions(-) diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index 90fe89ffa..3c40e578c 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -149,46 +149,17 @@ llarp_nodedb::UpdateAsyncIfNewer(llarp::RouterContact rc, return false; } -/// insert and write to disk +/// insert bool -llarp_nodedb::Insert(const llarp::RouterContact &rc, bool writeToDisk) +llarp_nodedb::Insert(const llarp::RouterContact &rc) { - if(writeToDisk) - { - std::array< byte_t, MAX_RC_SIZE > tmp; - llarp_buffer_t buf(tmp); - - if(!rc.BEncode(&buf)) - return false; - - buf.sz = buf.cur - buf.base; - auto filepath = getRCFilePath(rc.pubkey); - llarp::LogDebug("saving RC.pubkey ", filepath); - auto optional_ofs = llarp::util::OpenFileStream< std::ofstream >( - filepath, - std::ofstream::out | std::ofstream::binary | std::ofstream::trunc); - if(!optional_ofs) - return false; - auto &ofs = optional_ofs.value(); - ofs.write((char *)buf.base, buf.sz); - ofs.flush(); - ofs.close(); - if(!ofs) - { - llarp::LogError("Failed to write: ", filepath); - return false; - } - llarp::LogDebug("saved RC.pubkey: ", filepath); - } - { - llarp::util::Lock lock(&access); - auto itr = entries.find(rc.pubkey.as_array()); - if(itr != entries.end()) - entries.erase(itr); - entries.emplace(rc.pubkey.as_array(), rc); - LogDebug("Added or updated RC for ", llarp::RouterID(rc.pubkey), - " to nodedb. Current nodedb count is: ", entries.size()); - } + llarp::util::Lock lock(&access); + auto itr = entries.find(rc.pubkey.as_array()); + if(itr != entries.end()) + entries.erase(itr); + entries.emplace(rc.pubkey.as_array(), rc); + LogDebug("Added or updated RC for ", llarp::RouterID(rc.pubkey), + " to nodedb. Current nodedb count is: ", entries.size()); return true; } diff --git a/llarp/nodedb.hpp b/llarp/nodedb.hpp index dec748c35..0886d7779 100644 --- a/llarp/nodedb.hpp +++ b/llarp/nodedb.hpp @@ -97,13 +97,12 @@ struct llarp_nodedb std::string getRCFilePath(const llarp::RouterID &pubkey) const; - /// insert and optionally write to disk + /// insert without writing to disk bool - Insert(const llarp::RouterContact &rc, bool writeToDisk = false) - LOCKS_EXCLUDED(access); + Insert(const llarp::RouterContact &rc) LOCKS_EXCLUDED(access); - /// unconditional insert and write to disk in background - /// updates the inserted time of the entry + /// invokes Insert() asynchronously with an optional completion + /// callback void InsertAsync(llarp::RouterContact rc, std::shared_ptr< llarp::Logic > l = nullptr, From 2607bdbee6e55a6e74322985b3949aff9bfb9dd7 Mon Sep 17 00:00:00 2001 From: Stephen Shelton Date: Wed, 15 Jan 2020 21:55:34 -0700 Subject: [PATCH 26/26] Bump version (0.6.3) --- llarp/constants/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/constants/version.h b/llarp/constants/version.h index 85c36dcf2..475dcfb77 100644 --- a/llarp/constants/version.h +++ b/llarp/constants/version.h @@ -8,7 +8,7 @@ #define LLARP_VERSION_MAJ 0 #define LLARP_VERSION_MIN 6 -#define LLARP_VERSION_PATCH 2 +#define LLARP_VERSION_PATCH 3 #define LLARP_DEFAULT_NETID "lokinet"