From 7c8c11a42aa7a56ef1e83f6fa1f5a4eaad5bec21 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 29 Jul 2019 08:14:35 -0400 Subject: [PATCH 1/5] correct cmake paramter for shellhooks --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c411c8f3..3df472c18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ option(DEBIAN "build for debian" ) option(TESTNET "testnet build" ) option(WITH_SHARED "build shared library") option(WITH_COVERAGE "generate coverage data") -option(WITH_SHELLHOOKS "enable shell hooks on compile time (dangerous)" OFF) +option(USE_SHELLHOOKS "enable shell hooks on compile time (dangerous)" OFF) option(WARNINGS_AS_ERRORS "treat all warnings as errors. turn off for development, on for release" OFF) include(cmake/target_link_libraries_system.cmake) From 9a8470bcc1da96948eab7aec7c5723c0b39ce246 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 29 Jul 2019 09:08:40 -0400 Subject: [PATCH 2/5] * const correctness * use std::map's upper_bound to find many closer entries * randomize key for exploration to allow many explore jobs in paralell --- llarp/dht/bucket.hpp | 22 ++++++++++------------ llarp/dht/context.cpp | 5 +++-- llarp/dht/kademlia.hpp | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/llarp/dht/bucket.hpp b/llarp/dht/bucket.hpp index d796bc480..9d4607d4c 100644 --- a/llarp/dht/bucket.hpp +++ b/llarp/dht/bucket.hpp @@ -108,8 +108,8 @@ namespace llarp return true; } - size_t expecting = N; - size_t sz = nodes.size(); + const size_t expecting = N; + const size_t sz = nodes.size(); while(N) { auto itr = nodes.begin(); @@ -137,7 +137,7 @@ namespace llarp continue; } - auto curDist = item.first ^ target; + const auto curDist = item.first ^ target; if(curDist < mindist) { mindist = curDist; @@ -151,19 +151,17 @@ namespace llarp GetManyNearExcluding(const Key_t& target, std::set< Key_t >& result, size_t N, const std::set< Key_t >& exclude) const { - std::set< Key_t > s(exclude.begin(), exclude.end()); - - Key_t peer; - while(N--) + auto itr = nodes.upper_bound(target); + while(itr != nodes.end() && N > 0) { - if(!FindCloseExcluding(target, peer, s)) + if(exclude.count(itr->first) == 0) { - return false; + result.insert(itr->first); + --N; } - s.insert(peer); - result.insert(peer); + ++itr; } - return true; + return N == 0; } void diff --git a/llarp/dht/context.cpp b/llarp/dht/context.cpp index c7c208f82..561d6b204 100644 --- a/llarp/dht/context.cpp +++ b/llarp/dht/context.cpp @@ -336,9 +336,10 @@ namespace llarp uint64_t txid = ++ids; TXOwner peer(askpeer, txid); TXOwner whoasked(OurKey(), txid); + RouterID K; + K.Randomize(); pendingExploreLookups().NewTX( - peer, whoasked, askpeer.as_array(), - new ExploreNetworkJob(askpeer.as_array(), this)); + peer, whoasked, K, new ExploreNetworkJob(askpeer.as_array(), this)); } void diff --git a/llarp/dht/kademlia.hpp b/llarp/dht/kademlia.hpp index 9e99594eb..8ddda64a8 100644 --- a/llarp/dht/kademlia.hpp +++ b/llarp/dht/kademlia.hpp @@ -9,7 +9,7 @@ namespace llarp { struct XorMetric { - const Key_t& us; + const Key_t us; XorMetric(const Key_t& ourKey) : us(ourKey) { From db2206664a6ca24cc8706ffc0b6b52720446dc51 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 29 Jul 2019 11:10:20 -0400 Subject: [PATCH 3/5] fix crashes in testnet --- llarp/dht/bucket.hpp | 7 +++--- llarp/path/path.cpp | 4 +++- llarp/path/pathbuilder.cpp | 44 ++++++++++++++++++++------------------ llarp/service/endpoint.cpp | 19 ++++++++-------- llarp/utp/session.cpp | 7 +++--- 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/llarp/dht/bucket.hpp b/llarp/dht/bucket.hpp index 9d4607d4c..a0998e052 100644 --- a/llarp/dht/bucket.hpp +++ b/llarp/dht/bucket.hpp @@ -151,8 +151,9 @@ namespace llarp GetManyNearExcluding(const Key_t& target, std::set< Key_t >& result, size_t N, const std::set< Key_t >& exclude) const { - auto itr = nodes.upper_bound(target); - while(itr != nodes.end() && N > 0) + auto itr = nodes.lower_bound(target); + const auto end = nodes.end(); + while(N > 0 && itr != end) { if(exclude.count(itr->first) == 0) { @@ -161,7 +162,7 @@ namespace llarp } ++itr; } - return N == 0; + return true; } void diff --git a/llarp/path/path.cpp b/llarp/path/path.cpp index b77ae1a72..43761a923 100644 --- a/llarp/path/path.cpp +++ b/llarp/path/path.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -145,7 +146,8 @@ namespace llarp if((currentStatus & LR_StatusRecord::SUCCESS) == 1) { llarp::LogDebug("LR_Status message processed, path build successful"); - HandlePathConfirmMessage(r); + auto self = shared_from_this(); + r->logic()->queue_func([=]() { self->HandlePathConfirmMessage(r); }); } else { diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index 4352b4a53..3ec75a504 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -15,12 +15,14 @@ namespace llarp { struct AsyncPathKeyExchangeContext + : std::enable_shared_from_this< AsyncPathKeyExchangeContext > { using Path_t = path::Path_ptr; using PathSet_t = path::PathSet_ptr; PathSet_t pathset = nullptr; Path_t path = nullptr; - using Handler = std::function< void(const AsyncPathKeyExchangeContext&) >; + using Handler = + std::function< void(std::shared_ptr< AsyncPathKeyExchangeContext >) >; Handler result; size_t idx = 0; @@ -96,13 +98,13 @@ namespace llarp { // farthest hop // TODO: encrypt junk frames because our public keys are not eligator - logic->queue_func(std::bind(result, *this)); + logic->queue_func(std::bind(result, shared_from_this())); } else { // next hop - worker->addJob( - std::bind(&AsyncPathKeyExchangeContext::GenerateNextKey, *this)); + worker->addJob(std::bind(&AsyncPathKeyExchangeContext::GenerateNextKey, + shared_from_this())); } } @@ -120,28 +122,28 @@ namespace llarp { LRCM.frames[i].Randomize(); } - pool->addJob( - std::bind(&AsyncPathKeyExchangeContext::GenerateNextKey, *this)); + pool->addJob(std::bind(&AsyncPathKeyExchangeContext::GenerateNextKey, + shared_from_this())); } }; static void - PathBuilderKeysGenerated(const AsyncPathKeyExchangeContext& ctx) + PathBuilderKeysGenerated(std::shared_ptr< AsyncPathKeyExchangeContext > ctx) { - if(!ctx.pathset->IsStopped()) + if(!ctx->pathset->IsStopped()) { - RouterID remote = ctx.path->Upstream(); - const ILinkMessage* msg = &ctx.LRCM; - if(ctx.router->SendToOrQueue(remote, msg)) + RouterID remote = ctx->path->Upstream(); + const ILinkMessage* msg = &ctx->LRCM; + if(ctx->router->SendToOrQueue(remote, msg)) { // persist session with router until this path is done - ctx.router->PersistSessionUntil(remote, ctx.path->ExpireTime()); + ctx->router->PersistSessionUntil(remote, ctx->path->ExpireTime()); // add own path - ctx.router->pathContext().AddOwnPath(ctx.pathset, ctx.path); - ctx.pathset->PathBuildStarted(ctx.path); + ctx->router->pathContext().AddOwnPath(ctx->pathset, ctx->path); + ctx->pathset->PathBuildStarted(ctx->path); } else - LogError(ctx.pathset->Name(), " failed to send LRCM to ", remote); + LogError(ctx->pathset->Name(), " failed to send LRCM to ", remote); } } @@ -419,15 +421,15 @@ namespace llarp return; lastBuild = Now(); // async generate keys - AsyncPathKeyExchangeContext ctx; - ctx.router = router; - ctx.pathset = GetSelf(); - auto path = std::make_shared< path::Path >(hops, this, roles); + auto ctx = std::make_shared< AsyncPathKeyExchangeContext >(); + ctx->router = router; + ctx->pathset = GetSelf(); + auto path = std::make_shared< path::Path >(hops, this, roles); LogInfo(Name(), " build ", path->HopsString()); path->SetBuildResultHook( [this](Path_ptr p) { this->HandlePathBuilt(p); }); - ctx.AsyncGenerateKeys(path, router->logic(), router->threadpool(), - &PathBuilderKeysGenerated); + ctx->AsyncGenerateKeys(path, router->logic(), router->threadpool(), + &PathBuilderKeysGenerated); } void diff --git a/llarp/service/endpoint.cpp b/llarp/service/endpoint.cpp index 8cc67a41a..e42519e0d 100644 --- a/llarp/service/endpoint.cpp +++ b/llarp/service/endpoint.cpp @@ -893,20 +893,21 @@ namespace llarp Endpoint::OnLookup(const Address& addr, const IntroSet* introset, const RouterID& endpoint) { - auto now = Now(); - auto& fails = m_state->m_ServiceLookupFails; - auto& lookups = m_state->m_PendingServiceLookups; + const auto now = router->Now(); + auto& fails = m_state->m_ServiceLookupFails; + auto& lookups = m_state->m_PendingServiceLookups; if(introset == nullptr || introset->IsExpired(now)) { LogError(Name(), " failed to lookup ", addr.ToString(), " from ", endpoint); fails[endpoint] = fails[endpoint] + 1; - // inform one - auto itr = lookups.find(addr); - if(itr != lookups.end()) + // inform all + auto range = lookups.equal_range(addr); + auto itr = range.first; + if(itr != range.second) { itr->second(addr, nullptr); - lookups.erase(itr); + itr = lookups.erase(itr); } return false; } @@ -1169,12 +1170,12 @@ namespace llarp if(c) { c->UpdateIntroSet(true); - for(auto& pending : traffic[r]) + for(auto& pending : m_state->m_PendingTraffic[r]) { c->AsyncEncryptAndSendTo(pending.Buffer(), pending.protocol); } } - traffic.erase(r); + m_state->m_PendingTraffic.erase(r); }, 5000, true); } diff --git a/llarp/utp/session.cpp b/llarp/utp/session.cpp index b8bdd07a1..5d4e7556e 100644 --- a/llarp/utp/session.cpp +++ b/llarp/utp/session.cpp @@ -52,24 +52,25 @@ namespace llarp do { auto& msg = sendq.front(); - while(msg.vecs.size() && sz >= msg.vecs.front().iov_len) + while(msg.vecs.size() > 0 && sz >= msg.vecs.front().iov_len) { sz -= msg.vecs.front().iov_len; msg.vecs.pop_front(); - msg.fragments.pop_front(); } if(msg.vecs.size() == 0) { msg.Delivered(); sendq.pop_front(); } - else + else if(sz) { auto& front = msg.vecs.front(); front.iov_len -= sz; front.iov_base = ((byte_t*)front.iov_base) + sz; return; } + else + return; } while(sendq.size()); } } From 60fbeca9d461f70cfddadb8695b497d9dcd554b3 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 29 Jul 2019 12:43:24 -0400 Subject: [PATCH 4/5] const correctness --- llarp/path/pathbuilder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llarp/path/pathbuilder.cpp b/llarp/path/pathbuilder.cpp index 3ec75a504..6a94a356c 100644 --- a/llarp/path/pathbuilder.cpp +++ b/llarp/path/pathbuilder.cpp @@ -132,7 +132,7 @@ namespace llarp { if(!ctx->pathset->IsStopped()) { - RouterID remote = ctx->path->Upstream(); + const RouterID remote = ctx->path->Upstream(); const ILinkMessage* msg = &ctx->LRCM; if(ctx->router->SendToOrQueue(remote, msg)) { From e1dd7ad97f93786c0764bb16d625d0d42df1a5e3 Mon Sep 17 00:00:00 2001 From: Jeff Becker Date: Mon, 29 Jul 2019 18:33:49 -0400 Subject: [PATCH 5/5] revert dht stuff --- llarp/dht/bucket.hpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/llarp/dht/bucket.hpp b/llarp/dht/bucket.hpp index a0998e052..d796bc480 100644 --- a/llarp/dht/bucket.hpp +++ b/llarp/dht/bucket.hpp @@ -108,8 +108,8 @@ namespace llarp return true; } - const size_t expecting = N; - const size_t sz = nodes.size(); + size_t expecting = N; + size_t sz = nodes.size(); while(N) { auto itr = nodes.begin(); @@ -137,7 +137,7 @@ namespace llarp continue; } - const auto curDist = item.first ^ target; + auto curDist = item.first ^ target; if(curDist < mindist) { mindist = curDist; @@ -151,16 +151,17 @@ namespace llarp GetManyNearExcluding(const Key_t& target, std::set< Key_t >& result, size_t N, const std::set< Key_t >& exclude) const { - auto itr = nodes.lower_bound(target); - const auto end = nodes.end(); - while(N > 0 && itr != end) + std::set< Key_t > s(exclude.begin(), exclude.end()); + + Key_t peer; + while(N--) { - if(exclude.count(itr->first) == 0) + if(!FindCloseExcluding(target, peer, s)) { - result.insert(itr->first); - --N; + return false; } - ++itr; + s.insert(peer); + result.insert(peer); } return true; }