lokinet/llarp/nodedb.cpp

571 lines
12 KiB
C++
Raw Normal View History

#include <nodedb.hpp>
2019-01-13 14:00:50 +00:00
#include <crypto/crypto.hpp>
2018-12-12 01:55:30 +00:00
#include <router_contact.hpp>
#include <util/buffer.hpp>
#include <util/encode.hpp>
#include <util/fs.hpp>
2019-09-01 12:10:49 +00:00
#include <util/logging/logger.hpp>
#include <util/mem.hpp>
2019-09-01 13:26:16 +00:00
#include <util/thread/logic.hpp>
#include <util/thread/thread_pool.hpp>
#include <fstream>
2018-06-10 14:05:48 +00:00
#include <unordered_map>
2019-07-30 23:42:13 +00:00
#include <utility>
2018-05-30 00:40:02 +00:00
2018-06-13 13:09:19 +00:00
static const char skiplist_subdirs[] = "0123456789abcdef";
2018-08-02 23:36:34 +00:00
static const std::string RC_FILE_EXT = ".signed";
2018-05-30 00:40:02 +00:00
2019-07-30 23:42:13 +00:00
llarp_nodedb::NetDBEntry::NetDBEntry(llarp::RouterContact value)
: rc(std::move(value)), inserted(llarp::time_now_ms())
{
}
bool
llarp_nodedb::Remove(const llarp::RouterID &pk)
{
bool removed = false;
RemoveIf([&](const llarp::RouterContact &rc) -> bool {
if(rc.pubkey == pk)
{
removed = true;
return true;
}
return false;
});
return removed;
}
2018-05-16 18:13:18 +00:00
void
llarp_nodedb::Clear()
{
llarp::util::Lock lock(&access);
entries.clear();
}
2018-04-08 12:18:16 +00:00
bool
llarp_nodedb::Get(const llarp::RouterID &pk, llarp::RouterContact &result)
{
llarp::util::Lock l(&access);
auto itr = entries.find(pk);
if(itr == entries.end())
return false;
result = itr->second.rc;
return true;
}
void
KillRCJobs(const std::set< std::string > &files)
{
for(const auto &file : files)
fs::remove(file);
}
void
llarp_nodedb::RemoveIf(
std::function< bool(const llarp::RouterContact &rc) > filter)
{
std::set< std::string > files;
{
llarp::util::Lock l(&access);
auto itr = entries.begin();
while(itr != entries.end())
{
if(filter(itr->second.rc))
{
files.insert(getRCFilePath(itr->second.rc.pubkey));
itr = entries.erase(itr);
}
else
++itr;
}
}
disk->addJob(std::bind(&KillRCJobs, files));
}
bool
llarp_nodedb::Has(const llarp::RouterID &pk)
{
llarp::util::Lock lock(&access);
return entries.find(pk) != entries.end();
}
2018-06-14 17:35:12 +00:00
/// skiplist directory is hex encoded first nibble
/// skiplist filename is <base32encoded>.snode.signed
std::string
llarp_nodedb::getRCFilePath(const llarp::RouterID &pubkey) const
{
char ftmp[68] = {0};
const char *hexname =
llarp::HexEncode< llarp::AlignedBuffer< 32 >, decltype(ftmp) >(pubkey,
ftmp);
std::string hexString(hexname);
std::string skiplistDir;
llarp::RouterID r(pubkey);
std::string fname = r.ToString();
skiplistDir += hexString[0];
fname += RC_FILE_EXT;
fs::path filepath = nodePath / skiplistDir / fname;
return filepath.string();
}
void
2019-05-22 16:20:50 +00:00
llarp_nodedb::InsertAsync(llarp::RouterContact rc,
std::shared_ptr< llarp::Logic > logic,
std::function< void(void) > completionHandler)
{
disk->addJob([this, rc, logic, completionHandler]() {
this->Insert(rc);
if(logic && completionHandler)
{
LogicCall(logic, completionHandler);
}
});
}
bool
llarp_nodedb::UpdateAsyncIfNewer(llarp::RouterContact rc,
std::shared_ptr< llarp::Logic > logic,
std::function< void(void) > completionHandler)
{
llarp::util::Lock lock(&access);
auto itr = entries.find(rc.pubkey);
if(itr == entries.end() || itr->second.rc.OtherIsNewer(rc))
{
InsertAsync(rc, logic, completionHandler);
return true;
}
2019-07-30 23:42:13 +00:00
if(itr != entries.end())
{
// insertion time is set on...insertion. But it should be updated here
// even if there is no insertion of a new RC, to show that the existing one
// is not "stale"
itr->second.inserted = llarp::time_now_ms();
}
return false;
}
/// insert
bool
llarp_nodedb::Insert(const llarp::RouterContact &rc)
{
llarp::util::Lock lock(&access);
auto itr = entries.find(rc.pubkey.as_array());
if(itr != entries.end())
entries.erase(itr);
entries.emplace(rc.pubkey.as_array(), rc);
LogDebug("Added or updated RC for ", llarp::RouterID(rc.pubkey),
" to nodedb. Current nodedb count is: ", entries.size());
return true;
}
2018-05-30 00:40:02 +00:00
ssize_t
llarp_nodedb::Load(const fs::path &path)
{
std::error_code ec;
if(!fs::exists(path, ec))
2018-06-13 12:58:51 +00:00
{
return -1;
2018-06-13 12:58:51 +00:00
}
ssize_t loaded = 0;
2018-06-13 12:58:51 +00:00
for(const char &ch : skiplist_subdirs)
2018-06-13 11:37:44 +00:00
{
if(!ch)
continue;
std::string p;
p += ch;
fs::path sub = path / p;
2018-05-30 00:40:02 +00:00
ssize_t l = loadSubdir(sub);
if(l > 0)
loaded += l;
2018-05-30 00:40:02 +00:00
}
2020-01-14 17:01:41 +00:00
m_NextSaveToDisk = llarp::time_now_ms() + m_SaveInterval;
return loaded;
}
2018-05-30 00:40:02 +00:00
void
llarp_nodedb::SaveAll()
{
std::array< byte_t, MAX_RC_SIZE > tmp;
llarp::util::Lock lock(&access);
for(const auto &item : entries)
{
llarp_buffer_t buf(tmp);
if(!item.second.rc.BEncode(&buf))
continue;
buf.sz = buf.cur - buf.base;
const auto filepath = getRCFilePath(item.second.rc.pubkey);
2019-06-24 16:39:03 +00:00
auto optional_ofs = llarp::util::OpenFileStream< std::ofstream >(
filepath,
std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
2019-06-24 16:39:03 +00:00
if(!optional_ofs)
continue;
auto &ofs = optional_ofs.value();
ofs.write((char *)buf.base, buf.sz);
ofs.flush();
ofs.close();
}
}
2020-01-14 17:01:41 +00:00
bool
llarp_nodedb::ShouldSaveToDisk(llarp_time_t now) const
{
if(now == 0)
now = llarp::time_now_ms();
return m_NextSaveToDisk > 0 && m_NextSaveToDisk <= now;
}
void
llarp_nodedb::AsyncFlushToDisk()
{
disk->addJob(std::bind(&llarp_nodedb::SaveAll, this));
2020-01-14 17:01:41 +00:00
m_NextSaveToDisk = llarp::time_now_ms() + m_SaveInterval;
}
ssize_t
llarp_nodedb::loadSubdir(const fs::path &dir)
{
ssize_t sz = 0;
llarp::util::IterDir(dir, [&](const fs::path &f) -> bool {
if(fs::is_regular_file(f) && loadfile(f))
sz++;
return true;
});
return sz;
}
2018-04-30 16:14:20 +00:00
bool
llarp_nodedb::loadfile(const fs::path &fpath)
{
if(fpath.extension() != RC_FILE_EXT)
return false;
llarp::RouterContact rc;
if(!rc.Read(fpath.string().c_str()))
{
llarp::LogError("failed to read file ", fpath);
return false;
2018-04-08 12:18:16 +00:00
}
if(!rc.Verify(llarp::time_now_ms()))
2018-05-30 00:40:02 +00:00
{
llarp::LogError(fpath, " contains invalid RC");
return false;
2018-05-30 00:40:02 +00:00
}
{
llarp::util::Lock lock(&access);
entries.emplace(rc.pubkey.as_array(), rc);
2018-04-08 12:18:16 +00:00
}
return true;
}
2018-04-30 16:14:20 +00:00
void
llarp_nodedb::visit(std::function< bool(const llarp::RouterContact &) > visit)
{
llarp::util::Lock lock(&access);
auto itr = entries.begin();
while(itr != entries.end())
{
if(!visit(itr->second.rc))
return;
++itr;
}
}
void
2019-06-20 14:00:04 +00:00
llarp_nodedb::VisitInsertedBefore(
std::function< void(const llarp::RouterContact &) > visit,
llarp_time_t insertedAfter)
{
llarp::util::Lock lock(&access);
auto itr = entries.begin();
while(itr != entries.end())
{
2019-06-20 14:00:04 +00:00
if(itr->second.inserted < insertedAfter)
visit(itr->second.rc);
++itr;
}
}
void
llarp_nodedb::RemoveStaleRCs(const std::set< llarp::RouterID > &keep,
llarp_time_t cutoff)
{
std::set< llarp::RouterID > removeStale;
// remove stale routers
VisitInsertedBefore(
[&](const llarp::RouterContact &rc) {
if(keep.find(rc.pubkey) != keep.end())
return;
LogInfo("removing stale router: ", llarp::RouterID(rc.pubkey));
removeStale.insert(rc.pubkey);
},
cutoff);
RemoveIf([&removeStale](const llarp::RouterContact &rc) -> bool {
return removeStale.count(rc.pubkey) > 0;
});
}
/*
bool
llarp_nodedb::Save()
{
auto itr = entries.begin();
while(itr != entries.end())
{
llarp::pubkey pk = itr->first;
llarp_rc *rc= itr->second;
2018-05-30 00:40:02 +00:00
itr++; // advance
2018-04-08 12:18:16 +00:00
}
return true;
}
*/
2018-04-08 12:18:16 +00:00
// call request hook
2018-06-13 11:37:44 +00:00
void
logic_threadworker_callback(void *user)
{
2019-07-30 23:42:13 +00:00
auto *verify_request = static_cast< llarp_async_verify_rc * >(user);
2018-12-29 15:44:25 +00:00
if(verify_request->hook)
verify_request->hook(verify_request);
}
// write it to disk
2018-06-13 11:37:44 +00:00
void
disk_threadworker_setRC(llarp_async_verify_rc *verify_request)
2018-06-13 11:37:44 +00:00
{
2018-08-30 18:48:43 +00:00
verify_request->valid = verify_request->nodedb->Insert(verify_request->rc);
2018-08-10 21:34:11 +00:00
if(verify_request->logic)
2018-12-10 14:14:55 +00:00
verify_request->logic->queue_job(
{verify_request, &logic_threadworker_callback});
}
// we run the crypto verify in the crypto threadpool worker
2018-06-13 11:37:44 +00:00
void
crypto_threadworker_verifyrc(void *user)
{
2019-07-30 23:42:13 +00:00
auto *verify_request = static_cast< llarp_async_verify_rc * >(user);
2018-09-06 11:46:19 +00:00
llarp::RouterContact rc = verify_request->rc;
verify_request->valid = rc.Verify(llarp::time_now_ms());
// if it's valid we need to set it
2018-09-06 11:46:19 +00:00
if(verify_request->valid && rc.IsPublicRouter())
{
2020-01-12 12:29:36 +00:00
if(verify_request->diskworker)
{
llarp::LogDebug("RC is valid, saving to disk");
verify_request->diskworker->addJob(
std::bind(&disk_threadworker_setRC, verify_request));
return;
}
}
2020-01-12 12:29:36 +00:00
// callback to logic thread
verify_request->logic->queue_job(
{verify_request, &logic_threadworker_callback});
}
2018-06-13 12:58:51 +00:00
void
nodedb_inform_load_rc(void *user)
{
2019-07-30 23:42:13 +00:00
auto *job = static_cast< llarp_async_load_rc * >(user);
2018-06-13 12:58:51 +00:00
job->hook(job);
}
2020-01-12 12:29:36 +00:00
void
llarp_nodedb_async_verify(struct llarp_async_verify_rc *job)
{
job->cryptoworker->addJob(std::bind(&crypto_threadworker_verifyrc, job));
}
2018-06-13 12:58:51 +00:00
void
nodedb_async_load_rc(void *user)
{
2019-07-30 23:42:13 +00:00
auto *job = static_cast< llarp_async_load_rc * >(user);
2018-06-13 12:58:51 +00:00
auto fpath = job->nodedb->getRCFilePath(job->pubkey);
job->loaded = job->nodedb->loadfile(fpath);
if(job->loaded)
{
2018-08-30 18:48:43 +00:00
job->nodedb->Get(job->pubkey, job->result);
2018-06-13 12:58:51 +00:00
}
2018-12-10 14:14:55 +00:00
job->logic->queue_job({job, &nodedb_inform_load_rc});
2018-06-13 12:58:51 +00:00
}
2018-08-31 12:46:54 +00:00
bool
llarp_nodedb::ensure_dir(const char *dir)
{
2018-04-30 16:14:20 +00:00
fs::path path(dir);
std::error_code ec;
if(!fs::exists(dir, ec))
fs::create_directory(path, ec);
2018-04-08 12:18:16 +00:00
if(ec)
return false;
2018-04-08 12:18:16 +00:00
if(!fs::is_directory(path))
return false;
2018-04-30 16:14:20 +00:00
for(const char &ch : skiplist_subdirs)
{
// this seems to be a problem on all targets
2018-08-02 20:50:16 +00:00
// perhaps cpp17::fs is just as screwed-up
// attempting to create a folder with no name
if(!ch)
return true;
2018-05-29 12:15:48 +00:00
std::string p;
p += ch;
fs::path sub = path / p;
2018-04-30 16:14:20 +00:00
fs::create_directory(sub, ec);
if(ec)
return false;
2018-04-08 12:18:16 +00:00
}
2018-04-30 16:14:20 +00:00
return true;
}
ssize_t
llarp_nodedb::LoadAll()
{
return Load(nodePath.c_str());
2018-04-30 16:14:20 +00:00
}
2018-05-30 20:56:47 +00:00
2018-06-19 17:11:24 +00:00
size_t
llarp_nodedb::num_loaded() const
{
absl::ReaderMutexLock l(&access);
return entries.size();
}
2018-11-14 18:02:27 +00:00
bool
llarp_nodedb::select_random_exit(llarp::RouterContact &result)
2018-11-14 18:02:27 +00:00
{
llarp::util::Lock lock(&access);
const auto sz = entries.size();
auto itr = entries.begin();
2018-11-14 18:02:27 +00:00
if(sz < 3)
return false;
auto idx = llarp::randint() % sz;
2018-11-14 18:02:27 +00:00
if(idx)
std::advance(itr, idx - 1);
while(itr != entries.end())
2018-11-14 18:02:27 +00:00
{
if(itr->second.rc.IsExit())
2018-11-14 18:02:27 +00:00
{
result = itr->second.rc;
2018-11-14 18:02:27 +00:00
return true;
}
++itr;
}
// wrap around
itr = entries.begin();
2018-11-14 18:02:27 +00:00
while(idx--)
{
if(itr->second.rc.IsExit())
2018-11-14 18:02:27 +00:00
{
result = itr->second.rc;
2018-11-14 18:02:27 +00:00
return true;
}
++itr;
}
return false;
}
2018-08-31 14:41:04 +00:00
bool
llarp_nodedb::select_random_hop(const llarp::RouterContact &prev,
llarp::RouterContact &result, size_t N)
2018-06-19 17:11:24 +00:00
{
llarp::util::Lock lock(&access);
2018-08-02 20:50:16 +00:00
/// checking for "guard" status for N = 0 is done by caller inside of
/// pathbuilder's scope
size_t sz = entries.size();
2018-09-06 20:31:58 +00:00
if(sz < 3)
2018-08-31 14:41:04 +00:00
return false;
2019-02-07 12:23:02 +00:00
if(!N)
return false;
2019-02-07 12:25:22 +00:00
llarp_time_t now = llarp::time_now_ms();
2019-02-07 12:23:02 +00:00
auto itr = entries.begin();
size_t pos = llarp::randint() % sz;
std::advance(itr, pos);
auto start = itr;
while(itr == entries.end())
2018-06-19 17:11:24 +00:00
{
if(prev.pubkey != itr->second.rc.pubkey)
2018-06-20 12:34:48 +00:00
{
if(itr->second.rc.addrs.size() && !itr->second.rc.IsExpired(now))
2018-09-06 20:31:58 +00:00
{
result = itr->second.rc;
2019-02-07 12:23:02 +00:00
return true;
2018-09-06 20:31:58 +00:00
}
2019-02-07 12:23:02 +00:00
}
itr++;
}
itr = entries.begin();
while(itr != start)
{
if(prev.pubkey != itr->second.rc.pubkey)
2019-02-07 12:23:02 +00:00
{
if(itr->second.rc.addrs.size() && !itr->second.rc.IsExpired(now))
{
result = itr->second.rc;
2018-08-31 14:41:04 +00:00
return true;
}
2019-02-07 12:23:02 +00:00
}
++itr;
2018-06-20 12:34:48 +00:00
}
2019-02-07 12:23:02 +00:00
return false;
2018-06-19 17:11:24 +00:00
}
bool
llarp_nodedb::select_random_hop_excluding(
llarp::RouterContact &result, const std::set< llarp::RouterID > &exclude)
{
llarp::util::Lock lock(&access);
/// checking for "guard" status for N = 0 is done by caller inside of
/// pathbuilder's scope
const size_t sz = entries.size();
if(sz < 3)
{
return false;
}
llarp_time_t now = llarp::time_now_ms();
auto itr = entries.begin();
size_t pos = llarp::randint() % sz;
std::advance(itr, pos);
auto start = itr;
while(itr == entries.end())
{
if(exclude.count(itr->first) == 0)
{
if(itr->second.rc.addrs.size() && !itr->second.rc.IsExpired(now))
{
result = itr->second.rc;
return true;
}
}
itr++;
}
itr = entries.begin();
while(itr != start)
{
if(exclude.count(itr->first) == 0)
{
if(itr->second.rc.addrs.size() && !itr->second.rc.IsExpired(now))
{
result = itr->second.rc;
return true;
}
}
++itr;
}
return false;
}