Archives
/
i2pd
mirror of https://github.com/PurpleI2P/i2pd
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

* i2p::fs migration: NetDb.*

Browse Source
pull/388/head
hagen 8 years ago
parent 6f4271c054
commit 6d74493491
2 changed files with 73 additions and 135 deletions
Show Stats Download Patch File Download Diff File

203
NetDb.cpp
Unescape Escape View File

@ -1,12 +1,13 @@
#include <string.h>
#include "I2PEndian.h"
#include <fstream>
#include <vector>
#include <boost/asio.hpp>
#include <openssl/rand.h>
#include <zlib.h>
#include "I2PEndian.h"
#include "Base.h"
#include "Log.h"
#include "FS.h"
#include "Timestamp.h"
#include "I2NPProtocol.h"
#include "Tunnel.h"
@ -22,7 +23,6 @@ namespace i2p
{
namespace data
{
const char NetDb::m_NetDbPath[] = "netDb";
NetDb netdb;
NetDb::NetDb (): m_IsRunning (false), m_Thread (nullptr), m_Reseeder (nullptr)
@ -250,30 +250,6 @@ namespace data
return it->second->SetUnreachable (unreachable);
}
// TODO: Move to reseed and/or scheduled tasks. (In java version, scheduler fix this as well as sort RIs.)
bool NetDb::CreateNetDb(boost::filesystem::path directory)
{
LogPrint (eLogInfo, "NetDb: storage directory doesn't exist, trying to create it.");
if (!boost::filesystem::create_directory (directory))
{
LogPrint (eLogError, "NetDb: failed to create directory ", directory);
return false;
}
// list of chars might appear in base64 string
const char * chars = GetBase64SubstitutionTable (); // 64 bytes
for (int i = 0; i < 64; i++)
{
auto p = directory / (std::string ("r") + chars[i]);
if (!boost::filesystem::exists (p) && !boost::filesystem::create_directory (p))
{
LogPrint (eLogError, "NetDb: failed to create directory ", p);
return false;
}
}
return true;
}
void NetDb::Reseed ()
{
if (!m_Reseeder)
@ -288,145 +264,108 @@ namespace data
LogPrint (eLogWarning, "NetDb: failed to reseed after 10 attempts");
}
void NetDb::Load ()
bool NetDb::LoadRouterInfo (const std::string & path)
{
boost::filesystem::path p(i2p::util::filesystem::GetDataDir() / m_NetDbPath);
if (!boost::filesystem::exists (p))
auto r = std::make_shared<RouterInfo>(path);
if (r->GetRouterIdentity () && !r->IsUnreachable () &&
(!r->UsesIntroducer () || m_LastLoad < r->GetTimestamp () + 3600*1000LL)) // 1 hour
{
// seems netDb doesn't exist yet
if (!CreateNetDb(p)) return;
r->DeleteBuffer ();
r->ClearProperties (); // properties are not used for regular routers
m_RouterInfos[r->GetIdentHash ()] = r;
if (r->IsFloodfill ())
m_Floodfills.push_back (r);
} else {
LogPrint(eLogWarning, "NetDb: Can't load RI from ", path, ", delete");
i2p::fs::Remove(path);
}
return true;
}
void NetDb::Load ()
{
// make sure we cleanup netDb from previous attempts
m_RouterInfos.clear ();
m_Floodfills.clear ();
// load routers now
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch ();
int numRouters = 0;
boost::filesystem::directory_iterator end;
for (boost::filesystem::directory_iterator it (p); it != end; ++it)
{
if (boost::filesystem::is_directory (it->status()))
{
for (boost::filesystem::directory_iterator it1 (it->path ()); it1 != end; ++it1)
{
#if BOOST_VERSION > 10500
const std::string& fullPath = it1->path().string();
#else
const std::string& fullPath = it1->path();
#endif
auto r = std::make_shared<RouterInfo>(fullPath);
if (r->GetRouterIdentity () && !r->IsUnreachable () &&
(!r->UsesIntroducer () || ts < r->GetTimestamp () + 3600*1000LL)) // 1 hour
{
r->DeleteBuffer ();
r->ClearProperties (); // properties are not used for regular routers
m_RouterInfos[r->GetIdentHash ()] = r;
if (r->IsFloodfill ())
m_Floodfills.push_back (r);
numRouters++;
}
else
{
if (boost::filesystem::exists (fullPath))
boost::filesystem::remove (fullPath);
}
}
}
}
LogPrint (eLogInfo, "NetDb: ", numRouters, " routers loaded (", m_Floodfills.size (), " floodfils)");
m_LastLoad = i2p::util::GetSecondsSinceEpoch();
std::vector<std::string> files;
i2p::fs::GetNetDB().Traverse(files);
for (auto path : files)
LoadRouterInfo(path);
LogPrint (eLogInfo, "NetDb: ", m_RouterInfos.size(), " routers loaded (", m_Floodfills.size (), " floodfils)");
}
void NetDb::SaveUpdated ()
{
auto GetFilePath = [](const boost::filesystem::path& directory, const RouterInfo * routerInfo)
{
std::string s(routerInfo->GetIdentHashBase64());
return directory / (std::string("r") + s[0]) / ("routerInfo-" + s + ".dat");
};
boost::filesystem::path fullDirectory (i2p::util::filesystem::GetDataDir() / m_NetDbPath);
int count = 0, deletedCount = 0;
int updatedCount = 0, deletedCount = 0;
auto total = m_RouterInfos.size ();
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch ();
uint64_t ts = i2p::util::GetMillisecondsSinceEpoch();
for (auto it: m_RouterInfos)
{
if (it.second->IsUpdated ())
{
std::string f = GetFilePath(fullDirectory, it.second.get()).string();
it.second->SaveToFile (f);
std::string ident = it.second->GetIdentHashBase64();
std::string path = i2p::fs::GetNetDB().Path(ident);
if (it.second->IsUpdated ()) {
it.second->SaveToFile (path);
it.second->SetUpdated (false);
it.second->SetUnreachable (false);
it.second->DeleteBuffer ();
count++;
updatedCount++;
continue;
}
else
{
// find & mark unreachable routers
if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) {
// RouterInfo expires after 1 hour if uses introducer
if (it.second->UsesIntroducer () && ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hour
it.second->SetUnreachable (true);
else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) // routers don't expire if less than 25 or uptime is less than 10 minutes
{
if (i2p::context.IsFloodfill ())
{
if (ts > it.second->GetTimestamp () + 3600*1000LL) // 1 hours
{
it.second->SetUnreachable (true);
total--;
}
it.second->SetUnreachable (true);
} else if (total > 75 && ts > (i2p::context.GetStartupTime () + 600)*1000LL) {
// routers don't expire if less than 25 or uptime is less than 10 minutes
if (i2p::context.IsFloodfill ()) {
if (ts > it.second->GetTimestamp () + 3600*1000LL) { // 1 hour
it.second->SetUnreachable (true);
total--;
}
else if (total > 300)
{
if (ts > it.second->GetTimestamp () + 30*3600*1000LL) // 30 hours
{
it.second->SetUnreachable (true);
total--;
}
} else if (total > 300) {
if (ts > it.second->GetTimestamp () + 30*3600*1000LL) { // 30 hours
it.second->SetUnreachable (true);
total--;
}
else if (total > 120)
{
if (ts > it.second->GetTimestamp () + 72*3600*1000LL) // 72 hours
{
it.second->SetUnreachable (true);
total--;
}
} else if (total > 120) {
if (ts > it.second->GetTimestamp () + 72*3600*1000LL) { // 72 hours
it.second->SetUnreachable (true);
total--;
}
}
if (it.second->IsUnreachable ())
{
total--;
// delete RI file
if (boost::filesystem::exists (GetFilePath (fullDirectory, it.second.get ())))
{
boost::filesystem::remove (GetFilePath (fullDirectory, it.second.get ()));
deletedCount++;
}
// delete from floodfills list
if (it.second->IsFloodfill ())
{
std::unique_lock<std::mutex> l(m_FloodfillsMutex);
m_Floodfills.remove (it.second);
}
}
if (it.second->IsUnreachable ()) {
total--;
// delete RI file
i2p::fs::GetNetDB().Remove(ident);
deletedCount++;
// delete from floodfills list
if (it.second->IsFloodfill ()) {
std::unique_lock<std::mutex> l(m_FloodfillsMutex);
m_Floodfills.remove (it.second);
}
}
}
if (count > 0)
LogPrint (eLogInfo, "NetDb: ", count, " new/updated routers saved");
}
} // m_RouterInfos iteration
if (updatedCount > 0)
LogPrint (eLogInfo, "NetDb: saved ", updatedCount, " new/updated routers");
if (deletedCount > 0)
{
LogPrint (eLogDebug, "NetDb: ", deletedCount, " routers deleted");
LogPrint (eLogInfo, "NetDb: deleting ", deletedCount, " unreachable routers");
// clean up RouterInfos table
std::unique_lock<std::mutex> l(m_RouterInfosMutex);
for (auto it = m_RouterInfos.begin (); it != m_RouterInfos.end ();)
{
if (it->second->IsUnreachable ())
{
if (it->second->IsUnreachable ()) {
it->second->SaveProfile ();
it = m_RouterInfos.erase (it);
continue;
}
else
it++;
it++;
}
}
}

5
NetDb.h
Unescape Escape View File

@ -8,7 +8,6 @@
#include <string>
#include <thread>
#include <mutex>
#include <boost/filesystem.hpp>
#include "Base.h"
#include "Queue.h"
#include "I2NPProtocol.h"
@ -23,7 +22,6 @@ namespace i2p
{
namespace data
{
class NetDb
{
public:
@ -69,8 +67,8 @@ namespace data
private:
bool CreateNetDb(boost::filesystem::path directory);
void Load ();
bool LoadRouterInfo (const std::string & path);
void SaveUpdated ();
void Run (); // exploratory thread
void Explore (int numDestinations);
@ -90,6 +88,7 @@ namespace data
std::list<std::shared_ptr<RouterInfo> > m_Floodfills;
bool m_IsRunning;
uint64_t m_LastLoad;
std::thread * m_Thread;
i2p::util::Queue<std::shared_ptr<const I2NPMessage> > m_Queue; // of I2NPDatabaseStoreMsg

Write Preview
Loading…
Cancel
Save