Archives/i2pd
2
0
Fork 0
mirror of https://github.com/PurpleI2P/i2pd.git synced 2024-11-16 00:12:43 +00:00
Code Issues Projects Releases Wiki Activity

moved save/delete profile disk oprations to separate threads

Browse Source
This commit is contained in:
orignal 2024-04-27 16:00:43 -04:00
parent 648a884a18
commit 733a4a2869
4 changed files with 72 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
libi2pd/NetDb.cpp
View File

@ -107,8 +107,8 @@ namespace data
i2p::util::SetThreadName("NetDB");
uint64_t lastManage = 0, lastExploratory = 0, lastManageRequest = 0;
uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds ();
int16_t profilesCleanupVariance = 0;
uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds (), lastObsoleteProfilesCleanup = lastProfilesCleanup;
int16_t profilesCleanupVariance = 0, obsoleteProfilesCleanVariance = 0;
while (m_IsRunning)
{
@ -171,12 +171,39 @@ namespace data
if (mts >= lastProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_AUTOCLEAN_TIMEOUT + profilesCleanupVariance)*1000)
{
m_RouterProfilesPool.CleanUpMt ();
if (m_PersistProfiles) PersistProfiles ();
DeleteObsoleteProfiles ();
if (m_PersistProfiles)
{
bool isSaving = m_SavingProfiles.valid ();
if (isSaving && m_SavingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
{
m_SavingProfiles.get ();
isSaving = false;
}
if (!isSaving)
m_SavingProfiles = PersistProfiles ();
else
LogPrint (eLogWarning, "NetDb: Can't persist profiles. Profiles are being saved to disk");
}
lastProfilesCleanup = mts;
profilesCleanupVariance = (rand () % (2 * i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE) - i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE);
}
if (mts >= lastObsoleteProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT + obsoleteProfilesCleanVariance)*1000)
{
bool isDeleting = m_DeletingProfiles.valid ();
if (isDeleting && m_DeletingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active?
{
m_DeletingProfiles.get ();
isDeleting = false;
}
if (!isDeleting)
m_DeletingProfiles = DeleteObsoleteProfiles ();
else
LogPrint (eLogWarning, "NetDb: Can't delete profiles. Profiles are being deleted from disk");
lastObsoleteProfilesCleanup = mts;
obsoleteProfilesCleanVariance = (rand () % (2 * i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE) - i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT);
}
if (mts >= lastExploratory + 30000) // exploratory every 30 seconds
{
auto numRouters = m_RouterInfos.size ();

2
libi2pd/NetDb.hpp
View File

@ -15,6 +15,7 @@
#include <string>
#include <thread>
#include <mutex>
#include <future>
#include "Base.h"
#include "Gzip.h"
@ -177,6 +178,7 @@ namespace data
NetDbRequests m_Requests;
bool m_PersistProfiles;
std::future<void> m_SavingProfiles, m_DeletingProfiles;
/** router info we are bootstrapping from or nullptr if we are not currently doing that*/
std::shared_ptr<RouterInfo> m_FloodfillBootstrap;

51
libi2pd/Profiling.cpp
View File

@ -260,7 +260,13 @@ namespace data
g_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64);
}
void PersistProfiles ()
static void SaveProfilesToDisk (std::list<std::pair<i2p::data::IdentHash, std::shared_ptr<RouterProfile> > >&& profiles)
{
for (auto& it: profiles)
if (it.second) it.second->Save (it.first);
}
std::future<void> PersistProfiles ()
{
auto ts = GetTime ();
std::list<std::pair<i2p::data::IdentHash, std::shared_ptr<RouterProfile> > > tmp;
@ -278,8 +284,9 @@ namespace data
it++;
}
}
for (auto& it: tmp)
if (it.second) it.second->Save (it.first);
if (!tmp.empty ())
return std::async (std::launch::async, SaveProfilesToDisk, std::move (tmp));
return std::future<void>();
}
void SaveProfiles ()
@ -287,8 +294,7 @@ namespace data
std::unordered_map<i2p::data::IdentHash, std::shared_ptr<RouterProfile> > tmp;
{
std::unique_lock<std::mutex> l(g_ProfilesMutex);
tmp = g_Profiles;
g_Profiles.clear ();
std::swap (tmp, g_Profiles);
}
auto ts = GetTime ();
for (auto& it: tmp)
@ -296,7 +302,26 @@ namespace data
it.second->Save (it.first);
}
void DeleteObsoleteProfiles ()
static void DeleteFilesFromDisk (std::vector<std::string>&& files)
{
struct stat st;
std::time_t now = std::time(nullptr);
for (const auto& path: files)
{
if (stat(path.c_str(), &st) != 0)
{
LogPrint(eLogWarning, "Profiling: Can't stat(): ", path);
continue;
}
if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600)
{
LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path);
i2p::fs::Remove(path);
}
}
}
std::future<void> DeleteObsoleteProfiles ()
{
{
auto ts = GetTime ();
@ -310,21 +335,9 @@ namespace data
}
}
struct stat st;
std::time_t now = std::time(nullptr);
std::vector<std::string> files;
g_ProfilesStorage.Traverse(files);
for (const auto& path: files) {
if (stat(path.c_str(), &st) != 0) {
LogPrint(eLogWarning, "Profiling: Can't stat(): ", path);
continue;
}
if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) {
LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path);
i2p::fs::Remove(path);
}
}
return std::async (std::launch::async, DeleteFilesFromDisk, std::move (files));
}
}
}

11
libi2pd/Profiling.h
View File

@ -10,6 +10,7 @@
#define PROFILING_H__
#include <memory>
#include <future>
#include <boost/date_time/posix_time/posix_time.hpp>
#include "Identity.h"
@ -31,8 +32,10 @@ namespace data
const char PEER_PROFILE_USAGE_CONNECTED[] = "connected";
const int PEER_PROFILE_EXPIRATION_TIMEOUT = 36; // in hours (1.5 days)
const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 3 * 3600; // in seconds (3 hours)
const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 3600; // in seconds (1 hour)
const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 1500; // in seconds (25 minutes)
const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 900; // in seconds (15 minutes)
const int PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT = 5400; // in seconds (1.5 hours)
const int PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE = 2400; // in seconds (40 minutes)
const int PEER_PROFILE_DECLINED_RECENTLY_INTERVAL = 150; // in seconds (2.5 minutes)
const int PEER_PROFILE_PERSIST_INTERVAL = 3300; // in seconds (55 minutes)
const int PEER_PROFILE_UNREACHABLE_INTERVAL = 480; // in seconds (8 minutes)
@ -89,9 +92,9 @@ namespace data
std::shared_ptr<RouterProfile> GetRouterProfile (const IdentHash& identHash);
bool IsRouterBanned (const IdentHash& identHash); // check only existing profiles
void InitProfilesStorage ();
void DeleteObsoleteProfiles ();
std::future<void> DeleteObsoleteProfiles ();
void SaveProfiles ();
void PersistProfiles ();
std::future<void> PersistProfiles ();
}
}