Archives
/
lokinet
mirror of https://github.com/oxen-io/lokinet
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

add conf.d directory for config overrides (#1410)

Browse Source 
* add conf.d directory for config overrides

* refactor llarp::Config

* add explicit constructor with datadir as parameter
* have all config files be passed as std::optional
* make Config::LoadDefault private and use std::optional in Config::Load to remove ambiguity
* update rest of codebase to reflect above changes

* fix pybind

* rename bootstrap config skipBootstrap to seednode as it's more descriptive
* make seednode configurable
* make pybind layer compile
* make pybind layer run
pull/1418/head
Jeff 4 years ago committed by GitHub
parent 22acf0a537
commit 12eb32a816
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 154 additions and 99 deletions
Show Stats Download Patch File Download Diff File

33
daemon/main.cpp
Unescape Escape View File

@ -250,20 +250,25 @@ uninstall_win32_daemon()
/// this sets up, configures and runs the main context
static void
run_main_context(const fs::path confFile, const llarp::RuntimeOptions opts)
run_main_context(std::optional<fs::path> confFile, const llarp::RuntimeOptions opts)
{
try
{
// this is important, can downgrade from Info though
llarp::LogDebug("Running from: ", fs::current_path().string());
llarp::LogInfo("Using config file: ", confFile);
llarp::Config conf;
if (!conf.Load(confFile, opts.isRouter, confFile.parent_path()))
std::unique_ptr<llarp::Config> conf;
if (confFile.has_value())
{
llarp::LogInfo("Using config file: ", *confFile);
conf = std::make_unique<llarp::Config>(confFile->parent_path());
}
else
{
conf = std::make_unique<llarp::Config>(llarp::GetDefaultDataDir());
}
if (!conf->Load(confFile, opts.isRouter))
throw std::runtime_error{"Config file parsing failed"};
ctx = std::make_shared<llarp::Context>();
ctx->Configure(conf);
ctx->Configure(*conf);
signal(SIGINT, handle_signal);
signal(SIGTERM, handle_signal);
@ -350,7 +355,7 @@ lokinet_main(int argc, char* argv[])
bool genconfigOnly = false;
bool overwrite = false;
fs::path configFile;
std::optional<fs::path> configFile;
try
{
auto result = options.parse(argc, argv);
@ -427,21 +432,21 @@ lokinet_main(int argc, char* argv[])
return 1;
}
if (!configFile.empty())
if (configFile.has_value())
{
// when we have an explicit filepath
fs::path basedir = configFile.parent_path();
fs::path basedir = configFile->parent_path();
if (genconfigOnly)
{
llarp::ensureConfig(basedir, configFile, overwrite, opts.isRouter);
llarp::ensureConfig(basedir, *configFile, overwrite, opts.isRouter);
}
else
{
std::error_code ec;
if (!fs::exists(configFile, ec))
if (!fs::exists(*configFile, ec))
{
llarp::LogError("Config file not found ", configFile);
llarp::LogError("Config file not found ", *configFile);
return 1;
}

76
llarp/config/config.cpp
Unescape Escape View File

@ -892,6 +892,14 @@ namespace llarp
{
(void)params;
conf.defineOption<bool>(
"bootstrap",
"seed-node",
Default{false},
Comment{"Whether or not to run as a seed node. We will not have any bootstrap routers "
"configured."},
AssignmentAcceptor(seednode));
conf.defineOption<std::string>(
"bootstrap",
"add-node",
@ -973,35 +981,62 @@ namespace llarp
});
}
Config::Config(fs::path datadir) : m_DataDir(std::move(datadir))
{}
constexpr auto GetOverridesDir = [](auto datadir) -> fs::path { return datadir / "conf.d"; };
void
Config::Save()
{
const auto overridesDir = GetOverridesDir(m_DataDir);
if (not fs::exists(overridesDir))
fs::create_directory(overridesDir);
m_Parser.Save();
}
void
Config::Override(std::string section, std::string key, std::string value)
{
m_Parser.AddOverride(std::move(section), std::move(key), std::move(value));
m_Parser.AddOverride(GetOverridesDir(m_DataDir) / "overrides.ini", section, key, value);
}
void
Config::LoadOverrides()
{
const auto overridesDir = GetOverridesDir(m_DataDir);
if (fs::exists(overridesDir))
{
util::IterDir(overridesDir, [&](const fs::path& overrideFile) {
if (overrideFile.extension() == ".ini")
{
m_Parser.LoadFile(overrideFile);
}
return true;
});
}
}
bool
Config::Load(const fs::path fname, bool isRelay, fs::path defaultDataDir)
Config::Load(std::optional<fs::path> fname, bool isRelay)
{
if (not fname.has_value())
return LoadDefault(isRelay);
try
{
ConfigGenParameters params;
params.isRelay = isRelay;
params.defaultDataDir = std::move(defaultDataDir);
params.defaultDataDir = m_DataDir;
ConfigDefinition conf{isRelay};
initializeConfig(conf, params);
addBackwardsCompatibleConfigOptions(conf);
m_Parser.Clear();
if (!m_Parser.LoadFile(fname))
if (!m_Parser.LoadFile(*fname))
{
return false;
}
LoadOverrides();
m_Parser.IterAll([&](std::string_view section, const SectionValues_t& values) {
for (const auto& pair : values)
@ -1012,10 +1047,6 @@ namespace llarp
conf.acceptAllOptions();
// TODO: better way to support inter-option constraints
if (router.m_maxConnectedRouters < router.m_minConnectedRouters)
throw std::invalid_argument("[router]:min-connections must be <= [router]:max-connections");
return true;
}
catch (const std::exception& e)
@ -1026,17 +1057,26 @@ namespace llarp
}
bool
Config::LoadDefault(bool isRelay, fs::path dataDir)
Config::LoadDefault(bool isRelay)
{
try
{
ConfigGenParameters params;
params.isRelay = isRelay;
params.defaultDataDir = std::move(dataDir);
params.defaultDataDir = m_DataDir;
ConfigDefinition conf{isRelay};
initializeConfig(conf, params);
m_Parser.Clear();
LoadOverrides();
m_Parser.IterAll([&](std::string_view section, const SectionValues_t& values) {
for (const auto& pair : values)
{
conf.addConfigValue(section, pair.first, pair.second);
}
});
conf.acceptAllOptions();
return true;
@ -1103,12 +1143,12 @@ namespace llarp
llarp::LogInfo("Attempting to create config file, asRouter: ", asRouter, " path: ", confFile);
llarp::Config config;
llarp::Config config{defaultDataDir};
std::string confStr;
if (asRouter)
confStr = config.generateBaseRouterConfig(std::move(defaultDataDir));
confStr = config.generateBaseRouterConfig();
else
confStr = config.generateBaseClientConfig(std::move(defaultDataDir));
confStr = config.generateBaseClientConfig();
// open a filestream
auto stream = llarp::util::OpenFileStream<std::ofstream>(confFile.c_str(), std::ios::binary);
@ -1168,11 +1208,11 @@ namespace llarp
}
std::string
Config::generateBaseClientConfig(fs::path defaultDataDir)
Config::generateBaseClientConfig()
{
ConfigGenParameters params;
params.isRelay = false;
params.defaultDataDir = std::move(defaultDataDir);
params.defaultDataDir = m_DataDir;
llarp::ConfigDefinition def{false};
initializeConfig(def, params);
@ -1188,11 +1228,11 @@ namespace llarp
}
std::string
Config::generateBaseRouterConfig(fs::path defaultDataDir)
Config::generateBaseRouterConfig()
{
ConfigGenParameters params;
params.isRelay = true;
params.defaultDataDir = std::move(defaultDataDir);
params.defaultDataDir = m_DataDir;
llarp::ConfigDefinition def{true};
initializeConfig(def, params);

39
llarp/config/config.hpp
Unescape Escape View File

@ -166,8 +166,7 @@ namespace llarp
struct BootstrapConfig
{
std::vector<fs::path> routers;
/// for unit tests
bool skipBootstrap = false;
bool seednode;
void
defineConfigOptions(ConfigDefinition& conf, const ConfigGenParameters& params);
};
@ -184,6 +183,10 @@ namespace llarp
struct Config
{
explicit Config(fs::path datadir);
~Config() = default;
RouterConfig router;
NetworkConfig network;
ConnectConfig connect;
@ -205,10 +208,23 @@ namespace llarp
void
addBackwardsCompatibleConfigOptions(ConfigDefinition& conf);
// Load a config from the given file
// Load a config from the given file if the config file is not provided LoadDefault is called
bool
Load(const fs::path fname, bool isRelay, fs::path defaultDataDir);
Load(std::optional<fs::path> fname = std::nullopt, bool isRelay = false);
std::string
generateBaseClientConfig();
std::string
generateBaseRouterConfig();
void
Save();
void
Override(std::string section, std::string key, std::string value);
private:
/// Load (initialize) a default config.
///
/// This delegates to the ConfigDefinition to generate a default config,
@ -221,22 +237,13 @@ namespace llarp
/// @param dataDir is a path representing a directory to be used as the data dir
/// @return true on success, false otherwise
bool
LoadDefault(bool isRelay, fs::path dataDir);
std::string
generateBaseClientConfig(fs::path defaultDataDir);
std::string
generateBaseRouterConfig(fs::path defaultDataDir);
LoadDefault(bool isRelay);
void
Save();
LoadOverrides();
void
Override(std::string section, std::string key, std::string value);
private:
ConfigParser m_Parser;
const fs::path m_DataDir;
};
void

24
llarp/config/ini.cpp
Unescape Escape View File

@ -142,28 +142,26 @@ namespace llarp
}
void
ConfigParser::AddOverride(std::string section, std::string key, std::string value)
ConfigParser::AddOverride(fs::path fpath, std::string section, std::string key, std::string value)
{
m_Overrides[section].emplace(key, value);
auto& data = m_Overrides[fpath];
data[section].emplace(key, value);
}
void
ConfigParser::Save()
{
// if we have no overrides keep the config the same on disk
if (m_Overrides.empty())
return;
std::ofstream ofs(m_FileName);
// write existing config data
ofs.write(m_Data.data(), m_Data.size());
// write overrides
ofs << std::endl << std::endl << "# overrides" << std::endl;
for (const auto& [section, values] : m_Overrides)
for (const auto& [fname, overrides] : m_Overrides)
{
ofs << std::endl << "[" << section << "]" << std::endl;
for (const auto& [key, value] : values)
std::ofstream ofs(fname);
for (const auto& [section, values] : overrides)
{
ofs << key << "=" << value << std::endl;
ofs << std::endl << "[" << section << "]" << std::endl;
for (const auto& [key, value] : values)
{
ofs << key << "=" << value << std::endl;
}
}
}
m_Overrides.clear();

8
llarp/config/ini.hpp
Unescape Escape View File

@ -40,11 +40,11 @@ namespace llarp
bool
VisitSection(const char* name, std::function<bool(const SectionValues_t&)> visit) const;
/// add a config option that is appended at the end of the config buffer with no comments
/// add a config option that is appended in another file
void
AddOverride(std::string section, std::string key, std::string value);
AddOverride(fs::path file, std::string section, std::string key, std::string value);
/// save config and any overrides to the file it was loaded from
/// save config overrides
void
Save();
@ -54,7 +54,7 @@ namespace llarp
std::vector<char> m_Data;
Config_impl_t m_Config;
Config_impl_t m_Overrides;
std::unordered_map<fs::path, Config_impl_t, util::FileHash> m_Overrides;
fs::path m_FileName;
};

3
llarp/constants/files.hpp
Unescape Escape View File

@ -22,6 +22,7 @@ namespace llarp
{
#ifdef _WIN32
const fs::path homedir = getenv("APPDATA");
return homedir / "lokinet";
#else
fs::path homedir;
@ -36,8 +37,8 @@ namespace llarp
homedir = "/var/lib/lokinet";
return homedir;
}
#endif
return homedir / ".lokinet";
#endif
}
inline fs::path

2
llarp/router/router.cpp
Unescape Escape View File

@ -496,7 +496,7 @@ namespace llarp
{
configRouters.push_back(defaultBootstrapFile);
}
else if (not conf.bootstrap.skipBootstrap)
else if (not conf.bootstrap.seednode)
{
LogError("No bootstrap files specified in config file, and the default");
LogError("bootstrap file ", defaultBootstrapFile, " does not exist.");

10
llarp/util/fs.hpp
Unescape Escape View File

@ -27,6 +27,16 @@ namespace llarp
{
namespace util
{
struct FileHash
{
size_t
operator()(const fs::path& f) const
{
std::hash<std::string> h;
return h(f.string());
}
};
using error_code_t = std::error_code;
/// Ensure that a file exists and has correct permissions

8
pybind/llarp/config.cpp
Unescape Escape View File

@ -16,7 +16,7 @@ namespace llarp
{
using Config_ptr = std::shared_ptr<Config>;
py::class_<Config, Config_ptr>(mod, "Config")
.def(py::init<>())
.def(py::init<std::string>())
.def_readwrite("router", &Config::router)
.def_readwrite("network", &Config::network)
.def_readwrite("connect", &Config::connect)
@ -25,10 +25,7 @@ namespace llarp
.def_readwrite("lokid", &Config::lokid)
.def_readwrite("bootstrap", &Config::bootstrap)
.def_readwrite("logging", &Config::logging)
.def("LoadFile", &Config::Load)
.def("LoadDefault", [](Config& self, bool isRelay, std::string dir) {
return self.LoadDefault(isRelay, dir);
});
.def("Load", &Config::Load);
py::class_<RouterConfig>(mod, "RouterConfig")
.def(py::init<>())
@ -106,6 +103,7 @@ namespace llarp
py::class_<BootstrapConfig>(mod, "BootstrapConfig")
.def(py::init<>())
.def_readwrite("seednode", &BootstrapConfig::seednode)
.def_property(
"routers",
[](BootstrapConfig& self) {

15
test/crypto/test_llarp_key_manager.cpp
Unescape Escape View File

@ -112,8 +112,8 @@ TEST_F(KeyManagerTest, TestBackupFileByMoving_FailsIfBackupNamesAreExausted)
TEST_F(KeyManagerTest, TestInitialize_MakesKeyfiles)
{
llarp::Config conf;
conf.LoadDefault(false, {});
llarp::Config conf{fs::current_path()};
conf.Load();
KeyManager keyManager;
ASSERT_TRUE(keyManager.initialize(conf, true, true));
@ -128,9 +128,9 @@ TEST_F(KeyManagerTest, TestInitialize_MakesKeyfiles)
TEST_F(KeyManagerTest, TestInitialize_RespectsGenFlag)
{
llarp::Config conf;
conf.LoadDefault(false, {});
llarp::Config conf{fs::current_path()};
conf.Load();
KeyManager keyManager;
ASSERT_FALSE(keyManager.initialize(conf, false, true));
@ -143,8 +143,9 @@ TEST_F(KeyManagerTest, TestInitialize_RespectsGenFlag)
TEST_F(KeyManagerTest, TestInitialize_DetectsBadRcFile)
{
llarp::Config conf;
conf.LoadDefault(false, {});
llarp::Config conf{fs::current_path()};
conf.Load();
conf.lokid.whitelistRouters = false;
std::fstream f;

6
test/exit/test_llarp_exit_context.cpp
Unescape Escape View File

@ -11,12 +11,12 @@ static const llarp::RuntimeOptions opts = {.background = false, .debug = false,
std::shared_ptr<llarp::Context>
make_context()
{
llarp::Config conf{};
conf.LoadDefault(true, fs::current_path());
llarp::Config conf{fs::current_path()};
conf.Load(std::nullopt, true);
// set testing defaults
conf.network.m_endpointType = "null";
conf.bootstrap.skipBootstrap = true;
conf.bootstrap.seednode = true;
conf.api.m_enableRPCServer = false;
conf.lokid.whitelistRouters = false;
conf.router.m_publicAddress = llarp::IpAddress("1.1.1.1");

23
test/hive/hive.py
Unescape Escape View File

@ -48,14 +48,13 @@ class RouterHive(object):
print("not removing dir %s because it doesn't start with /tmp/" % self.tmpdir)
return False
def AddRelay(self, index):
dirname = "%s/relays/%d" % (self.tmpdir, index)
makedirs("%s/nodedb" % dirname, exist_ok=True)
config = pyllarp.Config()
config.LoadDefault(True, dirname);
config = pyllarp.Config(dirname)
config.Load(None, True)
port = index + 30000
tunname = "lokihive%d" % index
@ -65,34 +64,31 @@ class RouterHive(object):
config.router.nickname = "Router%d" % index
config.router.overrideAddress('127.0.0.1:{}'.format(port))
config.router.blockBogons = False
config.router.enablePeerStats = True
config.network.enableProfiling = False
config.network.routerProfilesFile = "%s/profiles.dat" % dirname
config.network.endpointType = 'null'
config.links.addInboundLink("lo", AF_INET, port);
config.links.setOutboundLink("lo", AF_INET, port + 10000);
# config.dns.options = {"local-dns": ("127.3.2.1:%d" % port)}
if index != 0:
if index == 0:
config.bootstrap.seednode = True
else:
config.bootstrap.routers = ["%s/relays/0/self.signed" % self.tmpdir]
config.api.enableRPCServer = False
config.lokid.whitelistRouters = False
print("adding relay at index %d" % port);
print("adding relay at index %d" % index)
self.hive.AddRelay(config)
def AddClient(self, index):
dirname = "%s/clients/%d" % (self.tmpdir, index)
makedirs("%s/nodedb" % dirname, exist_ok=True)
config = pyllarp.Config()
config.LoadDefault(False, dirname);
config = pyllarp.Config(dirname)
config.Load(None, False);
port = index + 50000
tunname = "lokihive%d" % index
@ -102,7 +98,6 @@ class RouterHive(object):
config.router.blockBogons = False
config.network.enableProfiling = False
config.network.routerProfilesFile = "%s/profiles.dat" % dirname
config.network.endpointType = 'null'
config.links.setOutboundLink("lo", AF_INET, port + 10000);

6
test/regress/2020-06-08-key-backup-bug.cpp
Unescape Escape View File

@ -11,11 +11,11 @@ llarp::RuntimeOptions opts = {false, false, false};
static std::shared_ptr<llarp::Context>
make_context(std::optional<fs::path> keyfile)
{
llarp::Config conf;
conf.LoadDefault(opts.isRouter, {});
llarp::Config conf{fs::current_path()};
conf.Load(std::nullopt, opts.isRouter);
conf.network.m_endpointType = "null";
conf.network.m_keyfile = keyfile;
conf.bootstrap.skipBootstrap = true;
conf.bootstrap.seednode = true;
conf.api.m_enableRPCServer = false;
auto context = std::make_shared<llarp::Context>();

Write Preview
Loading…
Cancel
Save