Archives
/
lokinet
mirror of https://github.com/oxen-io/lokinet
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

add conf.d directory for config overrides (#1410)

Browse Source 
* add conf.d directory for config overrides

* refactor llarp::Config

* add explicit constructor with datadir as parameter
* have all config files be passed as std::optional
* make Config::LoadDefault private and use std::optional in Config::Load to remove ambiguity
* update rest of codebase to reflect above changes

* fix pybind

* rename bootstrap config skipBootstrap to seednode as it's more descriptive
* make seednode configurable
* make pybind layer compile
* make pybind layer run
pull/1418/head
Jeff 4 years ago committed by GitHub
parent 22acf0a537
commit 12eb32a816
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 154 additions and 99 deletions
Show Stats Download Patch File Download Diff File

33
daemon/main.cpp
Unescape Escape View File

@ -250,20 +250,25 @@ uninstall_win32_daemon()
/// this sets up, configures and runs the main context /// this sets up, configures and runs the main context
static void static void
run_main_context(const fs::path confFile, const llarp::RuntimeOptions opts) run_main_context(std::optional<fs::path> confFile, const llarp::RuntimeOptions opts)
{ {
try try
{ {
// this is important, can downgrade from Info though std::unique_ptr<llarp::Config> conf;
llarp::LogDebug("Running from: ", fs::current_path().string()); if (confFile.has_value())
llarp::LogInfo("Using config file: ", confFile); {
llarp::LogInfo("Using config file: ", *confFile);
llarp::Config conf; conf = std::make_unique<llarp::Config>(confFile->parent_path());
if (!conf.Load(confFile, opts.isRouter, confFile.parent_path())) }
else
{
conf = std::make_unique<llarp::Config>(llarp::GetDefaultDataDir());
}
if (!conf->Load(confFile, opts.isRouter))
throw std::runtime_error{"Config file parsing failed"}; throw std::runtime_error{"Config file parsing failed"};
ctx = std::make_shared<llarp::Context>(); ctx = std::make_shared<llarp::Context>();
ctx->Configure(conf); ctx->Configure(*conf);
signal(SIGINT, handle_signal); signal(SIGINT, handle_signal);
signal(SIGTERM, handle_signal); signal(SIGTERM, handle_signal);
@ -350,7 +355,7 @@ lokinet_main(int argc, char* argv[])
bool genconfigOnly = false; bool genconfigOnly = false;
bool overwrite = false; bool overwrite = false;
fs::path configFile; std::optional<fs::path> configFile;
try try
{ {
auto result = options.parse(argc, argv); auto result = options.parse(argc, argv);
@ -427,21 +432,21 @@ lokinet_main(int argc, char* argv[])
return 1; return 1;
} }
if (!configFile.empty()) if (configFile.has_value())
{ {
// when we have an explicit filepath // when we have an explicit filepath
fs::path basedir = configFile.parent_path(); fs::path basedir = configFile->parent_path();
if (genconfigOnly) if (genconfigOnly)
{ {
llarp::ensureConfig(basedir, configFile, overwrite, opts.isRouter); llarp::ensureConfig(basedir, *configFile, overwrite, opts.isRouter);
} }
else else
{ {
std::error_code ec; std::error_code ec;
if (!fs::exists(configFile, ec)) if (!fs::exists(*configFile, ec))
{ {
llarp::LogError("Config file not found ", configFile); llarp::LogError("Config file not found ", *configFile);
return 1; return 1;
} }

76
llarp/config/config.cpp
Unescape Escape View File

@ -892,6 +892,14 @@ namespace llarp
{ {
(void)params; (void)params;
conf.defineOption<bool>(
"bootstrap",
"seed-node",
Default{false},
Comment{"Whether or not to run as a seed node. We will not have any bootstrap routers "
"configured."},
AssignmentAcceptor(seednode));
conf.defineOption<std::string>( conf.defineOption<std::string>(
"bootstrap", "bootstrap",
"add-node", "add-node",
@ -973,35 +981,62 @@ namespace llarp
}); });
} }
Config::Config(fs::path datadir) : m_DataDir(std::move(datadir))
{}
constexpr auto GetOverridesDir = [](auto datadir) -> fs::path { return datadir / "conf.d"; };
void void
Config::Save() Config::Save()
{ {
const auto overridesDir = GetOverridesDir(m_DataDir);
if (not fs::exists(overridesDir))
fs::create_directory(overridesDir);
m_Parser.Save(); m_Parser.Save();
} }
void void
Config::Override(std::string section, std::string key, std::string value) Config::Override(std::string section, std::string key, std::string value)
{ {
m_Parser.AddOverride(std::move(section), std::move(key), std::move(value)); m_Parser.AddOverride(GetOverridesDir(m_DataDir) / "overrides.ini", section, key, value);
}
void
Config::LoadOverrides()
{
const auto overridesDir = GetOverridesDir(m_DataDir);
if (fs::exists(overridesDir))
{
util::IterDir(overridesDir, [&](const fs::path& overrideFile) {
if (overrideFile.extension() == ".ini")
{
m_Parser.LoadFile(overrideFile);
}
return true;
});
}
} }
bool bool
Config::Load(const fs::path fname, bool isRelay, fs::path defaultDataDir) Config::Load(std::optional<fs::path> fname, bool isRelay)
{ {
if (not fname.has_value())
return LoadDefault(isRelay);
try try
{ {
ConfigGenParameters params; ConfigGenParameters params;
params.isRelay = isRelay; params.isRelay = isRelay;
params.defaultDataDir = std::move(defaultDataDir); params.defaultDataDir = m_DataDir;
ConfigDefinition conf{isRelay}; ConfigDefinition conf{isRelay};
initializeConfig(conf, params); initializeConfig(conf, params);
addBackwardsCompatibleConfigOptions(conf); addBackwardsCompatibleConfigOptions(conf);
m_Parser.Clear(); m_Parser.Clear();
if (!m_Parser.LoadFile(fname)) if (!m_Parser.LoadFile(*fname))
{ {
return false; return false;
} }
LoadOverrides();
m_Parser.IterAll([&](std::string_view section, const SectionValues_t& values) { m_Parser.IterAll([&](std::string_view section, const SectionValues_t& values) {
for (const auto& pair : values) for (const auto& pair : values)
@ -1012,10 +1047,6 @@ namespace llarp
conf.acceptAllOptions(); conf.acceptAllOptions();
// TODO: better way to support inter-option constraints
if (router.m_maxConnectedRouters < router.m_minConnectedRouters)
throw std::invalid_argument("[router]:min-connections must be <= [router]:max-connections");
return true; return true;
} }
catch (const std::exception& e) catch (const std::exception& e)
@ -1026,17 +1057,26 @@ namespace llarp
} }
bool bool
Config::LoadDefault(bool isRelay, fs::path dataDir) Config::LoadDefault(bool isRelay)
{ {
try try
{ {
ConfigGenParameters params; ConfigGenParameters params;
params.isRelay = isRelay; params.isRelay = isRelay;
params.defaultDataDir = std::move(dataDir); params.defaultDataDir = m_DataDir;
ConfigDefinition conf{isRelay}; ConfigDefinition conf{isRelay};
initializeConfig(conf, params); initializeConfig(conf, params);
m_Parser.Clear();
LoadOverrides();
m_Parser.IterAll([&](std::string_view section, const SectionValues_t& values) {
for (const auto& pair : values)
{
conf.addConfigValue(section, pair.first, pair.second);
}
});
conf.acceptAllOptions(); conf.acceptAllOptions();
return true; return true;
@ -1103,12 +1143,12 @@ namespace llarp
llarp::LogInfo("Attempting to create config file, asRouter: ", asRouter, " path: ", confFile); llarp::LogInfo("Attempting to create config file, asRouter: ", asRouter, " path: ", confFile);
llarp::Config config; llarp::Config config{defaultDataDir};
std::string confStr; std::string confStr;
if (asRouter) if (asRouter)
confStr = config.generateBaseRouterConfig(std::move(defaultDataDir)); confStr = config.generateBaseRouterConfig();
else else
confStr = config.generateBaseClientConfig(std::move(defaultDataDir)); confStr = config.generateBaseClientConfig();
// open a filestream // open a filestream
auto stream = llarp::util::OpenFileStream<std::ofstream>(confFile.c_str(), std::ios::binary); auto stream = llarp::util::OpenFileStream<std::ofstream>(confFile.c_str(), std::ios::binary);
@ -1168,11 +1208,11 @@ namespace llarp
} }
std::string std::string
Config::generateBaseClientConfig(fs::path defaultDataDir) Config::generateBaseClientConfig()
{ {
ConfigGenParameters params; ConfigGenParameters params;
params.isRelay = false; params.isRelay = false;
params.defaultDataDir = std::move(defaultDataDir); params.defaultDataDir = m_DataDir;
llarp::ConfigDefinition def{false}; llarp::ConfigDefinition def{false};
initializeConfig(def, params); initializeConfig(def, params);
@ -1188,11 +1228,11 @@ namespace llarp
} }
std::string std::string
Config::generateBaseRouterConfig(fs::path defaultDataDir) Config::generateBaseRouterConfig()
{ {
ConfigGenParameters params; ConfigGenParameters params;
params.isRelay = true; params.isRelay = true;
params.defaultDataDir = std::move(defaultDataDir); params.defaultDataDir = m_DataDir;
llarp::ConfigDefinition def{true}; llarp::ConfigDefinition def{true};
initializeConfig(def, params); initializeConfig(def, params);

39
llarp/config/config.hpp
Unescape Escape View File

@ -166,8 +166,7 @@ namespace llarp
struct BootstrapConfig struct BootstrapConfig
{ {
std::vector<fs::path> routers; std::vector<fs::path> routers;
/// for unit tests bool seednode;
bool skipBootstrap = false;
void void
defineConfigOptions(ConfigDefinition& conf, const ConfigGenParameters& params); defineConfigOptions(ConfigDefinition& conf, const ConfigGenParameters& params);
}; };
@ -184,6 +183,10 @@ namespace llarp
struct Config struct Config
{ {
explicit Config(fs::path datadir);
~Config() = default;
RouterConfig router; RouterConfig router;
NetworkConfig network; NetworkConfig network;
ConnectConfig connect; ConnectConfig connect;
@ -205,10 +208,23 @@ namespace llarp
void void
addBackwardsCompatibleConfigOptions(ConfigDefinition& conf); addBackwardsCompatibleConfigOptions(ConfigDefinition& conf);
// Load a config from the given file // Load a config from the given file if the config file is not provided LoadDefault is called
bool bool
Load(const fs::path fname, bool isRelay, fs::path defaultDataDir); Load(std::optional<fs::path> fname = std::nullopt, bool isRelay = false);
std::string
generateBaseClientConfig();
std::string
generateBaseRouterConfig();
void
Save();
void
Override(std::string section, std::string key, std::string value);
private:
/// Load (initialize) a default config. /// Load (initialize) a default config.
/// ///
/// This delegates to the ConfigDefinition to generate a default config, /// This delegates to the ConfigDefinition to generate a default config,
@ -221,22 +237,13 @@ namespace llarp
/// @param dataDir is a path representing a directory to be used as the data dir /// @param dataDir is a path representing a directory to be used as the data dir
/// @return true on success, false otherwise /// @return true on success, false otherwise
bool bool
LoadDefault(bool isRelay, fs::path dataDir); LoadDefault(bool isRelay);
std::string
generateBaseClientConfig(fs::path defaultDataDir);
std::string
generateBaseRouterConfig(fs::path defaultDataDir);
void void
Save(); LoadOverrides();
void
Override(std::string section, std::string key, std::string value);
private:
ConfigParser m_Parser; ConfigParser m_Parser;
const fs::path m_DataDir;
}; };
void void

24
llarp/config/ini.cpp
Unescape Escape View File

@ -142,28 +142,26 @@ namespace llarp
} }
void void
ConfigParser::AddOverride(std::string section, std::string key, std::string value) ConfigParser::AddOverride(fs::path fpath, std::string section, std::string key, std::string value)
{ {
m_Overrides[section].emplace(key, value); auto& data = m_Overrides[fpath];
data[section].emplace(key, value);
} }
void void
ConfigParser::Save() ConfigParser::Save()
{ {
// if we have no overrides keep the config the same on disk
if (m_Overrides.empty())
return;
std::ofstream ofs(m_FileName);
// write existing config data
ofs.write(m_Data.data(), m_Data.size());
// write overrides // write overrides
ofs << std::endl << std::endl << "# overrides" << std::endl; for (const auto& [fname, overrides] : m_Overrides)
for (const auto& [section, values] : m_Overrides)
{ {
ofs << std::endl << "[" << section << "]" << std::endl; std::ofstream ofs(fname);
for (const auto& [key, value] : values) for (const auto& [section, values] : overrides)
{ {
ofs << key << "=" << value << std::endl; ofs << std::endl << "[" << section << "]" << std::endl;
for (const auto& [key, value] : values)
{
ofs << key << "=" << value << std::endl;
}
} }
} }
m_Overrides.clear(); m_Overrides.clear();

8
llarp/config/ini.hpp
Unescape Escape View File

@ -40,11 +40,11 @@ namespace llarp
bool bool
VisitSection(const char* name, std::function<bool(const SectionValues_t&)> visit) const; VisitSection(const char* name, std::function<bool(const SectionValues_t&)> visit) const;
/// add a config option that is appended at the end of the config buffer with no comments /// add a config option that is appended in another file
void void
AddOverride(std::string section, std::string key, std::string value); AddOverride(fs::path file, std::string section, std::string key, std::string value);
/// save config and any overrides to the file it was loaded from /// save config overrides
void void
Save(); Save();
@ -54,7 +54,7 @@ namespace llarp
std::vector<char> m_Data; std::vector<char> m_Data;
Config_impl_t m_Config; Config_impl_t m_Config;
Config_impl_t m_Overrides; std::unordered_map<fs::path, Config_impl_t, util::FileHash> m_Overrides;
fs::path m_FileName; fs::path m_FileName;
}; };

3
llarp/constants/files.hpp
Unescape Escape View File

@ -22,6 +22,7 @@ namespace llarp
{ {
#ifdef _WIN32 #ifdef _WIN32
const fs::path homedir = getenv("APPDATA"); const fs::path homedir = getenv("APPDATA");
return homedir / "lokinet";
#else #else
fs::path homedir; fs::path homedir;
@ -36,8 +37,8 @@ namespace llarp
homedir = "/var/lib/lokinet"; homedir = "/var/lib/lokinet";
return homedir; return homedir;
} }
#endif
return homedir / ".lokinet"; return homedir / ".lokinet";
#endif
} }
inline fs::path inline fs::path

2
llarp/router/router.cpp
Unescape Escape View File

@ -496,7 +496,7 @@ namespace llarp
{ {
configRouters.push_back(defaultBootstrapFile); configRouters.push_back(defaultBootstrapFile);
} }
else if (not conf.bootstrap.skipBootstrap) else if (not conf.bootstrap.seednode)
{ {
LogError("No bootstrap files specified in config file, and the default"); LogError("No bootstrap files specified in config file, and the default");
LogError("bootstrap file ", defaultBootstrapFile, " does not exist."); LogError("bootstrap file ", defaultBootstrapFile, " does not exist.");

10
llarp/util/fs.hpp
Unescape Escape View File

@ -27,6 +27,16 @@ namespace llarp
{ {
namespace util namespace util
{ {
struct FileHash
{
size_t
operator()(const fs::path& f) const
{
std::hash<std::string> h;
return h(f.string());
}
};
using error_code_t = std::error_code; using error_code_t = std::error_code;
/// Ensure that a file exists and has correct permissions /// Ensure that a file exists and has correct permissions

8
pybind/llarp/config.cpp
Unescape Escape View File

@ -16,7 +16,7 @@ namespace llarp
{ {
using Config_ptr = std::shared_ptr<Config>; using Config_ptr = std::shared_ptr<Config>;
py::class_<Config, Config_ptr>(mod, "Config") py::class_<Config, Config_ptr>(mod, "Config")
.def(py::init<>()) .def(py::init<std::string>())
.def_readwrite("router", &Config::router) .def_readwrite("router", &Config::router)
.def_readwrite("network", &Config::network) .def_readwrite("network", &Config::network)
.def_readwrite("connect", &Config::connect) .def_readwrite("connect", &Config::connect)
@ -25,10 +25,7 @@ namespace llarp
.def_readwrite("lokid", &Config::lokid) .def_readwrite("lokid", &Config::lokid)
.def_readwrite("bootstrap", &Config::bootstrap) .def_readwrite("bootstrap", &Config::bootstrap)
.def_readwrite("logging", &Config::logging) .def_readwrite("logging", &Config::logging)
.def("LoadFile", &Config::Load) .def("Load", &Config::Load);
.def("LoadDefault", [](Config& self, bool isRelay, std::string dir) {
return self.LoadDefault(isRelay, dir);
});
py::class_<RouterConfig>(mod, "RouterConfig") py::class_<RouterConfig>(mod, "RouterConfig")
.def(py::init<>()) .def(py::init<>())
@ -106,6 +103,7 @@ namespace llarp
py::class_<BootstrapConfig>(mod, "BootstrapConfig") py::class_<BootstrapConfig>(mod, "BootstrapConfig")
.def(py::init<>()) .def(py::init<>())
.def_readwrite("seednode", &BootstrapConfig::seednode)
.def_property( .def_property(
"routers", "routers",
[](BootstrapConfig& self) { [](BootstrapConfig& self) {

15
test/crypto/test_llarp_key_manager.cpp
Unescape Escape View File

@ -112,8 +112,8 @@ TEST_F(KeyManagerTest, TestBackupFileByMoving_FailsIfBackupNamesAreExausted)
TEST_F(KeyManagerTest, TestInitialize_MakesKeyfiles) TEST_F(KeyManagerTest, TestInitialize_MakesKeyfiles)
{ {
llarp::Config conf; llarp::Config conf{fs::current_path()};
conf.LoadDefault(false, {}); conf.Load();
KeyManager keyManager; KeyManager keyManager;
ASSERT_TRUE(keyManager.initialize(conf, true, true)); ASSERT_TRUE(keyManager.initialize(conf, true, true));
@ -128,9 +128,9 @@ TEST_F(KeyManagerTest, TestInitialize_MakesKeyfiles)
TEST_F(KeyManagerTest, TestInitialize_RespectsGenFlag) TEST_F(KeyManagerTest, TestInitialize_RespectsGenFlag)
{ {
llarp::Config conf; llarp::Config conf{fs::current_path()};
conf.LoadDefault(false, {}); conf.Load();
KeyManager keyManager; KeyManager keyManager;
ASSERT_FALSE(keyManager.initialize(conf, false, true)); ASSERT_FALSE(keyManager.initialize(conf, false, true));
@ -143,8 +143,9 @@ TEST_F(KeyManagerTest, TestInitialize_RespectsGenFlag)
TEST_F(KeyManagerTest, TestInitialize_DetectsBadRcFile) TEST_F(KeyManagerTest, TestInitialize_DetectsBadRcFile)
{ {
llarp::Config conf; llarp::Config conf{fs::current_path()};
conf.LoadDefault(false, {}); conf.Load();
conf.lokid.whitelistRouters = false; conf.lokid.whitelistRouters = false;
std::fstream f; std::fstream f;

6
test/exit/test_llarp_exit_context.cpp
Unescape Escape View File

@ -11,12 +11,12 @@ static const llarp::RuntimeOptions opts = {.background = false, .debug = false,
std::shared_ptr<llarp::Context> std::shared_ptr<llarp::Context>
make_context() make_context()
{ {
llarp::Config conf{}; llarp::Config conf{fs::current_path()};
conf.LoadDefault(true, fs::current_path()); conf.Load(std::nullopt, true);
// set testing defaults // set testing defaults
conf.network.m_endpointType = "null"; conf.network.m_endpointType = "null";
conf.bootstrap.skipBootstrap = true; conf.bootstrap.seednode = true;
conf.api.m_enableRPCServer = false; conf.api.m_enableRPCServer = false;
conf.lokid.whitelistRouters = false; conf.lokid.whitelistRouters = false;
conf.router.m_publicAddress = llarp::IpAddress("1.1.1.1"); conf.router.m_publicAddress = llarp::IpAddress("1.1.1.1");

23
test/hive/hive.py
Unescape Escape View File

@ -48,14 +48,13 @@ class RouterHive(object):
print("not removing dir %s because it doesn't start with /tmp/" % self.tmpdir) print("not removing dir %s because it doesn't start with /tmp/" % self.tmpdir)
return False return False
def AddRelay(self, index): def AddRelay(self, index):
dirname = "%s/relays/%d" % (self.tmpdir, index) dirname = "%s/relays/%d" % (self.tmpdir, index)
makedirs("%s/nodedb" % dirname, exist_ok=True) makedirs("%s/nodedb" % dirname, exist_ok=True)
config = pyllarp.Config() config = pyllarp.Config(dirname)
config.LoadDefault(True, dirname); config.Load(None, True)
port = index + 30000 port = index + 30000
tunname = "lokihive%d" % index tunname = "lokihive%d" % index
@ -65,34 +64,31 @@ class RouterHive(object):
config.router.nickname = "Router%d" % index config.router.nickname = "Router%d" % index
config.router.overrideAddress('127.0.0.1:{}'.format(port)) config.router.overrideAddress('127.0.0.1:{}'.format(port))
config.router.blockBogons = False config.router.blockBogons = False
config.router.enablePeerStats = True
config.network.enableProfiling = False config.network.enableProfiling = False
config.network.routerProfilesFile = "%s/profiles.dat" % dirname
config.network.endpointType = 'null' config.network.endpointType = 'null'
config.links.addInboundLink("lo", AF_INET, port); config.links.addInboundLink("lo", AF_INET, port);
config.links.setOutboundLink("lo", AF_INET, port + 10000); config.links.setOutboundLink("lo", AF_INET, port + 10000);
# config.dns.options = {"local-dns": ("127.3.2.1:%d" % port)} # config.dns.options = {"local-dns": ("127.3.2.1:%d" % port)}
if index == 0:
if index != 0: config.bootstrap.seednode = True
else:
config.bootstrap.routers = ["%s/relays/0/self.signed" % self.tmpdir] config.bootstrap.routers = ["%s/relays/0/self.signed" % self.tmpdir]
config.api.enableRPCServer = False config.api.enableRPCServer = False
config.lokid.whitelistRouters = False config.lokid.whitelistRouters = False
print("adding relay at index %d" % index)
print("adding relay at index %d" % port);
self.hive.AddRelay(config) self.hive.AddRelay(config)
def AddClient(self, index): def AddClient(self, index):
dirname = "%s/clients/%d" % (self.tmpdir, index) dirname = "%s/clients/%d" % (self.tmpdir, index)
makedirs("%s/nodedb" % dirname, exist_ok=True) makedirs("%s/nodedb" % dirname, exist_ok=True)
config = pyllarp.Config() config = pyllarp.Config(dirname)
config.LoadDefault(False, dirname); config.Load(None, False);
port = index + 50000 port = index + 50000
tunname = "lokihive%d" % index tunname = "lokihive%d" % index
@ -102,7 +98,6 @@ class RouterHive(object):
config.router.blockBogons = False config.router.blockBogons = False
config.network.enableProfiling = False config.network.enableProfiling = False
config.network.routerProfilesFile = "%s/profiles.dat" % dirname
config.network.endpointType = 'null' config.network.endpointType = 'null'
config.links.setOutboundLink("lo", AF_INET, port + 10000); config.links.setOutboundLink("lo", AF_INET, port + 10000);

6
test/regress/2020-06-08-key-backup-bug.cpp
Unescape Escape View File

@ -11,11 +11,11 @@ llarp::RuntimeOptions opts = {false, false, false};
static std::shared_ptr<llarp::Context> static std::shared_ptr<llarp::Context>
make_context(std::optional<fs::path> keyfile) make_context(std::optional<fs::path> keyfile)
{ {
llarp::Config conf; llarp::Config conf{fs::current_path()};
conf.LoadDefault(opts.isRouter, {}); conf.Load(std::nullopt, opts.isRouter);
conf.network.m_endpointType = "null"; conf.network.m_endpointType = "null";
conf.network.m_keyfile = keyfile; conf.network.m_keyfile = keyfile;
conf.bootstrap.skipBootstrap = true; conf.bootstrap.seednode = true;
conf.api.m_enableRPCServer = false; conf.api.m_enableRPCServer = false;
auto context = std::make_shared<llarp::Context>(); auto context = std::make_shared<llarp::Context>();

Write Preview
Loading…
Cancel
Save