SRV Record handling for introsets (#1331)

* update loki-mq submodule for tuple support

* srv record reply implementation

still need to encode srv records into intro sets / router contacts
as well as decode from them and match against queried service.proto

* inverted condition fix in config code

* SRV record struct (de-)serialization for intro sets

* parsing and using srv records from config (for/in introsets)

* adopt str utils from core and use for srv parsing

* changes to repeat requests

no longer drop repeat requests on the floor, but do not make
an *actual* request for them if one is in progress.

do not call reply hook for each reply for a request, as
each userland request is actually made into several lokinet
requests and this would result in duplicate replies.

* fetch SRVs from introsets for .loki

* make format

* dns and srv fixes, srv appears to be working
pull/1332/head
Thomas Winget 4 years ago committed by GitHub
parent 521765b5a8
commit b1c14af938
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

2
external/loki-mq vendored

@ -1 +1 @@
Subproject commit 07b31bd8a1b39a7de7913b91aab7b8e1e12e928b
Subproject commit 30faadf01a561be8bda1b9fd78cd606bb209576a

@ -106,6 +106,7 @@ add_library(liblokinet
dns/rr.cpp
dns/serialize.cpp
dns/server.cpp
dns/srv_data.cpp
dns/unbound_resolver.cpp
consensus/table.cpp

@ -322,9 +322,17 @@ namespace llarp
throw std::invalid_argument(stringify("Invalid RouterID: ", arg));
auto itr = m_snodeBlacklist.emplace(std::move(id));
if (itr.second)
if (not itr.second)
throw std::invalid_argument(stringify("Duplicate blacklist-snode: ", arg));
});
conf.defineOption<std::string>("network", "srv", false, true, "", [this](std::string arg) {
llarp::dns::SRVData newSRV;
if (not newSRV.fromString(arg))
throw std::invalid_argument(stringify("Invalid SRV Record string: ", arg));
m_SRVRecords.push_back(std::move(newSRV));
});
}
void

@ -14,6 +14,7 @@
#include <net/ip_range_map.hpp>
#include <service/address.hpp>
#include <service/auth.hpp>
#include <dns/srv_data.hpp>
#include <cstdlib>
#include <functional>
@ -93,6 +94,8 @@ namespace llarp
std::optional<std::string> m_AuthMethod;
std::unordered_set<service::Address, service::Address::Hash> m_AuthWhitelist;
std::vector<llarp::dns::SRVData> m_SRVRecords;
// TODO:
// on-up
// on-down

@ -177,13 +177,20 @@ namespace llarp
T
fromString(const std::string& input)
{
std::istringstream iss(input);
T t;
iss >> t;
if (iss.fail())
throw std::invalid_argument(stringify(input, " is not a valid ", typeid(T).name()));
if constexpr (std::is_same_v<T, std::string>)
{
return input;
}
else
return t;
{
std::istringstream iss(input);
T t;
iss >> t;
if (iss.fail())
throw std::invalid_argument(stringify(input, " is not a valid ", typeid(T).name()));
else
return t;
}
}
std::string

@ -7,6 +7,7 @@ namespace llarp
{
namespace dns
{
constexpr uint16_t qTypeSRV = 33;
constexpr uint16_t qTypeAAAA = 28;
constexpr uint16_t qTypeTXT = 16;
constexpr uint16_t qTypeMX = 15;

@ -1,6 +1,7 @@
#include <dns/message.hpp>
#include <dns/dns.hpp>
#include <dns/srv_data.hpp>
#include <util/buffer.hpp>
#include <util/endian.hpp>
#include <util/logging/logger.hpp>
@ -268,10 +269,69 @@ namespace llarp
}
}
void
Message::AddSRVReply(std::vector<SRVData> records, RR_TTL_t ttl)
{
hdr_fields = reply_flags(hdr_fields);
const auto& question = questions[0];
for (const auto& srv : records)
{
if (not srv.IsValid())
{
AddNXReply();
return;
}
answers.emplace_back();
auto& rec = answers.back();
rec.rr_name = question.qname;
rec.rr_type = qTypeSRV;
rec.rr_class = qClassIN;
rec.ttl = ttl;
std::array<byte_t, 512> tmp = {{0}};
llarp_buffer_t buf(tmp);
buf.put_uint16(srv.priority);
buf.put_uint16(srv.weight);
buf.put_uint16(srv.port);
std::string target;
if (srv.target == "")
{
// get location of second dot (after service.proto) in qname
size_t pos = question.qname.find(".");
pos = question.qname.find(".", pos + 1);
target = question.qname.substr(pos + 1);
}
else
{
target = srv.target;
}
if (not EncodeName(&buf, target))
{
AddNXReply();
return;
}
buf.sz = buf.cur - buf.base;
rec.rData.resize(buf.sz);
memcpy(rec.rData.data(), buf.base, buf.sz);
}
}
void Message::AddNXReply(RR_TTL_t)
{
if (questions.size())
{
answers.clear();
authorities.clear();
additional.clear();
// authorative response with recursion available
hdr_fields = reply_flags(hdr_fields);
// don't allow recursion on this request

@ -9,6 +9,8 @@ namespace llarp
{
namespace dns
{
struct SRVData;
using MsgID_t = uint16_t;
using Fields_t = uint16_t;
using Count_t = uint16_t;
@ -66,6 +68,9 @@ namespace llarp
void
AddAReply(std::string name, RR_TTL_t ttl = 1);
void
AddSRVReply(std::vector<SRVData> records, RR_TTL_t ttl = 1);
void
AddNSReply(std::string name, RR_TTL_t ttl = 1);

@ -0,0 +1,102 @@
#include <dns/srv_data.hpp>
#include <util/str.hpp>
#include <util/logging/logger.hpp>
#include <limits>
namespace llarp::dns
{
bool
SRVData::IsValid() const
{
// if target is of first two forms outlined above
if (target == "." or target.size() == 0)
{
return true;
}
// check target size is not absurd
if (target.size() > TARGET_MAX_SIZE)
{
LogWarn("SRVData target larger than max size (", TARGET_MAX_SIZE, ")");
return false;
}
// does target end in .loki?
size_t pos = target.find(".loki");
if (pos != std::string::npos && pos == (target.size() - 5))
{
return true;
}
// does target end in .snode?
pos = target.find(".snode");
if (pos != std::string::npos && pos == (target.size() - 6))
{
return true;
}
// if we're here, target is invalid
LogWarn("SRVData invalid");
return false;
}
SRVTuple
SRVData::toTuple() const
{
return std::make_tuple(service_proto, priority, weight, port, target);
}
SRVData
SRVData::fromTuple(SRVTuple tuple)
{
SRVData s;
std::tie(s.service_proto, s.priority, s.weight, s.port, s.target) = std::move(tuple);
return s;
}
bool
SRVData::fromString(std::string_view srvString)
{
LogDebug("SRVData::fromString(\"", srvString, "\")");
// split on spaces, discard trailing empty strings
auto splits = split(srvString, " ", false);
if (splits.size() != 5 && splits.size() != 4)
{
LogWarn("SRV record should have either 4 or 5 space-separated parts");
return false;
}
service_proto = splits[0];
if (not parse_int(splits[1], priority))
{
LogWarn("SRV record failed to parse \"", splits[1], "\" as uint16_t (priority)");
return false;
}
if (not parse_int(splits[2], weight))
{
LogWarn("SRV record failed to parse \"", splits[2], "\" as uint16_t (weight)");
return false;
}
if (not parse_int(splits[3], port))
{
LogWarn("SRV record failed to parse \"", splits[3], "\" as uint16_t (port)");
return false;
}
if (splits.size() == 5)
target = splits[4];
else
target = "";
return IsValid();
}
} // namespace llarp::dns

@ -0,0 +1,62 @@
#pragma once
#include <dns/name.hpp>
#include <dns/serialize.hpp>
#include <tuple>
#include <string_view>
namespace llarp::dns
{
typedef std::tuple<std::string, uint16_t, uint16_t, uint16_t, std::string> SRVTuple;
struct SRVData
{
static constexpr size_t TARGET_MAX_SIZE = 200;
std::string service_proto; // service and protocol may as well be together
uint16_t priority;
uint16_t weight;
uint16_t port;
// target string for the SRV record to point to
// options:
// empty - refer to query name
// dot - authoritative "no such service available"
// any other .loki or .snode - target is that .loki or .snode
std::string target;
// do some basic validation on the target string
// note: this is not a conclusive, regex solution,
// but rather some sanity/safety checks
bool
IsValid() const;
SRVTuple
toTuple() const;
static SRVData
fromTuple(SRVTuple tuple);
/* bind-like formatted string for SRV records in config file
*
* format:
* srv=service.proto priority weight port target
*
* exactly one space character between parts.
*
* target can be empty, in which case the space after port should
* be omitted. if this is the case, the target is
* interpreted as the .loki or .snode of the current context.
*
* if target is not empty, it must be either
* - simply a full stop (dot/period) OR
* - a name within the .loki or .snode subdomains. a target
* specified in this manner must not end with a full stop.
*/
bool
fromString(std::string_view srvString);
};
} // namespace llarp::dns

@ -13,6 +13,7 @@
#include <ev/ev.hpp>
#include <router/abstractrouter.hpp>
#include <service/context.hpp>
#include <service/outbound_context.hpp>
#include <service/endpoint_state.hpp>
#include <service/outbound_context.hpp>
#include <util/meta/memfn.hpp>
@ -22,6 +23,8 @@
#include <util/str.hpp>
#include <dns/srv_data.hpp>
namespace llarp
{
namespace handlers
@ -308,6 +311,35 @@ namespace llarp
},
2s);
};
auto ReplyToLokiSRVWhenReady = [self = this, reply = reply](
service::Address addr, auto msg) -> bool {
using service::Address;
using service::OutboundContext;
return self->EnsurePathToService(
addr,
[=](const Address&, OutboundContext* ctx) {
if (ctx == nullptr)
return;
const auto& introset = ctx->GetCurrentIntroSet();
std::vector<llarp::dns::SRVData> records;
size_t numRecords = introset.SRVs.size();
if (numRecords > 0)
{
records.reserve(numRecords);
for (const auto& record : introset.SRVs)
{
records.push_back(std::move(llarp::dns::SRVData::fromTuple(record)));
}
}
msg->AddSRVReply(records);
reply(*msg);
},
2s);
};
std::string qname;
if (msg.answers.size() > 0)
{
@ -479,6 +511,23 @@ namespace llarp
reply(msg);
return true;
}
// TODO: SRV Record
else if (msg.questions[0].qtype == dns::qTypeSRV)
{
llarp::service::Address addr;
if (is_localhost_loki(msg))
{
msg.AddNXReply();
reply(msg);
return true;
}
else if (addr.FromString(qname, ".loki"))
{
llarp::LogWarn("SRV request for: ", qname);
return ReplyToLokiSRVWhenReady(addr, std::make_shared<dns::Message>(msg));
}
}
else
{
msg.AddNXReply();

@ -196,8 +196,6 @@ namespace llarp
RegenAndPublishIntroSet();
}
m_state->m_RemoteLookupFilter.Decay(now);
// expire snode sessions
EndpointUtil::ExpireSNodeSessions(now, m_state->m_SNodeSessions);
// expire pending tx
@ -414,7 +412,6 @@ namespace llarp
bool
Endpoint::Start()
{
m_state->m_RemoteLookupFilter.DecayInterval(500ms);
// how can I tell if a m_Identity isn't loaded?
if (!m_DataHandler)
{
@ -995,18 +992,28 @@ namespace llarp
}
}
// filter check for address
if (not m_state->m_RemoteLookupFilter.Insert(remote))
return false;
// add response hook to list for address.
m_state->m_PendingServiceLookups.emplace(remote, hook);
auto& lookups = m_state->m_PendingServiceLookups;
auto& lookupTimes = m_state->m_LastServiceLookupTimes;
const auto now = Now();
// if most recent lookup was within last INTROSET_LOOKUP_RETRY_COOLDOWN
// just add callback to the list and return
if (lookupTimes.find(remote) != lookupTimes.end()
&& now < (lookupTimes[remote] + INTROSET_LOOKUP_RETRY_COOLDOWN))
return true;
const auto paths = GetManyPathsWithUniqueEndpoints(this, NumParallelLookups);
using namespace std::placeholders;
size_t lookedUp = 0;
const dht::Key_t location = remote.ToKey();
uint64_t order = 0;
// flag to only add callback to list of callbacks for
// address once.
bool hookAdded = false;
for (const auto& path : paths)
{
for (size_t count = 0; count < RequestsPerLookup; ++count)
@ -1030,14 +1037,18 @@ namespace llarp
order++;
if (job->SendRequestViaPath(path, Router()))
{
lookups.emplace(remote, hook);
lookedUp++;
if (not hookAdded)
{
// if any of the lookups is successful, set last lookup time
lookupTimes[remote] = now;
hookAdded = true;
}
}
else
LogError(Name(), " send via path failed for lookup");
}
}
return lookedUp == (NumParallelLookups * RequestsPerLookup);
return hookAdded;
}
bool

@ -67,6 +67,8 @@ namespace llarp
static constexpr auto INTROSET_PUBLISH_RETRY_INTERVAL = 5s;
static constexpr auto INTROSET_LOOKUP_RETRY_COOLDOWN = 3s;
struct Endpoint : public path::Builder, public ILookupHolder, public IDataHandler
{
static const size_t MAX_OUTBOUND_CONTEXT_COUNT = 4;

@ -17,6 +17,12 @@ namespace llarp
m_Keyfile = conf.m_keyfile->string();
m_SnodeBlacklist = conf.m_snodeBlacklist;
m_ExitEnabled = conf.m_AllowExit;
for (const auto& record : conf.m_SRVRecords)
{
m_IntroSet.SRVs.push_back(record.toTuple());
}
// TODO:
/*
if (k == "on-up")

@ -67,6 +67,7 @@ namespace llarp
SNodeSessions m_SNodeSessions;
std::unordered_multimap<Address, PathEnsureHook, Address::Hash> m_PendingServiceLookups;
std::unordered_map<Address, llarp_time_t, Address::Hash> m_LastServiceLookupTimes;
std::unordered_map<RouterID, uint32_t, RouterID::Hash> m_ServiceLookupFails;
@ -88,8 +89,6 @@ namespace llarp
std::unordered_map<Tag, CachedTagResult, Tag::Hash> m_PrefetchedTags;
util::DecayingHashSet<Address> m_RemoteLookupFilter;
bool
Configure(const NetworkConfig& conf);

@ -2,6 +2,8 @@
#include <crypto/crypto.hpp>
#include <path/path.hpp>
#include <lokimq/bt_serialize.h>
namespace llarp
{
namespace service
@ -170,6 +172,28 @@ namespace llarp
if (!BEncodeMaybeReadDictEntry("n", topic, read, key, buf))
return false;
if (key == "s")
{
byte_t* begin = buf->cur;
if (not bencode_discard(buf))
return false;
byte_t* end = buf->cur;
std::string_view srvString(reinterpret_cast<char*>(begin), end - begin);
try
{
lokimq::bt_deserialize(srvString, SRVs);
}
catch (const lokimq::bt_deserialize_invalid& err)
{
LogError("Error decoding SRV records from IntroSet: ", err.what());
return false;
}
read = true;
}
if (!BEncodeMaybeReadDictInt("t", T, read, key, buf))
return false;
@ -215,6 +239,16 @@ namespace llarp
if (!BEncodeWriteDictEntry("n", topic, buf))
return false;
}
if (SRVs.size())
{
std::string serial = lokimq::bt_serialize(SRVs);
if (!bencode_write_bytestring(buf, "s", 1))
return false;
if (!buf->write(serial.begin(), serial.end()))
return false;
}
// Timestamp published
if (!BEncodeWriteDictInt("t", T.count(), buf))
return false;

@ -9,6 +9,7 @@
#include <util/bencode.hpp>
#include <util/time.hpp>
#include <util/status.hpp>
#include <dns/srv_data.hpp>
#include <optional>
#include <algorithm>
@ -30,6 +31,7 @@ namespace llarp
std::vector<Introduction> I;
PQPubKey K;
Tag topic;
std::vector<llarp::dns::SRVTuple> SRVs;
llarp_time_t T = 0s;
std::optional<PoW> W;
Signature Z;

@ -117,6 +117,12 @@ namespace llarp
std::string
Name() const override;
const IntroSet&
GetCurrentIntroSet() const
{
return currentIntroSet;
}
private:
/// swap remoteIntro with next intro
void

@ -3,6 +3,7 @@
#include <algorithm>
#include <cctype>
#include <cstring>
#include <cassert>
#include <string>
#include <set>
@ -95,4 +96,127 @@ namespace llarp
return splits;
}
using namespace std::literals;
std::vector<std::string_view>
split(std::string_view str, const std::string_view delim, bool trim)
{
std::vector<std::string_view> results;
// Special case for empty delimiter: splits on each character boundary:
if (delim.empty())
{
results.reserve(str.size());
for (size_t i = 0; i < str.size(); i++)
results.emplace_back(str.data() + i, 1);
return results;
}
for (size_t pos = str.find(delim); pos != std::string_view::npos; pos = str.find(delim))
{
if (!trim || !results.empty() || pos > 0)
results.push_back(str.substr(0, pos));
str.remove_prefix(pos + delim.size());
}
if (!trim || str.size())
results.push_back(str);
else
while (!results.empty() && results.back().empty())
results.pop_back();
return results;
}
std::vector<std::string_view>
split_any(std::string_view str, const std::string_view delims, bool trim)
{
if (delims.empty())
return split(str, delims, trim);
std::vector<std::string_view> results;
for (size_t pos = str.find_first_of(delims); pos != std::string_view::npos;
pos = str.find_first_of(delims))
{
if (!trim || !results.empty() || pos > 0)
results.push_back(str.substr(0, pos));
size_t until = str.find_first_not_of(delims, pos + 1);
if (until == std::string_view::npos)
str.remove_prefix(str.size());
else
str.remove_prefix(until);
}
if (!trim || str.size())
results.push_back(str);
else
while (!results.empty() && results.back().empty())
results.pop_back();
return results;
}
void
trim(std::string_view& s)
{
constexpr auto simple_whitespace = " \t\r\n"sv;
auto pos = s.find_first_not_of(simple_whitespace);
if (pos == std::string_view::npos)
{ // whole string is whitespace
s.remove_prefix(s.size());
return;
}
s.remove_prefix(pos);
pos = s.find_last_not_of(simple_whitespace);
assert(pos != std::string_view::npos);
s.remove_suffix(s.size() - (pos + 1));
}
std::string
lowercase_ascii_string(std::string src)
{
for (char& ch : src)
if (ch >= 'A' && ch <= 'Z')
ch = ch + ('a' - 'A');
return src;
}
std::string
friendly_duration(std::chrono::nanoseconds dur)
{
std::ostringstream os;
bool some = false;
if (dur >= 24h)
{
os << dur / 24h << 'd';
dur %= 24h;
some = true;
}
if (dur >= 1h || some)
{
os << dur / 1h << 'h';
dur %= 1h;
some = true;
}
if (dur >= 1min || some)
{
os << dur / 1min << 'm';
dur %= 1min;
some = true;
}
if (some)
{
// If we have >= minutes then don't bother with fractional seconds
os << dur / 1s << 's';
}
else
{
double seconds = std::chrono::duration<double>(dur).count();
os.precision(3);
if (dur >= 1s)
os << seconds << "s";
else if (dur >= 1ms)
os << seconds * 1000 << "ms";
else if (dur >= 1us)
os << seconds * 1'000'000 << u8"µs";
else
os << seconds * 1'000'000'000 << "ns";
}
return os.str();
}
} // namespace llarp

@ -4,6 +4,9 @@
#include <string_view>
#include <sstream>
#include <vector>
#include <chrono>
#include <iterator>
#include <charconv>
namespace llarp
{
@ -41,6 +44,116 @@ namespace llarp
std::vector<std::string_view>
split(const std::string_view str, char delimiter);
using namespace std::literals;
/// Returns true if the first string is equal to the second string, compared case-insensitively.
inline bool
string_iequal(std::string_view s1, std::string_view s2)
{
return std::equal(s1.begin(), s1.end(), s2.begin(), s2.end(), [](char a, char b) {
return std::tolower(static_cast<unsigned char>(a))
== std::tolower(static_cast<unsigned char>(b));
});
}
/// Returns true if the first string matches any of the given strings case-insensitively.
/// Arguments must be string literals, std::string, or std::string_views
template <typename S1, typename... S>
bool
string_iequal_any(const S1& s1, const S&... s)
{
return (... || string_iequal(s1, s));
}
/// Returns true if the first argument begins with the second argument
inline bool
starts_with(std::string_view str, std::string_view prefix)
{
return str.substr(0, prefix.size()) == prefix;
}
/// Returns true if the first argument ends with the second argument
inline bool
ends_with(std::string_view str, std::string_view suffix)
{
return str.size() >= suffix.size() && str.substr(str.size() - suffix.size()) == suffix;
}
/// Splits a string on some delimiter string and returns a vector of string_view's pointing into
/// the pieces of the original string. The pieces are valid only as long as the original string
/// remains valid. Leading and trailing empty substrings are not removed. If delim is empty you
/// get back a vector of string_views each viewing one character. If `trim` is true then leading
/// and trailing empty values will be suppressed.
///
/// auto v = split("ab--c----de", "--"); // v is {"ab", "c", "", "de"}
/// auto v = split("abc", ""); // v is {"a", "b", "c"}
/// auto v = split("abc", "c"); // v is {"ab", ""}
/// auto v = split("abc", "c", true); // v is {"ab"}
/// auto v = split("-a--b--", "-"); // v is {"", "a", "", "b", "", ""}
/// auto v = split("-a--b--", "-", true); // v is {"a", "", "b"}
///
std::vector<std::string_view>
split(std::string_view str, std::string_view delim, bool trim = false);
/// Splits a string on any 1 or more of the given delimiter characters and returns a vector of
/// string_view's pointing into the pieces of the original string. If delims is empty this works
/// the same as split(). `trim` works like split (suppresses leading and trailing empty string
/// pieces).
///
/// auto v = split_any("abcdedf", "dcx"); // v is {"ab", "e", "f"}
std::vector<std::string_view>
split_any(std::string_view str, std::string_view delims, bool trim = false);
/// Joins [begin, end) with a delimiter and returns the resulting string. Elements can be
/// anything that can be sent to an ostream via `<<`.
template <typename It>
std::string
join(std::string_view delimiter, It begin, It end)
{
std::ostringstream o;
if (begin != end)
o << *begin++;
while (begin != end)
o << delimiter << *begin++;
return o.str();
}
/// Wrapper around the above that takes a container and passes c.begin(), c.end() to the above.
template <typename Container>
std::string
join(std::string_view delimiter, const Container& c)
{
return join(delimiter, c.begin(), c.end());
}
/// Simple version of whitespace trimming: mutates the given string view to remove leading
/// space, \t, \r, \n. (More exotic and locale-dependent whitespace is not removed).
void
trim(std::string_view& s);
/// Parses an integer of some sort from a string, requiring that the entire string be consumed
/// during parsing. Return false if parsing failed, sets `value` and returns true if the entire
/// string was consumed.
template <typename T>
bool
parse_int(const std::string_view str, T& value, int base = 10)
{
T tmp;
auto* strend = str.data() + str.size();
auto [p, ec] = std::from_chars(str.data(), strend, tmp, base);
if (ec != std::errc() || p != strend)
return false;
value = tmp;
return true;
}
std::string
lowercase_ascii_string(std::string src);
/// Converts a duration into a human friendlier string.
std::string
friendly_duration(std::chrono::nanoseconds dur);
} // namespace llarp
#endif

Loading…
Cancel
Save