Crank oxen-mq to (1.2.)11; switch to oxen-encoding

- Update oxen-mq submodule to latest stable
- Add oxen-encoding submodule
- Convert all oxenmq encoding usage to oxenc
- Modernize cmake handling of oxenmq/oxenc
pull/1858/head
Jason Rhinelander 2 years ago
parent 257f48a166
commit d02558350a
No known key found for this signature in database
GPG Key ID: C4992CE7A88D4262

5
.gitmodules vendored

@ -35,4 +35,7 @@
[submodule "external/ngtcp2"]
path = external/ngtcp2
url = https://github.com/ngtcp2/ngtcp2.git
branch = v0.1.0
branch = v0.1.0
[submodule "external/oxen-encoding"]
path = external/oxen-encoding
url = https://github.com/oxen-io/oxen-encoding.git

@ -105,7 +105,6 @@ include(cmake/target_link_libraries_system.cmake)
include(cmake/add_import_library.cmake)
include(cmake/add_log_tag.cmake)
include(cmake/libatomic.cmake)
include(cmake/link_dep_libs.cmake)
if (STATIC_LINK)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX})
@ -180,15 +179,24 @@ if(NOT TARGET sodium)
export(TARGETS sodium NAMESPACE sodium:: FILE sodium-exports.cmake)
endif()
option(FORCE_OXENC_SUBMODULE "force using oxen-encoding submodule" OFF)
if(NOT FORCE_OXENC_SUBMODULE)
pkg_check_modules(OXENC liboxenc>=1.0.1 IMPORTED_TARGET)
endif()
if(OXENC_FOUND)
add_library(oxenc::oxenc ALIAS PkgConfig::OXENC)
message(STATUS "Found system liboxenc ${OXENC_VERSION}")
else()
message(STATUS "using oxen-encoding submodule")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/oxen-encoding)
endif()
option(FORCE_OXENMQ_SUBMODULE "force using oxenmq submodule" OFF)
if(NOT FORCE_OXENMQ_SUBMODULE)
pkg_check_modules(OXENMQ liboxenmq>=1.2.4)
pkg_check_modules(OXENMQ liboxenmq>=1.2.4 IMPORTED_TARGET)
endif()
if(OXENMQ_FOUND)
add_library(oxenmq INTERFACE)
link_dep_libs(oxenmq INTERFACE "${OXENMQ_LIBRARY_DIRS}" ${OXENMQ_LIBRARIES})
target_include_directories(oxenmq INTERFACE ${OXENMQ_INCLUDE_DIRS})
add_library(oxenmq::oxenmq ALIAS oxenmq)
add_library(oxenmq::oxenmq ALIAS PkgConfig::OXENMQ)
message(STATUS "Found system liboxenmq ${OXENMQ_VERSION}")
else()
message(STATUS "using oxenmq submodule")

@ -1,13 +0,0 @@
# Properly links a target to a list of library names by finding the given libraries. Takes:
# - a target
# - a linktype (e.g. INTERFACE, PUBLIC, PRIVATE)
# - a library search path (or "" for defaults)
# - any number of library names
function(link_dep_libs target linktype libdirs)
foreach(lib ${ARGN})
find_library(link_lib-${lib} NAMES ${lib} PATHS ${libdirs})
if(link_lib-${lib})
target_link_libraries(${target} ${linktype} ${link_lib-${lib}})
endif()
endforeach()
endfunction()

@ -22,6 +22,7 @@ if(SUBMODULE_CHECK)
check_submodule(pybind11)
check_submodule(sqlite_orm)
check_submodule(oxen-mq)
check_submodule(oxen-encoding)
check_submodule(uvw)
check_submodule(cpr)
check_submodule(ngtcp2)

@ -0,0 +1 @@
Subproject commit 077cbff7a38ce5c538d9653ffb119855151791c5

2
external/oxen-mq vendored

@ -1 +1 @@
Subproject commit 51754037ea19204610751c2ea8ae72b7ed6c1818
Subproject commit 5c72a57eca120750ecf557ce5a668fb38242956b

@ -7,7 +7,7 @@
#include <iterator>
#include <oxenmq/hex.h>
#include <oxenc/hex.h>
#include <sodium/crypto_sign.h>
#include <sodium/crypto_sign_ed25519.h>
@ -20,14 +20,14 @@ namespace llarp
{
if (str.size() != 2 * size())
return false;
oxenmq::from_hex(str.begin(), str.end(), begin());
oxenc::from_hex(str.begin(), str.end(), begin());
return true;
}
std::string
PubKey::ToString() const
{
return oxenmq::to_hex(begin(), end());
return oxenc::to_hex(begin(), end());
}
bool

@ -1,5 +1,5 @@
#include "findname.hpp"
#include <oxenmq/bt_serialize.h>
#include <oxenc/bt_serialize.h>
#include <llarp/dht/context.hpp>
#include "gotname.hpp"
#include <llarp/router/abstractrouter.hpp>
@ -16,7 +16,7 @@ namespace llarp::dht
bool
FindNameMessage::BEncode(llarp_buffer_t* buf) const
{
const auto data = oxenmq::bt_serialize(oxenmq::bt_dict{
const auto data = oxenc::bt_serialize(oxenc::bt_dict{
{"A", "N"sv},
{"H", std::string_view{(char*)NameHash.data(), NameHash.size()}},
{"T", TxID}});

@ -1,5 +1,5 @@
#include "gotname.hpp"
#include <oxenmq/bt_serialize.h>
#include <oxenc/bt_serialize.h>
#include <llarp/dht/context.hpp>
#include <llarp/router/abstractrouter.hpp>
#include <llarp/path/path_context.hpp>
@ -19,8 +19,8 @@ namespace llarp::dht
GotNameMessage::BEncode(llarp_buffer_t* buf) const
{
const std::string nonce((const char*)result.nonce.data(), result.nonce.size());
const auto data = oxenmq::bt_serialize(
oxenmq::bt_dict{{"A", "M"sv}, {"D", result.ciphertext}, {"N", nonce}, {"T", TxID}});
const auto data = oxenc::bt_serialize(
oxenc::bt_dict{{"A", "M"sv}, {"D", result.ciphertext}, {"N", nonce}, {"T", TxID}});
return buf->write(data.begin(), data.end());
}

@ -4,7 +4,7 @@
#include <limits>
#include <oxenmq/bt_serialize.h>
#include <oxenc/bt_serialize.h>
#include "llarp/util/bencode.h"
#include "llarp/util/types.hpp"
@ -106,7 +106,7 @@ namespace llarp::dns
bool
SRVData::BEncode(llarp_buffer_t* buf) const
{
const std::string data = oxenmq::bt_serialize(toTuple());
const std::string data = oxenc::bt_serialize(toTuple());
return buf->write(data.begin(), data.end());
}
@ -122,11 +122,11 @@ namespace llarp::dns
try
{
SRVTuple tuple{};
oxenmq::bt_deserialize(srvString, tuple);
oxenc::bt_deserialize(srvString, tuple);
*this = fromTuple(std::move(tuple));
return IsValid();
}
catch (const oxenmq::bt_deserialize_invalid&)
catch (const oxenc::bt_deserialize_invalid&)
{
return false;
};

@ -14,7 +14,7 @@
#include <optional>
#include <unordered_set>
#include <set>
#include "oxenmq/variant.h"
#include "oxenc/variant.h"
namespace llarp
{

@ -4,7 +4,7 @@
#include <llarp/net/ip_packet.hpp>
#include <set>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
namespace llarp
{

@ -274,7 +274,7 @@ namespace llarp
{
std::string_view bdata{data.data(), data.size()};
LogDebug(Name(), " parsing address map data: ", bdata);
const auto parsed = oxenmq::bt_deserialize<oxenmq::bt_dict>(bdata);
const auto parsed = oxenc::bt_deserialize<oxenc::bt_dict>(bdata);
for (const auto& [key, value] : parsed)
{
huint128_t ip{};
@ -1011,7 +1011,7 @@ namespace llarp
addrmap[ip.ToString()] = a.ToString();
}
}
const auto data = oxenmq::bt_serialize(addrmap);
const auto data = oxenc::bt_serialize(addrmap);
maybe->write(data.data(), data.size());
}
}

@ -768,7 +768,7 @@ namespace llarp
pos += sizeof(rxid);
auto p2 = pos + ShortHash::SIZE;
assert(p2 == data.data() + XMITOverhead);
LogTrace("rxid=", rxid, " sz=", sz, " h=", oxenmq::to_hex(pos, p2), " from ", m_RemoteAddr);
LogTrace("rxid=", rxid, " sz=", sz, " h=", oxenc::to_hex(pos, p2), " from ", m_RemoteAddr);
m_LastRX = m_Parent->Now();
{
// check for replay

@ -10,7 +10,7 @@
#include <llarp/util/logging/buffer.hpp>
#include <oxenmq/base32z.h>
#include <oxenc/base32z.h>
#include <mutex>
#include <memory>
@ -823,7 +823,7 @@ extern "C"
lokinet_hex_to_base32z(const char* hex)
{
std::string_view hexview{hex};
if (not oxenmq::is_hex(hexview))
if (not oxenc::is_hex(hexview))
return nullptr;
const size_t byte_len = hexview.size() / 2;
@ -834,9 +834,9 @@ extern "C"
// Write the bytes into the *end* of the buffer so that when we rewrite the final b32z chars
// into the buffer we won't overwrite any byte values until after we've consumed them.
char* bytepos = end - byte_len;
oxenmq::from_hex(hexview.begin(), hexview.end(), bytepos);
oxenc::from_hex(hexview.begin(), hexview.end(), bytepos);
// In-place conversion into the buffer
oxenmq::to_base32z(bytepos, end, buf.get());
oxenc::to_base32z(bytepos, end, buf.get());
return buf.release(); // leak the buffer to the caller
}

@ -1,6 +1,6 @@
#include "ip_range.hpp"
#include "oxenmq/bt_serialize.h"
#include "oxenc/bt_serialize.h"
#include "llarp/util/bencode.h"
@ -9,7 +9,7 @@ namespace llarp
bool
IPRange::BEncode(llarp_buffer_t* buf) const
{
const auto str = oxenmq::bt_serialize(ToString());
const auto str = oxenc::bt_serialize(ToString());
return buf->write(str.begin(), str.end());
}
@ -24,7 +24,7 @@ namespace llarp
std::string str;
try
{
oxenmq::bt_deserialize(data, str);
oxenc::bt_deserialize(data, str);
}
catch (std::exception&)
{

@ -93,7 +93,7 @@ namespace llarp
fs::path
NodeDB::GetPathForPubkey(RouterID pubkey) const
{
std::string hexString = oxenmq::to_hex(pubkey.begin(), pubkey.end());
std::string hexString = oxenc::to_hex(pubkey.begin(), pubkey.end());
std::string skiplistDir;
const llarp::RouterID r{pubkey};

@ -1,7 +1,7 @@
#include "types.hpp"
#include <llarp/util/str.hpp>
#include <oxenmq/bt_serialize.h>
#include <oxenc/bt_serialize.h>
#include <stdexcept>
namespace llarp
@ -103,7 +103,7 @@ namespace llarp
{
if (not buf)
throw std::runtime_error("PeerStats: Can't use null buf");
const oxenmq::bt_dict data = {
const oxenc::bt_dict data = {
{NumConnectionAttemptsKey, numConnectionAttempts},
{NumConnectionSuccessesKey, numConnectionSuccesses},
{NumConnectionRejectionsKey, numConnectionRejections},
@ -120,7 +120,7 @@ namespace llarp
{LeastRCRemainingLifetimeKey, leastRCRemainingLifetime.count()},
{LastRCUpdatedKey, lastRCUpdated.count()},
};
const auto serialized = oxenmq::bt_serialize(data);
const auto serialized = oxenc::bt_serialize(data);
if (not buf->write(serialized.begin(), serialized.end()))
throw std::runtime_error("PeerStats: buffer too small");
}

@ -3,7 +3,7 @@
#include <llarp/util/logging/buffer.hpp>
#include <llarp/util/logging/logger.hpp>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
#include <llarp/service/address.hpp>
#include <llarp/service/endpoint.hpp>
#include <llarp/ev/ev_libuv.hpp>

@ -15,8 +15,8 @@
#include <uvw/timer.h>
#include <iterator>
#include <oxenmq/hex.h>
#include <oxenmq/bt_serialize.h>
#include <oxenc/hex.h>
#include <oxenc/bt_serialize.h>
extern "C"
{
@ -35,7 +35,7 @@ namespace llarp::quic
std::ostream&
operator<<(std::ostream& o, const ConnectionID& c)
{
return o << oxenmq::to_hex(c.data, c.data + c.datalen);
return o << oxenc::to_hex(c.data, c.data + c.datalen);
}
ConnectionID
@ -297,7 +297,7 @@ namespace llarp::quic
conn.endpoint.make_stateless_reset_token(cid, token);
LogDebug(
"make stateless reset token ",
oxenmq::to_hex(token, token + NGTCP2_STATELESS_RESET_TOKENLEN));
oxenc::to_hex(token, token + NGTCP2_STATELESS_RESET_TOKENLEN));
return 0;
}
@ -1093,7 +1093,7 @@ namespace llarp::quic
uint16_t port;
try
{
oxenmq::bt_dict_consumer meta{lokinet_metadata};
oxenc::bt_dict_consumer meta{lokinet_metadata};
// '#' contains the port the client wants us to forward to
if (!meta.skip_until("#"))
{
@ -1108,7 +1108,7 @@ namespace llarp::quic
}
LogDebug("decoded lokinet tunnel port = ", port);
}
catch (const oxenmq::bt_deserialize_invalid& c)
catch (const oxenc::bt_deserialize_invalid& c)
{
LogWarn("transport params lokinet metadata is invalid: ", c.what());
return NGTCP2_ERR_TRANSPORT_PARAM;
@ -1191,7 +1191,7 @@ namespace llarp::quic
// reserved field code that QUIC parsers must ignore); currently we only include the port in
// here (from the client to tell the server what it's trying to reach, and reflected from
// the server for the client to verify).
std::string lokinet_metadata = bt_serialize(oxenmq::bt_dict{
std::string lokinet_metadata = bt_serialize(oxenc::bt_dict{
{"#", tunnel_port},
});
copy_and_advance(buf, lokinet_metadata_code);

@ -12,7 +12,7 @@
#include <variant>
#include <uvw/timer.h>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
extern "C"
{

@ -2,8 +2,7 @@
#include <llarp/util/logging/buffer.hpp>
#include <llarp/util/logging/logger.hpp>
#include <oxenmq/hex.h>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
#include <uvw/loop.h>
#include <stdexcept>

@ -7,7 +7,7 @@
#include <functional>
#include <string_view>
#include <type_traits>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
#include <vector>
#include <optional>
#include <uvw/async.h>

@ -170,7 +170,7 @@ namespace llarp::quic
{
LogWarn(
"Remote connection returned invalid initial byte (0x",
oxenmq::to_hex(bdata.begin(), bdata.begin() + 1),
oxenc::to_hex(bdata.begin(), bdata.begin() + 1),
"); dropping connection");
stream.close(tunnel::ERROR_BAD_INIT);
client.close();

@ -10,7 +10,7 @@
#include "util/printer.hpp"
#include "util/time.hpp"
#include <oxenmq/bt_serialize.h>
#include <oxenc/bt_serialize.h>
#include <fstream>
#include "util/fs.hpp"
@ -267,7 +267,7 @@ namespace llarp
try
{
std::string_view buf_view(reinterpret_cast<char*>(buf->cur), buf->size_left());
oxenmq::bt_list_consumer btlist(buf_view);
oxenc::bt_list_consumer btlist(buf_view);
uint64_t outer_version = btlist.consume_integer<uint64_t>();
@ -301,7 +301,7 @@ namespace llarp
}
bool
RouterContact::DecodeVersion_1(oxenmq::bt_list_consumer& btlist)
RouterContact::DecodeVersion_1(oxenc::bt_list_consumer& btlist)
{
auto signature_string = btlist.consume_string_view();
signed_bt_dict = btlist.consume_dict_data();

@ -18,10 +18,10 @@
#define MAX_RC_SIZE (1024)
#define NICKLEN (32)
namespace oxenmq
namespace oxenc
{
class bt_list_consumer;
} // namespace oxenmq
} // namespace oxenc
namespace llarp
{
@ -226,7 +226,7 @@ namespace llarp
DecodeVersion_0(llarp_buffer_t* buf);
bool
DecodeVersion_1(oxenmq::bt_list_consumer& btlist);
DecodeVersion_1(oxenc::bt_list_consumer& btlist);
};
inline std::ostream&

@ -1,5 +1,5 @@
#include "router_id.hpp"
#include <oxenmq/base32z.h>
#include <oxenc/base32z.h>
namespace llarp
{
@ -8,7 +8,7 @@ namespace llarp
std::string
RouterID::ToString() const
{
std::string b32 = oxenmq::to_base32z(begin(), end());
std::string b32 = oxenc::to_base32z(begin(), end());
b32 += SNODE_TLD;
return b32;
}
@ -17,7 +17,7 @@ namespace llarp
RouterID::ShortString() const
{
// 5 bytes produces exactly 8 base32z characters:
return oxenmq::to_base32z(begin(), begin() + 5);
return oxenc::to_base32z(begin(), begin() + 5);
}
util::StatusObject
@ -38,9 +38,9 @@ namespace llarp
// - must end in a 1-bit value: 'o' or 'y' (i.e. 10000 or 00000)
// - must have 51 preceeding base32z chars
// - thus we get 51*5+1 = 256 bits = 32 bytes of output
if (str.size() != 52 || !oxenmq::is_base32z(str) || !(str.back() == 'o' || str.back() == 'y'))
if (str.size() != 52 || !oxenc::is_base32z(str) || !(str.back() == 'o' || str.back() == 'y'))
return false;
oxenmq::from_base32z(str.begin(), str.end(), begin());
oxenc::from_base32z(str.begin(), str.end(), begin());
return true;
}
} // namespace llarp

@ -334,8 +334,8 @@ namespace llarp
{
service::EncryptedName result;
const auto j = nlohmann::json::parse(data[1]);
result.ciphertext = oxenmq::from_hex(j["encrypted_value"].get<std::string>());
const auto nonce = oxenmq::from_hex(j["nonce"].get<std::string>());
result.ciphertext = oxenc::from_hex(j["encrypted_value"].get<std::string>());
const auto nonce = oxenc::from_hex(j["nonce"].get<std::string>());
if (nonce.size() != result.nonce.size())
{
throw std::invalid_argument(stringify(
@ -392,7 +392,7 @@ namespace llarp
}
std::vector<std::string> routerIdStrings;
oxenmq::bt_deserialize(msg.data[0], routerIdStrings);
oxenc::bt_deserialize(msg.data[0], routerIdStrings);
std::vector<RouterID> routerIds;
routerIds.reserve(routerIdStrings.size());

@ -1,6 +1,6 @@
#include "address.hpp"
#include <llarp/crypto/crypto.hpp>
#include <oxenmq/base32z.h>
#include <oxenc/base32z.h>
#include <algorithm>
namespace llarp::service
@ -26,7 +26,7 @@ namespace llarp::service
str = subdomain;
str += '.';
}
str += oxenmq::to_base32z(begin(), end());
str += oxenc::to_base32z(begin(), end());
str += tld;
return str;
}
@ -56,10 +56,10 @@ namespace llarp::service
// - must end in a 1-bit value: 'o' or 'y' (i.e. 10000 or 00000)
// - must have 51 preceeding base32z chars
// - thus we get 51*5+1 = 256 bits = 32 bytes of output
if (str.size() != 52 || !oxenmq::is_base32z(str) || !(str.back() == 'o' || str.back() == 'y'))
if (str.size() != 52 || !oxenc::is_base32z(str) || !(str.back() == 'o' || str.back() == 'y'))
return false;
oxenmq::from_base32z(str.begin(), str.end(), begin());
oxenc::from_base32z(str.begin(), str.end(), begin());
return true;
}

@ -20,14 +20,12 @@
#include <optional>
#include <unordered_map>
#include <variant>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
#include "endpoint_types.hpp"
#include "llarp/endpoint_base.hpp"
#include "auth.hpp"
#include <oxenmq/variant.h>
#include <llarp/vpn/egres_packet_router.hpp>
// minimum time between introset shifts

@ -17,7 +17,7 @@
#include <set>
#include <unordered_map>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
namespace llarp
{

@ -2,7 +2,7 @@
#include <llarp/crypto/crypto.hpp>
#include <llarp/path/path.hpp>
#include <oxenmq/bt_serialize.h>
#include <oxenc/bt_serialize.h>
namespace llarp::service
{
@ -235,9 +235,9 @@ namespace llarp::service
try
{
oxenmq::bt_deserialize(srvString, SRVs);
oxenc::bt_deserialize(srvString, SRVs);
}
catch (const oxenmq::bt_deserialize_invalid& err)
catch (const oxenc::bt_deserialize_invalid& err)
{
LogError("Error decoding SRV records from IntroSet: ", err.what());
return false;
@ -321,7 +321,7 @@ namespace llarp::service
// srv records
if (not SRVs.empty())
{
std::string serial = oxenmq::bt_serialize(SRVs);
std::string serial = oxenc::bt_serialize(SRVs);
if (!bencode_write_bytestring(buf, "s", 1))
return false;
if (!buf->write(serial.begin(), serial.end()))

@ -8,7 +8,7 @@
#include "address.hpp"
#include <llarp/router_id.hpp>
#include <oxenmq/variant.h>
#include <oxenc/variant.h>
namespace llarp::service
{

@ -5,7 +5,7 @@
#include <llarp/util/meta/traits.hpp>
#include "printer.hpp"
#include <oxenmq/hex.h>
#include <oxenc/hex.h>
#include <array>
#include <cstddef>
@ -72,7 +72,7 @@ namespace llarp
friend std::ostream&
operator<<(std::ostream& out, const AlignedBuffer& self)
{
return out << oxenmq::to_hex(self.begin(), self.end());
return out << oxenc::to_hex(self.begin(), self.end());
}
/// bitwise NOT
@ -270,21 +270,21 @@ namespace llarp
std::string
ToHex() const
{
return oxenmq::to_hex(begin(), end());
return oxenc::to_hex(begin(), end());
}
std::string
ShortHex() const
{
return oxenmq::to_hex(begin(), begin() + 4);
return oxenc::to_hex(begin(), begin() + 4);
}
bool
FromHex(std::string_view str)
{
if (str.size() != 2 * size() || !oxenmq::is_hex(str))
if (str.size() != 2 * size() || !oxenc::is_hex(str))
return false;
oxenmq::from_hex(str.begin(), str.end(), begin());
oxenc::from_hex(str.begin(), str.end(), begin());
return true;
}

@ -416,10 +416,10 @@ namespace llarp::vpn
if (parts[1].find_first_not_of('0') == std::string::npos and parts[0] != ifname)
{
const auto& ip = parts[2];
if ((ip.size() == sizeof(uint32_t) * 2) and oxenmq::is_hex(ip))
if ((ip.size() == sizeof(uint32_t) * 2) and oxenc::is_hex(ip))
{
huint32_t x{};
oxenmq::from_hex(ip.begin(), ip.end(), reinterpret_cast<char*>(&x.h));
oxenc::from_hex(ip.begin(), ip.end(), reinterpret_cast<char*>(&x.h));
gateways.emplace_back(x);
}
}

@ -11,9 +11,9 @@ namespace llarp
.def(
"FromHex",
[](RouterID* r, const std::string& hex) {
if (hex.size() != 2 * r->size() || !oxenmq::is_hex(hex))
if (hex.size() != 2 * r->size() || !oxenc::is_hex(hex))
throw std::runtime_error("FromHex requires a 64-digit hex string");
oxenmq::from_hex(hex.begin(), hex.end(), r->data());
oxenc::from_hex(hex.begin(), hex.end(), r->data());
})
.def("__repr__", &RouterID::ToString)
.def("__str__", &RouterID::ToString)

Loading…
Cancel
Save