From a7c42ab2bdf62cb7ee3deeef6ad4c7fe4613b62a Mon Sep 17 00:00:00 2001 From: Jason Rhinelander Date: Mon, 29 Jun 2020 18:46:25 -0300 Subject: [PATCH] Use lokimq's hex/base32z encoding/decoding --- llarp/CMakeLists.txt | 4 +- llarp/crypto/types.cpp | 10 +- llarp/nodedb.cpp | 5 +- llarp/router/router.cpp | 1 - llarp/router_id.cpp | 29 ++-- llarp/router_id.hpp | 2 +- llarp/service/address.cpp | 48 +++--- llarp/service/address.hpp | 3 +- llarp/util/aligned.hpp | 16 +- llarp/util/encode.cpp | 18 --- llarp/util/encode.hpp | 230 --------------------------- pybind/llarp/router_id.cpp | 6 +- test/CMakeLists.txt | 1 - test/util/test_llarp_util_encode.cpp | 21 --- 14 files changed, 73 insertions(+), 321 deletions(-) delete mode 100644 llarp/util/encode.cpp delete mode 100644 llarp/util/encode.hpp delete mode 100644 test/util/test_llarp_util_encode.cpp diff --git a/llarp/CMakeLists.txt b/llarp/CMakeLists.txt index 12e2adbbe..e87cb6673 100644 --- a/llarp/CMakeLists.txt +++ b/llarp/CMakeLists.txt @@ -4,7 +4,6 @@ add_library(lokinet-util ${CMAKE_CURRENT_BINARY_DIR}/constants/version.cpp util/bencode.cpp util/buffer.cpp - util/encode.cpp util/fs.cpp util/json.cpp util/logging/android_logger.cpp @@ -34,6 +33,7 @@ target_link_libraries(lokinet-util PUBLIC nlohmann_json::nlohmann_json filesystem date::date + lokimq ) if(ANDROID) @@ -222,7 +222,7 @@ if(WITH_HIVE) target_sources(liblokinet PRIVATE tooling/router_hive.cpp) endif() -target_link_libraries(liblokinet PUBLIC cxxopts lokinet-platform lokinet-util lokinet-cryptography lokimq) +target_link_libraries(liblokinet PUBLIC cxxopts lokinet-platform lokinet-util lokinet-cryptography) target_link_libraries(liblokinet PRIVATE libunbound) diff --git a/llarp/crypto/types.cpp b/llarp/crypto/types.cpp index 6a0ab1287..1cf5ca3fd 100644 --- a/llarp/crypto/types.cpp +++ b/llarp/crypto/types.cpp @@ -7,6 +7,8 @@ #include +#include + #include #include #include @@ -16,14 +18,16 @@ namespace llarp bool PubKey::FromString(const std::string& str) { - return HexDecode(str.c_str(), begin(), size()); + if (str.size() != 2 * size()) + return false; + lokimq::from_hex(str.begin(), str.end(), begin()); + return true; } std::string PubKey::ToString() const { - char buf[(PUBKEYSIZE + 1) * 2] = {0}; - return HexEncode(*this, buf); + return lokimq::to_hex(begin(), end()); } bool diff --git a/llarp/nodedb.cpp b/llarp/nodedb.cpp index bbbda8205..3fa4f7eea 100644 --- a/llarp/nodedb.cpp +++ b/llarp/nodedb.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -144,9 +143,7 @@ llarp_nodedb::FindClosestTo(const llarp::dht::Key_t& location, uint32_t numRoute std::string llarp_nodedb::getRCFilePath(const llarp::RouterID& pubkey) const { - char ftmp[68] = {0}; - const char* hexname = llarp::HexEncode, decltype(ftmp)>(pubkey, ftmp); - std::string hexString(hexname); + std::string hexString = lokimq::to_hex(pubkey.begin(), pubkey.end()); std::string skiplistDir; llarp::RouterID r(pubkey); diff --git a/llarp/router/router.cpp b/llarp/router/router.cpp index 0113b14af..0778d658a 100644 --- a/llarp/router/router.cpp +++ b/llarp/router/router.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include diff --git a/llarp/router_id.cpp b/llarp/router_id.cpp index c22556d18..7b0d96731 100644 --- a/llarp/router_id.cpp +++ b/llarp/router_id.cpp @@ -1,18 +1,23 @@ #include +#include namespace llarp { + constexpr std::string_view SNODE_TLD = ".snode"; + std::string RouterID::ToString() const { - char stack[64] = {0}; - return std::string(llarp::Base32Encode(*this, stack)) + ".snode"; + std::string b32 = lokimq::to_base32z(begin(), end()); + b32 += SNODE_TLD; + return b32; } std::string RouterID::ShortString() const { - return ToString().substr(0, 8); + // 5 bytes produces exactly 8 base32z characters: + return lokimq::to_base32z(begin(), begin() + 5); } util::StatusObject @@ -23,13 +28,19 @@ namespace llarp } bool - RouterID::FromString(const std::string& str) + RouterID::FromString(std::string_view str) { - auto pos = str.find(".snode"); - if (pos == std::string::npos || pos == 0) - { + auto pos = str.find(SNODE_TLD); + if (pos != str.size() - SNODE_TLD.size()) + return false; + str.remove_suffix(SNODE_TLD.size()); + // Ensure we have something valid: + // - must end in a 1-bit value: 'o' or 'y' (i.e. 10000 or 00000) + // - must have 51 preceeding base32z chars + // - thus we get 51*5+1 = 256 bits = 32 bytes of output + if (str.size() != 52 || !lokimq::is_base32z(str) || !(str.back() == 'o' || str.back() == 'y')) return false; - } - return Base32Decode(str.substr(0, pos), *this); + lokimq::from_base32z(str.begin(), str.end(), begin()); + return true; } } // namespace llarp diff --git a/llarp/router_id.hpp b/llarp/router_id.hpp index ef58d92a4..71fbe5a93 100644 --- a/llarp/router_id.hpp +++ b/llarp/router_id.hpp @@ -34,7 +34,7 @@ namespace llarp ShortString() const; bool - FromString(const std::string& str); + FromString(std::string_view str); RouterID& operator=(const byte_t* ptr) diff --git a/llarp/service/address.cpp b/llarp/service/address.cpp index 44ff33153..4434e1c5a 100644 --- a/llarp/service/address.cpp +++ b/llarp/service/address.cpp @@ -1,5 +1,6 @@ #include #include +#include #include namespace llarp @@ -21,42 +22,47 @@ namespace llarp { if (!PermitTLD(tld)) return ""; - char tmp[(1 + 32) * 2] = {0}; - std::string str = Base32Encode(*this, tmp); + std::string str; if (subdomain.size()) - str = subdomain + "." + str; - return str + tld; + { + str = subdomain; + str += '.'; + } + str += lokimq::to_base32z(begin(), end()); + str += tld; + return str; } bool - Address::FromString(const std::string& str, const char* tld) + Address::FromString(std::string_view str, const char* tld) { if (!PermitTLD(tld)) return false; - static auto lowercase = [](const std::string s, bool stripDots) -> std::string { - std::string ret(s.size(), ' '); - std::transform(s.begin(), s.end(), ret.begin(), [stripDots](const char& ch) -> char { - if (ch == '.' && stripDots) - return ' '; - return ::tolower(ch); - }); - return ret.substr(0, ret.find_last_of(' ')); - }; + // Find, validate, and remove the .tld const auto pos = str.find_last_of('.'); if (pos == std::string::npos) return false; if (str.substr(pos) != tld) return false; - auto sub = str.substr(0, pos); - // set subdomains if they are there - const auto idx = sub.find_last_of('.'); + str = str.substr(0, pos); + + // copy subdomains if they are there (and strip them off) + const auto idx = str.find_last_of('.'); if (idx != std::string::npos) { - subdomain = lowercase(sub.substr(0, idx), false); - sub = sub.substr(idx + 1); + subdomain = str.substr(0, idx); + str.remove_prefix(idx + 1); } - // make sure it's lowercase - return Base32Decode(lowercase(sub, true), *this); + + // Ensure we have something valid: + // - must end in a 1-bit value: 'o' or 'y' (i.e. 10000 or 00000) + // - must have 51 preceeding base32z chars + // - thus we get 51*5+1 = 256 bits = 32 bytes of output + if (str.size() != 52 || !lokimq::is_base32z(str) || !(str.back() == 'o' || str.back() == 'y')) + return false; + + lokimq::from_base32z(str.begin(), str.end(), begin()); + return true; } dht::Key_t diff --git a/llarp/service/address.hpp b/llarp/service/address.hpp index 1633c30f3..c3aa77ca3 100644 --- a/llarp/service/address.hpp +++ b/llarp/service/address.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace llarp @@ -33,7 +34,7 @@ namespace llarp ToString(const char* tld = ".loki") const; bool - FromString(const std::string& str, const char* tld = ".loki"); + FromString(std::string_view str, const char* tld = ".loki"); Address() : AlignedBuffer<32>() { diff --git a/llarp/util/aligned.hpp b/llarp/util/aligned.hpp index 187447ecd..fa0726466 100644 --- a/llarp/util/aligned.hpp +++ b/llarp/util/aligned.hpp @@ -2,11 +2,12 @@ #define LLARP_ALIGNED_HPP #include -#include #include #include #include +#include + #include #include #include @@ -70,8 +71,7 @@ namespace llarp friend std::ostream& operator<<(std::ostream& out, const AlignedBuffer& self) { - char tmp[(sz * 2) + 1] = {0}; - return out << HexEncode(self, tmp); + return out << lokimq::to_hex(self.begin(), self.end()); } /// bitwise NOT @@ -255,20 +255,22 @@ namespace llarp std::string ToHex() const { - char strbuf[(1 + sz) * 2] = {0}; - return std::string(HexEncode(*this, strbuf)); + return lokimq::to_hex(begin(), end()); } std::string ShortHex() const { - return ToHex().substr(0, 8); + return lokimq::to_hex(begin(), begin() + 4); } bool FromHex(std::string_view str) { - return HexDecode(str.begin(), str.end(), data(), size()); + if (str.size() != 2 * size() || !lokimq::is_hex(str)) + return false; + lokimq::from_hex(str.begin(), str.end(), begin()); + return true; } std::ostream& diff --git a/llarp/util/encode.cpp b/llarp/util/encode.cpp deleted file mode 100644 index a8de4cfdf..000000000 --- a/llarp/util/encode.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include - -#include - -namespace llarp -{ - int - char2int(char input) - { - if (input >= '0' && input <= '9') - return input - '0'; - if (input >= 'A' && input <= 'F') - return input - 'A' + 10; - if (input >= 'a' && input <= 'f') - return input - 'a' + 10; - return 0; - } -} // namespace llarp diff --git a/llarp/util/encode.hpp b/llarp/util/encode.hpp deleted file mode 100644 index b0aab609e..000000000 --- a/llarp/util/encode.hpp +++ /dev/null @@ -1,230 +0,0 @@ -#ifndef LLARP_ENCODE_HPP -#define LLARP_ENCODE_HPP -#include -#include -#include -#include - -namespace llarp -{ - // from https://en.wikipedia.org/wiki/Base32#z-base-32 - static const char zbase32_alpha[] = {'y', 'b', 'n', 'd', 'r', 'f', 'g', '8', 'e', 'j', 'k', - 'm', 'c', 'p', 'q', 'x', 'o', 't', '1', 'u', 'w', 'i', - 's', 'z', 'a', '3', '4', '5', 'h', '7', '6', '9'}; - - static const std::unordered_map zbase32_reverse_alpha = { - {'y', 0}, {'b', 1}, {'n', 2}, {'d', 3}, {'r', 4}, {'f', 5}, {'g', 6}, {'8', 7}, - {'e', 8}, {'j', 9}, {'k', 10}, {'m', 11}, {'c', 12}, {'p', 13}, {'q', 14}, {'x', 15}, - {'o', 16}, {'t', 17}, {'1', 18}, {'u', 19}, {'w', 20}, {'i', 21}, {'s', 22}, {'z', 23}, - {'a', 24}, {'3', 25}, {'4', 26}, {'5', 27}, {'h', 28}, {'7', 29}, {'6', 30}, {'9', 31}}; - - template - static size_t - DecodeSize(size_t sz) - { - auto d = div(sz, a); - if (d.rem) - d.quot++; - return b * d.quot; - } - - static size_t - Base32DecodeSize(size_t sz) - { - return DecodeSize<5, 8>(sz); - } - - template - bool - Base32Decode(const Stack& stack, V& value) - { - int tmp = 0, bits = 0; - size_t idx = 0; - const size_t len = Base32DecodeSize(value.size()); - const size_t outLen = value.size(); - for (size_t i = 0; i < len; i++) - { - char ch = stack[i]; - if (ch) - { - auto itr = zbase32_reverse_alpha.find(ch); - if (itr == zbase32_reverse_alpha.end()) - return false; - ch = itr->second; - } - else - { - return idx == outLen; - } - tmp |= ch; - bits += 5; - if (bits >= 8) - { - if (idx >= outLen) - return false; - value[idx] = tmp >> (bits - 8); - bits -= 8; - idx++; - } - tmp <<= 5; - } - return idx == outLen; - } - - /// adapted from i2pd - template - const char* - Base32Encode(const V& value, Stack& stack) - { - size_t ret = 0, pos = 1; - int bits = 8; - uint32_t tmp = value[0]; - size_t len = value.size(); - while (ret < sizeof(stack) && (bits > 0 || pos < len)) - { - if (bits < 5) - { - if (pos < len) - { - tmp <<= 8; - tmp |= value[pos] & 0xFF; - pos++; - bits += 8; - } - else // last byte - { - tmp <<= (5 - bits); - bits = 5; - } - } - - bits -= 5; - int ind = (tmp >> bits) & 0x1F; - if (ret < sizeof(stack)) - { - stack[ret] = zbase32_alpha[ind]; - ret++; - } - else - return nullptr; - } - return &stack[0]; - } - - /// encode V as hex to stack - /// null terminate - /// return pointer to base of stack buffer on success otherwise returns - /// nullptr - template - const char* - HexEncode(const V& value, Stack& stack) - { - size_t idx = 0; - char* ptr = &stack[0]; - char* end = ptr + sizeof(stack); - while (idx < value.size()) - { - auto wrote = snprintf(ptr, end - ptr, "%.2x", value[idx]); - if (wrote == -1) - return nullptr; - ptr += wrote; - idx++; - } - *ptr = 0; - return &stack[0]; - } - - int - char2int(char input); - - template - bool - HexDecode(const char* src, OutputIt target, size_t sz) - { - while (*src && src[1] && sz) - { - *(target++) = char2int(*src) * 16 + char2int(src[1]); - src += 2; - --sz; - } - return sz == 0; - } - - template - bool - HexDecode(InputIt begin, InputIt end, OutputIt target, size_t sz) - { - while (begin != end && sz) - { - const auto first = *begin++; - const auto second = *begin++; - *(target++) = char2int(first) * 16 + char2int(second); - --sz; - } - return sz == 0; - } - - static const char base64_table[] = { - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', - 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', - 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', - 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; - - template - void - Base64Encode(OStream_t& out, const uint8_t* src, size_t len) - { - size_t i = 0; - size_t j = 0; - uint8_t buf[4] = {0}; - uint8_t tmp[3] = {0}; - while (len--) - { - tmp[i++] = *(src++); - if (3 == i) - { - buf[0] = (tmp[0] & 0xfc) >> 2; - buf[1] = ((tmp[0] & 0x03) << 4) + ((tmp[1] & 0xf0) >> 4); - buf[2] = ((tmp[1] & 0x0f) << 2) + ((tmp[2] & 0xc0) >> 6); - buf[3] = tmp[2] & 0x3f; - - // encode - for (i = 0; i < 4; ++i) - { - out << base64_table[buf[i]]; - } - // reset - i = 0; - } - } - - // remainder - if (i > 0) - { - // fill `tmp' with `\0' at most 3 times - for (j = i; j < 3; ++j) - { - tmp[j] = 0; - } - - // encode remainder - buf[0] = (tmp[0] & 0xfc) >> 2; - buf[1] = ((tmp[0] & 0x03) << 4) + ((tmp[1] & 0xf0) >> 4); - buf[2] = ((tmp[1] & 0x0f) << 2) + ((tmp[2] & 0xc0) >> 6); - buf[3] = tmp[2] & 0x3f; - for (j = 0; (j < i + 1); ++j) - { - out << base64_table[buf[j]]; - } - - // pad - while ((i++ < 3)) - { - out << '='; - } - } - } - -} // namespace llarp - -#endif diff --git a/pybind/llarp/router_id.cpp b/pybind/llarp/router_id.cpp index d371568aa..6c15f8814 100644 --- a/pybind/llarp/router_id.cpp +++ b/pybind/llarp/router_id.cpp @@ -10,8 +10,10 @@ namespace llarp py::class_(mod, "RouterID") .def( "FromHex", - [](RouterID* r, const std::string& hex) -> bool { - return HexDecode(hex.c_str(), r->data(), r->size()); + [](RouterID* r, const std::string& hex) { + if (hex.size() != 2 * r->size() || !lokimq::is_hex(hex)) + throw std::runtime_error("FromHex requires a 64-digit hex string"); + lokimq::from_hex(hex.begin(), hex.end(), r->data()); }) .def("__repr__", &RouterID::ToString) .def("__str__", &RouterID::ToString) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 45f0bbcbf..bd3714233 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -41,7 +41,6 @@ add_executable(testAll util/meta/test_llarp_util_traits.cpp util/test_llarp_util_aligned.cpp util/test_llarp_util_bencode.cpp - util/test_llarp_util_encode.cpp util/test_llarp_util_log_level.cpp util/thread/test_llarp_util_queue_manager.cpp util/thread/test_llarp_util_queue.cpp diff --git a/test/util/test_llarp_util_encode.cpp b/test/util/test_llarp_util_encode.cpp deleted file mode 100644 index cda10ba31..000000000 --- a/test/util/test_llarp_util_encode.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include - -#include -#include - -struct Base32Test : public ::testing::Test -{ - Base32Test() - { - } -}; - -TEST_F(Base32Test, Serialize) -{ - llarp::AlignedBuffer< 32 > addr, otherAddr; - addr.Randomize(); - char tmp[64] = {0}; - std::string encoded = llarp::Base32Encode(addr, tmp); - ASSERT_TRUE(llarp::Base32Decode(tmp, otherAddr)); - ASSERT_TRUE(otherAddr == addr); -}