From 4f90192e1a62c4ea3ce0cd4afb90b11e56407abd Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 3 Jan 2019 23:10:32 +0000 Subject: [PATCH] Fix lifetime issues in llarp_findOrCreate* function calls --- contrib/munin/lokinet-munin.py | 6 +- contrib/py/vanity/lokinet-vanity.py | 222 ++++++++++++++-------------- daemon/rcutil.cpp | 32 ++-- libutp/parse_log.py | 83 ++++++----- llarp/router.cpp | 17 +-- llarp/router.hpp | 4 +- 6 files changed, 180 insertions(+), 184 deletions(-) diff --git a/contrib/munin/lokinet-munin.py b/contrib/munin/lokinet-munin.py index ee1ce6de9..753cec9f2 100644 --- a/contrib/munin/lokinet-munin.py +++ b/contrib/munin/lokinet-munin.py @@ -19,7 +19,7 @@ def exit_sessions_main(): print("graph_category network") print("graph_info This graph shows the number of exit sessions on a lokinet exit") print("lokinet.exit.sessions.info Number of exit sessions") - print("lokinet.exit.sessions.label sessions")) + print("lokinet.exit.sessions.label sessions") else: count = 0 try: @@ -29,7 +29,6 @@ def exit_sessions_main(): pass print("lokinet.exit.sessions {}".format(outbound)) - def peers_main(): if len(sys.argv) == 2 and sys.argv[1] == 'config': @@ -64,4 +63,5 @@ if __name__ == '__main__': elif sys.argv[0] == 'lokinet-exit': exit_sessions_main() else: - print('please symlink this as `lokinet-peers` or `lokinet-exit` in munin plugins dir') + print( + 'please symlink this as `lokinet-peers` or `lokinet-exit` in munin plugins dir') diff --git a/contrib/py/vanity/lokinet-vanity.py b/contrib/py/vanity/lokinet-vanity.py index 09dc061ff..5fc1119dd 100644 --- a/contrib/py/vanity/lokinet-vanity.py +++ b/contrib/py/vanity/lokinet-vanity.py @@ -2,135 +2,137 @@ import bencode import sys import libnacl -import base64 import struct from io import BytesIO import time from multiprocessing import Process, Array, Value - def print_help(): - print('usage: {} keyfile.private prefix numthreads'.format(sys.argv[0])) - return 1 + print('usage: {} keyfile.private prefix numthreads'.format(sys.argv[0])) + return 1 + _zalpha = ['y', 'b', 'n', 'd', 'r', 'f', 'g', '8', - 'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x', - 'o', 't', '1', 'u', 'w', 'i', 's', 'z', - 'a', '3', '4', '5', 'h', '7', '6', '9'] + 'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x', + 'o', 't', '1', 'u', 'w', 'i', 's', 'z', + 'a', '3', '4', '5', 'h', '7', '6', '9'] + def zb32_encode(buf): - s = str() - bits = 0 - l = len(buf) - idx = 0 - tmp = buf[idx] - while bits > 0 or idx < l: - if bits < 5: - if idx < l: - tmp <<= 8 - tmp |= buf[idx] & 0xff - idx += 1 - bits += 8 - else: - tmp <<= 5 - bits - bits = 5 - bits -= 5 - s += _zalpha[(tmp >> bits) & 0x1f] - return s + s = str() + bits = 0 + l = len(buf) + idx = 0 + tmp = buf[idx] + while bits > 0 or idx < l: + if bits < 5: + if idx < l: + tmp <<= 8 + tmp |= buf[idx] & 0xff + idx += 1 + bits += 8 + else: + tmp <<= 5 - bits + bits = 5 + bits -= 5 + s += _zalpha[(tmp >> bits) & 0x1f] + return s def _gen_si(keys): - e = keys[b'e'][32:] - s = keys[b's'][32:] - v = keys[b'v'] - return {'e': e, 's':s, 'v':v} + e = keys[b'e'][32:] + s = keys[b's'][32:] + v = keys[b'v'] + return {'e': e, 's': s, 'v': v} class AddrGen: - def __init__(self, threads, keys, prefix): - self._inc = threads - self._keys = keys - self._c = Value('i') - self.sync = Array('i', 3) - self._procs = [] - self.prefix = prefix - - def runit(self): - for ch in self.prefix: - if ch not in _zalpha: - print("invalid prefix, {} not a valid character".format(ch)) - return None, None - print("find ^{}.loki".format(self.prefix)) - i = self._inc - while i > 0: - p = Process(target=self._gen_addr_tick, args=(self.prefix, abs(libnacl.randombytes_random()), abs(libnacl.randombytes_random()), _gen_si(self._keys))) - p.start() - self._procs.append(p) - i -=1 - return self._runner() - - def _gen_addr_tick(self, prefix, lo, hi, si): - print(prefix) - fd = BytesIO() - addr = '' - enc = bencode.BCodec(fd) - while self.sync[2] == 0: - si['x'] = struct.pack('>QQ', lo, hi) - fd.seek(0,0) - enc.encode(si) - pub = bytes(fd.getbuffer()) - addr = zb32_encode(libnacl.crypto_generichash(pub)) - if addr.startswith(prefix): - self.sync[2] = 1 - self.sync[0] = hi - self.sync[1] = lo - return - hi += self._inc - if hi == 0: - lo += 1 - self._c.value += 1 - - def _print_stats(self): - print('{} H/s'.format(self._c.value)) - self._c.value = 0 - - def _joinall(self): - for p in self._procs: - p.join() - - def _runner(self): - while self.sync[2] == 0: - time.sleep(1) - self._print_stats() - self._joinall() - fd = BytesIO() - enc = bencode.BCodec(fd) - hi = self.sync[0] - lo = self.sync[1] - si = _gen_si(self._keys) - si['x'] = struct.pack('>QQ', lo, hi) - enc.encode(si) - pub = bytes(fd.getbuffer()) - addr = zb32_encode(libnacl.crypto_generichash(pub)) - return si['x'], addr - + def __init__(self, threads, keys, prefix): + self._inc = threads + self._keys = keys + self._c = Value('i') + self.sync = Array('i', 3) + self._procs = [] + self.prefix = prefix + + def runit(self): + for ch in self.prefix: + if ch not in _zalpha: + print("invalid prefix, {} not a valid character".format(ch)) + return None, None + print("find ^{}.loki".format(self.prefix)) + i = self._inc + while i > 0: + p = Process(target=self._gen_addr_tick, args=(self.prefix, abs( + libnacl.randombytes_random()), abs(libnacl.randombytes_random()), _gen_si(self._keys))) + p.start() + self._procs.append(p) + i -= 1 + return self._runner() + + def _gen_addr_tick(self, prefix, lo, hi, si): + print(prefix) + fd = BytesIO() + addr = '' + enc = bencode.BCodec(fd) + while self.sync[2] == 0: + si['x'] = struct.pack('>QQ', lo, hi) + fd.seek(0, 0) + enc.encode(si) + pub = bytes(fd.getbuffer()) + addr = zb32_encode(libnacl.crypto_generichash(pub)) + if addr.startswith(prefix): + self.sync[2] = 1 + self.sync[0] = hi + self.sync[1] = lo + return + hi += self._inc + if hi == 0: + lo += 1 + self._c.value += 1 + + def _print_stats(self): + print('{} H/s'.format(self._c.value)) + self._c.value = 0 + + def _joinall(self): + for p in self._procs: + p.join() + + def _runner(self): + while self.sync[2] == 0: + time.sleep(1) + self._print_stats() + self._joinall() + fd = BytesIO() + enc = bencode.BCodec(fd) + hi = self.sync[0] + lo = self.sync[1] + si = _gen_si(self._keys) + si['x'] = struct.pack('>QQ', lo, hi) + enc.encode(si) + pub = bytes(fd.getbuffer()) + addr = zb32_encode(libnacl.crypto_generichash(pub)) + return si['x'], addr + def main(args): - if len(args) != 3: - return print_help() - keys = None - with open(args[0], 'rb') as fd: - dec = bencode.BCodec(fd) - keys = dec.decode() - runner = AddrGen(int(args[2]), keys, args[1]) - keys[b'x'], addr = runner.runit() - if addr: - print("found {}.loki".format(addr)) - with open(args[0], 'wb') as fd: - enc = bencode.BCodec(fd) - enc.encode(keys) + if len(args) != 3: + return print_help() + keys = None + with open(args[0], 'rb') as fd: + dec = bencode.BCodec(fd) + keys = dec.decode() + runner = AddrGen(int(args[2]), keys, args[1]) + keys[b'x'], addr = runner.runit() + if addr: + print("found {}.loki".format(addr)) + with open(args[0], 'wb') as fd: + enc = bencode.BCodec(fd) + enc.encode(keys) + if __name__ == '__main__': - main(sys.argv[1:]) \ No newline at end of file + main(sys.argv[1:]) diff --git a/daemon/rcutil.cpp b/daemon/rcutil.cpp index c270714c5..dce7d5a7c 100644 --- a/daemon/rcutil.cpp +++ b/daemon/rcutil.cpp @@ -261,8 +261,7 @@ main(int argc, char *argv[]) if(verifyMode) { - llarp::Crypto crypto; - llarp_crypto_init(&crypto); + llarp::Crypto crypto(llarp::Crypto::sodium{}); if(!rc.Read(rcfname)) { std::cout << "failed to read " << rcfname << std::endl; @@ -327,8 +326,7 @@ main(int argc, char *argv[]) // this is the only one... if(listMode) { - llarp::Crypto crypto; - llarp_crypto_init(&crypto); + llarp::Crypto crypto(llarp::Crypto::sodium{}); auto nodedb = llarp_nodedb_new(&crypto); llarp_nodedb_iter itr; itr.visit = [](llarp_nodedb_iter *i) -> bool { @@ -348,8 +346,7 @@ main(int argc, char *argv[]) std::cout << "no file to import" << std::endl; return 1; } - llarp::Crypto crypto; - llarp_crypto_init(&crypto); + llarp::Crypto crypto(llarp::Crypto::sodium{}); auto nodedb = llarp_nodedb_new(&crypto); if(!llarp_nodedb_ensure_dir(nodesdir)) { @@ -389,24 +386,21 @@ main(int argc, char *argv[]) // set updated timestamp rc.last_updated = llarp::time_now_ms(); // load longterm identity - llarp::Crypto crypt; - llarp_crypto_init(&crypt); + llarp::Crypto crypt(llarp::Crypto::sodium{}); // which is in daemon.ini config: router.encryption-privkey (defaults // "encryption.key") fs::path encryption_keyfile = "encryption.key"; llarp::SecretKey encryption; - llarp_findOrCreateEncryption(&crypt, encryption_keyfile.string().c_str(), - encryption); + llarp_findOrCreateEncryption(&crypt, encryption_keyfile, encryption); rc.enckey = llarp::seckey_topublic(encryption); // get identity public sig key fs::path ident_keyfile = "identity.key"; llarp::SecretKey identity; - llarp_findOrCreateIdentity(&crypt, ident_keyfile.string().c_str(), - identity); + llarp_findOrCreateIdentity(&crypt, ident_keyfile, identity); rc.pubkey = llarp::seckey_topublic(identity); @@ -438,15 +432,14 @@ main(int argc, char *argv[]) // set updated timestamp rc.last_updated = llarp::time_now_ms(); // load longterm identity - llarp::Crypto crypt; + llarp::Crypto crypt(llarp::Crypto::sodium{}); // no longer used? // llarp_crypto_libsodium_init(&crypt); llarp::SecretKey identityKey; // FIXME: Jeff requests we use this fs::path ident_keyfile = "identity.key"; - byte_t identity[SECKEYSIZE]; - llarp_findOrCreateIdentity(&crypt, ident_keyfile.string().c_str(), - identity); + llarp::SecretKey identity; + llarp_findOrCreateIdentity(&crypt, ident_keyfile, identity); // FIXME: update RC API // get identity public key @@ -467,10 +460,7 @@ main(int argc, char *argv[]) if(listMode) { - llarp::Crypto crypto; - // no longer used? - // llarp_crypto_libsodium_init(&crypto); - llarp_crypto_init(&crypto); + llarp::Crypto crypto(llarp::Crypto::sodium{}); auto nodedb = llarp_nodedb_new(&crypto); llarp_nodedb_iter itr; itr.visit = [](llarp_nodedb_iter *i) -> bool { @@ -517,7 +507,7 @@ main(int argc, char *argv[]) job->found = false; job->hook = &HandleDHTLocate; // llarp_rc_new(&job->result); - memcpy(job->target, binaryPK, PUBKEYSIZE); // set job's target + job->target = binaryPK; // set job's target // create query DHT request check_online_request *request = new check_online_request; diff --git a/libutp/parse_log.py b/libutp/parse_log.py index 44f69256e..b4be9abfb 100644 --- a/libutp/parse_log.py +++ b/libutp/parse_log.py @@ -1,4 +1,5 @@ -import os, sys, time +import os +import sys # usage: parse_log.py log-file [socket-index to focus on] @@ -7,15 +8,16 @@ socket_filter = None if len(sys.argv) >= 3: socket_filter = sys.argv[2].strip() -if socket_filter == None: +if socket_filter is None: print "scanning for socket with the most packets" file = open(sys.argv[1], 'rb') sockets = {} for l in file: - if not 'our_delay' in l: continue - + if not 'our_delay' in l: + continue + try: a = l.strip().split(" ") socket_index = a[1][:-1] @@ -39,14 +41,15 @@ if socket_filter == None: for i in items: print '%s: %d' % (i[0], i[1]) count += 1 - if count > 5: break - + if count > 5: + break + file.close() socket_filter = items[0][0] print '\nfocusing on socket %s' % socket_filter file = open(sys.argv[1], 'rb') -out_file = 'utp.out%s' % socket_filter; +out_file = 'utp.out%s' % socket_filter out = open(out_file, 'wb') delay_samples = 'dots lc rgb "blue"' @@ -58,24 +61,24 @@ window_size = 'steps lc rgb "sea-green"' rtt = 'lines lc rgb "light-blue"' metrics = { - 'our_delay':['our delay (ms)', 'x1y2', delay_samples], - 'upload_rate':['send rate (B/s)', 'x1y1', 'lines'], - 'max_window':['cwnd (B)', 'x1y1', cwnd], - 'target_delay':['target delay (ms)', 'x1y2', target_delay], - 'cur_window':['bytes in-flight (B)', 'x1y1', window_size], - 'cur_window_packets':['number of packets in-flight', 'x1y2', 'steps'], - 'packet_size':['current packet size (B)', 'x1y2', 'steps'], - 'rtt':['rtt (ms)', 'x1y2', rtt], - 'off_target':['off-target (ms)', 'x1y2', off_target], - 'delay_sum':['delay sum (ms)', 'x1y2', 'steps'], - 'their_delay':['their delay (ms)', 'x1y2', delay_samples], - 'get_microseconds':['clock (us)', 'x1y1', 'steps'], - 'wnduser':['advertised window size (B)', 'x1y1', 'steps'], - - 'delay_base':['delay base (us)', 'x1y1', delay_base], - 'their_delay_base':['their delay base (us)', 'x1y1', delay_base], - 'their_actual_delay':['their actual delay (us)', 'x1y1', delay_samples], - 'actual_delay':['actual_delay (us)', 'x1y1', delay_samples] + 'our_delay': ['our delay (ms)', 'x1y2', delay_samples], + 'upload_rate': ['send rate (B/s)', 'x1y1', 'lines'], + 'max_window': ['cwnd (B)', 'x1y1', cwnd], + 'target_delay': ['target delay (ms)', 'x1y2', target_delay], + 'cur_window': ['bytes in-flight (B)', 'x1y1', window_size], + 'cur_window_packets': ['number of packets in-flight', 'x1y2', 'steps'], + 'packet_size': ['current packet size (B)', 'x1y2', 'steps'], + 'rtt': ['rtt (ms)', 'x1y2', rtt], + 'off_target': ['off-target (ms)', 'x1y2', off_target], + 'delay_sum': ['delay sum (ms)', 'x1y2', 'steps'], + 'their_delay': ['their delay (ms)', 'x1y2', delay_samples], + 'get_microseconds': ['clock (us)', 'x1y1', 'steps'], + 'wnduser': ['advertised window size (B)', 'x1y1', 'steps'], + + 'delay_base': ['delay base (us)', 'x1y1', delay_base], + 'their_delay_base': ['their delay base (us)', 'x1y1', delay_base], + 'their_actual_delay': ['their actual delay (us)', 'x1y1', delay_samples], + 'actual_delay': ['actual_delay (us)', 'x1y1', delay_samples] } histogram_quantization = 1 @@ -115,7 +118,7 @@ for l in file: continue # if socket_index[:2] != '0x': # continue - + if socket_filter != None and socket_index != socket_filter: continue @@ -151,7 +154,7 @@ for l in file: # print time. Convert from milliseconds to seconds print >>out, '%f\t' % (float(t)/1000.), - #if t > 200000: + # if t > 200000: # break fill_columns = not columns @@ -164,12 +167,14 @@ for l in file: if n == "our_delay": bucket = v / histogram_quantization delay_histogram[bucket] = 1 + delay_histogram.get(bucket, 0) - if not n in metrics: continue + if not n in metrics: + continue if fill_columns: columns.append(n) if n == "max_window": window_size[socket_index] = v - print >>out, '%f\t' % int(reduce(lambda a,b: a+b, window_size.values())), + print >>out, '%f\t' % int( + reduce(lambda a, b: a+b, window_size.values())), else: print >>out, '%f\t' % v, print >>out, float(packet_loss * 8000), float(packet_timeout * 8000) @@ -179,8 +184,9 @@ for l in file: out.close() out = open('%s.histogram' % out_file, 'wb') -for d,f in delay_histogram.iteritems(): - print >>out, float(d*histogram_quantization) + histogram_quantization / 2, f +for d, f in delay_histogram.iteritems(): + print >>out, float(d*histogram_quantization) + \ + histogram_quantization / 2, f out.close() @@ -216,7 +222,7 @@ plot = [ 'y2': 'Time (ms)' }, { - 'data': ['their_actual_delay','their_delay_base'], + 'data': ['their_actual_delay', 'their_delay_base'], 'title': 'their_delay_base', 'y1': 'Time (us)', 'y2': '' @@ -253,9 +259,9 @@ print >>out, "set style data steps" #print >>out, "set yrange [0:*]" print >>out, "set y2range [*:*]" files += out_file + '.delays.png ' -#set hidden3d -#set title "Peer bandwidth distribution" -#set xlabel "Ratio" +# set hidden3d +# set title "Peer bandwidth distribution" +# set xlabel "Ratio" for p in plot: print >>out, 'set title "%s %s"' % (p['title'], title) @@ -274,9 +280,11 @@ for p in plot: print >>out, "plot", for c in p['data']: - if not c in metrics: continue + if not c in metrics: + continue i = columns.index(c) - print >>out, '%s"%s" using 1:%d title "%s-%s" axes %s with %s' % (comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]), + print >>out, '%s"%s" using 1:%d title "%s-%s" axes %s with %s' % ( + comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]), comma = ', ' print >>out, '' @@ -285,4 +293,3 @@ out.close() os.system("gnuplot utp.gnuplot") os.system("open %s" % files) - diff --git a/llarp/router.cpp b/llarp/router.cpp index 001393a2b..5ad9fc446 100644 --- a/llarp/router.cpp +++ b/llarp/router.cpp @@ -121,11 +121,10 @@ llarp_router_try_connect(llarp::Router *router, } bool -llarp_findOrCreateIdentity(llarp::Crypto *crypto, const char *fpath, +llarp_findOrCreateIdentity(llarp::Crypto *crypto, const fs::path &path, llarp::SecretKey &secretkey) { - llarp::LogDebug("find or create ", fpath); - fs::path path(fpath); + llarp::LogDebug("find or create ", path); std::error_code ec; if(!fs::exists(path, ec)) { @@ -151,11 +150,10 @@ llarp_findOrCreateIdentity(llarp::Crypto *crypto, const char *fpath, // C++ ... bool -llarp_findOrCreateEncryption(llarp::Crypto *crypto, const char *fpath, +llarp_findOrCreateEncryption(llarp::Crypto *crypto, const fs::path &path, llarp::SecretKey &encryption) { - llarp::LogDebug("find or create ", fpath); - fs::path path(fpath); + llarp::LogDebug("find or create ", path); std::error_code ec; if(!fs::exists(path, ec)) { @@ -397,15 +395,14 @@ namespace llarp { if(!EnsureEncryptionKey()) return false; - return llarp_findOrCreateIdentity(&crypto, ident_keyfile.string().c_str(), - identity); + return llarp_findOrCreateIdentity(&crypto, ident_keyfile, identity); } bool Router::EnsureEncryptionKey() { - return llarp_findOrCreateEncryption( - &crypto, encryption_keyfile.string().c_str(), encryption); + return llarp_findOrCreateEncryption(&crypto, encryption_keyfile, + encryption); } void diff --git a/llarp/router.hpp b/llarp/router.hpp index 41456c732..d3ebba2a4 100644 --- a/llarp/router.hpp +++ b/llarp/router.hpp @@ -33,11 +33,11 @@ #include bool -llarp_findOrCreateEncryption(llarp::Crypto *crypto, const char *fpath, +llarp_findOrCreateEncryption(llarp::Crypto *crypto, const fs::path &fpath, llarp::SecretKey &encryption); bool -llarp_findOrCreateIdentity(llarp::Crypto *crypto, const char *path, +llarp_findOrCreateIdentity(llarp::Crypto *crypto, const fs::path &path, llarp::SecretKey &secretkey); struct TryConnectJob;