Merge pull request #196 from michael-loki/fs_path_lifetime

Fix lifetime issues in llarp_findOrCreate* function calls
pull/198/head
Jeff 6 years ago committed by GitHub
commit 578cbbeb07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -19,7 +19,7 @@ def exit_sessions_main():
print("graph_category network") print("graph_category network")
print("graph_info This graph shows the number of exit sessions on a lokinet exit") print("graph_info This graph shows the number of exit sessions on a lokinet exit")
print("lokinet.exit.sessions.info Number of exit sessions") print("lokinet.exit.sessions.info Number of exit sessions")
print("lokinet.exit.sessions.label sessions")) print("lokinet.exit.sessions.label sessions")
else: else:
count = 0 count = 0
try: try:
@ -30,7 +30,6 @@ def exit_sessions_main():
print("lokinet.exit.sessions {}".format(outbound)) print("lokinet.exit.sessions {}".format(outbound))
def peers_main(): def peers_main():
if len(sys.argv) == 2 and sys.argv[1] == 'config': if len(sys.argv) == 2 and sys.argv[1] == 'config':
print("graph_title lokinet peers") print("graph_title lokinet peers")
@ -64,4 +63,5 @@ if __name__ == '__main__':
elif sys.argv[0] == 'lokinet-exit': elif sys.argv[0] == 'lokinet-exit':
exit_sessions_main() exit_sessions_main()
else: else:
print('please symlink this as `lokinet-peers` or `lokinet-exit` in munin plugins dir') print(
'please symlink this as `lokinet-peers` or `lokinet-exit` in munin plugins dir')

@ -2,135 +2,137 @@
import bencode import bencode
import sys import sys
import libnacl import libnacl
import base64
import struct import struct
from io import BytesIO from io import BytesIO
import time import time
from multiprocessing import Process, Array, Value from multiprocessing import Process, Array, Value
def print_help(): def print_help():
print('usage: {} keyfile.private prefix numthreads'.format(sys.argv[0])) print('usage: {} keyfile.private prefix numthreads'.format(sys.argv[0]))
return 1 return 1
_zalpha = ['y', 'b', 'n', 'd', 'r', 'f', 'g', '8', _zalpha = ['y', 'b', 'n', 'd', 'r', 'f', 'g', '8',
'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x', 'e', 'j', 'k', 'm', 'c', 'p', 'q', 'x',
'o', 't', '1', 'u', 'w', 'i', 's', 'z', 'o', 't', '1', 'u', 'w', 'i', 's', 'z',
'a', '3', '4', '5', 'h', '7', '6', '9'] 'a', '3', '4', '5', 'h', '7', '6', '9']
def zb32_encode(buf): def zb32_encode(buf):
s = str() s = str()
bits = 0 bits = 0
l = len(buf) l = len(buf)
idx = 0 idx = 0
tmp = buf[idx] tmp = buf[idx]
while bits > 0 or idx < l: while bits > 0 or idx < l:
if bits < 5: if bits < 5:
if idx < l: if idx < l:
tmp <<= 8 tmp <<= 8
tmp |= buf[idx] & 0xff tmp |= buf[idx] & 0xff
idx += 1 idx += 1
bits += 8 bits += 8
else: else:
tmp <<= 5 - bits tmp <<= 5 - bits
bits = 5 bits = 5
bits -= 5 bits -= 5
s += _zalpha[(tmp >> bits) & 0x1f] s += _zalpha[(tmp >> bits) & 0x1f]
return s return s
def _gen_si(keys): def _gen_si(keys):
e = keys[b'e'][32:] e = keys[b'e'][32:]
s = keys[b's'][32:] s = keys[b's'][32:]
v = keys[b'v'] v = keys[b'v']
return {'e': e, 's':s, 'v':v} return {'e': e, 's': s, 'v': v}
class AddrGen: class AddrGen:
def __init__(self, threads, keys, prefix): def __init__(self, threads, keys, prefix):
self._inc = threads self._inc = threads
self._keys = keys self._keys = keys
self._c = Value('i') self._c = Value('i')
self.sync = Array('i', 3) self.sync = Array('i', 3)
self._procs = [] self._procs = []
self.prefix = prefix self.prefix = prefix
def runit(self): def runit(self):
for ch in self.prefix: for ch in self.prefix:
if ch not in _zalpha: if ch not in _zalpha:
print("invalid prefix, {} not a valid character".format(ch)) print("invalid prefix, {} not a valid character".format(ch))
return None, None return None, None
print("find ^{}.loki".format(self.prefix)) print("find ^{}.loki".format(self.prefix))
i = self._inc i = self._inc
while i > 0: while i > 0:
p = Process(target=self._gen_addr_tick, args=(self.prefix, abs(libnacl.randombytes_random()), abs(libnacl.randombytes_random()), _gen_si(self._keys))) p = Process(target=self._gen_addr_tick, args=(self.prefix, abs(
p.start() libnacl.randombytes_random()), abs(libnacl.randombytes_random()), _gen_si(self._keys)))
self._procs.append(p) p.start()
i -=1 self._procs.append(p)
return self._runner() i -= 1
return self._runner()
def _gen_addr_tick(self, prefix, lo, hi, si):
print(prefix) def _gen_addr_tick(self, prefix, lo, hi, si):
fd = BytesIO() print(prefix)
addr = '' fd = BytesIO()
enc = bencode.BCodec(fd) addr = ''
while self.sync[2] == 0: enc = bencode.BCodec(fd)
si['x'] = struct.pack('>QQ', lo, hi) while self.sync[2] == 0:
fd.seek(0,0) si['x'] = struct.pack('>QQ', lo, hi)
enc.encode(si) fd.seek(0, 0)
pub = bytes(fd.getbuffer()) enc.encode(si)
addr = zb32_encode(libnacl.crypto_generichash(pub)) pub = bytes(fd.getbuffer())
if addr.startswith(prefix): addr = zb32_encode(libnacl.crypto_generichash(pub))
self.sync[2] = 1 if addr.startswith(prefix):
self.sync[0] = hi self.sync[2] = 1
self.sync[1] = lo self.sync[0] = hi
return self.sync[1] = lo
hi += self._inc return
if hi == 0: hi += self._inc
lo += 1 if hi == 0:
self._c.value += 1 lo += 1
self._c.value += 1
def _print_stats(self):
print('{} H/s'.format(self._c.value)) def _print_stats(self):
self._c.value = 0 print('{} H/s'.format(self._c.value))
self._c.value = 0
def _joinall(self):
for p in self._procs: def _joinall(self):
p.join() for p in self._procs:
p.join()
def _runner(self):
while self.sync[2] == 0: def _runner(self):
time.sleep(1) while self.sync[2] == 0:
self._print_stats() time.sleep(1)
self._joinall() self._print_stats()
fd = BytesIO() self._joinall()
enc = bencode.BCodec(fd) fd = BytesIO()
hi = self.sync[0] enc = bencode.BCodec(fd)
lo = self.sync[1] hi = self.sync[0]
si = _gen_si(self._keys) lo = self.sync[1]
si['x'] = struct.pack('>QQ', lo, hi) si = _gen_si(self._keys)
enc.encode(si) si['x'] = struct.pack('>QQ', lo, hi)
pub = bytes(fd.getbuffer()) enc.encode(si)
addr = zb32_encode(libnacl.crypto_generichash(pub)) pub = bytes(fd.getbuffer())
return si['x'], addr addr = zb32_encode(libnacl.crypto_generichash(pub))
return si['x'], addr
def main(args): def main(args):
if len(args) != 3: if len(args) != 3:
return print_help() return print_help()
keys = None keys = None
with open(args[0], 'rb') as fd: with open(args[0], 'rb') as fd:
dec = bencode.BCodec(fd) dec = bencode.BCodec(fd)
keys = dec.decode() keys = dec.decode()
runner = AddrGen(int(args[2]), keys, args[1]) runner = AddrGen(int(args[2]), keys, args[1])
keys[b'x'], addr = runner.runit() keys[b'x'], addr = runner.runit()
if addr: if addr:
print("found {}.loki".format(addr)) print("found {}.loki".format(addr))
with open(args[0], 'wb') as fd: with open(args[0], 'wb') as fd:
enc = bencode.BCodec(fd) enc = bencode.BCodec(fd)
enc.encode(keys) enc.encode(keys)
if __name__ == '__main__': if __name__ == '__main__':
main(sys.argv[1:]) main(sys.argv[1:])

@ -261,8 +261,7 @@ main(int argc, char *argv[])
if(verifyMode) if(verifyMode)
{ {
llarp::Crypto crypto; llarp::Crypto crypto(llarp::Crypto::sodium{});
llarp_crypto_init(&crypto);
if(!rc.Read(rcfname)) if(!rc.Read(rcfname))
{ {
std::cout << "failed to read " << rcfname << std::endl; std::cout << "failed to read " << rcfname << std::endl;
@ -327,8 +326,7 @@ main(int argc, char *argv[])
// this is the only one... // this is the only one...
if(listMode) if(listMode)
{ {
llarp::Crypto crypto; llarp::Crypto crypto(llarp::Crypto::sodium{});
llarp_crypto_init(&crypto);
auto nodedb = llarp_nodedb_new(&crypto); auto nodedb = llarp_nodedb_new(&crypto);
llarp_nodedb_iter itr; llarp_nodedb_iter itr;
itr.visit = [](llarp_nodedb_iter *i) -> bool { itr.visit = [](llarp_nodedb_iter *i) -> bool {
@ -348,8 +346,7 @@ main(int argc, char *argv[])
std::cout << "no file to import" << std::endl; std::cout << "no file to import" << std::endl;
return 1; return 1;
} }
llarp::Crypto crypto; llarp::Crypto crypto(llarp::Crypto::sodium{});
llarp_crypto_init(&crypto);
auto nodedb = llarp_nodedb_new(&crypto); auto nodedb = llarp_nodedb_new(&crypto);
if(!llarp_nodedb_ensure_dir(nodesdir)) if(!llarp_nodedb_ensure_dir(nodesdir))
{ {
@ -389,24 +386,21 @@ main(int argc, char *argv[])
// set updated timestamp // set updated timestamp
rc.last_updated = llarp::time_now_ms(); rc.last_updated = llarp::time_now_ms();
// load longterm identity // load longterm identity
llarp::Crypto crypt; llarp::Crypto crypt(llarp::Crypto::sodium{});
llarp_crypto_init(&crypt);
// which is in daemon.ini config: router.encryption-privkey (defaults // which is in daemon.ini config: router.encryption-privkey (defaults
// "encryption.key") // "encryption.key")
fs::path encryption_keyfile = "encryption.key"; fs::path encryption_keyfile = "encryption.key";
llarp::SecretKey encryption; llarp::SecretKey encryption;
llarp_findOrCreateEncryption(&crypt, encryption_keyfile.string().c_str(), llarp_findOrCreateEncryption(&crypt, encryption_keyfile, encryption);
encryption);
rc.enckey = llarp::seckey_topublic(encryption); rc.enckey = llarp::seckey_topublic(encryption);
// get identity public sig key // get identity public sig key
fs::path ident_keyfile = "identity.key"; fs::path ident_keyfile = "identity.key";
llarp::SecretKey identity; llarp::SecretKey identity;
llarp_findOrCreateIdentity(&crypt, ident_keyfile.string().c_str(), llarp_findOrCreateIdentity(&crypt, ident_keyfile, identity);
identity);
rc.pubkey = llarp::seckey_topublic(identity); rc.pubkey = llarp::seckey_topublic(identity);
@ -438,15 +432,14 @@ main(int argc, char *argv[])
// set updated timestamp // set updated timestamp
rc.last_updated = llarp::time_now_ms(); rc.last_updated = llarp::time_now_ms();
// load longterm identity // load longterm identity
llarp::Crypto crypt; llarp::Crypto crypt(llarp::Crypto::sodium{});
// no longer used? // no longer used?
// llarp_crypto_libsodium_init(&crypt); // llarp_crypto_libsodium_init(&crypt);
llarp::SecretKey identityKey; // FIXME: Jeff requests we use this llarp::SecretKey identityKey; // FIXME: Jeff requests we use this
fs::path ident_keyfile = "identity.key"; fs::path ident_keyfile = "identity.key";
byte_t identity[SECKEYSIZE]; llarp::SecretKey identity;
llarp_findOrCreateIdentity(&crypt, ident_keyfile.string().c_str(), llarp_findOrCreateIdentity(&crypt, ident_keyfile, identity);
identity);
// FIXME: update RC API // FIXME: update RC API
// get identity public key // get identity public key
@ -467,10 +460,7 @@ main(int argc, char *argv[])
if(listMode) if(listMode)
{ {
llarp::Crypto crypto; llarp::Crypto crypto(llarp::Crypto::sodium{});
// no longer used?
// llarp_crypto_libsodium_init(&crypto);
llarp_crypto_init(&crypto);
auto nodedb = llarp_nodedb_new(&crypto); auto nodedb = llarp_nodedb_new(&crypto);
llarp_nodedb_iter itr; llarp_nodedb_iter itr;
itr.visit = [](llarp_nodedb_iter *i) -> bool { itr.visit = [](llarp_nodedb_iter *i) -> bool {
@ -517,7 +507,7 @@ main(int argc, char *argv[])
job->found = false; job->found = false;
job->hook = &HandleDHTLocate; job->hook = &HandleDHTLocate;
// llarp_rc_new(&job->result); // llarp_rc_new(&job->result);
memcpy(job->target, binaryPK, PUBKEYSIZE); // set job's target job->target = binaryPK; // set job's target
// create query DHT request // create query DHT request
check_online_request *request = new check_online_request; check_online_request *request = new check_online_request;

@ -1,4 +1,5 @@
import os, sys, time import os
import sys
# usage: parse_log.py log-file [socket-index to focus on] # usage: parse_log.py log-file [socket-index to focus on]
@ -7,14 +8,15 @@ socket_filter = None
if len(sys.argv) >= 3: if len(sys.argv) >= 3:
socket_filter = sys.argv[2].strip() socket_filter = sys.argv[2].strip()
if socket_filter == None: if socket_filter is None:
print "scanning for socket with the most packets" print "scanning for socket with the most packets"
file = open(sys.argv[1], 'rb') file = open(sys.argv[1], 'rb')
sockets = {} sockets = {}
for l in file: for l in file:
if not 'our_delay' in l: continue if not 'our_delay' in l:
continue
try: try:
a = l.strip().split(" ") a = l.strip().split(" ")
@ -39,14 +41,15 @@ if socket_filter == None:
for i in items: for i in items:
print '%s: %d' % (i[0], i[1]) print '%s: %d' % (i[0], i[1])
count += 1 count += 1
if count > 5: break if count > 5:
break
file.close() file.close()
socket_filter = items[0][0] socket_filter = items[0][0]
print '\nfocusing on socket %s' % socket_filter print '\nfocusing on socket %s' % socket_filter
file = open(sys.argv[1], 'rb') file = open(sys.argv[1], 'rb')
out_file = 'utp.out%s' % socket_filter; out_file = 'utp.out%s' % socket_filter
out = open(out_file, 'wb') out = open(out_file, 'wb')
delay_samples = 'dots lc rgb "blue"' delay_samples = 'dots lc rgb "blue"'
@ -58,24 +61,24 @@ window_size = 'steps lc rgb "sea-green"'
rtt = 'lines lc rgb "light-blue"' rtt = 'lines lc rgb "light-blue"'
metrics = { metrics = {
'our_delay':['our delay (ms)', 'x1y2', delay_samples], 'our_delay': ['our delay (ms)', 'x1y2', delay_samples],
'upload_rate':['send rate (B/s)', 'x1y1', 'lines'], 'upload_rate': ['send rate (B/s)', 'x1y1', 'lines'],
'max_window':['cwnd (B)', 'x1y1', cwnd], 'max_window': ['cwnd (B)', 'x1y1', cwnd],
'target_delay':['target delay (ms)', 'x1y2', target_delay], 'target_delay': ['target delay (ms)', 'x1y2', target_delay],
'cur_window':['bytes in-flight (B)', 'x1y1', window_size], 'cur_window': ['bytes in-flight (B)', 'x1y1', window_size],
'cur_window_packets':['number of packets in-flight', 'x1y2', 'steps'], 'cur_window_packets': ['number of packets in-flight', 'x1y2', 'steps'],
'packet_size':['current packet size (B)', 'x1y2', 'steps'], 'packet_size': ['current packet size (B)', 'x1y2', 'steps'],
'rtt':['rtt (ms)', 'x1y2', rtt], 'rtt': ['rtt (ms)', 'x1y2', rtt],
'off_target':['off-target (ms)', 'x1y2', off_target], 'off_target': ['off-target (ms)', 'x1y2', off_target],
'delay_sum':['delay sum (ms)', 'x1y2', 'steps'], 'delay_sum': ['delay sum (ms)', 'x1y2', 'steps'],
'their_delay':['their delay (ms)', 'x1y2', delay_samples], 'their_delay': ['their delay (ms)', 'x1y2', delay_samples],
'get_microseconds':['clock (us)', 'x1y1', 'steps'], 'get_microseconds': ['clock (us)', 'x1y1', 'steps'],
'wnduser':['advertised window size (B)', 'x1y1', 'steps'], 'wnduser': ['advertised window size (B)', 'x1y1', 'steps'],
'delay_base':['delay base (us)', 'x1y1', delay_base], 'delay_base': ['delay base (us)', 'x1y1', delay_base],
'their_delay_base':['their delay base (us)', 'x1y1', delay_base], 'their_delay_base': ['their delay base (us)', 'x1y1', delay_base],
'their_actual_delay':['their actual delay (us)', 'x1y1', delay_samples], 'their_actual_delay': ['their actual delay (us)', 'x1y1', delay_samples],
'actual_delay':['actual_delay (us)', 'x1y1', delay_samples] 'actual_delay': ['actual_delay (us)', 'x1y1', delay_samples]
} }
histogram_quantization = 1 histogram_quantization = 1
@ -151,7 +154,7 @@ for l in file:
# print time. Convert from milliseconds to seconds # print time. Convert from milliseconds to seconds
print >>out, '%f\t' % (float(t)/1000.), print >>out, '%f\t' % (float(t)/1000.),
#if t > 200000: # if t > 200000:
# break # break
fill_columns = not columns fill_columns = not columns
@ -164,12 +167,14 @@ for l in file:
if n == "our_delay": if n == "our_delay":
bucket = v / histogram_quantization bucket = v / histogram_quantization
delay_histogram[bucket] = 1 + delay_histogram.get(bucket, 0) delay_histogram[bucket] = 1 + delay_histogram.get(bucket, 0)
if not n in metrics: continue if not n in metrics:
continue
if fill_columns: if fill_columns:
columns.append(n) columns.append(n)
if n == "max_window": if n == "max_window":
window_size[socket_index] = v window_size[socket_index] = v
print >>out, '%f\t' % int(reduce(lambda a,b: a+b, window_size.values())), print >>out, '%f\t' % int(
reduce(lambda a, b: a+b, window_size.values())),
else: else:
print >>out, '%f\t' % v, print >>out, '%f\t' % v,
print >>out, float(packet_loss * 8000), float(packet_timeout * 8000) print >>out, float(packet_loss * 8000), float(packet_timeout * 8000)
@ -179,8 +184,9 @@ for l in file:
out.close() out.close()
out = open('%s.histogram' % out_file, 'wb') out = open('%s.histogram' % out_file, 'wb')
for d,f in delay_histogram.iteritems(): for d, f in delay_histogram.iteritems():
print >>out, float(d*histogram_quantization) + histogram_quantization / 2, f print >>out, float(d*histogram_quantization) + \
histogram_quantization / 2, f
out.close() out.close()
@ -216,7 +222,7 @@ plot = [
'y2': 'Time (ms)' 'y2': 'Time (ms)'
}, },
{ {
'data': ['their_actual_delay','their_delay_base'], 'data': ['their_actual_delay', 'their_delay_base'],
'title': 'their_delay_base', 'title': 'their_delay_base',
'y1': 'Time (us)', 'y1': 'Time (us)',
'y2': '' 'y2': ''
@ -253,9 +259,9 @@ print >>out, "set style data steps"
#print >>out, "set yrange [0:*]" #print >>out, "set yrange [0:*]"
print >>out, "set y2range [*:*]" print >>out, "set y2range [*:*]"
files += out_file + '.delays.png ' files += out_file + '.delays.png '
#set hidden3d # set hidden3d
#set title "Peer bandwidth distribution" # set title "Peer bandwidth distribution"
#set xlabel "Ratio" # set xlabel "Ratio"
for p in plot: for p in plot:
print >>out, 'set title "%s %s"' % (p['title'], title) print >>out, 'set title "%s %s"' % (p['title'], title)
@ -274,9 +280,11 @@ for p in plot:
print >>out, "plot", print >>out, "plot",
for c in p['data']: for c in p['data']:
if not c in metrics: continue if not c in metrics:
continue
i = columns.index(c) i = columns.index(c)
print >>out, '%s"%s" using 1:%d title "%s-%s" axes %s with %s' % (comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]), print >>out, '%s"%s" using 1:%d title "%s-%s" axes %s with %s' % (
comma, out_file, i + 2, metrics[c][0], metrics[c][1], metrics[c][1], metrics[c][2]),
comma = ', ' comma = ', '
print >>out, '' print >>out, ''
@ -285,4 +293,3 @@ out.close()
os.system("gnuplot utp.gnuplot") os.system("gnuplot utp.gnuplot")
os.system("open %s" % files) os.system("open %s" % files)

@ -121,11 +121,10 @@ llarp_router_try_connect(llarp::Router *router,
} }
bool bool
llarp_findOrCreateIdentity(llarp::Crypto *crypto, const char *fpath, llarp_findOrCreateIdentity(llarp::Crypto *crypto, const fs::path &path,
llarp::SecretKey &secretkey) llarp::SecretKey &secretkey)
{ {
llarp::LogDebug("find or create ", fpath); llarp::LogDebug("find or create ", path);
fs::path path(fpath);
std::error_code ec; std::error_code ec;
if(!fs::exists(path, ec)) if(!fs::exists(path, ec))
{ {
@ -151,11 +150,10 @@ llarp_findOrCreateIdentity(llarp::Crypto *crypto, const char *fpath,
// C++ ... // C++ ...
bool bool
llarp_findOrCreateEncryption(llarp::Crypto *crypto, const char *fpath, llarp_findOrCreateEncryption(llarp::Crypto *crypto, const fs::path &path,
llarp::SecretKey &encryption) llarp::SecretKey &encryption)
{ {
llarp::LogDebug("find or create ", fpath); llarp::LogDebug("find or create ", path);
fs::path path(fpath);
std::error_code ec; std::error_code ec;
if(!fs::exists(path, ec)) if(!fs::exists(path, ec))
{ {
@ -397,15 +395,14 @@ namespace llarp
{ {
if(!EnsureEncryptionKey()) if(!EnsureEncryptionKey())
return false; return false;
return llarp_findOrCreateIdentity(&crypto, ident_keyfile.string().c_str(), return llarp_findOrCreateIdentity(&crypto, ident_keyfile, identity);
identity);
} }
bool bool
Router::EnsureEncryptionKey() Router::EnsureEncryptionKey()
{ {
return llarp_findOrCreateEncryption( return llarp_findOrCreateEncryption(&crypto, encryption_keyfile,
&crypto, encryption_keyfile.string().c_str(), encryption); encryption);
} }
void void

@ -33,11 +33,11 @@
#include <unordered_map> #include <unordered_map>
bool bool
llarp_findOrCreateEncryption(llarp::Crypto *crypto, const char *fpath, llarp_findOrCreateEncryption(llarp::Crypto *crypto, const fs::path &fpath,
llarp::SecretKey &encryption); llarp::SecretKey &encryption);
bool bool
llarp_findOrCreateIdentity(llarp::Crypto *crypto, const char *path, llarp_findOrCreateIdentity(llarp::Crypto *crypto, const fs::path &path,
llarp::SecretKey &secretkey); llarp::SecretKey &secretkey);
struct TryConnectJob; struct TryConnectJob;

Loading…
Cancel
Save