more path building stuff

pull/1/head
Jeff Becker 6 years ago
parent aa251cdd01
commit f68ffbc33d
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05

@ -62,7 +62,6 @@ testnet-configure: clean
cmake -GNinja -DCMAKE_BUILD_TYPE=Debug
testnet-build: testnet-configure
ninja clean
ninja
testnet: testnet-build

@ -9,37 +9,44 @@ from configparser import ConfigParser as CP
import os
svcNodeName = lambda id : 'svc-node-%03d' % id
clientNodeName = lambda id : 'client-node-%03d' % id
def svcNodeName(id): return 'svc-node-%03d' % id
def clientNodeName(id): return 'client-node-%03d' % id
def main():
ap = AP()
ap.add_argument('--dir', type=str, default='testnet_tmp')
ap.add_argument('--svc', type=int, default=20, help='number of service nodes')
ap.add_argument('--svc', type=int, default=20,
help='number of service nodes')
ap.add_argument('--baseport', type=int, default=19000)
ap.add_argument('--clients', type=int, default=200, help='number of client nodes')
ap.add_argument('--clients', type=int, default=200,
help='number of client nodes')
ap.add_argument('--bin', type=str, required=True)
ap.add_argument('--out', type=str, required=True)
ap.add_argument('--connect', type=int, default=5)
ap.add_argument('--connect', type=int, default=10)
args = ap.parse_args()
basedir = os.path.abspath(args.dir)
for nodeid in range(args.svc):
config = CP()
config['bind'] = {
'lo' : str(args.baseport + nodeid)
'lo': str(args.baseport + nodeid)
}
config['netdb'] = {
'dir' : 'netdb'
'dir': 'netdb'
}
config['connect'] = {}
for otherid in range(args.svc):
if otherid != nodeid:
name = svcNodeName(otherid)
config['connect'][name] = os.path.join(basedir, name, 'rc.signed')
config['connect'][name] = os.path.join(
basedir, name, 'rc.signed')
d = os.path.join(args.dir, svcNodeName(nodeid))
if not os.path.exists(d):
os.mkdir(d)
@ -50,22 +57,22 @@ def main():
for nodeid in range(args.clients):
config = CP()
config['netdb'] = {
'dir' : 'netdb'
'dir': 'netdb'
}
config['connect'] = {}
for otherid in range(args.svc):
if otherid % args.connect == 0:
name = svcNodeName(otherid)
config['connect'][name] = os.path.join(basedir, name, 'rc.signed')
config['connect'][name] = os.path.join(
basedir, name, 'rc.signed')
d = os.path.join(args.dir, clientNodeName(nodeid))
if not os.path.exists(d):
os.mkdir(d)
fp = os.path.join(d, 'daemon.ini')
with open(fp, 'w') as f:
config.write(f)
with open(args.out, 'w') as f:
f.write('''[program:svc-node]
directory = {}
@ -86,6 +93,7 @@ process_name = client-node-%(process_num)03d
numprocs = {}
'''.format(os.path.join(args.dir, 'client-node-%(process_num)03d'), args.bin, args.clients))
f.write('[supervisord]\ndirectory=.\n')
if __name__ == '__main__':
main()

@ -26,6 +26,7 @@ extern "C" {
#define SIGSIZE 64
#define TUNNONCESIZE 32
#define HMACSIZE 32
#define PATHIDSIZE 16
/*
typedef byte_t llarp_pubkey_t[PUBKEYSIZE];

@ -4,6 +4,7 @@
#include <llarp/bencode.h>
#include <llarp/buffer.h>
#include <sodium.h>
#include <vector>
namespace llarp
{
@ -13,19 +14,18 @@ namespace llarp
Encrypted() = default;
Encrypted(const byte_t* buf, size_t sz);
Encrypted(size_t sz);
~Encrypted();
bool
BEncode(llarp_buffer_t* buf) const
{
return bencode_write_bytestring(buf, data, size);
return bencode_write_bytestring(buf, _data.data(), _data.size());
}
void
Randomize()
{
if(data)
randombytes(data, size);
if(_data.size())
randombytes(_data.data(), _data.size());
}
bool
@ -36,11 +36,8 @@ namespace llarp
return false;
if(strbuf.sz == 0)
return false;
if(data)
delete[] data;
size = strbuf.sz;
data = new byte_t[size];
memcpy(data, strbuf.base, size);
_data.resize(strbuf.sz);
memcpy(_data.data(), strbuf.base, _data.size());
return true;
}
@ -50,12 +47,29 @@ namespace llarp
return &m_Buffer;
}
byte_t* data = nullptr;
size_t size = 0;
size_t
size()
{
return _data.size();
}
size_t
size() const
{
return _data.size();
}
byte_t*
data()
{
return _data.data();
}
std::vector< byte_t > _data;
private:
llarp_buffer_t m_Buffer;
};
}
} // namespace llarp
#endif

@ -44,7 +44,6 @@ namespace llarp
ctx->handler(ctx->frame, ctx->user);
else
{
delete ctx->frame;
ctx->handler(nullptr, ctx->user);
}
}
@ -68,8 +67,8 @@ namespace llarp
// TODO: should we own otherKey?
otherKey = other;
frame = new EncryptedFrame(buf.sz);
memcpy(frame->data + PUBKEYSIZE + TUNNONCESIZE + SHORTHASHSIZE, buf.base,
buf.sz);
memcpy(frame->data() + PUBKEYSIZE + TUNNONCESIZE + SHORTHASHSIZE,
buf.base, buf.sz);
user = u;
llarp_threadpool_queue_job(worker, {this, &Encrypt});
}

@ -0,0 +1,18 @@
#ifndef LLARP_ENDPOINT_HANDLER_HPP
#define LLARP_ENDPOINT_HANDLER_HPP
#include <llarp/buffer.h>
namespace llarp
{
// hidden service endpoint handler
struct IEndpointHandler
{
~IEndpointHandler(){};
virtual void
HandleMessage(llarp_buffer_t buf) = 0;
};
} // namespace llarp
#endif

@ -56,6 +56,11 @@ llarp_nodedb_get_random_rc(struct llarp_nodedb *n, struct llarp_rc *result);
void
llarp_nodedb_select_random_hop(struct llarp_nodedb *n, struct llarp_rc *result,
size_t N);
/// return number of RC loaded
size_t
llarp_nodedb_num_loaded(struct llarp_nodedb *n);
/**
put an rc into the node db
overwrites with new contents if already present

@ -8,12 +8,23 @@
extern "C" {
#endif
struct llarp_path_hop
{
struct llarp_rc router;
byte_t nextHop[PUBKEYSIZE];
byte_t sessionkey[SHAREDKEYSIZE];
byte_t pathid[PATHIDSIZE];
};
struct llarp_path_hops
{
struct llarp_rc routers[MAXHOPS];
struct llarp_path_hop hops[MAXHOPS];
size_t numHops;
};
void
llarp_path_hops_free(struct llarp_path_hops* hops);
#ifdef __cplusplus
}
#endif

@ -5,11 +5,14 @@
#include <llarp/time.h>
#include <llarp/aligned.hpp>
#include <llarp/crypto.hpp>
#include <llarp/endpoint.hpp>
#include <llarp/messages/relay_ack.hpp>
#include <llarp/messages/relay_commit.hpp>
#include <llarp/path_types.hpp>
#include <llarp/router_id.hpp>
#include <list>
#include <map>
#include <mutex>
#include <unordered_map>
#include <vector>
@ -91,6 +94,14 @@ namespace llarp
bool
Expired(llarp_time_t now) const;
// forward data in upstream direction
void
ForwardUpstream(llarp_buffer_t X, const TunnelNonce& Y, llarp_router* r);
// forward data in downstream direction
void
ForwardDownstream(llarp_buffer_t X, const TunnelNonce& Y, llarp_router* r);
};
/// configuration for a single hop when building a path
@ -113,12 +124,35 @@ namespace llarp
PathHopConfig();
};
enum PathStatus
{
ePathBuilding,
ePathEstablished,
ePathTimeout,
ePathExpired
};
/// A path we made
struct Path
{
typedef std::vector< PathHopConfig > HopList;
HopList hops;
llarp_time_t buildStarted;
PathStatus status;
Path(llarp_path_hops* path);
void
EncryptAndSend(llarp_buffer_t buf, llarp_router* r);
void
DecryptAndRecv(llarp_buffer_t buf, IEndpointHandler* handler);
const PathID_t&
PathID() const;
RouterID
Upstream();
};
template < typename User >
@ -132,7 +166,7 @@ namespace llarp
llarp_threadpool* worker = nullptr;
llarp_logic* logic = nullptr;
llarp_crypto* crypto = nullptr;
LR_CommitMessage LRCM;
LR_CommitMessage* LRCM = nullptr;
static void
HandleDone(void* user)
@ -157,6 +191,7 @@ namespace llarp
hop.commkey))
{
llarp::Error("Failed to generate shared key for path build");
delete ctx->user;
delete ctx;
return;
}
@ -165,7 +200,7 @@ namespace llarp
LR_CommitRecord record;
auto& frame = ctx->LRCM.frames[ctx->idx];
auto& frame = ctx->LRCM->frames[ctx->idx];
++ctx->idx;
if(ctx->idx < ctx->path->hops.size())
{
@ -180,6 +215,7 @@ namespace llarp
{
// failed to encode?
llarp::Error("Failed to generate Commit Record");
delete ctx->user;
delete ctx;
return;
}
@ -210,11 +246,12 @@ namespace llarp
user = u;
result = func;
worker = pool;
LRCM = new LR_CommitMessage;
for(size_t idx = 0; idx < MAXHOPS; ++idx)
{
LRCM.frames.emplace_back(256);
LRCM.frames.back().Randomize();
LRCM->frames.emplace_back(256);
LRCM->frames.back().Randomize();
}
llarp_threadpool_queue_job(pool, {this, &GenerateNextKey});
}
@ -227,56 +264,6 @@ namespace llarp
ePathBuildReject
};
/// path selection algorithm
struct IPathSelectionAlgorithm
{
virtual ~IPathSelectionAlgorithm(){};
/// select full path given an empty hop list to end at target
virtual bool
SelectFullPathTo(Path::HopList& hops, const RouterID& target) = 0;
/// report to path builder the result of a path build
/// can be used to "improve" path building algoirthm in the
/// future
virtual void
ReportPathBuildStatus(const Path::HopList& hops, const RouterID& target,
PathBuildStatus status){};
};
class PathBuildJob
{
public:
PathBuildJob(llarp_router* router, IPathSelectionAlgorithm* selector);
~PathBuildJob();
void
Start();
private:
typedef AsyncPathKeyExchangeContext< PathBuildJob > KeyExchanger;
LR_CommitMessage*
BuildLRCM();
static void
KeysGenerated(KeyExchanger* ctx);
llarp_router* router;
IPathSelectionAlgorithm* m_HopSelector;
KeyExchanger m_KeyExchanger;
};
/// a pool of paths for a hidden service
struct PathPool
{
PathPool(llarp_router* router);
~PathPool();
/// build a new path to a router by identity key
PathBuildJob*
BuildNewPathTo(const RouterID& router);
};
struct PathContext
{
PathContext(llarp_router* router);
@ -300,28 +287,30 @@ namespace llarp
bool
HandleRelayCommit(const LR_CommitMessage* msg);
bool
HandleRelayAck(const LR_AckMessage* msg);
void
PutTransitHop(const TransitHop& hop);
bool
ForwardLRCM(const RouterID& nextHop, std::deque< EncryptedFrame >& frames);
void
ForwradLRUM(const PathID_t& id, const RouterID& from, llarp_buffer_t X,
const TunnelNonce& nonce);
void
ForwradLRDM(const PathID_t& id, const RouterID& from, llarp_buffer_t X,
const TunnelNonce& nonce);
bool
HopIsUs(const PubKey& k) const;
void
AddOwnPath(Path* p);
typedef std::unordered_multimap< PathID_t, TransitHop, PathIDHash >
TransitHopsMap_t;
typedef std::pair< std::mutex, TransitHopsMap_t > SyncTransitMap_t;
typedef std::map< PathID_t, Path* > OwnedPathsMap_t;
typedef std::pair< std::mutex, OwnedPathsMap_t > SyncOwnedPathsMap_t;
llarp_threadpool*
Worker();
@ -340,6 +329,7 @@ namespace llarp
private:
llarp_router* m_Router;
SyncTransitMap_t m_TransitPaths;
SyncOwnedPathsMap_t m_OurPaths;
bool m_AllowTransit;
};

@ -1,11 +1,12 @@
#ifndef LLARP_PATH_TYPES_HPP
#define LLARP_PATH_TYPES_HPP
#include <llarp/crypto.h>
#include <llarp/aligned.hpp>
namespace llarp
{
typedef AlignedBuffer< 16 > PathID_t;
typedef AlignedBuffer< PATHIDSIZE > PathID_t;
}
#endif

@ -49,10 +49,8 @@ struct llarp_pathbuild_job
struct llarp_pathbuilder_context* context;
// path hop selection
llarp_pathbuilder_select_hop_func selectHop;
// result handler
llarp_pathbuilder_hook result;
// encryption secret key for hidden service
byte_t* secretkey;
// called when the path build started
llarp_pathbuilder_hook pathBuildStarted;
// path
struct llarp_path_hops hops;
};

@ -5,29 +5,21 @@
namespace llarp
{
Encrypted::Encrypted(const byte_t* buf, size_t sz)
Encrypted::Encrypted(const byte_t* buf, size_t sz) : _data(sz)
{
size = sz;
data = new byte_t[sz];
if(buf)
memcpy(data, buf, sz);
memcpy(data(), buf, sz);
else
llarp::Zero(data, sz);
m_Buffer.base = data;
m_Buffer.cur = data;
m_Buffer.sz = size;
llarp::Zero(data(), sz);
m_Buffer.base = data();
m_Buffer.cur = data();
m_Buffer.sz = size();
}
Encrypted::Encrypted(size_t sz) : Encrypted(nullptr, sz)
{
}
Encrypted::~Encrypted()
{
if(data)
delete[] data;
}
bool
EncryptedFrame::EncryptInPlace(byte_t* ourSecretKey, byte_t* otherPubkey,
llarp_crypto* crypto)
@ -38,7 +30,7 @@ namespace llarp
// <32 bytes pubkey>
// <N bytes encrypted payload>
//
byte_t* hash = data;
byte_t* hash = data();
byte_t* nonce = hash + SHORTHASHSIZE;
byte_t* pubkey = nonce + TUNNONCESIZE;
byte_t* body = pubkey + PUBKEYSIZE;
@ -52,7 +44,7 @@ namespace llarp
llarp_buffer_t buf;
buf.base = body;
buf.cur = buf.base;
buf.sz = size - EncryptedFrame::OverheadSize;
buf.sz = size() - EncryptedFrame::OverheadSize;
// set our pubkey
memcpy(pubkey, llarp::seckey_topublic(ourSecretKey), PUBKEYSIZE);
@ -75,7 +67,7 @@ namespace llarp
// generate message auth
buf.base = nonce;
buf.cur = buf.base;
buf.sz = size - SHORTHASHSIZE;
buf.sz = size() - SHORTHASHSIZE;
if(!MDS(hash, buf, shared))
{
@ -88,9 +80,9 @@ namespace llarp
bool
EncryptedFrame::DecryptInPlace(byte_t* ourSecretKey, llarp_crypto* crypto)
{
if(size <= size_t(EncryptedFrame::OverheadSize))
if(size() <= size_t(EncryptedFrame::OverheadSize))
{
llarp::Warn("encrypted frame too small, ", size,
llarp::Warn("encrypted frame too small, ", size(),
" <= ", size_t(EncryptedFrame::OverheadSize));
return false;
}
@ -100,7 +92,7 @@ namespace llarp
// <32 bytes pubkey>
// <N bytes encrypted payload>
//
byte_t* hash = data;
byte_t* hash = data();
byte_t* nonce = hash + SHORTHASHSIZE;
byte_t* otherPubkey = nonce + TUNNONCESIZE;
byte_t* body = otherPubkey + PUBKEYSIZE;
@ -113,7 +105,7 @@ namespace llarp
llarp_buffer_t buf;
buf.base = nonce;
buf.cur = buf.base;
buf.sz = size - SHORTHASHSIZE;
buf.sz = size() - SHORTHASHSIZE;
SharedSecret shared;
ShortHash digest;
@ -138,7 +130,7 @@ namespace llarp
buf.base = body;
buf.cur = body;
buf.sz = size - EncryptedFrame::OverheadSize;
buf.sz = size() - EncryptedFrame::OverheadSize;
if(!Decrypt(buf, shared, nonce))
{

@ -591,7 +591,7 @@ namespace iwp
void
queue_tx(uint64_t id, transit_message *msg)
{
tx[id] = msg;
tx.insert(std::make_pair(id, msg));
msg->generate_xmit(sendqueue, txflags);
}
@ -1921,7 +1921,7 @@ namespace iwp
delete link;
}
}
}
} // namespace iwp
extern "C" {
void

@ -292,7 +292,6 @@ nodedb_async_load_rc(void *user)
}
extern "C" {
struct llarp_nodedb *
llarp_nodedb_new(struct llarp_crypto *crypto)
{
@ -352,7 +351,8 @@ llarp_nodedb_load_dir(struct llarp_nodedb *n, const char *dir)
void
llarp_nodedb_async_verify(struct llarp_async_verify_rc *job)
{
// switch to crypto threadpool and continue with crypto_threadworker_verifyrc
// switch to crypto threadpool and continue with
// crypto_threadworker_verifyrc
llarp_threadpool_queue_job(job->cryptoworker,
{job, &crypto_threadworker_verifyrc});
}
@ -373,4 +373,24 @@ llarp_nodedb_get_rc(struct llarp_nodedb *n, const byte_t *pk)
return nullptr;
}
size_t
llarp_nodedb_num_loaded(struct llarp_nodedb *n)
{
return n->entries.size();
}
void
llarp_nodedb_select_random_hop(struct llarp_nodedb *n, struct llarp_rc *result,
size_t N)
{
/// TODO: check for "guard" status for N = 0?
auto sz = n->entries.size();
auto itr = n->entries.begin();
if(sz > 1)
{
std::advance(itr, rand() % (sz - 1));
}
llarp_rc_copy(result, &itr->second);
}
} // end extern

@ -10,7 +10,7 @@ namespace llarp
for(size_t idx = 0; idx < h->numHops; ++idx)
{
hops.emplace_back();
llarp_rc_copy(&hops[idx].router, &h->routers[idx]);
llarp_rc_copy(&hops[idx].router, &h->hops[idx].router);
}
}
@ -101,6 +101,12 @@ namespace llarp
map.second.emplace(k, v);
}
void
PathContext::AddOwnPath(Path* path)
{
MapPut(m_OurPaths, path->PathID(), path);
}
bool
PathContext::HasTransitHop(const TransitHopInfo& info)
{
@ -148,4 +154,17 @@ namespace llarp
: pathID(record.pathid), upstream(record.nextHop), downstream(down)
{
}
const PathID_t&
Path::PathID() const
{
return hops[0].pathID;
}
RouterID
Path::Upstream()
{
return hops[0].router.pubkey;
}
} // namespace llarp

@ -20,18 +20,25 @@ namespace llarp
pathbuilder_generated_keys(
AsyncPathKeyExchangeContext< llarp_pathbuild_job >* ctx)
{
llarp::Debug("Generated keys for build");
llarp::Debug("Generated LRCM");
auto router = ctx->user->router;
router->SendToOrQueue(ctx->path->Upstream(), ctx->LRCM);
ctx->path->status = ePathBuilding;
router->paths.AddOwnPath(ctx->path);
ctx->user->pathBuildStarted(ctx->user);
}
void
pathbuilder_start_build(void* user)
{
// select hops
llarp_pathbuild_job* job = static_cast< llarp_pathbuild_job* >(user);
size_t idx = 0;
// select hops
size_t idx = 0;
while(idx < job->hops.numHops)
{
job->selectHop(job->router->nodedb, &job->hops.routers[idx], idx);
auto rc = &job->hops.hops[idx].router;
llarp_rc_clear(rc);
job->selectHop(job->router->nodedb, rc, idx);
++idx;
}
@ -42,13 +49,6 @@ namespace llarp
ctx->AsyncGenerateKeys(new Path(&job->hops), job->router->logic,
job->router->tp, job, &pathbuilder_generated_keys);
// free rc
idx = 0;
while(idx < job->hops.numHops)
{
llarp_rc_free(&job->hops.routers[idx]);
++idx;
}
}
} // namespace llarp
@ -77,6 +77,8 @@ void
llarp_pathbuilder_build_path(struct llarp_pathbuild_job* job)
{
job->router = job->context->router;
if(job->selectHop == nullptr)
job->selectHop = &llarp_nodedb_select_random_hop;
llarp_logic_queue_job(job->router->logic,
{job, &llarp::pathbuilder_start_build});
}

@ -237,7 +237,7 @@ namespace llarp
llarp::Info("Accepted ", self->hop.info);
self->context->PutTransitHop(self->hop);
size_t sz = self->frames.front().size;
size_t sz = self->frames.front().size();
// we pop the front element it was ours
self->frames.pop_front();
// put our response on the end

@ -33,6 +33,7 @@ llarp_router::llarp_router()
, paths(this)
, dht(llarp_dht_context_new(this))
, inbound_msg_parser(this)
, explorePool(llarp_pathbuilder_context_new(this, dht))
{
llarp_rc_clear(&rc);
@ -53,7 +54,7 @@ llarp_router::HandleRecvLinkMessage(llarp_link_session *session,
bool
llarp_router::SendToOrQueue(const llarp::RouterID &remote,
std::vector< llarp::ILinkMessage * > msgs)
llarp::ILinkMessage *msg)
{
llarp_link *chosen = nullptr;
if(!outboundLink->has_session_to(outboundLink, remote))
@ -70,11 +71,9 @@ llarp_router::SendToOrQueue(const llarp::RouterID &remote,
else
chosen = outboundLink;
for(const auto &msg : msgs)
{
// this will create an entry in the obmq if it's not already there
outboundMesssageQueue[remote].push(msg);
}
// this will create an entry in the obmq if it's not already there
outboundMesssageQueue[remote].push(msg);
if(!chosen)
{
@ -359,13 +358,43 @@ llarp_router::handle_router_ticker(void *user, uint64_t orig, uint64_t left)
self->ScheduleTicker(orig);
}
void
llarp_router::HandleExploritoryPathBuildStarted(llarp_pathbuild_job *job)
{
delete job;
}
void
llarp_router::BuildExploritoryPath()
{
llarp_pathbuild_job *job = new llarp_pathbuild_job;
job->context = explorePool;
job->selectHop = selectHopFunc;
job->hops.numHops = 4;
job->user = this;
job->pathBuildStarted = &HandleExploritoryPathBuildStarted;
llarp_pathbuilder_build_path(job);
}
void
llarp_router::Tick()
{
llarp::Debug("tick router");
paths.ExpirePaths();
llarp_pathbuild_job job;
llarp_pathbuilder_build_path(&job);
// TODO: don't do this if we have enough paths already
if(inboundLinks.size() == 0)
{
auto N = llarp_nodedb_num_loaded(nodedb);
if(N > 5)
{
BuildExploritoryPath();
}
else
{
llarp::Warn("not enough nodes known to build exploritory paths, have ", N,
" nodes");
}
}
llarp_link_session_iter iter;
iter.user = this;
iter.visit = &send_padded_message;

@ -72,6 +72,7 @@ struct llarp_router
llarp::InboundMessageParser inbound_msg_parser;
llarp_pathbuilder_select_hop_func selectHopFunc = nullptr;
llarp_pathbuilder_context *explorePool = nullptr;
llarp_link *outboundLink = nullptr;
std::list< llarp_link * > inboundLinks;
@ -144,8 +145,7 @@ struct llarp_router
/// NOT threadsafe
/// MUST be called in the logic thread
bool
SendToOrQueue(const llarp::RouterID &remote,
std::vector< llarp::ILinkMessage * > msgs);
SendToOrQueue(const llarp::RouterID &remote, llarp::ILinkMessage *msg);
/// sendto or drop
void
@ -171,6 +171,9 @@ struct llarp_router
void
Tick();
void
BuildExploritoryPath();
/// schedule ticker to call i ms from now
void
ScheduleTicker(uint64_t i = 1000);
@ -207,6 +210,9 @@ struct llarp_router
static void
HandleDHTLookupForSendTo(llarp_router_lookup_job *job);
static void
HandleExploritoryPathBuildStarted(llarp_pathbuild_job *job);
};
#endif

Loading…
Cancel
Save