Merge remote-tracking branch 'ryan/master' into dev

i2p.rocks.signed.txt

@@ -0,0 +1 @@
+d1:ald1:ci1e1:d3:IWP1:e32:£8XÐG†-ûKZC@ž&ñÜAºÖ Q<>¨‘D1:i22:::ffff:162.243.164.2231:pi1090e1:vi0eee1:k32:àÕúçšÌÑ,ñ@ÔãL˜ª>jÒϳ1[cÇ™,…¡1:ui0e1:vi0e1:z64:¯²Û8Ú@SvHàDØ—‹U'<27>RÊŸ!é	¶„äi~Ý	ÂØ[&}öÆh!öÉ|Mˆ^g¼<67>s#xf­MÖ²>˜‡e

include/llarp/crypto_async.h

@@ -15,7 +15,7 @@
 extern "C" {
 #endif
 
-/// context for doing asynchronous crpytography for iwp
+/// context for doing asynchronous cryptography for iwp
 /// with a worker threadpool
 /// defined in crypto_async.cpp
 struct llarp_async_iwp;
@@ -42,7 +42,7 @@ struct iwp_async_keygen
 {
   /// internal wire protocol async configuration
   struct llarp_async_iwp *iwp;
-  /// a customizable pointer to pass data to iteration functor
+  /// a pointer to pass ourself to thread worker
   void *user;
   /// destination key buffer
   uint8_t *keybuf;
@@ -167,6 +167,7 @@ struct iwp_async_frame
   /// true if decryption succeded
   bool success;
   struct llarp_async_iwp *iwp;
+  /// a pointer to pass ourself
   void *user;
   /// current session key
   byte_t *sessionkey;

include/llarp/nodedb.h

@@ -3,6 +3,13 @@
 #include <llarp/common.h>
 #include <llarp/crypto.h>
 #include <llarp/router_contact.h>
+
+/**
+ * nodedb.h
+ *
+ * persistent storage API for router contacts
+ */
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -75,11 +82,24 @@ struct llarp_async_verify_rc;
 
 typedef void (*llarp_async_verify_rc_hook_func)(struct llarp_async_verify_rc *);
 
+/// verify rc request
 struct llarp_async_verify_rc
 {
+  /// async_verify_context
   void *user;
+  /// nodedb storage
+  struct llarp_nodedb *nodedb;
+  // llarp_logic for llarp_logic_queue_job
+  struct llarp_logic *logic; // includes a llarp_threadpool
+  struct llarp_crypto *crypto;
+  struct llarp_threadpool *cryptoworker;
+  struct llarp_threadpool *diskworker;
+
+  /// router contact (should this be a pointer?)
   struct llarp_rc rc;
+  /// result
   bool valid;
+  /// hook
   llarp_async_verify_rc_hook_func hook;
 };
 

llarp/crypto_async.cpp

@@ -1,4 +1,5 @@
 #include <llarp/crypto_async.h>
+#include <llarp/router_contact.h>
 #include <llarp/mem.h>
 #include <string.h>
 #include <llarp/crypto.hpp>
@@ -26,7 +27,9 @@ namespace iwp
   {
     iwp_async_keygen *keygen = static_cast< iwp_async_keygen * >(user);
     keygen->iwp->crypto->encryption_keygen(keygen->keybuf);
-    llarp_thread_job job = {.user = user, .work = &inform_keygen};
+    llarp_thread_job job;
+    job.user = user;
+    job.work = &inform_keygen;
     llarp_logic_queue_job(keygen->iwp->logic, job);
   }
 
@@ -477,4 +480,5 @@ llarp_async_iwp_free(struct llarp_async_iwp *iwp)
 {
   delete iwp;
 }
+
 }

llarp/dht.cpp

@@ -6,6 +6,7 @@
 #include <sodium.h>
 
 #include <set>
+#include <algorithm>    // std::find
 
 namespace llarp
 {
@@ -414,7 +415,10 @@ namespace llarp
     void
     Context::RemovePendingLookup(const Key_t &owner, uint64_t id)
     {
-      auto itr = pendingTX.find({owner, id});
+      TXOwner search;
+      search.requester = owner;
+      search.txid = id;
+      auto itr = pendingTX.find(search);
       if(itr == pendingTX.end())
         return;
       pendingTX.erase(itr);
@@ -423,7 +427,10 @@ namespace llarp
     SearchJob *
     Context::FindPendingTX(const Key_t &owner, uint64_t id)
     {
-      auto itr = pendingTX.find({owner, id});
+      TXOwner search;
+      search.requester = owner;
+      search.txid = id;
+      auto itr = pendingTX.find(search);
       if(itr == pendingTX.end())
         return nullptr;
       else
@@ -480,7 +487,12 @@ namespace llarp
                           const Key_t &askpeer, llarp_router_lookup_job *job)
     {
       auto id                   = ++ids;
-      pendingTX[{whoasked, id}] = SearchJob(whoasked, target, job);
+      
+      TXOwner ownerKey;
+      ownerKey.requester = whoasked;
+      ownerKey.txid = id;
+
+      pendingTX[ownerKey] = SearchJob(whoasked, target, job);
 
       llarp::Info("Asking ", askpeer, " for router ", target, " for ",
                   whoasked);

llarp/ev_kqueue.hpp

@@ -5,17 +5,24 @@
 
 #if __FreeBSD__
 // kqueue / kevent
-//#  include <sys/types.h> // already in net.h
 #  include <sys/event.h>
 #  include <sys/time.h>
 #endif
 
-//#include <sys/socket.h>
-//#include <ifaddrs.h>
+#if (__APPLE__ && __MACH__)
+// kqueue / kevent
+#  include <sys/event.h>
+#  include <sys/time.h>
+#endif
+
+// MacOS needs this
+#ifndef SOCK_NONBLOCK
+#  include <fcntl.h>
+#  define SOCK_NONBLOCK O_NONBLOCK
+#endif
 
 // original upstream
 #include <unistd.h>
-
 #include <cstdio>
 #include "ev.hpp"
 #include "logger.hpp"
@@ -61,9 +68,9 @@ namespace llarp
         default:
           return -1;
       }
-      ssize_t sent = ::sendto(fd, data, sz, SOCK_NONBLOCK, to, slen);
+      ssize_t sent = ::sendto(fd, data, sz, 0, to, slen);
       if(sent == -1)
-        perror("sendto()");
+        perror("kqueue sendto()");
       return sent;
     }
   };
@@ -152,6 +159,7 @@ struct llarp_kqueue_loop : public llarp_ev_loop
   udp_bind(const sockaddr* addr)
   {
     socklen_t slen;
+    llarp::Debug(__FILE__, "kqueue bind affam", addr->sa_family);
     switch(addr->sa_family)
     {
       case AF_INET:
@@ -191,7 +199,11 @@ struct llarp_kqueue_loop : public llarp_ev_loop
       }
     }
     llarp::Addr a(*addr);
-    llarp::Info(__FILE__, "bind to ", a.to_string());
+    llarp::Info(__FILE__, "bind to ", a);
+    // FreeBSD handbook said to do this
+    if (addr->sa_family == AF_INET && INADDR_ANY)
+      a._addr4.sin_addr.s_addr = htonl(INADDR_ANY);
+
     if(bind(fd, addr, slen) == -1)
     {
       perror("bind()");

llarp/iwp_link.cpp

@@ -1316,9 +1316,10 @@ namespace iwp
     UnmapAddr(const llarp::Addr &src)
     {
       lock_t lock(m_Connected_Mutex);
+      // std::unordered_map< llarp::pubkey, llarp::Addr, llarp::pubkeyhash >
       auto itr = std::find_if(
           m_Connected.begin(), m_Connected.end(),
-          [src](const auto &item) -> bool { return src == item.second; });
+                              [src](const std::pair<llarp::pubkey, llarp::Addr> &item) -> bool { return src == item.second; });
       if(itr == std::end(m_Connected))
         return;
 
@@ -1727,6 +1728,7 @@ namespace iwp
         break;
         // TODO: AF_PACKET
       default:
+        llarp::Error(__FILE__, "unsupported address family", af);
         return false;
     }
 
@@ -1823,7 +1825,7 @@ namespace iwp
         link->put_session(dst, s);
       }
       s->establish_job = job;
-      s->frame.alive();
+      s->frame.alive(); // mark it alive
       s->introduce(job->ai.enc_key);
     }
     return true;

llarp/mem_std.cpp

@@ -3,6 +3,11 @@
 #include <cstdlib>
 #include <cstring>
 
+struct llarp_alloc {
+    void *(*alloc)(struct llarp_alloc *mem, size_t sz, size_t align);
+    void (*free)(struct llarp_alloc *mem, void *ptr);
+};
+
 namespace llarp
 {
   void *

llarp/nodedb.cpp

@@ -1,11 +1,15 @@
 #include <llarp/nodedb.h>
 #include <llarp/router_contact.h>
+#include <llarp/crypto_async.h>
+
 #include <fstream>
 #include <llarp/crypto.hpp>
 #include <unordered_map>
 #include "buffer.hpp"
 #include "fs.hpp"
 #include "mem.hpp"
+#include "encode.hpp"
+#include "logger.hpp"
 
 static const char skiplist_subdirs[] = "0123456789ABCDEF";
 
@@ -29,6 +33,91 @@ struct llarp_nodedb
     }
   }
 
+  inline llarp::pubkey getPubKeyFromRC(llarp_rc *rc)
+  {
+    llarp::pubkey pk;
+    memcpy(pk.data(), rc->pubkey, pk.size());
+    return pk;
+  }
+
+  llarp_rc *getRC(llarp::pubkey pk)
+  {
+    return entries[pk];
+  }
+
+  bool pubKeyExists(llarp_rc *rc)
+  {
+    // extract pk from rc
+    llarp::pubkey pk = getPubKeyFromRC(rc);
+    // return true if we found before end
+    return entries.find(pk) != entries.end();
+  }
+
+  bool check(llarp_rc *rc)
+  {
+    if (!pubKeyExists(rc))
+    {
+      // we don't have it
+      return false;
+    }
+    llarp::pubkey pk = getPubKeyFromRC(rc);
+
+    // TODO: zero out any fields you don't want to compare
+
+    // serialize both and memcmp
+    byte_t nodetmp[MAX_RC_SIZE];
+    auto nodebuf = llarp::StackBuffer< decltype(nodetmp) >(nodetmp);
+    if (llarp_rc_bencode(entries[pk], &nodebuf))
+    {
+      byte_t paramtmp[MAX_RC_SIZE];
+      auto parambuf = llarp::StackBuffer< decltype(paramtmp) >(paramtmp);
+      if (llarp_rc_bencode(rc, &parambuf))
+      {
+        if (memcmp(&parambuf, &nodebuf, MAX_RC_SIZE) == 0)
+        {
+          return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  bool setRC(llarp_rc *rc) {
+    byte_t tmp[MAX_RC_SIZE];
+    auto buf = llarp::StackBuffer< decltype(tmp) >(tmp);
+
+    // extract pk from rc
+    llarp::pubkey pk = getPubKeyFromRC(rc);
+
+    // set local db
+    entries[pk] = rc;
+
+    if (llarp_rc_bencode(rc, &buf))
+    {
+      char ftmp[68] = {0};
+      const char *hexname =
+      llarp::HexEncode< llarp::pubkey, decltype(ftmp) >(pk, ftmp);
+      std::string filename(hexname);
+      filename.append(".signed.txt");
+      llarp::Info("saving RC.pubkey ", filename);
+      // write buf to disk
+      //auto filename = hexStr(pk.data(), sizeof(pk)) + ".rc";
+      // FIXME: path?
+      //printf("filename[%s]\n", filename.c_str());
+      std::ofstream ofs (filename, std::ofstream::out & std::ofstream::binary & std::ofstream::trunc);
+      ofs.write((char *)buf.base, buf.sz);
+      ofs.close();
+      if (!ofs)
+      {
+        llarp::Error("Failed to write", filename);
+        return false;
+      }
+      llarp::Info("saved RC.pubkey", filename);
+      return true;
+    }
+    return false;
+  }
+
   ssize_t
   Load(const fs::path &path)
   {
@@ -54,6 +143,18 @@ struct llarp_nodedb
     return loaded;
   }
 
+  ssize_t
+  loadSubdir(const fs::path &dir)
+  {
+    ssize_t sz = 0;
+    for(auto &path : fs::directory_iterator(dir))
+    {
+      if(loadfile(path))
+        sz++;
+    }
+    return sz;
+  }
+
   bool
   loadfile(const fs::path &fpath)
   {
@@ -91,19 +192,57 @@ struct llarp_nodedb
     return false;
   }
 
-  ssize_t
-  loadSubdir(const fs::path &dir)
+  /*
+  bool Save()
   {
-    ssize_t sz = 0;
-    for(auto &path : fs::directory_iterator(dir))
+    auto itr = entries.begin();
+    while(itr != entries.end())
     {
-      if(loadfile(path))
-        sz++;
+      llarp::pubkey pk = itr->first;
+      llarp_rc *rc= itr->second;
+
+      itr++; // advance
     }
-    return sz;
+    return true;
   }
+  */
 };
 
+// call request hook
+void logic_threadworker_callback(void *user) {
+  llarp_async_verify_rc *verify_request =
+    static_cast < llarp_async_verify_rc * >(user);
+  verify_request->hook(verify_request);
+}
+
+// write it to disk
+void disk_threadworker_setRC(void *user) {
+  llarp_async_verify_rc *verify_request =
+    static_cast < llarp_async_verify_rc * >(user);
+  verify_request->valid = verify_request->nodedb->setRC(&verify_request->rc);
+  llarp_logic_queue_job(verify_request->logic, { verify_request, &logic_threadworker_callback });
+}
+
+// we run the crypto verify in the crypto threadpool worker
+void crypto_threadworker_verifyrc(void *user)
+{
+  llarp_async_verify_rc *verify_request =
+    static_cast< llarp_async_verify_rc * >(user);
+  verify_request->valid = llarp_rc_verify_sig(verify_request->crypto, &verify_request->rc);
+  // if it's valid we need to set it
+  if (verify_request->valid)
+  {
+    llarp::Debug("RC is valid, saving to disk");
+    llarp_threadpool_queue_job(verify_request->diskworker,
+                               { verify_request, &disk_threadworker_setRC });
+  } else {
+    // callback to logic thread
+    llarp::Warn("RC is not valid, can't save to disk");
+    llarp_logic_queue_job(verify_request->logic,
+                          { verify_request, &logic_threadworker_callback });
+  }
+}
+
 extern "C" {
 
 struct llarp_nodedb *
@@ -164,6 +303,18 @@ llarp_nodedb_async_verify(struct llarp_nodedb *nodedb,
                           struct llarp_threadpool *diskworker,
                           struct llarp_async_verify_rc *job)
 {
+  // TODO: ask jeff is safe to remove the parameters
+  // we expect the following to be already set up at this point: user (context: router, llarp_link_establish_job), rc, hook
+  // do additional job set up
+  /*
+  job->logic = logic;
+  job->crypto = crypto;
+  job->cryptoworker = cryptoworker;
+  job->diskworker = diskworker;
+  job->nodedb = nodedb;
+  */
+  // switch to crypto threadpool and continue with crypto_threadworker_verifyrc
+  llarp_threadpool_queue_job(cryptoworker, { job, &crypto_threadworker_verifyrc });
 }
 
 bool
@@ -171,5 +322,5 @@ llarp_nodedb_find_rc(struct llarp_nodedb *nodedb, struct llarp_rc *dst,
                      const byte_t *k)
 {
   return false;
-}
-}
+} // end function
+} // end extern

llarp/router.cpp

@@ -267,6 +267,7 @@ llarp_router::on_verify_server_rc(llarp_async_verify_rc *job)
   llarp::async_verify_context *ctx =
       static_cast< llarp::async_verify_context * >(job->user);
   auto router = ctx->router;
+  llarp::Info("rc verified? ", job->valid?"valid":"invalid");
   if(!job->valid)
   {
     llarp::Warn("invalid server RC");
@@ -451,12 +452,18 @@ llarp_router::async_verify_RC(llarp_link_session *session,
                               bool isExpectingClient,
                               llarp_link_establish_job *establish_job)
 {
-  llarp_async_verify_rc *job = new llarp_async_verify_rc{
-      new llarp::async_verify_context{this, establish_job},
-      {},
-      false,
-      nullptr,
-  };
+  llarp_async_verify_rc *job = new llarp_async_verify_rc;
+  job->user = new llarp::async_verify_context{this, establish_job};
+  job->rc = {};
+  job->valid = false;
+  job->hook = nullptr;
+
+  job->nodedb = nodedb;
+  job->logic = logic;
+  job->crypto = &crypto;
+  job->cryptoworker = tp;
+  job->diskworker = disk;
+
   llarp_rc_copy(&job->rc, session->get_remote_router(session));
   if(isExpectingClient)
     job->hook = &llarp_router::on_verify_client_rc;

llarp/router_contact.cpp

@@ -134,6 +134,9 @@ llarp_rc_bdecode(struct llarp_rc *rc, llarp_buffer_t *buff)
 bool
 llarp_rc_verify_sig(struct llarp_crypto *crypto, struct llarp_rc *rc)
 {
+  // maybe we should copy rc before modifying it
+  // would that make it more thread safe?
+  // jeff agrees
   bool result = false;
   llarp::Signature sig;
   byte_t tmp[MAX_RC_SIZE];