Merge pull request #391 from majestrate/staging

Staging
This commit is contained in:
Jeff 2019-03-11 10:13:28 -04:00 committed by GitHub
commit e2989c9ea4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 193 additions and 63 deletions

5
.gitignore vendored
View File

@ -51,3 +51,8 @@ build2/
/contrib/lokinet-bootstrap-winnt/data.enc
/contrib/lokinet-bootstrap-winnt/out.bine
default.profraw
# ctags shit
GTAGS
GRTAGS
GPATH

View File

@ -59,11 +59,10 @@ add_subdirectory(vendor/nlohmann)
# still need the headers unconditionally
set(ABSEIL_DIR vendor/abseil-cpp)
include_directories(${ABSEIL_DIR})
if (NOT WIN32)
add_compile_options(-fPIC)
add_subdirectory(${ABSEIL_DIR})
endif(NOT WIN32)
add_subdirectory(${ABSEIL_DIR})
# turns off those annoying warnings for
# target-specific crypto code paths not
@ -168,7 +167,7 @@ if(ASAN)
endif(ASAN)
if(SHADOW)
set(WITH_STATIC OFF)
set(WITH_SHARED ON)
add_compile_options(-fPIC)
if("${SHADOW_ROOT}" STREQUAL "")
set(SHADOW_ROOT "$ENV{HOME}/.shadow")
@ -182,12 +181,10 @@ if(SHADOW)
set(CMAKE_MODULE_PATH "${SHADOW_ROOT}/share/cmake/Modules")
include_directories(${CMAKE_MODULE_PATH})
include(ShadowTools)
add_compile_options( -fno-inline -fno-strict-aliasing )
add_definitions(-DTESTNET=true)
add_compile_options(-fno-inline -fno-strict-aliasing )
add_definitions(-DTESTNET=1)
add_definitions(-DSHADOW_TESTNET)
include_directories(${SHADOW_ROOT}/include)
else()
set(WITH_STATIC ON)
endif(SHADOW)
if(CMAKE_BUILD_TYPE MATCHES "[Dd][Ee][Bb][Uu][Gg]")
@ -346,11 +343,19 @@ endif(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
add_log_tag(${ABYSS_EXE})
add_log_tag(${ABYSS_LIB})
if (WIN32)
set(EXE_LIBS ${STATIC_LIB} ws2_32 iphlpapi)
else()
set(EXE_LIBS ${STATIC_LIB} cppbackport libutp)
endif(WIN32)
if(SHADOW)
add_shadow_plugin(shadow-plugin-${SHARED_LIB} ${EXE_SRC})
target_include_directories(shadow-plugin-${SHARED_LIB} PUBLIC ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/llarp ${PROJECT_SOURCE_DIR}/crypto/include ${PROJECT_SOURCE_DIR}/vendor/cppbackport-master/lib)
target_link_libraries(shadow-plugin-${SHARED_LIB} ${STATIC_LIB} ${LIBS})
install(TARGETS shadow-plugin-${SHARED_LIB} DESTINATION plugins)
set(LOKINET_SHADOW shadow-plugin-${SHARED_LIB})
set(LOKINET_SHADOW_LIBS ${SHARED_LIB})
add_shadow_plugin(${LOKINET_SHADOW} ${EXE_SRC})
target_link_libraries(${LOKINET_SHADOW} ${LOKINET_SHADOW_LIBS})
target_include_directories(${LOKINET_SHADOW} PUBLIC ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/llarp ${PROJECT_SOURCE_DIR}/crypto/include ${PROJECT_SOURCE_DIR}/vendor/cppbackport-master/lib)
else()
if(NOT WIN32)
add_executable(${EXE} ${EXE_SRC})
@ -371,12 +376,7 @@ else()
install(CODE "execute_process(COMMAND setcap cap_net_admin,cap_net_bind_service=+eip ${CMAKE_INSTALL_PREFIX}/bin/lokinet)")
endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
if (WIN32)
target_link_libraries(${EXE} PUBLIC ${STATIC_LIB} ws2_32 iphlpapi)
else()
target_link_libraries(${EXE} PUBLIC ${STATIC_LIB} cppbackport libutp)
endif(WIN32)
target_link_libraries(${EXE} PUBLIC ${EXE_LIBS})
if(ANDROID)
add_library(${ANDROID_LIB} SHARED jni/lokinet_android.cpp)

View File

@ -22,6 +22,7 @@ SHADOW_LOG=$(REPO)/shadow.log.txt
SHADOW_SRC ?= $(HOME)/local/shadow
SHADOW_PARSE ?= $(PYTHON) $(SHADOW_SRC)/src/tools/parse-shadow.py - -m 0 --packet-data
SHADOW_PLOT ?= $(PYTHON) $(SHADOW_SRC)/src/tools/plot-shadow.py -d $(REPO) LokiNET -c $(SHADOW_CONFIG) -r 10000 -e '.*'
SHADOW_OPTS ?=
TESTNET_ROOT=/tmp/lokinet_testnet_tmp
TESTNET_CONF=$(TESTNET_ROOT)/supervisor.conf
@ -82,11 +83,11 @@ BUILD_ROOT = $(REPO)/build
SCAN_BUILD ?= scan-build
CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DUSE_LIBABYSS=$(JSONRPC) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) '$(REPO)'")
CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DUSE_LIBABYSS=$(JSONRPC) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
ANALYZE_CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "$(SCAN_BUILD) cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DUSE_LIBABYSS=$(JSONRPC) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) '$(REPO)'")
ANALYZE_CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "$(SCAN_BUILD) cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DUSE_LIBABYSS=$(JSONRPC) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
COVERAGE_CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DUSE_LIBABYSS=$(JSONRPC) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DWITH_COVERAGE=yes '$(REPO)'")
COVERAGE_CONFIG_CMD = $(shell /bin/echo -n "cd '$(BUILD_ROOT)' && " ; /bin/echo -n "cmake -G'$(CMAKE_GEN)' -DCMAKE_CROSSCOMPILING=$(CROSS) -DSTATIC_LINK_RUNTIME=$(STATIC_LINK) -DUSE_NETNS=$(NETNS) -DUSE_AVX2=$(AVX2) -DUSE_LIBABYSS=$(JSONRPC) -DNON_PC_TARGET=$(NON_PC_TARGET) -DWITH_SHARED=$(SHARED_LIB) -DWITH_COVERAGE=yes -DCMAKE_EXPORT_COMPILE_COMMANDS=ON '$(REPO)'")
TARGETS = $(REPO)/lokinet
SIGS = $(TARGETS:=.sig)
@ -139,8 +140,8 @@ shadow-build: shadow-configure
shadow-run: shadow-build
$(PYTHON) $(REPO)/contrib/shadow/genconf.py $(SHADOW_CONFIG)
cp $(SHADOW_PLUGIN) $(REPO)
bash -c "$(SHADOW_BIN) -w $$(cat /proc/cpuinfo | grep processor | wc -l) $(SHADOW_CONFIG) | $(SHADOW_PARSE)"
cp $(SHADOW_PLUGIN) $(REPO)/libshadow-plugin-lokinet.so
$(SHADOW_BIN) $(SHADOW_OPTS) $(SHADOW_CONFIG) | $(SHADOW_PARSE)
shadow-plot: shadow-run
$(SHADOW_PLOT)
@ -184,14 +185,16 @@ android-gradle: android-gradle-prepare
android: android-gradle
cp -f $(ANDROID_DIR)/build/outputs/apk/*.apk $(REPO)
windows-configure: clean
windows-release-configure: clean
mkdir -p '$(BUILD_ROOT)'
$(CONFIG_CMD) -DCMAKE_CROSSCOMPILING=ON -DCMAKE_TOOLCHAIN_FILE='$(REPO)/contrib/cross/mingw.cmake' -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=i686-w64-mingw32-gcc-win32 -DCMAKE_CXX_COMPILER=i686-w64-mingw32-g++-win32 -DCMAKE_ASM_FLAGS='$(ASFLAGS)' -DCMAKE_C_FLAGS='$(CFLAGS)' -DCMAKE_CXX_FLAGS='$(CXXFLAGS)'
$(CONFIG_CMD) -DCMAKE_TOOLCHAIN_FILE='$(REPO)/contrib/cross/mingw.cmake' -DCMAKE_BUILD_TYPE=Release -DCMAKE_ASM_FLAGS='$(ASFLAGS)' -DCMAKE_C_FLAGS='$(CFLAGS)' -DCMAKE_CXX_FLAGS='$(CXXFLAGS)'
windows: windows-configure
windows-release: windows-release-configure
$(MAKE) -C '$(BUILD_ROOT)'
cp '$(BUILD_ROOT)/lokinet.exe' '$(REPO)/lokinet.exe'
windows: windows-release
abyss: debug
$(ABYSS_EXE)

View File

@ -27,7 +27,7 @@ LLARP - Low Latency Anon Routing Protocol
state sponsored network attacks. The former is an inherit property of low latency
computer networks that I personally do not think is possible to properly fully
"solve". The latter is a threat that lies outside the scope of what the current
toolset that is available to me at the moment.
toolset that is available to me at the moment provides.
This project does not attempt to be a magical application level cure-all for

View File

@ -203,17 +203,18 @@ set(LIB_SRC
service/types.cpp
service/vanity.cpp
)
if(TESTNET)
set(LIB_SRC ${LIB_SRC} testnet.c)
endif()
add_library(${STATIC_LIB} STATIC ${LIB_SRC})
target_link_libraries(${STATIC_LIB} PUBLIC ${ABYSS_LIB} ${PLATFORM_LIB} ${UTIL_LIB} ${CRYPTOGRAPHY_LIB} libutp ${LIBS})
if(TESTNET)
target_sources(${STATIC_LIB} PUBLIC testnet.c)
endif()
set(LIBS ${LIBS} libutp)
target_link_libraries(${STATIC_LIB} PUBLIC ${ABYSS_LIB} ${PLATFORM_LIB} ${UTIL_LIB} ${CRYPTOGRAPHY_LIB} ${LIBS})
if(WITH_SHARED)
add_library(${SHARED_LIB} SHARED ${LIB_SRC})
target_link_libraries(${SHARED_LIB} PUBLIC ${ABYSS_LIB} ${CRYPTOGRAPHY_LIB} ${LIBS} ${UTIL_LIB} ${PLATFORM_LIB} Threads::Threads)
set(LIBS ${LIBS} Threads::Threads)
target_link_libraries(${SHARED_LIB} PUBLIC ${ABYSS_LIB} ${CRYPTOGRAPHY_LIB} ${UTIL_LIB} ${PLATFORM_LIB} ${LIBS})
if (WIN32)
target_link_libraries(${SHARED_LIB} PUBLIC ws2_32 iphlpapi)
else()

View File

@ -12,13 +12,12 @@ static CRITICAL_SECTION HandlerMtx;
std::list< win32_tun_io* > tun_listeners;
void
begin_tun_loop(int nThreads)
win32_tun_io::begin_tun_loop(int nThreads)
{
kThreadPool = new HANDLE[nThreads];
for(int i = 0; i < nThreads; ++i)
{
kThreadPool[i] =
CreateThread(nullptr, 0, &tun_ev_loop, nullptr, 0, nullptr);
kThreadPool[i] = CreateThread(nullptr, 0, &tun_ev_loop, this, 0, nullptr);
}
llarp::LogInfo("created ", nThreads, " threads for TUN event queue");
poolSize = nThreads;
@ -124,11 +123,48 @@ win32_tun_io::read(byte_t* buf, size_t sz)
ReadFile(tunif->tun_fd, buf, sz, nullptr, &pkt->pkt);
}
static void
tun_ev_tick_and_flush(void* user)
{
llarp_tun_io* tun = static_cast< llarp_tun_io* >(user);
if(tun->tick)
tun->tick(tun);
tun->flush(tun);
}
struct tun_pkt_t
{
std::vector< byte_t > pkt;
llarp_tun_io* tun;
tun_pkt_t(llarp_tun_io* t, const byte_t* buf, size_t sz) : tun(t), pkt(sz)
{
std::copy_n(buf, sz, pkt.begin());
}
static void
recv_pkt(void* user)
{
tun_pkt_t* pkt = static_cast< tun_pkt_t* >(user);
pkt->Recv();
delete pkt;
}
private:
void
Recv()
{
llarp_buffer_t buf(pkt);
if(tun->recvpkt)
tun->recvpkt(tun, buf);
}
}
// and now the event loop itself
extern "C" DWORD FAR PASCAL
tun_ev_loop(void* unused)
tun_ev_loop(void* user)
{
UNREFERENCED_PARAMETER(unused);
win32_tun_io* tun_io = static_cast< win32_tun_io* >(user);
DWORD size = 0;
OVERLAPPED* ovl = nullptr;
@ -148,11 +184,14 @@ tun_ev_loop(void* unused)
// of the tun logic
for(const auto& tun : tun_listeners)
{
/*
EnterCriticalSection(&HandlerMtx);
if(tun->t->tick)
tun->t->tick(tun->t);
tun->flush_write();
LeaveCriticalSection(&HandlerMtx);
*/
tun_io->logic->queue_job({tun->t, &tun_ev_tick_and_flush});
}
continue; // let's go at it once more
}
@ -173,8 +212,10 @@ tun_ev_loop(void* unused)
continue;
}
// EnterCriticalSection(&HandlerMtx);
if(ev->t->recvpkt)
ev->t->recvpkt(ev->t, llarp_buffer_t(pkt->buf, size));
tun_pkt_t* recv_pkt = new tun_pkt_t(ev->t, pkt->buf, size);
tun_io->logic->queue_job({recv_pkt, &tun_pkt_t::recv_pkt});
// if(ev->t->recvpkt)
// ev->t->recvpkt(ev->t, llarp_buffer_t(pkt->buf, size));
ev->read(ev->readbuf, sizeof(ev->readbuf));
// LeaveCriticalSection(&HandlerMtx);
}
@ -185,11 +226,14 @@ tun_ev_loop(void* unused)
ev->read(ev->readbuf, sizeof(ev->readbuf));
// LeaveCriticalSection(&HandlerMtx);
}
/*
EnterCriticalSection(&HandlerMtx);
if(ev->t->tick)
ev->t->tick(ev->t);
ev->flush_write();
LeaveCriticalSection(&HandlerMtx);
*/
tun_io->logic->queue_job({ev->t, &tun_ev_tick_and_flush});
delete pkt; // don't leak
}
llarp::LogDebug("exit TUN event loop thread from system managed thread pool");
@ -657,4 +701,4 @@ llarp_win32_loop::stop()
upoll_destroy(upollfd);
upollfd = nullptr;
llarp::LogDebug("destroy upoll");
}
}

View File

@ -23,14 +23,11 @@ struct asio_evt_pkt
};
extern "C" DWORD FAR PASCAL
tun_ev_loop(void* unused);
tun_ev_loop(void* ev);
void
exit_tun_loop();
void
begin_tun_loop(int nThreads);
namespace llarp
{
struct udp_listener : public ev_io
@ -60,10 +57,12 @@ namespace llarp
struct win32_tun_io
{
llarp_tun_io* t;
llarp::Logic* logic;
device* tunif;
byte_t readbuf[EV_READ_BUF_SZ] = {0};
win32_tun_io(llarp_tun_io* tio) : t(tio), tunif(tuntap_init()){};
win32_tun_io(llarp_tun_io* tio, llarp::Logic* l)
: t(tio), logic(l), tunif(tuntap_init()){};
bool
queue_write(const byte_t* buf, size_t sz);
@ -152,4 +151,4 @@ struct llarp_win32_loop : public llarp_ev_loop
stop();
};
#endif
#endif

View File

@ -67,6 +67,11 @@ namespace llarp
{
return db->Get(m_ExitRouter, cur);
}
else if(hop == numHops - 2)
{
return db->select_random_hop_excluding(cur,
{prev.pubkey, m_ExitRouter});
}
else
return path::Builder::SelectHop(db, prev, cur, hop, roles);
}

View File

@ -184,7 +184,6 @@ namespace llarp
QueueInboundPacketForExit(const llarp_buffer_t& buf)
{
ManagedBuffer copy{buf};
return m_NetworkToUserPktQueue.EmplaceIf(
[&](llarp::net::IPv4Packet& pkt) -> bool {
if(!pkt.Load(copy.underlying))

View File

@ -123,7 +123,10 @@ namespace llarp
++itr;
}
else
{
LogInfo("pending session at ", itr->first, " timed out");
itr = m_Pending.erase(itr);
}
}
}
}
@ -166,14 +169,6 @@ namespace llarp
return false;
}
void
ILinkLayer::RemovePending(ILinkSession* s)
{
Lock l(&m_PendingMutex);
llarp::Addr remote = s->GetRemoteEndpoint();
m_Pending.erase(remote);
}
util::StatusObject
ILinkLayer::ExtractStatus() const
{

View File

@ -197,8 +197,8 @@ namespace llarp
SessionRenegotiateHandler SessionRenegotiate;
/// called by link session to remove a pending session who is timed out
void
RemovePending(ILinkSession* s) LOCKS_EXCLUDED(m_PendingMutex);
// void
// RemovePending(ILinkSession* s) LOCKS_EXCLUDED(m_PendingMutex);
private:
static void

View File

@ -228,6 +228,8 @@ namespace llarp
bool
Session::IsTimedOut(llarp_time_t now) const
{
if(state == eConnecting)
return false;
if(state == eClose)
return true;
if(now <= lastActive)
@ -358,7 +360,6 @@ namespace llarp
utp_close(arg->socket);
else
session->Close();
link->RemovePending(session);
}
return 0;
}

View File

@ -468,3 +468,47 @@ llarp_nodedb::select_random_hop(const llarp::RouterContact &prev,
}
return false;
}
bool
llarp_nodedb::select_random_hop_excluding(
llarp::RouterContact &result, const std::set< llarp::RouterID > &exclude)
{
llarp::util::Lock lock(&access);
/// checking for "guard" status for N = 0 is done by caller inside of
/// pathbuilder's scope
size_t sz = entries.size();
if(sz < 3)
return false;
llarp_time_t now = llarp::time_now_ms();
auto itr = entries.begin();
size_t pos = llarp::randint() % sz;
std::advance(itr, pos);
auto start = itr;
while(itr == entries.end())
{
if(exclude.count(itr->first) == 0)
{
if(itr->second.addrs.size() && !itr->second.IsExpired(now))
{
result = itr->second;
return true;
}
}
itr++;
}
itr = entries.begin();
while(itr != start)
{
if(exclude.count(itr->first) == 0)
{
if(itr->second.addrs.size() && !itr->second.IsExpired(now))
{
result = itr->second;
return true;
}
}
++itr;
}
return false;
}

View File

@ -9,6 +9,8 @@
#include <absl/base/thread_annotations.h>
#include <set>
/**
* nodedb.hpp
*
@ -113,6 +115,11 @@ struct llarp_nodedb
llarp::RouterContact &result, size_t N)
LOCKS_EXCLUDED(access);
bool
select_random_hop_excluding(llarp::RouterContact &result,
const std::set< llarp::RouterID > &exclude)
LOCKS_EXCLUDED(access);
static bool
ensure_dir(const char *dir);
};

View File

@ -450,6 +450,15 @@ namespace llarp
return hops[0].rc.pubkey;
}
std::string
Path::HopsString() const
{
std::stringstream ss;
for(const auto& hop : hops)
ss << RouterID(hop.rc.pubkey) << " -> ";
return ss.str();
}
void
Path::EnterState(PathStatus st, llarp_time_t now)
{

View File

@ -362,6 +362,9 @@ namespace llarp
return _status;
}
std::string
HopsString() const;
llarp_time_t
LastRemoteActivityAt() const override
{

View File

@ -213,11 +213,15 @@ namespace llarp
size_t tries = 10;
do
{
cur.Clear();
--tries;
if(db->select_random_hop(prev, cur, hop))
return true;
} while(router->routerProfiling().IsBad(cur.pubkey) && tries > 0);
return false;
{
if(!router->routerProfiling().IsBad(cur.pubkey))
return true;
}
} while(tries > 0);
return tries > 0;
}
bool

View File

@ -292,7 +292,7 @@ namespace llarp
void
PathSet::HandlePathBuildTimeout(Path* p)
{
LogInfo("path build for ", p->Name(), " has timed out");
LogInfo("path build ", p->HopsString(), " timed out");
}
bool

View File

@ -70,7 +70,7 @@ namespace llarp
bool
RouterProfile::IsGood(uint64_t chances) const
{
if(connectTimeoutCount > 3)
if(connectTimeoutCount > chances)
return connectTimeoutCount <= connectGoodCount
&& (pathSuccessCount * chances) >= pathFailCount;
else

View File

@ -131,6 +131,12 @@ namespace llarp
&& netID == other.netID;
}
bool
operator<(const RouterContact &other) const
{
return pubkey < other.pubkey;
}
bool
operator!=(const RouterContact &other) const
{

View File

@ -1854,6 +1854,11 @@ namespace llarp
return false;
}
}
else if(hop == numHops - 2)
{
return db->select_random_hop_excluding(
cur, {prev.pubkey, m_NextIntro.router});
}
(void)roles;
return path::Builder::SelectHop(db, prev, cur, hop, roles);
}