Merge pull request #1078 from jagerman/travis-bionic-and-arm

New travis arches + required endian fixes, libsodium build replacement

.travis.yml

@@ -1,4 +1,5 @@
 language: cpp
+dist: bionic
 
 cache: ccache
 
@@ -7,35 +8,45 @@ matrix:
   include:
   - name: "lint check"
     os: linux
-    dist: xenial
     compiler: gcc
     env: MAKE_TARGET=format-verify PATH="/usr/lib/llvm-8/bin:$PATH"
     addons:
       apt:
         sources:
-          - llvm-toolchain-xenial-8
+          - llvm-toolchain-bionic-8
         packages:
           - clang-format-8
   - name: "make debug (linux/gcc)"
     os: linux
-    dist: xenial
     compiler: gcc
     env: BUILD_TYPE=Debug IS_NOTIFICATION=1
+    addons: &core_apt_addons
+      apt:
+        packages:
+        - build-essential
+        - cmake
+        - curl
+        - git
+        - libcap-dev
+        - libcurl4-openssl-dev
+        - libuv1-dev
+        - ninja-build
+        - libsodium-dev
   - name: "make release (linux/gcc)"
     os: linux
-    dist: xenial
     compiler: gcc
     env: BUILD_TYPE=Release
+    addons: *core_apt_addons
   - name: "make debug (linux/clang)"
     os: linux
-    dist: xenial
     compiler: clang
     env: BUILD_TYPE=Debug
+    addons: *core_apt_addons
   - name: "make release (linux/clang)"
     os: linux
-    dist: xenial
     compiler: clang
     env: BUILD_TYPE=Release
+    addons: *core_apt_addons
   - name: "make debug (macOS/clang)"
     os: osx
     osx_image: xcode10.2
@@ -44,6 +55,21 @@ matrix:
     os: osx
     osx_image: xcode10.2
     env: BUILD_TYPE=Release PATH="/usr/local/opt/ccache/libexec:$PATH"
+  - name: "make release (linux/gcc/arm64)"
+    arch: arm64
+    compiler: gcc
+    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
+  - name: "make release (linux/gcc/s390x)"
+    arch: s390x
+    compiler: gcc
+    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
+  - name: "make release (linux/gcc/ppc64le)"
+    arch: ppc64le
+    compiler: gcc
+    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
 #  - name: "make iOS"
 #    os: osx
 #    osx_image: xcode10.2
@@ -77,11 +103,12 @@ matrix:
     dist: xenial
     compiler: clang
     env: BUILD_TYPE=Debug XSAN=memory
-  - name: "router docker image"
+    addons: *core_apt_addons
-    os: linux
+#  - name: "router docker image"
-    dist: xenial
+#    os: linux
-    env: DOCKER_FILE=docker/router.Dockerfile
+#    dist: xenial
-    services: docker
+#    env: DOCKER_FILE=docker/router.Dockerfile
+#    services: docker
   - name: "make windows docker image"
     os: linux
     dist: xenial
@@ -92,49 +119,19 @@ matrix:
     dist: xenial
     env: DOCKER_FILE=docker/gcc-trunk.Dockerfile
     services: docker
-  - name: "native windows debug"
-    os: windows
-    env: BUILD_TYPE=Debug
-  - name: "native windows release"
-    os: windows
-    env: BUILD_TYPE=Release
-  allow_failures:
-  - name: "make windows docker image"
-    os: linux
-    dist: xenial
-    env: DOCKER_FILE=docker/alpine-windows.Dockerfile
-    services: docker
-  - name: "native windows debug"
-    os: windows
-    env: BUILD_TYPE=Debug
-  - name: "native windows release"
-    os: windows
-    env: BUILD_TYPE=Release
 #  - name: "native windows debug"
 #    os: windows
 #    env: BUILD_TYPE=Debug
 #  - name: "native windows release"
 #    os: windows
 #    env: BUILD_TYPE=Release
-#  - os: linux
+  allow_failures:
-#    dist: xenial
+  - name: "make windows docker image"
-#    env: DOCKER_FILE=docker/alpine-windows.Dockerfile
+  - name: "native windows debug"
-#    services: docker
+  - name: "native windows release"
   - name: "router docker image"
-    os: linux
+  - name: "gcc trunk"
-    dist: xenial
+  - arch: ppc64le
-    env: DOCKER_FILE=docker/router.Dockerfile
-    services: docker
-  - os: linux
-    dist: xenial
-    env: DOCKER_FILE=docker/gcc-trunk.Dockerfile
-    services: docker
-#  - os: osx
-#    osx_image: xcode10.2
-#    env: MAKE_TARGET=windows PATH="/usr/local/opt/ccache/libexec:$PATH"
-#  - os: osx
-#    osx_image: xcode10.2
-#    env: MAKE_TARGET=windows-release PATH="/usr/local/opt/ccache/libexec:$PATH"
 
 env:
   global:
@@ -142,24 +139,6 @@ env:
     - STATIC_LINK=OFF
 
 addons:
-  apt:
-    packages:
-    - binutils-gold
-    - build-essential
-    - clang-format
-    - cmake
-    - curl
-    - docker-ce
-    - g++-mingw-w64
-    - g++-mingw-w64-x86-64
-    - gcc-mingw-w64-base
-    - git
-    - libcap-dev
-    - libcurl4-openssl-dev
-    - libuv1-dev
-    - mingw-w64 mingw-w64-common
-    - ninja-build
-    - libsodium-dev
   homebrew:
     packages:
     - ccache
@@ -182,11 +161,11 @@ before_install:
 
 script:
   - if [ "$TRAVIS_OS_NAME" == "windows" ]; then
-      travis_wait cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && make test';
+      cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && make test';
     elif [[ ! -z $DOCKER_FILE ]]; then
-      travis_wait docker build -f $DOCKER_FILE .;
+      docker build -f $DOCKER_FILE .;
     else
-      travis_wait make DOWNLOAD_SODIUM=ON ${MAKE_TARGET:-test};
+      make DOWNLOAD_SODIUM=ON ${MAKE_TARGET:-test};
     fi
 
 after_script:

CMakeLists.txt

@@ -162,7 +162,11 @@ endif(WOW64_CROSS_COMPILE OR WIN64_CROSS_COMPILE)
 if(DEBIAN)
   add_definitions(-DDEBIAN)
 elseif(NATIVE_BUILD)
-  set(CRYPTO_FLAGS -march=native -mtune=native)
+  if(CMAKE_SYSTEM_PROCESSOR STREQUAL ppc64le)
+    set(CRYPTO_FLAGS -mcpu=native -mtune=native)
+  else()
+    set(CRYPTO_FLAGS -march=native -mtune=native)
+  endif()
 elseif(NOT NON_PC_TARGET)
   if (USE_AVX2)
     set(CRYPTO_FLAGS -march=haswell -mtune=haswell -mfpmath=sse)

cmake/DownloadLibSodium.cmake

@@ -1,63 +1,31 @@
 set(LIBSODIUM_PREFIX ${CMAKE_BINARY_DIR}/libsodium)
-set(LIBSODIUM_SRC ${LIBSODIUM_PREFIX}/libsodium-1.0.18)
+set(LIBSODIUM_URL https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz)
-set(LIBSODIUM_TARBALL ${LIBSODIUM_PREFIX}/libsodium-1.0.18.tar.gz)
+set(LIBSODIUM_HASH SHA512=17e8638e46d8f6f7d024fe5559eccf2b8baf23e143fadd472a7d29d228b186d86686a5e6920385fe2020729119a5f12f989c3a782afbd05a8db4819bb18666ef)
-set(LIBSODIUM_URL https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz)
+
 if(SODIUM_TARBALL_URL)
     # make a build time override of the tarball url so we can fetch it if the original link goes away
     set(LIBSODIUM_URL ${SODIUM_TARBALL_URL})
 endif()
-set(SODIUM_PRETEND_TO_BE_CONFIGURED ON)
-file(DOWNLOAD
-    ${LIBSODIUM_URL}
-    ${LIBSODIUM_TARBALL}
-    EXPECTED_HASH SHA512=17e8638e46d8f6f7d024fe5559eccf2b8baf23e143fadd472a7d29d228b186d86686a5e6920385fe2020729119a5f12f989c3a782afbd05a8db4819bb18666ef
-    SHOW_PROGRESS)
 
-execute_process(COMMAND tar -xzf ${LIBSODIUM_TARBALL} -C ${LIBSODIUM_PREFIX})
+
-if(WIN32)
+file(MAKE_DIRECTORY ${LIBSODIUM_PREFIX}/include)
-  message("patch -p0 -d ${LIBSODIUM_SRC} < ${CMAKE_SOURCE_DIR}/llarp/win32/libsodium-1.0.18-win32.patch")
+
-  execute_process(COMMAND "patch -p0 -d ${LIBSODIUM_SRC} < ${CMAKE_SOURCE_DIR}/llarp/win32/libsodium-1.0.18-win32.patch")
+include(ExternalProject)
-endif()
+include(ProcessorCount)
-file(GLOB_RECURSE sodium_sources
+ExternalProject_Add(libsodium_external
-    ${LIBSODIUM_SRC}/src/libsodium/*.c
+    BUILD_IN_SOURCE ON
-    ${LIBSODIUM_SRC}/src/libsodium/*.h
+    PREFIX ${LIBSODIUM_PREFIX}
+    URL ${LIBSODIUM_URL}
+    URL_HASH ${LIBSODIUM_HASH}
+    CONFIGURE_COMMAND ./configure --prefix=${LIBSODIUM_PREFIX} --enable-static --disable-shared
+    BUILD_COMMAND make -j${PROCESSOR_COUNT}
+    INSTALL_COMMAND ${MAKE}
+    BUILD_BYPRODUCTS ${LIBSODIUM_PREFIX}/lib/libsodium.a ${LIBSODIUM_PREFIX}/include
     )
-add_library(sodium_vendor ${sodium_sources})
 
-set_target_properties(sodium_vendor
+add_library(sodium_vendor STATIC IMPORTED GLOBAL)
-    PROPERTIES
+add_dependencies(sodium_vendor libsodium_external)
-        C_STANDARD 99
+set_target_properties(sodium_vendor PROPERTIES
-)
+    IMPORTED_LOCATION ${LIBSODIUM_PREFIX}/lib/libsodium.a
-
+    INTERFACE_INCLUDE_DIRECTORIES ${LIBSODIUM_PREFIX}/include
-target_include_directories(sodium_vendor
+    )
-    PUBLIC
-        ${LIBSODIUM_SRC}/src/libsodium/include
-    PRIVATE
-        ${LIBSODIUM_SRC}/src/libsodium/include/sodium
-)
-
-target_compile_definitions(sodium_vendor
-    PUBLIC
-        $<$<NOT:$<BOOL:${BUILD_SHARED_LIBS}>>:SODIUM_STATIC>
-        $<$<BOOL:${SODIUM_MINIMAL}>:SODIUM_LIBRARY_MINIMAL>
-    PRIVATE
-        $<$<BOOL:${BUILD_SHARED_LIBS}>:SODIUM_DLL_EXPORT>
-        $<$<BOOL:${SODIUM_ENABLE_BLOCKING_RANDOM}>:USE_BLOCKING_RANDOM>
-        $<$<BOOL:${SODIUM_MINIMAL}>:MINIMAL>
-        $<$<BOOL:${SODIUM_PRETEND_TO_BE_CONFIGURED}>:CONFIGURED>
-)
-
-# Variables that need to be exported to version.h.in
-set(VERSION_ORIG "${VERSION}") # an included module sets things in the calling scope :(
-set(VERSION 1.0.18)
-set(SODIUM_LIBRARY_VERSION_MAJOR 10)
-set(SODIUM_LIBRARY_VERSION_MINOR 3)
-
-configure_file(
-  ${LIBSODIUM_SRC}/src/libsodium/include/sodium/version.h.in
-  ${LIBSODIUM_SRC}/src/libsodium/include/sodium/version.h
-)
-
-target_sources(sodium_vendor PRIVATE ${LIBSODIUM_SRC}/src/libsodium/include/sodium/version.h)
-
-set(VERSION "${VERSION_ORIG}")

crypto/CMakeLists.txt

@@ -66,7 +66,10 @@ target_link_libraries(${CRYPTOGRAPHY_LIB} PRIVATE cryptography_avx_lib)
 
 option(DOWNLOAD_SODIUM "Allow libsodium to be downloaded and built locally if not found on the system" OFF)
 
-find_package(Sodium 1.0.18)
+# Allow -DDOWNLOAD_SODIUM=FORCE to download without even checking for a local libsodium
+if(NOT DOWNLOAD_SODIUM STREQUAL "FORCE")
+  find_package(Sodium 1.0.18)
+endif()
 
 if(sodium_FOUND)
   target_include_directories(${CRYPTOGRAPHY_LIB} PUBLIC ${sodium_INCLUDE_DIR})

docker/alpine-windows.Dockerfile

@@ -6,4 +6,4 @@ RUN apk update && \
 WORKDIR /src/
 COPY . /src/
 
-RUN make windows-release NINJA=ninja STATIC_LINK=ON
+RUN make windows-release NINJA=ninja STATIC_LINK=ON DOWNLOAD_SODIUM=ON

docker/gcc-trunk.Dockerfile

@@ -9,6 +9,6 @@ COPY . /src/
 
 RUN mkdir build && \
     cd build && \
-    cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=ON && \
+    cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=ON -DDOWNLOAD_SODIUM=ON && \
     ninja -k0 && \
     ./test/testAll

docker/router.Dockerfile

@@ -7,7 +7,7 @@ RUN apk update && \
 WORKDIR /src/
 COPY . /src/
 
-RUN make NINJA=ninja STATIC_LINK=ON BUILD_TYPE=Release
+RUN make NINJA=ninja STATIC_LINK=ON BUILD_TYPE=Release DOWNLOAD_SODIUM=ON
 RUN ./lokinet-bootstrap ${bootstrap}
 
 FROM alpine:latest

llarp/CMakeLists.txt

@@ -279,7 +279,7 @@ if(WITH_SHARED)
 endif()
 
 if (WARNINGS_AS_ERRORS)
-  set(WARN_FLAGS -Wall -Wextra -Wshadow -Werror)
+  set(WARN_FLAGS -Wall -Wextra -Werror)
   target_compile_options(${UTIL_LIB} PUBLIC ${WARN_FLAGS})
   target_compile_options(${PLATFORM_LIB} PUBLIC ${WARN_FLAGS})
   target_compile_options(${STATIC_LIB} PUBLIC ${WARN_FLAGS})

llarp/net/ip.cpp

@@ -31,14 +31,9 @@ namespace llarp
     IPPacket::In6ToHUInt(in6_addr addr)
     {
       uint32_t *ptr = in6_uint32_ptr(addr);
-#if __BYTE_ORDER == __BIG_ENDIAN
-      return huint128_t{ptr[0]} | (huint128_t{ptr[1]} << 32)
-          | (huint128_t{ptr[2]} << 64) | (huint128_t{ptr[3]} << 96);
-#else
       return huint128_t{ntohl(ptr[3])} | (huint128_t{ntohl(ptr[2])} << 32)
           | (huint128_t{ntohl(ptr[1])} << 64)
           | (huint128_t{ntohl(ptr[0])} << 96);
-#endif
     }
 
     in6_addr

llarp/net/ip.hpp

@@ -14,10 +14,10 @@
 // anything not win32
 struct ip_header
 {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#ifdef __LITTLE_ENDIAN__
   unsigned int ihl : 4;
   unsigned int version : 4;
-#elif __BYTE_ORDER == __BIG_ENDIAN
+#elif defined(__BIG_ENDIAN__)
   unsigned int version : 4;
   unsigned int ihl : 4;
 #else

llarp/net/net.hpp

@@ -138,11 +138,7 @@ namespace llarp
   constexpr huint32_t
   ipaddr_ipv4_bits(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
   {
-#if __BYTE_ORDER == __ORDER_BIG_ENDIAN__
-    return huint32_t{(a) | (b << 8) | (c << 16) | (d << 24)};
-#else
     return huint32_t{(d) | (c << 8) | (b << 16) | (a << 24)};
-#endif
   }
 
   IPRange

llarp/util/endian.hpp

@@ -94,6 +94,16 @@ be64toh(uint64_t big64);
 
 #endif
 
+#if !defined(__LITTLE_ENDIAN__) && defined(__BYTE_ORDER) \
+    && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif !defined(__BIG_ENDIAN__) && defined(__BYTE_ORDER) \
+    && defined(__BIG_ENDIAN) && __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#elif !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#error "Error: don't know which endian this is"
+#endif
+
 inline uint16_t
 buf16toh(const void *buf)
 {
@@ -193,7 +203,7 @@ htole64buf(void *buf, uint64_t big64)
 inline absl::uint128
 ntoh128(absl::uint128 i)
 {
-#if __BYTE_ORDER == __BIG_ENDIAN
+#ifdef __BIG_ENDIAN__
   return i;
 #else
   const auto loSwapped = htobe64(absl::Uint128Low64(i));

test/router/test_llarp_router_version.cpp

@@ -52,7 +52,7 @@ TEST_F(TestRouterVersion, TestBEncode)
 {
   llarp::RouterVersion v1235( {1, 2, 3}, 5);
 
-  std::array< byte_t, 128 > tmp;
+  std::array< byte_t, 128 > tmp{};
   llarp_buffer_t buf(tmp);
 
   EXPECT_TRUE(v1235.BEncode(&buf));

test/test_llarp_router_contact.cpp

@@ -37,7 +37,7 @@ TEST_F(RCTest, TestSignVerify)
 
   rc.enckey = encr.toPublic();
   rc.pubkey = sign.toPublic();
-  rc.exits.emplace_back(rc.pubkey, nuint32_t{50000});
+  rc.exits.emplace_back(rc.pubkey, nuint32_t{0x08080808});
   ASSERT_TRUE(rc.netID == netid);
   ASSERT_TRUE(rc.netID == NetID::DefaultValue());