Merge pull request #1078 from jagerman/travis-bionic-and-arm

New travis arches + required endian fixes, libsodium build replacement
5 years ago · 0271f178d2
parent e44449b2d5 a1813b6164
commit 0271f178d2
14 changed files with 97 additions and 142 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,4 +1,5 @@
 language: cpp
+dist: bionic

 cache: ccache

@ -7,35 +8,45 @@ matrix:
  include:
  - name: "lint check"
    os: linux
-    dist: xenial
    compiler: gcc
    env: MAKE_TARGET=format-verify PATH="/usr/lib/llvm-8/bin:$PATH"
    addons:
      apt:
        sources:
-          - llvm-toolchain-xenial-8
+          - llvm-toolchain-bionic-8
        packages:
          - clang-format-8
  - name: "make debug (linux/gcc)"
    os: linux
-    dist: xenial
    compiler: gcc
    env: BUILD_TYPE=Debug IS_NOTIFICATION=1
+    addons: &core_apt_addons
+      apt:
+        packages:
+        - build-essential
+        - cmake
+        - curl
+        - git
+        - libcap-dev
+        - libcurl4-openssl-dev
+        - libuv1-dev
+        - ninja-build
+        - libsodium-dev
  - name: "make release (linux/gcc)"
    os: linux
-    dist: xenial
    compiler: gcc
    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
  - name: "make debug (linux/clang)"
    os: linux
-    dist: xenial
    compiler: clang
    env: BUILD_TYPE=Debug
+    addons: *core_apt_addons
  - name: "make release (linux/clang)"
    os: linux
-    dist: xenial
    compiler: clang
    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
  - name: "make debug (macOS/clang)"
    os: osx
    osx_image: xcode10.2
@ -44,6 +55,21 @@ matrix:
    os: osx
    osx_image: xcode10.2
    env: BUILD_TYPE=Release PATH="/usr/local/opt/ccache/libexec:$PATH"
+  - name: "make release (linux/gcc/arm64)"
+    arch: arm64
+    compiler: gcc
+    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
+  - name: "make release (linux/gcc/s390x)"
+    arch: s390x
+    compiler: gcc
+    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
+  - name: "make release (linux/gcc/ppc64le)"
+    arch: ppc64le
+    compiler: gcc
+    env: BUILD_TYPE=Release
+    addons: *core_apt_addons
 #  - name: "make iOS"
 #    os: osx
 #    osx_image: xcode10.2
@ -77,11 +103,12 @@ matrix:
    dist: xenial
    compiler: clang
    env: BUILD_TYPE=Debug XSAN=memory
-  - name: "router docker image"
-    os: linux
-    dist: xenial
-    env: DOCKER_FILE=docker/router.Dockerfile
-    services: docker
+    addons: *core_apt_addons
+#  - name: "router docker image"
+#    os: linux
+#    dist: xenial
+#    env: DOCKER_FILE=docker/router.Dockerfile
+#    services: docker
  - name: "make windows docker image"
    os: linux
    dist: xenial
@ -92,49 +119,19 @@ matrix:
    dist: xenial
    env: DOCKER_FILE=docker/gcc-trunk.Dockerfile
    services: docker
-  - name: "native windows debug"
-    os: windows
-    env: BUILD_TYPE=Debug
-  - name: "native windows release"
-    os: windows
-    env: BUILD_TYPE=Release
-  allow_failures:
-  - name: "make windows docker image"
-    os: linux
-    dist: xenial
-    env: DOCKER_FILE=docker/alpine-windows.Dockerfile
-    services: docker
-  - name: "native windows debug"
-    os: windows
-    env: BUILD_TYPE=Debug
-  - name: "native windows release"
-    os: windows
-    env: BUILD_TYPE=Release
 #  - name: "native windows debug"
 #    os: windows
 #    env: BUILD_TYPE=Debug
 #  - name: "native windows release"
 #    os: windows
 #    env: BUILD_TYPE=Release
-#  - os: linux
-#    dist: xenial
-#    env: DOCKER_FILE=docker/alpine-windows.Dockerfile
-#    services: docker
+  allow_failures:
+  - name: "make windows docker image"
+  - name: "native windows debug"
+  - name: "native windows release"
  - name: "router docker image"
-    os: linux
-    dist: xenial
-    env: DOCKER_FILE=docker/router.Dockerfile
-    services: docker
-  - os: linux
-    dist: xenial
-    env: DOCKER_FILE=docker/gcc-trunk.Dockerfile
-    services: docker
-#  - os: osx
-#    osx_image: xcode10.2
-#    env: MAKE_TARGET=windows PATH="/usr/local/opt/ccache/libexec:$PATH"
-#  - os: osx
-#    osx_image: xcode10.2
-#    env: MAKE_TARGET=windows-release PATH="/usr/local/opt/ccache/libexec:$PATH"
+  - name: "gcc trunk"
+  - arch: ppc64le

 env:
  global:
@ -142,24 +139,6 @@ env:
    - STATIC_LINK=OFF

 addons:
-  apt:
-    packages:
-    - binutils-gold
-    - build-essential
-    - clang-format
-    - cmake
-    - curl
-    - docker-ce
-    - g++-mingw-w64
-    - g++-mingw-w64-x86-64
-    - gcc-mingw-w64-base
-    - git
-    - libcap-dev
-    - libcurl4-openssl-dev
-    - libuv1-dev
-    - mingw-w64 mingw-w64-common
-    - ninja-build
-    - libsodium-dev
  homebrew:
    packages:
    - ccache
@ -182,11 +161,11 @@ before_install:

 script:
  - if [ "$TRAVIS_OS_NAME" == "windows" ]; then
-      travis_wait cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && make test';
+      cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && make test';
    elif [[ ! -z $DOCKER_FILE ]]; then
-      travis_wait docker build -f $DOCKER_FILE .;
+      docker build -f $DOCKER_FILE .;
    else
-      travis_wait make DOWNLOAD_SODIUM=ON ${MAKE_TARGET:-test};
+      make DOWNLOAD_SODIUM=ON ${MAKE_TARGET:-test};
    fi

 after_script:
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -162,7 +162,11 @@ endif(WOW64_CROSS_COMPILE OR WIN64_CROSS_COMPILE)
 if(DEBIAN)
  add_definitions(-DDEBIAN)
 elseif(NATIVE_BUILD)
-  set(CRYPTO_FLAGS -march=native -mtune=native)
+  if(CMAKE_SYSTEM_PROCESSOR STREQUAL ppc64le)
+    set(CRYPTO_FLAGS -mcpu=native -mtune=native)
+  else()
+    set(CRYPTO_FLAGS -march=native -mtune=native)
+  endif()
 elseif(NOT NON_PC_TARGET)
  if (USE_AVX2)
    set(CRYPTO_FLAGS -march=haswell -mtune=haswell -mfpmath=sse)
--- a/cmake/DownloadLibSodium.cmake
+++ b/cmake/DownloadLibSodium.cmake
@ -1,63 +1,31 @@
 set(LIBSODIUM_PREFIX ${CMAKE_BINARY_DIR}/libsodium)
-set(LIBSODIUM_SRC ${LIBSODIUM_PREFIX}/libsodium-1.0.18)
-set(LIBSODIUM_TARBALL ${LIBSODIUM_PREFIX}/libsodium-1.0.18.tar.gz)
-set(LIBSODIUM_URL https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz)
+set(LIBSODIUM_URL https://github.com/jedisct1/libsodium/releases/download/1.0.18-RELEASE/libsodium-1.0.18.tar.gz https://download.libsodium.org/libsodium/releases/libsodium-1.0.18.tar.gz)
+set(LIBSODIUM_HASH SHA512=17e8638e46d8f6f7d024fe5559eccf2b8baf23e143fadd472a7d29d228b186d86686a5e6920385fe2020729119a5f12f989c3a782afbd05a8db4819bb18666ef)
+
 if(SODIUM_TARBALL_URL)
    # make a build time override of the tarball url so we can fetch it if the original link goes away
    set(LIBSODIUM_URL ${SODIUM_TARBALL_URL})
 endif()
-set(SODIUM_PRETEND_TO_BE_CONFIGURED ON)
-file(DOWNLOAD
-    ${LIBSODIUM_URL}
-    ${LIBSODIUM_TARBALL}
-    EXPECTED_HASH SHA512=17e8638e46d8f6f7d024fe5559eccf2b8baf23e143fadd472a7d29d228b186d86686a5e6920385fe2020729119a5f12f989c3a782afbd05a8db4819bb18666ef
-    SHOW_PROGRESS)
-
-execute_process(COMMAND tar -xzf ${LIBSODIUM_TARBALL} -C ${LIBSODIUM_PREFIX})
-if(WIN32)
-  message("patch -p0 -d ${LIBSODIUM_SRC} < ${CMAKE_SOURCE_DIR}/llarp/win32/libsodium-1.0.18-win32.patch")
-  execute_process(COMMAND "patch -p0 -d ${LIBSODIUM_SRC} < ${CMAKE_SOURCE_DIR}/llarp/win32/libsodium-1.0.18-win32.patch")
-endif()
-file(GLOB_RECURSE sodium_sources
-    ${LIBSODIUM_SRC}/src/libsodium/*.c
-    ${LIBSODIUM_SRC}/src/libsodium/*.h
-    )
-add_library(sodium_vendor ${sodium_sources})
-
-set_target_properties(sodium_vendor
-    PROPERTIES
-        C_STANDARD 99
-)

-target_include_directories(sodium_vendor
-    PUBLIC
-        ${LIBSODIUM_SRC}/src/libsodium/include
-    PRIVATE
-        ${LIBSODIUM_SRC}/src/libsodium/include/sodium
-)

-target_compile_definitions(sodium_vendor
-    PUBLIC
-        $<$<NOT:$<BOOL:${BUILD_SHARED_LIBS}>>:SODIUM_STATIC>
-        $<$<BOOL:${SODIUM_MINIMAL}>:SODIUM_LIBRARY_MINIMAL>
-    PRIVATE
-        $<$<BOOL:${BUILD_SHARED_LIBS}>:SODIUM_DLL_EXPORT>
-        $<$<BOOL:${SODIUM_ENABLE_BLOCKING_RANDOM}>:USE_BLOCKING_RANDOM>
-        $<$<BOOL:${SODIUM_MINIMAL}>:MINIMAL>
-        $<$<BOOL:${SODIUM_PRETEND_TO_BE_CONFIGURED}>:CONFIGURED>
-)
+file(MAKE_DIRECTORY ${LIBSODIUM_PREFIX}/include)

-# Variables that need to be exported to version.h.in
-set(VERSION_ORIG "${VERSION}") # an included module sets things in the calling scope :(
-set(VERSION 1.0.18)
-set(SODIUM_LIBRARY_VERSION_MAJOR 10)
-set(SODIUM_LIBRARY_VERSION_MINOR 3)
-
-configure_file(
-  ${LIBSODIUM_SRC}/src/libsodium/include/sodium/version.h.in
-  ${LIBSODIUM_SRC}/src/libsodium/include/sodium/version.h
-)
-
-target_sources(sodium_vendor PRIVATE ${LIBSODIUM_SRC}/src/libsodium/include/sodium/version.h)
+include(ExternalProject)
+include(ProcessorCount)
+ExternalProject_Add(libsodium_external
+    BUILD_IN_SOURCE ON
+    PREFIX ${LIBSODIUM_PREFIX}
+    URL ${LIBSODIUM_URL}
+    URL_HASH ${LIBSODIUM_HASH}
+    CONFIGURE_COMMAND ./configure --prefix=${LIBSODIUM_PREFIX} --enable-static --disable-shared
+    BUILD_COMMAND make -j${PROCESSOR_COUNT}
+    INSTALL_COMMAND ${MAKE}
+    BUILD_BYPRODUCTS ${LIBSODIUM_PREFIX}/lib/libsodium.a ${LIBSODIUM_PREFIX}/include
+    )

-set(VERSION "${VERSION_ORIG}")
+add_library(sodium_vendor STATIC IMPORTED GLOBAL)
+add_dependencies(sodium_vendor libsodium_external)
+set_target_properties(sodium_vendor PROPERTIES
+    IMPORTED_LOCATION ${LIBSODIUM_PREFIX}/lib/libsodium.a
+    INTERFACE_INCLUDE_DIRECTORIES ${LIBSODIUM_PREFIX}/include
+    )
--- a/crypto/CMakeLists.txt
+++ b/crypto/CMakeLists.txt
@ -66,7 +66,10 @@ target_link_libraries(${CRYPTOGRAPHY_LIB} PRIVATE cryptography_avx_lib)

 option(DOWNLOAD_SODIUM "Allow libsodium to be downloaded and built locally if not found on the system" OFF)

-find_package(Sodium 1.0.18)
+# Allow -DDOWNLOAD_SODIUM=FORCE to download without even checking for a local libsodium
+if(NOT DOWNLOAD_SODIUM STREQUAL "FORCE")
+  find_package(Sodium 1.0.18)
+endif()

 if(sodium_FOUND)
  target_include_directories(${CRYPTOGRAPHY_LIB} PUBLIC ${sodium_INCLUDE_DIR})
--- a/docker/alpine-windows.Dockerfile
+++ b/docker/alpine-windows.Dockerfile
@ -6,4 +6,4 @@ RUN apk update && \
 WORKDIR /src/
 COPY . /src/

-RUN make windows-release NINJA=ninja STATIC_LINK=ON
+RUN make windows-release NINJA=ninja STATIC_LINK=ON DOWNLOAD_SODIUM=ON
--- a/docker/gcc-trunk.Dockerfile
+++ b/docker/gcc-trunk.Dockerfile
@ -9,6 +9,6 @@ COPY . /src/

 RUN mkdir build && \
    cd build && \
-    cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=ON && \
+    cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=ON -DDOWNLOAD_SODIUM=ON && \
    ninja -k0 && \
    ./test/testAll
--- a/docker/router.Dockerfile
+++ b/docker/router.Dockerfile
@ -7,7 +7,7 @@ RUN apk update && \
 WORKDIR /src/
 COPY . /src/

-RUN make NINJA=ninja STATIC_LINK=ON BUILD_TYPE=Release
+RUN make NINJA=ninja STATIC_LINK=ON BUILD_TYPE=Release DOWNLOAD_SODIUM=ON
 RUN ./lokinet-bootstrap ${bootstrap}

 FROM alpine:latest
--- a/llarp/CMakeLists.txt
+++ b/llarp/CMakeLists.txt
@ -279,7 +279,7 @@ if(WITH_SHARED)
 endif()

 if (WARNINGS_AS_ERRORS)
-  set(WARN_FLAGS -Wall -Wextra -Wshadow -Werror)
+  set(WARN_FLAGS -Wall -Wextra -Werror)
  target_compile_options(${UTIL_LIB} PUBLIC ${WARN_FLAGS})
  target_compile_options(${PLATFORM_LIB} PUBLIC ${WARN_FLAGS})
  target_compile_options(${STATIC_LIB} PUBLIC ${WARN_FLAGS})
--- a/llarp/net/ip.cpp
+++ b/llarp/net/ip.cpp
@ -31,14 +31,9 @@ namespace llarp
    IPPacket::In6ToHUInt(in6_addr addr)
    {
      uint32_t *ptr = in6_uint32_ptr(addr);
-#if __BYTE_ORDER == __BIG_ENDIAN
-      return huint128_t{ptr[0]} | (huint128_t{ptr[1]} << 32)
-          | (huint128_t{ptr[2]} << 64) | (huint128_t{ptr[3]} << 96);
-#else
      return huint128_t{ntohl(ptr[3])} | (huint128_t{ntohl(ptr[2])} << 32)
          | (huint128_t{ntohl(ptr[1])} << 64)
          | (huint128_t{ntohl(ptr[0])} << 96);
-#endif
    }

    in6_addr
--- a/llarp/net/ip.hpp
+++ b/llarp/net/ip.hpp
@ -14,10 +14,10 @@
 // anything not win32
 struct ip_header
 {
-#if __BYTE_ORDER == __LITTLE_ENDIAN
+#ifdef __LITTLE_ENDIAN__
  unsigned int ihl : 4;
  unsigned int version : 4;
-#elif __BYTE_ORDER == __BIG_ENDIAN
+#elif defined(__BIG_ENDIAN__)
  unsigned int version : 4;
  unsigned int ihl : 4;
 #else
--- a/llarp/net/net.hpp
+++ b/llarp/net/net.hpp
@ -138,11 +138,7 @@ namespace llarp
  constexpr huint32_t
  ipaddr_ipv4_bits(uint32_t a, uint32_t b, uint32_t c, uint32_t d)
  {
-#if __BYTE_ORDER == __ORDER_BIG_ENDIAN__
-    return huint32_t{(a) | (b << 8) | (c << 16) | (d << 24)};
-#else
    return huint32_t{(d) | (c << 8) | (b << 16) | (a << 24)};
-#endif
  }

  IPRange
--- a/llarp/util/endian.hpp
+++ b/llarp/util/endian.hpp
@ -94,6 +94,16 @@ be64toh(uint64_t big64);

 #endif

+#if !defined(__LITTLE_ENDIAN__) && defined(__BYTE_ORDER) \
+    && defined(__LITTLE_ENDIAN) && __BYTE_ORDER == __LITTLE_ENDIAN
+#define __LITTLE_ENDIAN__
+#elif !defined(__BIG_ENDIAN__) && defined(__BYTE_ORDER) \
+    && defined(__BIG_ENDIAN) && __BYTE_ORDER == __BIG_ENDIAN
+#define __BIG_ENDIAN__
+#elif !defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+#error "Error: don't know which endian this is"
+#endif
+
 inline uint16_t
 buf16toh(const void *buf)
 {
@ -193,7 +203,7 @@ htole64buf(void *buf, uint64_t big64)
 inline absl::uint128
 ntoh128(absl::uint128 i)
 {
-#if __BYTE_ORDER == __BIG_ENDIAN
+#ifdef __BIG_ENDIAN__
  return i;
 #else
  const auto loSwapped = htobe64(absl::Uint128Low64(i));
--- a/test/router/test_llarp_router_version.cpp
+++ b/test/router/test_llarp_router_version.cpp
@ -52,7 +52,7 @@ TEST_F(TestRouterVersion, TestBEncode)
 {
  llarp::RouterVersion v1235( {1, 2, 3}, 5);

-  std::array< byte_t, 128 > tmp;
+  std::array< byte_t, 128 > tmp{};
  llarp_buffer_t buf(tmp);

  EXPECT_TRUE(v1235.BEncode(&buf));
--- a/test/test_llarp_router_contact.cpp
+++ b/test/test_llarp_router_contact.cpp
@ -37,7 +37,7 @@ TEST_F(RCTest, TestSignVerify)

  rc.enckey = encr.toPublic();
  rc.pubkey = sign.toPublic();
-  rc.exits.emplace_back(rc.pubkey, nuint32_t{50000});
+  rc.exits.emplace_back(rc.pubkey, nuint32_t{0x08080808});
  ASSERT_TRUE(rc.netID == netid);
  ASSERT_TRUE(rc.netID == NetID::DefaultValue());