From 53dfc570eb6742014bc4f34eaa63ea2b7b699496 Mon Sep 17 00:00:00 2001 From: Jack Ivanov <17044561+jackivanov@users.noreply.github.com> Date: Mon, 13 Jan 2020 17:20:40 +0100 Subject: [PATCH] Github Actions (#1681) --- .github/workflows/main.yml | 160 +++++++++++++++++++++++++++++++++++++ .travis.yml | 129 ------------------------------ server.yml | 4 +- tests/cloud-init.sh | 10 +-- tests/local-deploy.sh | 6 +- tests/pre-deploy.sh | 3 + tests/ssh-tunnel.sh | 4 +- tests/update-users.sh | 4 +- 8 files changed, 177 insertions(+), 143 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..3b9d885 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,160 @@ +name: Main + +on: [push, pull_request] + +jobs: + lint: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + sudo apt update -y + python -m pip install --upgrade pip + pip install -r requirements.txt + sudo snap install shellcheck + pip install ansible-lint + + - name: Checks and linters + run: | + /snap/bin/shellcheck algo install.sh + ansible-playbook main.yml --syntax-check + ansible-lint -v *.yml roles/{local,cloud-*}/*/*.yml + + scripted-deploy: + runs-on: ubuntu-16.04 + strategy: + matrix: + UBUNTU_VERSION: ["18.04", "19.04", "19.10"] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + sudo apt update -y + sudo add-apt-repository -yu ppa:wireguard/wireguard + sudo apt install -y \ + python3-pip \ + lxd \ + expect-dev \ + debootstrap \ + tree \ + bridge-utils \ + dnsutils \ + build-essential \ + libssl-dev \ + libffi-dev \ + python3-dev \ + linux-headers-$(uname -r) \ + wireguard \ + libxml2-utils \ + crudini \ + fping \ + strongswan \ + libstrongswan-standard-plugins \ + resolvconf + + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + - name: Provision + env: + DEPLOY: cloud-init + UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }} + run: | + ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' + # sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 80/" config.cfg + sudo -E ./tests/pre-deploy.sh + + - name: Deployment + run: | + until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done + ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & ) + until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do + echo 'Cloud init is not finished. Sleep for 30 seconds'; + sleep 30; + done + sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml + sudo lxc exec algo -- tar zcf /root/algo-configs.tar -C /opt/algo/configs/ . + sudo lxc file pull algo/root/algo-configs.tar ./ + sudo tar -C ./configs -zxf algo-configs.tar + + - name: Tests + run: | + set -x + sudo -E bash -x ./tests/wireguard-client.sh + sudo env "PATH=$PATH" ./tests/ipsec-client.sh + sudo ./tests/ssh-tunnel.sh + + local-deploy: + runs-on: ubuntu-16.04 + strategy: + matrix: + UBUNTU_VERSION: ["18.04", "19.04", "19.10"] + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-python@v1 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + set -x + sudo add-apt-repository -yu ppa:wireguard/wireguard + sudo add-apt-repository -yu ppa:ubuntu-lxc/stable + sudo apt update -y + sudo apt install -y \ + python3-pip \ + lxd \ + expect-dev \ + debootstrap \ + tree \ + bridge-utils \ + dnsutils \ + build-essential \ + libssl-dev \ + libffi-dev \ + python3-dev \ + linux-headers-$(uname -r) \ + wireguard \ + libxml2-utils \ + crudini \ + fping \ + strongswan \ + libstrongswan-standard-plugins \ + resolvconf + + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements.txt + + - name: Provision + env: + DEPLOY: docker + UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }} + run: | + ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' + sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 80/" config.cfg + sudo -E ./tests/pre-deploy.sh + + - name: Deployment + env: + DEPLOY: docker + UBUNTU_VERSION: ${{ matrix.UBUNTU_VERSION }} + run: | + docker build -t local/algo . + ./tests/local-deploy.sh + ./tests/update-users.sh + + - name: Tests + run: | + set -x + sudo bash -x ./tests/wireguard-client.sh + sudo env "PATH=$PATH" bash -x ./tests/ipsec-client.sh + sudo bash -x ./tests/ssh-tunnel.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 462fb6e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,129 +0,0 @@ ---- -language: python -python: "3.7" -dist: xenial - -services: - - docker - -addons: - apt: - sources: &default_sources - - sourceline: 'ppa:ubuntu-lxc/stable' - - sourceline: 'ppa:wireguard/wireguard' - packages: &default_packages - - python3-pip - - lxd - - expect-dev - - debootstrap - - tree - - bridge-utils - - dnsutils - - build-essential - - libssl-dev - - libffi-dev - - python3-dev - - linux-headers-$(uname -r) - - wireguard - - libxml2-utils - - crudini - - fping - - strongswan - - libstrongswan-standard-plugins - -cache: - directories: - - $HOME/lxc/ - pip: true - -before_cache: - - mkdir $HOME/lxc - - sudo tar cf $HOME/lxc/cache.tar /var/lib/lxd/images/ - - sudo chown $USER. $HOME/lxc/cache.tar - -custom_scripts: - provisioning: &provisioning - - ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' - - sudo ./tests/pre-deploy.sh - - 'sed -i "s/^reduce_mtu:\s0$/reduce_mtu: 20/" config.cfg' - tests: &tests - - sudo ./tests/wireguard-client.sh - - sudo env "PATH=$PATH" ./tests/ipsec-client.sh - - sudo ./tests/ssh-tunnel.sh - -stages: - - &tests-and-linters - stage: Tests - name: code checks and linters - addons: - apt: - packages: - - shellcheck - script: - - pip install ansible-lint - - shellcheck algo install.sh - - ansible-playbook main.yml --syntax-check - - ansible-lint -v *.yml roles/{local,cloud-*}/*/*.yml - - - &deploy-local - stage: Deploy - name: local deployment from docker - addons: - apt: - sources: *default_sources - packages: *default_packages - before_install: *provisioning - before_script: - - docker build -t travis/algo . - - ./tests/local-deploy.sh - - ./tests/update-users.sh - script: *tests - - - &deploy-cloudinit - stage: Deploy - name: cloud-init deployment - addons: - apt: - sources: *default_sources - packages: *default_packages - env: DEPLOY=cloud-init - before_install: *provisioning - before_script: - - until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done - - ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & ) - - | - until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do - echo 'Cloud init is not finished. Sleep for 30 seconds'; - sleep 30; - done - - sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml - - sudo lxc exec algo -- tar zcf /root/algo-configs.tar -C /opt/algo/configs/ . - - sudo lxc file pull algo/root/algo-configs.tar ./ - - sudo tar -C ./configs -zxf algo-configs.tar - script: *tests - -matrix: - fast_finish: true - include: - - <<: *tests-and-linters - - <<: *deploy-local - name: 'Ubuntu 18.04: local deployment from docker' - env: DEPLOY=docker UBUNTU_VERSION=18.04 - - <<: *deploy-local - name: 'Ubuntu 19.04: local deployment from docker' - env: DEPLOY=docker UBUNTU_VERSION=19.04 - - <<: *deploy-local - name: 'Ubuntu 19.10: local deployment from docker' - env: DEPLOY=docker UBUNTU_VERSION=19.10 - - <<: *deploy-cloudinit - name: 'Ubuntu 18.04: cloud-init deployment' - env: DEPLOY=cloud-init UBUNTU_VERSION=18.04 - - <<: *deploy-cloudinit - name: 'Ubuntu 19.04: cloud-init deployment' - env: DEPLOY=cloud-init UBUNTU_VERSION=19.04 - - <<: *deploy-cloudinit - name: 'Ubuntu 19.10: cloud-init deployment' - env: DEPLOY=cloud-init UBUNTU_VERSION=19.10 - -notifications: - email: false diff --git a/server.yml b/server.yml index 12d6175..782d713 100644 --- a/server.yml +++ b/server.yml @@ -87,8 +87,8 @@ ipsec_enabled: {{ ipsec_enabled }} wireguard_enabled: {{ wireguard_enabled }} {% if tests|default(false)|bool %} - ca_password: {{ CA_password }} - p12_password: {{ p12_export_password }} + ca_password: '{{ CA_password }}' + p12_password: '{{ p12_export_password }}' {% endif %} become: false delegate_to: localhost diff --git a/tests/cloud-init.sh b/tests/cloud-init.sh index f8de251..256fbe1 100755 --- a/tests/cloud-init.sh +++ b/tests/cloud-init.sh @@ -9,9 +9,9 @@ export DNS_ADBLOCKING=true export SSH_TUNNELING=true export ENDPOINT=10.0.8.100 export USERS=desktop,user1,user2 -export EXTRA_VARS='install_headers=false tests=true apparmor_enabled=false local_service_ip=172.16.0.1' -export ANSIBLE_EXTRA_ARGS='--skip-tags apparmor' -export REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG:-trailofbits/algo}} -export REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH:-master}} +export EXTRA_VARS='install_headers=false tests=true local_service_ip=172.16.0.1' +export ANSIBLE_EXTRA_ARGS='' +export REPO_SLUG=${REPOSITORY:-trailofbits/algo} +export REPO_BRANCH=${BRANCH:-master} -curl -s https://raw.githubusercontent.com/${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG}}/${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}/install.sh | sudo -E bash -x" +curl -s https://raw.githubusercontent.com/${REPOSITORY:-trailofbits/algo}/${BRANCH:-master}/install.sh | sudo -E bash -x" diff --git a/tests/local-deploy.sh b/tests/local-deploy.sh index 71458a1..e3acc6c 100755 --- a/tests/local-deploy.sh +++ b/tests/local-deploy.sh @@ -2,11 +2,11 @@ set -ex -DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 apparmor_enabled=false ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1" +DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1" if [ "${DEPLOY}" == "docker" ] then - docker run -it -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" travis/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags apparmor" + docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags debug" else - ansible-playbook main.yml -e "${DEPLOY_ARGS}" --skip-tags apparmor + ansible-playbook main.yml -e "${DEPLOY_ARGS}" fi diff --git a/tests/pre-deploy.sh b/tests/pre-deploy.sh index e56922d..dc51008 100755 --- a/tests/pre-deploy.sh +++ b/tests/pre-deploy.sh @@ -8,6 +8,9 @@ tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache." cp -f tests/lxd-bridge /etc/default/lxd-bridge cp -f tests/algo.conf /etc/default/algo.conf +export REPOSITORY=${GITHUB_REPOSITORY} +export BRANCH=${GITHUB_REF#refs/heads/} + if [[ "$DEPLOY" == "cloud-init" ]]; then bash tests/cloud-init.sh | lxc profile set default user.user-data - else diff --git a/tests/ssh-tunnel.sh b/tests/ssh-tunnel.sh index fba8039..0b491b9 100755 --- a/tests/ssh-tunnel.sh +++ b/tests/ssh-tunnel.sh @@ -2,7 +2,7 @@ set -euxo pipefail -PASS=$(grep ^p12_password: configs/10.0.8.100/.config.yml | awk '{print $2}') +PASS=$(grep ^p12_password: configs/10.0.8.100/.config.yml | awk '{print $2}' | cut -f2 -d\') ssh-keygen -p -P ${PASS} -N '' -f configs/10.0.8.100/ssh-tunnel/desktop.pem @@ -10,6 +10,6 @@ ssh -o StrictHostKeyChecking=no -D 127.0.0.1:1080 -f -q -C -N desktop@10.0.8.100 git config --global http.proxy 'socks5://127.0.0.1:1080' -git clone -vv https://github.com/trailofbits/algo /tmp/ssh-tunnel-check +for i in {1..10}; do git clone -vv https://github.com/trailofbits/algo /tmp/ssh-tunnel-check && break || sleep 1; done echo "SSH tunneling tests passed" diff --git a/tests/update-users.sh b/tests/update-users.sh index 6f561cd..baa8c82 100755 --- a/tests/update-users.sh +++ b/tests/update-users.sh @@ -6,7 +6,7 @@ USER_ARGS="{ 'server': '10.0.8.100', 'users': ['desktop', 'user1', 'user2'], 'lo if [ "${DEPLOY}" == "docker" ] then - docker run -it -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "USER_ARGS=${USER_ARGS}" travis/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook users.yml -e \"${USER_ARGS}\" -t update-users" + docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "USER_ARGS=${USER_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook users.yml -e \"${USER_ARGS}\" -t update-users --skip-tags debug -vvvvv" else ansible-playbook users.yml -e "${USER_ARGS}" -t update-users fi @@ -23,7 +23,7 @@ if sudo openssl crl -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/c exit 1 fi -if sudo openssl x509 -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/certs/user1.crt | grep CN=user1 +if sudo openssl x509 -inform pem -noout -text -in configs/10.0.8.100/ipsec/.pki/certs/user1.crt | grep -E "CN(=|\s+=\s+)user1" then echo "The new user exists" else