diff --git a/.travis.yml b/.travis.yml index 47a58a9..7a2c67d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,11 @@ --- language: python python: "2.7" -sudo: required -dist: trusty +dist: xenial services: - docker -matrix: - fast_finish: true - addons: apt: sources: @@ -41,35 +37,52 @@ before_cache: - sudo tar cf $HOME/lxc/cache.tar /var/lib/lxd/images/ - sudo chown $USER. $HOME/lxc/cache.tar -env: - - LXC_NAME=docker LXC_DISTRO=ubuntu LXC_RELEASE=18.04 - -before_install: - - test "${LXC_NAME}" != "docker" && sudo modprobe wireguard || docker build -t travis/algo . +matrix: + fast_finish: true + include: + - stage: Test + name: local deployment from docker + script: + - docker build -t travis/algo . + - sudo tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache." + - sudo cp -f tests/lxd-bridge /etc/default/lxd-bridge + - ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' + - echo -e "#cloud-config\nssh_authorized_keys:\n - $(cat ~/.ssh/id_rsa.pub)" | sudo lxc profile set default user.user-data - + - sudo service lxd restart + - sudo lxc launch ubuntu:18.04 algo + - until host algo.lxd 10.0.8.1 -t A; do sleep 3; done + - export LXC_IP="$(dig algo.lxd @10.0.8.1 +short)" + - pip install -r requirements.txt + - pip install ansible-lint + - gem install awesome_bot + - ansible-playbook --version + - tree . -L 2 + - ansible-playbook main.yml --syntax-check + - ./tests/local-deploy.sh + - ./tests/update-users.sh -install: - - sudo tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache." - - ssh-keygen -f ~/.ssh/id_rsa -t rsa -N '' - - chmod 0644 ~/.ssh/config - - echo -e "#cloud-config\nssh_authorized_keys:\n - $(cat ~/.ssh/id_rsa.pub)" | sudo lxc profile set default user.user-data - - - sudo cp -f tests/lxd-bridge /etc/default/lxd-bridge - - sudo service lxd restart - - sudo lxc launch ${LXC_DISTRO}:${LXC_RELEASE} ${LXC_NAME} - - until host ${LXC_NAME}.lxd 10.0.8.1 -t A; do sleep 3; done - - export LXC_IP="$(dig ${LXC_NAME}.lxd @10.0.8.1 +short)" - - pip install -r requirements.txt - - pip install ansible-lint - - gem install awesome_bot - - ansible-playbook --version - - tree . -L 2 + - stage: Test + name: cloud-init deployment + script: + - sudo tar xf $HOME/lxc/cache.tar -C / || echo "Didn't extract cache." + - sudo cp -f tests/lxd-bridge /etc/default/lxd-bridge + - sudo service lxd restart + - bash tests/cloud-init.sh | sudo lxc profile set default user.user-data - + - sudo lxc profile show default + - sudo lxc launch ubuntu:18.04 algo + - until sudo lxc exec algo -- test -f /var/log/cloud-init-output.log; do echo 'Log file not found, Sleep for 3 seconds'; sleep 3; done + - ( sudo lxc exec algo -- tail -f /var/log/cloud-init-output.log & ) + - | + until sudo lxc exec algo -- test -f /var/lib/cloud/data/result.json; do + echo 'Cloud init is not finished. Sleep for 30 seconds'; + sleep 30; + done + - sudo lxc exec algo -- test -f /opt/algo/configs/localhost/.config.yml -script: +# script: # - awesome_bot --allow-dupe --skip-save-results *.md docs/*.md --white-list paypal.com,do.co,microsoft.com,https://github.com/trailofbits/algo/archive/master.zip,https://github.com/trailofbits/algo/issues/new # - shellcheck algo # - ansible-lint main.yml users.yml deploy_client.yml - - ansible-playbook main.yml --syntax-check - - ./tests/local-deploy.sh - - ./tests/update-users.sh notifications: email: false diff --git a/docs/deploy-from-script-or-cloud-init-to-localhost.md b/docs/deploy-from-script-or-cloud-init-to-localhost.md new file mode 100644 index 0000000..57e3ed6 --- /dev/null +++ b/docs/deploy-from-script-or-cloud-init-to-localhost.md @@ -0,0 +1,58 @@ +# Deploy from script or cloud-init + +You can use `install.sh` to prepare the environment and deploy AlgoVPN on the local Ubuntu server in one shot using cloud-init or run the script directly on the server. The script doesn't configure any parameters in your cloud, so it's on your own to configure related [firewall rules](faq.md#what-inbound-ports-are-used), a floating ip address and other resources you may need. + +## Cloud init deployment + +You can copy-paste the snippet below to the user data (cloud-init or startup script) field when creating a new server. For now it is only possible for [DigitalOcean](https://www.digitalocean.com/docs/droplets/resources/metadata/), Amazon [EC2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html) and [Lightsail](https://lightsail.aws.amazon.com/ls/docs/en/articles/lightsail-how-to-configure-server-additional-data-shell-script), [Google Cloud](https://cloud.google.com/compute/docs/startupscript) and [Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init). + +``` +#!/bin/bash +curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x +``` +The command will prepare the environment and install AlgoVPN with default parameters. If you want to modify the behaviour you may define additional variables. + +## Variables + +`METHOD` - which method of the deployment to use. Possible values are local and cloud. Default: cloud. The cloud method is intended to use in cloud-init deployments only. If you are not using cloud-init to deploy the server you have to use the local method +`ONDEMAND_CELLULAR` - "Connect On Demand" when connected to cellular networks. Bollean. Default: false +`ONDEMAND_WIFI` - "Connect On Demand" when connected to Wi-Fi. Default: false +`ONDEMAND_WIFI_EXCLUDE` - List the names of any trusted Wi-Fi networks where macOS/iOS IPsec clients should not use "Connect On Demand". Comma-separated list. +`WINDOWS` - To support Windows 10 or Linux Desktop clients. Default: false +`STORE_CAKEY` - To retain the CA key. (required to add users in the future, but less secure). Default: false +`LOCAL_DNS` - To install an ad blocking DNS resolver. Default: false +`SSH_TUNNELING` - Enable SSH tunneling for each user. Default: false +`ENDPOINT` - The public IP address or domain name of your server: (IMPORTANT! This is used to verify the certificate). It will be gathered automatically for DigitalOcean, AWS, GCE or Azure if the `METHOD` is cloud. Otherwise you need to define this variable according to your public IP address. +`USERS` - list of VPN users. Comma-separated list. +`REPO_SLUG` - Owner and repository that used to get the installation scripts from. Default: trailofbits/algo +`REPO_BRANCH` - Branch for `REPO_SLUG`. Default: master +`EXTRA_VARS` - Additional extra variables. +`ANSIBLE_EXTRA_ARGS` - Any available ansible parameters. ie: `--skip-tags apparmor` + +## Examples + +##### How to customise a cloud-init deployment by variables + +``` +#!/bin/bash +export ONDEMAND_CELLULAR=true +export WINDOWS=true +export SSH_TUNNELING=true +curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x +``` + +##### How to deploy locally without using cloud-init + +``` +export METHOD=local +export ONDEMAND_CELLULAR=true +curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x +``` + +##### How to deploy a server using arguments + +The arguments order as per [variables](#variables) above + +``` +curl -s https://raw.githubusercontent.com/trailofbits/algo/master/install.sh | sudo bash -x -s local true false _null true true true true myvpnserver.com +``` diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..248f378 --- /dev/null +++ b/install.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env sh + +set -ex + +METHOD="${1:-${METHOD:-cloud}}" +ONDEMAND_CELLULAR="${2:-${ONDEMAND_CELLULAR:-false}}" +ONDEMAND_WIFI="${3:-${ONDEMAND_WIFI:-false}}" +ONDEMAND_WIFI_EXCLUDE="${4:-${ONDEMAND_WIFI_EXCLUDE:-_null}}" +WINDOWS="${5:-${WINDOWS:-false}}" +STORE_CAKEY="${6:-${STORE_CAKEY:-false}}" +LOCAL_DNS="${7:-${LOCAL_DNS:-false}}" +SSH_TUNNELING="${8:-${SSH_TUNNELING:-false}}" +ENDPOINT="${9:-${ENDPOINT:-localhost}}" +USERS="${10:-${USERS:-user1}}" +REPO_SLUG="${11:-${REPO_SLUG:-trailofbits/algo}}" +REPO_BRANCH="${12:-${REPO_BRANCH:-master}}" +EXTRA_VARS="${13:-${EXTRA_VARS:-placeholder=null}}" +ANSIBLE_EXTRA_ARGS="${14:-${ANSIBLE_EXTRA_ARGS}}" + +cd /opt/ + +installRequirements() { + apt-get update + apt-get install \ + software-properties-common \ + git \ + build-essential \ + libssl-dev \ + libffi-dev \ + python-dev \ + python-pip \ + python-setuptools \ + python-virtualenv \ + bind9-host \ + jq -y +} + +getAlgo() { + [ ! -d "algo" ] && git clone https://github.com/${REPO_SLUG} algo + cd algo + + git checkout ${REPO_BRANCH} + + python -m virtualenv --python=`which python2` .venv + . .venv/bin/activate + python -m pip install -U pip virtualenv + python -m pip install -r requirements.txt +} + +publicIpFromInterface() { + echo "Couldn't find a valid ipv4 address, using the first IP found on the interfaces as the endpoint." + DEFAULT_INTERFACE="$(ip -4 route list match default | grep -Eo "dev .*" | awk '{print $2}')" + ENDPOINT=$(ip -4 addr sh dev eth0 | grep -w inet | head -n1 | awk '{print $2}' | grep -oE '\b([0-9]{1,3}\.){3}[0-9]{1,3}\b') + export ENDPOINT=$ENDPOINT + echo "Using ${ENDPOINT} as the endpoint" +} + +publicIpFromMetadata() { + if curl -s http://169.254.169.254/metadata/v1/vendor-data | grep DigitalOcean >/dev/null; then + PROVIDER="digitalocean" + ENDPOINT="$(curl -s http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address)" + elif test "$(curl -s http://169.254.169.254/latest/meta-data/services/domain)" = "amazonaws.com"; then + PROVIDER="amazon" + ENDPOINT="$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)" + elif host -t A -W 10 metadata.google.internal 127.0.0.53 >/dev/null; then + PROVIDER="gce" + ENDPOINT="$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")" + elif test "$(curl -s -H Metadata:true 'http://169.254.169.254/metadata/instance/compute/publisher/?api-version=2017-04-02&format=text')" = "Canonical"; then + PROVIDER="azure" + ENDPOINT="$(curl -H Metadata:true 'http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2017-04-02&format=text')" + fi + + if echo ${ENDPOINT} | grep -oE "\b([0-9]{1,3}\.){3}[0-9]{1,3}\b"; then + export ENDPOINT=$ENDPOINT + echo "Using ${ENDPOINT} as the endpoint" + else + publicIpFromInterface + fi +} + +deployAlgo() { + getAlgo + + cd /opt/algo + . .venv/bin/activate + + export HOME=/root + export ANSIBLE_LOCAL_TEMP=/root/.ansible/tmp + export ANSIBLE_REMOTE_TEMP=/root/.ansible/tmp + + ansible-playbook main.yml \ + -e provider=local \ + -e ondemand_cellular=${ONDEMAND_CELLULAR} \ + -e ondemand_wifi=${ONDEMAND_WIFI} \ + -e ondemand_wifi_exclude=${ONDEMAND_WIFI_EXCLUDE} \ + -e windows=${WINDOWS} \ + -e store_cakey=${STORE_CAKEY} \ + -e local_dns=${LOCAL_DNS} \ + -e ssh_tunneling=${SSH_TUNNELING} \ + -e endpoint=$ENDPOINT \ + -e users=$(echo "$USERS" | jq -Rc 'split(",")') \ + -e server=localhost \ + -e ssh_user=root \ + -e "${EXTRA_VARS}" \ + --skip-tags debug ${ANSIBLE_EXTRA_ARGS} | + tee /var/log/algo.log +} + +if test $METHOD = "cloud"; then + publicIpFromMetadata +fi + +installRequirements + +deployAlgo diff --git a/tests/cloud-init.sh b/tests/cloud-init.sh new file mode 100755 index 0000000..2d95c99 --- /dev/null +++ b/tests/cloud-init.sh @@ -0,0 +1,17 @@ +#!/bin/bash +echo "#!/bin/bash +export METHOD=local +export ONDEMAND_CELLULAR=true +export ONDEMAND_WIFI=true +export ONDEMAND_WIFI_EXCLUDE=test1,test2 +export WINDOWS=true +export STORE_CAKEY=true +export LOCAL_DNS=true +export ENDPOINT=algo.lxc +export USERS=user1,user2 +export EXTRA_VARS='install_headers=false tests=true apparmor_enabled=false' +export ANSIBLE_EXTRA_ARGS='--skip-tags apparmor' +export REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG:-trailofbits/algo}} +export REPO_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH:-master}} + +curl -s https://raw.githubusercontent.com/${TRAVIS_PULL_REQUEST_SLUG:-${TRAVIS_REPO_SLUG}}/${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}/install.sh | sudo -E bash -x"