From 4ea6c4ad5d1cac14d63f758f8fa6909b77a0b758 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 12 Jun 2020 15:29:50 +0200 Subject: [PATCH] Various build optimizations --- docker/dev/Dockerfile | 12 ++--- docker/dev/docker_update.sh | 4 ++ docker/dev/test_deploy.sh | 4 ++ docker/prod/Dockerfile | 58 +++++++++++++++++++++ docker/{dev => prod}/Dockerfile.aarch64 | 5 +- docker/{dev => prod}/Dockerfile.armv7hf | 6 ++- docker/{dev => prod}/Dockerfile.libc | 4 +- docker/{dev => prod}/deploy.sh | 6 ++- docs/src/contributing_docker_development.md | 15 +++++- docs/src/contributing_local_development.md | 6 +++ server/Cargo.lock | 2 - server/Cargo.toml | 11 ++-- 12 files changed, 111 insertions(+), 22 deletions(-) create mode 100644 docker/prod/Dockerfile rename docker/{dev => prod}/Dockerfile.aarch64 (96%) rename docker/{dev => prod}/Dockerfile.armv7hf (94%) rename docker/{dev => prod}/Dockerfile.libc (96%) rename docker/{dev => prod}/deploy.sh (96%) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index d9ffc2f38..fbec25367 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -21,17 +21,13 @@ COPY server/Cargo.toml server/Cargo.lock ./ RUN sudo chown -R rust:rust . RUN mkdir -p ./src/bin \ && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs -RUN cargo build --release +RUN cargo build RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server* COPY server/src ./src/ COPY server/migrations ./migrations/ -# Build for release -RUN cargo build --frozen --release - -# Get diesel-cli on there just in case -# RUN cargo install diesel_cli --no-default-features --features postgres - +# Build for debug +RUN cargo build FROM ekidd/rust-musl-builder:1.42.0-openssl11 as docs WORKDIR /app @@ -47,7 +43,7 @@ RUN apk add libpq # Copy resources COPY server/config/defaults.hjson /config/defaults.hjson -COPY --from=rust /app/server/target/x86_64-unknown-linux-musl/release/lemmy_server /app/lemmy +COPY --from=rust /app/server/target/x86_64-unknown-linux-musl/debug/lemmy_server /app/lemmy COPY --from=docs /app/docs/book/ /app/dist/documentation/ COPY --from=node /app/ui/dist /app/dist diff --git a/docker/dev/docker_update.sh b/docker/dev/docker_update.sh index 9d0f45429..21e92ba88 100755 --- a/docker/dev/docker_update.sh +++ b/docker/dev/docker_update.sh @@ -1,2 +1,6 @@ #!/bin/sh +set -e + +export COMPOSE_DOCKER_CLI_BUILD=1 +export DOCKER_BUILDKIT=1 docker-compose up -d --no-deps --build diff --git a/docker/dev/test_deploy.sh b/docker/dev/test_deploy.sh index 1ae91f654..ce85f60c6 100755 --- a/docker/dev/test_deploy.sh +++ b/docker/dev/test_deploy.sh @@ -1,4 +1,8 @@ #!/bin/sh +set -e + +export COMPOSE_DOCKER_CLI_BUILD=1 +export DOCKER_BUILDKIT=1 # Rebuilding dev docker docker-compose build diff --git a/docker/prod/Dockerfile b/docker/prod/Dockerfile new file mode 100644 index 000000000..b9eec4565 --- /dev/null +++ b/docker/prod/Dockerfile @@ -0,0 +1,58 @@ +FROM node:10-jessie as node + +WORKDIR /app/ui + +# Cache deps +COPY ui/package.json ui/yarn.lock ./ +RUN yarn install --pure-lockfile + +# Build +COPY ui /app/ui +RUN yarn build + +FROM ekidd/rust-musl-builder:1.42.0-openssl11 as rust + +# Cache deps +WORKDIR /app +RUN sudo chown -R rust:rust . +RUN USER=root cargo new server +WORKDIR /app/server +COPY server/Cargo.toml server/Cargo.lock ./ +RUN sudo chown -R rust:rust . +RUN mkdir -p ./src/bin \ + && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs +RUN cargo build --release +RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server* +COPY server/src ./src/ +COPY server/migrations ./migrations/ + +# Build for release +RUN cargo build --frozen --release + +# reduce binary size +RUN strip /app/server/target/x86_64-unknown-linux-musl/release/lemmy_server + +FROM ekidd/rust-musl-builder:1.42.0-openssl11 as docs +WORKDIR /app +COPY docs ./docs +RUN sudo chown -R rust:rust . +RUN mdbook build docs/ + + +FROM alpine:3.10 + +# Install libpq for postgres +RUN apk add libpq + +# Copy resources +COPY server/config/defaults.hjson /config/defaults.hjson +COPY --from=rust /app/server/target/x86_64-unknown-linux-musl/release/lemmy_server /app/lemmy +COPY --from=docs /app/docs/book/ /app/dist/documentation/ +COPY --from=node /app/ui/dist /app/dist + +RUN addgroup -g 1000 lemmy +RUN adduser -D -s /bin/sh -u 1000 -G lemmy lemmy +RUN chown lemmy:lemmy /app/lemmy +USER lemmy +EXPOSE 8536 +CMD ["/app/lemmy"] diff --git a/docker/dev/Dockerfile.aarch64 b/docker/prod/Dockerfile.aarch64 similarity index 96% rename from docker/dev/Dockerfile.aarch64 rename to docker/prod/Dockerfile.aarch64 index 9636a5907..7a1c833fc 100644 --- a/docker/dev/Dockerfile.aarch64 +++ b/docker/prod/Dockerfile.aarch64 @@ -40,10 +40,11 @@ COPY server/src ./src/ COPY server/migrations ./migrations/ RUN rm -f ./target/release/deps/lemmy_server* ; rm -f ./target/debug/deps/lemmy_server* - # build for release RUN cargo build --frozen --release -# RUN cargo build --frozen + +# reduce binary size +RUN strip /app/server/target/release/lemmy_server # Get diesel-cli on there just in case # RUN cargo install diesel_cli --no-default-features --features postgres diff --git a/docker/dev/Dockerfile.armv7hf b/docker/prod/Dockerfile.armv7hf similarity index 94% rename from docker/dev/Dockerfile.armv7hf rename to docker/prod/Dockerfile.armv7hf index c2c9084c1..efd937a3c 100644 --- a/docker/dev/Dockerfile.armv7hf +++ b/docker/prod/Dockerfile.armv7hf @@ -43,8 +43,10 @@ RUN rm -f ./target/release/deps/lemmy_server* ; rm -f ./target/debug/deps/lemmy_ # build for release -#RUN cargo build --frozen --release -RUN cargo build --frozen +RUN cargo build --frozen --release + +# reduce binary size +RUN strip /app/server/target/debug/lemmy_server # Get diesel-cli on there just in case # RUN cargo install diesel_cli --no-default-features --features postgres diff --git a/docker/dev/Dockerfile.libc b/docker/prod/Dockerfile.libc similarity index 96% rename from docker/dev/Dockerfile.libc rename to docker/prod/Dockerfile.libc index 6348342fd..9c7502827 100644 --- a/docker/dev/Dockerfile.libc +++ b/docker/prod/Dockerfile.libc @@ -43,7 +43,9 @@ COPY server/migrations ./migrations/ # workaround for https://github.com/rust-lang/rust/issues/62896 #RUN RUSTFLAGS='-Ccodegen-units=1' cargo build --release RUN cargo build --release --frozen -#RUN cargo build --frozen + +# reduce binary size +RUN strip /app/server/target/release/lemmy_server # Get diesel-cli on there just in case # RUN cargo install diesel_cli --no-default-features --features postgres diff --git a/docker/dev/deploy.sh b/docker/prod/deploy.sh similarity index 96% rename from docker/dev/deploy.sh rename to docker/prod/deploy.sh index 09b21b0b7..0ad47bcd8 100755 --- a/docker/dev/deploy.sh +++ b/docker/prod/deploy.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -e git checkout master # Import translations @@ -20,7 +21,7 @@ git add "server/src/version.rs" echo $new_tag > "ansible/VERSION" git add "ansible/VERSION" -cd docker/dev || exit +cd docker/prod || exit # Changing the docker-compose prod sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../prod/docker-compose.yml @@ -32,6 +33,9 @@ git add ../../ansible/templates/docker-compose.yml git commit -m"Version $new_tag" git tag $new_tag +export COMPOSE_DOCKER_CLI_BUILD=1 +export DOCKER_BUILDKIT=1 + # Rebuilding docker docker-compose build docker tag dev_lemmy:latest dessalines/lemmy:x64-$new_tag diff --git a/docs/src/contributing_docker_development.md b/docs/src/contributing_docker_development.md index 092398219..afa051072 100644 --- a/docs/src/contributing_docker_development.md +++ b/docs/src/contributing_docker_development.md @@ -3,11 +3,22 @@ ## Running ```bash +sudo apt install git docker-compose git clone https://github.com/LemmyNet/lemmy cd lemmy/docker/dev -./docker_update.sh # This builds and runs it, updating for your changes +sudo docker-compose up --no-deps --build ``` and go to http://localhost:8536. -Note that compile times when changing `Cargo.toml` are relatively long with Docker, because builds can't be incrementally cached. If this is a problem for you, you should use [Local Development](contributing_local_development.md). +To speed up the Docker compile, add the following to `/etc/docker/daemon.json` and restart Docker. +``` +{ + "features": { + "buildkit": true + } +} +``` + +If the build is still too slow, you will have to use a +[local build](contributing_local_development.md) instead. diff --git a/docs/src/contributing_local_development.md b/docs/src/contributing_local_development.md index f801caf32..066386f50 100644 --- a/docs/src/contributing_local_development.md +++ b/docs/src/contributing_local_development.md @@ -56,6 +56,12 @@ Then open [localhost:4444](http://localhost:4444) in your browser. It will auto- any frontend files. For backend coding, you will have to rerun `cargo run`. You can use `cargo check` as a faster way to find compilation errors. +To speed up incremental builds, you can add the following to `~/.cargo/config`: +``` +[target.x86_64-unknown-linux-gnu] +rustflags = ["-Clink-arg=-fuse-ld=lld"] +``` + Note that this setup doesn't include image uploads or link previews (provided by pict-rs and iframely respectively). If you want to test those, you should use the [Docker development](contributing_docker_development.md). diff --git a/server/Cargo.lock b/server/Cargo.lock index 6d6eb944a..f91b665b8 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -413,7 +413,6 @@ name = "attohttpc" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "flate2 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2201,7 +2200,6 @@ name = "serde_json" version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/server/Cargo.toml b/server/Cargo.toml index 44ba15235..43933e531 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -4,16 +4,19 @@ version = "0.0.1" authors = ["Dessalines "] edition = "2018" +[profile.release] +lto = true + [dependencies] diesel = { version = "1.4.2", features = ["postgres","chrono", "r2d2", "64-column-tables"] } diesel_migrations = "1.4.0" dotenv = "0.15.0" bcrypt = "0.7.0" activitypub = "0.2.0" -chrono = { version = "0.4.7", features = ["serde"] } +chrono = "0.4.7" failure = "0.1.8" -serde_json = { version = "1.0.52", features = ["preserve_order"]} -serde = { version = "1.0.105", features = ["derive"] } +serde_json = "1.0.52" +serde = "1.0.105" actix = "0.9.0" actix-web = "2.0.0" actix-files = "0.2.1" @@ -34,7 +37,7 @@ rss = "1.9.0" htmlescape = "0.3.1" config = {version = "0.10.1", default-features = false, features = ["hjson"] } percent-encoding = "2.1.0" -attohttpc = { version = "0.14.0", default-features = false, features = ["tls-rustls", "compress"] } +attohttpc = { version = "0.14.0", default-features = false, features = ["tls-rustls"] } comrak = "0.7" tokio = "0.2.20" futures = "0.3.4"