Refactored docker-compose file.

build_sqlite.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+csv_file="./data/torrents.csv"
+torrent_files_csv="../torrent_files.csv"
+db_file="${TORRENTS_CSV_DB_FILE:-./torrents.db}"
+build_files=false
+
+while getopts ":f" opt; do
+  case $opt in
+    f)
+      build_files=true
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      ;;
+  esac
+done
+
+echo "Creating temporary torrents.db file..."
+
+# Remove double quotes for csv import
+sed 's/\"//g' $csv_file > torrents_removed_quotes.csv
+
+# Sort by seeders desc before insert
+sort --field-separator=';' --key=5 -nr -o torrents_removed_quotes.csv torrents_removed_quotes.csv
+
+touch db_tmp
+
+sqlite3 -batch db_tmp <<"EOF"
+drop table if exists torrents;
+create table torrents(
+  "infohash" TEXT,
+  "name" TEXT,
+  "size_bytes" INTEGER,
+  "created_unix" INTEGER,
+  "seeders" INTEGER,
+  "leechers" INTEGER,
+  "completed" INTEGER,
+  "scraped_date" INTEGER
+);
+.separator ";"
+.import torrents_removed_quotes.csv torrents
+UPDATE torrents SET completed=NULL WHERE completed = '';
+EOF
+rm torrents_removed_quotes.csv
+
+if $build_files ; then
+  # Cache torrent files
+  echo "Building files DB from $torrent_files_csv ..."
+
+  # Remove double quotes for csv import
+  sed 's/\"//g' $torrent_files_csv > torrent_files_removed_quotes.csv
+
+  # Removing those with too many ;
+  awk -F \; 'NF == 4' <torrent_files_removed_quotes.csv > torrent_files_temp_2
+
+  rm torrent_files_removed_quotes.csv
+  mv torrent_files_temp_2 torrent_files_temp
+
+sqlite3 -batch db_tmp<<EOF
+create table files_tmp(
+"infohash" TEXT,
+"index_" INTEGER,
+"path" TEXT,
+"size_bytes" INTEGER
+);
+.separator ";"
+.import torrent_files_temp files_tmp
+
+-- Filling the extra columns
+create table files(
+"infohash" TEXT,
+"index_" INTEGER,
+"path" TEXT,
+"size_bytes" INTEGER,
+"created_unix" INTEGER,
+"seeders" INTEGER,
+"leechers" INTEGER,
+"completed" INTEGER,
+"scraped_date" INTEGER
+);
+insert into files
+select files_tmp.infohash,
+files_tmp.index_,
+files_tmp.path,
+files_tmp.size_bytes,
+torrents.created_unix,
+torrents.seeders,
+torrents.leechers,
+torrents.completed,
+torrents.scraped_date
+from files_tmp
+inner join torrents on files_tmp.infohash = torrents.infohash
+order by torrents.seeders desc, files_tmp.size_bytes desc;
+delete from files where seeders is null;
+drop table files_tmp;
+EOF
+rm torrent_files_temp
+fi
+
+mv db_tmp $db_file
+echo "Done."

docker/dev/Dockerfile

@@ -1,40 +1,71 @@
+ARG RUST_BUILDER_IMAGE=ekidd/rust-musl-builder:stable
+
+# Front end
 FROM node:10-jessie as node
 WORKDIR /app/ui
 
 # Cache deps
-COPY server/ui/package.json server/ui/yarn.lock ./
+COPY ui/package.json ui/yarn.lock ./
 RUN yarn install --pure-lockfile
 
-# Build 
-COPY server/ui /app/ui
+# Build the UI
+COPY ./ui /app/ui
 RUN yarn build
 
-FROM rust:1.42 as rust
+# Build the torrents.db file
+FROM ubuntu:groovy as db_file_builder
+WORKDIR /app
+RUN apt-get -y update
+RUN apt-get -y install sqlite3
+COPY ./data ./data
+COPY ./build_sqlite.sh .
+RUN ./build_sqlite.sh
 
-# Install musl
-RUN apt-get update
-RUN apt-get install musl-tools -y
-RUN rustup target add x86_64-unknown-linux-musl
+# Rust cargo chef build
+FROM $RUST_BUILDER_IMAGE as planner
+WORKDIR /app
+# We only pay the installation cost once, 
+# it will be cached from the second build onwards
+RUN cargo install cargo-chef --version 0.1.6
+COPY ./Cargo.toml ./Cargo.lock .
+COPY ./src src
+RUN ls
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM $RUST_BUILDER_IMAGE as cacher
+
+ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
 
-# Cache deps
 WORKDIR /app
-RUN USER=root cargo new server
-WORKDIR /app/server/service
-COPY server/service/Cargo.toml server/service/Cargo.lock ./
-RUN  mkdir -p ./src/bin \
-  && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs 
-RUN RUSTFLAGS=-Clinker=musl-gcc cargo build --release --target=x86_64-unknown-linux-musl
-RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/torrents-csv-*
-COPY server/service/src ./src/
-
-# build for release
-RUN RUSTFLAGS=-Clinker=musl-gcc cargo build --frozen --release --target=x86_64-unknown-linux-musl
-
-FROM alpine:3.10
+RUN cargo install cargo-chef --version 0.1.6
+COPY --from=planner /app/recipe.json ./recipe.json
+RUN cargo chef cook --target ${CARGO_BUILD_TARGET} --recipe-path recipe.json
+
+FROM $RUST_BUILDER_IMAGE as builder
+
+ARG CARGO_BUILD_TARGET=x86_64-unknown-linux-musl
+ARG RUSTRELEASEDIR="debug"
+
+WORKDIR /app
+COPY ./Cargo.toml ./Cargo.lock .
+COPY ./src src
+# Copy over the cached dependencies
+COPY --from=cacher /app/target target
+COPY --from=cacher /home/rust/.cargo /home/rust/.cargo
+RUN cargo build
+
+# reduce binary size
+RUN strip ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/torrents-csv-service
+
+RUN cp ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/torrents-csv-service /app/torrents-csv-service
+
+# The runner
+FROM alpine:3.12
   
 # Copy resources
-COPY --from=rust /app/server/service/target/x86_64-unknown-linux-musl/release/torrents-csv-service /app/
+COPY --from=builder /app/torrents-csv-service /app/
 COPY --from=node /app/ui/dist /app/dist
+COPY --from=db_file_builder /app/torrents.db /app/torrents.db
 RUN addgroup -g 1000 torrents-csv-service
 RUN adduser -D -s /bin/sh -u 1000 -G torrents-csv-service torrents-csv-service
 RUN chown torrents-csv-service:torrents-csv-service /app/torrents-csv-service

docker/dev/docker-compose.yml

@@ -2,9 +2,7 @@ version: '3.7'
 
 services:
   torrents-csv:
-    build: 
-      context: ../../
-      dockerfile: docker/dev/Dockerfile
+    image: torrents-csv-server:latest
     restart: always
     ports:
       - "8902:8080"

docker/dev/docker_update.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-sudo docker build ../../ --file ../dev/Dockerfile -t torrents-csv:latest
+sudo docker build ../../ --file ../dev/Dockerfile -t torrents-csv-server:latest
 sudo docker-compose up -d

src/main.rs

@@ -49,7 +49,7 @@ async fn index() -> Result<NamedFile, actix_web::error::Error> {
 }
 
 fn front_end_dir() -> String {
-  env::var("TORRENTS_CSV_FRONT_END_DIR").unwrap_or_else(|_| "../ui/dist".to_string())
+  env::var("TORRENTS_CSV_FRONT_END_DIR").unwrap_or_else(|_| "./ui/dist".to_string())
 }
 
 fn torrents_db_file() -> String {