From 17b01a76273ebde8f2529276285c28be1cc2bb3d Mon Sep 17 00:00:00 2001
From: AbdBarho <ka70911@gmail.com>
Date: Fri, 16 Sep 2022 20:07:50 +0200
Subject: [PATCH] Parallel Downloads (#74)

---
 services/download/Dockerfile  |  2 +-
 services/download/download.sh | 28 +++-------------------------
 services/download/links.txt   | 12 ++++++++++++
 3 files changed, 16 insertions(+), 26 deletions(-)
 create mode 100644 services/download/links.txt

diff --git a/services/download/Dockerfile b/services/download/Dockerfile
index 6abfdfc..9c111b1 100644
--- a/services/download/Dockerfile
+++ b/services/download/Dockerfile
@@ -1,6 +1,6 @@
 FROM bash:alpine3.15
 
-RUN apk add parallel
+RUN apk add parallel aria2
 COPY . /docker
 RUN chmod +x /docker/download.sh
 ENTRYPOINT ["/docker/download.sh"]
diff --git a/services/download/download.sh b/services/download/download.sh
index f20b597..35048c0 100755
--- a/services/download/download.sh
+++ b/services/download/download.sh
@@ -2,32 +2,10 @@
 
 set -Eeuo pipefail
 
-# [[ "$(sha256sum -b $file | head -c 64)" == "$sha" ]]
+echo "Downloading, this might take a while..."
 
-declare -A MODELS
-
-MODELS['model.ckpt']='https://www.googleapis.com/storage/v1/b/aai-blog-files/o/sd-v1-4.ckpt?alt=media'
-MODELS['GFPGANv1.3.pth']='https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth'
-MODELS['RealESRGAN_x4plus.pth']='https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth'
-MODELS['RealESRGAN_x4plus_anime_6B.pth']='https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth'
-MODELS['LDSR.yaml']='https://heibox.uni-heidelberg.de/f/31a76b13ea27482981b4/?dl=1'
-MODELS['LDSR.ckpt']='https://heibox.uni-heidelberg.de/f/578df07c8fc04ffbadf3/?dl=1'
-
-echo "Downloading..."
-
-for file in "${!MODELS[@]}"; do
-  url=${MODELS[$file]}
-  full_path="/cache/models/$file"
-
-  if [[ -f "$full_path" ]]; then
-    echo "- $file exists"
-    continue
-  fi
-
-  mkdir -p $(dirname $full_path)
-  wget --tries=10 -c -O $full_path $url
-done
+aria2c --input-file /docker/links.txt --dir /cache/models --continue
 
 echo "Checking SHAs..."
 
-time parallel --will-cite -a /docker/checksums.sha256 "echo -n {} | sha256sum -c"
+parallel --will-cite -a /docker/checksums.sha256 "echo -n {} | sha256sum -c"
diff --git a/services/download/links.txt b/services/download/links.txt
new file mode 100644
index 0000000..b728297
--- /dev/null
+++ b/services/download/links.txt
@@ -0,0 +1,12 @@
+https://www.googleapis.com/storage/v1/b/aai-blog-files/o/sd-v1-4.ckpt?alt=media
+  out=model.ckpt
+https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth
+  out=GFPGANv1.3.pth
+https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth
+  out=RealESRGAN_x4plus.pth
+https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth
+  out=RealESRGAN_x4plus_anime_6B.pth
+https://heibox.uni-heidelberg.de/f/31a76b13ea27482981b4/?dl=1
+  out=LDSR.yaml
+https://heibox.uni-heidelberg.de/f/578df07c8fc04ffbadf3/?dl=1
+  out=LDSR.ckpt