|
|
|
@ -21,24 +21,25 @@ Running without a warrior
|
|
|
|
|
-------------------------
|
|
|
|
|
To run this outside the warrior, clone this repository, cd into its directory and run:
|
|
|
|
|
|
|
|
|
|
pip install --upgrade seesaw
|
|
|
|
|
python3 -m pip install setuptools wheel
|
|
|
|
|
python3 -m pip install --upgrade seesaw zstandard requests
|
|
|
|
|
./get-wget-lua.sh
|
|
|
|
|
|
|
|
|
|
then start downloading with:
|
|
|
|
|
|
|
|
|
|
run-pipeline pipeline.py --concurrent 2 YOURNICKHERE
|
|
|
|
|
run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE
|
|
|
|
|
|
|
|
|
|
For more options, run:
|
|
|
|
|
|
|
|
|
|
run-pipeline --help
|
|
|
|
|
run-pipeline3 --help
|
|
|
|
|
|
|
|
|
|
If you don't have root access and/or your version of pip is very old, you can replace "pip install --upgrade seesaw" with:
|
|
|
|
|
|
|
|
|
|
wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python get-pip.py --user ; ~/.local/bin/pip install --upgrade --user seesaw
|
|
|
|
|
wget https://raw.github.com/pypa/pip/master/contrib/get-pip.py ; python3 get-pip.py --user ; ~/.local/bin/pip3 install --upgrade --user seesaw
|
|
|
|
|
|
|
|
|
|
so that pip and seesaw are installed in your home, then run
|
|
|
|
|
|
|
|
|
|
~/.local/bin/run-pipeline pipeline.py --concurrent 2 YOURNICKHERE
|
|
|
|
|
~/.local/bin/run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE
|
|
|
|
|
|
|
|
|
|
Running multiple instances on different IPs
|
|
|
|
|
-------------------------------------------
|
|
|
|
@ -49,7 +50,7 @@ Use the `--context-value` argument to pass in `bind_address=123.4.5.6` (replace
|
|
|
|
|
|
|
|
|
|
Example of running 2 threads, no web interface, and Wget binding of IP address:
|
|
|
|
|
|
|
|
|
|
run-pipeline pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6
|
|
|
|
|
run-pipeline3 pipeline.py --concurrent 2 YOURNICKHERE --disable-web-server --context-value bind_address=123.4.5.6
|
|
|
|
|
|
|
|
|
|
Distribution-specific setup
|
|
|
|
|
-------------------------
|
|
|
|
@ -60,9 +61,10 @@ Package `libzstd-dev` version 1.4.4 is required which is currently available fro
|
|
|
|
|
adduser --system --group --shell /bin/bash archiveteam
|
|
|
|
|
echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list
|
|
|
|
|
apt-get update \
|
|
|
|
|
&& apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \
|
|
|
|
|
&& apt-get install -y git-core libgnutls-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \
|
|
|
|
|
&& apt-get -t buster-backports install zstd libzstd-dev libzstd1
|
|
|
|
|
pip install --upgrade seesaw zstandard
|
|
|
|
|
python3 -m pip install setuptools wheel
|
|
|
|
|
python3 -m pip install --upgrade seesaw zstandard requests
|
|
|
|
|
su -c "cd /home/archiveteam; git clone https://github.com/ArchiveTeam/reddit-grab.git; cd reddit-grab; ./get-wget-lua.sh" archiveteam
|
|
|
|
|
screen su -c "cd /home/archiveteam/reddit-grab/; run-pipeline pipeline.py --concurrent 2 --address '127.0.0.1' YOURNICKHERE" archiveteam
|
|
|
|
|
[... ctrl+A D to detach ...]
|
|
|
|
@ -72,7 +74,7 @@ In __Debian Jessie, Ubuntu 18.04 Bionic and above__, the `libgnutls-dev` package
|
|
|
|
|
adduser --system --group --shell /bin/bash archiveteam
|
|
|
|
|
echo deb http://deb.debian.org/debian buster-backports main contrib > /etc/apt/sources.list.d/backports.list
|
|
|
|
|
apt-get update \
|
|
|
|
|
&& apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen python-dev python-pip bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync \
|
|
|
|
|
&& apt-get install -y git-core libgnutls28-dev lua5.1 liblua5.1-0 liblua5.1-0-dev screen bzip2 zlib1g-dev flex autoconf autopoint texinfo gperf lua-socket rsync automake pkg-config python3-dev python3-pip build-essential \
|
|
|
|
|
&& apt-get -t buster-backports install zstd libzstd-dev libzstd1
|
|
|
|
|
[... pretty much the same as above ...]
|
|
|
|
|
|
|
|
|
|