Removing file search, using new db import method.

pull/3/head
Heretic 1 year ago
parent 29017cae1b
commit 7923956322

166
Cargo.lock generated

@ -247,9 +247,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.66"
version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"
checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
[[package]]
name = "askama_escape"
@ -257,17 +257,6 @@ version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.1.0"
@ -339,9 +328,9 @@ checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad"
[[package]]
name = "cc"
version = "1.0.77"
version = "1.0.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d"
dependencies = [
"jobserver",
]
@ -360,9 +349,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
[[package]]
name = "cookie"
version = "0.16.1"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917"
checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb"
dependencies = [
"percent-encoding",
"time",
@ -431,17 +420,38 @@ dependencies = [
[[package]]
name = "env_logger"
version = "0.9.3"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
dependencies = [
"atty",
"humantime",
"is-terminal",
"log",
"regex",
"termcolor",
]
[[package]]
name = "errno"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
dependencies = [
"errno-dragonfly",
"libc",
"winapi",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "fallible-iterator"
version = "0.2.0"
@ -569,9 +579,9 @@ dependencies = [
[[package]]
name = "hermit-abi"
version = "0.1.19"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
dependencies = [
"libc",
]
@ -631,11 +641,33 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "io-lifetimes"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "is-terminal"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189"
dependencies = [
"hermit-abi",
"io-lifetimes",
"rustix",
"windows-sys",
]
[[package]]
name = "itoa"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
[[package]]
name = "jobserver"
@ -654,9 +686,9 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388"
[[package]]
name = "libc"
version = "0.2.138"
version = "0.2.139"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
[[package]]
name = "libsqlite3-sys"
@ -669,6 +701,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4"
[[package]]
name = "local-channel"
version = "0.1.3"
@ -751,9 +789,9 @@ dependencies = [
[[package]]
name = "num_cpus"
version = "1.14.0"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi",
"libc",
@ -761,9 +799,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.16.0"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
[[package]]
name = "parking_lot"
@ -790,9 +828,9 @@ dependencies = [
[[package]]
name = "paste"
version = "1.0.9"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1"
checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
[[package]]
name = "percent-encoding"
@ -826,18 +864,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "proc-macro2"
version = "1.0.47"
version = "1.0.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.21"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
dependencies = [
"proc-macro2",
]
@ -904,9 +942,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.7.0"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
dependencies = [
"aho-corasick",
"memchr",
@ -942,11 +980,25 @@ dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "0.36.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4feacf7db682c6c329c4ede12649cd36ecab0f3be5b7d74e6a20304725db4549"
dependencies = [
"bitflags",
"errno",
"io-lifetimes",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "ryu"
version = "1.0.11"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
[[package]]
name = "scheduled-thread-pool"
@ -965,21 +1017,21 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "semver"
version = "1.0.14"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
[[package]]
name = "serde"
version = "1.0.148"
version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc"
checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
[[package]]
name = "serde_derive"
version = "1.0.148"
version = "1.0.152"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c"
checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
dependencies = [
"proc-macro2",
"quote",
@ -988,9 +1040,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.89"
version = "1.0.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
dependencies = [
"itoa",
"ryu",
@ -1056,9 +1108,9 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.105"
version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
dependencies = [
"proc-macro2",
"quote",
@ -1118,9 +1170,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.22.0"
version = "1.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3"
checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"
dependencies = [
"autocfg",
"bytes",
@ -1131,7 +1183,7 @@ dependencies = [
"pin-project-lite",
"signal-hook-registry",
"socket2",
"winapi",
"windows-sys",
]
[[package]]
@ -1188,9 +1240,9 @@ dependencies = [
[[package]]
name = "typenum"
version = "1.15.0"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
[[package]]
name = "unicase"
@ -1209,9 +1261,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
[[package]]
name = "unicode-ident"
version = "1.0.5"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
[[package]]
name = "unicode-normalization"

@ -7,13 +7,13 @@ edition = "2018"
[dependencies]
actix-web = "4.2.1"
actix-files = "0.6.2"
env_logger = "0.9.3"
env_logger = "0.10.0"
serde = "1.0.148"
serde_json = "1.0.89"
serde_derive = "1.0.148"
serde = "1.0.152"
serde_json = "1.0.91"
serde_derive = "1.0.152"
anyhow = "1.0.66"
anyhow = "1.0.68"
r2d2 = "0.8.10"
r2d2_sqlite = "0.21.0"

@ -23,7 +23,7 @@ Made with [Rust](https://www.rust-lang.org), [ripgrep](https://github.com/BurntS
### Docker
```
wget https://codeberg.org/heretic/torrents-csv-server/raw/branch/main/docker/prod/docker-compose.yml
wget https://git.torrents-csv.ml/heretic/torrents-csv-server/raw/branch/main/docker/prod/docker-compose.yml
docker-compose up -d
```
@ -32,7 +32,7 @@ And goto http://localhost:8902
### Docker Development
```
git clone --recurse-submodules https://codeberg.org/heretic/torrents-csv-server
git clone --recurse-submodules https://git.torrents-csv.ml/heretic/torrents-csv-server
cd torrents-csv-server/docker/dev
./docker_update.sh
```
@ -46,7 +46,7 @@ cd torrents-csv-server/docker/dev
### Running
```
git clone --recurse-submodules https://codeberg.org/heretic/torrents-csv-server
git clone --recurse-submodules https://git.torrents-csv.ml/heretic/torrents-csv-server
cd torrents-csv-server
./search.sh "bleh season 1"
bleh season 1 (1993-)

@ -1,101 +0,0 @@
#!/bin/bash
csv_file="./data/torrents.csv"
torrent_files_csv="../torrent_files.csv"
db_file="${TORRENTS_CSV_DB_FILE:-./torrents.db}"
build_files=false
while getopts ":f" opt; do
case $opt in
f)
build_files=true
;;
\?)
echo "Invalid option: -$OPTARG" >&2
;;
esac
done
echo "Creating temporary torrents.db file..."
# Remove double quotes for csv import
sed 's/\"//g' $csv_file > torrents_removed_quotes.csv
# Sort by seeders desc before insert
sort --field-separator=';' --key=5 -nr -o torrents_removed_quotes.csv torrents_removed_quotes.csv
touch db_tmp
sqlite3 -batch db_tmp <<"EOF"
drop table if exists torrents;
create table torrents(
"infohash" TEXT,
"name" TEXT,
"size_bytes" INTEGER,
"created_unix" INTEGER,
"seeders" INTEGER,
"leechers" INTEGER,
"completed" INTEGER,
"scraped_date" INTEGER
);
.separator ";"
.import torrents_removed_quotes.csv torrents
UPDATE torrents SET completed=NULL WHERE completed = '';
EOF
rm torrents_removed_quotes.csv
if $build_files ; then
# Cache torrent files
echo "Building files DB from $torrent_files_csv ..."
# Remove double quotes for csv import
sed 's/\"//g' $torrent_files_csv > torrent_files_removed_quotes.csv
# Removing those with too many ;
awk -F \; 'NF == 4' <torrent_files_removed_quotes.csv > torrent_files_temp_2
rm torrent_files_removed_quotes.csv
mv torrent_files_temp_2 torrent_files_temp
sqlite3 -batch db_tmp<<EOF
create table files_tmp(
"infohash" TEXT,
"index_" INTEGER,
"path" TEXT,
"size_bytes" INTEGER
);
.separator ";"
.import torrent_files_temp files_tmp
-- Filling the extra columns
create table files(
"infohash" TEXT,
"index_" INTEGER,
"path" TEXT,
"size_bytes" INTEGER,
"created_unix" INTEGER,
"seeders" INTEGER,
"leechers" INTEGER,
"completed" INTEGER,
"scraped_date" INTEGER
);
insert into files
select files_tmp.infohash,
files_tmp.index_,
files_tmp.path,
files_tmp.size_bytes,
torrents.created_unix,
torrents.seeders,
torrents.leechers,
torrents.completed,
torrents.scraped_date
from files_tmp
inner join torrents on files_tmp.infohash = torrents.infohash
order by torrents.seeders desc, files_tmp.size_bytes desc;
delete from files where seeders is null;
drop table files_tmp;
EOF
rm torrent_files_temp
fi
mv db_tmp $db_file
echo "Done."

@ -1 +1 @@
Subproject commit ced44d4978dcfb6b1fd74ee23188923a0c63e5bb
Subproject commit 4b439891b73231ce235592888bb916946b8d5a79

@ -13,11 +13,10 @@ RUN yarn build
# Build the torrents.db file
FROM alpine:3 as db_file_builder
RUN apk add sqlite bash
WORKDIR /app
RUN apk add sed sqlite bash coreutils
COPY ./data ./data
COPY ./build_sqlite.sh .
RUN ./build_sqlite.sh
COPY ./data .
RUN ./import_to_sqlite.sh
FROM $RUST_BUILDER_IMAGE as chef
USER root

@ -14,11 +14,10 @@ RUN yarn build
# Build the torrents.db file
FROM alpine:3 as db_file_builder
RUN apk add sqlite bash
WORKDIR /app
RUN apk add sed sqlite bash coreutils
COPY ./data ./data
COPY ./build_sqlite.sh .
RUN ./build_sqlite.sh
COPY ./data .
RUN ./import_to_sqlite.sh
FROM $RUST_BUILDER_IMAGE as chef
USER root

@ -14,7 +14,6 @@ use actix_web::{middleware, web, web::Data, App, HttpResponse, HttpServer};
use anyhow::{anyhow, Error};
use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::params;
use serde_json::Value;
use std::{cmp, env, io, ops::Deref};
const DEFAULT_SIZE: usize = 25;
@ -89,7 +88,7 @@ async fn search(
fn search_query(
query: web::Query<SearchQuery>,
conn: r2d2::PooledConnection<SqliteConnectionManager>,
) -> Result<Value, Error> {
) -> Result<Vec<Torrent>, Error> {
let q = query.q.trim();
if q.is_empty() || q.len() < 3 || q == "2020" {
return Err(anyhow!("{{\"error\": \"{}\"}}", "Empty query".to_string()));
@ -105,15 +104,7 @@ fn search_query(
q, type_, page, size
);
let res = if type_ == "file" {
let results = torrent_file_search(conn, q, size, offset)?;
serde_json::to_value(results)?
} else {
let results = torrent_search(conn, q, size, offset)?;
serde_json::to_value(results)?
};
Ok(res)
torrent_search(conn, q, size, offset)
}
#[derive(Debug, Serialize, Deserialize)]
@ -134,7 +125,8 @@ fn torrent_search(
size: usize,
offset: usize,
) -> Result<Vec<Torrent>, Error> {
let stmt_str = "select * from torrents where name like '%' || ?1 || '%' limit ?2, ?3";
let stmt_str =
"select * from torrent where name like '%' || ?1 || '%' order by seeders desc limit ?2, ?3";
let mut stmt = conn.prepare(stmt_str)?;
let torrent_iter = stmt.query_map(
params![
@ -163,55 +155,6 @@ fn torrent_search(
Ok(torrents)
}
#[derive(Debug, Serialize, Deserialize)]
struct File {
infohash: String,
index_: u32,
path: String,
size_bytes: isize,
created_unix: u32,
seeders: u32,
leechers: u32,
completed: Option<u32>,
scraped_date: u32,
}
fn torrent_file_search(
conn: r2d2::PooledConnection<SqliteConnectionManager>,
query: &str,
size: usize,
offset: usize,
) -> Result<Vec<File>, Error> {
let stmt_str = "select * from files where path like '%' || ?1 || '%' limit ?2, ?3";
let mut stmt = conn.prepare(stmt_str)?;
let file_iter = stmt.query_map(
params![
query.replace(' ', "%"),
offset.to_string(),
size.to_string(),
],
|row| {
Ok(File {
infohash: row.get(0)?,
index_: row.get(1)?,
path: row.get(2)?,
size_bytes: row.get(3)?,
created_unix: row.get(4)?,
seeders: row.get(5)?,
leechers: row.get(6)?,
completed: row.get(7)?,
scraped_date: row.get(8)?,
})
},
)?;
let mut files = Vec::new();
for file in file_iter {
files.push(file?);
}
Ok(files)
}
#[cfg(test)]
mod tests {
use r2d2_sqlite::SqliteConnectionManager;

Loading…
Cancel
Save