Remove import TPB dump tool

The Pirate Bay no longer provides dumps AFAIK.

Commit 8fdc8d6 added crawling of top 48h TPB torrents list.
pull/27/head
Urban Guacamole 4 years ago
parent 8fdc8d6508
commit 3304722c76

@ -28,9 +28,9 @@ Here's what the setup looks like rn:
The programs create their own tables in the DB that they need. Database name is "nextgen".
What I did first after getting the server up and running was importing the TPB dump. Download https://thepiratebay.org/static/dump/csv/torrent_dump_full.csv.gz to the import-tpb-dump directory and run `go run`.
What I did first after getting the server up and running was importing the TPB dump. This is sadly no longer possible, since TPB stopped providing the dump.
There is a complete database dump available in torrentparadise-staticbackup.torrent, so you don't have to do that. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed.
There is a database dump available in torrentparadise-staticbackup.torrent. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed.
# Usage

@ -1,3 +0,0 @@
go build
scp ./import-tpb-dump user@server:/home/nextgen/
ssh user@server sudo -u nextgen /home/nextgen/import-tpb-dump

@ -1,84 +0,0 @@
package main
import (
"compress/gzip"
"database/sql"
"encoding/base64"
"encoding/csv"
"encoding/hex"
"io"
"log"
"os"
"time"
"github.com/lib/pq"
)
func main() {
f, err := os.Open("/home/nextgen/torrent_dump_full.csv.gz")
if err != nil {
log.Fatal(err)
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
log.Fatal(err)
}
defer gr.Close()
db := initDb()
cr := csv.NewReader(gr)
cr.LazyQuotes = true
cr.Comma = rune(';')
const layout = "2006-Jan-02 15:04:05"
if err != nil {
log.Fatal(err)
}
_, err = cr.Read() // read first line and throw it away
if err != nil {
log.Fatal(err)
}
for {
line, error := cr.Read()
if error == io.EOF {
break
} else if error != nil {
if perr, ok := err.(*csv.ParseError); ok && perr.Err == csv.ErrFieldCount {
log.Println(err)
}
}
added, err := time.Parse(layout, line[0])
if err != nil {
log.Println(err)
}
ihBytes, _ := base64.StdEncoding.DecodeString(line[1])
ih := hex.EncodeToString(ihBytes)
_, err = db.Exec("INSERT INTO torrent (infohash, name, length, added) VALUES ($1, $2, $3, $4)", ih, line[2], line[3], added)
if err, ok := err.(*pq.Error); ok { //dark magic
if err.Code != "23505" {
log.Fatal(err)
}
}
}
}
func initDb() *sql.DB {
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS torrent (
infohash char(40) PRIMARY KEY NOT NULL,
name varchar NOT NULL,
length bigint,
added timestamp DEFAULT current_timestamp
)`)
if err != nil {
log.Fatal(err)
}
return db
}
Loading…
Cancel
Save