Remove import TPB dump tool
The Pirate Bay no longer provides dumps AFAIK.
Commit 8fdc8d6
added crawling of top 48h TPB torrents list.
pull/27/head
parent
8fdc8d6508
commit
3304722c76
@ -1,3 +0,0 @@
|
||||
go build
|
||||
scp ./import-tpb-dump user@server:/home/nextgen/
|
||||
ssh user@server sudo -u nextgen /home/nextgen/import-tpb-dump
|
@ -1,84 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/csv"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
func main() {
|
||||
f, err := os.Open("/home/nextgen/torrent_dump_full.csv.gz")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
gr, err := gzip.NewReader(f)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer gr.Close()
|
||||
|
||||
db := initDb()
|
||||
|
||||
cr := csv.NewReader(gr)
|
||||
cr.LazyQuotes = true
|
||||
cr.Comma = rune(';')
|
||||
const layout = "2006-Jan-02 15:04:05"
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
_, err = cr.Read() // read first line and throw it away
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
for {
|
||||
line, error := cr.Read()
|
||||
if error == io.EOF {
|
||||
break
|
||||
} else if error != nil {
|
||||
if perr, ok := err.(*csv.ParseError); ok && perr.Err == csv.ErrFieldCount {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
added, err := time.Parse(layout, line[0])
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
ihBytes, _ := base64.StdEncoding.DecodeString(line[1])
|
||||
ih := hex.EncodeToString(ihBytes)
|
||||
_, err = db.Exec("INSERT INTO torrent (infohash, name, length, added) VALUES ($1, $2, $3, $4)", ih, line[2], line[3], added)
|
||||
if err, ok := err.(*pq.Error); ok { //dark magic
|
||||
if err.Code != "23505" {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func initDb() *sql.DB {
|
||||
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
|
||||
db, err := sql.Open("postgres", connStr)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS torrent (
|
||||
infohash char(40) PRIMARY KEY NOT NULL,
|
||||
name varchar NOT NULL,
|
||||
length bigint,
|
||||
added timestamp DEFAULT current_timestamp
|
||||
)`)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return db
|
||||
}
|
Loading…
Reference in New Issue