You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
126 lines
3.5 KiB
Go
126 lines
3.5 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"log"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/etix/goscrape"
|
|
_ "github.com/lib/pq"
|
|
)
|
|
|
|
const retryLimit = 10
|
|
const waitTime = 500 // in ms
|
|
var trackers = [4]string{"udp://tracker.coppersurfer.tk:6969", "udp://tracker.internetwarriors.net:1337", "udp://tracker.opentrackr.org:1337", "udp://tracker.pirateparty.gr:6969/announce"}
|
|
|
|
func main() {
|
|
db := initDb()
|
|
trackerResponses := make(chan trackerResponse, 100)
|
|
trackerRequests := make(chan []string, 1000)
|
|
|
|
for _, tracker := range trackers {
|
|
go runScraper(trackerRequests, trackerResponses, tracker, waitTime)
|
|
}
|
|
|
|
go runPersister(trackerResponses, db)
|
|
|
|
rows, err := db.Query("SELECT infohash FROM torrent WHERE NOT EXISTS (SELECT FROM peercount WHERE infohash = torrent.infohash)")
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
var infohashes []string
|
|
for rows.Next() {
|
|
var infohash string
|
|
if err := rows.Scan(&infohash); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
if len(infohashes) < 74 {
|
|
infohashes = append(infohashes, infohash)
|
|
} else {
|
|
trackerRequests <- infohashes
|
|
infohashes = []string{}
|
|
}
|
|
}
|
|
trackerRequests <- infohashes
|
|
for len(trackerRequests) > 0 {
|
|
time.Sleep(2 * time.Second)
|
|
log.Println("Tracker requests left to send: " + strconv.Itoa(len(trackerRequests)))
|
|
}
|
|
close(trackerRequests)
|
|
|
|
for len(trackerResponses) > 0 {
|
|
time.Sleep(2 * time.Second)
|
|
log.Println("Tracker responses left to save: " + strconv.Itoa(len(trackerResponses)))
|
|
}
|
|
time.Sleep(time.Duration(waitTime*retryLimit) * time.Millisecond)
|
|
close(trackerResponses)
|
|
}
|
|
|
|
//Runs a tracker that scrapes the given tracker. Takes requests from trackerRequests and sends responses to trackerResponses
|
|
//waittime is in miliseconds
|
|
func runScraper(trackerRequests chan []string, trackerResponses chan trackerResponse, tracker string, waittime int) {
|
|
s, err := goscrape.New(tracker)
|
|
s.SetTimeout(time.Duration(waitTime) * time.Millisecond)
|
|
s.SetRetryLimit(retryLimit)
|
|
if err != nil {
|
|
log.Fatal("Error:", err)
|
|
}
|
|
for req := range trackerRequests {
|
|
infohashes := make([][]byte, len(req))
|
|
for i, v := range req {
|
|
if len(v) != 40 { //infohashes are 40 chars long in string representation.
|
|
panic("Infohash in trackerRequest with index " + strconv.Itoa(i) + " isn't 40 chars long, it's " + strconv.Itoa(len(v)) + " long.")
|
|
}
|
|
infohashes[i] = []byte(v)
|
|
}
|
|
res, err := s.Scrape(infohashes...)
|
|
if err != nil {
|
|
log.Println(err)
|
|
} else {
|
|
trackerResponses <- trackerResponse{tracker, res}
|
|
}
|
|
|
|
time.Sleep(time.Duration(waittime) * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func runPersister(trackerResponses chan trackerResponse, db *sql.DB) {
|
|
for res := range trackerResponses {
|
|
for _, scrapeResult := range res.scrapeResult {
|
|
_, err := db.Exec("INSERT INTO peercount (infohash, tracker, seeders, leechers, completed) VALUES ($1, $2, $3, $4, $5)", scrapeResult.Infohash, res.tracker, scrapeResult.Seeders, scrapeResult.Leechers, scrapeResult.Completed)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func initDb() *sql.DB {
|
|
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
|
|
db, err := sql.Open("postgres", connStr)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS peercount (
|
|
infohash char(40),
|
|
tracker varchar,
|
|
seeders int NOT NULL,
|
|
leechers int NOT NULL,
|
|
completed int NOT NULL,
|
|
scraped timestamp DEFAULT current_timestamp,
|
|
PRIMARY KEY (infohash, tracker, scraped)
|
|
)`)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return db
|
|
}
|
|
|
|
type trackerResponse struct {
|
|
tracker string
|
|
scrapeResult []*goscrape.ScrapeResult
|
|
}
|