Add TPB recent video to crawl-rss

crawl-rss/main.go

@@ -31,6 +31,10 @@ func main() {
 		for _, torrent := range torrents {
 			addTorrent(db, torrent, crawled)
 		}
+		torrents = CrawlTPBVideoRecent()
+		for _, torrent := range torrents {
+			addTorrent(db, torrent, crawled)
+		}
 		if i%10 == 0 {
 			torrents = CrawlTPB48hTop()
 			for _, torrent := range torrents {
@@ -41,8 +45,8 @@ func main() {
 			}
 		}
 		i++
-		time.Sleep(time.Minute * 60)
 		go refresh(db)
+		time.Sleep(time.Minute * 60)
 	}
 }
 
@@ -53,7 +57,7 @@ func refresh(db *sql.DB) {
 
 func addTorrent(db *sql.DB, torr Torrent, crawled map[string]bool) {
 	if !(crawled[string(torr.Infohash)]) {
-		_, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", torr.Infohash, torr.Name, torr.Length)
+		_, err := db.Exec("INSERT INTO torrent (infohash, name, length) VALUES ($1, $2, $3)", strings.ToLower(torr.Infohash), torr.Name, torr.Length)
 		if err, ok := err.(*pq.Error); ok { //dark magic
 			if err.Code != "23505" {
 				log.Fatal(err)

crawl-rss/tpb.go

@@ -7,11 +7,14 @@ import (
 	"net/http"
 )
 
-// CrawlTPB48hTop crawls https://apibay.org/precompiled/data_top100_48h.json
 func CrawlTPB48hTop() []Torrent {
 	return parseApibayJSON("https://apibay.org/precompiled/data_top100_48h.json")
 }
 
+func CrawlTPBVideoRecent() []Torrent {
+	return parseApibayJSON("https://apibay.org/q.php?q=category%3A200")
+}
+
 func parseApibayJSON(url string) []Torrent {
 	httpresp, err := http.Get(url)
 	if err != nil {