Add crawl of TPB 48h top to crawl-rss
I still miss most of the trending torrents because I source new torrents mainly from the DHT.pull/27/head
parent
6f9f970979
commit
8fdc8d6508
@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// CrawlTPB48hTop crawls https://apibay.org/precompiled/data_top100_48h.json
|
||||
func CrawlTPB48hTop() []Torrent {
|
||||
return parseApibayJSON("https://apibay.org/precompiled/data_top100_48h.json")
|
||||
}
|
||||
|
||||
func parseApibayJSON(url string) []Torrent {
|
||||
httpresp, err := http.Get(url)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return nil
|
||||
}
|
||||
defer httpresp.Body.Close()
|
||||
body, err := ioutil.ReadAll(httpresp.Body)
|
||||
|
||||
var resp []ApibayTorrent
|
||||
err = json.Unmarshal(body, &resp)
|
||||
|
||||
var torrents []Torrent
|
||||
for _, apibayTorr := range resp {
|
||||
torrents = append(torrents, Torrent{apibayTorr.Info_hash, apibayTorr.Name, apibayTorr.Size})
|
||||
}
|
||||
return torrents
|
||||
}
|
||||
|
||||
// ApibayTorrent Structure returned from apibay. For unmarshaling from JSON. Not all fields that are returned from Apibay are in this struct; YAGNI
|
||||
type ApibayTorrent struct {
|
||||
ID int
|
||||
Info_hash string
|
||||
Name string
|
||||
Size int
|
||||
Added int
|
||||
}
|
@ -0,0 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCrawlTPB48hTop(t *testing.T) {
|
||||
torrents := CrawlTPB48hTop()
|
||||
if len(torrents) < 1 {
|
||||
t.Error("no torrents crawled from tpb")
|
||||
}
|
||||
for i, torrent := range torrents {
|
||||
if torrent.Length < 10 {
|
||||
t.Error("bad length of torrent "+strconv.Itoa(i))
|
||||
}
|
||||
if len(torrent.Name) < 2 {
|
||||
t.Error("weirdly short name of torrent "+strconv.Itoa(i))
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue