Add crawl of TPB 48h top to crawl-rss
I still miss most of the trending torrents because I source new torrents mainly from the DHT.pull/27/head
parent
6f9f970979
commit
8fdc8d6508
@ -0,0 +1,41 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CrawlTPB48hTop crawls https://apibay.org/precompiled/data_top100_48h.json
|
||||||
|
func CrawlTPB48hTop() []Torrent {
|
||||||
|
return parseApibayJSON("https://apibay.org/precompiled/data_top100_48h.json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseApibayJSON(url string) []Torrent {
|
||||||
|
httpresp, err := http.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
log.Println(err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer httpresp.Body.Close()
|
||||||
|
body, err := ioutil.ReadAll(httpresp.Body)
|
||||||
|
|
||||||
|
var resp []ApibayTorrent
|
||||||
|
err = json.Unmarshal(body, &resp)
|
||||||
|
|
||||||
|
var torrents []Torrent
|
||||||
|
for _, apibayTorr := range resp {
|
||||||
|
torrents = append(torrents, Torrent{apibayTorr.Info_hash, apibayTorr.Name, apibayTorr.Size})
|
||||||
|
}
|
||||||
|
return torrents
|
||||||
|
}
|
||||||
|
|
||||||
|
// ApibayTorrent Structure returned from apibay. For unmarshaling from JSON. Not all fields that are returned from Apibay are in this struct; YAGNI
|
||||||
|
type ApibayTorrent struct {
|
||||||
|
ID int
|
||||||
|
Info_hash string
|
||||||
|
Name string
|
||||||
|
Size int
|
||||||
|
Added int
|
||||||
|
}
|
@ -0,0 +1,21 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCrawlTPB48hTop(t *testing.T) {
|
||||||
|
torrents := CrawlTPB48hTop()
|
||||||
|
if len(torrents) < 1 {
|
||||||
|
t.Error("no torrents crawled from tpb")
|
||||||
|
}
|
||||||
|
for i, torrent := range torrents {
|
||||||
|
if torrent.Length < 10 {
|
||||||
|
t.Error("bad length of torrent "+strconv.Itoa(i))
|
||||||
|
}
|
||||||
|
if len(torrent.Name) < 2 {
|
||||||
|
t.Error("weirdly short name of torrent "+strconv.Itoa(i))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue