diff --git a/.gitignore b/.gitignore index 6a4f7d3..eca9b10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,7 @@ generated/ node_modules/ -db.sqlite3 -dump.sql +dump.csv ipfs.html about.html copyright.html torrent_dump_full.csv.gz -index-generator/main.js \ No newline at end of file diff --git a/README.md b/README.md index 2092511..0d4c93c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,13 @@ # About ## What is this? -This is a repository of all the tools I use to build and run torrent-paradise.ml. Some people asked for a source, so I'm just putting this out here. I did make *some* effort to clean it up. The 'code name' of the project is nextgen (next gen torrent search), so don't be surprised if it comes up somewhere. +If you don't know what Torrent Paradise is, see the [website](https://torrent-paradise.ml/about.html). + +This is a repository of all the tools I use to build and run torrent-paradise.ml. The 'code name' of the project is nextgen (next gen torrent search), so don't be surprised if it comes up somewhere. ## Can you help me? Maybe, open an issue. Be sure to demonstrate an effort that you tried to solve the problem yourself. -## This is a big mess. Fix it maybe? -WIP ❤️ - # Setup Here's what the setup looks like rn: @@ -24,30 +23,28 @@ Here's what the setup looks like rn: - user with username nextgen on the server - my laptop w/ Linux - Go toolchain installed - - node v10.9.0 & npm - -Schema for the database is sth like this (taken from index-generator/README, runs on sqlite, probably also on pg.) -```sql -CREATE TABLE peercount ( infohash char(40), tracker varchar, seeders int, leechers int, completed int, scraped timestamp); - -CREATE TABLE torrent( infohash char(40), name varchar, length bigint, added timestamp); -``` - + - node v10.15 & npm + - Python 3 (required only for index-generator/fix-metajson.py) +The programs create their own tables in the DB that they need. Database name is "nextgen". What I did first after getting the server up and running was importing the TPB dump. Download https://thepiratebay.org/static/dump/csv/torrent_dump_full.csv.gz to the import-tpb-dump directory and run `go run`. -I probably forgot sth. Open an issue! - # Usage ## Generate the index -This is a half-broken process that is partially described in update-index.sh. Read the script to understand what it does. +See `update-index.sh`. Before running it for the first time, you should create the materialized view *fresh*. For instructions, see the first paragraph of snippets.sql. ## Spider the DHT -Run `go build` in spider/ to compile and scp the binary it to the server. You can use the systemd service file in `spider/spider.service` to start the spider on boot. +Run `go build` in spider/ to compile and scp the binary it to the server. You can use the systemd service file in `spider/spider.service` to start the spider on server boot. + +## Scraping trackers for seed/leech data + +Run `go build` in tracker-scraper/ to compile and scp the binary it to the server. Run it every time you want to fetch new seed/leech data for all torrents. + +`tracker-scraper` saves the results into the *trackerdata* table in database. # Contributing diff --git a/index-generator/README b/index-generator/README deleted file mode 100644 index a0226b0..0000000 --- a/index-generator/README +++ /dev/null @@ -1,23 +0,0 @@ -Takes a sqlite DB (export of postgres that is used by spider, tracker-scraper and import-tpb-dump) and generates the index, using the ipfsearch-index library. - -Getting sqlite dump from the postgres: -nextgen@ipfsearch: ~$ pg_dump --data-only --inserts nextgen > dump.sql - -# remove header from dump (manually) - -sed -i -e 's/public.peercount/peercount/g' dump.sql -sed -i -e 's/public.torrent/torrent/g' dump.sql -tail -n +2 dump.sql > newdump.sql -mv newdump.sql dump.sql -gzip dump.sql - -# copy dump.sql.gz to index-generator directory, unzip - -user@localhost $ scp user@server:/home/nextgen/dump.sql.gz . -$ sqlite3 db.sqlite3 - -sqlite> CREATE TABLE peercount ( infohash char(40), tracker varchar, seeders int, leechers int, completed int, scraped timestamp); -sqlite> CREATE TABLE torrent( infohash char(40), name varchar, length bigint, added timestamp); -sqlite> BEGIN; -sqlite> .read dump.sql -sqlite> END; \ No newline at end of file diff --git a/index-generator/fix-metajson.py b/index-generator/fix-metajson.py index f17d1f4..366476a 100644 --- a/index-generator/fix-metajson.py +++ b/index-generator/fix-metajson.py @@ -19,7 +19,7 @@ def count_torrents_in_index(inxpath, blocksize): meta = json.load(io.open(sys.argv[1]+".meta.json","r")) meta["resultPage"] = "resultpage" meta["entries"] = count_torrents_in_index(sys.argv[1],1000) -meta["inxUrlBase"] = "website/generated/inx" -meta["invUrlBase"] = "website/generated/inv" +meta["inxURLBase"] = "generated/inx" +meta["invURLBase"] = "generated/inv" json.dump(meta,io.open(sys.argv[1]+".meta.json", "w")) diff --git a/index-generator/main.ts b/index-generator/main.ts index 4a138c2..71437bc 100644 --- a/index-generator/main.ts +++ b/index-generator/main.ts @@ -1,31 +1,41 @@ const ipfsearch = require("ipfsearch-index") -const sqlite3 = require('sqlite3') - -const db = new sqlite3.Database('db.sqlite3'); +const parse = require("csv-parse") +const fs = require("fs") let indexer = new ipfsearch.Indexer() let i = 0 -db.each("SELECT torrent.infohash, torrent.name, torrent.length, torrent.added, peercount.seeders, peercount.leechers, peercount.completed FROM torrent INNER JOIN peercount on torrent.infohash = peercount.infohash ORDER BY peercount.scraped DESC", function(err, row){ - if(err) console.error(err) - if(row["seeders"] > 0){ - indexer.addToIndex(new Torrent(row["infohash"], row["name"], row["length"], row["seeders"], row["leechers"], row["completed"])) +const parser = parse() +fs.createReadStream("dump.csv").pipe(parser) + +parser.on('readable', function () { + let record + while (record = parser.read()) { + if (parseInt(record[3]) > 0) { + indexer.addToIndex(new Torrent(record[0], record[1], parseInt(record[2]), parseInt(record[3]), parseInt(record[4]), parseInt(record[5]))) + i++ + } } - i++ -},function(err,num){ - console.log("Read all "+i+" records.") - console.log("Persisting "+num+" records.") - indexer.persist("generated/inv", "generated/inx", "", "Torrent Paradise torrent index","todo", 1000) +}) + +parser.on('error', function (err) { + console.error(err.message) +}) + +parser.on('end', function () { + console.log("Read all " + i + " records. Persisting."); + indexer.persist("../website/generated/inv", "../website/generated/inx", "Urban Guacamole", "Torrent Paradise index", "", 1000); + }) class Torrent extends ipfsearch.Document { len: number - s : number - l : number - c : number + s: number + l: number + c: number - constructor(id : string, text : string, size : number, seeders : number, leechers : number, completed : number){ - super(id,text) + constructor(id: string, text: string, size: number, seeders: number, leechers: number, completed: number) { + super(id, text) this.len = size this.s = seeders this.l = leechers diff --git a/index-generator/package-lock.json b/index-generator/package-lock.json index 8165cd5..acfc2c4 100644 --- a/index-generator/package-lock.json +++ b/index-generator/package-lock.json @@ -5,933 +5,24 @@ "requires": true, "dependencies": { "@types/node": { - "version": "10.12.18", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz", - "integrity": "sha512-fh+pAqt4xRzPfqA6eh3Z2y6fyZavRIumvjhaCL753+TVkGKGhpPeyrJG2JftD0T9q4GF00KjefsQ+PQNDdWQaQ==" + "version": "10.12.29", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.29.tgz", + "integrity": "sha512-J/tnbnj8HcsBgCe2apZbdUpQ7hs4d7oZNTYA5bekWdP0sr2NGsOpI/HRdDroEi209tEvTcTtxhD0FfED3DhEcw==" }, - "abbrev": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" - }, - "ajv": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.6.2.tgz", - "integrity": "sha512-FBHEW6Jf5TB9MGBgUUA9XHkTbjXYfAUjY43ACMfmdMRHniyoMHjHjzD50OK8LGDWQwp4rWEsIq5kEqq7rvIM1g==", - "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" - }, - "are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "buffer-writer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", - "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "chownr": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", - "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==" - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" - }, - "combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" - }, - "detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" - }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "requires": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", - "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "fs-minipass": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", - "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", - "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", - "requires": { - "ajv": "^6.5.5", - "har-schema": "^2.0.0" - } - }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - } - }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "ignore-walk": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz", - "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", - "requires": { - "minimatch": "^3.0.4" - } - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "csv-parse": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.3.3.tgz", + "integrity": "sha512-bZ+AZjm2LlWEp5+TKeFeXDldduCUUaxEif+KUv+zvAwmCvCKTqeSHVEyxztGCQ6OE+87ObRq4NsCmg91SuJbhQ==" }, "ipfsearch-index": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/ipfsearch-index/-/ipfsearch-index-2.5.1.tgz", - "integrity": "sha512-NscBi32MjuYGVoD7UZNJ0bupbx10lm7CbFMrPWPz27lwLenepVjwEe8GRSvZnjUxPQDuVZvAHMcXotLHMmBMlA==" - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "mime-db": { - "version": "1.37.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", - "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==" - }, - "mime-types": { - "version": "2.1.21", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", - "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", - "requires": { - "mime-db": "~1.37.0" - } - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - }, - "minipass": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", - "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", - "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "nan": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", - "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA==" - }, - "needle": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/needle/-/needle-2.2.4.tgz", - "integrity": "sha512-HyoqEb4wr/rsoaIDfTH2aVL9nWtQqba2/HvMv+++m8u0dz808MaagKILxtfeSN7QU7nvbQ79zk3vYOJp9zsNEA==", - "requires": { - "debug": "^2.1.2", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - } - }, - "node-pre-gyp": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.11.0.tgz", - "integrity": "sha512-TwWAOZb0j7e9eGaf9esRx3ZcLaE5tQ2lvYy1pb5IAaG1a2e2Kv5Lms1Y4hpj+ciXJRofIxxlt5haeQ/2ANeE0Q==", - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.1", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.2.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - }, - "dependencies": { - "semver": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", - "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==" - } - } - }, - "nopt": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", - "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", - "requires": { - "abbrev": "1", - "osenv": "^0.1.4" - } - }, - "npm-bundled": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.5.tgz", - "integrity": "sha512-m/e6jgWu8/v5niCUKQi9qQl8QdeEduFA96xHDDzFGqly0OOjI7c+60KM/2sppfnUU9JJagf+zs+yGhqSOFj71g==" - }, - "npm-packlist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.2.0.tgz", - "integrity": "sha512-7Mni4Z8Xkx0/oegoqlcao/JpPCPEMtUvsmB0q7mgvlMinykJLSRTYuFqoQLYgGY8biuxIeiHO+QNJKbCfljewQ==", - "requires": { - "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" - } - }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" - }, - "os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" - }, - "osenv": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", - "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", - "requires": { - "os-homedir": "^1.0.0", - "os-tmpdir": "^1.0.0" - } - }, - "packet-reader": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-0.3.1.tgz", - "integrity": "sha1-zWLmCvjX/qinBexP+ZCHHEaHHyc=" - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "pg": { - "version": "7.8.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-7.8.0.tgz", - "integrity": "sha512-yS3C9YD+ft0H7G47uU0eKajgTieggCXdA+Fxhm5G+wionY6kPBa8BEVDwPLMxQvkRkv3/LXiFEqjZm9gfxdW+g==", - "requires": { - "buffer-writer": "2.0.0", - "packet-reader": "0.3.1", - "pg-connection-string": "0.1.3", - "pg-pool": "^2.0.4", - "pg-types": "~2.0.0", - "pgpass": "1.x", - "semver": "4.3.2" - } - }, - "pg-connection-string": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz", - "integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc=" - }, - "pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" - }, - "pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha512-hod2zYQxM8Gt482q+qONGTYcg/qVcV32VHVPtktbBJs0us3Dj7xibISw0BAAXVMCzt8A/jhfJvpZaxUlqtqs0g==" - }, - "pg-types": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.0.0.tgz", - "integrity": "sha512-THUD7gQll5tys+5eQ8Rvs7DjHiIC3bLqixk3gMN9Hu8UrCBAOjf35FoI39rTGGc3lM2HU/R+Knpxvd11mCwOMA==", - "requires": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.0", - "postgres-interval": "^1.1.0" - } - }, - "pgpass": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", - "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", - "requires": { - "split": "^1.0.0" - } - }, - "postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" - }, - "postgres-bytea": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", - "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" - }, - "postgres-date": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.3.tgz", - "integrity": "sha1-4tiXAu/bJY/52c7g/pG9BpdSV6g=" - }, - "postgres-interval": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz", - "integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==", - "requires": { - "xtend": "^4.0.0" - } - }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "psl": { - "version": "1.1.31", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", - "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" - } - } - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", - "requires": { - "glob": "^7.1.3" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "sax": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" - }, - "semver": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", - "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" - }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" - }, - "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" - }, - "split": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", - "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", - "requires": { - "through": "2" - } - }, - "sqlite3": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-4.0.6.tgz", - "integrity": "sha512-EqBXxHdKiwvNMRCgml86VTL5TK1i0IKiumnfxykX0gh6H6jaKijAXvE9O1N7+omfNSawR2fOmIyJZcfe8HYWpw==", - "requires": { - "nan": "~2.10.0", - "node-pre-gyp": "^0.11.0", - "request": "^2.87.0" - } - }, - "sshpk": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.0.tgz", - "integrity": "sha512-Zhev35/y7hRMcID/upReIvRse+I9SVhyVre/KTJSJQWMz3C3+G+HpO7m1wK/yckEtujKZ7dS4hkVxAnmHaIGVQ==", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" - }, - "tar": { - "version": "4.4.8", - "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", - "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", - "requires": { - "chownr": "^1.1.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.3.4", - "minizlib": "^1.1.1", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.2", - "yallist": "^3.0.2" - } - }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - } - } - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/ipfsearch-index/-/ipfsearch-index-2.5.5.tgz", + "integrity": "sha512-BiUFE2uTOHvftvKESurAIBq5HAPHrnJgoIRQ+iEKbBs+X5db7aotL0FDq0YU2KH+NRuDBQbnyw1Fn5xI0YQxnA==" }, "typescript": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.2.2.tgz", - "integrity": "sha512-VCj5UiSyHBjwfYacmDuc/NOk4QQixbE+Wn7MFJuS0nRuPQbof132Pw4u53dm264O8LPc2MVsc7RJNml5szurkg==" - }, - "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", - "requires": { - "punycode": "^2.1.0" - } - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "requires": { - "string-width": "^1.0.2 || 2" - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - }, - "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "3.3.3333", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.3.3333.tgz", + "integrity": "sha512-JjSKsAfuHBE/fB2oZ8NxtRTk5iGcg6hkYXMnZ3Wc+b2RSqejEqTaem11mHASMnFilHrax3sLK0GDzcJrekZYLw==" } } } diff --git a/index-generator/package.json b/index-generator/package.json index c3cd658..b2c9da9 100644 --- a/index-generator/package.json +++ b/index-generator/package.json @@ -4,11 +4,10 @@ "description": "", "main": "main.js", "dependencies": { - "@types/node": "^10.12.18", - "ipfsearch-index": "^2.5.1", - "pg": "^7.8.0", - "sqlite3": "^4.0.6", - "typescript": "^3.2.2" + "@types/node": "^10.12.29", + "csv-parse": "^4.3.3", + "ipfsearch-index": "^2.5.5", + "typescript": "^3.3.3333" }, "devDependencies": {}, "scripts": { diff --git a/snippets.sql b/snippets.sql new file mode 100644 index 0000000..c216d5e --- /dev/null +++ b/snippets.sql @@ -0,0 +1,17 @@ +-- SELECT for the index-ready materialized view "fresh" +select distinct on (infohash) trackerdata.infohash, torrent.name, torrent.length, seeders, leechers, completed, tracker from trackerdata inner join torrent on (trackerdata.infohash = torrent.infohash) order by infohash, scraped asc, seeders desc; +-- to create this view, run: create materialized view AS (paste from above) + +select sum(seeders),tracker from trackerdata group by tracker; + +get highest seed/leech count found for every torrent, using data from tracker with most seeds +select distinct on (infohash) infohash, seeders, leechers, completed, tracker from trackerdata where completed != 0 order by infohash, scraped asc, seeders desc; + + + +generate top 100 by seeders: +select * from (select distinct on (trackerdata.infohash) trackerdata.infohash, torrent.name, seeders, leechers from trackerdata inner join torrent on (trackerdata.infohash = torrent.infohash) where completed != 0 order by infohash, seeders desc) as subquery order by seeders desc limit 100; + +SELECT added::date, count(infohash) +from torrent where added > '2019-01-15'::date +group by added::date order by count desc; \ No newline at end of file diff --git a/tracker-scraper/tracker-scraper.go b/tracker-scraper/tracker-scraper.go index b33e0ef..814facc 100644 --- a/tracker-scraper/tracker-scraper.go +++ b/tracker-scraper/tracker-scraper.go @@ -4,28 +4,47 @@ import ( "database/sql" "log" "strconv" + "sync/atomic" "time" "github.com/etix/goscrape" _ "github.com/lib/pq" ) -const retryLimit = 10 +const retryLimit = 3 const waitTime = 500 // in ms -var trackers = [4]string{"udp://tracker.coppersurfer.tk:6969", "udp://tracker.internetwarriors.net:1337", "udp://tracker.opentrackr.org:1337", "udp://tracker.pirateparty.gr:6969/announce"} +var trackers = [3]string{"udp://tracker.coppersurfer.tk:6969", "udp://tracker.internetwarriors.net:1337", "udp://tracker.pirateparty.gr:6969"} func main() { db := initDb() trackerResponses := make(chan trackerResponse, 100) - trackerRequests := make(chan []string, 1000) + trackerRequests := make(map[string]chan []string) + + var counter uint64 //count of torrents scraped + + quitCounter := make(chan bool) + go func() { + for { + select { + case <-quitCounter: + return + default: + log.Println("Torrents scraped so far: " + strconv.Itoa(int(atomic.LoadUint64(&counter)))) + time.Sleep(2 * time.Second) + } + } + }() for _, tracker := range trackers { - go runScraper(trackerRequests, trackerResponses, tracker, waitTime) + trackerRequests[tracker] = make(chan []string, 1000) + go runScraper(trackerRequests[tracker], trackerResponses, tracker, waitTime, &counter) } - go runPersister(trackerResponses, db) + datestamp := time.Now().Local().Format("2006-01-02") + + go runPersister(trackerResponses, db, datestamp) - rows, err := db.Query("SELECT infohash FROM torrent WHERE NOT EXISTS (SELECT FROM peercount WHERE infohash = torrent.infohash)") + rows, err := db.Query("SELECT infohash FROM torrent WHERE NOT EXISTS (SELECT FROM trackerdata WHERE infohash = torrent.infohash AND scraped = '" + datestamp + "')") if err != nil { log.Fatal(err) } @@ -39,16 +58,32 @@ func main() { if len(infohashes) < 74 { infohashes = append(infohashes, infohash) } else { - trackerRequests <- infohashes + for _, tracker := range trackers { + trackerRequests[tracker] <- infohashes + } infohashes = []string{} } } - trackerRequests <- infohashes + for _, tracker := range trackers { + trackerRequests[tracker] <- infohashes + } + + quitCounter <- true + for len(trackerRequests) > 0 { time.Sleep(2 * time.Second) - log.Println("Tracker requests left to send: " + strconv.Itoa(len(trackerRequests))) + var left int + for i, tracker := range trackers { + left = len(trackerRequests[tracker]) + if left != 0 { + log.Println("Tracker " + strconv.Itoa(i) + " requests left to send: " + strconv.Itoa(len(trackerRequests[tracker]))) + } + } + } + + for _, tracker := range trackers { + close(trackerRequests[tracker]) } - close(trackerRequests) for len(trackerResponses) > 0 { time.Sleep(2 * time.Second) @@ -60,7 +95,7 @@ func main() { //Runs a tracker that scrapes the given tracker. Takes requests from trackerRequests and sends responses to trackerResponses //waittime is in miliseconds -func runScraper(trackerRequests chan []string, trackerResponses chan trackerResponse, tracker string, waittime int) { +func runScraper(trackerRequests chan []string, trackerResponses chan trackerResponse, tracker string, waittime int, counter *uint64) { s, err := goscrape.New(tracker) s.SetTimeout(time.Duration(waitTime) * time.Millisecond) s.SetRetryLimit(retryLimit) @@ -79,6 +114,7 @@ func runScraper(trackerRequests chan []string, trackerResponses chan trackerResp if err != nil { log.Println(err) } else { + atomic.AddUint64(counter, uint64(len(infohashes))) trackerResponses <- trackerResponse{tracker, res} } @@ -86,10 +122,10 @@ func runScraper(trackerRequests chan []string, trackerResponses chan trackerResp } } -func runPersister(trackerResponses chan trackerResponse, db *sql.DB) { +func runPersister(trackerResponses chan trackerResponse, db *sql.DB, datestamp string) { for res := range trackerResponses { for _, scrapeResult := range res.scrapeResult { - _, err := db.Exec("INSERT INTO peercount (infohash, tracker, seeders, leechers, completed) VALUES ($1, $2, $3, $4, $5)", scrapeResult.Infohash, res.tracker, scrapeResult.Seeders, scrapeResult.Leechers, scrapeResult.Completed) + _, err := db.Exec("INSERT INTO trackerdata (infohash, tracker, seeders, leechers, completed, scraped) VALUES ($1, $2, $3, $4, $5, $6)", scrapeResult.Infohash, res.tracker, scrapeResult.Seeders, scrapeResult.Leechers, scrapeResult.Completed, datestamp) if err != nil { log.Fatal(err) } @@ -104,14 +140,19 @@ func initDb() *sql.DB { log.Fatal(err) } - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS peercount ( + /*_, err = db.Exec(`CREATE TYPE tracker AS ENUM ('udp://tracker.coppersurfer.tk:6969', 'udp://tracker.internetwarriors.net:1337', 'udp://tracker.opentrackr.org:1337', 'udp://tracker.pirateparty.gr:6969')`) + if err != nil { + log.Fatal(err) + }*/ + + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS trackerdata ( infohash char(40), - tracker varchar, + tracker tracker, seeders int NOT NULL, leechers int NOT NULL, completed int NOT NULL, - scraped timestamp DEFAULT current_timestamp, - PRIMARY KEY (infohash, tracker, scraped) + scraped char(10), + PRIMARY KEY (infohash, scraped, tracker) )`) if err != nil { log.Fatal(err) diff --git a/update-index.sh b/update-index.sh index bc51bf2..fbb4eaa 100755 --- a/update-index.sh +++ b/update-index.sh @@ -1,25 +1,14 @@ # This script updates the index. Testing and uploading to server/IPFS is done manually. echo "Scraping trackers for seed/leech data" -mosh nextgen@server "~/tracker-scraper" -echo "Generating SQL dump" -ssh nextgen@server pg_dump --data-only --inserts nextgen > index-generator/dump.sql +mosh nextgen@server "~/tracker-scraper" # you can use ssh instead of mosh aswell -sed -i -e 's/public.peercount/peercount/g' index-generator/dump.sql -sed -i -e 's/public.torrent/torrent/g' index-generator/dump.sql -tail -n +21 index-generator/dump.sql > index-generator/newdump.sql # remove headers -mv index-generator/newdump.sql index-generator/dump.sql -rm index-generator/db.sqlite3 -echo "Preparing sqlite DB" -sqlite3 index-generator/db.sqlite3 "CREATE TABLE peercount ( infohash char(40), tracker varchar, seeders int, leechers int, completed int, scraped timestamp, ws boolean);" -sqlite3 index-generator/db.sqlite3 "CREATE TABLE torrent( infohash char(40), name varchar, length bigint, added timestamp);" -echo """Do the following: +ssh nextgen@server "psql -c 'REFRESH MATERIALIZED VIEW fresh'" + +echo "Generating index dump" +rm index-generator/dump.csv +ssh nextgen@server "psql -c '\copy (select * from fresh) to stdout with format csv'" > index-generator/dump.csv -sqlite> BEGIN; -sqlite> .read index-generator/dump.sql -sqlite> END;""" -sqlite3 index-generator/db.sqlite3 -echo "Generating index now..." (cd index-generator; node --max-old-space-size=10000 main.js) python3 index-generator/fix-metajson.py website/generated/inx