commit 4f7fe8805e99d0a0ce1f1bfabedab2a752a03cf9 Author: Kevin Lynx Date: Mon Jul 1 22:42:14 2013 +0800 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..d18f843 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +## dhtcrawler2 + +This git branch maintain pre-compiled erlang files to start dhtcrawler2 directly. + +## Usage + +* install Erlang R16B or newer +* download mongodb and start mongodb first + + mongod --dbpath your-database-path --setParameter textSearchEnabled=true + +* start **crawler**, on Windows, just click `win_start_crawler.bat` +* start **hash_reader**, on Windows, just click `win_start_hash.bat` +* start **httpd**, on Windows, just click `win_start_http.bat` +* wait several minutes and checkout `localhost:8000` + + + diff --git a/create_bin.bat b/create_bin.bat new file mode 100644 index 0000000..a5b897f --- /dev/null +++ b/create_bin.bat @@ -0,0 +1,16 @@ +cd .. +mkdir bin\deps\bson\ebin +mkdir bin\deps\mongodb\ebin +mkdir bin\deps\kdht\ebin +mkdir bin\deps\ibrowse\ebin +copy deps\bson\ebin\*.* bin\deps\bson\ebin\ +copy deps\mongodb\ebin\*.* bin\deps\mongodb\ebin\ +copy deps\kdht\ebin\*.* bin\deps\kdht\ebin\ +copy deps\ibrowse\ebin\*.* bin\deps\ibrowse\ebin\ +mkdir bin\www +copy www\*.* bin\www\ +copy tools\*.* bin\ +mkdir bin\priv +mkdir bin\ebin +copy ebin\*.* bin\ebin\ +pause diff --git a/deps/bson/ebin/bson.app b/deps/bson/ebin/bson.app new file mode 100644 index 0000000..9fa5b43 --- /dev/null +++ b/deps/bson/ebin/bson.app @@ -0,0 +1,7 @@ +{application, bson, + [{description, "BSON are JSON-like objects with a standard binary serialization. See bsonspec.org."}, + {vsn, "0"}, + {modules, [bson, bson_binary, bson_tests]}, + {registered, []}, + {applications, [kernel, stdlib]} + ]}. diff --git a/deps/bson/ebin/bson.beam b/deps/bson/ebin/bson.beam new file mode 100644 index 0000000..af6f0bf Binary files /dev/null and b/deps/bson/ebin/bson.beam differ diff --git a/deps/bson/ebin/bson_binary.beam b/deps/bson/ebin/bson_binary.beam new file mode 100644 index 0000000..c21b23f Binary files /dev/null and b/deps/bson/ebin/bson_binary.beam differ diff --git a/deps/bson/ebin/bson_tests.beam b/deps/bson/ebin/bson_tests.beam new file mode 100644 index 0000000..9f8a5f5 Binary files /dev/null and b/deps/bson/ebin/bson_tests.beam differ diff --git a/deps/ibrowse/ebin/ibrowse.app b/deps/ibrowse/ebin/ibrowse.app new file mode 100644 index 0000000..e537fa0 --- /dev/null +++ b/deps/ibrowse/ebin/ibrowse.app @@ -0,0 +1,9 @@ +{application,ibrowse, + [{description,"Erlang HTTP client application"}, + {vsn,"4.0.1"}, + {registered,[ibrowse_sup,ibrowse]}, + {applications,[kernel,stdlib]}, + {env,[]}, + {mod,{ibrowse_app,[]}}, + {modules,[ibrowse,ibrowse_app,ibrowse_http_client,ibrowse_lb, + ibrowse_lib,ibrowse_sup]}]}. diff --git a/deps/ibrowse/ebin/ibrowse.beam b/deps/ibrowse/ebin/ibrowse.beam new file mode 100644 index 0000000..54e2aa6 Binary files /dev/null and b/deps/ibrowse/ebin/ibrowse.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_app.beam b/deps/ibrowse/ebin/ibrowse_app.beam new file mode 100644 index 0000000..62fb1d5 Binary files /dev/null and b/deps/ibrowse/ebin/ibrowse_app.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_http_client.beam b/deps/ibrowse/ebin/ibrowse_http_client.beam new file mode 100644 index 0000000..34ed0ad Binary files /dev/null and b/deps/ibrowse/ebin/ibrowse_http_client.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_lb.beam b/deps/ibrowse/ebin/ibrowse_lb.beam new file mode 100644 index 0000000..e920334 Binary files /dev/null and b/deps/ibrowse/ebin/ibrowse_lb.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_lib.beam b/deps/ibrowse/ebin/ibrowse_lib.beam new file mode 100644 index 0000000..1f9d905 Binary files /dev/null and b/deps/ibrowse/ebin/ibrowse_lib.beam differ diff --git a/deps/ibrowse/ebin/ibrowse_sup.beam b/deps/ibrowse/ebin/ibrowse_sup.beam new file mode 100644 index 0000000..0f74506 Binary files /dev/null and b/deps/ibrowse/ebin/ibrowse_sup.beam differ diff --git a/deps/kdht/ebin/.gitignore b/deps/kdht/ebin/.gitignore new file mode 100644 index 0000000..17278c0 --- /dev/null +++ b/deps/kdht/ebin/.gitignore @@ -0,0 +1 @@ +*.beam diff --git a/deps/kdht/ebin/kdht.app b/deps/kdht/ebin/kdht.app new file mode 100644 index 0000000..d898285 --- /dev/null +++ b/deps/kdht/ebin/kdht.app @@ -0,0 +1,10 @@ +{application, kdht, [ + {description, "DHT library"}, + {vsn, git}, + {registered, []}, + {applications, [kernel, stdlib]}, + {modules, [bencode, bucket, dht_id, dht_net, dht_state, kdht_sup, msg, + search, storage, timer_monitor, vlog]}, + {mod, {}} +]}. + diff --git a/deps/mongodb/ebin/mongo.beam b/deps/mongodb/ebin/mongo.beam new file mode 100644 index 0000000..b7c4d61 Binary files /dev/null and b/deps/mongodb/ebin/mongo.beam differ diff --git a/deps/mongodb/ebin/mongo_app.beam b/deps/mongodb/ebin/mongo_app.beam new file mode 100644 index 0000000..190225e Binary files /dev/null and b/deps/mongodb/ebin/mongo_app.beam differ diff --git a/deps/mongodb/ebin/mongo_connection.beam b/deps/mongodb/ebin/mongo_connection.beam new file mode 100644 index 0000000..982951d Binary files /dev/null and b/deps/mongodb/ebin/mongo_connection.beam differ diff --git a/deps/mongodb/ebin/mongo_cursor.beam b/deps/mongodb/ebin/mongo_cursor.beam new file mode 100644 index 0000000..58cf89e Binary files /dev/null and b/deps/mongodb/ebin/mongo_cursor.beam differ diff --git a/deps/mongodb/ebin/mongo_id_server.beam b/deps/mongodb/ebin/mongo_id_server.beam new file mode 100644 index 0000000..ef86195 Binary files /dev/null and b/deps/mongodb/ebin/mongo_id_server.beam differ diff --git a/deps/mongodb/ebin/mongo_pool.beam b/deps/mongodb/ebin/mongo_pool.beam new file mode 100644 index 0000000..d4a7d6d Binary files /dev/null and b/deps/mongodb/ebin/mongo_pool.beam differ diff --git a/deps/mongodb/ebin/mongo_protocol.beam b/deps/mongodb/ebin/mongo_protocol.beam new file mode 100644 index 0000000..3ecf7b7 Binary files /dev/null and b/deps/mongodb/ebin/mongo_protocol.beam differ diff --git a/deps/mongodb/ebin/mongo_sup.beam b/deps/mongodb/ebin/mongo_sup.beam new file mode 100644 index 0000000..ce8c0e7 Binary files /dev/null and b/deps/mongodb/ebin/mongo_sup.beam differ diff --git a/deps/mongodb/ebin/mongodb.app b/deps/mongodb/ebin/mongodb.app new file mode 100644 index 0000000..754e8db --- /dev/null +++ b/deps/mongodb/ebin/mongodb.app @@ -0,0 +1,9 @@ +{application,mongodb, + [{description,"Client interface to MongoDB, also known as the driver. See www.mongodb.org"}, + {vsn,"v0.3.1-96-g613f157"}, + {registered,[]}, + {applications,[kernel,stdlib,bson]}, + {mod,{mongo_app,[]}}, + {modules,[mongo,mongo_app,mongo_connection,mongo_cursor, + mongo_id_server,mongo_pool,mongo_protocol, + mongo_sup]}]}. diff --git a/ebin/bt_conn.beam b/ebin/bt_conn.beam new file mode 100644 index 0000000..6a1d74a Binary files /dev/null and b/ebin/bt_conn.beam differ diff --git a/ebin/bt_message.beam b/ebin/bt_message.beam new file mode 100644 index 0000000..cc4beb2 Binary files /dev/null and b/ebin/bt_message.beam differ diff --git a/ebin/crawler_app.beam b/ebin/crawler_app.beam new file mode 100644 index 0000000..e770ddd Binary files /dev/null and b/ebin/crawler_app.beam differ diff --git a/ebin/crawler_http.beam b/ebin/crawler_http.beam new file mode 100644 index 0000000..29b75be Binary files /dev/null and b/ebin/crawler_http.beam differ diff --git a/ebin/crawler_stats.beam b/ebin/crawler_stats.beam new file mode 100644 index 0000000..a1c860d Binary files /dev/null and b/ebin/crawler_stats.beam differ diff --git a/ebin/crawler_sup.beam b/ebin/crawler_sup.beam new file mode 100644 index 0000000..7cb6c68 Binary files /dev/null and b/ebin/crawler_sup.beam differ diff --git a/ebin/db_hash.beam b/ebin/db_hash.beam new file mode 100644 index 0000000..a959f7a Binary files /dev/null and b/ebin/db_hash.beam differ diff --git a/ebin/db_hash_reader.beam b/ebin/db_hash_reader.beam new file mode 100644 index 0000000..6f58b15 Binary files /dev/null and b/ebin/db_hash_reader.beam differ diff --git a/ebin/db_hash_reader_sup.beam b/ebin/db_hash_reader_sup.beam new file mode 100644 index 0000000..2f7b2e9 Binary files /dev/null and b/ebin/db_hash_reader_sup.beam differ diff --git a/ebin/db_store_mongo.beam b/ebin/db_store_mongo.beam new file mode 100644 index 0000000..05e4314 Binary files /dev/null and b/ebin/db_store_mongo.beam differ diff --git a/ebin/db_system.beam b/ebin/db_system.beam new file mode 100644 index 0000000..045e0f4 Binary files /dev/null and b/ebin/db_system.beam differ diff --git a/ebin/dht_monitor.beam b/ebin/dht_monitor.beam new file mode 100644 index 0000000..534f6d8 Binary files /dev/null and b/ebin/dht_monitor.beam differ diff --git a/ebin/dhtcrawler.app b/ebin/dhtcrawler.app new file mode 100644 index 0000000..000a14a --- /dev/null +++ b/ebin/dhtcrawler.app @@ -0,0 +1,13 @@ +{application,dhtcrawler, + [{description,"A DHT crawler to index magnet hash to torrent"}, + {vsn,"6dde34b"}, + {registered,[dht_crawler_sup]}, + {applications,[kernel,stdlib,crypto,public_key,ssl,inets,bson, + mongodb]}, + {mod,{crawler_app,[]}}, + {modules,[bt_conn,bt_message,crawler_app,crawler_http, + crawler_stats,crawler_sup,db_hash,db_hash_reader, + db_hash_reader_sup,db_store_mongo,db_system, + dht_monitor,hash_reader_stats,http_handler, + string_split,time_util,tor_download, + tor_download_stats,torrent_file,urldecode,vlog]}]}. diff --git a/ebin/hash_reader_stats.beam b/ebin/hash_reader_stats.beam new file mode 100644 index 0000000..1d96aa3 Binary files /dev/null and b/ebin/hash_reader_stats.beam differ diff --git a/ebin/http_handler.beam b/ebin/http_handler.beam new file mode 100644 index 0000000..abed71c Binary files /dev/null and b/ebin/http_handler.beam differ diff --git a/ebin/string_split.beam b/ebin/string_split.beam new file mode 100644 index 0000000..5a2a483 Binary files /dev/null and b/ebin/string_split.beam differ diff --git a/ebin/time_util.beam b/ebin/time_util.beam new file mode 100644 index 0000000..20abc1b Binary files /dev/null and b/ebin/time_util.beam differ diff --git a/ebin/tor_download.beam b/ebin/tor_download.beam new file mode 100644 index 0000000..755c58d Binary files /dev/null and b/ebin/tor_download.beam differ diff --git a/ebin/tor_download_stats.beam b/ebin/tor_download_stats.beam new file mode 100644 index 0000000..9840142 Binary files /dev/null and b/ebin/tor_download_stats.beam differ diff --git a/ebin/torrent_file.beam b/ebin/torrent_file.beam new file mode 100644 index 0000000..a3c167b Binary files /dev/null and b/ebin/torrent_file.beam differ diff --git a/ebin/urldecode.beam b/ebin/urldecode.beam new file mode 100644 index 0000000..189760e Binary files /dev/null and b/ebin/urldecode.beam differ diff --git a/ebin/vlog.beam b/ebin/vlog.beam new file mode 100644 index 0000000..950fe60 Binary files /dev/null and b/ebin/vlog.beam differ diff --git a/win_start_crawler.bat b/win_start_crawler.bat new file mode 100644 index 0000000..c7cbd08 --- /dev/null +++ b/win_start_crawler.bat @@ -0,0 +1,2 @@ +erl -pa ebin -noshell -s crawler_app start + diff --git a/win_start_hash.bat b/win_start_hash.bat new file mode 100644 index 0000000..67d86bf --- /dev/null +++ b/win_start_hash.bat @@ -0,0 +1 @@ +erl -pa ebin -noshell -s db_hash_reader_sup start_standalone localhost 27017 10 diff --git a/win_start_http.bat b/win_start_http.bat new file mode 100644 index 0000000..caa6e2f --- /dev/null +++ b/win_start_http.bat @@ -0,0 +1,2 @@ +erl -pa ebin -noshell -s crawler_http start + diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..a2b9dbf --- /dev/null +++ b/www/index.html @@ -0,0 +1,66 @@ + + + + +Yet Another Magnet Search Engine + + + +
+ +
+
+
+ + +
+
+
+

+Try AVI + American + Iron Man +

+
+ + +
+
+ + + diff --git a/www/page.temp b/www/page.temp new file mode 100644 index 0000000..9836b3d --- /dev/null +++ b/www/page.temp @@ -0,0 +1,59 @@ + + + + +Yet Another Magnet Search Engine + + + +
+ +
+
+
+ + +
+
+
+~s +
+ +
+
+ + +