first commit
Kevin Lynx
11 years ago
commit
4f7fe8805e
@ -0,0 +1,18 @@
|
||||
## dhtcrawler2
|
||||
|
||||
This git branch maintain pre-compiled erlang files to start dhtcrawler2 directly.
|
||||
|
||||
## Usage
|
||||
|
||||
* install Erlang R16B or newer
|
||||
* download mongodb and start mongodb first
|
||||
|
||||
mongod --dbpath your-database-path --setParameter textSearchEnabled=true
|
||||
|
||||
* start **crawler**, on Windows, just click `win_start_crawler.bat`
|
||||
* start **hash_reader**, on Windows, just click `win_start_hash.bat`
|
||||
* start **httpd**, on Windows, just click `win_start_http.bat`
|
||||
* wait several minutes and checkout `localhost:8000`
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,16 @@
|
||||
cd ..
|
||||
mkdir bin\deps\bson\ebin
|
||||
mkdir bin\deps\mongodb\ebin
|
||||
mkdir bin\deps\kdht\ebin
|
||||
mkdir bin\deps\ibrowse\ebin
|
||||
copy deps\bson\ebin\*.* bin\deps\bson\ebin\
|
||||
copy deps\mongodb\ebin\*.* bin\deps\mongodb\ebin\
|
||||
copy deps\kdht\ebin\*.* bin\deps\kdht\ebin\
|
||||
copy deps\ibrowse\ebin\*.* bin\deps\ibrowse\ebin\
|
||||
mkdir bin\www
|
||||
copy www\*.* bin\www\
|
||||
copy tools\*.* bin\
|
||||
mkdir bin\priv
|
||||
mkdir bin\ebin
|
||||
copy ebin\*.* bin\ebin\
|
||||
pause
|
||||
@ -0,0 +1,7 @@
|
||||
{application, bson,
|
||||
[{description, "BSON are JSON-like objects with a standard binary serialization. See bsonspec.org."},
|
||||
{vsn, "0"},
|
||||
{modules, [bson, bson_binary, bson_tests]},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib]}
|
||||
]}.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,9 @@
|
||||
{application,ibrowse,
|
||||
[{description,"Erlang HTTP client application"},
|
||||
{vsn,"4.0.1"},
|
||||
{registered,[ibrowse_sup,ibrowse]},
|
||||
{applications,[kernel,stdlib]},
|
||||
{env,[]},
|
||||
{mod,{ibrowse_app,[]}},
|
||||
{modules,[ibrowse,ibrowse_app,ibrowse_http_client,ibrowse_lb,
|
||||
ibrowse_lib,ibrowse_sup]}]}.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1 @@
|
||||
*.beam
|
||||
@ -0,0 +1,10 @@
|
||||
{application, kdht, [
|
||||
{description, "DHT library"},
|
||||
{vsn, git},
|
||||
{registered, []},
|
||||
{applications, [kernel, stdlib]},
|
||||
{modules, [bencode, bucket, dht_id, dht_net, dht_state, kdht_sup, msg,
|
||||
search, storage, timer_monitor, vlog]},
|
||||
{mod, {}}
|
||||
]}.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,9 @@
|
||||
{application,mongodb,
|
||||
[{description,"Client interface to MongoDB, also known as the driver. See www.mongodb.org"},
|
||||
{vsn,"v0.3.1-96-g613f157"},
|
||||
{registered,[]},
|
||||
{applications,[kernel,stdlib,bson]},
|
||||
{mod,{mongo_app,[]}},
|
||||
{modules,[mongo,mongo_app,mongo_connection,mongo_cursor,
|
||||
mongo_id_server,mongo_pool,mongo_protocol,
|
||||
mongo_sup]}]}.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,13 @@
|
||||
{application,dhtcrawler,
|
||||
[{description,"A DHT crawler to index magnet hash to torrent"},
|
||||
{vsn,"6dde34b"},
|
||||
{registered,[dht_crawler_sup]},
|
||||
{applications,[kernel,stdlib,crypto,public_key,ssl,inets,bson,
|
||||
mongodb]},
|
||||
{mod,{crawler_app,[]}},
|
||||
{modules,[bt_conn,bt_message,crawler_app,crawler_http,
|
||||
crawler_stats,crawler_sup,db_hash,db_hash_reader,
|
||||
db_hash_reader_sup,db_store_mongo,db_system,
|
||||
dht_monitor,hash_reader_stats,http_handler,
|
||||
string_split,time_util,tor_download,
|
||||
tor_download_stats,torrent_file,urldecode,vlog]}]}.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@ -0,0 +1,2 @@
|
||||
erl -pa ebin -noshell -s crawler_app start
|
||||
|
||||
@ -0,0 +1 @@
|
||||
erl -pa ebin -noshell -s db_hash_reader_sup start_standalone localhost 27017 10
|
||||
@ -0,0 +1,2 @@
|
||||
erl -pa ebin -noshell -s crawler_http start
|
||||
|
||||
@ -0,0 +1,66 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Yet Another Magnet Search Engine</title>
|
||||
<style type="text/css" media="screen">
|
||||
body { margin:0; padding:0; }
|
||||
#wrap {
|
||||
width: 980px;
|
||||
margin: 5px auto;
|
||||
}
|
||||
#header {
|
||||
height: 30px;
|
||||
}
|
||||
#main {
|
||||
margin: 20px 10px;
|
||||
}
|
||||
#content {
|
||||
min-height: 130px;
|
||||
padding: 20px 5px;
|
||||
}
|
||||
#footer{
|
||||
text-align:center;
|
||||
font-size: 13px;
|
||||
margin-top: 20px;
|
||||
padding-top: 5px;
|
||||
border-top: 1px solid #888;
|
||||
}
|
||||
.search-box #s_text {
|
||||
width: 200px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="wrap">
|
||||
<div id="header">
|
||||
<h2><a style="text-decoration:none;" href="/">Magnet Search Engine</a></h2>
|
||||
</div>
|
||||
<div id="main">
|
||||
<div class="search-box">
|
||||
<form id="s_box" method="get" action="/e/http_handler:search" >
|
||||
<input id="s_text" name="q" type="text" />
|
||||
<input id="s_submit" value="Search" type="submit" />
|
||||
</form>
|
||||
</div>
|
||||
<div id="content">
|
||||
<p>
|
||||
Try <a href="/e/http_handler:search?q=avi">AVI</a>
|
||||
<a href="/e/http_handler:search?q=american">American</a>
|
||||
<a href="/e/http_handler:search?q=iron+man">Iron Man</a>
|
||||
</p>
|
||||
</div>
|
||||
<div id="footer">
|
||||
<a href="/e/http_handler:recent">New</a>
|
||||
<a href="/e/http_handler:top">Top</a>
|
||||
<a href="/e/http_handler:stats">Stats</a>
|
||||
<span> © Kevin Lynx 2013</span>
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
document.getElementsByTagName('input')[0].focus();
|
||||
</script>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@ -0,0 +1,59 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Yet Another Magnet Search Engine</title>
|
||||
<style type="text/css" media="screen">
|
||||
body { margin:0; padding:0; }
|
||||
#wrap {
|
||||
width: 980px;
|
||||
margin: 5px auto;
|
||||
}
|
||||
#header {
|
||||
height: 30px;
|
||||
}
|
||||
#main {
|
||||
margin: 20px 10px;
|
||||
}
|
||||
#content {
|
||||
min-height: 130px;
|
||||
padding: 20px 5px;
|
||||
}
|
||||
#footer{
|
||||
text-align:center;
|
||||
font-size: 13px;
|
||||
margin-top: 20px;
|
||||
padding-top: 5px;
|
||||
border-top: 1px solid #888;
|
||||
}
|
||||
.search-box #s_text {
|
||||
width: 200px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="wrap">
|
||||
<div id="header">
|
||||
<h2><a style="text-decoration:none;" href="/">Magnet Search Engine</a></h2>
|
||||
</div>
|
||||
<div id="main">
|
||||
<div class="search-box">
|
||||
<form id="s_box" method="get" action="/e/http_handler:search" >
|
||||
<input id="s_text" value="~s" name="q" type="text" />
|
||||
<input id="s_submit" value="Search" type="submit" />
|
||||
</form>
|
||||
</div>
|
||||
<div id="content">
|
||||
~s
|
||||
</div>
|
||||
<div id="footer">
|
||||
<a href="/e/http_handler:recent">New</a>
|
||||
<a href="/e/http_handler:top">Top</a>
|
||||
<a href="/e/http_handler:stats">Stats</a>
|
||||
<span> © Kevin Lynx 2013</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Loading…
Reference in New Issue