add rmmseg, to segment chinese texts, add a tool to convert the existing torrent file names

11 years ago · 269584c708
parent 676d354515
commit 269584c708
66 changed files with 132952 additions and 5 deletions
--- a/deps/bson/ebin/bson.beam
+++ b/deps/bson/ebin/bson.beam
--- a/deps/bson/ebin/bson_binary.beam
+++ b/deps/bson/ebin/bson_binary.beam
--- a/deps/bson/ebin/bson_tests.beam
+++ b/deps/bson/ebin/bson_tests.beam
--- a/deps/ibrowse/ebin/ibrowse.beam
+++ b/deps/ibrowse/ebin/ibrowse.beam
--- a/deps/ibrowse/ebin/ibrowse_app.beam
+++ b/deps/ibrowse/ebin/ibrowse_app.beam
--- a/deps/ibrowse/ebin/ibrowse_http_client.beam
+++ b/deps/ibrowse/ebin/ibrowse_http_client.beam
--- a/deps/ibrowse/ebin/ibrowse_lb.beam
+++ b/deps/ibrowse/ebin/ibrowse_lb.beam
--- a/deps/ibrowse/ebin/ibrowse_lib.beam
+++ b/deps/ibrowse/ebin/ibrowse_lib.beam
--- a/deps/ibrowse/ebin/ibrowse_sup.beam
+++ b/deps/ibrowse/ebin/ibrowse_sup.beam
--- a/deps/kdht/ebin/bencode.beam
+++ b/deps/kdht/ebin/bencode.beam
--- a/deps/kdht/ebin/bucket.beam
+++ b/deps/kdht/ebin/bucket.beam
--- a/deps/kdht/ebin/dht_id.beam
+++ b/deps/kdht/ebin/dht_id.beam
--- a/deps/kdht/ebin/dht_net.beam
+++ b/deps/kdht/ebin/dht_net.beam
--- a/deps/kdht/ebin/dht_state.beam
+++ b/deps/kdht/ebin/dht_state.beam
--- a/deps/kdht/ebin/kdht_sup.beam
+++ b/deps/kdht/ebin/kdht_sup.beam
--- a/deps/kdht/ebin/msg.beam
+++ b/deps/kdht/ebin/msg.beam
--- a/deps/kdht/ebin/search.beam
+++ b/deps/kdht/ebin/search.beam
--- a/deps/kdht/ebin/storage.beam
+++ b/deps/kdht/ebin/storage.beam
--- a/deps/kdht/ebin/timer_monitor.beam
+++ b/deps/kdht/ebin/timer_monitor.beam
--- a/deps/kdht/ebin/vlog.beam
+++ b/deps/kdht/ebin/vlog.beam
--- a/deps/mongodb/ebin/mongo.beam
+++ b/deps/mongodb/ebin/mongo.beam
--- a/deps/mongodb/ebin/mongo_app.beam
+++ b/deps/mongodb/ebin/mongo_app.beam
--- a/deps/mongodb/ebin/mongo_connection.beam
+++ b/deps/mongodb/ebin/mongo_connection.beam
--- a/deps/mongodb/ebin/mongo_cursor.beam
+++ b/deps/mongodb/ebin/mongo_cursor.beam
--- a/deps/mongodb/ebin/mongo_id_server.beam
+++ b/deps/mongodb/ebin/mongo_id_server.beam
--- a/deps/mongodb/ebin/mongo_pool.beam
+++ b/deps/mongodb/ebin/mongo_pool.beam
--- a/deps/mongodb/ebin/mongo_protocol.beam
+++ b/deps/mongodb/ebin/mongo_protocol.beam
--- a/deps/mongodb/ebin/mongo_sup.beam
+++ b/deps/mongodb/ebin/mongo_sup.beam
--- a/ebin/bt_conn.beam
+++ b/ebin/bt_conn.beam
--- a/ebin/bt_message.beam
+++ b/ebin/bt_message.beam
--- a/ebin/config.beam
+++ b/ebin/config.beam
--- a/ebin/crawler_app.beam
+++ b/ebin/crawler_app.beam
--- a/ebin/crawler_http.beam
+++ b/ebin/crawler_http.beam
--- a/ebin/crawler_stats.beam
+++ b/ebin/crawler_stats.beam
--- a/ebin/crawler_sup.beam
+++ b/ebin/crawler_sup.beam
--- a/ebin/db_daterange.beam
+++ b/ebin/db_daterange.beam
--- a/ebin/db_frontend.beam
+++ b/ebin/db_frontend.beam
--- a/ebin/db_hash.beam
+++ b/ebin/db_hash.beam
--- a/ebin/db_loc_torrent.beam
+++ b/ebin/db_loc_torrent.beam
--- a/ebin/db_store_mongo.beam
+++ b/ebin/db_store_mongo.beam
--- a/ebin/db_system.beam
+++ b/ebin/db_system.beam
--- a/ebin/dht_monitor.beam
+++ b/ebin/dht_monitor.beam
--- a/ebin/dhtcrawler.app
+++ b/ebin/dhtcrawler.app
@ -1,6 +1,6 @@
 {application,dhtcrawler,
             [{description,"A DHT crawler to index magnet hash to torrent"},
-              {vsn,"84ec8fd"},
+              {vsn,"77893f0"},
              {registered,[dht_crawler_sup]},
              {applications,[kernel,stdlib,crypto,public_key,ssl,inets,bson,
                             mongodb]},
@ -10,7 +10,8 @@
                        db_hash,db_loc_torrent,db_store_mongo,db_system,
                        dht_monitor,hash_reader,hash_reader_stats,
                        hash_reader_sup,http_cache,http_handler,loc_torrent,
-                        loc_torrent_cache,loc_torrent_sup,string_split,
-                        time_util,tor_builder,tor_download,tor_download_stats,
-                        tor_location_reader,tor_location_writer,torrent_file,
-                        urldecode,vlog]}]}.
+                        loc_torrent_cache,loc_torrent_sup,name_seger,rmmseg,
+                        string_split,time_util,tor_builder,tor_download,
+                        tor_download_stats,tor_location,tor_location_reader,
+                        tor_location_writer,torrent_file,transfer,urldecode,
+                        vlog]}]}.
--- a/ebin/hash_reader.beam
+++ b/ebin/hash_reader.beam
--- a/ebin/hash_reader_stats.beam
+++ b/ebin/hash_reader_stats.beam
--- a/ebin/hash_reader_sup.beam
+++ b/ebin/hash_reader_sup.beam
--- a/ebin/http_cache.beam
+++ b/ebin/http_cache.beam
--- a/ebin/http_handler.beam
+++ b/ebin/http_handler.beam
--- a/ebin/loc_torrent.beam
+++ b/ebin/loc_torrent.beam
--- a/ebin/loc_torrent_cache.beam
+++ b/ebin/loc_torrent_cache.beam
--- a/ebin/loc_torrent_sup.beam
+++ b/ebin/loc_torrent_sup.beam
--- a/ebin/name_seger.beam
+++ b/ebin/name_seger.beam
--- a/ebin/rmmseg.beam
+++ b/ebin/rmmseg.beam
--- a/ebin/string_split.beam
+++ b/ebin/string_split.beam
--- a/ebin/time_util.beam
+++ b/ebin/time_util.beam
--- a/ebin/tor_builder.beam
+++ b/ebin/tor_builder.beam
--- a/ebin/tor_download.beam
+++ b/ebin/tor_download.beam
--- a/ebin/tor_download_stats.beam
+++ b/ebin/tor_download_stats.beam
--- a/ebin/tor_location_reader.beam
+++ b/ebin/tor_location_reader.beam
--- a/ebin/tor_location_writer.beam
+++ b/ebin/tor_location_writer.beam
--- a/ebin/torrent_file.beam
+++ b/ebin/torrent_file.beam
--- a/ebin/urldecode.beam
+++ b/ebin/urldecode.beam
--- a/ebin/vlog.beam
+++ b/ebin/vlog.beam
--- a/priv/chars.dic
+++ b/priv/chars.dic
--- a/priv/rmmseg.dll
+++ b/priv/rmmseg.dll
--- a/priv/words.dic
+++ b/priv/words.dic