Done major code changes and reviews
danieleperera
4 years ago
parent
b76af6d57e
commit
e2b7c8346d
@ -1,86 +0,0 @@
|
||||
# This is an example ThreatIngestor config file with some preconfigured RSS
|
||||
# sources, feeding extracted artifacts into a CSV file.
|
||||
|
||||
general:
|
||||
# Run forever, check feeds once an hour.
|
||||
daemon: True
|
||||
sleep: 10
|
||||
onion_validation: ([a-z2-7]{16,56}\.onion)
|
||||
blacklist: pedo,xxx,infant,loli,porn,child,abuse,sex,drug,cocaine,dope,zoo,daddy,daughter,boy,girl,young,muder,cocks,year,old
|
||||
interestingKeywords: t.me,rss,xml,atom,dataleak,breach,blog,ransomware,source code,data breach,bank,logs,hack
|
||||
elasticsearch:
|
||||
index: darkweb
|
||||
port : 9200
|
||||
host : 127.0.0.1
|
||||
|
||||
sources:
|
||||
# A few threat intel blogs to get you started!
|
||||
- name: simple-text-file
|
||||
module: simplefile
|
||||
filename: onion_master_list.txt
|
||||
|
||||
# - name: source-gist
|
||||
# module: gist
|
||||
# url: https://gist.github.com/search?l=Text&q=.onion
|
||||
|
||||
# - name: source-reddit
|
||||
# module: reddit
|
||||
# url: https://api.pushshift.io/reddit/search/comment/?subreddit=onions&limit=1000000
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: pastebin
|
||||
# module: pastebin-account
|
||||
# url: https://gist.github.com/search?l=Text&q=.onion
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: hunchly-report
|
||||
# module: gmail-hunchly
|
||||
# url: https://gist.github.com/search?l=Text&q=.onion
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: onionland-search
|
||||
# module: collect-onions
|
||||
# url: http://3bbaaaccczcbdddz.onion/discover
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: torch
|
||||
# module: collect-onions
|
||||
# url: http://xmh57jrzrnw6insl.onion
|
||||
# feed_type: messy
|
||||
|
||||
|
||||
operators:
|
||||
- name: simple-html
|
||||
module: html
|
||||
socks5:
|
||||
http: 'socks5h://127.0.0.1:9050'
|
||||
https: 'socks5h://127.0.0.1:9050'
|
||||
TorController:
|
||||
port: 9051
|
||||
password: your-torcontroller-password-here
|
||||
|
||||
- name: simple-screenshot
|
||||
module: screenshot
|
||||
screenshots_path: null
|
||||
|
||||
- name: onionscan-go
|
||||
module: onionscan
|
||||
binpath: /home/tony/go/bin/onionscan
|
||||
|
||||
|
||||
# - name: yara-rule
|
||||
# module: yara
|
||||
# filename: categories.yar
|
||||
# base_score: 50
|
||||
#
|
||||
# - name: regex-match
|
||||
# module: regex
|
||||
# keywords: test,test2
|
||||
# base_score: 20
|
||||
|
||||
notifiers:
|
||||
# Simple telegram notifier
|
||||
- name: telegram-notifer
|
||||
module: telegram
|
||||
chat_id:
|
||||
token:
|
||||
@ -0,0 +1,106 @@
|
||||
# This is an example OnionIngestor config file with some preconfigured configurations
|
||||
# Storage Engines elasticsearch and telegram are configured
|
||||
|
||||
general:
|
||||
# Run forever, check feeds once an hour.
|
||||
daemon: True
|
||||
sleep: 10
|
||||
onion_validation: ([a-z2-7]{16,56}\.onion)
|
||||
blacklist: pedo,porn,child
|
||||
interestingKeywords: your,keywords,here
|
||||
save-thread: no # Use a separate thread to save onions
|
||||
|
||||
monitor:
|
||||
filename: monitoring.txt
|
||||
|
||||
sources:
|
||||
# A few threat intel blogs to get you started!
|
||||
- name: simple-text-file
|
||||
module: simplefile
|
||||
filename: onion_master_list.txt
|
||||
|
||||
# - name: source-gist
|
||||
# module: gist
|
||||
# url: https://gist.github.com/search?l=Text&q=.onion
|
||||
|
||||
# - name: source-reddit
|
||||
# module: reddit
|
||||
# url: https://api.pushshift.io/reddit/search/comment/?subreddit=onions&limit=1000000
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: pastebin
|
||||
# module: pastebin-account
|
||||
# url: https://gist.github.com/search?l=Text&q=.onion
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: hunchly-report
|
||||
# module: gmail-hunchly
|
||||
# url: https://gist.github.com/search?l=Text&q=.onion
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: onionland-search
|
||||
# module: collect-onions
|
||||
# url: http://3bbaaaccczcbdddz.onion/discover
|
||||
# feed_type: messy
|
||||
#
|
||||
# - name: torch
|
||||
# module: collect-onions
|
||||
# url: http://xmh57jrzrnw6insl.onion
|
||||
# feed_type: messy
|
||||
|
||||
|
||||
operators:
|
||||
- name: simple-html
|
||||
module: html
|
||||
timeout: 300
|
||||
retries: 2
|
||||
interestingKeywords: your,keywords,here
|
||||
socks5:
|
||||
http: 'socks5h://127.0.0.1:9050'
|
||||
https: 'socks5h://127.0.0.1:9050'
|
||||
TorController:
|
||||
port: 9051
|
||||
password: your-tor-controller-password
|
||||
|
||||
- name: onionscan-go
|
||||
module: onionscan
|
||||
binpath: your-onionscan-binary-path
|
||||
|
||||
- name: simple-screenshot
|
||||
module: screenshot
|
||||
screenshots_path: null
|
||||
|
||||
# - name: yara-rule
|
||||
# module: yara
|
||||
# filename: categories.yar
|
||||
# base_score: 50
|
||||
#
|
||||
# - name: regex-match
|
||||
# module: regex
|
||||
# keywords: test,test2
|
||||
# base_score: 20
|
||||
|
||||
database_Engines:
|
||||
- name: telegram-notifer #Simple Telegram notifier
|
||||
module: telegram
|
||||
chat_id: your-telegram-chat-id
|
||||
token: your-telegram-token
|
||||
|
||||
- name: elasticsearch
|
||||
module: elasticsearch
|
||||
index: your-index-name
|
||||
port : 9200
|
||||
host : 127.0.0.1
|
||||
|
||||
# - name: email
|
||||
# module: send_email
|
||||
# alert: no # Enable/disable email alerts
|
||||
# from: alert@example.com
|
||||
# to: alert@example.com
|
||||
# server: 127.0.0.1 # Address of the server (hostname or IP)
|
||||
# port: 25 # Outgoing SMTP port: 25, 587, ...
|
||||
# tls: no # Enable/disable tls support
|
||||
# username: '' # (optional) Username for authentication. Leave blank for no authentication.
|
||||
# password: '' # (optional) Password for authentication. Leave blank for no authentication.
|
||||
# subject: '[onioningestor] - {subject}'
|
||||
# size-limit: 1048576 # Size limit for pastie, above it's sent as attachement
|
||||
@ -0,0 +1,149 @@
|
||||
import sys
|
||||
import time
|
||||
import schedule
|
||||
import threading
|
||||
|
||||
class StorageScheduler():
|
||||
def __init__(self, storage, **kwargs):
|
||||
self.storage = storage
|
||||
self.name = self.storage.name
|
||||
|
||||
def save_pastie(self, pastie, timeout):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class StorageSync(StorageScheduler):
|
||||
### synchronously save onions ###
|
||||
def save_pastie(self, pastie, timeout):
|
||||
self.storage.save_pastie(pastie)
|
||||
|
||||
|
||||
# LATER: implement an async class
|
||||
class StorageThread(threading.Thread, StorageScheduler):
|
||||
def __init__(self, logger, storage, **kwargs):
|
||||
threading.Thread.__init__(self)
|
||||
StorageScheduler.__init__(self, storage, **kwargs)
|
||||
self.logger = logger
|
||||
try:
|
||||
size = int(kwargs['queue_size'])
|
||||
except Exception:
|
||||
size = 0
|
||||
self.queue = Queue(size)
|
||||
self.kill_received = False
|
||||
|
||||
def run(self):
|
||||
self.logger.info('{0}: Thread for saving pasties started'.format(self.name))
|
||||
# loop over the queue
|
||||
while not self.kill_received:
|
||||
# pastie = None
|
||||
try:
|
||||
# grabs pastie from queue
|
||||
pastie = self.queue.get(True, 5)
|
||||
# save the pasties in each storage
|
||||
self.storage.save_pastie(pastie)
|
||||
except Empty:
|
||||
pass
|
||||
# catch unknown errors
|
||||
except Exception as e:
|
||||
self.logger.error("{0}: Thread for saving pasties crashed unexpectectly, recovering...: {1}".format(self.name, e))
|
||||
self.logger.debug(traceback.format_exc())
|
||||
finally:
|
||||
# to be on the safe side of gf
|
||||
del(pastie)
|
||||
# signals to queue job is done
|
||||
self.queue.task_done()
|
||||
self.logger.info('{0}: Thread for saving pasties terminated'.format(self.name))
|
||||
|
||||
def save_pastie(self, pastie, timeout):
|
||||
try:
|
||||
self.logger.debug('{0}: queueing pastie {1} for saving'.format(self.name, pastie.url))
|
||||
self.queue.put(pastie, True, timeout)
|
||||
except Full:
|
||||
self.logger.error('{0}: unable to save pastie[{1}]: queue is full'.format(self.name, pastie.url))
|
||||
|
||||
|
||||
class StorageDispatcher():
|
||||
"""Dispatcher will then take care of dispatching onions to the right databases.
|
||||
Each database thread will read in the task and will handle it."""
|
||||
def __init__(self, logger):
|
||||
self.logger = logger
|
||||
self.__storage = []
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def add_storage(self, thread_storage):
|
||||
self.__storage.append(thread_storage)
|
||||
|
||||
def save_pastie(self, pastie, timeout=5):
|
||||
self.logger.debug('Saving to database')
|
||||
for t in self.__storage:
|
||||
t.save_pastie(pastie, timeout)
|
||||
|
||||
class PastieStorage():
|
||||
def __init__(self, **kwargs):
|
||||
self.lookup = kwargs.get('lookup', False)
|
||||
self.name = kwargs.get('name', self.__class__.__name__)
|
||||
try:
|
||||
self.logger.debug('{0}: initializing storage backend'.format(self.name))
|
||||
self.__init_storage__(**kwargs)
|
||||
except Exception as e:
|
||||
self.logger.error('{0}: unable to initialize storage backend: {1}'.format(self.name, e))
|
||||
raise
|
||||
|
||||
def format_directory(self, directory):
|
||||
d = datetime.now()
|
||||
year = str(d.year)
|
||||
month = str(d.month)
|
||||
# prefix month and day with "0" if it is only one digit
|
||||
if len(month) < 2:
|
||||
month = "0" + month
|
||||
day = str(d.day)
|
||||
if len(day) < 2:
|
||||
day = "0" + day
|
||||
return directory + os.sep + year + os.sep + month + os.sep + day
|
||||
|
||||
def __init_storage__(self, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def __save_pastie__(self, pastie):
|
||||
raise NotImplementedError
|
||||
|
||||
def save_pastie(self, pastie):
|
||||
try:
|
||||
start = time.time()
|
||||
self.logger.debug('{0}: saving pastie[{1}]'.format(self.name, pastie.url))
|
||||
self.__save_pastie__(pastie)
|
||||
delta = time.time() - start
|
||||
self.logger.debug('{0}: pastie[{1}] saved in {2}s'.format(self.name, pastie.url, delta))
|
||||
except Exception as e:
|
||||
self.logger.error('{0}: unable to save pastie[{1}]: {2}'.format(self.name, pastie.url, e))
|
||||
raise
|
||||
|
||||
def __seen_pastie__(self, pastie_id, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def seen_pastie(self, pastie_id, **kwargs):
|
||||
if not self.lookup:
|
||||
return False
|
||||
try:
|
||||
start = time.time()
|
||||
self.logger.debug('{0}: looking up pastie[{1}]'.format(self.name, pastie_id))
|
||||
res = self.__seen_pastie__(pastie_id, **kwargs)
|
||||
delta = time.time() - start
|
||||
self.logger.debug('{0}: pastie[{1}] looked-up in {2}s'.format(self.name, pastie_id, delta))
|
||||
return res
|
||||
except Exception as e:
|
||||
self.logger.error('{0}: unable to lookup pastie[{1}]: {2}'.format(self.name, pastie_id, e))
|
||||
raise
|
||||
|
||||
class Notifier(object):
|
||||
def __init__(self, logger, **kwargs):
|
||||
self.logger = logger
|
||||
|
||||
def send(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
def scheduledEvery(self, time="10:30"):
|
||||
self.logger.info(f'Scheduled task everyday as {time}')
|
||||
schedule.every().day.at(time).do(self.send)
|
||||
schedule.run_pending()
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
import sys
|
||||
import requests
|
||||
|
||||
from onioningestor.databases import PastieStorage
|
||||
|
||||
class Plugin(PastieStorage):
|
||||
|
||||
def __init__(self, logger, **kwargs):
|
||||
# kwargs = {'name': 'telegram-notifer', 'chat_id': 111111, 'token': 'XXXX'}
|
||||
self.name = kwargs.get('name')
|
||||
self.logger = logger
|
||||
self.token = kwargs.get('token')
|
||||
self.chat_id = kwargs.get('chat_id')
|
||||
|
||||
def __save_pastie__(self, pastie):
|
||||
message = '''
|
||||
HiddenSite: {site}
|
||||
Source : {url}
|
||||
Monitor : {content}
|
||||
Status : {status}
|
||||
'''.format(
|
||||
site=pastie.url,
|
||||
url=pastie.source,
|
||||
content=pastie.monitor,
|
||||
status=pastie.status)
|
||||
|
||||
url = 'https://api.telegram.org/bot{0}/sendMessage'.format(self.token)
|
||||
try:
|
||||
self.logger.debug('Sending message to telegram {} for pastie_id {}'.format(url, pastie))
|
||||
requests.post(url, data={'chat_id': self.chat_id, 'text':message})
|
||||
except Exception as e:
|
||||
self.logger.warning("Failed to alert through telegram: {0}".format(e))
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
|
||||
|
||||
class notifier(object):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def scheduledEvery(self, time, summary):
|
||||
pass
|
||||
@ -0,0 +1,42 @@
|
||||
from datetime import datetime as dt
|
||||
|
||||
class Onion(object):
|
||||
"""Onion class"""
|
||||
def __init__(self, url, source, type, status, monitor, denylist):
|
||||
self.url = url
|
||||
self.source = source
|
||||
self.type = type
|
||||
self.status = status
|
||||
self.monitor = monitor
|
||||
self.denylist = denylist
|
||||
self.datetime = dt.now()
|
||||
|
||||
def simpleHTML(self, response):
|
||||
self.simpleHTML = response
|
||||
# if any match update denylist
|
||||
|
||||
def onionscan(self, response):
|
||||
self.onionscan = response
|
||||
|
||||
def asdict(self):
|
||||
d = {
|
||||
'hiddenService':self.url,
|
||||
'source':self.source,
|
||||
'type':self.type,
|
||||
'status':self.status,
|
||||
'monitor': self.monitor,
|
||||
'denylist': self.denylist,
|
||||
'dateFound': self.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")+"Z",
|
||||
'simpleHTML': self.simpleHTML,
|
||||
'onionscan':self.onionscan
|
||||
}
|
||||
return d
|
||||
|
||||
def __lt__(self, other):
|
||||
return self.datetime < other.datetime
|
||||
|
||||
def __str__(self):
|
||||
return self.url
|
||||
|
||||
def __repr__(self):
|
||||
return self.url
|
||||
@ -0,0 +1,261 @@
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import traceback
|
||||
import subprocess
|
||||
from uuid import uuid4
|
||||
from pathlib import Path
|
||||
from datetime import datetime as dt
|
||||
from json.decoder import JSONDecodeError
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from threading import Timer
|
||||
|
||||
import requests
|
||||
|
||||
from stem.control import Controller
|
||||
from stem import Signal
|
||||
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.firefox.options import Options
|
||||
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
|
||||
|
||||
from onioningestor.operators import Operator
|
||||
|
||||
|
||||
class Plugin(Operator):
|
||||
"""OnionScraper main work logic.
|
||||
|
||||
Handles reading the config file, calling sources, maintaining state and
|
||||
sending artifacts to operators.
|
||||
"""
|
||||
def __init__(self, logger, **kwargs):
|
||||
self.logger = logger
|
||||
self.logger.info('Initializing OnionScanner')
|
||||
screenshots = kwargs.pop('screenshots_path', None)
|
||||
if screenshots:
|
||||
self.screenshots = Path(screenshots)
|
||||
else:
|
||||
self.screenshots = Path(__file__).parents[1]/'screenshots'
|
||||
self.onionscan = kwargs['binpath']
|
||||
self.timeout = int(kwargs['timeout'])
|
||||
self.proxy = kwargs['socks5']
|
||||
self.torControl = kwargs['TorController']
|
||||
self.retries = int(kwargs['retries'])
|
||||
self.headers ={
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:72.0) Gecko/20100101 Firefox/72.0',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language':'en-US,en;q=0.5',
|
||||
'DNT': '1', 'Connection':
|
||||
'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1'
|
||||
}
|
||||
|
||||
blacklist = kwargs['blacklist'].split(',')
|
||||
self.blacklist = re.compile('|'.join([re.escape(word) for word in blacklist]), re.IGNORECASE)
|
||||
keywords = kwargs['interestingKeywords'].split(',')
|
||||
self.keywords = re.compile('|'.join([re.escape(word) for word in keywords]), re.IGNORECASE)
|
||||
self.session = self.get_tor_session()
|
||||
|
||||
def response(self, status, content, onion):
|
||||
"""
|
||||
status: success/failure
|
||||
content: dict
|
||||
onion: str
|
||||
return: dict
|
||||
"""
|
||||
return {'status': status, 'data': content, 'onion': onion}
|
||||
|
||||
def parseDoc(self, data):
|
||||
data['onionscan'].pop('simpleReport', None)
|
||||
crawls = data['onionscan'].pop('crawls', None)
|
||||
hiddenService = data['onionscan'].pop('hiddenService', None)
|
||||
data['onionscan']['crawls'] = [*crawls]
|
||||
data['hiddenService'] = hiddenService
|
||||
for onion in crawls.keys():
|
||||
print(onion)
|
||||
with open('/home/tony/Projects/OnionScraper_v2/onion_master_list.txt', 'a') as fp:
|
||||
fp.write("%s\n" % onion)
|
||||
#q.enqueue(self.crawl, onion)
|
||||
#with open('test.json', 'w', encoding='utf-8') as f:
|
||||
# json.dump(data, f, ensure_ascii=False, indent=4)
|
||||
return data
|
||||
|
||||
def format_directory(self, directory):
|
||||
d = dt.now()
|
||||
year = str(d.year)
|
||||
month = str(d.month)
|
||||
# prefix month and day with "0" if it is only one digit
|
||||
if len(month) < 2:
|
||||
month = "0" + month
|
||||
day = str(d.day)
|
||||
if len(day) < 2:
|
||||
day = "0" + day
|
||||
save_path = directory/year/month/day
|
||||
if not os.path.isdir(save_path):
|
||||
self.logger.info("[*] Creating directory to save screenshots")
|
||||
os.makedirs(save_path)
|
||||
|
||||
return save_path
|
||||
|
||||
def take_screenshot(self, save_path, onion):
|
||||
binary = FirefoxBinary('/home/tony/Projects/OnionScraper/geckodriver')
|
||||
fp = webdriver.FirefoxProfile()
|
||||
fp.set_preference('network.proxy.type', 1)
|
||||
fp.set_preference('network.proxy.socks', '127.0.0.1')
|
||||
fp.set_preference('network.proxy.socks_port', 9050)
|
||||
fp.set_preference('network.proxy.socks_remote_dns', True)
|
||||
|
||||
options = Options()
|
||||
options.headless = True
|
||||
driver = webdriver.Firefox(
|
||||
executable_path='/home/tony/Projects/OnionScraper/geckodriver',
|
||||
options=options,
|
||||
firefox_profile=fp
|
||||
)
|
||||
url = 'http://' + onion
|
||||
driver.get(url)
|
||||
uid = str(uuid4()).split('-')[0]
|
||||
filename = f"{onion}_screenshot_{uid}.png"
|
||||
f_name = f"{save_path}/{filename}"
|
||||
driver.save_screenshot(f_name)
|
||||
|
||||
driver.quit()
|
||||
|
||||
if os.path.isfile(f_name):
|
||||
self.logger.info(f'[*] Screenshot was taken. {f_name}')
|
||||
dateScreenshoted = dt.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')+ 'Z'
|
||||
result = {'dateScreenshoted':dateScreenshoted,'filename':filename}
|
||||
return self.response("success",result,onion)
|
||||
else:
|
||||
self.logger.error('[x] Unable to take screenshot')
|
||||
return self.response("failure",None,onion)
|
||||
|
||||
def get_tor_session(self):
|
||||
try:
|
||||
s = requests.session()
|
||||
s.proxies = self.proxy
|
||||
s.headers.update(self.headers)
|
||||
except Exception as e:
|
||||
self.logger.error(e)
|
||||
self.logger.debug(traceback.print_exc())
|
||||
return s
|
||||
|
||||
# signal TOR for a new connection
|
||||
def renew_connection(self):
|
||||
with Controller.from_port(port = self.torControl['port']) as controller:
|
||||
# Now we switch TOR identities to make sure we have a good connection
|
||||
self.logger.info('Getting new Tor IP')
|
||||
# authenticate to our local TOR controller
|
||||
controller.authenticate(self.torControl['password'])
|
||||
# send the signal for a new identity
|
||||
controller.signal(Signal.NEWNYM)
|
||||
# wait for the new identity to be initialized
|
||||
time.sleep(controller.get_newnym_wait())
|
||||
session = self.get_tor_session()
|
||||
self.logger.info(f"IP is {session.get('http://httpbin.org/ip').json()['origin']}")
|
||||
|
||||
def handle_timeout(self, process, onion):
|
||||
#
|
||||
# Handle a timeout from the onionscan process.
|
||||
#
|
||||
|
||||
try:
|
||||
# kill the onionscan process
|
||||
process.kill()
|
||||
self.logger.info("[!!!] Killed the onionscan process.")
|
||||
except:
|
||||
pass
|
||||
self.renew_connection()
|
||||
return
|
||||
|
||||
def run_sessions(self, onion):
|
||||
retry = 0
|
||||
result = None
|
||||
while True:
|
||||
try:
|
||||
url = 'http://'+onion
|
||||
self.logger.info(url)
|
||||
content = self.session.get(url)
|
||||
if content.status_code == 200:
|
||||
result = content.json()
|
||||
except JSONDecodeError as e:
|
||||
self.logger.debug(f'JSONDecodeError {e}')
|
||||
result = content.text
|
||||
except Exception as e:
|
||||
self.logger.error(e)
|
||||
self.logger.debug(traceback.print_exc())
|
||||
finally:
|
||||
if result:
|
||||
return self.response("success",result,onion)
|
||||
else:
|
||||
self.logger.info('[x] No results found retrying ...')
|
||||
retry += 1
|
||||
self.renew_connection()
|
||||
if retry > self.retries:
|
||||
self.logger.error('[x] Max retries exceeded')
|
||||
return self.response("failure",None, onion)
|
||||
|
||||
def run_onionscan(self, onion):
|
||||
self.logger.info("[*] Running onionscan on %s", onion)
|
||||
|
||||
# fire up onionscan
|
||||
process = subprocess.Popen([self.onionscan,"--webport=0","--jsonReport","--simpleReport=false",onion],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
|
||||
|
||||
# start the timer and let it run till timeout minutes
|
||||
process_timer = Timer(300,self.handle_timeout,args=[process,onion])
|
||||
process_timer.start()
|
||||
|
||||
# wait for the onion scan results
|
||||
stdout = process.communicate()[0]
|
||||
|
||||
# we have received valid results so we can kill the timer
|
||||
if process_timer.is_alive():
|
||||
process_timer.cancel()
|
||||
try:
|
||||
return self.response("success",json.loads(stdout),onion)
|
||||
except json.decoder.JSONDecodeError:
|
||||
pass
|
||||
|
||||
self.logger.info("[!!!] Process timed out for %s", onion)
|
||||
|
||||
return self.response("failure",None, onion)
|
||||
|
||||
def handle_onion(self, onion_tuple):
|
||||
onion = onion_tuple.url
|
||||
self.logger.info(f'Processing {onion} with onionscan')
|
||||
try:
|
||||
blacklist_URL = self.blacklist.search(onion)
|
||||
if blacklist_URL:
|
||||
self.logger.info(f"[X] Blocked by blacklist => matched keyword {blacklist_URL.group()}")
|
||||
else:
|
||||
self.logger.debug("[*] URL blacklist test: PASSED")
|
||||
results = self.run_onionscan(onion)
|
||||
if results['status'] == 'success':# and results['data']['webDetected'] == 'true':
|
||||
content = self.run_sessions(onion)
|
||||
if content['status'] == 'success':
|
||||
blacklist_CONTENT = self.blacklist.search(content['data'])
|
||||
if blacklist_CONTENT:
|
||||
self.logger.info(f"[X] Blocked by blacklist content => matched keyword {blacklist_CONTENT.group()}")
|
||||
else:
|
||||
self.logger.debug("[*] CONTENT blacklist test: PASSED")
|
||||
screenshot = self.take_screenshot(self.format_directory(self.screenshots), onion)
|
||||
self.logger.info("Indexing!")
|
||||
doc = {
|
||||
'onionscan':json.loads(results['data']),
|
||||
'html':content['data'],
|
||||
'screenshots':screenshot['data'],
|
||||
'interestingKeywords':self.interestingKeywords.findall(content['data'])
|
||||
}
|
||||
return self.parseDoc(doc)
|
||||
|
||||
else:
|
||||
self.logger.info(f"[x] hidden service {onion} is not active")
|
||||
except Exception as e:
|
||||
self.logger.error(e)
|
||||
self.logger.error(traceback.print_exc())
|
||||
finally:
|
||||
pass
|
||||
#sys.exit(0)
|
||||
Loading…
Reference in New Issue