Merge pull request #142 from creekorful/66-add-kubernetes-config-files

Add kubernetes config files

README.md

@@ -9,7 +9,7 @@ Bathyscaphe is a Go written, fast, highly configurable, cloud-native dark web cr
 To start the crawler, one just need to execute the following command:
 
 ```sh
-$ ./scripts/start.sh
+$ ./scripts/docker/start.sh
 ```
 
 and wait for all containers to start.
@@ -21,7 +21,7 @@ and wait for all containers to start.
 
 # How to initiate crawling
 
-One can use the RabbitMQ dashhboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue**
+One can use the RabbitMQ dashboard available at localhost:15003, and publish a new JSON object in the **crawlingQueue**
 .
 
 The object should look like this:
@@ -38,7 +38,7 @@ If one want to speed up the crawling, he can scale the instance of crawling comp
 This may be done by issuing the following command after the crawler is started:
 
 ```sh
-$ ./scripts/scale.sh crawler=5
+$ ./scripts/docker/start.sh -d --scale crawler=5
 ```
 
 this will set the number of crawler instance to 5.

deployments/k8s/blacklister.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: blacklister-deployment
+  labels:
+    app: blacklister
+spec:
+  replicas: 5
+  selector:
+    matchLabels:
+      app: blacklister
+  template:
+    metadata:
+      labels:
+        app: blacklister
+    spec:
+      containers:
+        - name: blacklister
+          image: creekorful/bs-blacklister:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --config-api
+            - http://configapi-service
+            - --cache-srv
+            - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
+            - --tor-proxy
+            - torproxy-service:9050

deployments/k8s/configapi.yaml

@@ -0,0 +1,47 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: configapi-deployment
+  labels:
+    app: configapi
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: configapi
+  template:
+    metadata:
+      labels:
+        app: configapi
+    spec:
+      containers:
+        - name: configapi
+          image: creekorful/bs-configapi:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --cache-srv
+            - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379
+            - --default-value
+            - forbidden-hostnames=[]
+            - --default-value
+            - allowed-mime-types=[{"content-type":"text","extensions":["html","php","aspx", "htm"]}]
+            - --default-value
+            - refresh-delay={"delay":0}
+            - --default-value
+            - blacklist-config={"threshold":5, "ttl":1200}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: configapi-service
+spec:
+  selector:
+    app: configapi
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080

deployments/k8s/crawler.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: crawler-deployment
+  labels:
+    app: crawler
+spec:
+  replicas: 20
+  selector:
+    matchLabels:
+      app: crawler
+  template:
+    metadata:
+      labels:
+        app: crawler
+    spec:
+      containers:
+        - name: crawler
+          image: creekorful/bs-crawler:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --tor-proxy
+            - torproxy-service:9050
+            - --config-api
+            - http://configapi-service

deployments/k8s/helm/elastic-values.yaml

@@ -0,0 +1,2 @@
+# Config file to switch to single node
+replicas: 1

deployments/k8s/helm/rabbitmq-values.yaml

@@ -0,0 +1,3 @@
+auth:
+  username: bathyscaphe
+  password: nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC

deployments/k8s/helm/redis-values.yaml

@@ -0,0 +1 @@
+password: WtVcUp9yebNnwjuE2LuumJM52AJ9sZ

deployments/k8s/indexer-es.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: indexer-es-deployment
+  labels:
+    app: indexer-es
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: indexer-es
+  template:
+    metadata:
+      labels:
+        app: indexer-es
+    spec:
+      containers:
+        - name: indexer
+          image: creekorful/bs-indexer:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --event-prefetch
+            - '20'
+            - --config-api
+            - http://configapi-service
+            - --index-driver
+            - elastic
+            - --index-dest
+            - http://elasticsearch-master:9200

deployments/k8s/scheduler.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scheduler-deployment
+  labels:
+    app: scheduler
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: scheduler
+  template:
+    metadata:
+      labels:
+        app: scheduler
+    spec:
+      containers:
+        - name: scheduler
+          image: creekorful/bs-scheduler:latest
+          args:
+            - --log-level
+            - debug
+            - --event-srv
+            - amqp://bathyscaphe:nHEsR9MpJfxLwwkKP48mhMtGFTBqMTfC@rabbitmq:5672
+            - --event-prefetch
+            - '20'
+            - --config-api
+            - http://configapi-service
+            - --cache-srv
+            - redis://default:WtVcUp9yebNnwjuE2LuumJM52AJ9sZ@redis-master:6379

deployments/k8s/torproxy.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: torproxy-deployment
+  labels:
+    app: torproxy
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: torproxy
+  template:
+    metadata:
+      labels:
+        app: torproxy
+    spec:
+      containers:
+        - name: torproxy
+          image: dperson/torproxy:latest
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: torproxy-service
+spec:
+  selector:
+    app: torproxy
+  ports:
+    - protocol: TCP
+      port: 9050

scripts/delete.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-./scripts/exec.sh down "$@"

scripts/docker/delete.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./scripts/docker/exec.sh down "$@"

scripts/docker/log.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./scripts/docker/exec.sh logs "$@"

scripts/docker/ps.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./scripts/docker/exec.sh ps "$@"

scripts/docker/start.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./scripts/docker/exec.sh up "$@"

scripts/docker/stop.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./scripts/docker/exec.sh stop "$@"

scripts/helpers/blacklist-hostnames.py

@@ -46,11 +46,11 @@ print("there is {} forbidden hostnames defined in ConfigAPI".format(len(forbidde
 # Merge the lists while preventing duplicates
 for forbidden_hostname in forbidden_hostnames:
     add_if_not_exist(new_hostnames, forbidden_hostname['hostname'])
-print("there is {} forbidden hostnames now".format(len(forbidden_hostnames)))
+print("there is {} forbidden hostnames now".format(len(new_hostnames)))
 
 # Update ConfigAPI
 headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
-r = requests.put(config_api_uri + "/config/forbidden-hostnames", json.dumps(forbidden_hostnames), headers=headers)
+r = requests.put(config_api_uri + "/config/forbidden-hostnames", json.dumps(new_hostnames), headers=headers)
 
 if r.ok:
     print("successfully updated forbidden hostnames")

scripts/k8s/bootstrap.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+helm repo add bitnami https://charts.bitnami.com/bitnami
+helm repo add elastic https://helm.elastic.co
+
+kubectl create namespace bathyscaphe
+
+helm install --namespace bathyscaphe redis -f deployments/k8s/helm/redis-values.yaml bitnami/redis
+helm install --namespace bathyscaphe rabbitmq -f deployments/k8s/helm/rabbitmq-values.yaml bitnami/rabbitmq
+helm install --namespace bathyscaphe elasticsearch elastic/elasticsearch
+helm install --namespace bathyscaphe kibana elastic/kibana
+
+# Install our resources
+kubectl -n bathyscaphe apply -f deployments/k8s/torproxy.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/configapi.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/crawler.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/scheduler.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/blacklister.yaml
+kubectl -n bathyscaphe apply -f deployments/k8s/indexer-es.yaml

scripts/log.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-./scripts/exec.sh logs "$@"

scripts/ps.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-./scripts/exec.sh ps "$@"

scripts/scale.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-./scripts/exec.sh scale "$@"

scripts/start.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-./scripts/exec.sh up "$@"

scripts/stop.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-./scripts/exec.sh stop "$@"