Merge branch '13-create-a-docker-compose-to-run-rmq-the-bus-and-the-api' into 'dev'

Resolve "Create a docker-compose to run RMQ, the bus and the API"

Closes #17 and #13

See merge request !7

.env

+# DATA_DIR=
+# SOAR_TOKEN_LIFETIME=
+# SOAR_TOKEN_SECRET=
\ No newline at end of file

.gitignore

 clients.json
 examples/
-rmq.log
+rmq.log
\ No newline at end of file
+Pipfile
+Pipfile.lock
\ No newline at end of file

Pipfile

-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-pubsubpy = "*"
-pika = "*"
-pyrabbit = "*"
-flask = "*"
-flask-restful = "*"
-marshmallow = "*"
-bson = "*"
-flask-cors = "*"
-cryptography = "*"
-
-[dev-packages]
-
-[requires]
-python_version = "3.8"

README.md

@@ -2,14 +2,6 @@
 
 Communications Backbone by C2 Team (NOC)
 
-## DJ prototype 
-
-I did this in freedom-fire-day so it doesn't currently follow the flask template. 
-
-It's also a bit weird because it writes to a local ignored `clients.json` file 
-instead of using a database. I did this as a placeholder because we've not yet 
-decided what the infrastructure looks like.
-
 ### Data flow 
 
 - Client A sends to `client-a-outbox` (or POSTs to API /send - not yet implemented)
@@ -31,14 +23,22 @@ which queues it reads from.
 
 Subsequent requests to the client endpoint return the client_id but not the secret.
 
-### Setup 
+### Running via docker-compose
+
+Using `docker-compose` will mean that everything is setup automatically, this includes the `rabbitmq` container, the backbone API, and the backbone bus. The `run-compose.sh` script has been provided to simplify this even further - all you have to do is set whatever env vars you need in the `.env` file and then run `./run-compose.sh` (the defaults in `.env` are fine for local dev work, but ones not labelled `optional` will need setting in a production setting). The env vars are:
+
+- `DATA_DIR` - Where to mount the volume of the API container on your local system. This defaults to the result of `pwd`, which should be within the `communications-backbone` repo
+- `SOAR_TOKEN_LIFETIME` (Optional) - The number of hours until a newly created token expires
+- `SOAR_TOKEN_SECRET` (Optional) - A secret key used to encrypt/decrypt token data. If specified the value should be set using TokenModel.getKey()
+
+### Running manually
+
+#### Setup 
 
 ```
 pipenv install  
 ```
 
-### Running 
-
 #### RabbitMQ 
 
 `docker run --rm -p 5672:5672 -d --hostname rmq --name rmq rabbitmq:management`

api.py

 from flask import Flask
+from flask_cors import CORS
 from flask_restful import Api
+
 from endpoints.clients import Client, ClientList
+from endpoints.notify import Notify
 from endpoints.receive import Receive
 from endpoints.send import Send
-from endpoints.notify import Notify
 from endpoints.token import Token
-from flask_cors import CORS
 from models.token import TokenModel
 
+import os
+
 token = TokenModel()
 token.setSecret()
 
@@ -22,5 +25,7 @@ api.add_resource(Send, "/send")
 api.add_resource(Notify, "/notify")
 api.add_resource(Token, "/token")
 
+flask_host = os.getenv("FLASK_HOST", "localhost") # Sets to whatever MQ_HOST is, or defaults to localhost
+
 if __name__ == "__main__":
-    app.run(debug=True, port=8087)
+    app.run(debug=False, port=8087, host=flask_host)

docker/Dockerfile

+FROM python:alpine3.17
+
+WORKDIR /app
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+ENTRYPOINT [ "python" ]
\ No newline at end of file

docker/docker-compose.yaml

+version: '3.8'
+services:
+  rabbitmq__local:
+    image: rabbitmq:management
+    restart: unless-stopped
+    ports:
+      - "5672:5672"
+      # - "15672:15672" # Admin web console
+    expose:
+      - "5672"
+    container_name: rmq
+
+  soar_bus:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    restart: unless-stopped
+    depends_on:
+      - rabbitmq__local
+    environment:
+      - MQ_HOST=rmq
+    volumes:
+      - ../:/app
+    command: "soar_bus.py"
+    container_name: soar_bus
+
+  soar_api:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    restart: unless-stopped
+    ports:
+      - "8087:8087"
+    expose:
+      - "8087"
+    depends_on:
+      - rabbitmq__local
+    environment:
+      - MQ_HOST=rmq
+      - FLASK_HOST=0.0.0.0
+    volumes:
+      - ../:/app
+    command: "api.py"
+    container_name: soar_api
\ No newline at end of file

endpoints/auth_resource.py

 import json
+import os
+
 from flask_restful import Resource, abort
+
 from models.token import TokenModel
 
+
 class AuthResource(Resource):
 
     def __init__(self): 
         self.token = TokenModel()
-        with open("clients.json", "r") as clients_file:
+        with open("./data/clients.json", "r") as clients_file:
             self.clients = json.load(clients_file)
 
     def auth(self, request): 

endpoints/clients.py

@@ -13,7 +13,7 @@ class ClientSchema(Schema):
 
 
 class ClientsFile:
-    file = "clients.json"
+    file = "./data/clients.json"
     mtime = 0
     clients = {}
     parser = None

endpoints/token.py

 import json 
 from flask_restful import Resource, request, abort
 from marshmallow import Schema, fields
-import pika
 from models.token import TokenModel
 
-
 class TokenQuerySchema(Schema):
     client_id = fields.Str(required=True)
     secret = fields.Str(required=True)
@@ -18,7 +16,7 @@ class Token(Resource):
     def __init__(self):
         self.schema = TokenQuerySchema()
         self.model = TokenModel()
-        with open("clients.json", "r") as clients_file:
+        with open("./data/clients.json", "r") as clients_file:
             self.clients = json.load(clients_file)
     
     def get(self):

requirements.txt

+-i https://pypi.org/simple
+amqp==5.1.1 ; python_version >= '3.6'
+aniso8601==9.0.1
+bson==0.5.10
+cffi==1.15.1
+click==8.1.3 ; python_version >= '3.7'
+cryptography==38.0.3
+flask==2.2.2
+flask-cors==3.0.10
+flask-restful==0.3.9
+future==0.18.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httplib2==0.21.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+importlib-metadata==5.1.0 ; python_version < '3.10'
+itsdangerous==2.1.2 ; python_version >= '3.7'
+jinja2==3.1.2 ; python_version >= '3.7'
+kombu==5.2.4 ; python_version >= '3.7'
+markupsafe==2.1.1 ; python_version >= '3.7'
+marshmallow==3.19.0
+packaging==21.3 ; python_version >= '3.6'
+pika==1.3.1
+pubsubpy==2.3.0
+pycparser==2.21
+pyparsing==3.0.9 ; python_full_version >= '3.6.8'
+pyrabbit==1.1.0
+python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pytz==2022.6
+six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+vine==5.0.0 ; python_version >= '3.6'
+werkzeug==2.2.2 ; python_version >= '3.7'
+zipp==3.10.0 ; python_version >= '3.7'

rmq.py

 import json
+import os
 
 import pika
 
-host='localhost' # TODO Handle host being passed in (https://git.noc.ac.uk/communications-backbone-system/communications-backbone/-/issues/17)
+host = os.getenv("MQ_HOST", "localhost") # Sets to whatever MQ_HOST is, or defaults to localhost
 
 # -------------------------------------------------------------------------------------------------------------------------------------------------------------
 
 def pika_connect(host):
-    connection = pika.BlockingConnection(pika.ConnectionParameters(host))
-    channel = connection.channel()
+    try:
+        connection = pika.BlockingConnection(pika.ConnectionParameters(host))
+    except Exception:
+        connection = None
+
+    if connection is not None:
+        channel = connection.channel()
+    else:
+        print("ERROR: Pika has been unable to connect to host '%s'. Is RabbitMQ running?" % host)
+        raise Exception("ERROR: Pika has been unable to connect to host '%s'. Is RabbitMQ running?" % host)
+
     return connection, channel
 
 
@@ -154,7 +164,6 @@ def subscribe(queue_name, exchange_name, topic=None):
     # setup bindings between queue and exchange, 
     # exchange_type is either 'fanout' or 'topic' based on if the topic arg is passed
     connection, channel = pika_connect(host=host)
-
     setup_queue(channel=channel, queue_name=queue_name)
 
     if topic is None:

run-compose.sh

+#! /usr/bin/env bash
+
+set -a
+source .env
+
+usage() {
+    echo "usage: ./run-compose.sh [<rebuild>]"
+    echo "    rebuild - will force docker-compose to rebuild the images before spinning them up."
+}
+
+if [[ -z "${DATA_DIR}" ]]; then
+    DATA_DIR=$(pwd)
+fi
+
+while [ -n "$1" ]; do
+    case $1 in
+    rebuild)
+        EXTRA_ARGS="--build"
+        ;;
+    *)
+        usage
+        exit 0
+        ;;
+    esac
+    shift
+done
+
+docker-compose -f docker/docker-compose.yaml up $EXTRA_ARGS
\ No newline at end of file

soar_bus.py

@@ -10,8 +10,9 @@
 # soar-broadcast - admin messages forwarded to all client-inboxes regardless of subscriptions
 
 import concurrent.futures
+
 from endpoints.clients import ClientsFile
-from rmq import publish, subscribe, broadcast, forward
+from rmq import broadcast, forward, publish, subscribe
 
 THREADS = []
 EXCHANGES = {
@@ -21,6 +22,7 @@ EXCHANGES = {
 
 
 def main():
+    print("Starting SOAR bus...")
     clients_file = ClientsFile()
     clients = clients_file.get()
 
@@ -45,6 +47,7 @@ def main():
                 EXCHANGES.get("publish"),
                 client["subscription"] # topic
             )
+            THREADS.append(thread)
             thread = executor.submit(
                 subscribe, 
                 f"{id}-inbox",
@@ -55,6 +58,14 @@ def main():
             # TODO - add optional webhook target to client and post to webhook target
             # if present
 
+        # Make sure the threads are actually running, error if not,
+        # this allows the SOAR Bus to actually wait for RMQ to start running
+        for thread in THREADS:
+            thread.result()
+            try:
+                print(thread.result())
+            except Exception as e:
+                print(e)
 
 if __name__ == "__main__":
     main()