generate_schema_config.py

from formats.header import message_header
from formats.message import message_schema
from formats.mission_plan import mission_plan_schema
from formats.mission_plan_encoded import mission_plan_encoded_schema
from formats.observation import observation_schema
from formats.observation_encoded import observation_encoded_schema
from formats.payload import payload_schema
from formats.planning_configuration import planning_configuration_schema
from formats.platform_status import platform_status_schema
from formats.waypoints import waypoints_schema
from formats.platform_status_encoded import platform_status_encoded_schema
from formats.survey import survey_schema
from formats.survey_encoded import survey_encoded_schema
from formats.acknowledgement import acknowledgement_schema
from formats.alert import alert_schema
from formats.instruction_set import config_file_schema, instruction_set_schema

from flasgger import Swagger
from flask import Flask

import argparse
import json
import os
import re
import requests
from urllib.parse import urlparse


# Enable running on domain sub-path
URL_PREFIX = os.getenv("URL_PREFIX", "")
# Allow env override of default host
FLASK_HOST = os.getenv("FLASK_HOST", "localhost")
# Allow env override of default port
FLASK_PORT = os.getenv("FLASK_PORT", 5000)
# Switch on debug mode if env var is truthy
FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() in ("true", "1", "t")


def get_swagger_config(reload=False):
    if reload:
        print("Reload specified: Ignoring cached refs")

    swagger_config = {
        "openapi": "3.0.2",
        "swagger_ui": True,
        "specs_route": "/",
        "info": {
            "title": "SoAR Backbone Message Formats",
            "version": "1.0",
            "description": "SoAR message protocol in schemas",
        },
        "specs": [
            {
                "endpoint": "swagger",
                "route": "/soar_protocol.json",
            }
        ],
        "url_prefix": URL_PREFIX,
        "paths": {},
        "components": {
            "schemas": {
                "MESSAGE": message_schema,
                "header": message_header,
                "payload": payload_schema,
                "mission_plan": mission_plan_schema,
                "mission_plan_encoded": mission_plan_encoded_schema,
                "observation": observation_schema,
                "observation_encoded": observation_encoded_schema,
                "planning_configuration": planning_configuration_schema,
                "platform_status": platform_status_schema,
                "platform_status_encoded": platform_status_encoded_schema,
                "survey": survey_schema,
                "survey_encoded": survey_encoded_schema,
                "acknowledgement": acknowledgement_schema,
                "alert": alert_schema,
                "instruction_set": instruction_set_schema,
                "config_file": config_file_schema,
                "waypoints": waypoints_schema,
            }
        },
    }
    import_remote_refs(swagger_config, reload)
    return swagger_config


def resolve_ref(ref):
    """
    Get schema URL, parse JSON
    Return None if either fails
    """
    try:
        res = requests.get(ref)
        if res.status_code == 200:
            return res.json()
        else:
            return None
    except (json.JSONDecodeError, ValueError):
        return None


def rename_ref(ref):
    """
    Convert remote ref URL into a name that can
    be used for a local ref in the schema
    Remote the URL scheme and replace / with .
    """
    # remove url scheme
    deschemed = re.sub(r"^[htps]*\:*[/]{2}", "", ref)
    # replace / with . since the name will be in a path
    return re.sub(r"[/]", ".", deschemed)


def nested_replace(source, key, value, replace_with):
    """
    Find all instances of a key value pair in a nested
    dictionary and replace the value with replace_with
    """
    for k, v in source.items():
        if k == key and v == value:
            source[k] = replace_with
        elif type(v) is list:
            for item in v:
                if type(item) is dict:
                    nested_replace(item, key, value, replace_with)
        if type(v) is dict:
            nested_replace(v, key, value, replace_with)


def downgrade_schema_30x_compatible(schema):
    """
    The published GeoJSON schemas are OpenAPI v3.1

    Moving to v3.1 is not trivial
    There isn't a CommonJS validator for v3.1
    There isn't a python source of the v3.0.x defs

    Remove $id and $schema
    Remove oneOf: [{type:null}]
    Iterate over oneOf and items child schemas
    """
    if "$id" in schema:
        del schema["$id"]
    if "$schema" in schema:
        del schema["$schema"]
    if "properties" in schema:
        for propConfig in schema["properties"].values():
            if "oneOf" in propConfig:
                try:
                    propConfig["oneOf"].remove({"type": "null"})
                except ValueError:
                    pass
                for child_schema in propConfig["oneOf"]:
                    downgrade_schema_30x_compatible(child_schema)
            if "items" in propConfig:
                downgrade_schema_30x_compatible(propConfig["items"])


def get_remote_ref_cache_path(remote_ref):
    parsed_ref = urlparse(remote_ref)
    return f"remotes/{parsed_ref.hostname}{parsed_ref.path}"


def get_cached_ref(remote_ref):
    ref_path = get_remote_ref_cache_path(remote_ref)
    ref = None
    if os.path.exists(ref_path):
        print(f"loading cached ref: {remote_ref}")
        with open(ref_path, "r") as ref_file:
            ref = json.load(ref_file)
    return ref


def store_cached_ref(remote_ref, definition):
    ref_path = get_remote_ref_cache_path(remote_ref)
    ref_dirs = re.sub(r"\/[^\/]+$", "", ref_path)
    os.makedirs(ref_dirs, 0o775, True)
    with open(ref_path, "w") as ref_file:
        json.dump(definition, ref_file, indent=2)


def inject_schema(schema, remote_ref, reload=False):
    """
    Given a parent schema and a remote ref

    1. get the remote ref schema
    2. create a local reference name (without path separators)
    3. insert into components.schemas
    4. replace remote references with local references

    returns True if resolved and injected
    """
    local_name = rename_ref(remote_ref)
    local_ref = f"#/components/schemas/{local_name}"
    # get schema from cache if present
    ref_schema = None if reload else get_cached_ref(remote_ref)
    if not ref_schema:
        print(f"ref not cached: {remote_ref}")
        ref_schema = resolve_ref(remote_ref)
        downgrade_schema_30x_compatible(ref_schema)
        store_cached_ref(remote_ref, ref_schema)

    if ref_schema is not None:
        nested_replace(schema, "$ref", remote_ref, local_ref)
        schema["components"]["schemas"][local_name] = ref_schema
        return True
    else:
        return False


def import_remote_refs(swagger_config, reload=False):
    """
    inject the following remote refs into the schema
    and replace the remote refs with local refs

    returns True if all schemas resolved and injected
    """
    # For some reason importing Feature or FeatureCollection
    # makes the schema fail to validate

    ref_imports = [
        "https://geojson.org/schema/FeatureCollection.json",
        "https://geojson.org/schema/Feature.json",
        "https://geojson.org/schema/LineString.json",
        "https://geojson.org/schema/MultiLineString.json",
        "https://geojson.org/schema/MultiPoint.json",
        "https://geojson.org/schema/MultiPolygon.json",
        "https://geojson.org/schema/Point.json",
        "https://geojson.org/schema/Polygon.json",
    ]

    return all([inject_schema(swagger_config, ref, reload) for ref in ref_imports])


def configure_flask(swagger_config):
    """
    Setup a flask app, load flasgger

    and then patch to remove invalid
    definitions:{} object
    """
    app = Flask(__name__)
    Swagger(app, config=swagger_config, merge=True)

    # Replace schema route to remove invalid
    # definitions: {}
    # Should be fixed if Flassger 0.9.7 is released
    #
    # The last release of flasgger was Aug 2020
    # This bug was fixed in Nov 2021
    # There is a pre-release from May 2023
    # Until the fix gets released we have to
    # remove the invalid definitions object
    # from the spec
    @app.after_request
    def after_request_decorator(response):
        """
        I didn't want to mess with flasgger so
        this blunt workaround that runs on every
        route and then checks whether it's required
        """
        is_response = type(response).__name__ == "Response"
        is_json = is_response and response.content_type == "application/json"
        if is_json:
            parsed = response.json
            if "definitions" in parsed:
                del parsed["definitions"]
            response.data = json.dumps(parsed)

        return response

    return app


def serve(swagger_config):
    """
    Run as local flask app on FLASK_PORT|5000
    """
    app = configure_flask(swagger_config)
    app.run(debug=FLASK_DEBUG, host=FLASK_HOST, port=FLASK_PORT)


def compile_schema(swagger_config):
    """Extract the output schema from flasgger

    The only way I have found to do this is to
    use a test client to make the GET request
    for the page

    The function that returns the definition
    can't be called outside the flask app context
    """
    app = configure_flask(swagger_config)
    route = swagger_config["specs"][0]["route"]
    client = app.test_client()
    response = client.get(route)
    spec = response.json
    return spec


def write_schema(swagger_config, file_path):
    """
    Dump schema to specified file
    """
    spec = compile_schema(swagger_config)
    json_schema = json.dumps(spec, indent=2)

    with open(file_path, "w") as f:
        f.write(json_schema)


def get_options():
    """
    Parse script arguments
    """
    parser = argparse.ArgumentParser(
        description="Generate the schema",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        "-s",
        "--serve",
        dest="run_flask",
        action="store_true",
        help="Run flask app",
        default=False,
    )
    parser.add_argument(
        "-f",
        "--file",
        dest="output_file",
        action="store_true",
        help="Save output to schema file",
        default=False,
    )
    parser.add_argument(
        "-r",
        "--reload",
        dest="reload_schemas",
        action="store_true",
        help="Overwrite local copies of remote reference schemas",
        default=False,
    )
    parser.add_argument("filename", nargs="?", default="project/soar/swagger.json")
    args = parser.parse_args()
    config = vars(args)
    # If no flag is specified default to running the flask server
    if not (config["run_flask"] or config["output_file"]):
        config["run_flask"] = True
    return config


if __name__ == "__main__":
    # Parse script args
    config = get_options()

    swagger_config = get_swagger_config(config.get("reload_schemas"))

    # Output compiled schema
    if config.get("output_file"):
        write_schema(swagger_config, config.get("filename"))

    # Run flask app
    if config.get("run_flask"):
        serve(swagger_config)