Resolve "Test validating messages with references to GeoJson schema in both JS and python"

Closes #45 (closed)

formats/planning_configuration.py

 region_schema = {
     "type": "object",
     "properties": {
-        "geometry_coordinates": {
-            "type": "array",
-            "example": [
-                [
-                    [-4.1777839187560915, 50.34173405662855],
-                    [-4.1777839187560915, 50.33820949229701],
-                    [-4.143667777943875, 50.33820949229701],
-                    [-4.143667777943875, 50.34173405662855],
-                    [-4.1777839187560915, 50.34173405662855],
-                ]
-            ],
+        "geometry": {
+            "$ref": "https://geojson.org/schema/Polygon.json",
         },
     },
-    "description": "Using GEOJSON, exact 4-point region (rectangle shaped - 5 points)",
+    "description": "GeoJSON Polygon",
 }
 
 squad_metadata_schema = {

generate_schema_config.py

 from flasgger import Swagger
 from flask import Flask
 
+import json
 import os
+import re
+import requests
+import argparse
 
-app = Flask(__name__)
 
-url_prefix = os.getenv("URL_PREFIX", "")
+# Enable running on domain sub-path
+URL_PREFIX = os.getenv("URL_PREFIX", "")
+# Allow env override of default host
+FLASK_HOST = os.getenv("FLASK_HOST", "localhost")
+# Allow env override of default port 
+FLASK_PORT = os.getenv("FLASK_PORT", 5000) 
+
 
 swagger_config = {
     "openapi": "3.0.2",
             "route": "/soar_protocol.json",
         }
     ],
-    "url_prefix": url_prefix,
+    "url_prefix": URL_PREFIX,
     "paths": {},
     "components": {
         "schemas": {
     },
 }
 
-swag = Swagger(app, config=swagger_config, merge=True)
 
-flask_host = os.getenv(
-    "FLASK_HOST", "localhost"
-)  # Sets to whatever FLASK_HOST is, or defaults to localhost
+def resolve_ref(ref): 
+    """
+    Get schema URL, parse JSON 
+    Return None if either fails
+    """
+    try:
+        res = requests.get(ref)
+        if res.status_code == 200:
+            return res.json()
+        else: 
+            return None
+    except (json.JSONDecodeError, ValueError):
+        return None
+        
+
+def rename_ref(ref): 
+    """
+    Convert remote ref URL into a name that can 
+    be used for a local ref in the schema 
+    Remote the URL scheme and replace / with .
+    """
+    # remove url scheme
+    deschemed = re.sub(r"^[htps]*\:*[/]{2}", "", ref)
+    # replace / with . since the name will be in a path
+    return re.sub(r"[/]", ".", deschemed)
+
+
+def nested_replace(source, key, value, replace_with):
+    """
+    Find all instances of a key value pair in a nested 
+    dictionary and replace the value with replace_with
+    """ 
+    for k,v in source.items():
+        if k == key and v == value:
+            source[k] = replace_with
+        elif type(v) is list:
+            for item in v:
+                if type(item) is dict:
+                    nested_replace(item, key, value, replace_with)
+        if type(v) is dict:
+            nested_replace(v, key, value, replace_with)
+
+
+def inject_schema(schema, remote_ref):
+    """
+    Given a parent schema and a remote ref
+    
+    1. get the remote ref schema
+    2. create a local reference name (without path separators)
+    3. insert into components.schemas 
+    4. replace remote references with local references 
+
+    
+    returns True if resolved and injected
+    """ 
+    local_name = rename_ref(remote_ref)
+    local_ref = f"#/components/schemas/{local_name}"
+    ref_schema = resolve_ref(remote_ref)
+    if (ref_schema is not None):
+        nested_replace(schema, "$ref", remote_ref, local_ref)
+        schema["components"]["schemas"][local_name] = ref_schema
+        return True
+    else: 
+        return False    
+
+
+def import_remote_refs():
+    """
+    inject the following remote refs into the schema 
+    and replace the remote refs with local refs
+
+    returns True if all schemas resolved and injected 
+    """
+    ref_imports = [
+        "https://geojson.org/schema/Feature.json",
+        "https://geojson.org/schema/FeatureCollection.json",
+        "https://geojson.org/schema/LineString.json",
+        "https://geojson.org/schema/Point.json",
+        "https://geojson.org/schema/Polygon.json",
+    ]
+
+    return all([
+        inject_schema(swagger_config, ref)
+        for ref in ref_imports
+    ])
+
+
+import_remote_refs()
+
+
+def serve():
+    """
+    Run as local flask app on port 5000
+    """
+    app = Flask(__name__)
+    Swagger(app, config=swagger_config, merge=True)
+    
+    app.run(debug=False, host=FLASK_HOST, port=FLASK_PORT)
+
+
+def write_schema(swagger_config, file_path):
+    """
+    Dump schema to specified file
+    """
+    json_schema = json.dumps(swagger_config, indent=2)
+    with open(file_path, "w") as f: 
+        f.write(json_schema)
+
+
+def get_options():
+    """
+    Parse script arguments
+    """
+    parser = argparse.ArgumentParser(description="Generate the schema",
+                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-s", "--serve", dest="run_flask", action="store_true", help="Run flask app", default=False)
+    parser.add_argument("-f", "--file", dest="output_file", action="store_true", help="Save output to schema file", default=False)
+    args = parser.parse_args()
+    config = vars(args)
+    # If no flag is specified default to running the flask server
+    if (all(v is False for v in config.values())):
+        config["run_flask"] = True
+    return config    
+
+
+if __name__ == "__main__": 
+    # Parse script args 
+    config = get_options()    
+    
+    # Output compiled schema
+    if config.get("output_file"):
+        write_schema(swagger_config, "project/soar/swagger.json")
+
+    # Run flask app
+    if config.get("run_flask"):
+        serve()
 
-if __name__ == "__main__":
-    app.run(debug=False, host=flask_host)
+    
\ No newline at end of file

requirements.txt

 Flask
 flask-restx
 flasgger
+requests

[Dan Jones]

This is the experiment I did during the design sprint with compiling the geojson schemas into the backbone message format at build time so messages can be compiled against local refs derived from the published geojson schemas.

[Dan Jones]

GeoJSON Point accepts a minimum of 2 coordinates but sets no maximum.

If we use GeoJSON Point to specify lat/long do we want to include altitude/depth and if so do we just have a convention that says [x(long), y(lat), z(depth), a(?)(altitude)] so you can provide points as 2D, 3D, or 4D as required?

Alternatively do we just use GeoJSON Point for lat/long and keep depth/altitude as separate named parameters? Or do we have a 3D Point [x,y,z] with a parameter that says whether z refers to depth or altitude?

[Dan Jones]

Hmm - Importing the individual geographies works OK but for some reason importing the Feature or FeatureCollection schemas fails and I can't immediately see why.

We can probably do without FeatureCollection but I think we need Feature for the primitive list because we want primitives to accept different geometry types.

So far I've removed $id and $schema from the published schemas

I think removing properties that don't alter the structure or validation of the data is probably fine. I might be able to make Feature work with a bit more of that.

If I have to do anything else to make Feature work then I think it needs a different answer.

[Dan Jones]

This is a bug with our current pinned version of openapi-spec-validator and can be fixed by upgrading the package.