Resolve "Test validating messages with references to GeoJson schema in both JS and python"

Closes #45 (closed)

formats/planning_configuration.py

 region_schema = {
     "type": "object",
     "properties": {
-        "geometry_coordinates": {
-            "type": "array",
-            "example": [
-                [
-                    [-4.1777839187560915, 50.34173405662855],
-                    [-4.1777839187560915, 50.33820949229701],
-                    [-4.143667777943875, 50.33820949229701],
-                    [-4.143667777943875, 50.34173405662855],
-                    [-4.1777839187560915, 50.34173405662855],
-                ]
-            ],
+        "geometry": {
+            "$ref": "https://geojson.org/schema/Polygon.json",
         },
     },
-    "description": "Using GEOJSON, exact 4-point region (rectangle shaped - 5 points)",
+    "description": "GeoJSON Polygon",
 }
 
 squad_metadata_schema = {

.flake8

+[flake8]
+# Black formats code with max line length of 88
+# Bugbears B950 warning allows lines to be 10% longer than flake8's max
+max-line-length = 80
+
+# Errors and warnings to report
+select =
+    C,
+    E,
+    F,
+    W,
+    B,
+    # Allow max line length to be exceeded by 10% before warning.
+    B950
+
+# Errors and warnings to ignore
+ignore =
+    # Black puts space around slice operator which flake8 errors on.
+    E203,
+    # No error on max line length exceeded as B950 is doing this for us.
+    E501,
+    # Black puts line breaks before binary operators so don't warn this.
+    W503
+    B036
+    E999
+
+# Folders to ignore
+exclude =
+    .git,
+    *.pyc,
+    __pycache__,
+    ./docker,
+    ./docs,
+    ./kustomize
+    ./requirements.txt
+    ./requirements-dev.txt
+    ./CHANGELOG.md 
+    ./CONTRIBUTING.md

generate_schema_config.py

 from flasgger import Swagger
 from flask import Flask
 
+import json
 import os
+import re
+import requests
 
-app = Flask(__name__)
 
 url_prefix = os.getenv("URL_PREFIX", "")
 
     },
 }
 
-swag = Swagger(app, config=swagger_config, merge=True)
+
+def resolve_ref(ref):
+    """
+    Get schema URL, parse JSON
+    Return None if either fails
+    """
+    try:
+        res = requests.get(ref)
+        if res.status_code == 200:
+            return res.json()
+        else:
+            return None
+    except (json.JSONDecodeError, ValueError):
+        return None
+
+
+def rename_ref(ref):
+    """
+    Convert remote ref URL into a name that can
+    be used for a local ref in the schema
+    Remote the URL scheme and replace / with .
+    """
+    # remove url scheme
+    deschemed = re.sub(r"^[htps]*\:*[/]{2}", "", ref)
+    # replace / with . since the name will be in a path
+    return re.sub(r"[/]", ".", deschemed)
+
+
+def nested_replace(source, key, value, replace_with):
+    """
+    Find all instances of a key value pair in a nested
+    dictionary and replace the value with replace_with
+    """
+    for k, v in source.items():
+        if k == key and v == value:
+            source[k] = replace_with
+        elif type(v) is list:
+            for item in v:
+                if type(item) is dict:
+                    nested_replace(item, key, value, replace_with)
+        if type(v) is dict:
+            nested_replace(v, key, value, replace_with)
+
+
+def inject_schema(schema, remote_ref):
+    """
+    Given a parent schema and a remote ref
+
+    1. get the remote ref schema
+    2. create a local reference name (without path separators)
+    3. insert into components.schemas
+    4. replace remote references with local references
+
+
+    returns True if resolved and injected
+    """
+    local_name = rename_ref(remote_ref)
+    local_ref = f"#/components/schemas/{local_name}"
+    ref_schema = resolve_ref(remote_ref)
+    if ref_schema is not None:
+        nested_replace(schema, "$ref", remote_ref, local_ref)
+        schema["components"]["schemas"][local_name] = ref_schema
+        return True
+    else:
+        return False
+
+
+def import_remote_refs():
+    """
+    inject the following remote refs into the schema
+    and replace the remote refs with local refs
+
+    returns True if all schemas resolved and injected
+    """
+    ref_imports = [
+        "https://geojson.org/schema/Feature.json",
+        "https://geojson.org/schema/FeatureCollection.json",
+        "https://geojson.org/schema/LineString.json",
+        "https://geojson.org/schema/Point.json",
+        "https://geojson.org/schema/Polygon.json",
+    ]
+
+    return all([inject_schema(swagger_config, ref) for ref in ref_imports])
+
+
+import_remote_refs()
+
+app = Flask(__name__)
+Swagger(app, config=swagger_config, merge=True)
 
 flask_host = os.getenv(
     "FLASK_HOST", "localhost"

requirements.txt

 Flask
 flask-restx
 flasgger
+requests

[Dan Jones]

This is the experiment I did during the design sprint with compiling the geojson schemas into the backbone message format at build time so messages can be compiled against local refs derived from the published geojson schemas.

[Dan Jones]

GeoJSON Point accepts a minimum of 2 coordinates but sets no maximum.

If we use GeoJSON Point to specify lat/long do we want to include altitude/depth and if so do we just have a convention that says [x(long), y(lat), z(depth), a(?)(altitude)] so you can provide points as 2D, 3D, or 4D as required?

Alternatively do we just use GeoJSON Point for lat/long and keep depth/altitude as separate named parameters? Or do we have a 3D Point [x,y,z] with a parameter that says whether z refers to depth or altitude?

[Dan Jones]

Hmm - Importing the individual geographies works OK but for some reason importing the Feature or FeatureCollection schemas fails and I can't immediately see why.

We can probably do without FeatureCollection but I think we need Feature for the primitive list because we want primitives to accept different geometry types.

So far I've removed $id and $schema from the published schemas

I think removing properties that don't alter the structure or validation of the data is probably fine. I might be able to make Feature work with a bit more of that.

If I have to do anything else to make Feature work then I think it needs a different answer.

[Dan Jones]

This is a bug with our current pinned version of openapi-spec-validator and can be fixed by upgrading the package.