Skip to content

GitLab

Commit 52334de6 authored by Dan Jones's avatar Dan Jones
Browse files

Merge branch '70-refactor-geojson-loading-for-serving-from-nucleus' into '67-mas-dt' 

Resolve "Refactor GeoJSON loading for serving from nucleus"

See merge request !39
parents 8b622f22 a1d6143c
dev 80-add-informational-type-primitive 87-remove-comment-for-platform-status-state action-refactor-test master v2.0.0 v2.0.0-beta.11 v2.0.0-beta.10 v2.0.0-beta.9 v2.0.0-beta.8 v2.0.0-beta.7 v2.0.0-beta.6 v2.0.0-beta.5 v2.0.0-beta.4 v2.0.0-beta.3 v2.0.0-beta.2 v2.0.0-beta.1
3 merge requests!59Release v2.0.0,!39Resolve "Refactor GeoJSON loading for serving from nucleus",!36Resolve "Create temporary mas-dt branch for initial schema refactor"
Pipeline #231512 passed with stages
in 1 minute and 3 seconds
Hide whitespace changes
Inline Side-by-side
Showing with 1297 additions and 10 deletions
CHANGELOG.md
View file @ 52334de6
......@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
### Changed
- Store remote schemas as committed local files
This prevents the live runtime loading untested changes
- Refactored planning_configuration message definition
- Remove region_of_interest and exclusion_zones
- Add primitives array of classified geojson Features
......@@ -49,7 +51,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
### Added
JSON schema definitions for the SoAR project
- outer wrapper MESSAGE definition
- outer wrapper MESSAGE definition
- header object
- payload object
- acknowledgement
......
generate_schema_config.py
View file @ 52334de6
......@@ -19,6 +19,7 @@ import json
import os
import re
import requests
from urllib.parse import urlparse
# Enable running on domain sub-path
......@@ -31,7 +32,10 @@ FLASK_PORT = os.getenv("FLASK_PORT", 5000)
FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() in ("true", "1", "t")
def get_swagger_config():
def get_swagger_config(reload=False):
if reload:
print("Reload specified: Ignoring cached refs")
swagger_config = {
"openapi": "3.0.2",
"swagger_ui": True,
......@@ -115,7 +119,7 @@ def get_swagger_config():
}
},
}
import_remote_refs(swagger_config)
import_remote_refs(swagger_config, reload)
return swagger_config
......@@ -191,7 +195,30 @@ def downgrade_schema_30x_compatible(schema):
downgrade_schema_30x_compatible(propConfig["items"])
def inject_schema(schema, remote_ref):
def get_remote_ref_cache_path(remote_ref):
parsed_ref = urlparse(remote_ref)
return f"remotes/{parsed_ref.hostname}{parsed_ref.path}"
def get_cached_ref(remote_ref):
ref_path = get_remote_ref_cache_path(remote_ref)
ref = None
if os.path.exists(ref_path):
print(f"loading cached ref: {remote_ref}")
with open(ref_path, "r") as ref_file:
ref = json.load(ref_file)
return ref
def store_cached_ref(remote_ref, definition):
ref_path = get_remote_ref_cache_path(remote_ref)
ref_dirs = re.sub(r"\/[^\/]+$", "", ref_path)
os.makedirs(ref_dirs, 0o775, True)
with open(ref_path, "w") as ref_file:
json.dump(definition, ref_file, indent=2)
def inject_schema(schema, remote_ref, reload=False):
"""
Given a parent schema and a remote ref
......@@ -204,8 +231,14 @@ def inject_schema(schema, remote_ref):
"""
local_name = rename_ref(remote_ref)
local_ref = f"#/components/schemas/{local_name}"
ref_schema = resolve_ref(remote_ref)
downgrade_schema_30x_compatible(ref_schema)
# get schema from cache if present
ref_schema = None if reload else get_cached_ref(remote_ref)
if not ref_schema:
print(f"ref not cached: {remote_ref}")
ref_schema = resolve_ref(remote_ref)
downgrade_schema_30x_compatible(ref_schema)
store_cached_ref(remote_ref, ref_schema)
if ref_schema is not None:
nested_replace(schema, "$ref", remote_ref, local_ref)
schema["components"]["schemas"][local_name] = ref_schema
......@@ -214,7 +247,7 @@ def inject_schema(schema, remote_ref):
return False
def import_remote_refs(swagger_config):
def import_remote_refs(swagger_config, reload=False):
"""
inject the following remote refs into the schema
and replace the remote refs with local refs
......@@ -235,7 +268,7 @@ def import_remote_refs(swagger_config):
"https://geojson.org/schema/Polygon.json",
]
return all([inject_schema(swagger_config, ref) for ref in ref_imports])
return all([inject_schema(swagger_config, ref, reload) for ref in ref_imports])
def configure_flask(swagger_config):
......@@ -339,6 +372,14 @@ def get_options():
help="Save output to schema file",
default=False,
)
parser.add_argument(
"-r",
"--reload",
dest="reload_schemas",
action="store_true",
help="Overwrite local copies of remote reference schemas",
default=False,
)
parser.add_argument("filename", nargs="?", default="project/soar/swagger.json")
args = parser.parse_args()
config = vars(args)
......@@ -349,11 +390,11 @@ def get_options():
if __name__ == "__main__":
swagger_config = get_swagger_config()
# Parse script args
config = get_options()
swagger_config = get_swagger_config(config.get("reload_schemas"))
# Output compiled schema
if config.get("output_file"):
write_schema(swagger_config, config.get("filename"))
......
remotes/README.md 0 → 100644
View file @ 52334de6
# Local store for remote refs
You don't want to retrieve remote refs live for reliability and change
control.
The schema retains the references to the remotes.
The actual schemas are stored and committed locally.
This means if there are breaking changes you can decide how and when to
move to the new definitions.
The validation is then running against a collection of local refs.
\ No newline at end of file
remotes/geojson.org/schema/Feature.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON Feature",
"type": "object",
"required": [
"type",
"properties",
"geometry"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Feature"
]
},
"id": {
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
},
"properties": {
"oneOf": [
{
"type": "object"
}
]
},
"geometry": {
"oneOf": [
{
"title": "GeoJSON Point",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Point"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON LineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"LineString"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON Polygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Polygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPoint",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPoint"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiLineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiLineString"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPolygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPolygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON GeometryCollection",
"type": "object",
"required": [
"type",
"geometries"
],
"properties": {
"type": {
"type": "string",
"enum": [
"GeometryCollection"
]
},
"geometries": {
"type": "array",
"items": {
"oneOf": [
{
"title": "GeoJSON Point",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Point"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON LineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"LineString"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON Polygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Polygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPoint",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPoint"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiLineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiLineString"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPolygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPolygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
]
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
]
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/FeatureCollection.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON FeatureCollection",
"type": "object",
"required": [
"type",
"features"
],
"properties": {
"type": {
"type": "string",
"enum": [
"FeatureCollection"
]
},
"features": {
"type": "array",
"items": {
"title": "GeoJSON Feature",
"type": "object",
"required": [
"type",
"properties",
"geometry"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Feature"
]
},
"id": {
"oneOf": [
{
"type": "number"
},
{
"type": "string"
}
]
},
"properties": {
"oneOf": [
{
"type": "object"
}
]
},
"geometry": {
"oneOf": [
{
"title": "GeoJSON Point",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Point"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON LineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"LineString"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON Polygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Polygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPoint",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPoint"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiLineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiLineString"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPolygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPolygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON GeometryCollection",
"type": "object",
"required": [
"type",
"geometries"
],
"properties": {
"type": {
"type": "string",
"enum": [
"GeometryCollection"
]
},
"geometries": {
"type": "array",
"items": {
"oneOf": [
{
"title": "GeoJSON Point",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Point"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON LineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"LineString"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON Polygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Polygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPoint",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPoint"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiLineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiLineString"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
},
{
"title": "GeoJSON MultiPolygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPolygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
]
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
]
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/LineString.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON LineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"LineString"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/MultiLineString.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON MultiLineString",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiLineString"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/MultiPoint.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON MultiPoint",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPoint"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/MultiPolygon.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON MultiPolygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"MultiPolygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/Point.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON Point",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Point"
]
},
"coordinates": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
remotes/geojson.org/schema/Polygon.json 0 → 100644
View file @ 52334de6
{
"title": "GeoJSON Polygon",
"type": "object",
"required": [
"type",
"coordinates"
],
"properties": {
"type": {
"type": "string",
"enum": [
"Polygon"
]
},
"coordinates": {
"type": "array",
"items": {
"type": "array",
"minItems": 4,
"items": {
"type": "array",
"minItems": 2,
"items": {
"type": "number"
}
}
}
},
"bbox": {
"type": "array",
"minItems": 4,
"items": {
"type": "number"
}
}
}
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment