diff --git a/generate_schema_config.py b/generate_schema_config.py index cb3b2380c8fe019a91f2f6015b0bcd2558f9a33b..09687e3e03344915bc267e456e03eba4af0996a5 100644 --- a/generate_schema_config.py +++ b/generate_schema_config.py @@ -19,6 +19,7 @@ import json import os import re import requests +from urllib.parse import urlparse # Enable running on domain sub-path @@ -31,7 +32,10 @@ FLASK_PORT = os.getenv("FLASK_PORT", 5000) FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() in ("true", "1", "t") -def get_swagger_config(): +def get_swagger_config(reload=False): + if reload: + print("Reload specified: Ignoring cached refs") + swagger_config = { "openapi": "3.0.2", "swagger_ui": True, @@ -115,7 +119,7 @@ def get_swagger_config(): } }, } - import_remote_refs(swagger_config) + import_remote_refs(swagger_config, reload) return swagger_config @@ -191,7 +195,30 @@ def downgrade_schema_30x_compatible(schema): downgrade_schema_30x_compatible(propConfig["items"]) -def inject_schema(schema, remote_ref): +def get_remote_ref_cache_path(remote_ref): + parsed_ref = urlparse(remote_ref) + return f"remotes/{parsed_ref.hostname}{parsed_ref.path}" + + +def get_cached_ref(remote_ref): + ref_path = get_remote_ref_cache_path(remote_ref) + ref = None + if os.path.exists(ref_path): + print(f"loading cached ref: {remote_ref}") + with open(ref_path, 'r') as ref_file: + ref = json.load(ref_file) + return ref + + +def store_cached_ref(remote_ref, definition): + ref_path = get_remote_ref_cache_path(remote_ref) + ref_dirs = re.sub(r'\/[^\/]+$', '', ref_path) + os.makedirs(ref_dirs, 0o775, True) + with open(ref_path, 'w') as ref_file: + json.dump(definition, ref_file) + + +def inject_schema(schema, remote_ref, reload=False): """ Given a parent schema and a remote ref @@ -204,8 +231,14 @@ def inject_schema(schema, remote_ref): """ local_name = rename_ref(remote_ref) local_ref = f"#/components/schemas/{local_name}" - ref_schema = resolve_ref(remote_ref) - downgrade_schema_30x_compatible(ref_schema) + # get schema from cache if present + ref_schema = None if reload else get_cached_ref(remote_ref) + if not ref_schema: + print(f"ref not cached: {remote_ref}") + ref_schema = resolve_ref(remote_ref) + downgrade_schema_30x_compatible(ref_schema) + store_cached_ref(remote_ref, ref_schema) + if ref_schema is not None: nested_replace(schema, "$ref", remote_ref, local_ref) schema["components"]["schemas"][local_name] = ref_schema @@ -214,7 +247,7 @@ def inject_schema(schema, remote_ref): return False -def import_remote_refs(swagger_config): +def import_remote_refs(swagger_config, reload=False): """ inject the following remote refs into the schema and replace the remote refs with local refs @@ -235,7 +268,7 @@ def import_remote_refs(swagger_config): "https://geojson.org/schema/Polygon.json", ] - return all([inject_schema(swagger_config, ref) for ref in ref_imports]) + return all([inject_schema(swagger_config, ref, reload) for ref in ref_imports]) def configure_flask(swagger_config): @@ -339,6 +372,14 @@ def get_options(): help="Save output to schema file", default=False, ) + parser.add_argument( + "-r", + "--reload", + dest="reload_schemas", + action="store_true", + help="Overwrite local copies of remote reference schemas", + default=False, + ) parser.add_argument("filename", nargs="?", default="project/soar/swagger.json") args = parser.parse_args() config = vars(args) @@ -349,11 +390,11 @@ def get_options(): if __name__ == "__main__": - swagger_config = get_swagger_config() - # Parse script args config = get_options() + swagger_config = get_swagger_config(config.get('reload_schemas')) + # Output compiled schema if config.get("output_file"): write_schema(swagger_config, config.get("filename")) diff --git a/remotes/README.md b/remotes/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ecfffd47ad9e9f63aa5b81234fc3d6db53b36857 --- /dev/null +++ b/remotes/README.md @@ -0,0 +1,13 @@ +# Local store for remote refs + +You don't want to retrieve remote refs live for reliability and change +control. + +The schema retains the references to the remotes. + +The actual schemas are stored and committed locally. + +This means if there are breaking changes you can decide how and when to +move to the new definitions. + +The validation is then running against a collection of local refs. \ No newline at end of file diff --git a/remotes/geojson.org/schema/Feature.json b/remotes/geojson.org/schema/Feature.json new file mode 100644 index 0000000000000000000000000000000000000000..13e3a55e4d60454ad7aab239deef3c61ec170026 --- /dev/null +++ b/remotes/geojson.org/schema/Feature.json @@ -0,0 +1 @@ +{"title": "GeoJSON Feature", "type": "object", "required": ["type", "properties", "geometry"], "properties": {"type": {"type": "string", "enum": ["Feature"]}, "id": {"oneOf": [{"type": "number"}, {"type": "string"}]}, "properties": {"oneOf": [{"type": "object"}]}, "geometry": {"oneOf": [{"title": "GeoJSON Point", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Point"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "number"}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON LineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["LineString"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON Polygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Polygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPoint", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPoint"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiLineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiLineString"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPolygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPolygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON GeometryCollection", "type": "object", "required": ["type", "geometries"], "properties": {"type": {"type": "string", "enum": ["GeometryCollection"]}, "geometries": {"type": "array", "items": {"oneOf": [{"title": "GeoJSON Point", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Point"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "number"}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON LineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["LineString"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON Polygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Polygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPoint", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPoint"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiLineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiLineString"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPolygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPolygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}]}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}]}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/FeatureCollection.json b/remotes/geojson.org/schema/FeatureCollection.json new file mode 100644 index 0000000000000000000000000000000000000000..ecdfee1c0367cb50d92b915e05f0df7e6d678d0d --- /dev/null +++ b/remotes/geojson.org/schema/FeatureCollection.json @@ -0,0 +1 @@ +{"title": "GeoJSON FeatureCollection", "type": "object", "required": ["type", "features"], "properties": {"type": {"type": "string", "enum": ["FeatureCollection"]}, "features": {"type": "array", "items": {"title": "GeoJSON Feature", "type": "object", "required": ["type", "properties", "geometry"], "properties": {"type": {"type": "string", "enum": ["Feature"]}, "id": {"oneOf": [{"type": "number"}, {"type": "string"}]}, "properties": {"oneOf": [{"type": "object"}]}, "geometry": {"oneOf": [{"title": "GeoJSON Point", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Point"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "number"}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON LineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["LineString"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON Polygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Polygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPoint", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPoint"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiLineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiLineString"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPolygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPolygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON GeometryCollection", "type": "object", "required": ["type", "geometries"], "properties": {"type": {"type": "string", "enum": ["GeometryCollection"]}, "geometries": {"type": "array", "items": {"oneOf": [{"title": "GeoJSON Point", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Point"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "number"}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON LineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["LineString"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON Polygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Polygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPoint", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPoint"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiLineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiLineString"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}, {"title": "GeoJSON MultiPolygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPolygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}]}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}]}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/LineString.json b/remotes/geojson.org/schema/LineString.json new file mode 100644 index 0000000000000000000000000000000000000000..a573e44aad45ef5a527a22357433cd2fbc98a77e --- /dev/null +++ b/remotes/geojson.org/schema/LineString.json @@ -0,0 +1 @@ +{"title": "GeoJSON LineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["LineString"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/MultiLineString.json b/remotes/geojson.org/schema/MultiLineString.json new file mode 100644 index 0000000000000000000000000000000000000000..b459b30b355034fc73754d6d44fa803348dd459b --- /dev/null +++ b/remotes/geojson.org/schema/MultiLineString.json @@ -0,0 +1 @@ +{"title": "GeoJSON MultiLineString", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiLineString"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/MultiPoint.json b/remotes/geojson.org/schema/MultiPoint.json new file mode 100644 index 0000000000000000000000000000000000000000..8520decd5f44a2a9d0d648179e50393602d6855f --- /dev/null +++ b/remotes/geojson.org/schema/MultiPoint.json @@ -0,0 +1 @@ +{"title": "GeoJSON MultiPoint", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPoint"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/MultiPolygon.json b/remotes/geojson.org/schema/MultiPolygon.json new file mode 100644 index 0000000000000000000000000000000000000000..4369fef29c517251036adc5a9ba31700dc42036d --- /dev/null +++ b/remotes/geojson.org/schema/MultiPolygon.json @@ -0,0 +1 @@ +{"title": "GeoJSON MultiPolygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["MultiPolygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/Point.json b/remotes/geojson.org/schema/Point.json new file mode 100644 index 0000000000000000000000000000000000000000..8c94619ee6b8ac646e6d01e8d86105d96269afa3 --- /dev/null +++ b/remotes/geojson.org/schema/Point.json @@ -0,0 +1 @@ +{"title": "GeoJSON Point", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Point"]}, "coordinates": {"type": "array", "minItems": 2, "items": {"type": "number"}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file diff --git a/remotes/geojson.org/schema/Polygon.json b/remotes/geojson.org/schema/Polygon.json new file mode 100644 index 0000000000000000000000000000000000000000..dedbdca617cf1375e99f61ec549afa8454ae2523 --- /dev/null +++ b/remotes/geojson.org/schema/Polygon.json @@ -0,0 +1 @@ +{"title": "GeoJSON Polygon", "type": "object", "required": ["type", "coordinates"], "properties": {"type": {"type": "string", "enum": ["Polygon"]}, "coordinates": {"type": "array", "items": {"type": "array", "minItems": 4, "items": {"type": "array", "minItems": 2, "items": {"type": "number"}}}}, "bbox": {"type": "array", "minItems": 4, "items": {"type": "number"}}}} \ No newline at end of file