From 83e5783e37f28bbe0a59ad8bdeeba1bb167c97c6 Mon Sep 17 00:00:00 2001
From: Dan Jones <danjon@noc.ac.uk>
Date: Fri, 23 Aug 2024 10:27:26 +0100
Subject: [PATCH] ci: run both js and python tests in ci
- use discover to find python unittests
- choose filename for output compiled schema
- use python+node container for JS tests
- compile ignored test schema before running all tests
---
.gitignore | 1 +
.gitlab-ci.yml | 1 +
CHANGELOG.md | 4 +++
README.md | 16 +++++++++---
docker/docker-compose-test.yaml | 2 +-
generate_schema_config.py | 4 +--
gitlab/test-js.yml | 13 ++++++++++
tests-js/docker/Dockerfile | 11 +++++++-
tests-js/soar-examples.test.js | 2 +-
tests/test_schemas.py | 45 +++++++++++++++++++++++++--------
10 files changed, 80 insertions(+), 19 deletions(-)
create mode 100644 gitlab/test-js.yml
diff --git a/.gitignore b/.gitignore
index 41e951a..fc272ee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
__pycache__/
*.pyc
+tests/test_swagger.json
\ No newline at end of file
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 513b309..d4e696e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,6 +2,7 @@ include:
- project: communications-backbone-system/backbone-infrastructure-config
ref: master
file: gitlab/all.yml
+ - local: /gitlab/test-js.yml
variables:
DOCKER_IMAGE_NAME: backbone-message-format
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5ca1898..c1eda55 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- Refactor schema script to remove invalid definitions object automatically
- Refactor generate_schema_config script to output file on -f flag
+- Use discover to find all unit tests
+- Run tests against current format definitions
+- Test that formats match saved schema
+- Run python and javascript tests in CI
## [v0.1.0] - 2023-03-24
diff --git a/README.md b/README.md
index 1588222..081c9d8 100644
--- a/README.md
+++ b/README.md
@@ -78,14 +78,22 @@ python3 generate_schema_config.py -f
2. Commit any changes to the `backbone-message-format/project/<project_name>/swagger.json`
## Run Tests
+
+We have found slightly different behaviour between the OpenAPI validators in python and javascript.
+
+By running tests in both languages we should protect against messages that pass validation in
+one client but fail in another.
+
Run both tests below:
-1. Test 1
+
+1. Test 1 - Use python validators
```
-python3 -m unittest tests/test_schemas.py
+python3 -m unittest discover
```
-2. Test 2
+2. Test 2 - Use javascript validators
```
-cd tests-js/docker; docker compose up --build
+# Compile schema and run javascript validation tests in docker
+python3 test-js.py
```
## Quick Links
diff --git a/docker/docker-compose-test.yaml b/docker/docker-compose-test.yaml
index 2461af3..6caad2f 100644
--- a/docker/docker-compose-test.yaml
+++ b/docker/docker-compose-test.yaml
@@ -12,4 +12,4 @@ services:
- PYTHONDONTWRITEBYTECODE=1
volumes:
- ../:/app
- command: "python3 -m unittest tests/test_schemas.py"
\ No newline at end of file
+ command: "python3 -m unittest discover"
diff --git a/generate_schema_config.py b/generate_schema_config.py
index 07e9fdf..fa58bb1 100644
--- a/generate_schema_config.py
+++ b/generate_schema_config.py
@@ -28,7 +28,6 @@ FLASK_PORT = os.getenv("FLASK_PORT", 5000)
# Switch on debug mode if env var is truthy
FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() in ("true", "1", "t")
-
swagger_config = {
"openapi": "3.0.2",
"swagger_ui": True,
@@ -214,6 +213,7 @@ def get_options():
help="Save output to schema file",
default=False,
)
+ parser.add_argument("filename", nargs="?", default="project/soar/swagger.json")
args = parser.parse_args()
config = vars(args)
# If no flag is specified default to running the flask server
@@ -228,7 +228,7 @@ if __name__ == "__main__":
# Output compiled schema
if config.get("output_file"):
- write_schema(swagger_config, "project/soar/swagger.json")
+ write_schema(swagger_config, config.get("filename"))
# Run flask app
if config.get("run_flask"):
diff --git a/gitlab/test-js.yml b/gitlab/test-js.yml
new file mode 100644
index 0000000..d773334
--- /dev/null
+++ b/gitlab/test-js.yml
@@ -0,0 +1,13 @@
+test-js:
+ stage: test
+ script:
+ - cd tests-js/docker
+ - docker compose up --build
+ tags:
+ - shell
+ rules:
+ - if: '$CI_SKIP_TESTS == "1"'
+ when: never
+ - if: '$CI_IGNORE_TESTS == "1"'
+ allow_failure: true
+ - when: on_success
diff --git a/tests-js/docker/Dockerfile b/tests-js/docker/Dockerfile
index 8e06b59..27076ba 100644
--- a/tests-js/docker/Dockerfile
+++ b/tests-js/docker/Dockerfile
@@ -1,15 +1,24 @@
-FROM node:18.7.0-alpine
+FROM python:3.9.16-alpine
+
+# Add node for running JS validator tests
+RUN apk add --update nodejs npm
WORKDIR /app/tests-js
COPY tests-js/package.json /app/tests-js/package.json
+RUN npm install -g yarn
+
RUN yarn install
WORKDIR /app
COPY . /app
+# compile test schema from message
+RUN pip install -r requirements.txt
+RUN python generate_schema_config.py -f tests/test_swagger.json
+
WORKDIR /app/tests-js
CMD [ 'yarn', 'test' ]
diff --git a/tests-js/soar-examples.test.js b/tests-js/soar-examples.test.js
index 22abf84..56141d1 100644
--- a/tests-js/soar-examples.test.js
+++ b/tests-js/soar-examples.test.js
@@ -3,7 +3,7 @@ const Validator = require('swagger-model-validator');
const OpenAPISchemaValidator = require('openapi-schema-validator').default;
const getSchema = () => {
- const schema = require(`${__dirname}/../project/soar/swagger.json`);
+ const schema = require(`${__dirname}/../tests/test_swagger.json`);
return schema;
};
diff --git a/tests/test_schemas.py b/tests/test_schemas.py
index 2f9d465..421c48b 100644
--- a/tests/test_schemas.py
+++ b/tests/test_schemas.py
@@ -5,12 +5,28 @@ from jsonschema.validators import RefResolver
import unittest
import json
import os
+from generate_schema_config import write_schema, swagger_config
-SCHEMA_DIR = "project/soar/swagger.json"
MOCK_DATA_DIR = "examples/"
+class SchemaTestCase(unittest.TestCase):
+ def load_schema(self, file_path=None):
+ if file_path:
+ schema_path = file_path
+ else:
+ schema_path = os.getenv("SCHEMA_PATH")
+ schema, _ = read_from_filename(schema_path)
+ return schema
+
+ @classmethod
+ def setUpClass(cls):
+ test_schema_path = "tests/test_swagger.json"
+ os.environ["SCHEMA_PATH"] = test_schema_path
+ write_schema(swagger_config, test_schema_path)
+
+
class SchemaError(Exception):
"""
Test config specs of swagger.json for projects
@@ -19,26 +35,37 @@ class SchemaError(Exception):
def __init__(self, exception_type, message):
self.type = exception_type
self.message = message
- super().__init__(self.message)
+ super().__init__(message)
-class TestSpecs(unittest.TestCase):
+class TestCompiledSoarSpec(SchemaTestCase):
+ """
+ Test the saved version of the compiled schema has been updated
+ """
+
+ def test_compiled_spec_matches_definitions(self):
+ print("TEST: definitions match project/soar/swagger.json")
+ derived_schema = self.load_schema()
+ saved_schema = self.load_schema("project/soar/swagger.json")
+ self.assertEqual(saved_schema, derived_schema)
+
+
+class TestSpecs(SchemaTestCase):
def test_swagger_specs(self):
"""
Test specs (swagger.json generated from generate_schema_config.py)
"""
- schema, spec_url = read_from_filename("tests/fixtures/swagger.json")
+ print("TEST: compiled schema matches openapi v3 spec")
+ schema = self.load_schema()
self.assertIsNone(openapi_v30_spec_validator.validate(schema))
-class TestAllMessageExamples(unittest.TestCase):
+class TestAllMessageExamples(SchemaTestCase):
def test_schema_specs(self):
"""
Test specs (swagger.json generated from generate_schema_config.py)
"""
- schema_ref = open(SCHEMA_DIR)
- schema = json.load(schema_ref)
- schema_ref.close()
+ schema = self.load_schema()
partner_dir_list = os.listdir(MOCK_DATA_DIR)
@@ -70,8 +97,6 @@ class TestAllMessageExamples(unittest.TestCase):
f.close()
print("Done.")
- schema_ref.close()
-
if __name__ == "__main__":
unittest.main()
--
GitLab