ci: run both js and python tests in ci

- use discover to find python unittests
- choose filename for output compiled schema
- use python+node container for JS tests
- compile ignored test schema before running all tests

.gitignore

 __pycache__/
 *.pyc
 
+tests/test_swagger.json
\ No newline at end of file

.gitlab-ci.yml

@@ -2,6 +2,7 @@ include:
   - project: communications-backbone-system/backbone-infrastructure-config
     ref: master
     file: gitlab/all.yml
+  - local: /gitlab/test-js.yml
 
 variables:
   DOCKER_IMAGE_NAME: backbone-message-format

CHANGELOG.md

@@ -23,6 +23,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 - Refactor schema script to remove invalid definitions object automatically
 - Refactor generate_schema_config script to output file on -f flag 
+- Use discover to find all unit tests 
+- Run tests against current format definitions 
+- Test that formats match saved schema 
+- Run python and javascript tests in CI
 
 ## [v0.1.0] - 2023-03-24
 

README.md

@@ -78,14 +78,22 @@ python3 generate_schema_config.py -f
 2. Commit any changes to the `backbone-message-format/project/<project_name>/swagger.json`
 
 ## Run Tests
+
+We have found slightly different behaviour between the OpenAPI validators in python and javascript.
+
+By running tests in both languages we should protect against messages that pass validation in 
+one client but fail in another. 
+
 Run both tests below:
-1. Test 1
+
+1. Test 1 - Use python validators
 ```
-python3 -m unittest tests/test_schemas.py
+python3 -m unittest discover
 ```
-2. Test 2
+2. Test 2 - Use javascript validators
 ```
-cd tests-js/docker; docker compose up --build
+# Compile schema and run javascript validation tests in docker
+python3 test-js.py
 ```
 
 ## Quick Links

docker/docker-compose-test.yaml

@@ -12,4 +12,4 @@ services:
       - PYTHONDONTWRITEBYTECODE=1
     volumes:
       - ../:/app
-    command: "python3 -m unittest tests/test_schemas.py"
\ No newline at end of file
+    command: "python3 -m unittest discover"

generate_schema_config.py

@@ -28,7 +28,6 @@ FLASK_PORT = os.getenv("FLASK_PORT", 5000)
 # Switch on debug mode if env var is truthy
 FLASK_DEBUG = os.getenv("FLASK_DEBUG", "False").lower() in ("true", "1", "t")
 
-
 swagger_config = {
     "openapi": "3.0.2",
     "swagger_ui": True,
@@ -214,6 +213,7 @@ def get_options():
         help="Save output to schema file",
         default=False,
     )
+    parser.add_argument("filename", nargs="?", default="project/soar/swagger.json")
     args = parser.parse_args()
     config = vars(args)
     # If no flag is specified default to running the flask server
@@ -228,7 +228,7 @@ if __name__ == "__main__":
 
     # Output compiled schema
     if config.get("output_file"):
-        write_schema(swagger_config, "project/soar/swagger.json")
+        write_schema(swagger_config, config.get("filename"))
 
     # Run flask app
     if config.get("run_flask"):

gitlab/test-js.yml

+test-js:
+  stage: test
+  script:
+    - cd tests-js/docker
+    - docker compose up --build
+  tags:
+    - shell
+  rules:
+    - if: '$CI_SKIP_TESTS == "1"'
+      when: never
+    - if: '$CI_IGNORE_TESTS == "1"'
+      allow_failure: true
+    - when: on_success

tests-js/docker/Dockerfile

-FROM node:18.7.0-alpine 
+FROM python:3.9.16-alpine
+
+# Add node for running JS validator tests 
+RUN apk add --update nodejs npm
 
 WORKDIR /app/tests-js
 
 COPY tests-js/package.json /app/tests-js/package.json 
 
+RUN npm install -g yarn
+
 RUN yarn install 
 
 WORKDIR /app 
 
 COPY . /app
 
+# compile test schema from message 
+RUN pip install -r requirements.txt
+RUN python generate_schema_config.py -f tests/test_swagger.json
+
 WORKDIR /app/tests-js
 
 CMD [ 'yarn', 'test' ]

tests-js/soar-examples.test.js

@@ -3,7 +3,7 @@ const Validator = require('swagger-model-validator');
 const OpenAPISchemaValidator = require('openapi-schema-validator').default;
 
 const getSchema = () => {
-  const schema = require(`${__dirname}/../project/soar/swagger.json`);
+  const schema = require(`${__dirname}/../tests/test_swagger.json`);
   return schema;
 };
 

tests/test_schemas.py

@@ -5,12 +5,28 @@ from jsonschema.validators import RefResolver
 import unittest
 import json
 import os
+from generate_schema_config import write_schema, swagger_config
 
 
-SCHEMA_DIR = "project/soar/swagger.json"
 MOCK_DATA_DIR = "examples/"
 
 
+class SchemaTestCase(unittest.TestCase):
+    def load_schema(self, file_path=None):
+        if file_path:
+            schema_path = file_path
+        else:
+            schema_path = os.getenv("SCHEMA_PATH")
+        schema, _ = read_from_filename(schema_path)
+        return schema
+
+    @classmethod
+    def setUpClass(cls):
+        test_schema_path = "tests/test_swagger.json"
+        os.environ["SCHEMA_PATH"] = test_schema_path
+        write_schema(swagger_config, test_schema_path)
+
+
 class SchemaError(Exception):
     """
     Test config specs of swagger.json for projects
@@ -19,26 +35,37 @@ class SchemaError(Exception):
     def __init__(self, exception_type, message):
         self.type = exception_type
         self.message = message
-        super().__init__(self.message)
+        super().__init__(message)
 
 
-class TestSpecs(unittest.TestCase):
+class TestCompiledSoarSpec(SchemaTestCase):
+    """
+    Test the saved version of the compiled schema has been updated
+    """
+
+    def test_compiled_spec_matches_definitions(self):
+        print("TEST: definitions match project/soar/swagger.json")
+        derived_schema = self.load_schema()
+        saved_schema = self.load_schema("project/soar/swagger.json")
+        self.assertEqual(saved_schema, derived_schema)
+
+
+class TestSpecs(SchemaTestCase):
     def test_swagger_specs(self):
         """
         Test specs (swagger.json generated from generate_schema_config.py)
         """
-        schema, spec_url = read_from_filename("tests/fixtures/swagger.json")
+        print("TEST: compiled schema matches openapi v3 spec")
+        schema = self.load_schema()
         self.assertIsNone(openapi_v30_spec_validator.validate(schema))
 
 
-class TestAllMessageExamples(unittest.TestCase):
+class TestAllMessageExamples(SchemaTestCase):
     def test_schema_specs(self):
         """
         Test specs (swagger.json generated from generate_schema_config.py)
         """
-        schema_ref = open(SCHEMA_DIR)
-        schema = json.load(schema_ref)
-        schema_ref.close()
+        schema = self.load_schema()
 
         partner_dir_list = os.listdir(MOCK_DATA_DIR)
 
@@ -70,8 +97,6 @@ class TestAllMessageExamples(unittest.TestCase):
                         f.close()
                         print("Done.")
 
-        schema_ref.close()
-
 
 if __name__ == "__main__":
     unittest.main()