Add SPEC_TEST_READY env var

importers/test/test_djornl_parser.py

@@ -238,11 +238,3 @@ def test_duplicate_cluster_data(self):
             cluster_data,
             self.json_data["load_clusters"]
         )
-
-    def test_the_full_shebang(self):
-
-        RES_ROOT_DATA_PATH = os.path.join(_TEST_DIR, 'djornl', 'test_data')
-        parser = self.init_parser_with_path(RES_ROOT_DATA_PATH)
-
-        parser.load_data()
-        self.assertEqual(True, parser.load_data())

importers/test/test_djornl_parser_integration.py

@@ -0,0 +1,27 @@
+"""
+Tests for the DJORNL Parser
+
+At the present time, this just ensures that the files are parsed correctly;
+it does not check data loading into the db.
+"""
+import unittest
+import os
+
+from importers.djornl.parser import DJORNL_Parser
+from spec.test.helpers import modified_environ, check_spec_test_env
+
+_TEST_DIR = '/app/spec/test'
+
+
+class Test_DJORNL_Parser_Integration(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        check_spec_test_env()
+
+    def test_the_full_shebang(self):
+
+        with modified_environ(RES_ROOT_DATA_PATH=os.path.join(_TEST_DIR, 'djornl', 'test_data')):
+            parser = DJORNL_Parser()
+            parser.load_data()
+            self.assertEqual(True, parser.load_data())

relation_engine_server/test/test_api_v1.py

@@ -121,6 +121,10 @@ def test_update_specs(self):
         resp_json = resp.json()
         self.assertEqual(resp.status_code, 200)
         self.assertTrue(len(resp_json['status']))
+
+        # delete the SPEC_TEST_READY env var as it is no longer true
+        os.environ.pop('SPEC_TEST_READY', None)
+
         # Test that the indexes get created and not duplicated
         url = _CONF['db_url'] + '/_api/index'
         auth = (_CONF['db_user'], _CONF['db_pass'])
@@ -161,7 +165,7 @@ def test_list_data_sources(self):
         # /data_sources is used by the UI and requires slightly different response formatting
         # /specs/data_sources is in the standard /specs format used by collections and stored_queries
 
-        data_sources = ['djornl', 'envo_ontology', 'go_ontology', 'gtdb', 'ncbi_taxonomy', 'rdp_taxonomy']
+        data_sources = ['ncbi_taxonomy']
 
         # /spec/data_sources endpoint
         def check_resp_json_spec_endpoint(self, resp):

spec/test/helpers.py

@@ -8,6 +8,10 @@
 import os
 import requests
 import sys
+import shutil
+from relation_engine_server.utils.wait_for import wait_for_api
+from relation_engine_server.utils.pull_spec import download_specs
+from relation_engine_server.utils.config import get_config as get_re_config
 
 
 @functools.lru_cache(maxsize=1)
@@ -62,6 +66,21 @@ def create_test_docs(coll_name, docs, update_on_dupe=False):
     return resp
 
 
+def check_spec_test_env():
+    """ ensure that the environment is prepared for running the spec tests """
+    if os.environ.get('SPEC_TEST_READY', None) is None:
+        wait_for_api()
+        _CONF = get_re_config()
+        # Remove the spec directory, ignoring if it is already missing
+        shutil.rmtree(_CONF['spec_paths']['root'], ignore_errors=True)
+        # Recreate the spec directory so we have a clean slate, avoiding name conflicts
+        os.makedirs(_CONF['spec_paths']['root'])
+        # copy the contents of /app/spec into /spec/repo
+        shutil.copytree('/app/spec', _CONF['spec_paths']['repo'])
+        download_specs()
+        os.environ.update({'SPEC_TEST_READY': "Done"})
+
+
 def capture_stdout(function, *args, **kwargs):
     """capture and return the standard output from a function"""
     io_stdout = io.StringIO()

spec/test/stored_queries/test_djornl.py

@@ -5,9 +5,8 @@
 import unittest
 import os
 
-from spec.test.helpers import get_config, modified_environ, create_test_docs, run_query
+from spec.test.helpers import get_config, modified_environ, create_test_docs, run_query, check_spec_test_env
 from importers.djornl.parser import DJORNL_Parser
-from relation_engine_server.utils.wait_for import wait_for_api
 
 _CONF = get_config()
 _TEST_DIR = '/app/spec/test'
@@ -27,7 +26,7 @@ class Test_DJORNL_Stored_Queries(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
 
-        wait_for_api()
+        check_spec_test_env()
         # import the results file
         results_file = os.path.join(_TEST_DIR, 'djornl', 'results.json')
         with open(results_file) as fh:

spec/test/stored_queries/test_list_test_vertices.py

@@ -1,8 +1,7 @@
 import unittest
 import requests
 
-from spec.test.helpers import create_test_docs, get_config
-from relation_engine_server.utils.wait_for import wait_for_api
+from spec.test.helpers import create_test_docs, get_config, check_spec_test_env
 
 _CONF = get_config()
 _QUERY_URL = _CONF['re_api_url'] + '/api/v1/query_results?view=list_test_vertices'
@@ -12,8 +11,7 @@ class TestListTestVertices(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        # Wait for the API to come online
-        wait_for_api()
+        check_spec_test_env()
 
     def test_valid(self):
         """Test a valid query."""

spec/test/stored_queries/test_ncbi_tax.py

@@ -6,8 +6,7 @@
 import unittest
 import requests
 
-from spec.test.helpers import get_config, assert_subset, create_test_docs
-from relation_engine_server.utils.wait_for import wait_for_api
+from spec.test.helpers import get_config, assert_subset, create_test_docs, check_spec_test_env
 
 _CONF = get_config()
 _NOW = int(time.time() * 1000)
@@ -19,7 +18,7 @@ class TestNcbiTax(unittest.TestCase):
     def setUpClass(cls):
         """Create test documents"""
 
-        wait_for_api()
+        check_spec_test_env()
 
         taxon_docs = [
             {'_key': '1', 'scientific_name': 'Bacteria', 'rank': 'Domain', 'strain': False},

spec/test/stored_queries/test_taxonomy.py

@@ -6,8 +6,7 @@
 import unittest
 import requests
 
-from spec.test.helpers import get_config, assert_subset, create_test_docs
-from relation_engine_server.utils.wait_for import wait_for_api
+from spec.test.helpers import get_config, assert_subset, create_test_docs, check_spec_test_env
 
 _CONF = get_config()
 _NOW = int(time.time() * 1000)
@@ -19,7 +18,7 @@ class TestTaxonomy(unittest.TestCase):
     def setUpClass(cls):
         """Create test documents"""
 
-        wait_for_api()
+        check_spec_test_env()
         taxon_docs = [
             {'_key': '1', 'scientific_name': 'Bacteria', 'rank': 'Domain', 'strain': False},
             {'_key': '2', 'scientific_name': 'Firmicutes', 'rank': 'Phylum', 'strain': False},

spec/test/stored_queries/test_ws.py

@@ -4,8 +4,7 @@
 import unittest
 import json
 import requests
-from spec.test.helpers import get_config, create_test_docs
-from relation_engine_server.utils.wait_for import wait_for_api
+from spec.test.helpers import get_config, create_test_docs, check_spec_test_env
 
 _CONF = get_config()
 
@@ -34,7 +33,7 @@ def setUpClass(cls):
         Create all test data.
         """
 
-        wait_for_api()
+        check_spec_test_env()
 
         ws_object_version = [
             _ws_obj(1, 1, 1),  # root/origin object