diff --git a/project/excel/translator_testing_model.xlsx b/project/excel/translator_testing_model.xlsx index 64a7ee9..efaf6ef 100644 Binary files a/project/excel/translator_testing_model.xlsx and b/project/excel/translator_testing_model.xlsx differ diff --git a/project/graphql/translator_testing_model.graphql b/project/graphql/translator_testing_model.graphql index 953a3b7..11e7f60 100644 --- a/project/graphql/translator_testing_model.graphql +++ b/project/graphql/translator_testing_model.graphql @@ -18,10 +18,10 @@ type AcceptanceTestAsset inV1: Boolean wellKnown: Boolean testReference: Uriorcurie - runnerSettings: [String]! testMetadata: TestMetadata! id: Uriorcurie! tags: [String] + testRunnerSettings: [String] mustPassDate: Date mustPassEnvironment: TestEnvEnum scientificQuestion: String @@ -39,17 +39,15 @@ type AcceptanceTestCase id: Uriorcurie! name: String description: String - testEnv: TestEnvEnum + testRunnerSettings: [String] queryType: QueryTypeEnum preconditions: [Precondition] trapiTemplate: TrapiTemplateEnum - components: [ComponentEnum] testCaseObjective: TestObjectiveEnum testCaseSource: TestSourceEnum testCasePredicateName: String testCasePredicateId: String testCaseInputId: Uriorcurie - testCaseRunnerSettings: [String]! tags: [String] testAssets: [AcceptanceTestAsset]! } @@ -60,6 +58,7 @@ type AcceptanceTestSuite name: String description: String tags: [String] + testRunnerSettings: [String] testMetadata: TestMetadata! testPersona: TestPersonaEnum testCases: [TestCase] @@ -70,74 +69,13 @@ type BenchmarkTestSuite { } -type ComplianceTestCase - { - id: Uriorcurie! - name: String - description: String - testEnv: TestEnvEnum - queryType: QueryTypeEnum - testAssets: [TestAsset]! - preconditions: [Precondition] - trapiTemplate: TrapiTemplateEnum - components: [ComponentEnum] - testCaseObjective: TestObjectiveEnum - testCaseSource: TestSourceEnum - testCasePredicateName: String - testCasePredicateId: String - testCaseInputId: Uriorcurie - testCaseRunnerSettings: [String]! - tags: [String] - trapiVersion: String - biolinkVersion: String - } - -type KnowledgeGraphNavigationTestCase - { - id: Uriorcurie! - name: String - description: String - testEnv: TestEnvEnum - queryType: QueryTypeEnum - testAssets: [TestAsset]! - preconditions: [Precondition] - trapiTemplate: TrapiTemplateEnum - components: [ComponentEnum] - testCaseObjective: TestObjectiveEnum - testCaseSource: TestSourceEnum - testCasePredicateName: String - testCasePredicateId: String - testCaseInputId: Uriorcurie - testCaseRunnerSettings: [String]! - tags: [String] - } - -type OneHopTestCase - { - id: Uriorcurie! - name: String - description: String - testEnv: TestEnvEnum - queryType: QueryTypeEnum - testAssets: [TestAsset]! - preconditions: [Precondition] - trapiTemplate: TrapiTemplateEnum - components: [ComponentEnum] - testCaseObjective: TestObjectiveEnum - testCaseSource: TestSourceEnum - testCasePredicateName: String - testCasePredicateId: String - testCaseInputId: Uriorcurie - testCaseRunnerSettings: [String]! - tags: [String] - } - type OneHopTestSuite { id: Uriorcurie! name: String description: String tags: [String] + testRunnerSettings: [String] testMetadata: TestMetadata! testPersona: TestPersonaEnum testCases: [TestCase] @@ -150,6 +88,7 @@ type Precondition name: String description: String tags: [String] + testRunnerSettings: [String] } type Qualifier @@ -163,18 +102,16 @@ type QuantitativeTestCase id: Uriorcurie! name: String description: String - testEnv: TestEnvEnum + testRunnerSettings: [String] queryType: QueryTypeEnum testAssets: [TestAsset]! preconditions: [Precondition] trapiTemplate: TrapiTemplateEnum - components: [ComponentEnum] testCaseObjective: TestObjectiveEnum testCaseSource: TestSourceEnum testCasePredicateName: String testCasePredicateId: String testCaseInputId: Uriorcurie - testCaseRunnerSettings: [String]! tags: [String] } @@ -184,6 +121,7 @@ type StandardsComplianceTestSuite name: String description: String tags: [String] + testRunnerSettings: [String] testMetadata: TestMetadata! testPersona: TestPersonaEnum testCases: [TestCase] @@ -210,10 +148,10 @@ type TestAsset inV1: Boolean wellKnown: Boolean testReference: Uriorcurie - runnerSettings: [String]! testMetadata: TestMetadata! id: Uriorcurie! tags: [String] + testRunnerSettings: [String] } type TestCase @@ -221,18 +159,16 @@ type TestCase id: Uriorcurie! name: String description: String - testEnv: TestEnvEnum + testRunnerSettings: [String] queryType: QueryTypeEnum testAssets: [TestAsset]! preconditions: [Precondition] trapiTemplate: TrapiTemplateEnum - components: [ComponentEnum] testCaseObjective: TestObjectiveEnum testCaseSource: TestSourceEnum testCasePredicateName: String testCasePredicateId: String testCaseInputId: Uriorcurie - testCaseRunnerSettings: [String]! tags: [String] } @@ -242,6 +178,7 @@ type TestCaseResult name: String description: String tags: [String] + testRunnerSettings: [String] testSuiteId: Uriorcurie testCase: TestCase testCaseResult: TestCaseResultEnum @@ -267,10 +204,10 @@ type TestEdgeData inV1: Boolean wellKnown: Boolean testReference: Uriorcurie - runnerSettings: [String]! testMetadata: TestMetadata! id: Uriorcurie! tags: [String] + testRunnerSettings: [String] } interface TestEntity @@ -279,6 +216,7 @@ interface TestEntity name: String description: String tags: [String] + testRunnerSettings: [String] } type TestEntityParameter @@ -293,6 +231,7 @@ type TestMetadata name: String description: String tags: [String] + testRunnerSettings: [String] testSource: TestSourceEnum testReference: Uriorcurie testObjective: TestObjectiveEnum @@ -305,6 +244,7 @@ type TestOutput name: String description: String tags: [String] + testRunnerSettings: [String] testCaseId: String pks: [TestResultPKSet] } @@ -315,6 +255,7 @@ type TestResultPKSet name: String description: String tags: [String] + testRunnerSettings: [String] parentPk: String mergedPk: String aragorn: String @@ -324,22 +265,17 @@ type TestResultPKSet improving: String } -type TestRunnerConfiguration - { - id: Uriorcurie! - name: String - description: String - testRunParameters: [TestEntityParameter] - tags: [String] - } - type TestRunSession { id: Uriorcurie! name: String description: String tags: [String] + testRunnerSettings: [String] + components: [ComponentEnum] + testEnv: TestEnvEnum testRunnerName: String + testRunParameters: [TestEntityParameter] testEntities: [TestEntity] testCaseResults: [TestCaseResult] timestamp: Datetime @@ -351,6 +287,7 @@ type TestSuite name: String description: String tags: [String] + testRunnerSettings: [String] testMetadata: TestMetadata! testPersona: TestPersonaEnum testCases: [TestCase] @@ -363,6 +300,7 @@ type TestSuiteSpecification name: String description: String tags: [String] + testRunnerSettings: [String] testDataFileLocator: Uriorcurie testDataFileFormat: FileFormatEnum } diff --git a/project/jsonld/translator_testing_model.context.jsonld b/project/jsonld/translator_testing_model.context.jsonld index 41c4a2c..cc48e5d 100644 --- a/project/jsonld/translator_testing_model.context.jsonld +++ b/project/jsonld/translator_testing_model.context.jsonld @@ -1,7 +1,7 @@ { "comments": { "description": "Auto generated by LinkML jsonld context generator", - "generation_date": "2024-03-18T12:26:22", + "generation_date": "2024-03-21T14:12:54", "source": "translator_testing_model.yaml" }, "@context": { diff --git a/project/jsonld/translator_testing_model.jsonld b/project/jsonld/translator_testing_model.jsonld index 62cd6bf..7cdc68d 100644 --- a/project/jsonld/translator_testing_model.jsonld +++ b/project/jsonld/translator_testing_model.jsonld @@ -442,6 +442,14 @@ { "text": "QuantitativeTest", "description": "Quantitative test" + }, + { + "text": "OneHopTest", + "description": "One Hop Test of knowledge graph navigation" + }, + { + "text": "StandardsValidationTest", + "description": "TRAPI and Biolink Model (\"reasoner-validator\") validation" } ] }, @@ -834,7 +842,7 @@ { "name": "test_entity_parameters", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_entity_parameters", - "description": "One or more parameters documenting target characteristics of a TestEntity.", + "description": "One or more 'tag = value' parameters documenting target characteristics of a TestEntity.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_entity_parameters", "multivalued": true, @@ -1339,39 +1347,20 @@ "@type": "SlotDefinition" }, { - "name": "runner_settings", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/runner_settings", - "description": "Settings for the test harness for TestAsset", + "name": "test_runner_settings", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_runner_settings", + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "aliases": [ "Settings" ], - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/runner_settings", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_runner_settings", "multivalued": true, - "owner": "TestAsset", - "domain_of": [ - "TestAsset" - ], - "range": "string", - "required": true, - "@type": "SlotDefinition" - }, - { - "name": "test_case_runner_settings", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_case_runner_settings", - "description": "Settings for the test harness for TestCase", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "aliases": [ - "Settings" - ], - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_case_runner_settings", - "multivalued": true, - "owner": "TestCase", + "owner": "TestEntity", "domain_of": [ - "TestCase" + "TestEntity" ], "range": "string", - "required": true, "@type": "SlotDefinition" }, { @@ -1572,9 +1561,9 @@ "description": "Deployment environment within which the associated TestSuite is run.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_env", - "owner": "TestCase", + "owner": "TestRunSession", "domain_of": [ - "TestCase" + "TestRunSession" ], "range": "TestEnvEnum", "@type": "SlotDefinition" @@ -1642,9 +1631,9 @@ "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/components", "multivalued": true, - "owner": "TestCase", + "owner": "TestRunSession", "domain_of": [ - "TestCase" + "TestRunSession" ], "range": "ComponentEnum", "@type": "SlotDefinition" @@ -1655,10 +1644,6 @@ "description": "TRAPI version (SemVer string)", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/trapi_version", - "owner": "ComplianceTestCase", - "domain_of": [ - "ComplianceTestCase" - ], "range": "string", "@type": "SlotDefinition" }, @@ -1668,10 +1653,6 @@ "description": "Biolink Model release (SemVer string)", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/biolink_version", - "owner": "ComplianceTestCase", - "domain_of": [ - "ComplianceTestCase" - ], "range": "string", "@type": "SlotDefinition" }, @@ -1762,14 +1743,14 @@ { "name": "test_run_parameters", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_run_parameters", - "description": "Parameters for TestRunnerConfiguration that inform the TestHarness and TestRunners about the general characteristics of a test run.", + "description": "TestRunSession parameters informing a TestHarness about the global characteristics of TestRunner processing tests.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "test_entity_parameters", "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_run_parameters", "multivalued": true, - "owner": "TestRunnerConfiguration", + "owner": "TestRunSession", "domain_of": [ - "TestRunnerConfiguration" + "TestRunSession" ], "range": "TestEntityParameter", "inlined": true, @@ -1944,26 +1925,25 @@ "@type": "SlotDefinition" }, { - "name": "TestAsset_runner_settings", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/runner_settings", - "description": "Settings for the test harness, e.g. \"inferred\"", + "name": "TestAsset_test_runner_settings", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_runner_settings", + "description": "Scalar settings for the TestRunner, e.g. \"inferred\"", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "aliases": [ "Settings" ], - "is_a": "runner_settings", + "is_a": "test_runner_settings", "domain": "TestAsset", - "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/runner_settings", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_runner_settings", "multivalued": true, - "alias": "runner_settings", + "alias": "test_runner_settings", "owner": "TestAsset", "domain_of": [ "TestAsset" ], "is_usage_slot": true, - "usage_slot_name": "runner_settings", + "usage_slot_name": "test_runner_settings", "range": "string", - "required": true, "@type": "SlotDefinition" }, { @@ -2036,31 +2016,30 @@ "@type": "SlotDefinition" }, { - "name": "TestRunnerConfiguration_tags", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/tags", - "description": "One or more 'tags' slot values (slot inherited from TestEntity) should generally identify the TestRunner(s) using this configuration.", + "name": "TestRunSession_test_run_parameters", + "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_run_parameters", + "description": "Different TestRunners could expect additional global test configuration parameters, like the applicable TRAPI version (\"trapi_version\") or Biolink Model versions (\"biolink_version\").", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "mappings": [ - "http://schema.org/additionalType" - ], - "is_a": "tags", - "domain": "TestRunnerConfiguration", - "slot_uri": "http://schema.org/additionalType", + "is_a": "test_run_parameters", + "domain": "TestRunSession", + "slot_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_run_parameters", "multivalued": true, - "alias": "tags", - "owner": "TestRunnerConfiguration", + "alias": "test_run_parameters", + "owner": "TestRunSession", "domain_of": [ - "TestRunnerConfiguration" + "TestRunSession" ], "is_usage_slot": true, - "usage_slot_name": "tags", - "range": "string", + "usage_slot_name": "test_run_parameters", + "range": "TestEntityParameter", + "inlined": true, + "inlined_as_list": true, "@type": "SlotDefinition" }, { "name": "TestRunSession_test_entities", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/test_entities", - "description": "Different TestRunners may expect specific kinds of TestEntity as input. These 'test_entities' one or more instances of TestAsset, TestCase or TestSuite.", + "description": "Different TestRunners could expect specific kinds of TestEntity as an input. These 'test_entities' are one or more instances of TestAsset, TestCase or (preferably?) TestSuite.", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "test_entities", "domain": "TestRunSession", @@ -2115,7 +2094,8 @@ "id", "name", "description", - "tags" + "tags", + "test_runner_settings" ], "slot_usage": {}, "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestEntity", @@ -2133,6 +2113,7 @@ "name", "description", "tags", + "test_runner_settings", "test_source", "test_reference", "test_objective", @@ -2167,10 +2148,10 @@ "in_v1", "well_known", "test_reference", - "TestAsset_runner_settings", "test_metadata", "TestAsset_id", - "TestAsset_tags" + "TestAsset_tags", + "TestAsset_test_runner_settings" ], "slot_usage": {}, "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestAsset", @@ -2201,10 +2182,10 @@ "in_v1", "well_known", "test_reference", - "TestAsset_runner_settings", "test_metadata", "TestAsset_id", "TestAsset_tags", + "TestAsset_test_runner_settings", "must_pass_date", "must_pass_environment", "scientific_question", @@ -2245,10 +2226,10 @@ "in_v1", "well_known", "test_reference", - "TestAsset_runner_settings", "test_metadata", "TestAsset_id", - "TestAsset_tags" + "TestAsset_tags", + "TestAsset_test_runner_settings" ], "slot_usage": {}, "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestEdgeData", @@ -2264,7 +2245,8 @@ "id", "name", "description", - "tags" + "tags", + "test_runner_settings" ], "slot_usage": {}, "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/Precondition", @@ -2280,18 +2262,16 @@ "id", "name", "description", - "test_env", + "test_runner_settings", "query_type", "TestCase_test_assets", "preconditions", "trapi_template", - "components", "test_case_objective", "test_case_source", "test_case_predicate_name", "test_case_predicate_id", "test_case_input_id", - "test_case_runner_settings", "TestCase_tags" ], "slot_usage": {}, @@ -2308,17 +2288,15 @@ "id", "name", "description", - "test_env", + "test_runner_settings", "query_type", "preconditions", "trapi_template", - "components", "test_case_objective", "test_case_source", "test_case_predicate_name", "test_case_predicate_id", "test_case_input_id", - "test_case_runner_settings", "TestCase_tags", "AcceptanceTestCase_test_assets" ], @@ -2336,110 +2314,22 @@ "id", "name", "description", - "test_env", + "test_runner_settings", "query_type", "TestCase_test_assets", "preconditions", "trapi_template", - "components", "test_case_objective", "test_case_source", "test_case_predicate_name", "test_case_predicate_id", "test_case_input_id", - "test_case_runner_settings", "TestCase_tags" ], "slot_usage": {}, "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/QuantitativeTestCase", "@type": "ClassDefinition" }, - { - "name": "ComplianceTestCase", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/ComplianceTestCase", - "description": "TRAPI and Biolink Model standards compliance test", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestCase", - "slots": [ - "id", - "name", - "description", - "test_env", - "query_type", - "TestCase_test_assets", - "preconditions", - "trapi_template", - "components", - "test_case_objective", - "test_case_source", - "test_case_predicate_name", - "test_case_predicate_id", - "test_case_input_id", - "test_case_runner_settings", - "TestCase_tags", - "trapi_version", - "biolink_version" - ], - "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/ComplianceTestCase", - "@type": "ClassDefinition" - }, - { - "name": "KnowledgeGraphNavigationTestCase", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/KnowledgeGraphNavigationTestCase", - "description": "Knowledge Graph navigation integration test", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestCase", - "slots": [ - "id", - "name", - "description", - "test_env", - "query_type", - "TestCase_test_assets", - "preconditions", - "trapi_template", - "components", - "test_case_objective", - "test_case_source", - "test_case_predicate_name", - "test_case_predicate_id", - "test_case_input_id", - "test_case_runner_settings", - "TestCase_tags" - ], - "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/KnowledgeGraphNavigationTestCase", - "@type": "ClassDefinition" - }, - { - "name": "OneHopTestCase", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestCase", - "description": "'One Hop' Knowledge Graph navigation integration test", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "KnowledgeGraphNavigationTestCase", - "slots": [ - "id", - "name", - "description", - "test_env", - "query_type", - "TestCase_test_assets", - "preconditions", - "trapi_template", - "components", - "test_case_objective", - "test_case_source", - "test_case_predicate_name", - "test_case_predicate_id", - "test_case_input_id", - "test_case_runner_settings", - "TestCase_tags" - ], - "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestCase", - "@type": "ClassDefinition" - }, { "name": "TestSuiteSpecification", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestSuiteSpecification", @@ -2451,6 +2341,7 @@ "name", "description", "tags", + "test_runner_settings", "test_data_file_locator", "test_data_file_format" ], @@ -2469,6 +2360,7 @@ "name", "description", "tags", + "test_runner_settings", "test_metadata", "test_persona", "test_cases", @@ -2488,6 +2380,7 @@ "name", "description", "tags", + "test_runner_settings", "test_metadata", "test_persona", "test_cases", @@ -2517,6 +2410,7 @@ "name", "description", "tags", + "test_runner_settings", "test_metadata", "test_persona", "test_cases", @@ -2537,6 +2431,7 @@ "name", "description", "tags", + "test_runner_settings", "test_metadata", "test_persona", "test_cases", @@ -2546,23 +2441,6 @@ "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/OneHopTestSuite", "@type": "ClassDefinition" }, - { - "name": "TestRunnerConfiguration", - "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestRunnerConfiguration", - "description": "General configuration parameters and test data input for a single invocation of a TestRunner.", - "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", - "is_a": "TestEntity", - "slots": [ - "id", - "name", - "description", - "test_run_parameters", - "TestRunnerConfiguration_tags" - ], - "slot_usage": {}, - "class_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestRunnerConfiguration", - "@type": "ClassDefinition" - }, { "name": "TestCaseResult", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestCaseResult", @@ -2574,6 +2452,7 @@ "name", "description", "tags", + "test_runner_settings", "test_suite_id", "test_case", "test_case_result" @@ -2585,7 +2464,7 @@ { "name": "TestRunSession", "definition_uri": "https://w3id.org/TranslatorSRI/TranslatorTestingModel/TestRunSession", - "description": "A single invocation of a TestRunner.", + "description": "Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite).", "from_schema": "https://w3id.org/TranslatorSRI/TranslatorTestingModel", "is_a": "TestEntity", "slots": [ @@ -2593,7 +2472,11 @@ "name", "description", "tags", + "test_runner_settings", + "components", + "test_env", "test_runner_name", + "TestRunSession_test_run_parameters", "TestRunSession_test_entities", "test_case_results", "timestamp" @@ -2613,6 +2496,7 @@ "name", "description", "tags", + "test_runner_settings", "test_case_id", "pks" ], @@ -2631,6 +2515,7 @@ "name", "description", "tags", + "test_runner_settings", "parent_pk", "merged_pk", "aragorn", @@ -2646,9 +2531,9 @@ ], "metamodel_version": "1.7.0", "source_file": "translator_testing_model.yaml", - "source_file_date": "2024-03-18T12:26:09", - "source_file_size": 27461, - "generation_date": "2024-03-18T12:26:22", + "source_file_date": "2024-03-21T14:07:11", + "source_file_size": 27801, + "generation_date": "2024-03-21T14:12:54", "@type": "SchemaDefinition", "@context": [ "project/jsonld/translator_testing_model.context.jsonld", diff --git a/project/jsonschema/translator_testing_model.schema.json b/project/jsonschema/translator_testing_model.schema.json index 3524f75..ac8138a 100644 --- a/project/jsonschema/translator_testing_model.schema.json +++ b/project/jsonschema/translator_testing_model.schema.json @@ -86,13 +86,6 @@ "$ref": "#/$defs/NodeEnum", "description": "The node of the (templated) TRAPI query to replace" }, - "runner_settings": { - "description": "Settings for the test harness, e.g. \"inferred\"", - "items": { - "type": "string" - }, - "type": "array" - }, "scientific_question": { "description": "The full human-readable scientific question a SME would ask, which is encoded into the test asset.", "type": "string" @@ -122,6 +115,13 @@ "description": "Document URL where original test source particulars are registered (e.g. Github repo)", "type": "string" }, + "test_runner_settings": { + "description": "Scalar settings for the TestRunner, e.g. \"inferred\"", + "items": { + "type": "string" + }, + "type": "array" + }, "top_level": { "description": "The answer must return in these many results", "type": "integer" @@ -131,7 +131,6 @@ } }, "required": [ - "runner_settings", "test_metadata", "id" ], @@ -142,13 +141,6 @@ "additionalProperties": false, "description": "See AcceptanceTestAsset above for more details.", "properties": { - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, "description": { "description": "A human-readable description for a Test Entity", "type": "string" @@ -198,20 +190,16 @@ "test_case_predicate_name": { "type": "string" }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, "test_case_source": { "$ref": "#/$defs/TestSourceEnum", "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" }, "trapi_template": { "$ref": "#/$defs/TrapiTemplateEnum", @@ -220,7 +208,6 @@ }, "required": [ "test_assets", - "test_case_runner_settings", "id" ], "title": "AcceptanceTestCase", @@ -264,6 +251,13 @@ "$ref": "#/$defs/TestPersonaEnum", "description": "A Test persona describes the user or operational context of a given test." }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_suite_specification": { "$ref": "#/$defs/TestSuiteSpecification", "description": "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." @@ -282,102 +276,6 @@ "title": "BenchmarkTestSuite", "type": "object" }, - "ComplianceTestCase": { - "additionalProperties": false, - "description": "TRAPI and Biolink Model standards compliance test", - "properties": { - "biolink_version": { - "description": "Biolink Model release (SemVer string)", - "type": "string" - }, - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" - }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" - }, - "name": { - "description": "A human-readable name for a Test Entity", - "type": "string" - }, - "preconditions": { - "items": { - "type": "string" - }, - "type": "array" - }, - "query_type": { - "$ref": "#/$defs/QueryTypeEnum", - "description": "Type of TestCase query." - }, - "tags": { - "description": "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", - "items": { - "type": "string" - }, - "type": "array" - }, - "test_assets": { - "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", - "items": { - "$ref": "#/$defs/TestAsset" - }, - "type": "array" - }, - "test_case_input_id": { - "type": "string" - }, - "test_case_objective": { - "$ref": "#/$defs/TestObjectiveEnum", - "description": "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" - }, - "test_case_predicate_id": { - "type": "string" - }, - "test_case_predicate_name": { - "type": "string" - }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, - "test_case_source": { - "$ref": "#/$defs/TestSourceEnum", - "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." - }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." - }, - "trapi_template": { - "$ref": "#/$defs/TrapiTemplateEnum", - "description": "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." - }, - "trapi_version": { - "description": "TRAPI version (SemVer string)", - "type": "string" - } - }, - "required": [ - "test_assets", - "test_case_runner_settings", - "id" - ], - "title": "ComplianceTestCase", - "type": "object" - }, "ComponentEnum": { "description": "Translator components are identified by their InfoRes identifiers.", "title": "ComponentEnum", @@ -423,94 +321,6 @@ "title": "FileFormatEnum", "type": "string" }, - "KnowledgeGraphNavigationTestCase": { - "additionalProperties": false, - "description": "Knowledge Graph navigation integration test", - "properties": { - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" - }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" - }, - "name": { - "description": "A human-readable name for a Test Entity", - "type": "string" - }, - "preconditions": { - "items": { - "type": "string" - }, - "type": "array" - }, - "query_type": { - "$ref": "#/$defs/QueryTypeEnum", - "description": "Type of TestCase query." - }, - "tags": { - "description": "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", - "items": { - "type": "string" - }, - "type": "array" - }, - "test_assets": { - "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", - "items": { - "$ref": "#/$defs/TestAsset" - }, - "type": "array" - }, - "test_case_input_id": { - "type": "string" - }, - "test_case_objective": { - "$ref": "#/$defs/TestObjectiveEnum", - "description": "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" - }, - "test_case_predicate_id": { - "type": "string" - }, - "test_case_predicate_name": { - "type": "string" - }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, - "test_case_source": { - "$ref": "#/$defs/TestSourceEnum", - "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." - }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." - }, - "trapi_template": { - "$ref": "#/$defs/TrapiTemplateEnum", - "description": "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." - } - }, - "required": [ - "test_assets", - "test_case_runner_settings", - "id" - ], - "title": "KnowledgeGraphNavigationTestCase", - "type": "object" - }, "NodeEnum": { "description": "Target node of a Subject-Predicate-Object driven query", "enum": [ @@ -520,94 +330,6 @@ "title": "NodeEnum", "type": "string" }, - "OneHopTestCase": { - "additionalProperties": false, - "description": "'One Hop' Knowledge Graph navigation integration test", - "properties": { - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" - }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" - }, - "name": { - "description": "A human-readable name for a Test Entity", - "type": "string" - }, - "preconditions": { - "items": { - "type": "string" - }, - "type": "array" - }, - "query_type": { - "$ref": "#/$defs/QueryTypeEnum", - "description": "Type of TestCase query." - }, - "tags": { - "description": "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.", - "items": { - "type": "string" - }, - "type": "array" - }, - "test_assets": { - "description": "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.", - "items": { - "$ref": "#/$defs/TestAsset" - }, - "type": "array" - }, - "test_case_input_id": { - "type": "string" - }, - "test_case_objective": { - "$ref": "#/$defs/TestObjectiveEnum", - "description": "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" - }, - "test_case_predicate_id": { - "type": "string" - }, - "test_case_predicate_name": { - "type": "string" - }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, - "test_case_source": { - "$ref": "#/$defs/TestSourceEnum", - "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." - }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." - }, - "trapi_template": { - "$ref": "#/$defs/TrapiTemplateEnum", - "description": "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." - } - }, - "required": [ - "test_assets", - "test_case_runner_settings", - "id" - ], - "title": "OneHopTestCase", - "type": "object" - }, "OneHopTestSuite": { "additionalProperties": false, "description": "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness.", @@ -646,6 +368,13 @@ "$ref": "#/$defs/TestPersonaEnum", "description": "A Test persona describes the user or operational context of a given test." }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_suite_specification": { "$ref": "#/$defs/TestSuiteSpecification", "description": "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." @@ -680,6 +409,13 @@ "type": "string" }, "type": "array" + }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" } }, "required": [ @@ -708,13 +444,6 @@ "additionalProperties": false, "description": "Assumed additional model from Shervin's runner JSON here as an example. This schema is not yet complete.", "properties": { - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, "description": { "description": "A human-readable description for a Test Entity", "type": "string" @@ -764,20 +493,16 @@ "test_case_predicate_name": { "type": "string" }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, "test_case_source": { "$ref": "#/$defs/TestSourceEnum", "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" }, "trapi_template": { "$ref": "#/$defs/TrapiTemplateEnum", @@ -786,7 +511,6 @@ }, "required": [ "test_assets", - "test_case_runner_settings", "id" ], "title": "QuantitativeTestCase", @@ -848,6 +572,13 @@ "$ref": "#/$defs/TestPersonaEnum", "description": "A Test persona describes the user or operational context of a given test." }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_suite_specification": { "$ref": "#/$defs/TestSuiteSpecification", "description": "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." @@ -917,13 +648,6 @@ }, "type": "array" }, - "runner_settings": { - "description": "Settings for the test harness, e.g. \"inferred\"", - "items": { - "type": "string" - }, - "type": "array" - }, "semantic_severity": { "$ref": "#/$defs/SemanticSeverityEnum" }, @@ -945,12 +669,18 @@ "description": "Document URL where original test source particulars are registered (e.g. Github repo)", "type": "string" }, + "test_runner_settings": { + "description": "Scalar settings for the TestRunner, e.g. \"inferred\"", + "items": { + "type": "string" + }, + "type": "array" + }, "well_known": { "type": "boolean" } }, "required": [ - "runner_settings", "test_metadata", "id" ], @@ -961,13 +691,6 @@ "additionalProperties": false, "description": "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition.", "properties": { - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, "description": { "description": "A human-readable description for a Test Entity", "type": "string" @@ -1017,20 +740,16 @@ "test_case_predicate_name": { "type": "string" }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, "test_case_source": { "$ref": "#/$defs/TestSourceEnum", "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" }, "trapi_template": { "$ref": "#/$defs/TrapiTemplateEnum", @@ -1039,7 +758,6 @@ }, "required": [ "test_assets", - "test_case_runner_settings", "id" ], "title": "TestCase", @@ -1076,6 +794,13 @@ "$ref": "#/$defs/TestCaseResultEnum", "description": "Encoded result of a single test run of a given test case" }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_suite_id": { "description": "CURIE id of a TestSuite registered in the system.", "type": "string" @@ -1128,6 +853,13 @@ "$ref": "#/$defs/TestCaseResultEnum", "description": "Encoded result of a single test run of a given test case" }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_suite_id": { "description": "CURIE id of a TestSuite registered in the system.", "type": "string" @@ -1141,13 +873,6 @@ "additionalProperties": false, "description": "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition.", "properties": { - "components": { - "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", - "items": { - "$ref": "#/$defs/ComponentEnum" - }, - "type": "array" - }, "description": { "description": "A human-readable description for a Test Entity", "type": "string" @@ -1197,20 +922,16 @@ "test_case_predicate_name": { "type": "string" }, - "test_case_runner_settings": { - "description": "Settings for the test harness for TestCase", - "items": { - "type": "string" - }, - "type": "array" - }, "test_case_source": { "$ref": "#/$defs/TestSourceEnum", "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." }, - "test_env": { - "$ref": "#/$defs/TestEnvEnum", - "description": "Deployment environment within which the associated TestSuite is run." + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" }, "trapi_template": { "$ref": "#/$defs/TrapiTemplateEnum", @@ -1218,8 +939,7 @@ } }, "required": [ - "test_assets", - "test_case_runner_settings" + "test_assets" ], "title": "TestCase", "type": "object" @@ -1281,13 +1001,6 @@ }, "type": "array" }, - "runner_settings": { - "description": "Settings for the test harness, e.g. \"inferred\"", - "items": { - "type": "string" - }, - "type": "array" - }, "semantic_severity": { "$ref": "#/$defs/SemanticSeverityEnum" }, @@ -1309,12 +1022,18 @@ "description": "Document URL where original test source particulars are registered (e.g. Github repo)", "type": "string" }, + "test_runner_settings": { + "description": "Scalar settings for the TestRunner, e.g. \"inferred\"", + "items": { + "type": "string" + }, + "type": "array" + }, "well_known": { "type": "boolean" } }, "required": [ - "runner_settings", "test_metadata", "id" ], @@ -1399,6 +1118,13 @@ "description": "Document URL where original test source particulars are registered (e.g. Github repo)", "type": "string" }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_source": { "$ref": "#/$defs/TestSourceEnum", "description": "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." @@ -1415,7 +1141,9 @@ "enum": [ "AcceptanceTest", "BenchmarkTest", - "QuantitativeTest" + "QuantitativeTest", + "OneHopTest", + "StandardsValidationTest" ], "title": "TestObjectiveEnum", "type": "string" @@ -1453,6 +1181,13 @@ "test_case_id": { "description": "CURIE id of a TestCase registered in the system.", "type": "string" + }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" } }, "required": [ @@ -1513,6 +1248,13 @@ }, "type": "array" }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "unsecret": { "type": "string" } @@ -1525,8 +1267,15 @@ }, "TestRunSession": { "additionalProperties": false, - "description": "A single invocation of a TestRunner.", + "description": "Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite).", "properties": { + "components": { + "description": "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.", + "items": { + "$ref": "#/$defs/ComponentEnum" + }, + "type": "array" + }, "description": { "description": "A human-readable description for a Test Entity", "type": "string" @@ -1571,60 +1320,41 @@ } ] }, - "description": "Different TestRunners may expect specific kinds of TestEntity as input. These 'test_entities' one or more instances of TestAsset, TestCase or TestSuite.", + "description": "Different TestRunners could expect specific kinds of TestEntity as an input. These 'test_entities' are one or more instances of TestAsset, TestCase or (preferably?) TestSuite.", "type": "object" }, - "test_runner_name": { - "description": "Global system name of a TestRunner.", - "type": "string" - }, - "timestamp": { - "description": "Date time when a given entity was created.", - "format": "date-time", - "type": "string" - } - }, - "required": [ - "id" - ], - "title": "TestRunSession", - "type": "object" - }, - "TestRunnerConfiguration": { - "additionalProperties": false, - "description": "General configuration parameters and test data input for a single invocation of a TestRunner.", - "properties": { - "description": { - "description": "A human-readable description for a Test Entity", - "type": "string" + "test_env": { + "$ref": "#/$defs/TestEnvEnum", + "description": "Deployment environment within which the associated TestSuite is run." }, - "id": { - "description": "A unique identifier for a Test Entity", - "type": "string" + "test_run_parameters": { + "description": "Different TestRunners could expect additional global test configuration parameters, like the applicable TRAPI version (\"trapi_version\") or Biolink Model versions (\"biolink_version\").", + "items": { + "$ref": "#/$defs/TestEntityParameter" + }, + "type": "array" }, - "name": { - "description": "A human-readable name for a Test Entity", + "test_runner_name": { + "description": "Global system name of a TestRunner.", "type": "string" }, - "tags": { - "description": "One or more 'tags' slot values (slot inherited from TestEntity) should generally identify the TestRunner(s) using this configuration.", + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", "items": { "type": "string" }, "type": "array" }, - "test_run_parameters": { - "description": "Parameters for TestRunnerConfiguration that inform the TestHarness and TestRunners about the general characteristics of a test run.", - "items": { - "$ref": "#/$defs/TestEntityParameter" - }, - "type": "array" + "timestamp": { + "description": "Date time when a given entity was created.", + "format": "date-time", + "type": "string" } }, "required": [ "id" ], - "title": "TestRunnerConfiguration", + "title": "TestRunSession", "type": "object" }, "TestSourceEnum": { @@ -1679,6 +1409,13 @@ "$ref": "#/$defs/TestPersonaEnum", "description": "A Test persona describes the user or operational context of a given test." }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" + }, "test_suite_specification": { "$ref": "#/$defs/TestSuiteSpecification", "description": "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." @@ -1721,6 +1458,13 @@ "test_data_file_locator": { "description": "An web accessible file resource link to test entity data (e.g. a web accessible text file of Test Asset entries)", "type": "string" + }, + "test_runner_settings": { + "description": "Scalar parameters for the TestRunner processing a given TestEntity.", + "items": { + "type": "string" + }, + "type": "array" } }, "required": [ diff --git a/project/owl/translator_testing_model.owl.ttl b/project/owl/translator_testing_model.owl.ttl index 5fdf771..07a995b 100644 --- a/project/owl/translator_testing_model.owl.ttl +++ b/project/owl/translator_testing_model.owl.ttl @@ -33,38 +33,6 @@ ttm:BenchmarkTestSuite a owl:Class, skos:definition "JsonObj(is_a='TestSuite')" ; skos:inScheme . -ttm:ComplianceTestCase a owl:Class, - linkml:ClassDefinition ; - rdfs:label "ComplianceTestCase" ; - rdfs:subClassOf [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:trapi_version ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:biolink_version ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:trapi_version ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:biolink_version ], - [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:biolink_version ], - [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:trapi_version ], - ttm:TestCase ; - skos:definition "TRAPI and Biolink Model standards compliance test" ; - skos:inScheme . - -ttm:OneHopTestCase a owl:Class, - linkml:ClassDefinition ; - rdfs:label "OneHopTestCase" ; - rdfs:subClassOf ttm:KnowledgeGraphNavigationTestCase ; - skos:definition "'One Hop' Knowledge Graph navigation integration test" ; - skos:inScheme . - ttm:OneHopTestSuite a owl:Class, linkml:ClassDefinition ; rdfs:label "OneHopTestSuite" ; @@ -97,59 +65,61 @@ ttm:TestRunSession a owl:Class, linkml:ClassDefinition ; rdfs:label "TestRunSession" ; rdfs:subClassOf [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; + owl:minCardinality 0 ; owl:onProperty ttm:test_runner_name ], [ a owl:Restriction ; - owl:allValuesFrom owl:Thing ; - owl:onProperty ttm:test_entities ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:TestCaseResult ; - owl:onProperty ttm:test_case_results ], + owl:minCardinality 0 ; + owl:onProperty ttm:timestamp ], [ a owl:Restriction ; owl:maxCardinality 1 ; + owl:onProperty ttm:timestamp ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; owl:onProperty ttm:test_runner_name ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:test_case_results ], + owl:allValuesFrom ttm:ComponentEnum ; + owl:onProperty ttm:components ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:timestamp ], + owl:maxCardinality 1 ; + owl:onProperty ttm:test_runner_name ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:test_entities ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Datetime ; - owl:onProperty ttm:timestamp ], + owl:minCardinality 0 ; + owl:onProperty ttm:test_case_results ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:timestamp ], + owl:allValuesFrom ttm:TestCaseResult ; + owl:onProperty ttm:test_case_results ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_runner_name ], - ttm:TestEntity ; - skos:definition "A single invocation of a TestRunner." ; - skos:inScheme . - -ttm:TestRunnerConfiguration a owl:Class, - linkml:ClassDefinition ; - rdfs:label "TestRunnerConfiguration" ; - rdfs:subClassOf [ a owl:Restriction ; - owl:maxCardinality 1 ; owl:onProperty ttm:test_run_parameters ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:tags ], + owl:minCardinality 0 ; + owl:onProperty ttm:components ], + [ a owl:Restriction ; + owl:allValuesFrom owl:Thing ; + owl:onProperty ttm:test_entities ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; owl:onProperty ttm:test_run_parameters ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:allValuesFrom ttm:TestEnvEnum ; + owl:onProperty ttm:test_env ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:test_env ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; owl:onProperty ttm:test_run_parameters ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:Datetime ; + owl:onProperty ttm:timestamp ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:tags ], + owl:onProperty ttm:test_env ], ttm:TestEntity ; - skos:definition "General configuration parameters and test data input for a single invocation of a TestRunner." ; + skos:definition "Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite)." ; skos:inScheme . ttm:biolink_object_aspect_qualifier a owl:ObjectProperty, @@ -190,6 +160,12 @@ ttm:biolink_subject_direction_qualifier a owl:ObjectProperty, skos:definition "The direction of the subject of the test asset predicate" ; skos:inScheme . +ttm:biolink_version a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "biolink_version" ; + skos:definition "Biolink Model release (SemVer string)" ; + skos:inScheme . + ttm:results a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "results" ; @@ -197,110 +173,109 @@ ttm:results a owl:ObjectProperty, skos:definition "The results of a TestRunner run of one specific TestCase." ; skos:inScheme . +ttm:trapi_version a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "trapi_version" ; + skos:definition "TRAPI version (SemVer string)" ; + skos:inScheme . + ttm:AcceptanceTestAsset a owl:Class, linkml:ClassDefinition ; rdfs:label "AcceptanceTestAsset" ; rdfs:subClassOf [ a owl:Restriction ; + owl:allValuesFrom ttm:NodeEnum ; + owl:onProperty ttm:query_node ], + [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:notes ], + owl:onProperty ttm:answer_informal_concept ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:query_node ], + owl:onProperty ttm:string_entry ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:expected_result ], - [ a owl:Restriction ; - owl:minCardinality 0 ; owl:onProperty ttm:top_level ], [ a owl:Restriction ; - owl:allValuesFrom ttm:NodeEnum ; - owl:onProperty ttm:query_node ], + owl:maxCardinality 1 ; + owl:onProperty ttm:scientific_question ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:answer_informal_concept ], + owl:maxCardinality 1 ; + owl:onProperty ttm:must_pass_environment ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:string_entry ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:query_node ], + owl:allValuesFrom ttm:TestEnvEnum ; + owl:onProperty ttm:must_pass_environment ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:scientific_question ], + owl:allValuesFrom linkml:Integer ; + owl:onProperty ttm:top_level ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:scientific_question ], + owl:allValuesFrom ttm:ExpectedResultsEnum ; + owl:onProperty ttm:expected_result ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Date ; - owl:onProperty ttm:must_pass_date ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:string_entry ], [ a owl:Restriction ; - owl:allValuesFrom ttm:DirectionEnum ; - owl:onProperty ttm:direction ], + owl:maxCardinality 1 ; + owl:onProperty ttm:query_node ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; owl:onProperty ttm:answer_informal_concept ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:must_pass_date ], + owl:onProperty ttm:answer_informal_concept ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:direction ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:notes ], [ a owl:Restriction ; - owl:allValuesFrom ttm:TestEnvEnum ; + owl:minCardinality 0 ; owl:onProperty ttm:must_pass_environment ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:notes ], + owl:minCardinality 0 ; + owl:onProperty ttm:expected_result ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:notes ], + owl:minCardinality 0 ; + owl:onProperty ttm:must_pass_date ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:string_entry ], + owl:minCardinality 0 ; + owl:onProperty ttm:scientific_question ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:top_level ], + owl:onProperty ttm:notes ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:must_pass_date ], + owl:onProperty ttm:query_node ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:must_pass_environment ], + owl:onProperty ttm:direction ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:answer_informal_concept ], + owl:onProperty ttm:expected_result ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:scientific_question ], + owl:allValuesFrom linkml:Date ; + owl:onProperty ttm:must_pass_date ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Integer ; + owl:minCardinality 0 ; owl:onProperty ttm:top_level ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:direction ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:must_pass_environment ], + owl:maxCardinality 1 ; + owl:onProperty ttm:must_pass_date ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:expected_result ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:scientific_question ], [ a owl:Restriction ; - owl:allValuesFrom ttm:ExpectedResultsEnum ; - owl:onProperty ttm:expected_result ], + owl:minCardinality 0 ; + owl:onProperty ttm:notes ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:string_entry ], + owl:allValuesFrom ttm:DirectionEnum ; + owl:onProperty ttm:direction ], ttm:TestAsset ; skos:definition "Model derived from Jenn's test asset design and Shervin's runner JSON here as an example." ; skos:inScheme . -ttm:KnowledgeGraphNavigationTestCase a owl:Class, - linkml:ClassDefinition ; - rdfs:label "KnowledgeGraphNavigationTestCase" ; - rdfs:subClassOf ttm:TestCase ; - skos:definition "Knowledge Graph navigation integration test" ; - skos:inScheme . - a owl:Class, ttm:QueryTypeEnum ; rdfs:label "treats" ; @@ -311,19 +286,19 @@ ttm:TestOutput a owl:Class, rdfs:label "TestOutput" ; rdfs:subClassOf [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:pks ], + owl:onProperty ttm:test_case_id ], [ a owl:Restriction ; owl:allValuesFrom ttm:TestResultPKSet ; owl:onProperty ttm:pks ], [ a owl:Restriction ; - owl:maxCardinality 1 ; + owl:allValuesFrom linkml:String ; owl:onProperty ttm:test_case_id ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:maxCardinality 1 ; owl:onProperty ttm:test_case_id ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:test_case_id ], + owl:minCardinality 0 ; + owl:onProperty ttm:pks ], ttm:TestEntity ; skos:definition "The output of a TestRunner run of one specific TestCase." ; skos:inScheme . @@ -413,22 +388,22 @@ ttm:Qualifier a owl:Class, rdfs:label "Qualifier" ; rdfs:subClassOf [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:parameter ], - [ a owl:Restriction ; - owl:minCardinality 0 ; owl:onProperty ttm:value ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; owl:onProperty ttm:parameter ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:value ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:parameter ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; owl:onProperty ttm:value ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:parameter ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:value ], ttm:TestEntityParameter ; skos:inScheme . @@ -452,27 +427,27 @@ ttm:TestCaseResult a owl:Class, rdfs:label "TestCaseResult" ; rdfs:subClassOf [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_case ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; owl:onProperty ttm:test_suite_id ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:test_case_result ], + owl:maxCardinality 1 ; + owl:onProperty ttm:test_suite_id ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:test_case ], [ a owl:Restriction ; - owl:maxCardinality 1 ; + owl:minCardinality 0 ; owl:onProperty ttm:test_case_result ], [ a owl:Restriction ; owl:allValuesFrom ttm:TestCase ; owl:onProperty ttm:test_case ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:test_case_result ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_suite_id ], + owl:onProperty ttm:test_case ], [ a owl:Restriction ; - owl:maxCardinality 1 ; + owl:allValuesFrom linkml:Uriorcurie ; owl:onProperty ttm:test_suite_id ], [ a owl:Restriction ; owl:allValuesFrom ttm:TestCaseResultEnum ; @@ -500,23 +475,23 @@ ttm:TestEntityParameter a owl:Class, linkml:ClassDefinition ; rdfs:label "TestEntityParameter" ; rdfs:subClassOf [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:maxCardinality 1 ; owl:onProperty ttm:value ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:parameter ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; owl:onProperty ttm:value ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:parameter ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:parameter ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:value ] ; + owl:onProperty ttm:parameter ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:value ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:parameter ] ; skos:definition "A single 'tag = value' pair (where 'value' is a simple string)." ; skos:inScheme . @@ -580,11 +555,21 @@ ttm:TestEntityParameter a owl:Class, rdfs:label "BenchmarkTest" ; rdfs:subClassOf ttm:TestObjectiveEnum . + a owl:Class, + ttm:TestObjectiveEnum ; + rdfs:label "OneHopTest" ; + rdfs:subClassOf ttm:TestObjectiveEnum . + a owl:Class, ttm:TestObjectiveEnum ; rdfs:label "QuantitativeTest" ; rdfs:subClassOf ttm:TestObjectiveEnum . + a owl:Class, + ttm:TestObjectiveEnum ; + rdfs:label "StandardsValidationTest" ; + rdfs:subClassOf ttm:TestObjectiveEnum . + a owl:Class, ttm:TestPersonaEnum ; rdfs:label "All" ; @@ -610,67 +595,67 @@ ttm:TestResultPKSet a owl:Class, rdfs:label "TestResultPKSet" ; rdfs:subClassOf [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:merged_pk ], + owl:onProperty ttm:aragorn ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:aragorn ], + owl:onProperty ttm:bte ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:merged_pk ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:improving ], + owl:onProperty ttm:merged_pk ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; + owl:maxCardinality 1 ; owl:onProperty ttm:parent_pk ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:arax ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:improving ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:aragorn ], + owl:onProperty ttm:unsecret ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:improving ], - [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:aragorn ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:improving ], + owl:onProperty ttm:merged_pk ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:bte ], + owl:onProperty ttm:aragorn ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:arax ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:bte ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:aragorn ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:merged_pk ], + owl:onProperty ttm:arax ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:bte ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:merged_pk ], + owl:onProperty ttm:improving ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; owl:onProperty ttm:unsecret ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:unsecret ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:parent_pk ], - [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:unsecret ], + owl:onProperty ttm:bte ], [ a owl:Restriction ; owl:maxCardinality 1 ; + owl:onProperty ttm:arax ], + [ a owl:Restriction ; + owl:minCardinality 0 ; owl:onProperty ttm:parent_pk ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:arax ], + owl:onProperty ttm:parent_pk ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:arax ], + owl:minCardinality 0 ; + owl:onProperty ttm:unsecret ], ttm:TestEntity ; skos:definition "Primary keys for a given ARA result set from a SmokeTest result for a given TestCase." ; skos:inScheme . @@ -710,20 +695,20 @@ ttm:TestSuiteSpecification a owl:Class, linkml:ClassDefinition ; rdfs:label "TestSuiteSpecification" ; rdfs:subClassOf [ a owl:Restriction ; - owl:allValuesFrom ttm:FileFormatEnum ; - owl:onProperty ttm:test_data_file_format ], - [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:maxCardinality 1 ; owl:onProperty ttm:test_data_file_format ], [ a owl:Restriction ; owl:maxCardinality 1 ; + owl:onProperty ttm:test_data_file_locator ], + [ a owl:Restriction ; + owl:allValuesFrom ttm:FileFormatEnum ; owl:onProperty ttm:test_data_file_format ], [ a owl:Restriction ; owl:allValuesFrom linkml:Uriorcurie ; owl:onProperty ttm:test_data_file_locator ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:test_data_file_locator ], + owl:minCardinality 0 ; + owl:onProperty ttm:test_data_file_format ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:test_data_file_locator ], @@ -795,14 +780,6 @@ ttm:qualifiers a owl:ObjectProperty, skos:definition "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; skos:inScheme . -ttm:runner_settings a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "runner_settings" ; - rdfs:range linkml:String ; - skos:altLabel "Settings" ; - skos:definition "Settings for the test harness for TestAsset" ; - skos:inScheme . - ttm:test_case_results a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "test_case_results" ; @@ -810,14 +787,6 @@ ttm:test_case_results a owl:ObjectProperty, skos:definition "One or more instances of TestCaseResult." ; skos:inScheme . -ttm:test_case_runner_settings a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "test_case_runner_settings" ; - rdfs:range linkml:String ; - skos:altLabel "Settings" ; - skos:definition "Settings for the test harness for TestCase" ; - skos:inScheme . - ttm:test_cases a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "test_cases" ; @@ -836,183 +805,183 @@ ttm:test_entity_parameters a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "test_entity_parameters" ; rdfs:range ttm:TestEntityParameter ; - skos:definition "One or more parameters documenting target characteristics of a TestEntity." ; + skos:definition "One or more 'tag = value' parameters documenting target characteristics of a TestEntity." ; skos:inScheme . ttm:TestAsset a owl:Class, linkml:ClassDefinition ; rdfs:label "TestAsset" ; rdfs:subClassOf [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:predicate_name ], - [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:qualifiers ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:input_category ], + owl:onProperty ttm:association ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:runner_settings ], + owl:allValuesFrom linkml:Boolean ; + owl:onProperty ttm:in_v1 ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:test_metadata ], + owl:allValuesFrom ttm:TestIssueEnum ; + owl:onProperty ttm:test_issue ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:in_v1 ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:input_id ], + owl:onProperty ttm:test_reference ], [ a owl:Restriction ; - owl:allValuesFrom ttm:predicate_type ; + owl:maxCardinality 1 ; owl:onProperty ttm:predicate_id ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:predicate_id ], + owl:onProperty ttm:output_id ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:expected_output ], + owl:minCardinality 0 ; + owl:onProperty ttm:output_category ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:semantic_severity ], + owl:allValuesFrom linkml:Uriorcurie ; + owl:onProperty ttm:input_id ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:id ], + owl:onProperty ttm:input_id ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:output_category ], + owl:minCardinality 0 ; + owl:onProperty ttm:predicate_id ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:predicate_name ], + owl:onProperty ttm:input_category ], [ a owl:Restriction ; - owl:allValuesFrom ttm:TestIssueEnum ; - owl:onProperty ttm:test_issue ], + owl:minCardinality 1 ; + owl:onProperty ttm:test_metadata ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:well_known ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:test_reference ], + owl:onProperty ttm:expected_output ], [ a owl:Restriction ; - owl:allValuesFrom ttm:concept_category ; - owl:onProperty ttm:output_category ], + owl:maxCardinality 1 ; + owl:onProperty ttm:id ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:output_id ], + owl:minCardinality 0 ; + owl:onProperty ttm:output_name ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:in_v1 ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:predicate_name ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:output_name ], + owl:onProperty ttm:expected_output ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Boolean ; - owl:onProperty ttm:well_known ], + owl:maxCardinality 1 ; + owl:onProperty ttm:test_issue ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:output_category ], + owl:onProperty ttm:input_id ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:input_id ], + owl:onProperty ttm:test_runner_settings ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:id ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:output_name ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:input_category ], + owl:allValuesFrom ttm:SemanticSeverityEnum ; + owl:onProperty ttm:semantic_severity ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:allValuesFrom ttm:concept_category ; + owl:onProperty ttm:output_category ], + [ a owl:Restriction ; + owl:allValuesFrom ttm:association_category ; owl:onProperty ttm:association ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:output_name ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; owl:onProperty ttm:test_issue ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:output_name ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:Boolean ; - owl:onProperty ttm:in_v1 ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:tags ], + owl:onProperty ttm:semantic_severity ], [ a owl:Restriction ; - owl:minCardinality 1 ; - owl:onProperty ttm:runner_settings ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:input_name ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:input_name ], + owl:onProperty ttm:input_category ], [ a owl:Restriction ; - owl:allValuesFrom ttm:association_category ; - owl:onProperty ttm:association ], + owl:allValuesFrom ttm:concept_category ; + owl:onProperty ttm:input_category ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:well_known ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; owl:onProperty ttm:input_name ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:expected_output ], + owl:onProperty ttm:association ], [ a owl:Restriction ; owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:input_id ], + owl:onProperty ttm:id ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:allValuesFrom linkml:Uriorcurie ; owl:onProperty ttm:test_reference ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:output_id ], + owl:onProperty ttm:predicate_name ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:output_id ], + owl:maxCardinality 1 ; + owl:onProperty ttm:semantic_severity ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:input_name ], + owl:maxCardinality 1 ; + owl:onProperty ttm:test_metadata ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:semantic_severity ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; owl:onProperty ttm:expected_output ], [ a owl:Restriction ; owl:allValuesFrom ttm:TestMetadata ; owl:onProperty ttm:test_metadata ], [ a owl:Restriction ; - owl:allValuesFrom ttm:concept_category ; - owl:onProperty ttm:input_category ], + owl:maxCardinality 1 ; + owl:onProperty ttm:output_name ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:tags ], + owl:allValuesFrom linkml:Uriorcurie ; + owl:onProperty ttm:output_id ], [ a owl:Restriction ; - owl:allValuesFrom ttm:SemanticSeverityEnum ; - owl:onProperty ttm:semantic_severity ], + owl:allValuesFrom ttm:predicate_type ; + owl:onProperty ttm:predicate_id ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:predicate_id ], + owl:onProperty ttm:in_v1 ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:test_reference ], + owl:maxCardinality 1 ; + owl:onProperty ttm:output_id ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:predicate_name ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:well_known ], [ a owl:Restriction ; owl:minCardinality 1 ; owl:onProperty ttm:id ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_runner_settings ], [ a owl:Restriction ; owl:allValuesFrom ttm:Qualifier ; owl:onProperty ttm:qualifiers ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:predicate_name ], + owl:allValuesFrom linkml:Boolean ; + owl:onProperty ttm:well_known ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:in_v1 ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:tags ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:well_known ], + owl:onProperty ttm:input_name ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_issue ], - [ a owl:Restriction ; - owl:minCardinality 1 ; - owl:onProperty ttm:test_metadata ], + owl:onProperty ttm:tags ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:association ], + owl:onProperty ttm:output_category ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:test_reference ], ttm:TestEntity ; skos:definition "Represents a Test Asset, which is a single specific instance of TestCase-agnostic semantic parameters representing the specification of a Translator test target with inputs and (expected) outputs." ; skos:inScheme . @@ -1021,41 +990,41 @@ ttm:TestMetadata a owl:Class, linkml:ClassDefinition ; rdfs:label "TestMetadata" ; rdfs:subClassOf [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:test_annotations ], + owl:minCardinality 0 ; + owl:onProperty ttm:test_objective ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:test_objective ], [ a owl:Restriction ; - owl:allValuesFrom ttm:TestObjectiveEnum ; - owl:onProperty ttm:test_objective ], + owl:allValuesFrom ttm:TestSourceEnum ; + owl:onProperty ttm:test_source ], [ a owl:Restriction ; owl:allValuesFrom linkml:Uriorcurie ; owl:onProperty ttm:test_reference ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_annotations ], - [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:test_reference ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; owl:onProperty ttm:test_source ], [ a owl:Restriction ; - owl:allValuesFrom ttm:TestSourceEnum ; + owl:maxCardinality 1 ; owl:onProperty ttm:test_source ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_source ], + owl:onProperty ttm:test_annotations ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:maxCardinality 1 ; + owl:onProperty ttm:test_annotations ], + [ a owl:Restriction ; + owl:allValuesFrom ttm:TestObjectiveEnum ; owl:onProperty ttm:test_objective ], [ a owl:Restriction ; - owl:maxCardinality 1 ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_annotations ], + [ a owl:Restriction ; + owl:minCardinality 0 ; owl:onProperty ttm:test_reference ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:test_annotations ], + owl:onProperty ttm:test_reference ], ttm:TestEntity ; skos:definition "Represents metadata related to (external SME, SMURF, Translator feedback, large scale batch, etc.) like the provenance of test assets, cases and/or suites." ; skos:inScheme . @@ -1069,38 +1038,38 @@ ttm:TestSuite a owl:Class, linkml:ClassDefinition ; rdfs:label "TestSuite" ; rdfs:subClassOf [ a owl:Restriction ; - owl:allValuesFrom ttm:TestSuiteSpecification ; + owl:minCardinality 0 ; owl:onProperty ttm:test_suite_specification ], [ a owl:Restriction ; - owl:allValuesFrom ttm:TestMetadata ; - owl:onProperty ttm:test_metadata ], + owl:allValuesFrom ttm:TestCase ; + owl:onProperty ttm:test_cases ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:test_persona ], - [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:test_cases ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:test_suite_specification ], + [ a owl:Restriction ; + owl:allValuesFrom ttm:TestSuiteSpecification ; + owl:onProperty ttm:test_suite_specification ], [ a owl:Restriction ; owl:minCardinality 1 ; owl:onProperty ttm:test_metadata ], + [ a owl:Restriction ; + owl:allValuesFrom ttm:TestPersonaEnum ; + owl:onProperty ttm:test_persona ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:test_persona ], + [ a owl:Restriction ; + owl:allValuesFrom ttm:TestMetadata ; + owl:onProperty ttm:test_metadata ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:test_metadata ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:TestCase ; - owl:onProperty ttm:test_cases ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_suite_specification ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:TestPersonaEnum ; - owl:onProperty ttm:test_persona ], + owl:onProperty ttm:test_cases ], ttm:TestEntity ; skos:definition "Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_suite_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present." ; skos:inScheme . @@ -1129,12 +1098,6 @@ ttm:association a owl:ObjectProperty, skos:definition "Specific Biolink Model association 'category' which applies to the test asset defined knowledge statement" ; skos:inScheme . -ttm:biolink_version a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "biolink_version" ; - skos:definition "Biolink Model release (SemVer string)" ; - skos:inScheme . - ttm:bte a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "bte" ; @@ -1418,7 +1381,7 @@ ttm:test_run_parameters a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "test_run_parameters" ; rdfs:subPropertyOf ttm:test_entity_parameters ; - skos:definition "Parameters for TestRunnerConfiguration that inform the TestHarness and TestRunners about the general characteristics of a test run." ; + skos:definition "TestRunSession parameters informing a TestHarness about the global characteristics of TestRunner processing tests." ; skos:inScheme . ttm:test_runner_name a owl:ObjectProperty, @@ -1469,12 +1432,6 @@ ttm:trapi_template a owl:ObjectProperty, skos:definition "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; skos:inScheme . -ttm:trapi_version a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "trapi_version" ; - skos:definition "TRAPI version (SemVer string)" ; - skos:inScheme . - ttm:unsecret a owl:ObjectProperty, linkml:SlotDefinition ; rdfs:label "unsecret" ; @@ -1502,6 +1459,14 @@ ttm:test_assets a owl:ObjectProperty, skos:definition "List of explicitly enumerated Test Assets. The class attributes of TestAsset would be included in the TestCase versus being referred to by the identifier (curie) of the TestAsset. That is, this would be a list of objects (in JSONSchema serialization) versus a list of strings (where each string is an identifier pointing to another class)." ; skos:inScheme . +ttm:test_runner_settings a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "test_runner_settings" ; + rdfs:range linkml:String ; + skos:altLabel "Settings" ; + skos:definition "Scalar parameters for the TestRunner processing a given TestEntity." ; + skos:inScheme . + ttm:DirectionEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; @@ -1520,169 +1485,106 @@ ttm:NodeEnum a owl:Class, linkml:permissible_values , . -ttm:id a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "id" ; - rdfs:range linkml:Uriorcurie ; - skos:definition "A unique identifier for a Test Entity" ; - skos:inScheme . - -ttm:parameter a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "parameter" ; - skos:definition "Name of a TestParameter." ; - skos:inScheme . - -ttm:test_metadata a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "test_metadata" ; - rdfs:range ttm:TestMetadata ; - skos:definition "Test metadata describes the external provenance, cross-references and objectives for a given test." ; - skos:inScheme . - -ttm:test_reference a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "test_reference" ; - rdfs:range linkml:Uriorcurie ; - skos:altLabel "GitHubIssue" ; - skos:definition "Document URL where original test source particulars are registered (e.g. Github repo)" ; - skos:inScheme . - -ttm:value a owl:ObjectProperty, - linkml:SlotDefinition ; - rdfs:label "value" ; - skos:definition "(String) value of a TestParameter." ; - skos:inScheme . - -ttm:FileFormatEnum a owl:Class, - linkml:EnumDefinition ; - owl:unionOf ( ) ; - linkml:permissible_values , - , - . - -ttm:SemanticSeverityEnum a owl:Class, - linkml:EnumDefinition ; - owl:unionOf ( ) ; - linkml:permissible_values , - , - . - ttm:TestCase a owl:Class, linkml:ClassDefinition ; rdfs:label "TestCase" ; rdfs:subClassOf [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:trapi_template ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:QueryTypeEnum ; owl:onProperty ttm:query_type ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:test_env ], + owl:allValuesFrom ttm:Precondition ; + owl:onProperty ttm:preconditions ], [ a owl:Restriction ; owl:minCardinality 0 ; owl:onProperty ttm:test_case_input_id ], [ a owl:Restriction ; - owl:allValuesFrom ttm:TestObjectiveEnum ; - owl:onProperty ttm:test_case_objective ], + owl:allValuesFrom owl:Thing ; + owl:onProperty ttm:test_assets ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:test_case_predicate_id ], + owl:onProperty ttm:test_case_predicate_name ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_case_objective ], + owl:onProperty ttm:test_case_predicate_name ], [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:test_case_source ], + owl:minCardinality 0 ; + owl:onProperty ttm:tags ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:query_type ], + owl:onProperty ttm:trapi_template ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:allValuesFrom ttm:TrapiTemplateEnum ; owl:onProperty ttm:trapi_template ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:components ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_case_predicate_name ], [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:maxCardinality 1 ; owl:onProperty ttm:test_case_predicate_id ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; - owl:onProperty ttm:test_case_predicate_name ], + owl:onProperty ttm:tags ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:test_case_objective ], [ a owl:Restriction ; owl:allValuesFrom ttm:TestSourceEnum ; owl:onProperty ttm:test_case_source ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:query_type ], + owl:onProperty ttm:test_case_source ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_case_source ], + owl:onProperty ttm:query_type ], [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:test_case_runner_settings ], + owl:allValuesFrom ttm:QueryTypeEnum ; + owl:onProperty ttm:query_type ], + [ a owl:Restriction ; + owl:maxCardinality 1 ; + owl:onProperty ttm:test_case_input_id ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:test_case_objective ], [ a owl:Restriction ; - owl:minCardinality 1 ; - owl:onProperty ttm:test_case_runner_settings ], + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_case_predicate_id ], [ a owl:Restriction ; - owl:allValuesFrom ttm:Precondition ; + owl:minCardinality 0 ; owl:onProperty ttm:preconditions ], - [ a owl:Restriction ; - owl:minCardinality 1 ; - owl:onProperty ttm:test_assets ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:test_case_input_id ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:tags ], + owl:onProperty ttm:test_case_predicate_id ], [ a owl:Restriction ; - owl:allValuesFrom owl:Thing ; + owl:minCardinality 1 ; owl:onProperty ttm:test_assets ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:ComponentEnum ; - owl:onProperty ttm:components ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:test_case_predicate_name ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:test_case_predicate_id ], + owl:onProperty ttm:trapi_template ], [ a owl:Restriction ; - owl:minCardinality 0 ; - owl:onProperty ttm:preconditions ], + owl:allValuesFrom ttm:TestObjectiveEnum ; + owl:onProperty ttm:test_case_objective ], [ a owl:Restriction ; - owl:maxCardinality 1 ; + owl:allValuesFrom linkml:Uriorcurie ; owl:onProperty ttm:test_case_input_id ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:TestEnvEnum ; - owl:onProperty ttm:test_env ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:test_env ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:test_case_predicate_name ], - [ a owl:Restriction ; - owl:allValuesFrom ttm:TrapiTemplateEnum ; - owl:onProperty ttm:trapi_template ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; - owl:onProperty ttm:tags ], + owl:onProperty ttm:test_case_source ], ttm:TestEntity ; skos:definition "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition." ; skos:inScheme . -ttm:TestCaseResultEnum a owl:Class, - linkml:EnumDefinition ; - owl:unionOf ( ) ; - linkml:permissible_values , - , - . +ttm:id a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "id" ; + rdfs:range linkml:Uriorcurie ; + skos:definition "A unique identifier for a Test Entity" ; + skos:inScheme . + +ttm:parameter a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "parameter" ; + skos:definition "Name of a TestParameter." ; + skos:inScheme . ttm:tags a owl:ObjectProperty, linkml:SlotDefinition ; @@ -1690,6 +1592,48 @@ ttm:tags a owl:ObjectProperty, skos:definition "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; skos:inScheme . +ttm:test_metadata a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "test_metadata" ; + rdfs:range ttm:TestMetadata ; + skos:definition "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + skos:inScheme . + +ttm:test_reference a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "test_reference" ; + rdfs:range linkml:Uriorcurie ; + skos:altLabel "GitHubIssue" ; + skos:definition "Document URL where original test source particulars are registered (e.g. Github repo)" ; + skos:inScheme . + +ttm:value a owl:ObjectProperty, + linkml:SlotDefinition ; + rdfs:label "value" ; + skos:definition "(String) value of a TestParameter." ; + skos:inScheme . + +ttm:FileFormatEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + , + . + +ttm:SemanticSeverityEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + , + . + +ttm:TestCaseResultEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + , + . + ttm:ExpectedOutputEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; @@ -1699,13 +1643,6 @@ ttm:ExpectedOutputEnum a owl:Class, , . -ttm:TestObjectiveEnum a owl:Class, - linkml:EnumDefinition ; - owl:unionOf ( ) ; - linkml:permissible_values , - , - . - ttm:TestPersonaEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; @@ -1718,37 +1655,43 @@ ttm:TestEntity a owl:Class, linkml:ClassDefinition ; rdfs:label "TestEntity" ; rdfs:subClassOf [ a owl:Restriction ; - owl:minCardinality 0 ; + owl:allValuesFrom linkml:String ; + owl:onProperty ttm:test_runner_settings ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:String ; owl:onProperty ttm:name ], [ a owl:Restriction ; owl:maxCardinality 1 ; - owl:onProperty ttm:description ], - [ a owl:Restriction ; - owl:allValuesFrom linkml:String ; owl:onProperty ttm:name ], [ a owl:Restriction ; - owl:allValuesFrom linkml:Uriorcurie ; - owl:onProperty ttm:id ], + owl:maxCardinality 1 ; + owl:onProperty ttm:description ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; owl:onProperty ttm:tags ], + [ a owl:Restriction ; + owl:minCardinality 0 ; + owl:onProperty ttm:tags ], [ a owl:Restriction ; owl:minCardinality 1 ; owl:onProperty ttm:id ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:description ], + owl:onProperty ttm:test_runner_settings ], [ a owl:Restriction ; owl:maxCardinality 1 ; owl:onProperty ttm:id ], - [ a owl:Restriction ; - owl:maxCardinality 1 ; - owl:onProperty ttm:name ], [ a owl:Restriction ; owl:minCardinality 0 ; - owl:onProperty ttm:tags ], + owl:onProperty ttm:name ], [ a owl:Restriction ; owl:allValuesFrom linkml:String ; + owl:onProperty ttm:description ], + [ a owl:Restriction ; + owl:allValuesFrom linkml:Uriorcurie ; + owl:onProperty ttm:id ], + [ a owl:Restriction ; + owl:minCardinality 0 ; owl:onProperty ttm:description ] ; skos:definition "Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing." ; skos:inScheme . @@ -1780,6 +1723,15 @@ ttm:TestIssueEnum a owl:Class, , . +ttm:TestObjectiveEnum a owl:Class, + linkml:EnumDefinition ; + owl:unionOf ( ) ; + linkml:permissible_values , + , + , + , + . + ttm:TestSourceEnum a owl:Class, linkml:EnumDefinition ; owl:unionOf ( ) ; diff --git a/project/protobuf/translator_testing_model.proto b/project/protobuf/translator_testing_model.proto index be7c114..dcd3cb8 100644 --- a/project/protobuf/translator_testing_model.proto +++ b/project/protobuf/translator_testing_model.proto @@ -19,10 +19,10 @@ message AcceptanceTestAsset boolean inV1 = 0 boolean wellKnown = 0 uriorcurie testReference = 0 - repeated string runnerSettings = 0 testMetadata testMetadata = 0 uriorcurie id = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 date mustPassDate = 0 testEnvEnum mustPassEnvironment = 0 string scientificQuestion = 0 @@ -40,17 +40,15 @@ message AcceptanceTestCase uriorcurie id = 0 string name = 0 string description = 0 - testEnvEnum testEnv = 0 + repeated string testRunnerSettings = 0 queryTypeEnum queryType = 0 repeated precondition preconditions = 0 trapiTemplateEnum trapiTemplate = 0 - repeated componentEnum components = 0 testObjectiveEnum testCaseObjective = 0 testSourceEnum testCaseSource = 0 string testCasePredicateName = 0 string testCasePredicateId = 0 uriorcurie testCaseInputId = 0 - repeated string testCaseRunnerSettings = 0 repeated string tags = 0 repeated acceptanceTestAsset testAssets = 0 } @@ -60,73 +58,12 @@ message AcceptanceTestSuite string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 testMetadata testMetadata = 0 testPersonaEnum testPersona = 0 repeated testCase testCases = 0 testSuiteSpecification testSuiteSpecification = 0 } -// TRAPI and Biolink Model standards compliance test -message ComplianceTestCase - { - uriorcurie id = 0 - string name = 0 - string description = 0 - testEnvEnum testEnv = 0 - queryTypeEnum queryType = 0 - repeated testAsset testAssets = 0 - repeated precondition preconditions = 0 - trapiTemplateEnum trapiTemplate = 0 - repeated componentEnum components = 0 - testObjectiveEnum testCaseObjective = 0 - testSourceEnum testCaseSource = 0 - string testCasePredicateName = 0 - string testCasePredicateId = 0 - uriorcurie testCaseInputId = 0 - repeated string testCaseRunnerSettings = 0 - repeated string tags = 0 - string trapiVersion = 0 - string biolinkVersion = 0 - } -// Knowledge Graph navigation integration test -message KnowledgeGraphNavigationTestCase - { - uriorcurie id = 0 - string name = 0 - string description = 0 - testEnvEnum testEnv = 0 - queryTypeEnum queryType = 0 - repeated testAsset testAssets = 0 - repeated precondition preconditions = 0 - trapiTemplateEnum trapiTemplate = 0 - repeated componentEnum components = 0 - testObjectiveEnum testCaseObjective = 0 - testSourceEnum testCaseSource = 0 - string testCasePredicateName = 0 - string testCasePredicateId = 0 - uriorcurie testCaseInputId = 0 - repeated string testCaseRunnerSettings = 0 - repeated string tags = 0 - } -// 'One Hop' Knowledge Graph navigation integration test -message OneHopTestCase - { - uriorcurie id = 0 - string name = 0 - string description = 0 - testEnvEnum testEnv = 0 - queryTypeEnum queryType = 0 - repeated testAsset testAssets = 0 - repeated precondition preconditions = 0 - trapiTemplateEnum trapiTemplate = 0 - repeated componentEnum components = 0 - testObjectiveEnum testCaseObjective = 0 - testSourceEnum testCaseSource = 0 - string testCasePredicateName = 0 - string testCasePredicateId = 0 - uriorcurie testCaseInputId = 0 - repeated string testCaseRunnerSettings = 0 - repeated string tags = 0 - } // Test case for testing the integrity of "One Hop" knowledge graph retrievals sensa legacy SRI_Testing harness. message OneHopTestSuite { @@ -134,6 +71,7 @@ message OneHopTestSuite string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 testMetadata testMetadata = 0 testPersonaEnum testPersona = 0 repeated testCase testCases = 0 @@ -146,6 +84,7 @@ message Precondition string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 } message Qualifier { @@ -158,18 +97,16 @@ message QuantitativeTestCase uriorcurie id = 0 string name = 0 string description = 0 - testEnvEnum testEnv = 0 + repeated string testRunnerSettings = 0 queryTypeEnum queryType = 0 repeated testAsset testAssets = 0 repeated precondition preconditions = 0 trapiTemplateEnum trapiTemplate = 0 - repeated componentEnum components = 0 testObjectiveEnum testCaseObjective = 0 testSourceEnum testCaseSource = 0 string testCasePredicateName = 0 string testCasePredicateId = 0 uriorcurie testCaseInputId = 0 - repeated string testCaseRunnerSettings = 0 repeated string tags = 0 } // Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model. @@ -179,6 +116,7 @@ message StandardsComplianceTestSuite string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 testMetadata testMetadata = 0 testPersonaEnum testPersona = 0 repeated testCase testCases = 0 @@ -205,10 +143,10 @@ message TestAsset boolean inV1 = 0 boolean wellKnown = 0 uriorcurie testReference = 0 - repeated string runnerSettings = 0 testMetadata testMetadata = 0 uriorcurie id = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 } // Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition. message TestCase @@ -216,18 +154,16 @@ message TestCase uriorcurie id = 0 string name = 0 string description = 0 - testEnvEnum testEnv = 0 + repeated string testRunnerSettings = 0 queryTypeEnum queryType = 0 repeated testAsset testAssets = 0 repeated precondition preconditions = 0 trapiTemplateEnum trapiTemplate = 0 - repeated componentEnum components = 0 testObjectiveEnum testCaseObjective = 0 testSourceEnum testCaseSource = 0 string testCasePredicateName = 0 string testCasePredicateId = 0 uriorcurie testCaseInputId = 0 - repeated string testCaseRunnerSettings = 0 repeated string tags = 0 } // The outcome of a TestRunner run of one specific TestCase. @@ -237,6 +173,7 @@ message TestCaseResult string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 uriorcurie testSuiteId = 0 testCase testCase = 0 testCaseResultEnum testCaseResult = 0 @@ -262,10 +199,10 @@ message TestEdgeData boolean inV1 = 0 boolean wellKnown = 0 uriorcurie testReference = 0 - repeated string runnerSettings = 0 testMetadata testMetadata = 0 uriorcurie id = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 } // A single 'tag = value' pair (where 'value' is a simple string). message TestEntityParameter @@ -280,6 +217,7 @@ message TestMetadata string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 testSourceEnum testSource = 0 uriorcurie testReference = 0 testObjectiveEnum testObjective = 0 @@ -292,6 +230,7 @@ message TestOutput string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 string testCaseId = 0 repeated testResultPKSet pks = 0 } @@ -302,6 +241,7 @@ message TestResultPKSet string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 string parentPk = 0 string mergedPk = 0 string aragorn = 0 @@ -310,23 +250,18 @@ message TestResultPKSet string bte = 0 string improving = 0 } -// General configuration parameters and test data input for a single invocation of a TestRunner. -message TestRunnerConfiguration - { - uriorcurie id = 0 - string name = 0 - string description = 0 - repeated testEntityParameter testRunParameters = 0 - repeated string tags = 0 - } -// A single invocation of a TestRunner. +// Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite). message TestRunSession { uriorcurie id = 0 string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 + repeated componentEnum components = 0 + testEnvEnum testEnv = 0 string testRunnerName = 0 + repeated testEntityParameter testRunParameters = 0 repeated testEntity testEntities = 0 repeated testCaseResult testCaseResults = 0 datetime timestamp = 0 @@ -338,6 +273,7 @@ message TestSuite string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 testMetadata testMetadata = 0 testPersonaEnum testPersona = 0 repeated testCase testCases = 0 @@ -350,6 +286,7 @@ message TestSuiteSpecification string name = 0 string description = 0 repeated string tags = 0 + repeated string testRunnerSettings = 0 uriorcurie testDataFileLocator = 0 fileFormatEnum testDataFileFormat = 0 } diff --git a/project/shacl/translator_testing_model.shacl.ttl b/project/shacl/translator_testing_model.shacl.ttl index 8d70cb8..dcad69f 100644 --- a/project/shacl/translator_testing_model.shacl.ttl +++ b/project/shacl/translator_testing_model.shacl.ttl @@ -8,108 +8,96 @@ ttm:AcceptanceTestCase a sh:NodeShape ; sh:closed true ; sh:description "See AcceptanceTestAsset above for more details." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:preconditions ], - [ sh:datatype xsd:string ; + sh:property [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 8 ; + sh:order 6 ; sh:path ttm:test_case_predicate_name ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:preconditions ], [ sh:maxCount 1 ; - sh:order 10 ; + sh:order 8 ; sh:path ttm:test_case_input_id ], - [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; + [ sh:class ttm:AcceptanceTestAsset ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 1 ; + sh:path ttm:test_assets ], + [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:test_case_objective ], + sh:order 7 ; + sh:path ttm:test_case_predicate_id ], + [ sh:datatype xsd:string ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 13 ; + sh:path ttm:test_runner_settings ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 12 ; + sh:order 9 ; sh:path schema1:identifier ], [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; sh:maxCount 1 ; - sh:order 7 ; + sh:order 5 ; sh:path ttm:test_case_source ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 13 ; - sh:path schema1:name ], - [ sh:datatype xsd:string ; + [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; + sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:test_case_predicate_id ], + sh:order 3 ; + sh:path ttm:trapi_template ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 14 ; + sh:order 11 ; sh:path schema1:description ], - [ sh:class ttm:AcceptanceTestAsset ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_assets ], [ sh:datatype xsd:string ; - sh:description "Settings for the test harness for TestCase" ; - sh:minCount 1 ; - sh:order 11 ; - sh:path ttm:test_case_runner_settings ], - [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; - sh:in [ rdf:rest () ] ; - sh:order 5 ; - sh:path ttm:components ], + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 10 ; + sh:path schema1:name ], [ sh:description "Type of TestCase query." ; sh:in ( "treats" ) ; sh:maxCount 1 ; - sh:order 1 ; + sh:order 0 ; sh:path ttm:query_type ], - [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; - sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; + [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; + sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" "OneHopTest" "StandardsValidationTest" ) ; sh:maxCount 1 ; sh:order 4 ; - sh:path ttm:trapi_template ], + sh:path ttm:test_case_objective ], [ sh:datatype xsd:string ; sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; - sh:order 15 ; - sh:path schema1:additionalType ], - [ sh:description "Deployment environment within which the associated TestSuite is run." ; - sh:in ( "dev" "ci" "test" "prod" ) ; - sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:test_env ] ; + sh:order 12 ; + sh:path schema1:additionalType ] ; sh:targetClass ttm:AcceptanceTestCase . ttm:AcceptanceTestSuite a sh:NodeShape ; sh:closed true ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:TestMetadata ; - sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 0 ; - sh:path ttm:test_metadata ], - [ sh:class ttm:TestSuiteSpecification ; - sh:description "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." ; + sh:property [ sh:description "A Test persona describes the user or operational context of a given test." ; + sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; sh:maxCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:test_suite_specification ], + sh:order 1 ; + sh:path ttm:test_persona ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 5 ; sh:path schema1:name ], - [ sh:description "A unique identifier for a Test Entity" ; + [ sh:class ttm:TestSuiteSpecification ; + sh:description "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 4 ; - sh:path schema1:identifier ], + sh:nodeKind sh:IRI ; + sh:order 3 ; + sh:path ttm:test_suite_specification ], + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:test_cases ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; @@ -119,16 +107,22 @@ ttm:AcceptanceTestSuite a sh:NodeShape ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 7 ; sh:path schema1:additionalType ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_cases ], - [ sh:description "A Test persona describes the user or operational context of a given test." ; - sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:test_persona ] ; + sh:minCount 1 ; + sh:order 4 ; + sh:path schema1:identifier ], + [ sh:datatype xsd:string ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 8 ; + sh:path ttm:test_runner_settings ], + [ sh:class ttm:TestMetadata ; + sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 0 ; + sh:path ttm:test_metadata ] ; sh:targetClass ttm:AcceptanceTestSuite . ttm:BenchmarkTestSuite a sh:NodeShape ; @@ -137,265 +131,151 @@ ttm:BenchmarkTestSuite a sh:NodeShape ; sh:ignoredProperties ( rdf:type ) ; sh:targetClass ttm:BenchmarkTestSuite . -ttm:ComplianceTestCase a sh:NodeShape ; +ttm:OneHopTestSuite a sh:NodeShape ; sh:closed true ; - sh:description "TRAPI and Biolink Model standards compliance test" ; + sh:description "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; - sh:maxCount 1 ; - sh:order 8 ; - sh:path ttm:test_case_objective ], - [ sh:maxCount 1 ; - sh:order 12 ; - sh:path ttm:test_case_input_id ], - [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; - sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; + sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:trapi_template ], - [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; - sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; + sh:order 5 ; + sh:path schema1:name ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 7 ; + sh:path schema1:additionalType ], + [ sh:class ttm:TestMetadata ; + sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:test_case_source ], + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 0 ; + sh:path ttm:test_metadata ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 16 ; + sh:order 6 ; sh:path schema1:description ], - [ sh:class ttm:TestAsset ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 4 ; - sh:path ttm:test_assets ], - [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; - sh:in [ rdf:rest () ] ; - sh:order 7 ; - sh:path ttm:components ], [ sh:datatype xsd:string ; - sh:description "Settings for the test harness for TestCase" ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 8 ; + sh:path ttm:test_runner_settings ], + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:test_cases ], + [ sh:class ttm:TestSuiteSpecification ; + sh:description "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." ; + sh:maxCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 3 ; + sh:path ttm:test_suite_specification ], + [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; sh:minCount 1 ; - sh:order 13 ; - sh:path ttm:test_case_runner_settings ], - [ sh:description "Type of TestCase query." ; + sh:order 4 ; + sh:path schema1:identifier ], + [ sh:description "A Test persona describes the user or operational context of a given test." ; + sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_persona ] ; + sh:targetClass ttm:OneHopTestSuite . + +ttm:QuantitativeTestCase a sh:NodeShape ; + sh:closed true ; + sh:description "Assumed additional model from Shervin's runner JSON here as an example. This schema is not yet complete." ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:description "Type of TestCase query." ; sh:in ( "treats" ) ; sh:maxCount 1 ; - sh:order 3 ; + sh:order 0 ; sh:path ttm:query_type ], [ sh:datatype xsd:string ; sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; - sh:order 17 ; + sh:order 12 ; sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "Biolink Model release (SemVer string)" ; - sh:maxCount 1 ; + [ sh:class ttm:TestAsset ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; sh:order 1 ; - sh:path ttm:biolink_version ], + sh:path ttm:test_assets ], + [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; + sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; + sh:maxCount 1 ; + sh:order 5 ; + sh:path ttm:test_case_source ], + [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; + sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; + sh:maxCount 1 ; + sh:order 3 ; + sh:path ttm:trapi_template ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 15 ; + sh:order 10 ; sh:path schema1:name ], [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; sh:order 11 ; - sh:path ttm:test_case_predicate_id ], + sh:path schema1:description ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:preconditions ], + [ sh:maxCount 1 ; + sh:order 8 ; + sh:path ttm:test_case_input_id ], [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:test_case_predicate_name ], + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 13 ; + sh:path ttm:test_runner_settings ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 14 ; + sh:order 9 ; sh:path schema1:identifier ], - [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 5 ; - sh:path ttm:preconditions ], - [ sh:description "Deployment environment within which the associated TestSuite is run." ; - sh:in ( "dev" "ci" "test" "prod" ) ; + [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:test_env ], + sh:order 6 ; + sh:path ttm:test_case_predicate_name ], + [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; + sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" "OneHopTest" "StandardsValidationTest" ) ; + sh:maxCount 1 ; + sh:order 4 ; + sh:path ttm:test_case_objective ], [ sh:datatype xsd:string ; - sh:description "TRAPI version (SemVer string)" ; sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:trapi_version ] ; - sh:targetClass ttm:ComplianceTestCase . + sh:order 7 ; + sh:path ttm:test_case_predicate_id ] ; + sh:targetClass ttm:QuantitativeTestCase . -ttm:KnowledgeGraphNavigationTestCase a sh:NodeShape ; +ttm:StandardsComplianceTestSuite a sh:NodeShape ; sh:closed true ; - sh:description "Knowledge Graph navigation integration test" ; + sh:description "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:TestAsset ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_assets ], - [ sh:description "Type of TestCase query." ; - sh:in ( "treats" ) ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:query_type ], - [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; - sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; - sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:test_case_source ], - [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; - sh:in [ rdf:rest () ] ; - sh:order 5 ; - sh:path ttm:components ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 13 ; - sh:path schema1:name ], - [ sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:test_case_input_id ], - [ sh:description "Deployment environment within which the associated TestSuite is run." ; - sh:in ( "dev" "ci" "test" "prod" ) ; - sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:test_env ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 8 ; - sh:path ttm:test_case_predicate_name ], - [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; + sh:property [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:test_case_objective ], - [ sh:datatype xsd:string ; - sh:description "Settings for the test harness for TestCase" ; sh:minCount 1 ; - sh:order 11 ; - sh:path ttm:test_case_runner_settings ], - [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; - sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; - sh:maxCount 1 ; sh:order 4 ; - sh:path ttm:trapi_template ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 14 ; - sh:path schema1:description ], - [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:preconditions ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:test_case_predicate_id ], + sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; - sh:order 15 ; - sh:path schema1:additionalType ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 12 ; - sh:path schema1:identifier ] ; - sh:targetClass ttm:KnowledgeGraphNavigationTestCase . - -ttm:OneHopTestCase a sh:NodeShape ; - sh:closed true ; - sh:description "'One Hop' Knowledge Graph navigation integration test" ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:test_case_predicate_id ], - [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; - sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; - sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:trapi_template ], - [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 8 ; + sh:path ttm:test_runner_settings ], + [ sh:description "A Test persona describes the user or operational context of a given test." ; + sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:test_case_objective ], + sh:order 1 ; + sh:path ttm:test_persona ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 14 ; + sh:order 6 ; sh:path schema1:description ], - [ sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:test_case_input_id ], - [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; - sh:in [ rdf:rest () ] ; - sh:order 5 ; - sh:path ttm:components ], - [ sh:description "Type of TestCase query." ; - sh:in ( "treats" ) ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:query_type ], - [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:preconditions ], - [ sh:class ttm:TestAsset ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_assets ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 8 ; - sh:path ttm:test_case_predicate_name ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; - sh:order 15 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "Settings for the test harness for TestCase" ; - sh:minCount 1 ; - sh:order 11 ; - sh:path ttm:test_case_runner_settings ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 13 ; - sh:path schema1:name ], - [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; - sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; - sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:test_case_source ], - [ sh:description "Deployment environment within which the associated TestSuite is run." ; - sh:in ( "dev" "ci" "test" "prod" ) ; - sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:test_env ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 12 ; - sh:path schema1:identifier ] ; - sh:targetClass ttm:OneHopTestCase . - -ttm:OneHopTestSuite a sh:NodeShape ; - sh:closed true ; - sh:description "Test case for testing the integrity of \"One Hop\" knowledge graph retrievals sensa legacy SRI_Testing harness." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 4 ; - sh:path schema1:identifier ], [ sh:class ttm:TestMetadata ; sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; sh:maxCount 1 ; @@ -403,261 +283,122 @@ ttm:OneHopTestSuite a sh:NodeShape ; sh:nodeKind sh:IRI ; sh:order 0 ; sh:path ttm:test_metadata ], - [ sh:description "A Test persona describes the user or operational context of a given test." ; - sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:test_persona ], [ sh:class ttm:TestSuiteSpecification ; sh:description "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; sh:order 3 ; sh:path ttm:test_suite_specification ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_cases ], [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 7 ; sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path schema1:description ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 5 ; - sh:path schema1:name ] ; - sh:targetClass ttm:OneHopTestSuite . + sh:path schema1:name ], + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:test_cases ] ; + sh:targetClass ttm:StandardsComplianceTestSuite . -ttm:QuantitativeTestCase a sh:NodeShape ; +ttm:TestEdgeData a sh:NodeShape ; sh:closed true ; - sh:description "Assumed additional model from Shervin's runner JSON here as an example. This schema is not yet complete." ; + sh:description "Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "Type of TestCase query." ; - sh:in ( "treats" ) ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:query_type ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 8 ; - sh:path ttm:test_case_predicate_name ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; - sh:order 15 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 14 ; - sh:path schema1:description ], - [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; - sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; - sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:trapi_template ], + sh:property [ sh:maxCount 1 ; + sh:order 7 ; + sh:path ttm:output_category ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:test_case_predicate_id ], - [ sh:maxCount 1 ; sh:order 10 ; - sh:path ttm:test_case_input_id ], - [ sh:datatype xsd:string ; - sh:description "Settings for the test harness for TestCase" ; - sh:minCount 1 ; - sh:order 11 ; - sh:path ttm:test_case_runner_settings ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 12 ; - sh:path schema1:identifier ], + sh:path ttm:expected_output ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 13 ; + sh:order 18 ; sh:path schema1:name ], - [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; - sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; - sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:test_case_source ], - [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; - sh:in [ rdf:rest () ] ; - sh:order 5 ; - sh:path ttm:components ], - [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:test_case_objective ], - [ sh:description "Deployment environment within which the associated TestSuite is run." ; - sh:in ( "dev" "ci" "test" "prod" ) ; + [ sh:datatype xsd:boolean ; sh:maxCount 1 ; + sh:order 13 ; + sh:path ttm:in_v1 ], + [ sh:maxCount 1 ; sh:order 0 ; - sh:path ttm:test_env ], - [ sh:class ttm:TestAsset ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_assets ], - [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:preconditions ] ; - sh:targetClass ttm:QuantitativeTestCase . - -ttm:StandardsComplianceTestSuite a sh:NodeShape ; - sh:closed true ; - sh:description "Test suite for testing Translator components against releases of standards like TRAPI and the Biolink Model." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A Test persona describes the user or operational context of a given test." ; - sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; + sh:path ttm:input_id ], + [ sh:in ( "High" "Low" "NotApplicable" ) ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:test_persona ], - [ sh:class ttm:TestMetadata ; - sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + sh:order 12 ; + sh:path ttm:semantic_severity ], + [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 0 ; - sh:path ttm:test_metadata ], + sh:order 1 ; + sh:path ttm:input_name ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 4 ; + sh:order 17 ; sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 5 ; - sh:path schema1:name ], - [ sh:class ttm:TestSuiteSpecification ; - sh:description "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." ; - sh:maxCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:test_suite_specification ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_cases ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path schema1:description ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ] ; - sh:targetClass ttm:StandardsComplianceTestSuite . - -ttm:TestEdgeData a sh:NodeShape ; - sh:closed true ; - sh:description "Represents a single Biolink Model compliant instance of a subject-predicate-object edge that can be used for testing." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 20 ; - sh:path schema1:description ], + sh:description "Scalar settings for the TestRunner, e.g. \"inferred\"" ; + sh:order 21 ; + sh:path ttm:test_runner_settings ], [ sh:maxCount 1 ; sh:order 3 ; sh:path ttm:predicate_id ], - [ sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:output_category ], - [ sh:in ( "High" "Low" "NotApplicable" ) ; - sh:maxCount 1 ; - sh:order 12 ; - sh:path ttm:semantic_severity ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 18 ; - sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 19 ; - sh:path schema1:name ], + sh:order 6 ; + sh:path ttm:output_name ], [ sh:description "Specific Biolink Model association 'category' which applies to the test asset defined knowledge statement" ; sh:maxCount 1 ; sh:order 8 ; sh:path ttm:association ], [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection" ; + sh:order 20 ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:output_name ], + sh:order 14 ; + sh:path ttm:well_known ], [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; sh:maxCount 1 ; sh:order 15 ; sh:path ttm:test_reference ], - [ sh:datatype xsd:boolean ; - sh:maxCount 1 ; - sh:order 13 ; - sh:path ttm:in_v1 ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection" ; - sh:order 21 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:input_name ], - [ sh:class ttm:Qualifier ; - sh:description "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; - sh:nodeKind sh:BlankNode ; - sh:order 9 ; - sh:path ttm:qualifiers ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:expected_output ], [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" "test_issue" ) ; sh:maxCount 1 ; sh:order 11 ; sh:path ttm:test_issue ], - [ sh:maxCount 1 ; - sh:order 5 ; - sh:path ttm:output_id ], [ sh:datatype xsd:string ; sh:maxCount 1 ; sh:order 4 ; sh:path ttm:predicate_name ], [ sh:datatype xsd:string ; - sh:description "Settings for the test harness, e.g. \"inferred\"" ; - sh:minCount 1 ; - sh:order 16 ; - sh:path ttm:runner_settings ], - [ sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:input_id ], - [ sh:datatype xsd:boolean ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 14 ; - sh:path ttm:well_known ], + sh:order 19 ; + sh:path schema1:description ], [ sh:class ttm:TestMetadata ; sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; sh:maxCount 1 ; sh:minCount 1 ; sh:nodeKind sh:IRI ; - sh:order 17 ; + sh:order 16 ; sh:path ttm:test_metadata ], [ sh:maxCount 1 ; sh:order 2 ; - sh:path ttm:input_category ] ; + sh:path ttm:input_category ], + [ sh:class ttm:Qualifier ; + sh:description "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; + sh:nodeKind sh:BlankNode ; + sh:order 9 ; + sh:path ttm:qualifiers ], + [ sh:maxCount 1 ; + sh:order 5 ; + sh:path ttm:output_id ] ; sh:targetClass ttm:TestEdgeData . ttm:TestOutput a sh:NodeShape ; @@ -665,125 +406,108 @@ ttm:TestOutput a sh:NodeShape ; sh:description "The output of a TestRunner run of one specific TestCase." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 5 ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:string ; sh:description "CURIE id of a TestCase registered in the system." ; sh:maxCount 1 ; sh:order 0 ; sh:path ttm:test_case_id ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 5 ; - sh:path schema1:additionalType ], + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 3 ; + sh:path schema1:name ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; sh:order 2 ; sh:path schema1:identifier ], - [ sh:class ttm:TestResultPKSet ; - sh:description "Primary keys for a given ARA result set from a SmokeTest result for a given TestCase." ; - sh:nodeKind sh:IRI ; - sh:order 1 ; - sh:path ttm:pks ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; sh:order 4 ; sh:path schema1:description ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 3 ; - sh:path schema1:name ] ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 6 ; + sh:path ttm:test_runner_settings ], + [ sh:class ttm:TestResultPKSet ; + sh:description "Primary keys for a given ARA result set from a SmokeTest result for a given TestCase." ; + sh:nodeKind sh:IRI ; + sh:order 1 ; + sh:path ttm:pks ] ; sh:targetClass ttm:TestOutput . ttm:TestRunSession a sh:NodeShape ; sh:closed true ; - sh:description "A single invocation of a TestRunner." ; + sh:description "Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite)." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:TestCaseResult ; - sh:description "One or more instances of TestCaseResult." ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_case_results ], - [ sh:datatype xsd:dateTime ; - sh:description "Date time when a given entity was created." ; - sh:maxCount 1 ; + sh:property [ sh:class ttm:TestEntityParameter ; + sh:description "Different TestRunners could expect additional global test configuration parameters, like the applicable TRAPI version (\"trapi_version\") or Biolink Model versions (\"biolink_version\")." ; + sh:nodeKind sh:BlankNode ; sh:order 3 ; - sh:path ttm:timestamp ], + sh:path ttm:test_run_parameters ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 8 ; + sh:path schema1:name ], [ sh:datatype xsd:string ; sh:description "Global system name of a TestRunner." ; sh:maxCount 1 ; - sh:order 0 ; + sh:order 2 ; sh:path ttm:test_runner_name ], + [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; + sh:in [ rdf:rest () ] ; + sh:order 0 ; + sh:path ttm:components ], [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 11 ; + sh:path ttm:test_runner_settings ], + [ sh:datatype xsd:dateTime ; + sh:description "Date time when a given entity was created." ; sh:maxCount 1 ; sh:order 6 ; - sh:path schema1:description ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 4 ; - sh:path schema1:identifier ], - [ sh:class ttm:TestEntity ; - sh:description "Different TestRunners may expect specific kinds of TestEntity as input. These 'test_entities' one or more instances of TestAsset, TestCase or TestSuite." ; - sh:nodeKind sh:IRI ; - sh:order 1 ; - sh:path ttm:test_entities ], + sh:path ttm:timestamp ], [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; + sh:order 10 ; sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + [ sh:description "Deployment environment within which the associated TestSuite is run." ; + sh:in ( "dev" "ci" "test" "prod" ) ; sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_env ], + [ sh:class ttm:TestCaseResult ; + sh:description "One or more instances of TestCaseResult." ; + sh:nodeKind sh:IRI ; sh:order 5 ; - sh:path schema1:name ] ; - sh:targetClass ttm:TestRunSession . - -ttm:TestRunnerConfiguration a sh:NodeShape ; - sh:closed true ; - sh:description "General configuration parameters and test data input for a single invocation of a TestRunner." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally identify the TestRunner(s) using this configuration." ; - sh:order 4 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 2 ; - sh:path schema1:name ], + sh:path ttm:test_case_results ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 1 ; + sh:order 7 ; sh:path schema1:identifier ], - [ sh:class ttm:TestEntityParameter ; - sh:description "Parameters for TestRunnerConfiguration that inform the TestHarness and TestRunners about the general characteristics of a test run." ; - sh:nodeKind sh:BlankNode ; - sh:order 0 ; - sh:path ttm:test_run_parameters ], + [ sh:class ttm:TestEntity ; + sh:description "Different TestRunners could expect specific kinds of TestEntity as an input. These 'test_entities' are one or more instances of TestAsset, TestCase or (preferably?) TestSuite." ; + sh:nodeKind sh:IRI ; + sh:order 4 ; + sh:path ttm:test_entities ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 3 ; + sh:order 9 ; sh:path schema1:description ] ; - sh:targetClass ttm:TestRunnerConfiguration . + sh:targetClass ttm:TestRunSession . ttm:TestSuite a sh:NodeShape ; sh:closed true ; sh:description "Specification of a set of Test Cases, one of either with a static list of 'test_cases' or a dynamic 'test_suite_specification' slot values. Note: at least one slot or the other, but generally not both(?) needs to be present." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 5 ; - sh:path schema1:name ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; sh:order 6 ; @@ -793,16 +517,6 @@ ttm:TestSuite a sh:NodeShape ; sh:minCount 1 ; sh:order 4 ; sh:path schema1:identifier ], - [ sh:description "A Test persona describes the user or operational context of a given test." ; - sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:test_persona ], - [ sh:class ttm:TestCase ; - sh:description "List of explicitly enumerated Test Cases." ; - sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_cases ], [ sh:class ttm:TestMetadata ; sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; sh:maxCount 1 ; @@ -810,171 +524,188 @@ ttm:TestSuite a sh:NodeShape ; sh:nodeKind sh:IRI ; sh:order 0 ; sh:path ttm:test_metadata ], + [ sh:class ttm:TestCase ; + sh:description "List of explicitly enumerated Test Cases." ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:test_cases ], + [ sh:description "A Test persona describes the user or operational context of a given test." ; + sh:in ( "All" "Clinical" "LookUp" "Mechanistic" ) ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_persona ], + [ sh:datatype xsd:string ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 8 ; + sh:path ttm:test_runner_settings ], [ sh:class ttm:TestSuiteSpecification ; sh:description "Declarative specification of a Test Suite of Test Cases whose generation is deferred, (i.e. within a Test Runner) or whose creation is achieved by stream processing of an external data source." ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; sh:order 3 ; - sh:path ttm:test_suite_specification ] ; + sh:path ttm:test_suite_specification ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 7 ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 5 ; + sh:path schema1:name ] ; sh:targetClass ttm:TestSuite . ttm:AcceptanceTestAsset a sh:NodeShape ; sh:closed true ; sh:description "Model derived from Jenn's test asset design and Shervin's runner JSON here as an example." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:class ttm:Qualifier ; - sh:description "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; - sh:nodeKind sh:BlankNode ; - sh:order 19 ; - sh:path ttm:qualifiers ], - [ sh:description "The deployment environment within which this test must pass." ; - sh:in ( "dev" "ci" "test" "prod" ) ; + sh:property [ sh:datatype xsd:integer ; + sh:description "The answer must return in these many results" ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:must_pass_environment ], - [ sh:description "The direction of the expected query result triple" ; - sh:in ( "increased" "decreased" ) ; + sh:order 7 ; + sh:path ttm:top_level ], + [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:direction ], - [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" "test_issue" ) ; + sh:order 16 ; + sh:path ttm:output_name ], + [ sh:description "The node of the (templated) TRAPI query to replace" ; + sh:in ( "subject" "object" ) ; sh:maxCount 1 ; - sh:order 21 ; - sh:path ttm:test_issue ], + sh:order 8 ; + sh:path ttm:query_node ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 11 ; - sh:path ttm:input_name ], + sh:order 14 ; + sh:path ttm:predicate_name ], [ sh:datatype xsd:string ; - sh:description "Settings for the test harness, e.g. \"inferred\"" ; - sh:minCount 1 ; - sh:order 26 ; - sh:path ttm:runner_settings ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 28 ; - sh:path schema1:identifier ], - [ sh:maxCount 1 ; - sh:order 17 ; - sh:path ttm:output_category ], - [ sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:input_id ], - [ sh:class ttm:TestMetadata ; - sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 27 ; - sh:path ttm:test_metadata ], - [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:order 23 ; - sh:path ttm:in_v1 ], - [ sh:maxCount 1 ; - sh:order 12 ; - sh:path ttm:input_category ], - [ sh:datatype xsd:integer ; - sh:description "The answer must return in these many results" ; + sh:order 20 ; + sh:path ttm:expected_output ], + [ sh:datatype xsd:string ; + sh:description "The full human-readable scientific question a SME would ask, which is encoded into the test asset." ; sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:top_level ], - [ sh:maxCount 1 ; - sh:order 15 ; - sh:path ttm:output_id ], + sh:order 2 ; + sh:path ttm:scientific_question ], [ sh:datatype xsd:string ; + sh:description "The object of the core triple to be tested" ; sh:maxCount 1 ; - sh:order 14 ; - sh:path ttm:predicate_name ], + sh:order 3 ; + sh:path ttm:string_entry ], [ sh:in ( "High" "Low" "NotApplicable" ) ; sh:maxCount 1 ; sh:order 22 ; sh:path ttm:semantic_severity ], + [ sh:maxCount 1 ; + sh:order 13 ; + sh:path ttm:predicate_id ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 28 ; + sh:path schema1:name ], + [ sh:description "Specific Biolink Model association 'category' which applies to the test asset defined knowledge statement" ; + sh:maxCount 1 ; + sh:order 18 ; + sh:path ttm:association ], [ sh:datatype xsd:string ; sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection" ; - sh:order 31 ; + sh:order 30 ; sh:path schema1:additionalType ], - [ sh:description "The expected result of the query" ; - sh:in ( "include_good" "exclude_bad" ) ; + [ sh:maxCount 1 ; + sh:order 12 ; + sh:path ttm:input_category ], + [ sh:maxCount 1 ; + sh:order 15 ; + sh:path ttm:output_id ], + [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:expected_result ], - [ sh:datatype xsd:string ; - sh:description "An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0" ; + sh:order 24 ; + sh:path ttm:well_known ], + [ sh:maxCount 1 ; + sh:order 17 ; + sh:path ttm:output_category ], + [ sh:description "The deployment environment within which this test must pass." ; + sh:in ( "dev" "ci" "test" "prod" ) ; sh:maxCount 1 ; - sh:order 5 ; - sh:path ttm:answer_informal_concept ], + sh:order 1 ; + sh:path ttm:must_pass_environment ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 30 ; + sh:order 29 ; sh:path schema1:description ], [ sh:datatype xsd:string ; + sh:description "Scalar settings for the TestRunner, e.g. \"inferred\"" ; + sh:order 31 ; + sh:path ttm:test_runner_settings ], + [ sh:description "The direction of the expected query result triple" ; + sh:in ( "increased" "decreased" ) ; sh:maxCount 1 ; - sh:order 20 ; - sh:path ttm:expected_output ], - [ sh:description "The node of the (templated) TRAPI query to replace" ; - sh:in ( "subject" "object" ) ; - sh:maxCount 1 ; - sh:order 8 ; - sh:path ttm:query_node ], - [ sh:datatype xsd:string ; - sh:description "The object of the core triple to be tested" ; + sh:order 4 ; + sh:path ttm:direction ], + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 3 ; - sh:path ttm:string_entry ], + sh:minCount 1 ; + sh:order 27 ; + sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "The full human-readable scientific question a SME would ask, which is encoded into the test asset." ; - sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:scientific_question ], - [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; + sh:description "An answer that is returned from the test case, note: this must be combined with the expected_result to form a complete answer. It might make sense to couple these in their own object instead of strictly sticking to the flat schema introduced by the spreadsheet here: https://docs.google.com/spreadsheets/d/1yj7zIchFeVl1OHqL_kE_pqvzNLmGml_FLbHDs-8Yvig/edit#gid=0" ; sh:maxCount 1 ; - sh:order 25 ; - sh:path ttm:test_reference ], + sh:order 5 ; + sh:path ttm:answer_informal_concept ], + [ sh:maxCount 1 ; + sh:order 10 ; + sh:path ttm:input_id ], [ sh:datatype xsd:date ; sh:description "The date by which this test must pass" ; sh:maxCount 1 ; sh:order 0 ; sh:path ttm:must_pass_date ], - [ sh:datatype xsd:string ; - sh:description "The notes of the query" ; + [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; sh:maxCount 1 ; - sh:order 9 ; - sh:path ttm:notes ], + sh:order 25 ; + sh:path ttm:test_reference ], [ sh:datatype xsd:boolean ; sh:maxCount 1 ; - sh:order 24 ; - sh:path ttm:well_known ], + sh:order 23 ; + sh:path ttm:in_v1 ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 29 ; - sh:path schema1:name ], - [ sh:datatype xsd:string ; + sh:order 11 ; + sh:path ttm:input_name ], + [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" "test_issue" ) ; sh:maxCount 1 ; - sh:order 16 ; - sh:path ttm:output_name ], - [ sh:description "Specific Biolink Model association 'category' which applies to the test asset defined knowledge statement" ; + sh:order 21 ; + sh:path ttm:test_issue ], + [ sh:class ttm:TestMetadata ; + sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; sh:maxCount 1 ; - sh:order 18 ; - sh:path ttm:association ], - [ sh:maxCount 1 ; - sh:order 13 ; - sh:path ttm:predicate_id ] ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 26 ; + sh:path ttm:test_metadata ], + [ sh:description "The expected result of the query" ; + sh:in ( "include_good" "exclude_bad" ) ; + sh:maxCount 1 ; + sh:order 6 ; + sh:path ttm:expected_result ], + [ sh:class ttm:Qualifier ; + sh:description "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; + sh:nodeKind sh:BlankNode ; + sh:order 19 ; + sh:path ttm:qualifiers ], + [ sh:datatype xsd:string ; + sh:description "The notes of the query" ; + sh:maxCount 1 ; + sh:order 9 ; + sh:path ttm:notes ] ; sh:targetClass ttm:AcceptanceTestAsset . ttm:TestCaseResult a sh:NodeShape ; sh:closed true ; sh:description "The outcome of a TestRunner run of one specific TestCase." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 4 ; - sh:path schema1:name ], - [ sh:description "Encoded result of a single test run of a given test case" ; + sh:property [ sh:description "Encoded result of a single test run of a given test case" ; sh:in ( "test_passed" "test_failed" "test_skipped" ) ; sh:maxCount 1 ; sh:order 2 ; @@ -988,69 +719,95 @@ ttm:TestCaseResult a sh:NodeShape ; sh:maxCount 1 ; sh:order 0 ; sh:path ttm:test_suite_id ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 6 ; - sh:path schema1:additionalType ], [ sh:class ttm:TestCase ; sh:description "Slot referencing a single TestCase." ; sh:maxCount 1 ; sh:nodeKind sh:IRI ; sh:order 1 ; sh:path ttm:test_case ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 4 ; + sh:path schema1:name ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; sh:order 3 ; - sh:path schema1:identifier ] ; + sh:path schema1:identifier ], + [ sh:datatype xsd:string ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 7 ; + sh:path ttm:test_runner_settings ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 6 ; + sh:path schema1:additionalType ] ; sh:targetClass ttm:TestCaseResult . ttm:TestEntity a sh:NodeShape ; sh:closed false ; sh:description "Abstract global 'identification' class shared as a parent with all major model classes within the data model for Translator testing." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:order 0 ; - sh:path schema1:identifier ], - [ sh:datatype xsd:string ; + sh:property [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 1 ; sh:path schema1:name ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 2 ; - sh:path schema1:description ], + sh:order 0 ; + sh:path schema1:identifier ], [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 3 ; - sh:path schema1:additionalType ] ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:string ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 4 ; + sh:path ttm:test_runner_settings ], + [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 2 ; + sh:path schema1:description ] ; sh:targetClass ttm:TestEntity . ttm:TestResultPKSet a sh:NodeShape ; sh:closed true ; sh:description "Primary keys for a given ARA result set from a SmokeTest result for a given TestCase." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; + sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 8 ; + sh:path schema1:name ], + [ sh:datatype xsd:string ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:merged_pk ], + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; sh:order 7 ; sh:path schema1:identifier ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:parent_pk ], + sh:order 5 ; + sh:path ttm:bte ], [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 10 ; - sh:path schema1:additionalType ], + sh:maxCount 1 ; + sh:order 4 ; + sh:path ttm:unsecret ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 11 ; + sh:path ttm:test_runner_settings ], + [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 8 ; - sh:path schema1:name ], + sh:order 9 ; + sh:path schema1:description ], [ sh:datatype xsd:string ; sh:maxCount 1 ; sh:order 2 ; @@ -1059,28 +816,115 @@ ttm:TestResultPKSet a sh:NodeShape ; sh:maxCount 1 ; sh:order 6 ; sh:path ttm:improving ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 10 ; + sh:path schema1:additionalType ], [ sh:datatype xsd:string ; sh:maxCount 1 ; sh:order 3 ; sh:path ttm:arax ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:merged_pk ], + sh:order 0 ; + sh:path ttm:parent_pk ] ; + sh:targetClass ttm:TestResultPKSet . + +ttm:TestAsset a sh:NodeShape ; + sh:closed true ; + sh:description "Represents a Test Asset, which is a single specific instance of TestCase-agnostic semantic parameters representing the specification of a Translator test target with inputs and (expected) outputs." ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:maxCount 1 ; + sh:order 3 ; + sh:path ttm:predicate_id ], + [ sh:maxCount 1 ; + sh:order 7 ; + sh:path ttm:output_category ], + [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection" ; + sh:order 20 ; + sh:path schema1:additionalType ], + [ sh:class ttm:Qualifier ; + sh:description "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; + sh:nodeKind sh:BlankNode ; + sh:order 9 ; + sh:path ttm:qualifiers ], + [ sh:datatype xsd:boolean ; + sh:maxCount 1 ; + sh:order 14 ; + sh:path ttm:well_known ], + [ sh:in ( "High" "Low" "NotApplicable" ) ; + sh:maxCount 1 ; + sh:order 12 ; + sh:path ttm:semantic_severity ], + [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" "test_issue" ) ; + sh:maxCount 1 ; + sh:order 11 ; + sh:path ttm:test_issue ], [ sh:datatype xsd:string ; + sh:description "Scalar settings for the TestRunner, e.g. \"inferred\"" ; + sh:order 21 ; + sh:path ttm:test_runner_settings ], + [ sh:datatype xsd:boolean ; + sh:maxCount 1 ; + sh:order 13 ; + sh:path ttm:in_v1 ], + [ sh:class ttm:TestMetadata ; + sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:nodeKind sh:IRI ; + sh:order 16 ; + sh:path ttm:test_metadata ], + [ sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:input_id ], + [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; sh:maxCount 1 ; + sh:order 15 ; + sh:path ttm:test_reference ], + [ sh:description "A unique identifier for a Test Entity" ; + sh:maxCount 1 ; + sh:minCount 1 ; + sh:order 17 ; + sh:path schema1:identifier ], + [ sh:maxCount 1 ; sh:order 5 ; - sh:path ttm:bte ], + sh:path ttm:output_id ], + [ sh:datatype xsd:string ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:input_name ], + [ sh:description "Specific Biolink Model association 'category' which applies to the test asset defined knowledge statement" ; + sh:maxCount 1 ; + sh:order 8 ; + sh:path ttm:association ], + [ sh:datatype xsd:string ; + sh:maxCount 1 ; + sh:order 4 ; + sh:path ttm:predicate_name ], [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 9 ; + sh:order 19 ; sh:path schema1:description ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:unsecret ] ; - sh:targetClass ttm:TestResultPKSet . + sh:order 6 ; + sh:path ttm:output_name ], + [ sh:maxCount 1 ; + sh:order 2 ; + sh:path ttm:input_category ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; + sh:maxCount 1 ; + sh:order 18 ; + sh:path schema1:name ], + [ sh:datatype xsd:string ; + sh:maxCount 1 ; + sh:order 10 ; + sh:path ttm:expected_output ] ; + sh:targetClass ttm:TestAsset . ttm:TestEntityParameter a sh:NodeShape ; sh:closed true ; @@ -1098,44 +942,58 @@ ttm:TestEntityParameter a sh:NodeShape ; sh:path ttm:parameter ] ; sh:targetClass ttm:TestEntityParameter . -ttm:Qualifier a sh:NodeShape ; +ttm:Precondition a sh:NodeShape ; sh:closed true ; + sh:description "Represents a precondition for a TestCase" ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; - sh:description "The 'value' of should be a suitable value generally drawn from an applicable Biolink Model (\"Enum\") value set of the specified Qualifier." ; + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 2 ; + sh:path schema1:description ], + [ sh:datatype xsd:string ; + sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; + sh:order 3 ; + sh:path schema1:additionalType ], + [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; sh:order 1 ; - sh:path ttm:value ], + sh:path schema1:name ], [ sh:datatype xsd:string ; - sh:description "The 'parameter' of a Qualifier should be a `qualifier` slot name from the Biolink Model ('biolink' namespace) 'biolink:qualifier' hierarchy." ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 4 ; + sh:path ttm:test_runner_settings ], + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; + sh:minCount 1 ; sh:order 0 ; - sh:path ttm:parameter ] ; - sh:targetClass ttm:Qualifier . + sh:path schema1:identifier ] ; + sh:targetClass ttm:Precondition . -ttm:TestSuiteSpecification a sh:NodeShape ; +ttm:Qualifier a sh:NodeShape ; sh:closed true ; - sh:description "Parameters for a Test Case instances either dynamically generated from some external source of Test Assets." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 2 ; - sh:path schema1:identifier ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + sh:property [ sh:datatype xsd:string ; + sh:description "The 'value' of should be a suitable value generally drawn from an applicable Biolink Model (\"Enum\") value set of the specified Qualifier." ; sh:maxCount 1 ; - sh:order 3 ; - sh:path schema1:name ], + sh:order 1 ; + sh:path ttm:value ], [ sh:datatype xsd:string ; + sh:description "The 'parameter' of a Qualifier should be a `qualifier` slot name from the Biolink Model ('biolink' namespace) 'biolink:qualifier' hierarchy." ; + sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:parameter ] ; + sh:targetClass ttm:Qualifier . + +ttm:TestSuiteSpecification a sh:NodeShape ; + sh:closed true ; + sh:description "Parameters for a Test Case instances either dynamically generated from some external source of Test Assets." ; + sh:ignoredProperties ( rdf:type ) ; + sh:property [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; sh:order 5 ; sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 4 ; - sh:path schema1:description ], [ sh:description "File format of test entity data (e.g. TSV, YAML or JSON)" ; sh:in ( "TSV" "YAML" "JSON" ) ; sh:maxCount 1 ; @@ -1144,252 +1002,143 @@ ttm:TestSuiteSpecification a sh:NodeShape ; [ sh:description "An web accessible file resource link to test entity data (e.g. a web accessible text file of Test Asset entries)" ; sh:maxCount 1 ; sh:order 0 ; - sh:path ttm:test_data_file_locator ] ; - sh:targetClass ttm:TestSuiteSpecification . - -ttm:TestAsset a sh:NodeShape ; - sh:closed true ; - sh:description "Represents a Test Asset, which is a single specific instance of TestCase-agnostic semantic parameters representing the specification of a Translator test target with inputs and (expected) outputs." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; - sh:maxCount 1 ; - sh:order 15 ; - sh:path ttm:test_reference ], - [ sh:maxCount 1 ; - sh:order 2 ; - sh:path ttm:input_category ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 20 ; - sh:path schema1:description ], - [ sh:in ( "High" "Low" "NotApplicable" ) ; - sh:maxCount 1 ; - sh:order 12 ; - sh:path ttm:semantic_severity ], + sh:path ttm:test_data_file_locator ], [ sh:datatype xsd:string ; + sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:predicate_name ], + sh:order 3 ; + sh:path schema1:name ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 18 ; + sh:order 2 ; sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; - sh:maxCount 1 ; - sh:order 19 ; - sh:path schema1:name ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:input_name ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; sh:order 6 ; - sh:path ttm:output_name ], - [ sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:output_category ], - [ sh:datatype xsd:string ; - sh:maxCount 1 ; - sh:order 10 ; - sh:path ttm:expected_output ], - [ sh:datatype xsd:boolean ; - sh:maxCount 1 ; - sh:order 13 ; - sh:path ttm:in_v1 ], - [ sh:datatype xsd:boolean ; - sh:maxCount 1 ; - sh:order 14 ; - sh:path ttm:well_known ], - [ sh:datatype xsd:string ; - sh:description "Settings for the test harness, e.g. \"inferred\"" ; - sh:minCount 1 ; - sh:order 16 ; - sh:path ttm:runner_settings ], - [ sh:description "Specific Biolink Model association 'category' which applies to the test asset defined knowledge statement" ; - sh:maxCount 1 ; - sh:order 8 ; - sh:path ttm:association ], - [ sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:input_id ], - [ sh:class ttm:Qualifier ; - sh:description "Optional qualifiers which constrain to the test asset defined knowledge statement. Note that this field records such qualifier slots and values as tag=value pairs, where the tag is the Biolink Model qualifier slot named and the value is an acceptable (Biolink Model enum?) value of the said qualifier slot." ; - sh:nodeKind sh:BlankNode ; - sh:order 9 ; - sh:path ttm:qualifiers ], - [ sh:class ttm:TestMetadata ; - sh:description "Test metadata describes the external provenance, cross-references and objectives for a given test." ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:nodeKind sh:IRI ; - sh:order 17 ; - sh:path ttm:test_metadata ], - [ sh:maxCount 1 ; - sh:order 3 ; - sh:path ttm:predicate_id ], - [ sh:maxCount 1 ; - sh:order 5 ; - sh:path ttm:output_id ], + sh:path ttm:test_runner_settings ], [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection" ; - sh:order 21 ; - sh:path schema1:additionalType ], - [ sh:in ( "causes not treats" "TMKP" "category too generic" "contraindications" "chemical roles" "test_issue" ) ; + sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 11 ; - sh:path ttm:test_issue ] ; - sh:targetClass ttm:TestAsset . + sh:order 4 ; + sh:path schema1:description ] ; + sh:targetClass ttm:TestSuiteSpecification . ttm:TestCase a sh:NodeShape ; sh:closed true ; sh:description "Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition." ; sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX." ; - sh:in [ rdf:rest () ] ; - sh:order 5 ; - sh:path ttm:components ], + sh:property [ sh:datatype xsd:string ; + sh:description "A human-readable description for a Test Entity" ; + sh:maxCount 1 ; + sh:order 11 ; + sh:path schema1:description ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 13 ; + sh:order 10 ; sh:path schema1:name ], + [ sh:description "Type of TestCase query." ; + sh:in ( "treats" ) ; + sh:maxCount 1 ; + sh:order 0 ; + sh:path ttm:query_type ], + [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; + sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" "OneHopTest" "StandardsValidationTest" ) ; + sh:maxCount 1 ; + sh:order 4 ; + sh:path ttm:test_case_objective ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 8 ; + sh:order 6 ; sh:path ttm:test_case_predicate_name ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; + [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; + sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; sh:maxCount 1 ; - sh:order 14 ; - sh:path schema1:description ], + sh:order 3 ; + sh:path ttm:trapi_template ], + [ sh:class ttm:Precondition ; + sh:nodeKind sh:IRI ; + sh:order 2 ; + sh:path ttm:preconditions ], [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; sh:minCount 1 ; - sh:order 12 ; + sh:order 9 ; sh:path schema1:identifier ], [ sh:datatype xsd:string ; - sh:description "Settings for the test harness for TestCase" ; - sh:minCount 1 ; - sh:order 11 ; - sh:path ttm:test_case_runner_settings ], - [ sh:datatype xsd:string ; - sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; - sh:order 15 ; - sh:path schema1:additionalType ], - [ sh:class ttm:Precondition ; - sh:nodeKind sh:IRI ; - sh:order 3 ; - sh:path ttm:preconditions ], - [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path ttm:test_case_objective ], + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 13 ; + sh:path ttm:test_runner_settings ], [ sh:maxCount 1 ; - sh:order 10 ; + sh:order 8 ; sh:path ttm:test_case_input_id ], - [ sh:description "Deployment environment within which the associated TestSuite is run." ; - sh:in ( "dev" "ci" "test" "prod" ) ; - sh:maxCount 1 ; - sh:order 0 ; - sh:path ttm:test_env ], - [ sh:description "A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory." ; - sh:in ( "ameliorates" "treats" "three_hop" "drug_treats_rare_disease" "drug-to-gene" ) ; - sh:maxCount 1 ; - sh:order 4 ; - sh:path ttm:trapi_template ], [ sh:datatype xsd:string ; sh:maxCount 1 ; - sh:order 9 ; + sh:order 7 ; sh:path ttm:test_case_predicate_id ], - [ sh:description "Type of TestCase query." ; - sh:in ( "treats" ) ; + [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; + sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:query_type ], + sh:order 5 ; + sh:path ttm:test_case_source ], + [ sh:datatype xsd:string ; + sh:description "One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection." ; + sh:order 12 ; + sh:path schema1:additionalType ], [ sh:class ttm:TestAsset ; sh:description "One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection." ; sh:minCount 1 ; sh:nodeKind sh:IRI ; - sh:order 2 ; - sh:path ttm:test_assets ], - [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; - sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; - sh:maxCount 1 ; - sh:order 7 ; - sh:path ttm:test_case_source ] ; + sh:order 1 ; + sh:path ttm:test_assets ] ; sh:targetClass ttm:TestCase . -ttm:Precondition a sh:NodeShape ; +ttm:TestMetadata a sh:NodeShape ; sh:closed true ; - sh:description "Represents a precondition for a TestCase" ; + sh:description "Represents metadata related to (external SME, SMURF, Translator feedback, large scale batch, etc.) like the provenance of test assets, cases and/or suites." ; sh:ignoredProperties ( rdf:type ) ; sh:property [ sh:datatype xsd:string ; sh:description "A human-readable description for a Test Entity" ; sh:maxCount 1 ; - sh:order 2 ; + sh:order 6 ; sh:path schema1:description ], + [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; + sh:maxCount 1 ; + sh:order 1 ; + sh:path ttm:test_reference ], + [ sh:class ttm:TestEntityParameter ; + sh:description "Metadata annotation." ; + sh:nodeKind sh:BlankNode ; + sh:order 3 ; + sh:path ttm:test_annotations ], [ sh:datatype xsd:string ; sh:description "A human-readable name for a Test Entity" ; sh:maxCount 1 ; - sh:order 1 ; + sh:order 5 ; sh:path schema1:name ], - [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 0 ; - sh:path schema1:identifier ], [ sh:datatype xsd:string ; sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 3 ; - sh:path schema1:additionalType ] ; - sh:targetClass ttm:Precondition . - -ttm:TestMetadata a sh:NodeShape ; - sh:closed true ; - sh:description "Represents metadata related to (external SME, SMURF, Translator feedback, large scale batch, etc.) like the provenance of test assets, cases and/or suites." ; - sh:ignoredProperties ( rdf:type ) ; - sh:property [ sh:description "A unique identifier for a Test Entity" ; - sh:maxCount 1 ; - sh:minCount 1 ; - sh:order 4 ; - sh:path schema1:identifier ], + sh:order 7 ; + sh:path schema1:additionalType ], [ sh:description "Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string." ; sh:in ( "SME" "SMURF" "GitHubUserFeedback" "TACT" "BenchMark" "TranslatorTeam" "TestDataLocation" ) ; sh:maxCount 1 ; sh:order 0 ; sh:path ttm:test_source ], - [ sh:description "Document URL where original test source particulars are registered (e.g. Github repo)" ; - sh:maxCount 1 ; - sh:order 1 ; - sh:path ttm:test_reference ], - [ sh:datatype xsd:string ; - sh:description "A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists." ; - sh:order 7 ; - sh:path schema1:additionalType ], - [ sh:datatype xsd:string ; - sh:description "A human-readable description for a Test Entity" ; - sh:maxCount 1 ; - sh:order 6 ; - sh:path schema1:description ], - [ sh:class ttm:TestEntityParameter ; - sh:description "Metadata annotation." ; - sh:nodeKind sh:BlankNode ; - sh:order 3 ; - sh:path ttm:test_annotations ], [ sh:description "Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)" ; - sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" ) ; + sh:in ( "AcceptanceTest" "BenchmarkTest" "QuantitativeTest" "OneHopTest" "StandardsValidationTest" ) ; sh:maxCount 1 ; sh:order 2 ; sh:path ttm:test_objective ], - [ sh:datatype xsd:string ; - sh:description "A human-readable name for a Test Entity" ; + [ sh:description "A unique identifier for a Test Entity" ; sh:maxCount 1 ; - sh:order 5 ; - sh:path schema1:name ] ; + sh:minCount 1 ; + sh:order 4 ; + sh:path schema1:identifier ], + [ sh:datatype xsd:string ; + sh:description "Scalar parameters for the TestRunner processing a given TestEntity." ; + sh:order 8 ; + sh:path ttm:test_runner_settings ] ; sh:targetClass ttm:TestMetadata . diff --git a/project/shex/translator_testing_model.shex b/project/shex/translator_testing_model.shex index b5f5b05..8a83059 100644 --- a/project/shex/translator_testing_model.shex +++ b/project/shex/translator_testing_model.shex @@ -92,34 +92,6 @@ linkml:Sparqlpath xsd:string ) } - CLOSED { - ( $ ( & ; - rdf:type [ ] ? ; - @linkml:String ? ; - @linkml:String ? - ) ; - rdf:type [ ] - ) -} - - ( - CLOSED { - ( $ ( & ; - rdf:type [ ] ? - ) ; - rdf:type [ ] - ) - } OR @ -) - - CLOSED { - ( $ ( & ; - rdf:type [ ] ? - ) ; - rdf:type [ ] - ) -} - CLOSED { ( $ ( & ; rdf:type [ ] ? @@ -182,9 +154,9 @@ linkml:Sparqlpath xsd:string @linkml:Boolean ? ; @linkml:Boolean ? ; @linkml:Uriorcurie ? ; - @linkml:String + ; @ ; - schema1:additionalType @linkml:String * + schema1:additionalType @linkml:String * ; + @linkml:String * ) ; rdf:type [ ] ) @@ -195,23 +167,20 @@ linkml:Sparqlpath xsd:string CLOSED { ( $ ( & ; rdf:type [ ] ? ; - @ ? ; @ ? ; @ + ; @ * ; @ ? ; - @ * ; @ ? ; @ ? ; @linkml:String ? ; @linkml:String ? ; @linkml:Uriorcurie ? ; - @linkml:String + ; schema1:additionalType @linkml:String * ) ; rdf:type [ ] ) - } OR @ OR @ OR @ OR @ + } OR @ OR @ ) CLOSED { @@ -235,13 +204,14 @@ linkml:Sparqlpath xsd:string ( @ OR @ OR @ OR @ OR @ OR @ OR @ - OR @ OR @ OR @ OR @ + OR @ OR @ OR @ ) { ( $ ( schema1:name @linkml:String ? ; schema1:description @linkml:String ? ; - schema1:additionalType @linkml:String * + schema1:additionalType @linkml:String * ; + @linkml:String * ) ; rdf:type [ ] ) @@ -294,20 +264,13 @@ linkml:Sparqlpath xsd:string ) } - CLOSED { - ( $ ( & ; - rdf:type [ ] ? ; - @ * ; - schema1:additionalType @linkml:String * - ) ; - rdf:type [ ] - ) -} - CLOSED { ( $ ( & ; rdf:type [ ] ? ; + @ * ; + @ ? ; @linkml:String ? ; + @ * ; @ * ; @ * ; @linkml:Datetime ? diff --git a/project/sqlschema/translator_testing_model.sql b/project/sqlschema/translator_testing_model.sql index 4c2b2ff..9da5d66 100644 --- a/project/sqlschema/translator_testing_model.sql +++ b/project/sqlschema/translator_testing_model.sql @@ -4,64 +4,10 @@ CREATE TABLE "AcceptanceTestCase" ( id TEXT NOT NULL, name TEXT, description TEXT, - test_env VARCHAR(4), - query_type VARCHAR(6), - preconditions TEXT, - trapi_template VARCHAR(24), - test_case_objective VARCHAR(16), - test_case_source VARCHAR(18), - test_case_predicate_name TEXT, - test_case_predicate_id TEXT, - test_case_input_id TEXT, - PRIMARY KEY (id) -); - -CREATE TABLE "ComplianceTestCase" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - test_env VARCHAR(4), - query_type VARCHAR(6), - test_assets TEXT NOT NULL, - preconditions TEXT, - trapi_template VARCHAR(24), - test_case_objective VARCHAR(16), - test_case_source VARCHAR(18), - test_case_predicate_name TEXT, - test_case_predicate_id TEXT, - test_case_input_id TEXT, - trapi_version TEXT, - biolink_version TEXT, - PRIMARY KEY (id) -); - -CREATE TABLE "KnowledgeGraphNavigationTestCase" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - test_env VARCHAR(4), - query_type VARCHAR(6), - test_assets TEXT NOT NULL, - preconditions TEXT, - trapi_template VARCHAR(24), - test_case_objective VARCHAR(16), - test_case_source VARCHAR(18), - test_case_predicate_name TEXT, - test_case_predicate_id TEXT, - test_case_input_id TEXT, - PRIMARY KEY (id) -); - -CREATE TABLE "OneHopTestCase" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - test_env VARCHAR(4), query_type VARCHAR(6), - test_assets TEXT NOT NULL, preconditions TEXT, trapi_template VARCHAR(24), - test_case_objective VARCHAR(16), + test_case_objective VARCHAR(23), test_case_source VARCHAR(18), test_case_predicate_name TEXT, test_case_predicate_id TEXT, @@ -86,12 +32,11 @@ CREATE TABLE "QuantitativeTestCase" ( id TEXT NOT NULL, name TEXT, description TEXT, - test_env VARCHAR(4), query_type VARCHAR(6), test_assets TEXT NOT NULL, preconditions TEXT, trapi_template VARCHAR(24), - test_case_objective VARCHAR(16), + test_case_objective VARCHAR(23), test_case_source VARCHAR(18), test_case_predicate_name TEXT, test_case_predicate_id TEXT, @@ -103,12 +48,11 @@ CREATE TABLE "TestCase" ( id TEXT NOT NULL, name TEXT, description TEXT, - test_env VARCHAR(4), query_type VARCHAR(6), test_assets TEXT NOT NULL, preconditions TEXT, trapi_template VARCHAR(24), - test_case_objective VARCHAR(16), + test_case_objective VARCHAR(23), test_case_source VARCHAR(18), test_case_predicate_name TEXT, test_case_predicate_id TEXT, @@ -128,7 +72,7 @@ CREATE TABLE "TestMetadata" ( description TEXT, test_source VARCHAR(18), test_reference TEXT, - test_objective VARCHAR(16), + test_objective VARCHAR(23), test_annotations TEXT, PRIMARY KEY (id) ); @@ -141,19 +85,13 @@ CREATE TABLE "TestOutput" ( PRIMARY KEY (id) ); -CREATE TABLE "TestRunnerConfiguration" ( - id TEXT NOT NULL, - name TEXT, - description TEXT, - test_run_parameters TEXT, - PRIMARY KEY (id) -); - CREATE TABLE "TestRunSession" ( id TEXT NOT NULL, name TEXT, description TEXT, + test_env VARCHAR(4), test_runner_name TEXT, + test_run_parameters TEXT, timestamp DATETIME, PRIMARY KEY (id) ); @@ -335,17 +273,10 @@ CREATE TABLE "TestSuite" ( FOREIGN KEY(test_suite_specification) REFERENCES "TestSuiteSpecification" (id) ); -CREATE TABLE "AcceptanceTestCase_components" ( +CREATE TABLE "AcceptanceTestCase_test_runner_settings" ( backref_id TEXT, - components VARCHAR, - PRIMARY KEY (backref_id, components), - FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestCase" (id) -); - -CREATE TABLE "AcceptanceTestCase_test_case_runner_settings" ( - backref_id TEXT, - test_case_runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, test_case_runner_settings), + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestCase" (id) ); @@ -356,69 +287,6 @@ CREATE TABLE "AcceptanceTestCase_tags" ( FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestCase" (id) ); -CREATE TABLE "ComplianceTestCase_components" ( - backref_id TEXT, - components VARCHAR, - PRIMARY KEY (backref_id, components), - FOREIGN KEY(backref_id) REFERENCES "ComplianceTestCase" (id) -); - -CREATE TABLE "ComplianceTestCase_test_case_runner_settings" ( - backref_id TEXT, - test_case_runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, test_case_runner_settings), - FOREIGN KEY(backref_id) REFERENCES "ComplianceTestCase" (id) -); - -CREATE TABLE "ComplianceTestCase_tags" ( - backref_id TEXT, - tags TEXT, - PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "ComplianceTestCase" (id) -); - -CREATE TABLE "KnowledgeGraphNavigationTestCase_components" ( - backref_id TEXT, - components VARCHAR, - PRIMARY KEY (backref_id, components), - FOREIGN KEY(backref_id) REFERENCES "KnowledgeGraphNavigationTestCase" (id) -); - -CREATE TABLE "KnowledgeGraphNavigationTestCase_test_case_runner_settings" ( - backref_id TEXT, - test_case_runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, test_case_runner_settings), - FOREIGN KEY(backref_id) REFERENCES "KnowledgeGraphNavigationTestCase" (id) -); - -CREATE TABLE "KnowledgeGraphNavigationTestCase_tags" ( - backref_id TEXT, - tags TEXT, - PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "KnowledgeGraphNavigationTestCase" (id) -); - -CREATE TABLE "OneHopTestCase_components" ( - backref_id TEXT, - components VARCHAR, - PRIMARY KEY (backref_id, components), - FOREIGN KEY(backref_id) REFERENCES "OneHopTestCase" (id) -); - -CREATE TABLE "OneHopTestCase_test_case_runner_settings" ( - backref_id TEXT, - test_case_runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, test_case_runner_settings), - FOREIGN KEY(backref_id) REFERENCES "OneHopTestCase" (id) -); - -CREATE TABLE "OneHopTestCase_tags" ( - backref_id TEXT, - tags TEXT, - PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "OneHopTestCase" (id) -); - CREATE TABLE "Precondition_tags" ( backref_id TEXT, tags TEXT, @@ -426,17 +294,17 @@ CREATE TABLE "Precondition_tags" ( FOREIGN KEY(backref_id) REFERENCES "Precondition" (id) ); -CREATE TABLE "QuantitativeTestCase_components" ( +CREATE TABLE "Precondition_test_runner_settings" ( backref_id TEXT, - components VARCHAR, - PRIMARY KEY (backref_id, components), - FOREIGN KEY(backref_id) REFERENCES "QuantitativeTestCase" (id) + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "Precondition" (id) ); -CREATE TABLE "QuantitativeTestCase_test_case_runner_settings" ( +CREATE TABLE "QuantitativeTestCase_test_runner_settings" ( backref_id TEXT, - test_case_runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, test_case_runner_settings), + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), FOREIGN KEY(backref_id) REFERENCES "QuantitativeTestCase" (id) ); @@ -447,17 +315,10 @@ CREATE TABLE "QuantitativeTestCase_tags" ( FOREIGN KEY(backref_id) REFERENCES "QuantitativeTestCase" (id) ); -CREATE TABLE "TestCase_components" ( - backref_id TEXT, - components VARCHAR, - PRIMARY KEY (backref_id, components), - FOREIGN KEY(backref_id) REFERENCES "TestCase" (id) -); - -CREATE TABLE "TestCase_test_case_runner_settings" ( +CREATE TABLE "TestCase_test_runner_settings" ( backref_id TEXT, - test_case_runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, test_case_runner_settings), + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), FOREIGN KEY(backref_id) REFERENCES "TestCase" (id) ); @@ -475,6 +336,13 @@ CREATE TABLE "TestMetadata_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestMetadata" (id) ); +CREATE TABLE "TestMetadata_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestMetadata" (id) +); + CREATE TABLE "TestOutput_tags" ( backref_id TEXT, tags TEXT, @@ -482,11 +350,11 @@ CREATE TABLE "TestOutput_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestOutput" (id) ); -CREATE TABLE "TestRunnerConfiguration_tags" ( +CREATE TABLE "TestOutput_test_runner_settings" ( backref_id TEXT, - tags TEXT, - PRIMARY KEY (backref_id, tags), - FOREIGN KEY(backref_id) REFERENCES "TestRunnerConfiguration" (id) + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestOutput" (id) ); CREATE TABLE "TestRunSession_tags" ( @@ -496,6 +364,20 @@ CREATE TABLE "TestRunSession_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestRunSession" (id) ); +CREATE TABLE "TestRunSession_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestRunSession" (id) +); + +CREATE TABLE "TestRunSession_components" ( + backref_id TEXT, + components VARCHAR, + PRIMARY KEY (backref_id, components), + FOREIGN KEY(backref_id) REFERENCES "TestRunSession" (id) +); + CREATE TABLE "TestRunSession_test_entities" ( backref_id TEXT, test_entities TEXT, @@ -510,11 +392,11 @@ CREATE TABLE "TestSuiteSpecification_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestSuiteSpecification" (id) ); -CREATE TABLE "AcceptanceTestAsset_runner_settings" ( +CREATE TABLE "TestSuiteSpecification_test_runner_settings" ( backref_id TEXT, - runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, runner_settings), - FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestAsset" (id) + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestSuiteSpecification" (id) ); CREATE TABLE "AcceptanceTestAsset_tags" ( @@ -524,6 +406,13 @@ CREATE TABLE "AcceptanceTestAsset_tags" ( FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestAsset" (id) ); +CREATE TABLE "AcceptanceTestAsset_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestAsset" (id) +); + CREATE TABLE "AcceptanceTestSuite_tags" ( backref_id TEXT, tags TEXT, @@ -531,6 +420,13 @@ CREATE TABLE "AcceptanceTestSuite_tags" ( FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestSuite" (id) ); +CREATE TABLE "AcceptanceTestSuite_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "AcceptanceTestSuite" (id) +); + CREATE TABLE "OneHopTestSuite_tags" ( backref_id TEXT, tags TEXT, @@ -538,6 +434,13 @@ CREATE TABLE "OneHopTestSuite_tags" ( FOREIGN KEY(backref_id) REFERENCES "OneHopTestSuite" (id) ); +CREATE TABLE "OneHopTestSuite_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "OneHopTestSuite" (id) +); + CREATE TABLE "StandardsComplianceTestSuite_tags" ( backref_id TEXT, tags TEXT, @@ -545,11 +448,11 @@ CREATE TABLE "StandardsComplianceTestSuite_tags" ( FOREIGN KEY(backref_id) REFERENCES "StandardsComplianceTestSuite" (id) ); -CREATE TABLE "TestAsset_runner_settings" ( +CREATE TABLE "StandardsComplianceTestSuite_test_runner_settings" ( backref_id TEXT, - runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, runner_settings), - FOREIGN KEY(backref_id) REFERENCES "TestAsset" (id) + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "StandardsComplianceTestSuite" (id) ); CREATE TABLE "TestAsset_tags" ( @@ -559,6 +462,13 @@ CREATE TABLE "TestAsset_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestAsset" (id) ); +CREATE TABLE "TestAsset_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestAsset" (id) +); + CREATE TABLE "TestCaseResult_tags" ( backref_id TEXT, tags TEXT, @@ -566,11 +476,11 @@ CREATE TABLE "TestCaseResult_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestCaseResult" (id) ); -CREATE TABLE "TestEdgeData_runner_settings" ( +CREATE TABLE "TestCaseResult_test_runner_settings" ( backref_id TEXT, - runner_settings TEXT NOT NULL, - PRIMARY KEY (backref_id, runner_settings), - FOREIGN KEY(backref_id) REFERENCES "TestEdgeData" (id) + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestCaseResult" (id) ); CREATE TABLE "TestEdgeData_tags" ( @@ -580,6 +490,13 @@ CREATE TABLE "TestEdgeData_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestEdgeData" (id) ); +CREATE TABLE "TestEdgeData_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestEdgeData" (id) +); + CREATE TABLE "TestResultPKSet_tags" ( backref_id TEXT, tags TEXT, @@ -587,9 +504,23 @@ CREATE TABLE "TestResultPKSet_tags" ( FOREIGN KEY(backref_id) REFERENCES "TestResultPKSet" (id) ); +CREATE TABLE "TestResultPKSet_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestResultPKSet" (id) +); + CREATE TABLE "TestSuite_tags" ( backref_id TEXT, tags TEXT, PRIMARY KEY (backref_id, tags), FOREIGN KEY(backref_id) REFERENCES "TestSuite" (id) ); + +CREATE TABLE "TestSuite_test_runner_settings" ( + backref_id TEXT, + test_runner_settings TEXT, + PRIMARY KEY (backref_id, test_runner_settings), + FOREIGN KEY(backref_id) REFERENCES "TestSuite" (id) +); diff --git a/src/data/examples/Bad-TestAsset-001.yaml b/src/data/examples/Bad-TestAsset-001.yaml index 451aa19..171adc6 100644 --- a/src/data/examples/Bad-TestAsset-001.yaml +++ b/src/data/examples/Bad-TestAsset-001.yaml @@ -10,6 +10,6 @@ input_id: PUBCHEM.COMPOUND:107970 predicate_name: treats output_id: MONDO:0005301 expected_output: Acceptable -runner_settings: +test_runner_settings: - "inferred" - "test" diff --git a/src/data/examples/Bad-TestAsset-002.yaml b/src/data/examples/Bad-TestAsset-002.yaml index ff7ef00..bb16005 100644 --- a/src/data/examples/Bad-TestAsset-002.yaml +++ b/src/data/examples/Bad-TestAsset-002.yaml @@ -11,6 +11,6 @@ description: Fingolimod treats multiple sclerosis predicate_name: treats output_id: MONDO:0005301 expected_output: Acceptable -runner_settings: +test_runner_settings: - "inferred" - "test" diff --git a/src/data/examples/Bad-TestAsset-003.yaml b/src/data/examples/Bad-TestAsset-003.yaml index 7a327f6..5a8f6dc 100644 --- a/src/data/examples/Bad-TestAsset-003.yaml +++ b/src/data/examples/Bad-TestAsset-003.yaml @@ -11,6 +11,6 @@ input_id: PUBCHEM.COMPOUND:107970 #predicate_name: treats output_id: MONDO:0005301 expected_output: Acceptable -runner_settings: +test_runner_settings: - "inferred" - "test" diff --git a/src/data/examples/Bad-TestAsset-004.yaml b/src/data/examples/Bad-TestAsset-004.yaml index 5f40e3a..7c1a41a 100644 --- a/src/data/examples/Bad-TestAsset-004.yaml +++ b/src/data/examples/Bad-TestAsset-004.yaml @@ -11,6 +11,6 @@ input_id: PUBCHEM.COMPOUND:107970 predicate_name: treats #output_id: MONDO:0005301 expected_output: Acceptable -runner_settings: +test_runner_settings: - "inferred" - "test" diff --git a/src/data/examples/Bad-TestAsset-005.yaml b/src/data/examples/Bad-TestAsset-005.yaml index bf1d611..a0d29b7 100644 --- a/src/data/examples/Bad-TestAsset-005.yaml +++ b/src/data/examples/Bad-TestAsset-005.yaml @@ -11,6 +11,6 @@ input_id: PUBCHEM.COMPOUND:107970 predicate_name: treats #output_id: MONDO:0005301 expected_output: Acceptable -runner_settings: +test_runner_settings: - "inferred" - "test" diff --git a/src/data/examples/Bad-TestAsset-006.yaml b/src/data/examples/Bad-TestAsset-006.yaml index 87416cf..64aa600 100644 --- a/src/data/examples/Bad-TestAsset-006.yaml +++ b/src/data/examples/Bad-TestAsset-006.yaml @@ -11,6 +11,6 @@ input_id: PUBCHEM.COMPOUND:107970 predicate_name: treats output_id: MONDO:0005301 expected_output: Acceptable -#runner_settings: +#test_runner_settings: # - "inferred" # - "test" diff --git a/src/data/examples/Bad-TestEntity-001.yaml b/src/data/examples/Bad-TestEntity-001.yaml index 9676668..dbe6245 100644 --- a/src/data/examples/Bad-TestEntity-001.yaml +++ b/src/data/examples/Bad-TestEntity-001.yaml @@ -11,6 +11,8 @@ # - id # - name # - description +# - tags +# - test_runner_settings # # where the slots are: # diff --git a/src/data/examples/Bad-TestEntity-002.yaml-uriorcurie_should_fail_but_does_not b/src/data/examples/Bad-TestEntity-002.yaml-uriorcurie_should_fail_but_does_not index f9357e2..3f8643b 100644 --- a/src/data/examples/Bad-TestEntity-002.yaml-uriorcurie_should_fail_but_does_not +++ b/src/data/examples/Bad-TestEntity-002.yaml-uriorcurie_should_fail_but_does_not @@ -11,6 +11,8 @@ # - id # - name # - description +# - tags +# - test_runner_settings # # where the slots are: # diff --git a/src/data/examples/Good-AcceptanceTestAsset-001.yaml b/src/data/examples/Good-AcceptanceTestAsset-001.yaml index 2dcfc1a..2a48722 100644 --- a/src/data/examples/Good-AcceptanceTestAsset-001.yaml +++ b/src/data/examples/Good-AcceptanceTestAsset-001.yaml @@ -31,7 +31,7 @@ expected_output: NeverShow test_metadata: id: "test:1" test_source: SMURF -runner_settings: +test_runner_settings: - "inferred" - "test" diff --git a/src/data/examples/Good-AcceptanceTestCase-001.yaml b/src/data/examples/Good-AcceptanceTestCase-001.yaml index 6bd3daf..ca6f646 100644 --- a/src/data/examples/Good-AcceptanceTestCase-001.yaml +++ b/src/data/examples/Good-AcceptanceTestCase-001.yaml @@ -13,7 +13,7 @@ id: example:AcceptanceTestCase001 name: foo bar description: This is an acceptance test case -test_case_runner_settings: +test_runner_settings: - "inferred" - "test" test_assets: @@ -24,7 +24,7 @@ test_assets: predicate_name: treats output_id: DRUGBANK:DB01592 expected_output: NeverShow - runner_settings: + test_runner_settings: - "inferred" - "test" test_metadata: diff --git a/src/data/examples/Good-TestAsset-001.json b/src/data/examples/Good-TestAsset-001.json index 5349a8f..f773550 100644 --- a/src/data/examples/Good-TestAsset-001.json +++ b/src/data/examples/Good-TestAsset-001.json @@ -6,7 +6,7 @@ "predicate_name": "treats", "output_id": "MONDO:0005301", "expected_output": "Acceptable", - "runner_settings": ["inferred", "test"], + "test_runner_settings": ["inferred", "test"], "test_metadata": { "id": "example:TestAsset001" } diff --git a/src/data/examples/Good-TestAsset-001.yaml b/src/data/examples/Good-TestAsset-001.yaml index 2d27cba..31fd44d 100644 --- a/src/data/examples/Good-TestAsset-001.yaml +++ b/src/data/examples/Good-TestAsset-001.yaml @@ -27,7 +27,6 @@ # - in_v1 # - well_known # - test_reference -# - runner_settings # slot_usage: # id: # aliases: ["AssetIdentifier"] @@ -36,7 +35,7 @@ # description: >- # One or more 'tags' slot values (inherited from TestEntity) should generally # be defined to specify TestAsset membership in a "Block List" collection -# runner_settings: +# test_runner_settings: # description: >- # Settings for the test harness, e.g. "inferred" # @@ -55,6 +54,6 @@ in_v1: True well_known: False test_metadata: id: "example:TestAsset001" -runner_settings: +test_runner_settings: - "inferred" - "test" \ No newline at end of file diff --git a/src/data/examples/Good-TestCase-001.json b/src/data/examples/Good-TestCase-001.json index cf3a57b..d4763fe 100644 --- a/src/data/examples/Good-TestCase-001.json +++ b/src/data/examples/Good-TestCase-001.json @@ -11,11 +11,11 @@ "predicate_name": "treats", "output_id": "MONDO:0005301", "expected_output": "Acceptable", - "runner_settings": ["inferred", "test"], + "test_runner_settings": ["inferred", "test"], "test_metadata": { "id": "example:TestMetadata001" } } ], - "test_case_runner_settings": ["inferred", "test"] + "test_runner_settings": ["inferred", "test"] } diff --git a/src/data/examples/Good-TestCase-001.yaml b/src/data/examples/Good-TestCase-001.yaml index 1fbdf65..2af86e1 100644 --- a/src/data/examples/Good-TestCase-001.yaml +++ b/src/data/examples/Good-TestCase-001.yaml @@ -43,12 +43,12 @@ test_assets: predicate_name: treats output_id: DRUGBANK:DB01592 expected_output: NeverShow - runner_settings: + test_runner_settings: - "inferred" - "test" test_metadata: id: test:1 test_source: SMURF -test_case_runner_settings: +test_runner_settings: - "inferred" - "test" \ No newline at end of file diff --git a/src/data/examples/Good-TestEntity-Complete.yaml b/src/data/examples/Good-TestEntity-Complete.yaml index ad412a8..2671e65 100644 --- a/src/data/examples/Good-TestEntity-Complete.yaml +++ b/src/data/examples/Good-TestEntity-Complete.yaml @@ -11,6 +11,8 @@ # - id # - name # - description +# - tags +# - test_runner_settings # # where the slots are: # diff --git a/src/translator_testing_model/datamodel/pydanticmodel.py b/src/translator_testing_model/datamodel/pydanticmodel.py index 9c11ed2..5a3cfdf 100644 --- a/src/translator_testing_model/datamodel/pydanticmodel.py +++ b/src/translator_testing_model/datamodel/pydanticmodel.py @@ -53,6 +53,10 @@ class TestObjectiveEnum(str, Enum): BenchmarkTest = "BenchmarkTest" # Quantitative test QuantitativeTest = "QuantitativeTest" + # One Hop Test of knowledge graph navigation + OneHopTest = "OneHopTest" + # TRAPI and Biolink Model ("reasoner-validator") validation + StandardsValidationTest = "StandardsValidationTest" @@ -242,6 +246,7 @@ class TestEntity(ConfiguredBaseModel): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestMetadata(TestEntity): @@ -256,6 +261,7 @@ class TestMetadata(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestAsset(TestEntity): @@ -278,12 +284,12 @@ class TestAsset(TestEntity): in_v1: Optional[bool] = Field(None) well_known: Optional[bool] = Field(None) test_reference: Optional[str] = Field(None, description="""Document URL where original test source particulars are registered (e.g. Github repo)""") - runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness, e.g. \"inferred\"""") test_metadata: TestMetadata = Field(..., description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar settings for the TestRunner, e.g. \"inferred\"""") class AcceptanceTestAsset(TestAsset): @@ -316,12 +322,12 @@ class AcceptanceTestAsset(TestAsset): in_v1: Optional[bool] = Field(None) well_known: Optional[bool] = Field(None) test_reference: Optional[str] = Field(None, description="""Document URL where original test source particulars are registered (e.g. Github repo)""") - runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness, e.g. \"inferred\"""") test_metadata: TestMetadata = Field(..., description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar settings for the TestRunner, e.g. \"inferred\"""") class TestEdgeData(TestAsset): @@ -344,12 +350,12 @@ class TestEdgeData(TestAsset): in_v1: Optional[bool] = Field(None) well_known: Optional[bool] = Field(None) test_reference: Optional[str] = Field(None, description="""Document URL where original test source particulars are registered (e.g. Github repo)""") - runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness, e.g. \"inferred\"""") test_metadata: TestMetadata = Field(..., description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar settings for the TestRunner, e.g. \"inferred\"""") class Precondition(TestEntity): @@ -360,140 +366,67 @@ class Precondition(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestCase(TestEntity): """ Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition. """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") test_case_predicate_name: Optional[str] = Field(None) test_case_predicate_id: Optional[str] = Field(None) test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class AcceptanceTestCase(TestCase): """ See AcceptanceTestAsset above for more details. """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") test_assets: List[AcceptanceTestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") test_case_predicate_name: Optional[str] = Field(None) test_case_predicate_id: Optional[str] = Field(None) test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class QuantitativeTestCase(TestCase): """ Assumed additional model from Shervin's runner JSON here as an example. This schema is not yet complete. """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") - query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") - test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") - preconditions: Optional[List[str]] = Field(default_factory=list) - trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") - test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") - test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") - test_case_predicate_name: Optional[str] = Field(None) - test_case_predicate_id: Optional[str] = Field(None) - test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") - - -class ComplianceTestCase(TestCase): - """ - TRAPI and Biolink Model standards compliance test - """ - trapi_version: Optional[str] = Field(None, description="""TRAPI version (SemVer string)""") - biolink_version: Optional[str] = Field(None, description="""Biolink Model release (SemVer string)""") - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") - query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") - test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") - preconditions: Optional[List[str]] = Field(default_factory=list) - trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") - test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") - test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") - test_case_predicate_name: Optional[str] = Field(None) - test_case_predicate_id: Optional[str] = Field(None) - test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") - - -class KnowledgeGraphNavigationTestCase(TestCase): - """ - Knowledge Graph navigation integration test - """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") - query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") - test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") - preconditions: Optional[List[str]] = Field(default_factory=list) - trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") - test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") - test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") - test_case_predicate_name: Optional[str] = Field(None) - test_case_predicate_id: Optional[str] = Field(None) - test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") - - -class OneHopTestCase(KnowledgeGraphNavigationTestCase): - """ - 'One Hop' Knowledge Graph navigation integration test - """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") test_case_predicate_name: Optional[str] = Field(None) test_case_predicate_id: Optional[str] = Field(None) test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestSuiteSpecification(TestEntity): @@ -506,6 +439,7 @@ class TestSuiteSpecification(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestSuite(TestEntity): @@ -520,6 +454,7 @@ class TestSuite(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class AcceptanceTestSuite(TestSuite): @@ -532,6 +467,7 @@ class AcceptanceTestSuite(TestSuite): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class BenchmarkTestSuite(ConfiguredBaseModel): @@ -553,6 +489,7 @@ class StandardsComplianceTestSuite(TestSuite): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class OneHopTestSuite(TestSuite): @@ -567,17 +504,7 @@ class OneHopTestSuite(TestSuite): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") - - -class TestRunnerConfiguration(TestEntity): - """ - General configuration parameters and test data input for a single invocation of a TestRunner. - """ - test_run_parameters: Optional[List[TestEntityParameter]] = Field(default_factory=list, description="""Parameters for TestRunnerConfiguration that inform the TestHarness and TestRunners about the general characteristics of a test run.""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally identify the TestRunner(s) using this configuration.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestCaseResult(TestEntity): @@ -591,20 +518,25 @@ class TestCaseResult(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestRunSession(TestEntity): """ - A single invocation of a TestRunner. + Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite). """ + components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") test_runner_name: Optional[str] = Field(None, description="""Global system name of a TestRunner.""") - test_entities: Optional[Dict[str, TestEntity]] = Field(default_factory=dict, description="""Different TestRunners may expect specific kinds of TestEntity as input. These 'test_entities' one or more instances of TestAsset, TestCase or TestSuite.""") + test_run_parameters: Optional[List[TestEntityParameter]] = Field(default_factory=list, description="""Different TestRunners could expect additional global test configuration parameters, like the applicable TRAPI version (\"trapi_version\") or Biolink Model versions (\"biolink_version\").""") + test_entities: Optional[Dict[str, TestEntity]] = Field(default_factory=dict, description="""Different TestRunners could expect specific kinds of TestEntity as an input. These 'test_entities' are one or more instances of TestAsset, TestCase or (preferably?) TestSuite.""") test_case_results: Optional[Dict[str, TestCaseResult]] = Field(default_factory=dict, description="""One or more instances of TestCaseResult.""") timestamp: Optional[datetime ] = Field(None, description="""Date time when a given entity was created.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestOutput(TestEntity): @@ -617,6 +549,7 @@ class TestOutput(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestResultPKSet(TestEntity): @@ -634,6 +567,7 @@ class TestResultPKSet(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") @@ -650,16 +584,12 @@ class TestResultPKSet(TestEntity): TestCase.update_forward_refs() AcceptanceTestCase.update_forward_refs() QuantitativeTestCase.update_forward_refs() -ComplianceTestCase.update_forward_refs() -KnowledgeGraphNavigationTestCase.update_forward_refs() -OneHopTestCase.update_forward_refs() TestSuiteSpecification.update_forward_refs() TestSuite.update_forward_refs() AcceptanceTestSuite.update_forward_refs() BenchmarkTestSuite.update_forward_refs() StandardsComplianceTestSuite.update_forward_refs() OneHopTestSuite.update_forward_refs() -TestRunnerConfiguration.update_forward_refs() TestCaseResult.update_forward_refs() TestRunSession.update_forward_refs() TestOutput.update_forward_refs() diff --git a/src/translator_testing_model/datamodel/pydanticmodel_v2.py b/src/translator_testing_model/datamodel/pydanticmodel_v2.py index 5cdd728..0997cfb 100644 --- a/src/translator_testing_model/datamodel/pydanticmodel_v2.py +++ b/src/translator_testing_model/datamodel/pydanticmodel_v2.py @@ -49,6 +49,10 @@ class TestObjectiveEnum(str, Enum): BenchmarkTest = "BenchmarkTest" # Quantitative test QuantitativeTest = "QuantitativeTest" + # One Hop Test of knowledge graph navigation + OneHopTest = "OneHopTest" + # TRAPI and Biolink Model ("reasoner-validator") validation + StandardsValidationTest = "StandardsValidationTest" @@ -238,6 +242,7 @@ class TestEntity(ConfiguredBaseModel): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestMetadata(TestEntity): @@ -252,6 +257,7 @@ class TestMetadata(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestAsset(TestEntity): @@ -274,12 +280,12 @@ class TestAsset(TestEntity): in_v1: Optional[bool] = Field(None) well_known: Optional[bool] = Field(None) test_reference: Optional[str] = Field(None, description="""Document URL where original test source particulars are registered (e.g. Github repo)""") - runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness, e.g. \"inferred\"""") test_metadata: TestMetadata = Field(..., description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar settings for the TestRunner, e.g. \"inferred\"""") class AcceptanceTestAsset(TestAsset): @@ -312,12 +318,12 @@ class AcceptanceTestAsset(TestAsset): in_v1: Optional[bool] = Field(None) well_known: Optional[bool] = Field(None) test_reference: Optional[str] = Field(None, description="""Document URL where original test source particulars are registered (e.g. Github repo)""") - runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness, e.g. \"inferred\"""") test_metadata: TestMetadata = Field(..., description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar settings for the TestRunner, e.g. \"inferred\"""") class TestEdgeData(TestAsset): @@ -340,12 +346,12 @@ class TestEdgeData(TestAsset): in_v1: Optional[bool] = Field(None) well_known: Optional[bool] = Field(None) test_reference: Optional[str] = Field(None, description="""Document URL where original test source particulars are registered (e.g. Github repo)""") - runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness, e.g. \"inferred\"""") test_metadata: TestMetadata = Field(..., description="""Test metadata describes the external provenance, cross-references and objectives for a given test.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a \"Block List\" collection""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar settings for the TestRunner, e.g. \"inferred\"""") class Precondition(TestEntity): @@ -356,140 +362,67 @@ class Precondition(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestCase(TestEntity): """ Represents a single enumerated instance of Test Case, derived from a given collection of one or more TestAsset instances (the value of the 'test_assets' slot) which define the 'inputs' and 'outputs' of the TestCase, used to probe a particular test condition. """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") test_case_predicate_name: Optional[str] = Field(None) test_case_predicate_id: Optional[str] = Field(None) test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class AcceptanceTestCase(TestCase): """ See AcceptanceTestAsset above for more details. """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") test_assets: List[AcceptanceTestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") test_case_predicate_name: Optional[str] = Field(None) test_case_predicate_id: Optional[str] = Field(None) test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class QuantitativeTestCase(TestCase): """ Assumed additional model from Shervin's runner JSON here as an example. This schema is not yet complete. """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") - query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") - test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") - preconditions: Optional[List[str]] = Field(default_factory=list) - trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") - test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") - test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") - test_case_predicate_name: Optional[str] = Field(None) - test_case_predicate_id: Optional[str] = Field(None) - test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") - - -class ComplianceTestCase(TestCase): - """ - TRAPI and Biolink Model standards compliance test - """ - trapi_version: Optional[str] = Field(None, description="""TRAPI version (SemVer string)""") - biolink_version: Optional[str] = Field(None, description="""Biolink Model release (SemVer string)""") - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") - query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") - test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") - preconditions: Optional[List[str]] = Field(default_factory=list) - trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") - test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") - test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") - test_case_predicate_name: Optional[str] = Field(None) - test_case_predicate_id: Optional[str] = Field(None) - test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") - - -class KnowledgeGraphNavigationTestCase(TestCase): - """ - Knowledge Graph navigation integration test - """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") - query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") - test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") - preconditions: Optional[List[str]] = Field(default_factory=list) - trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") - test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") - test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") - test_case_predicate_name: Optional[str] = Field(None) - test_case_predicate_id: Optional[str] = Field(None) - test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") - - -class OneHopTestCase(KnowledgeGraphNavigationTestCase): - """ - 'One Hop' Knowledge Graph navigation integration test - """ - test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") query_type: Optional[QueryTypeEnum] = Field(None, description="""Type of TestCase query.""") test_assets: List[TestAsset] = Field(default_factory=list, description="""One or more 'tags' slot values (inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in 'test_assets' slot (\"Block List\") collection.""") preconditions: Optional[List[str]] = Field(default_factory=list) trapi_template: Optional[TrapiTemplateEnum] = Field(None, description="""A template for a query, which can be used to generate a query for a test case. note: the current enumerated values for this slot come from the Benchmarks repo config/benchmarks.json \"templates\" collection and refer to the \"name\" field of each template. Templates themselves are currently stored in the config/[source_name]/templates directory.""") - components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") test_case_objective: Optional[TestObjectiveEnum] = Field(None, description="""Testing objective behind specified set of test particulars (e.g. acceptance pass/fail; benchmark; quantitative)""") test_case_source: Optional[TestSourceEnum] = Field(None, description="""Provenance of a specific set of test assets, cases and/or suites. Or, the person who cares about this, know about this. We would like this to be an ORCID eventually, but currently it is just a string.""") test_case_predicate_name: Optional[str] = Field(None) test_case_predicate_id: Optional[str] = Field(None) test_case_input_id: Optional[str] = Field(None) - test_case_runner_settings: List[str] = Field(default_factory=list, description="""Settings for the test harness for TestCase""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally be defined as filters to specify TestAsset membership in a \"Block List\" collection.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestSuiteSpecification(TestEntity): @@ -502,6 +435,7 @@ class TestSuiteSpecification(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestSuite(TestEntity): @@ -516,6 +450,7 @@ class TestSuite(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class AcceptanceTestSuite(TestSuite): @@ -528,6 +463,7 @@ class AcceptanceTestSuite(TestSuite): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class BenchmarkTestSuite(ConfiguredBaseModel): @@ -549,6 +485,7 @@ class StandardsComplianceTestSuite(TestSuite): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class OneHopTestSuite(TestSuite): @@ -563,17 +500,7 @@ class OneHopTestSuite(TestSuite): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") - - -class TestRunnerConfiguration(TestEntity): - """ - General configuration parameters and test data input for a single invocation of a TestRunner. - """ - test_run_parameters: Optional[List[TestEntityParameter]] = Field(default_factory=list, description="""Parameters for TestRunnerConfiguration that inform the TestHarness and TestRunners about the general characteristics of a test run.""") - id: str = Field(..., description="""A unique identifier for a Test Entity""") - name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") - description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") - tags: Optional[List[str]] = Field(default_factory=list, description="""One or more 'tags' slot values (slot inherited from TestEntity) should generally identify the TestRunner(s) using this configuration.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestCaseResult(TestEntity): @@ -587,20 +514,25 @@ class TestCaseResult(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestRunSession(TestEntity): """ - A single invocation of a TestRunner. + Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more instances of TestSuite). """ + components: Optional[List[ComponentEnum]] = Field(default_factory=list, description="""The component that this test case is intended to run against. Most often this is the ARS for acceptance tests, but for the Benchmarks repo integration, this can also be individual components of the system like Aragorn, or ARAX.""") + test_env: Optional[TestEnvEnum] = Field(None, description="""Deployment environment within which the associated TestSuite is run.""") test_runner_name: Optional[str] = Field(None, description="""Global system name of a TestRunner.""") - test_entities: Optional[Dict[str, TestEntity]] = Field(default_factory=dict, description="""Different TestRunners may expect specific kinds of TestEntity as input. These 'test_entities' one or more instances of TestAsset, TestCase or TestSuite.""") + test_run_parameters: Optional[List[TestEntityParameter]] = Field(default_factory=list, description="""Different TestRunners could expect additional global test configuration parameters, like the applicable TRAPI version (\"trapi_version\") or Biolink Model versions (\"biolink_version\").""") + test_entities: Optional[Dict[str, TestEntity]] = Field(default_factory=dict, description="""Different TestRunners could expect specific kinds of TestEntity as an input. These 'test_entities' are one or more instances of TestAsset, TestCase or (preferably?) TestSuite.""") test_case_results: Optional[Dict[str, TestCaseResult]] = Field(default_factory=dict, description="""One or more instances of TestCaseResult.""") timestamp: Optional[datetime ] = Field(None, description="""Date time when a given entity was created.""") id: str = Field(..., description="""A unique identifier for a Test Entity""") name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestOutput(TestEntity): @@ -613,6 +545,7 @@ class TestOutput(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") class TestResultPKSet(TestEntity): @@ -630,6 +563,7 @@ class TestResultPKSet(TestEntity): name: Optional[str] = Field(None, description="""A human-readable name for a Test Entity""") description: Optional[str] = Field(None, description="""A human-readable description for a Test Entity""") tags: Optional[List[str]] = Field(default_factory=list, description="""A human-readable tags for categorical memberships of a TestEntity (preferably a URI or CURIE). Typically used to aggregate instances of TestEntity into formally typed or ad hoc lists.""") + test_runner_settings: Optional[List[str]] = Field(default_factory=list, description="""Scalar parameters for the TestRunner processing a given TestEntity.""") @@ -646,16 +580,12 @@ class TestResultPKSet(TestEntity): TestCase.model_rebuild() AcceptanceTestCase.model_rebuild() QuantitativeTestCase.model_rebuild() -ComplianceTestCase.model_rebuild() -KnowledgeGraphNavigationTestCase.model_rebuild() -OneHopTestCase.model_rebuild() TestSuiteSpecification.model_rebuild() TestSuite.model_rebuild() AcceptanceTestSuite.model_rebuild() BenchmarkTestSuite.model_rebuild() StandardsComplianceTestSuite.model_rebuild() OneHopTestSuite.model_rebuild() -TestRunnerConfiguration.model_rebuild() TestCaseResult.model_rebuild() TestRunSession.model_rebuild() TestOutput.model_rebuild() diff --git a/src/translator_testing_model/datamodel/translator_testing_model.py b/src/translator_testing_model/datamodel/translator_testing_model.py index 65a0966..f4498ba 100644 --- a/src/translator_testing_model/datamodel/translator_testing_model.py +++ b/src/translator_testing_model/datamodel/translator_testing_model.py @@ -1,5 +1,5 @@ # Auto generated from translator_testing_model.yaml by pythongen.py version: 0.0.1 -# Generation date: 2024-03-18T12:26:26 +# Generation date: 2024-03-21T14:12:58 # Schema: Translator-Testing-Model # # id: https://w3id.org/TranslatorSRI/TranslatorTestingModel @@ -111,18 +111,6 @@ class QuantitativeTestCaseId(TestCaseId): pass -class ComplianceTestCaseId(TestCaseId): - pass - - -class KnowledgeGraphNavigationTestCaseId(TestCaseId): - pass - - -class OneHopTestCaseId(KnowledgeGraphNavigationTestCaseId): - pass - - class TestSuiteSpecificationId(TestEntityId): pass @@ -143,10 +131,6 @@ class OneHopTestSuiteId(TestSuiteId): pass -class TestRunnerConfigurationId(TestEntityId): - pass - - class TestCaseResultId(TestEntityId): pass @@ -227,6 +211,7 @@ class TestEntity(YAMLRoot): name: Optional[str] = None description: Optional[str] = None tags: Optional[Union[str, List[str]]] = empty_list() + test_runner_settings: Optional[Union[str, List[str]]] = empty_list() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): @@ -244,6 +229,10 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): self.tags = [self.tags] if self.tags is not None else [] self.tags = [v if isinstance(v, str) else str(v) for v in self.tags] + if not isinstance(self.test_runner_settings, list): + self.test_runner_settings = [self.test_runner_settings] if self.test_runner_settings is not None else [] + self.test_runner_settings = [v if isinstance(v, str) else str(v) for v in self.test_runner_settings] + super().__post_init__(**kwargs) @@ -302,7 +291,6 @@ class TestAsset(TestEntity): class_model_uri: ClassVar[URIRef] = TTM.TestAsset id: Union[str, TestAssetId] = None - runner_settings: Union[str, List[str]] = None test_metadata: Union[dict, TestMetadata] = None input_id: Optional[Union[str, URIorCURIE]] = None input_name: Optional[str] = None @@ -321,6 +309,7 @@ class TestAsset(TestEntity): well_known: Optional[Union[bool, Bool]] = None test_reference: Optional[Union[str, URIorCURIE]] = None tags: Optional[Union[str, List[str]]] = empty_list() + test_runner_settings: Optional[Union[str, List[str]]] = empty_list() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): @@ -328,12 +317,6 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.id, TestAssetId): self.id = TestAssetId(self.id) - if self._is_empty(self.runner_settings): - self.MissingRequiredField("runner_settings") - if not isinstance(self.runner_settings, list): - self.runner_settings = [self.runner_settings] if self.runner_settings is not None else [] - self.runner_settings = [v if isinstance(v, str) else str(v) for v in self.runner_settings] - if self._is_empty(self.test_metadata): self.MissingRequiredField("test_metadata") if not isinstance(self.test_metadata, TestMetadata): @@ -392,6 +375,10 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): self.tags = [self.tags] if self.tags is not None else [] self.tags = [v if isinstance(v, str) else str(v) for v in self.tags] + if not isinstance(self.test_runner_settings, list): + self.test_runner_settings = [self.test_runner_settings] if self.test_runner_settings is not None else [] + self.test_runner_settings = [v if isinstance(v, str) else str(v) for v in self.test_runner_settings] + super().__post_init__(**kwargs) @@ -408,7 +395,6 @@ class AcceptanceTestAsset(TestAsset): class_model_uri: ClassVar[URIRef] = TTM.AcceptanceTestAsset id: Union[str, AcceptanceTestAssetId] = None - runner_settings: Union[str, List[str]] = None test_metadata: Union[dict, TestMetadata] = None must_pass_date: Optional[Union[str, XSDDate]] = None must_pass_environment: Optional[Union[str, "TestEnvEnum"]] = None @@ -474,7 +460,6 @@ class TestEdgeData(TestAsset): class_model_uri: ClassVar[URIRef] = TTM.TestEdgeData id: Union[str, TestEdgeDataId] = None - runner_settings: Union[str, List[str]] = None test_metadata: Union[dict, TestMetadata] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): @@ -525,12 +510,9 @@ class TestCase(TestEntity): id: Union[str, TestCaseId] = None test_assets: Union[Dict[Union[str, TestAssetId], Union[dict, TestAsset]], List[Union[dict, TestAsset]]] = empty_dict() - test_case_runner_settings: Union[str, List[str]] = None - test_env: Optional[Union[str, "TestEnvEnum"]] = None query_type: Optional[Union[str, "QueryTypeEnum"]] = None preconditions: Optional[Union[Union[str, PreconditionId], List[Union[str, PreconditionId]]]] = empty_list() trapi_template: Optional[Union[str, "TrapiTemplateEnum"]] = None - components: Optional[Union[Union[str, "ComponentEnum"], List[Union[str, "ComponentEnum"]]]] = empty_list() test_case_objective: Optional[Union[str, "TestObjectiveEnum"]] = None test_case_source: Optional[Union[str, "TestSourceEnum"]] = None test_case_predicate_name: Optional[str] = None @@ -548,15 +530,6 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): self.MissingRequiredField("test_assets") self._normalize_inlined_as_list(slot_name="test_assets", slot_type=TestAsset, key_name="id", keyed=True) - if self._is_empty(self.test_case_runner_settings): - self.MissingRequiredField("test_case_runner_settings") - if not isinstance(self.test_case_runner_settings, list): - self.test_case_runner_settings = [self.test_case_runner_settings] if self.test_case_runner_settings is not None else [] - self.test_case_runner_settings = [v if isinstance(v, str) else str(v) for v in self.test_case_runner_settings] - - if self.test_env is not None and not isinstance(self.test_env, TestEnvEnum): - self.test_env = TestEnvEnum(self.test_env) - if self.query_type is not None and not isinstance(self.query_type, QueryTypeEnum): self.query_type = QueryTypeEnum(self.query_type) @@ -567,10 +540,6 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self.trapi_template is not None and not isinstance(self.trapi_template, TrapiTemplateEnum): self.trapi_template = TrapiTemplateEnum(self.trapi_template) - if not isinstance(self.components, list): - self.components = [self.components] if self.components is not None else [] - self.components = [v if isinstance(v, ComponentEnum) else ComponentEnum(v) for v in self.components] - if self.test_case_objective is not None and not isinstance(self.test_case_objective, TestObjectiveEnum): self.test_case_objective = TestObjectiveEnum(self.test_case_objective) @@ -606,7 +575,6 @@ class AcceptanceTestCase(TestCase): class_model_uri: ClassVar[URIRef] = TTM.AcceptanceTestCase id: Union[str, AcceptanceTestCaseId] = None - test_case_runner_settings: Union[str, List[str]] = None test_assets: Union[Dict[Union[str, AcceptanceTestAssetId], Union[dict, AcceptanceTestAsset]], List[Union[dict, AcceptanceTestAsset]]] = empty_dict() def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): @@ -636,7 +604,6 @@ class QuantitativeTestCase(TestCase): id: Union[str, QuantitativeTestCaseId] = None test_assets: Union[Dict[Union[str, TestAssetId], Union[dict, TestAsset]], List[Union[dict, TestAsset]]] = empty_dict() - test_case_runner_settings: Union[str, List[str]] = None def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if self._is_empty(self.id): @@ -647,89 +614,6 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): super().__post_init__(**kwargs) -@dataclass -class ComplianceTestCase(TestCase): - """ - TRAPI and Biolink Model standards compliance test - """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = TTM["ComplianceTestCase"] - class_class_curie: ClassVar[str] = "ttm:ComplianceTestCase" - class_name: ClassVar[str] = "ComplianceTestCase" - class_model_uri: ClassVar[URIRef] = TTM.ComplianceTestCase - - id: Union[str, ComplianceTestCaseId] = None - test_assets: Union[Dict[Union[str, TestAssetId], Union[dict, TestAsset]], List[Union[dict, TestAsset]]] = empty_dict() - test_case_runner_settings: Union[str, List[str]] = None - trapi_version: Optional[str] = None - biolink_version: Optional[str] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, ComplianceTestCaseId): - self.id = ComplianceTestCaseId(self.id) - - if self.trapi_version is not None and not isinstance(self.trapi_version, str): - self.trapi_version = str(self.trapi_version) - - if self.biolink_version is not None and not isinstance(self.biolink_version, str): - self.biolink_version = str(self.biolink_version) - - super().__post_init__(**kwargs) - - -@dataclass -class KnowledgeGraphNavigationTestCase(TestCase): - """ - Knowledge Graph navigation integration test - """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = TTM["KnowledgeGraphNavigationTestCase"] - class_class_curie: ClassVar[str] = "ttm:KnowledgeGraphNavigationTestCase" - class_name: ClassVar[str] = "KnowledgeGraphNavigationTestCase" - class_model_uri: ClassVar[URIRef] = TTM.KnowledgeGraphNavigationTestCase - - id: Union[str, KnowledgeGraphNavigationTestCaseId] = None - test_assets: Union[Dict[Union[str, TestAssetId], Union[dict, TestAsset]], List[Union[dict, TestAsset]]] = empty_dict() - test_case_runner_settings: Union[str, List[str]] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, KnowledgeGraphNavigationTestCaseId): - self.id = KnowledgeGraphNavigationTestCaseId(self.id) - - super().__post_init__(**kwargs) - - -@dataclass -class OneHopTestCase(KnowledgeGraphNavigationTestCase): - """ - 'One Hop' Knowledge Graph navigation integration test - """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = TTM["OneHopTestCase"] - class_class_curie: ClassVar[str] = "ttm:OneHopTestCase" - class_name: ClassVar[str] = "OneHopTestCase" - class_model_uri: ClassVar[URIRef] = TTM.OneHopTestCase - - id: Union[str, OneHopTestCaseId] = None - test_assets: Union[Dict[Union[str, TestAssetId], Union[dict, TestAsset]], List[Union[dict, TestAsset]]] = empty_dict() - test_case_runner_settings: Union[str, List[str]] = None - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, OneHopTestCaseId): - self.id = OneHopTestCaseId(self.id) - - super().__post_init__(**kwargs) - - @dataclass class TestSuiteSpecification(TestEntity): """ @@ -884,39 +768,6 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): super().__post_init__(**kwargs) -@dataclass -class TestRunnerConfiguration(TestEntity): - """ - General configuration parameters and test data input for a single invocation of a TestRunner. - """ - _inherited_slots: ClassVar[List[str]] = [] - - class_class_uri: ClassVar[URIRef] = TTM["TestRunnerConfiguration"] - class_class_curie: ClassVar[str] = "ttm:TestRunnerConfiguration" - class_name: ClassVar[str] = "TestRunnerConfiguration" - class_model_uri: ClassVar[URIRef] = TTM.TestRunnerConfiguration - - id: Union[str, TestRunnerConfigurationId] = None - test_run_parameters: Optional[Union[Union[dict, TestEntityParameter], List[Union[dict, TestEntityParameter]]]] = empty_list() - tags: Optional[Union[str, List[str]]] = empty_list() - - def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): - if self._is_empty(self.id): - self.MissingRequiredField("id") - if not isinstance(self.id, TestRunnerConfigurationId): - self.id = TestRunnerConfigurationId(self.id) - - if not isinstance(self.test_run_parameters, list): - self.test_run_parameters = [self.test_run_parameters] if self.test_run_parameters is not None else [] - self.test_run_parameters = [v if isinstance(v, TestEntityParameter) else TestEntityParameter(**as_dict(v)) for v in self.test_run_parameters] - - if not isinstance(self.tags, list): - self.tags = [self.tags] if self.tags is not None else [] - self.tags = [v if isinstance(v, str) else str(v) for v in self.tags] - - super().__post_init__(**kwargs) - - @dataclass class TestCaseResult(TestEntity): """ @@ -955,7 +806,8 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): @dataclass class TestRunSession(TestEntity): """ - A single invocation of a TestRunner. + Single run of a TestRunner in a given environment, with a specified set of test_entities (generally, one or more + instances of TestSuite). """ _inherited_slots: ClassVar[List[str]] = [] @@ -965,7 +817,10 @@ class TestRunSession(TestEntity): class_model_uri: ClassVar[URIRef] = TTM.TestRunSession id: Union[str, TestRunSessionId] = None + components: Optional[Union[Union[str, "ComponentEnum"], List[Union[str, "ComponentEnum"]]]] = empty_list() + test_env: Optional[Union[str, "TestEnvEnum"]] = None test_runner_name: Optional[str] = None + test_run_parameters: Optional[Union[Union[dict, TestEntityParameter], List[Union[dict, TestEntityParameter]]]] = empty_list() test_entities: Optional[Union[Dict[Union[str, TestEntityId], Union[dict, TestEntity]], List[Union[dict, TestEntity]]]] = empty_dict() test_case_results: Optional[Union[Dict[Union[str, TestCaseResultId], Union[dict, TestCaseResult]], List[Union[dict, TestCaseResult]]]] = empty_dict() timestamp: Optional[Union[str, XSDDateTime]] = None @@ -976,9 +831,20 @@ def __post_init__(self, *_: List[str], **kwargs: Dict[str, Any]): if not isinstance(self.id, TestRunSessionId): self.id = TestRunSessionId(self.id) + if not isinstance(self.components, list): + self.components = [self.components] if self.components is not None else [] + self.components = [v if isinstance(v, ComponentEnum) else ComponentEnum(v) for v in self.components] + + if self.test_env is not None and not isinstance(self.test_env, TestEnvEnum): + self.test_env = TestEnvEnum(self.test_env) + if self.test_runner_name is not None and not isinstance(self.test_runner_name, str): self.test_runner_name = str(self.test_runner_name) + if not isinstance(self.test_run_parameters, list): + self.test_run_parameters = [self.test_run_parameters] if self.test_run_parameters is not None else [] + self.test_run_parameters = [v if isinstance(v, TestEntityParameter) else TestEntityParameter(**as_dict(v)) for v in self.test_run_parameters] + self._normalize_inlined_as_dict(slot_name="test_entities", slot_type=TestEntity, key_name="id", keyed=True) self._normalize_inlined_as_dict(slot_name="test_case_results", slot_type=TestCaseResult, key_name="id", keyed=True) @@ -1112,6 +978,12 @@ class TestObjectiveEnum(EnumDefinitionImpl): QuantitativeTest = PermissibleValue( text="QuantitativeTest", description="Quantitative test") + OneHopTest = PermissibleValue( + text="OneHopTest", + description="One Hop Test of knowledge graph navigation") + StandardsValidationTest = PermissibleValue( + text="StandardsValidationTest", + description="TRAPI and Biolink Model (\"reasoner-validator\") validation") _defn = EnumDefinition( name="TestObjectiveEnum", @@ -1453,11 +1325,8 @@ class slots: slots.well_known = Slot(uri=TTM.well_known, name="well_known", curie=TTM.curie('well_known'), model_uri=TTM.well_known, domain=None, range=Optional[Union[bool, Bool]]) -slots.runner_settings = Slot(uri=TTM.runner_settings, name="runner_settings", curie=TTM.curie('runner_settings'), - model_uri=TTM.runner_settings, domain=None, range=Union[str, List[str]]) - -slots.test_case_runner_settings = Slot(uri=TTM.test_case_runner_settings, name="test_case_runner_settings", curie=TTM.curie('test_case_runner_settings'), - model_uri=TTM.test_case_runner_settings, domain=None, range=Union[str, List[str]]) +slots.test_runner_settings = Slot(uri=TTM.test_runner_settings, name="test_runner_settings", curie=TTM.curie('test_runner_settings'), + model_uri=TTM.test_runner_settings, domain=None, range=Optional[Union[str, List[str]]]) slots.must_pass_date = Slot(uri=TTM.must_pass_date, name="must_pass_date", curie=TTM.curie('must_pass_date'), model_uri=TTM.must_pass_date, domain=None, range=Optional[Union[str, XSDDate]]) @@ -1564,8 +1433,8 @@ class slots: slots.TestAsset_tags = Slot(uri=SCHEMA.additionalType, name="TestAsset_tags", curie=SCHEMA.curie('additionalType'), model_uri=TTM.TestAsset_tags, domain=TestAsset, range=Optional[Union[str, List[str]]]) -slots.TestAsset_runner_settings = Slot(uri=TTM.runner_settings, name="TestAsset_runner_settings", curie=TTM.curie('runner_settings'), - model_uri=TTM.TestAsset_runner_settings, domain=TestAsset, range=Union[str, List[str]]) +slots.TestAsset_test_runner_settings = Slot(uri=TTM.test_runner_settings, name="TestAsset_test_runner_settings", curie=TTM.curie('test_runner_settings'), + model_uri=TTM.TestAsset_test_runner_settings, domain=TestAsset, range=Optional[Union[str, List[str]]]) slots.TestCase_test_assets = Slot(uri=TTM.test_assets, name="TestCase_test_assets", curie=TTM.curie('test_assets'), model_uri=TTM.TestCase_test_assets, domain=TestCase, range=Union[Dict[Union[str, TestAssetId], Union[dict, TestAsset]], List[Union[dict, TestAsset]]]) @@ -1576,8 +1445,8 @@ class slots: slots.AcceptanceTestCase_test_assets = Slot(uri=TTM.test_assets, name="AcceptanceTestCase_test_assets", curie=TTM.curie('test_assets'), model_uri=TTM.AcceptanceTestCase_test_assets, domain=AcceptanceTestCase, range=Union[Dict[Union[str, AcceptanceTestAssetId], Union[dict, AcceptanceTestAsset]], List[Union[dict, AcceptanceTestAsset]]]) -slots.TestRunnerConfiguration_tags = Slot(uri=SCHEMA.additionalType, name="TestRunnerConfiguration_tags", curie=SCHEMA.curie('additionalType'), - model_uri=TTM.TestRunnerConfiguration_tags, domain=TestRunnerConfiguration, range=Optional[Union[str, List[str]]]) +slots.TestRunSession_test_run_parameters = Slot(uri=TTM.test_run_parameters, name="TestRunSession_test_run_parameters", curie=TTM.curie('test_run_parameters'), + model_uri=TTM.TestRunSession_test_run_parameters, domain=TestRunSession, range=Optional[Union[Union[dict, TestEntityParameter], List[Union[dict, TestEntityParameter]]]]) slots.TestRunSession_test_entities = Slot(uri=TTM.test_entities, name="TestRunSession_test_entities", curie=TTM.curie('test_entities'), model_uri=TTM.TestRunSession_test_entities, domain=TestRunSession, range=Optional[Union[Dict[Union[str, TestEntityId], Union[dict, TestEntity]], List[Union[dict, TestEntity]]]]) \ No newline at end of file diff --git a/src/translator_testing_model/schema/translator_testing_model.yaml b/src/translator_testing_model/schema/translator_testing_model.yaml index e4bf2bf..7dacbbb 100644 --- a/src/translator_testing_model/schema/translator_testing_model.yaml +++ b/src/translator_testing_model/schema/translator_testing_model.yaml @@ -88,6 +88,7 @@ classes: - name - description - tags + - test_runner_settings TestMetadata: description: >- @@ -107,8 +108,8 @@ classes: TestAsset: description: >- Represents a Test Asset, which is a single specific instance of - TestCase-agnostic semantic parameters representing the - specification of a Translator test target with inputs and (expected) outputs. + TestCase-agnostic semantic parameters representing the specification + of a Translator test target with inputs and (expected) outputs. is_a: TestEntity slots: - input_id @@ -127,7 +128,6 @@ classes: - in_v1 - well_known - test_reference - - runner_settings - test_metadata slot_usage: id: @@ -137,9 +137,9 @@ classes: description: >- One or more 'tags' slot values (inherited from TestEntity) should generally be defined to specify TestAsset membership in a "Block List" collection - runner_settings: + test_runner_settings: description: >- - Settings for the test harness, e.g. "inferred" + Scalar settings for the TestRunner, e.g. "inferred" AcceptanceTestAsset: is_a: TestAsset @@ -190,18 +190,15 @@ classes: TestCase, used to probe a particular test condition. is_a: TestEntity slots: - - test_env - query_type - test_assets - preconditions - trapi_template - - components - test_case_objective - test_case_source - test_case_predicate_name - test_case_predicate_id - test_case_input_id - - test_case_runner_settings slot_usage: test_assets: aliases: ["Block List"] @@ -230,32 +227,6 @@ classes: # test_assets: # range: QuantitativeTestAsset - ComplianceTestCase: - is_a: TestCase - description: >- - TRAPI and Biolink Model standards compliance test - slots: - - trapi_version - - biolink_version - slot_usage: -# test_assets: -# range: ComplianceTestAsset - - KnowledgeGraphNavigationTestCase: - is_a: TestCase - description: >- - Knowledge Graph navigation integration test - slot_usage: -# test_assets: -# range: KnowledgeGraphNavigationTestAsset - - OneHopTestCase: - is_a: KnowledgeGraphNavigationTestCase - description: >- - 'One Hop' Knowledge Graph navigation integration test - slot_usage: -# test_assets: -# range: OneHopTestAsset ##################### # Test Suite Models # @@ -304,19 +275,6 @@ classes: # TestRunner Models # ##################### - TestRunnerConfiguration: - description: >- - General configuration parameters and test data input - for a single invocation of a TestRunner. - is_a: TestEntity - slots: - - test_run_parameters - slot_usage: - tags: - description: >- - One or more 'tags' slot values (slot inherited from TestEntity) - should generally identify the TestRunner(s) using this configuration. - TestCaseResult: description: >- The outcome of a TestRunner run of one specific TestCase. @@ -328,18 +286,28 @@ classes: TestRunSession: description: >- - A single invocation of a TestRunner. + Single run of a TestRunner in a given environment, with a specified + set of test_entities (generally, one or more instances of TestSuite). is_a: TestEntity slots: + - components + - test_env - test_runner_name + - test_run_parameters - test_entities - test_case_results - timestamp slot_usage: + test_run_parameters: + description: >- + Different TestRunners could expect additional global test + configuration parameters, like the applicable TRAPI version + ("trapi_version") or Biolink Model versions ("biolink_version"). test_entities: description: >- - Different TestRunners may expect specific kinds of TestEntity as input. - These 'test_entities' one or more instances of TestAsset, TestCase or TestSuite. + Different TestRunners could expect specific kinds of TestEntity + as an input. These 'test_entities' are one or more instances of + TestAsset, TestCase or (preferably?) TestSuite. TestOutput: description: >- @@ -407,7 +375,7 @@ slots: test_entity_parameters: description: >- - One or more parameters documenting target characteristics of a TestEntity. + One or more 'tag = value' parameters documenting target characteristics of a TestEntity. range: TestEntityParameter multivalued: true inlined: true @@ -579,21 +547,13 @@ slots: aliases: ["Well Known"] range: boolean - runner_settings: + test_runner_settings: aliases: ["Settings"] description: >- - Settings for the test harness for TestAsset + Scalar parameters for the TestRunner processing a given TestEntity. range: string multivalued: true - required: true - - test_case_runner_settings: - aliases: ["Settings"] - description: >- - Settings for the test harness for TestCase - range: string - multivalued: true - required: true + required: false must_pass_date: description: >- @@ -778,8 +738,8 @@ slots: test_run_parameters: description: >- - Parameters for TestRunnerConfiguration that inform the TestHarness - and TestRunners about the general characteristics of a test run. + TestRunSession parameters informing a TestHarness about + the global characteristics of TestRunner processing tests. is_a: test_entity_parameters test_suite_id: @@ -865,6 +825,10 @@ enums: description: Semantic benchmarking QuantitativeTest: description: Quantitative test + OneHopTest: + description: One Hop Test of knowledge graph navigation + StandardsValidationTest: + description: TRAPI and Biolink Model ("reasoner-validator") validation TestEnvEnum: description: >-