From 664006d4c4316ffaa6bb9f0fc8098515fa57942e Mon Sep 17 00:00:00 2001 From: LucaFgr Date: Mon, 17 Nov 2025 07:56:50 +0000 Subject: [PATCH 1/9] Moving changes from json to inc_nlohmann_json --- .dotstop.dot | 3 +++ TSF/trustable/statements/JLS-02.md | 2 +- TSF/trustable/statements/JLS-10.md | 8 +++++++- TSF/trustable/statements/JLS-16.md | 10 +++++++++- TSF/trustable/statements/JLS-38.md | 20 ++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 TSF/trustable/statements/JLS-38.md diff --git a/.dotstop.dot b/.dotstop.dot index e8c2d8c40a..3812dade2a 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -67,6 +67,7 @@ digraph G { "JLS-33" [sha="16ebc7717e389ac1ca349ead591b4dc5b65997e8c5f78d58d6293cd75bbe0d28"]; "JLS-34" [sha="3484d9766deace45ecbc0d6892c7114ce7d97a51836399887500a318b3a88dc6"]; "JLS-35" [sha="b11006d1d5708c3aba84d4f06834ad965d6aebde8619306389a4f8fa655b2dcf"]; +"JLS-38" [sha="a7ab6cc546e4c9c02c6dc479b8fff29ef6f5be5459185daac4ad1117b2845115"]; "NJF-01" [sha="548dc86014e093974f68660942daa231271496a471885bbed092a375b3079bd8"]; "NJF-02" [sha="6ea015646d696e3f014390ff41612eab66ac940f20cf27ce933cbadf8482d526"]; "NJF-03" [sha="4bd1f8210b7bba9a248055a437f377d9da0b7576c5e3ed053606cf8b5b2febe3"]; @@ -407,8 +408,10 @@ digraph G { "TA-MISBEHAVIOURS" -> "JLS-31" [sha="ff3352e20146a81904da6d8d94b003b4e0acbc2a8a63a73ea017ea0535e45e79"]; "TA-RELEASES" -> "JLS-14" [sha="1202b9934353436fba927de6762cf62a8fc23ab0815a3c06f9d0a77b55830720"]; "TA-RELEASES" -> "JLS-21" [sha="5d57d2b547a841bb31f29034b785d9bec1ffb0e495d80e0e356a54391aa22e1b"]; +"TA-RELEASES" -> "JLS-38" [sha="ed0d250c8c191ac4fc03712a321bf399d26e932edabb13067fbcffae1640cb90"]; "TA-SUPPLY_CHAIN" -> "JLS-23" [sha=fe2b810e22c4da9911266183bc8679a56d8dd2d5a76624cd1f3ee329d9b93a08]; "TA-TESTS" -> "JLS-16" [sha=a4143b13d9ae2553534457603bdca9beb6cca0ee5b8b9bae50cefa97d2519702]; +"TA-TESTS" -> "JLS-02" [sha="e99cf5b009b3cdc149edc81b3454dddfaf69ab10f80e70ce698bcfb823b5fbd1"]; "TA-UPDATES" -> "JLS-06" [sha="7386ba4dfdca14a2b0c73b6b759ddeee66e0196f164322d552c2867e5c7a4b96"]; "TA-UPDATES" -> "JLS-07" [sha="9256bec79e828b44dd12d4298483bbab7ab24a1eb542c133ee5392ee5829cb7f"]; "TA-UPDATES" -> "JLS-12" [sha="45b7cf8eebee7a35ba39b3f990cefe3cbfd79c5f74415c5321026d64d89f5703"]; diff --git a/TSF/trustable/statements/JLS-02.md b/TSF/trustable/statements/JLS-02.md index 814585ea16..62312fbf92 100644 --- a/TSF/trustable/statements/JLS-02.md +++ b/TSF/trustable/statements/JLS-02.md @@ -25,4 +25,4 @@ score: Erikhu1: 1.0 --- -Fuzz testing is used in the original nlohmann/json repository (https://github.com/nlohmann/json) to uncover edge cases and failure modes throughout development. (https://github.com/nlohmann/json/blob/develop/tests/fuzzing.md) \ No newline at end of file +Fuzz testing is used in the nlohmann/json repository to uncover edge cases and failure modes throughout development. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-10.md b/TSF/trustable/statements/JLS-10.md index e6c0555d38..446c5bb226 100644 --- a/TSF/trustable/statements/JLS-10.md +++ b/TSF/trustable/statements/JLS-10.md @@ -1,6 +1,12 @@ --- level: 1.1 normative: true +references: + - type: website + url: "https://github.com/nlohmann/json/releases" + description: "List of nlohmann/json releases consisting of source code, build instructions, test code and test result summaries." --- -Every release of nlohmann/json includes source code, build instructions, tests and attestations. (TODO: Test result summary) \ No newline at end of file +Every release of the nlohmann/json library includes source code, build instructions, test code, test results summaries and attestations. + +NOTE: Currently, I cannot find any attestations for nlohmann/json. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-16.md b/TSF/trustable/statements/JLS-16.md index 9fee7fc698..daf78c64e8 100644 --- a/TSF/trustable/statements/JLS-16.md +++ b/TSF/trustable/statements/JLS-16.md @@ -3,14 +3,22 @@ references: - type: verbose_file path: "./TSF/docs/list_of_test_environments.md" comment: "The list of all test-cases together with their execution environments" + - type: website + url: "https://github.com/score-json/json/actions" + description: "Github actions page showing that score-json is using Github host environment." evidence: type: check_list_of_tests configuration: sources: - "./tests/src" - "./TSF/tests" + type: https_response_time + configuration: + target: 2.0 + urls: + - https://github.com/score-json/json/actions level: 1.1 normative: true --- -A list of tests, which is extracted from the test execution, is provided, along with a list of test environments. \ No newline at end of file +A list of tests, which is extracted from the test execution, is provided, along with a list of test environments, a list of fault induction tests and test construction configurations and results. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-38.md b/TSF/trustable/statements/JLS-38.md new file mode 100644 index 0000000000..d4a8b1046a --- /dev/null +++ b/TSF/trustable/statements/JLS-38.md @@ -0,0 +1,20 @@ +--- +level: 1.1 +normative: true +references: + - type: website + url: "https://json.nlohmann.me/integration/cmake/" + description: "cmake build management documentation for nlohmann/json" + - type: website + url: "https://json.nlohmann.me/integration/package_managers/" + description: "package manager documentation for nlohmann/json" +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/integration/cmake/" + - "https://json.nlohmann.me/integration/package_managers/" +--- + +Every release of the nlohmann/json library shall provide configuration management for build instructions and infrastructure. \ No newline at end of file From c875377e6427b1552757f389d6796d7cb4d2560d Mon Sep 17 00:00:00 2001 From: Erik Hu Date: Wed, 19 Nov 2025 10:40:58 +0100 Subject: [PATCH 2/9] add context files (#5) * add context files * remove references to checklist files * add answer fields * explain component evidence * cleanup --- .dotstop.dot | 16 ++ TSF/scripts/generate_report.sh | 2 +- TSF/trustable/assertions/TA-ANALYSIS.md | 3 - .../assertions/TA-ANALYSIS_CONTEXT.md | 91 ++++++++++++ TSF/trustable/assertions/TA-BEHAVIOURS.md | 3 - .../assertions/TA-BEHAVIOURS_CONTEXT.md | 88 +++++++++++ TSF/trustable/assertions/TA-CONFIDENCE.md | 3 - .../assertions/TA-CONFIDENCE_CONTEXT.md | 45 ++++++ TSF/trustable/assertions/TA-CONSTRAINTS.md | 3 - .../assertions/TA-CONSTRAINTS_CONTEXT.md | 90 ++++++++++++ TSF/trustable/assertions/TA-DATA.md | 3 - TSF/trustable/assertions/TA-DATA_CONTEXT.md | 64 ++++++++ TSF/trustable/assertions/TA-FIXES.md | 3 - TSF/trustable/assertions/TA-FIXES_CONTEXT.md | 104 +++++++++++++ TSF/trustable/assertions/TA-INDICATORS.md | 3 - .../assertions/TA-INDICATORS_CONTEXT.md | 104 +++++++++++++ TSF/trustable/assertions/TA-INPUTS.md | 3 - TSF/trustable/assertions/TA-INPUTS_CONTEXT.md | 117 +++++++++++++++ TSF/trustable/assertions/TA-ITERATIONS.md | 3 - .../assertions/TA-ITERATIONS_CONTEXT.md | 76 ++++++++++ TSF/trustable/assertions/TA-METHODOLOGIES.md | 3 - .../assertions/TA-METHODOLOGIES_CONTEXT.md | 72 +++++++++ TSF/trustable/assertions/TA-MISBEHAVIOURS.md | 3 - .../assertions/TA-MISBEHAVIOURS_CONTEXT.md | 138 ++++++++++++++++++ TSF/trustable/assertions/TA-RELEASES.md | 3 - .../assertions/TA-RELEASES_CONTEXT.md | 79 ++++++++++ TSF/trustable/assertions/TA-SUPPLY_CHAIN.md | 3 - .../assertions/TA-SUPPLY_CHAIN_CONTEXT.md | 67 +++++++++ TSF/trustable/assertions/TA-TESTS.md | 3 - TSF/trustable/assertions/TA-TESTS_CONTEXT.md | 52 +++++++ TSF/trustable/assertions/TA-UPDATES.md | 3 - .../assertions/TA-UPDATES_CONTEXT.md | 65 +++++++++ TSF/trustable/assertions/TA-VALIDATION.md | 3 - .../assertions/TA-VALIDATION_CONTEXT.md | 69 +++++++++ 34 files changed, 1338 insertions(+), 49 deletions(-) create mode 100644 TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-DATA_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-FIXES_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-INPUTS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-RELEASES_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-TESTS_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-UPDATES_CONTEXT.md create mode 100644 TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md diff --git a/.dotstop.dot b/.dotstop.dot index e8c2d8c40a..7e15da1524 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -182,21 +182,37 @@ digraph G { "PJD-02" [sha="2ad95a7543626869ed3195b95256c03a991df49e51acc6b54513a1ed32d63dc8"]; "PJD-03" [sha=fcc8152c7a04032d91becabc5577409773b64c381d09bdab7be954ca1916db64]; "TA-ANALYSIS" [sha="76cb774300c75d7b6a24c872fe691f2f0cd1d36cc4a2493e835f6b861bc79c34"]; +"TA-ANALYSIS_CONTEXT" [sha="6d4ba31be15b85d6836a7f79d2f7d9fef2bb9a5cc3809ea1173e7799fb73c08d"]; "TA-BEHAVIOURS" [sha="3ec27e29aa991978efe6a56267b98c2a08b27a4aff693d5cf2b01dfe72276570"]; +"TA-BEHAVIOURS_CONTEXT" [sha="5e82851069ccae86c01da3c636f6ff6f37ae101f4148d8593be282d277cefc29"]; "TA-CONFIDENCE" [sha=afda09331b2fc3b8d9b1cd921bee66251a65e5543a473c61eb03f9ea11d57eb5]; +"TA-CONFIDENCE_CONTEXT" [sha="cf43890902cf5fffab2dc1445c959da69893ec6dcfc6ae8267376db91a7bd452"]; "TA-CONSTRAINTS" [sha=cdee0ae34c33110044975efc981e4ac4d63d824aaaac78233b1f3828ef070da3]; +"TA-CONSTRAINTS_CONTEXT" [sha="ae3df98a3e78809e6461906dc19c51afae254214c884ff337e0879e44929fbe9"]; "TA-DATA" [sha="796e4b5851340906568a47c4436c1fa67ca1c48c98a8b6a16b19283d04cea799"]; +"TA-DATA_CONTEXT" [sha="ef12e98926137d14c7fa45094391da2d128de0061191921e6df8a8ddba84930e"]; "TA-FIXES" [sha="08f069034d1399c43c4321c5f05de72125f4155a8b68b9bbb2029cb679e6ad48"]; +"TA-FIXES_CONTEXT" [sha="a625f84bd6a86eaf5b6d199d04a96abe53cda0a6eefdfaae53d3557a0ec7e57c"]; "TA-INDICATORS" [sha=a5392ef1e3e40d82ef9a304d1304bf007fa85d961ab8ea268231cb212d763477]; +"TA-INDICATORS_CONTEXT" [sha="38f6ba205e232c70863e78f740b8c0ac63004cf2886fa311261ac9d849b26e16"]; "TA-INPUTS" [sha="6edcb6e0ea0a918f611d6644da7a28dd5c924a210984cd913e7ff558677a6ea6"]; +"TA-INPUTS_CONTEXT" [sha="4288c4a4d07801c1492203ee37390e494588032dbaa27c1b9506029e12ba1fbf"]; "TA-ITERATIONS" [sha=c445bfe866db71df67d4e87353d674b62abce19b52048fac37284d8065d67678]; +"TA-ITERATIONS_CONTEXT" [sha="78e116bb06c8699bc6366580696a9bd84613145bcc5db4caa9d04bb07ec0015d"]; "TA-METHODOLOGIES" [sha=d24f6055c79268b1f6e4bdf73951719f192d6f492a7376f85b349a95ccb2a319]; +"TA-METHODOLOGIES_CONTEXT" [sha="3cc5aad374f755859fb06ee534bf7dd18ab95a16f12a916614bf0d1048c2ffd2"]; "TA-MISBEHAVIOURS" [sha=d24fcdeee0ae0fa696f272dc39c8e9e37cce7fb9b2cfd07bcd1451b765be5c6e]; +"TA-MISBEHAVIOURS_CONTEXT" [sha="22df78c213228c60b0a85f3d82eb99803b2e6c15d7a67f5ab33cdfe3355e2715"]; "TA-RELEASES" [sha="3c924109e9916fb154eadbc2d733a8413ae551a1282b73de389b9ad7540a4e75"]; +"TA-RELEASES_CONTEXT" [sha="79103bd44a548248276fa7c21b010a59aa2a1246cb773bd2433abccb36cde817"]; "TA-SUPPLY_CHAIN" [sha="0629a5a339322874ad3d51c0c14219ede72195bf514abac82c95ebc3a685ae2c"]; +"TA-SUPPLY_CHAIN_CONTEXT" [sha="a847f61665fa3da0b3324cd8f0528ca56fba4813634033955d4c30db1d8bfa73"]; "TA-TESTS" [sha=afa5e61fc86f70f99f6c60b4f2b51ba7d486705f197048a7dc3fc8fea225385c]; +"TA-TESTS_CONTEXT" [sha="eb2ed76391364cfdc2546d8c3eac93b0c40220efc7c7d32b4eb6a2562da6ad34"]; "TA-UPDATES" [sha="9f0554c79d125a37c7e68b9efbb022dc4853a3e2f87c7d224d30c51f5b9b8435"]; +"TA-UPDATES_CONTEXT" [sha="296a9acece851901c0bcbdca1d09b26e517637099d6fba0e2d6c2715e153ac96"]; "TA-VALIDATION" [sha="20f6d87b89d9fbffe427a734ab70906c26ad787e53032a8c4f4d599227704be6"]; +"TA-VALIDATION_CONTEXT" [sha="5c545fe4beb6dff690ea80e0c1ea71abdf72d8c6d404ae456234c7749f7a8586"]; "TIJ-01" [sha="f14e40946ba0f957437e1d4eecd71b9625518b6fdee282e1dab4bb1b247f131a"]; "TIJ-01.1" [sha="5e43e22dd59a3ffb8e2304053f3150e2eb6ed44b3456acfade28df386b3a1e1c"]; "TIJ-01.2" [sha="2ca07a1a8d1fc3bd41de5866e22cfc08db1dcbd0ac3b203725d1ef3caa6e6cdd"]; diff --git a/TSF/scripts/generate_report.sh b/TSF/scripts/generate_report.sh index d2f6ae50d2..757e551b1b 100755 --- a/TSF/scripts/generate_report.sh +++ b/TSF/scripts/generate_report.sh @@ -15,7 +15,7 @@ mkdir -p "$TSF_REPORT_FOLDER" # -p ensures no error if the folder already exists # generate TSF report echo "Generating TSF report in: $TSF_REPORT_FOLDER" -trudag publish --validate --figures --output-dir "$TSF_REPORT_FOLDER" --dump data_store || exit 1 +trudag publish --validate --figures --all-bodies --output-dir "$TSF_REPORT_FOLDER" --dump data_store || exit 1 # generate TSF graph trudag plot -o "$TSF_REPORT_FOLDER/graph.svg" --url "$1/generated" || exit 1 diff --git a/TSF/trustable/assertions/TA-ANALYSIS.md b/TSF/trustable/assertions/TA-ANALYSIS.md index 0b42206cf1..5019ace3c5 100644 --- a/TSF/trustable/assertions/TA-ANALYSIS.md +++ b/TSF/trustable/assertions/TA-ANALYSIS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-ANALYSIS-CHECKLIST.md --- Collected data from tests and monitoring of deployed software in eclipse-score/inc_nlohmann_json is analysed according to specified objectives. \ No newline at end of file diff --git a/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md b/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md new file mode 100644 index 0000000000..e6dc6ee310 --- /dev/null +++ b/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md @@ -0,0 +1,91 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied to the extent that test data, and data collected +from monitoring of deployed versions of XYZ, has been analysed, and the results +used to inform the refinement of Expectations and risk analysis. + +The extent of the analysis is with sufficient precision to confirm that: + +- all Expectations (TA-BEHAVIOURS) are met +- all Misbehaviours (TA-MISBEHAVIOURS) are detected or mitigated +- all advance warning indicators (TA-INDICATORS) are monitored +- failure rates (calculated directly or inferred by statistics) are + within acceptable tolerance + +When tests reveal Misbehaviours missing from our analysis (TA-ANALYSIS), we update our Expectations (TA-BEHAVIOURS, TA-MISBEHAVIOURS). +Guided by confidence evaluations (TA-CONFIDENCE), we refine and repeat the analysis as needed. +Analysis results also inform confidence evaluations, allowing automatic generation through statistical modelling and defining Key Performance Indicators (KPIs) for consistent use across the TSF. + +For increased confidence in the analysis specification and results, they should be evaluated in terms of their reliability, relevance, and understandability. + +- Reliability: The analysis methods must be verified against both known good and bad data to ensure sufficient detection of false negatives and false positives. + Accuracy degradation across methods should be tracked and aggregated, making outcomes more easily verifiable and providing visibility into how changes to the system under test or to the analysis mechanisms affect the results. +- Relevance: The results must account for hardware and hardware/software interactions. + Calibration should address capacity, scalability, response time, latency, and throughput where applicable. + To further increase confidence in estimated failure rates, the analysis should also cover testing sufficiency (with statistical methods where appropriate), cascading failures including sequencing and concurrency, bug analysis, and comparison against expected results and variability. + The analysis should be automated and exercised repeatedly for timely feedback. +- Understandability: Both methods and results should be mapped to other analyses performed on the system (linked to TT_EXPECTATIONS) to ensure alignment with scope, abstraction levels, and partitioning, thereby guiding prioritisation. + Effectiveness also depends on user-friendliness and presentation (involving semi-formal structured forms, supported by diagrams and figures with clear legends). + +To gain increased confidence, test results should be shown to be reproducible. +Even with non-deterministic software, representative test setups must be ensured to produced reproducible results +within a defined threshold as specified by TT-EXPECTATIONS. +Reproducible test results also supports verification of toolchain updates (together with other measures in TA-FIXES), +by confirming that test results remain unchanged when no changes are intended. + +**Evidence** + +- Analysis of test data, including thresholds in relation to appropriate statistical properties. + - **Answer**: +- Analysis of failures + - **Answer**: +- Analysis of spikes and trends + - **Answer**: +- Validation of analysis methods used + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-ANALYSIS is based on KPIs +that may indicate problems in development, test, or production. + +**CHECKLIST** + +- What fraction of Expectations are covered by the test data? + - **Answer**: +- What fraction of Misbehaviours are covered by the monitored indicator data? + - **Answer**: +- How confident are we that the indicator data are accurate and timely? + - **Answer**: +- How reliable is the monitoring process? + - **Answer**: +- How well does the production data correlate with our test data? + - **Answer**: +- Are we publishing our data analysis? + - **Answer**: +- Are we comparing and analysing production data vs test? + - **Answer**: +- Are our results getting better, or worse? + - **Answer**: +- Are we addressing spikes/regressions? + - **Answer**: +- Do we have sensible/appropriate target failure rates? + - **Answer**: +- Do we need to check the targets? + - **Answer**: +- Are we achieving the targets? + - **Answer**: +- Are all underlying assumptions and target conditions for the analysis specified? + - **Answer**: +- Have the underlying assumptions been verified using known good data? + - **Answer**: +- Has the Misbehaviour identification process been verified using known bad data? + - **Answer**: +- Are results shown to be reproducible? + - **Answer**: + diff --git a/TSF/trustable/assertions/TA-BEHAVIOURS.md b/TSF/trustable/assertions/TA-BEHAVIOURS.md index e2b8b5c3ed..31c65544c9 100644 --- a/TSF/trustable/assertions/TA-BEHAVIOURS.md +++ b/TSF/trustable/assertions/TA-BEHAVIOURS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-BEHAVIOURS-CHECKLIST.md --- Expected or required behaviours for the nlohmann/json library are identified, specified, verified and validated based on analysis. diff --git a/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md b/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md new file mode 100644 index 0000000000..c85654a6ce --- /dev/null +++ b/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md @@ -0,0 +1,88 @@ +--- +level: 1.1 +normative: false +--- + +Although it is practically impossible to specify all of the necessary behaviours +and required properties for complex software, we must clearly specify the most +important of these (e.g. where harm could result if given criteria are not met), +and verify that these are correctly provided by XYZ. + +**Guidance** + +This assertion is satisfied to the extent that we have: + +- Determined which Behaviours are critical for consumers of XYZ and recorded + them as Expectations. +- Verified these Behaviours are achieved. + +Expectations could be verified by: + +- Functional testing for the system. +- Functional soak testing for the system. +- Specifying architecture and verifying its implementation with pre-merge + integration testing for components. +- Specifying components and verifying their implementation using pre-merge unit + testing. + +The number and combination of the above verification strategies will depend on +the scale of the project. For example, unit testing is more suitable for the +development of a small library than an OS. +Similarly, the verification strategy must align with the chosen development +methods and be supported by appropriate verification approaches and tools. + +Regardless of the chosen strategy, the reasoning behind it must be recorded in a +traceable way, linking breakdown and verification methods to the relevant +reasoning, abstraction levels, and design partitioning (including system +interfaces with users and hardware, or other system boundaries). + +Finally, the resulting system must be validated, with the foundation of +validation being a working system that has appropriately considered calibration +targets such as capacity, scalability, response time, latency, and throughput, +where applicable. +Without this, specification and verification efforts cannot be considered +sufficient. + +**Evidence** + +- List of Expectations + - **Answer**: +- Argument of sufficiency for break-down of expected behaviour for all + Expectations + - **Answer**: +- Validation and verification of expected behaviour + - **Answer**: + + +**Confidence scoring** + +Confidence scoring for TA-BEHAVIOURS is based on our confidence that the list of +Expectations is accurate and complete, that Expectations are verified by tests, +and that the resulting system and tests are validated by appropriate strategies. + +**Checklist** + +- How has the list of Expectations varied over time? + - **Answer**: +- How confident can we be that this list is comprehensive? + - **Answer**: +- Could some participants have incentives to manipulate information? + - **Answer**: +- Could there be whole categories of Expectations still undiscovered? + - **Answer**: +- Can we identify Expectations that have been understood but not specified? + - **Answer**: +- Can we identify some new Expectations, right now? + - **Answer**: +- How confident can we be that this list covers all critical requirements? + - **Answer**: +- How comprehensive is the list of tests? + - **Answer**: +- Is every Expectation covered by at least one implemented test? + - **Answer**: +- Are there any Expectations where we believe more coverage would help? + - **Answer**: +- How do dependencies affect Expectations, and are their properties verifiable? + - **Answer**: +- Are input analysis findings from components, tools, and data considered in relation to Expectations? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-CONFIDENCE.md b/TSF/trustable/assertions/TA-CONFIDENCE.md index 54e0006966..05a552823e 100644 --- a/TSF/trustable/assertions/TA-CONFIDENCE.md +++ b/TSF/trustable/assertions/TA-CONFIDENCE.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-CONFIDENCE-CHECKLIST.md --- Confidence in the nlohmann/json library is measured based on results of analysis. diff --git a/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md b/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md new file mode 100644 index 0000000000..efa1d013e1 --- /dev/null +++ b/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md @@ -0,0 +1,45 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +To quantify confidence, either a subjective assessment or a statistical argument must be presented for each statement and then systematically and repeatably aggregated to assess whether the final deliverable is fit for purpose. + +To improve the accuracy of confidence evaluations in reflecting reality, the following steps are necessary: + +- Break down high-level claims into smaller, recursive requests. +- Provide automated evaluations whenever possible, and rely on subjective assessments from appropriate parties when automation is not feasible. +- Aggregate confidence scores from evidence nodes. +- Continuously adjust prior confidence measures with new evidence, building on established values. + +Any confidence scores, whether tracked manually or statistically, must be based on documented review guidelines that are themselves reviewed and applied by appropriate parties. +These guidelines should focus on detecting inconsistencies in the reasoning and evidence linked to related Expectations, and on assessing the relevancy of all aspects considered. +As a result, the argument structure must reflect the project scope, which in turn should be captured in the set Expectations and linked to the project’s analysis, design considerations, and partitioning. +Within this structure, Statements must be ordered or weighted so that their relative importance and supporting reasoning are clear, with iteration scores capturing strengths and weaknesses and guiding decisions. + +As subjective assessments are replaced with statistical arguments and confidence scores are refined with new evidence, evaluation accuracy improves. +Over time, these scores reveal the project's capability to deliver on its objectives. +The process itself should be analysed to determine score maturity, with meta-analysis used to assess long-term trends in sourcing, accumulation, and weighting. + +**Evidence** + +- Confidence scores from other TA items + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-CONFIDENCE is based on quality of the confidence +scores given to Statements + +**Checklist** + +- What is the algorithm for combining/comparing the scores? + - **Answer**: +- How confident are we that this algorithm is fit for purpose? + - **Answer**: +- What are the trends for each score? + - **Answer**: +- How well do our scores correlate with external feedback signals? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-CONSTRAINTS.md b/TSF/trustable/assertions/TA-CONSTRAINTS.md index 6db0e6a836..0e86c0ce1e 100644 --- a/TSF/trustable/assertions/TA-CONSTRAINTS.md +++ b/TSF/trustable/assertions/TA-CONSTRAINTS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-CONSTRAINTS-CHECKLIST.md --- Constraints on adaptation and deployment of eclipse-score/inc_nlohmann_json are specified. diff --git a/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md b/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md new file mode 100644 index 0000000000..c57b098a9d --- /dev/null +++ b/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md @@ -0,0 +1,90 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +Constraints on reuse, reconfiguration, modification, and deployment are +specified to enhance the trustability of outputs. To ensure clarity, boundaries +on what the output cannot do - especially where common domain assumptions may +not hold - must be explicitly documented. These constraints are distinct from +misbehaviour mitigations; instead, they define the context within which the +system is designed to operate, including all modes and environmental +considerations. This upfront documentation clarifies intended use, highlights +known limitations, and prevents misinterpretation. + +These constraints, categorised into explicit limitations and assumptions of use, +guide both stakeholders and users (integrators, maintainers, operators, and +end-users). They define the intended scope and provide a clear interface for how +upstream and downstream systems can integrate, modify, install, reuse, or +reconfigure to achieve the desired output. The documentation must also specify +the contexts in which the integrity of existing Statements is preserved and +whether reimplementation is required, considering device maintenance +assumptions, including software updates and vulnerability mitigation. + +Crucially, these limitations are not unresolved defects from triage decisions +but deliberate exclusions based on design choices. Each omission should be +supported by a clear rationale (linked to relevant Expectations and analyses +with the appropriate architectural and abstraction levels) to ensure +transparency for future scope expansion and to guide both upstream and +downstream modifications. + +To remain effective in practice, constraints must consider user-friendliness in +relation to associated Misbehaviours (TA-MISBEHAVIOURS) and AWIs +(TA-INDICATORS): + +- Include mechanisms to prevent misuse (e.g., protecting runtime parameters from + corruption or unauthorized modification during both development and + operation), explicitly linking them to relevant Misbehaviours and their + analyses (as defined in TA-MISBEHAVIOURS). +- Present constraint-related data with emphasis on availability, clarity, and + transparent communication of defined safe states, along with the mechanisms + that transition the system into those states, ensuring they are connected to + the relevant AWIs (as defined in TA-INDICATORS). + +Finally, the documentation must establish and promote a clear process for +reporting bugs, issues, and requests. + +**Suggested evidence** + +- Installation manuals with worked examples + - **Answer**: +- Configuration manuals with worked examples + - **Answer**: +- Specification documentation with a clearly defined scope + - **Answer**: +- User guides detailing limitations in interfaces designed for expandability or + modularity + - **Answer**: +- Documented strategies used by external users to address constraints and + work with existing Statements + - **Answer**: + +**Confidence scoring** + +The reliability of these constraints should be assessed based on the absence of +contradictions and obvious pitfalls within the defined Statements. + +**Checklist** + +- Are the constraints grounded in realistic expectations, backed by real-world + examples? + - **Answer**: +- Do they effectively guide downstream consumers in expanding upon existing + Statements? + - **Answer**: +- Do they provide clear guidance for upstreams on reusing components with + well-defined claims? + - **Answer**: +- Are any Statements explicitly designated as not reusable or adaptable? + - **Answer**: +- Are there worked examples from downstream or upstream users demonstrating + these constraints in practice? + - **Answer**: +- Have there been any documented misunderstandings from users, and are these + visibly resolved? + - **Answer**: +- Do external users actively keep up with updates, and are they properly + notified of any changes? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-DATA.md b/TSF/trustable/assertions/TA-DATA.md index 53443a5f65..1babf29ae3 100644 --- a/TSF/trustable/assertions/TA-DATA.md +++ b/TSF/trustable/assertions/TA-DATA.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-DATA-CHECKLIST.md --- Data in eclipse-score/inc_nlohmann_json is collected from tests, and from monitoring of deployed software, according to specified objectives. diff --git a/TSF/trustable/assertions/TA-DATA_CONTEXT.md b/TSF/trustable/assertions/TA-DATA_CONTEXT.md new file mode 100644 index 0000000000..b918240144 --- /dev/null +++ b/TSF/trustable/assertions/TA-DATA_CONTEXT.md @@ -0,0 +1,64 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied if results from all tests and monitored deployments are captured accurately, ensuring: + +- Sufficient precision for meaningful analysis +- Enough contextual information to reproduce the setup (e.g., runner ID, software version SHA), though not necessarily the exact results + +Monitored deployments run in both production and development, validating monitoring mechanisms across environments and ensuring comparable results. +Collecting and retaining all data that support project claims (together with traceability to reasoning and specifications, and including both established and experimental indicators as well as test data from all environments) preserves evidence for selecting appropriate measures and enables historical analysis. + +To avoid misinterpretation, all data storage mechanisms and locations are documented, together with long-term storage strategies, so analyses can be reliably reproduced. +How this data is made accessible is assessed as part of TA-ITERATIONS. + +Storage strategies should account for foreseeable malicious activities and privacy considerations when handling sensitive data, including how the data is managed during transit and at rest, and whether it can be accessed in plaintext or only through appropriate tools (also considered for TA-INPUTS and TA-TESTS). + +Appropriate storage strategies safeguard availability across the product lifecycle, with emphasis on release-related data, and account for decommissioning, infrastructure teardown, and post-project backups. + +**Evidence** + +- Time-stamped and traceable result records for each test execution, linked to associated system under test version and specification references. + - **Answer**: +- List of monitored indicators, linked to associated specification version references. + - **Answer**: +- Time-stamped and traceable test-derived data for each indicator, linked to associated system under test version and indicator specifications references. + - **Answer**: +- List of monitored deployments, linked to associated version and configuration references. + - **Answer**: +- Time-stamped and traceable production data for each indicator, linked to associated deployment metadata and specification references. + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-DATA quantifies the completeness of test results +(including pass/fail and performance) and the availability of data from all +monitored deployments. + +**Checklist** + +- Is all test data stored with long-term accessibility? + - **Answer**: +- Is all monitoring data stored with long-term accessibility? + - **Answer**: +- Are extensible data models implemented? + - **Answer**: +- Is sensitive data handled correctly (broadcasted, stored, discarded, or anonymised) with appropriate encryption and redundancy? + - **Answer**: +- Are proper backup mechanisms in place? + - **Answer**: +- Are storage and backup limits tested? + - **Answer**: +- Are all data changes traceable? + - **Answer**: +- Are concurrent changes correctly managed and resolved? + - **Answer**: +- Is data accessible only to intended parties? + - **Answer**: +- Are any subsets of our data being published? + - **Answer**: + diff --git a/TSF/trustable/assertions/TA-FIXES.md b/TSF/trustable/assertions/TA-FIXES.md index bf0111cf83..f69511b73b 100644 --- a/TSF/trustable/assertions/TA-FIXES.md +++ b/TSF/trustable/assertions/TA-FIXES.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-FIXES-CHECKLIST.md --- In the nlohmann/json repository, known bugs or misbehaviours are analysed and triaged, and critical fixes or mitigations are implemented or applied. diff --git a/TSF/trustable/assertions/TA-FIXES_CONTEXT.md b/TSF/trustable/assertions/TA-FIXES_CONTEXT.md new file mode 100644 index 0000000000..2ea376dfb8 --- /dev/null +++ b/TSF/trustable/assertions/TA-FIXES_CONTEXT.md @@ -0,0 +1,104 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied to the extent that we have identified, triaged, and applied fixes or mitigations to faults in XYZ, as well as to bugs and publicly disclosed vulnerabilities identified in upstream dependencies. + +Confidence can be improved by assessing known faults, bugs, and vulnerabilities to establish their relevance and impact for XYZ. +An important aspect is documenting how issues are discovered and tracked, including identifying additional Misbehaviours (TA-MISBEHAVIOURS) that may require immediate mitigation measures (including recalls), and how such issues are communicated to users. + +In principle, this analysis should include not only the code in XYZ but also its dependencies (all the way down) and the tools and data used to construct the release. +In practice, however, the cost/benefit of this work must be weighed against: + +- the volume and quality of available bug and vulnerability reports +- the likelihood that our build, configuration, or use case is actually affected + +The triage process must be documented, reviewed, and evidenced as sufficient and consistently followed. +Documentation must make clear how prioritisation, assignment, and rejection (e.g., for duplicates) are handled, and how mitigations are tracked to completion in a timely manner appropriate to the project's claims and the issues discovered. + +Field incidents are a key source of high-priority Misbehaviours. +These require additional rigour to ensure appropriate and timely responses. +For every iteration and associated change, related issue resolutions must be documented with their impact (e.g., whether new Misbehaviours were found or parts of the analysis had to be redone) and linked to the specific change, ensuring visible traceability. +This information must remain available to support decision traceability throughout the project's lifetime (as considered in TA-DATA). + +As part of ongoing monitoring, the rate of incoming, resolved, and rejected issues across the project and its dependencies should be tracked for trends and anomalies, to identify shifts and to detect if a source of information is lost. + +**Evidence** + +- List of known bugs fixed since last release + - **Answer**: Provided in JLS-29 +- List of outstanding bugs still not fixed, with triage/prioritisation based + on severity/relevance/impact + - **Answer**: Provided in JLS-28 and JLS-11 +- List of known vulnerabilities fixed since last release + - **Answer**: Provided in JLS-29 +- List of outstanding known vulnerabilities still not fixed, with triage/prioritisation based + on severity/relevance/impact + - **Answer**: Provided in JLS-30, JLS-33 and AOU-29 +- List of XYZ component versions, showing where a newer version exists upstream + - **Answer**: Not relevant since nlohmann/json has no external components, as stated in JLS-34 +- List of component version updates since last release + - **Answer**: Not relevant as nlohmann/json has no external components, as stated in JLS-34 +- List of fixes applied to developed code since last release + - **Answer**: Provided in JLS-29 +- List of fixes for developed code that are outstanding, not applied yet + - **Answer**: Provided in JLS-11 +- List of XYZ faults outstanding (O) + - **Answer**: Provided in JLS-11 +- List of XYZ faults fixed since last release (F) + - **Answer**: Provided in JLS-29 +- List of XYZ faults mitigated since last release (M) + - **Answer**: Provided in JLS-29 + +**Confidence scoring** + +Confidence scoring for TA-FIXES can be based on + +- some function of [O, F, M] for XYZ +- number of outstanding relevant bugs from components +- bug triage results, accounting for undiscovered bugs +- number of outstanding known vulnerabilities +- triage results of publicly disclosed vulnerabilities, accounting for undiscovered bugs and vulnerabilities +- confidence that known fixes have been applied +- confidence that known mitigations have been applied +- previous confidence score for TA-FIXES + +Each iteration, we should improve the algorithm based on measurements + +**Checklist** + +- How many faults have we identified in XYZ? + - **Answer**: +- How many unknown faults remain to be found, based on the number that have + been processed so far? + - **Answer**: +- Is there any possibility that people could be motivated to manipulate the + lists (e.g. bug bonus or pressure to close). + - **Answer**: +- How many faults may be unrecorded (or incorrectly closed, or downplayed)? + - **Answer**: +- How do we collect lists of bugs and known vulnerabilities from components? + - **Answer**: +- How (and how often) do we check these lists for relevant bugs and known vulnerabilities? + - **Answer**: +- How confident can we be that the lists are honestly maintained? + - **Answer**: +- Could some participants have incentives to manipulate information? + - **Answer**: +- How confident are we that the lists are comprehensive? + - **Answer**: +- Could there be whole categories of bugs/vulnerabilities still undiscovered? + - **Answer**: +- How effective is our triage/prioritisation? + - **Answer**: +- How many components have never been updated? + - **Answer**: +- How confident are we that we could update them? + - **Answer**: +- How confident are we that outstanding fixes do not impact our Expectations? + - **Answer**: +- How confident are we that outstanding fixes do not address Misbehaviours? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-INDICATORS.md b/TSF/trustable/assertions/TA-INDICATORS.md index d97301176f..88dac1da9b 100644 --- a/TSF/trustable/assertions/TA-INDICATORS.md +++ b/TSF/trustable/assertions/TA-INDICATORS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-INDICATORS-CHECKLIST.md --- In eclipse-score/inc_nlohmann_json, advanced warning indicators for misbehaviours are identified, and monitoring mechanisms are specified, verified and validated based on analysis. \ No newline at end of file diff --git a/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md b/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md new file mode 100644 index 0000000000..20ed4d193e --- /dev/null +++ b/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md @@ -0,0 +1,104 @@ +--- +level: 1.1 +normative: false +--- + +Not all deviations from Expected Behaviour can be associated with a specific +condition. Therefore, we must have a strategy for managing deviations that +arise from unknown system states, process vulnerabilities or configurations. + +This is the role of _Advanced Warning Indicators (AWI)_. These are specific +metrics which correlate with deviations from Expected Behaviour and can be +monitored in real time. The system should return to a defined known-good +state when AWIs exceed defined tolerances. + +**Guidance** + +This assertion is met to the extent that: + +- We have identified indicators that are strongly correlated with observed + deviations from Expected Behaviour in testing and/or production. +- The system returns to a defined known-good state when AWIs exceed + defined tolerances. +- The mechanism for returning to the known-good state is verified. +- The selection of Advance Warning Indicators is validated against the set of + possible deviations from Expected behaviour. + +Note, the set of possible deviations from Expected behaviour _is not_ the same +as the set of Misbehaviours identified in TA-MISBEHAVIOURS, as it includes +deviations due to unknown causes. + +Deviations are easily determined by negating recorded Expectations. Potential +AWIs could be identified using source code analysis, risk analysis or incident +reports. A set of AWIs to be used in production should be identified by +monitoring candidate signals in all tests (functional, soak, stress) and +measuring correlation with deviations. + +Telematics, diagnostics, or manual proof testing are of little value without +mitigation. As such, AWI monitoring and mitigation should be automatic, +traceable back to analysis, and formally recorded to ensure information from +previously unidentified misbehaviours is captured in a structured way. + +The known-good state should be chosen with regard to the system's intended +consumers and/or context. Canonical examples are mechanisms like reboots, +resets, relaunches and restarts. The mechanism for returning to a known-good +state can be verified using fault induction tests. Incidences of AWIs triggering +a return to the known-good state in either testing or production should be +considered as a Misbehaviour in TA-MISBEHAVIOURS. Relying on AWIs alone is not +an acceptable mitigation strategy. TA-MISBEHAVIOURS and TA-INDICATORS are +treated separately for this reason. + +The selection of AWIs can be validated by analysing failure data. For instance, +a high number of instances of deviations with all AWIs in tolerance implies the +set of AWIs is incorrect, or the tolerance is too lax. + +**Evidence** + +- Risk analyses + - **Answer**: +- List of advance warning indicators + - **Answer**: +- List of Expectations for monitoring mechanisms + - **Answer**: +- List of implemented monitoring mechanisms + - **Answer**: +- List of identified misbehaviours without advance warning indicators + - **Answer**: +- List of advance warning indicators without implemented monitoring mechanisms + - **Answer**: +- Advance warning signal data as time series (see TA-DATA) + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-INDICATORS is based on confidence that the list of +indicators is comprehensive / complete, that the indicators are useful, and that +monitoring mechanisms have been implemented to collect the required data. + +**Checklist** + +- How appropriate/thorough are the analyses that led to the indicators? + - **Answer**: +- How confident can we be that the list of indicators is comprehensive? + - **Answer**: +- Could there be whole categories of warning indicators still missing? + - **Answer**: +- How has the list of advance warning indicators varied over time? + - **Answer**: +- How confident are we that the indicators are leading/predictive? + - **Answer**: +- Are there misbehaviours that have no advance warning indicators? + - **Answer**: +- Can we collect data for all indicators? + - **Answer**: +- Are the monitoring mechanisms used included in our Trustable scope? + - **Answer**: +- Are there gaps or trends in the data? + - **Answer**: +- If there are gaps or trends, are they analysed and addressed? + - **Answer**: +- Is the data actually predictive/useful? + - **Answer**: +- Are indicators from code, component, tool, or data inspections taken into +consideration? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-INPUTS.md b/TSF/trustable/assertions/TA-INPUTS.md index 93013745c1..30cedaba34 100644 --- a/TSF/trustable/assertions/TA-INPUTS.md +++ b/TSF/trustable/assertions/TA-INPUTS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-INPUTS-CHECKLIST.md --- All inputs to the nlohmann/json library are assessed, to identify potential risks and issues. diff --git a/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md b/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md new file mode 100644 index 0000000000..7721a6a705 --- /dev/null +++ b/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md @@ -0,0 +1,117 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +Anything that can influence the output of the XYZ project is considered an input. +This includes: + +- Software components used to implement specified features and meet defined Expectations +- Software tools, and their outputs, used for design, construction and verification +- Infrastructure that supports development and release processes + +All inputs (components, tools, data) and their dependencies (recursively) used to build and verify XYZ releases must be identified and assessed, since they are untrusted by default. + +Each input should be evaluated on verifiable merits, regardless of any claims it makes (including adherence to standards or guidance). +Evaluation must include the project's defined Expectations to ensure that inputs meet requirements, and that risks are recorded and addressed appropriately. + +For components, we need to consider how their misbehaviour might impact achieving project XYZ's Expectations. +Sources (e.g. bug databases, advisories) for known risks should be identified, their update frequency recorded, and tests defined for detecting them. +These form the inputs to TA-FIXES. + +For the tools used to construct and verify XYZ, we need to consider how their misbehaviour could: + +- Introduce unintended changes +- Fail to detect Misbehaviours during testing +- Produce misleading data used to design or verify the next iteration + +Where any input impacts are identified, consider: + +- How serious their impact might be, and whether Expectations or analysis outcomes are affected (severity) +- Whether they are detected by another tool, test, or manual check (detectability) + +Confidence in assessing severity and detectability can be supported by analysing development history and practices of each input to evaluate upstream sources (both third-party and first-party) for maintainability and sustainability (including, for example, testability, modularity and configurability) to reduce failure impact and support safe change. + +These qualities can be estimated through evidence of software engineering best practice, applied through: + +- Processes defining and following design, documentation and review guidelines, carried out manually (advocating simple design, reuse, structured coding constructs, and competent release management) +- Appropriate use of programming languages and their features, supported by tools such as static analysis, with regular improvement of their configurations + +For impacts with high severity or low detectability (or both), additional analysis should assess whether existing tests effectively detect Misbehaviours and their impacts. + +As a result, for example, any binary inputs without reproducible build steps or clear development history and maintenance processes should be treated as risks and mitigated appropriately. + +**Evidence** + +- List of components used to build XYZ, including: + - Whether content is provided as source or binary + - **Answer**: +- Record of component assessments: + - Originating project and version + - **Answer**: + - Date of assessments and identity of assessors + - **Answer**: + - Role of component in XYZ + - **Answer**: + - Sources of bug and risk data + - **Answer**: + - Potential misbehaviours and risks identified and assessed + - **Answer**: +- List of tools used to build and verify XYZ + - **Answer**: +- Record of tool assessments: + - Originating project and tool version + - **Answer**: + - Date of assessments and identity of assessors + - **Answer**: + - Role of the tool in XYZ releases + - **Answer**: + - Potential misbehaviours and impacts + - **Answer**: + - Detectability and severity of impacts + - **Answer**: +- Tests or measures to address identified impacts + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-INPUTS is based on the set of components and tools +identified, how many of (and how often) these have been assessed for their risk +and impact for XYZ, and the sources of risk and issue data identified. + +**Checklist** + +- Are there components that are not on the list? + - **Answer**: +- Are there assessments for all components? + - **Answer**: +- Has an assessment been done for the current version of the component? + - **Answer**: +- Have sources of bug and/or vulnerability data been identified? + - **Answer**: +- Have additional tests and/or Expectations been documented and linked to + component assessment? + - **Answer**: +- Are component tests run when integrating new versions of components? + - **Answer**: +- Are there tools that are not on the list? + - **Answer**: +- Are there impact assessments for all tools? + - **Answer**: +- Have tools with high impact been qualified? + - **Answer**: +- Were assessments or reviews done for the current tool versions? + - **Answer**: +- Have additional tests and/or Expectations been documented and linked to + tool assessments? + - **Answer**: +- Are tool tests run when integrating new versions of tools? + - **Answer**: +- Are tool and component tests included in release preparation? + - **Answer**: +- Can patches be applied, and then upstreamed for long-term maintenance? + - **Answer**: +- Do all dependencies comply with acceptable licensing terms? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-ITERATIONS.md b/TSF/trustable/assertions/TA-ITERATIONS.md index 378dbda92b..eeb5414204 100644 --- a/TSF/trustable/assertions/TA-ITERATIONS.md +++ b/TSF/trustable/assertions/TA-ITERATIONS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-ITERATIONS-CHECKLIST.md --- All constructed iterations of the nlohmann/json library include source code, build instructions, tests, results and attestations. diff --git a/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md b/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md new file mode 100644 index 0000000000..fab30edf85 --- /dev/null +++ b/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md @@ -0,0 +1,76 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is best satisfied by checking generated documentation to confirm that: + +- every iteration is a working product with evidence-backed, falsifiable Statements, together with documentation of confidence in those Statements and all required Trustable Statements. +- every iteration includes instructions for building and using the product +- all components, dependencies, tools, and data are identified in a manifest +- the manifest provides links to source code +- where source code is unavailable, the supplier is identified + +An iteration consists of each batch of changes accepted into the canonical version of the product. +How the canonical version is managed must be documented (for TT-CHANGES) alongside the product's Expectations. + +Every iteration must be usable as a standalone product, with verification and validation completed so that a hotfix could be released at any point. +Documentation generated alongside the product must include build and usage guidance together with the project's documented Expectations and supporting Statements, enabling any maintainer or user to reverify the state of the product and associated Statements. + +For each iteration, any changes must be accompanied by attestations and reasoning, explaining the tests performed and the review steps taken, together with their outcomes (e.g., results of source code inspections). +Any attestations and impact assessments must be traceable to the specific changes, authors, reviewers, and the review process documentation used. + +Collating and making available all appropriate data and documentation for every iteration must be automatable, so that the product's build can be reproduced and its analysis repeated end-to-end independently (best achieved using generated documentation and configuration as code). +All relevant data, including approval statuses and dates, must be stored long-term and analysed as part of TA-DATA. +For complex systems, the resulting information must be presented in a user-friendly, searchable, and accessible form. + +Given such transparent documentation and attestations for every iteration, it becomes possible to analyse product and development trends over time. +For releases, additional documentation should summarise all changes across the iterations since the previous release. + +**Evidence** + +- list of components with source + - source code + - **Answer**: + - build instructions + - **Answer**: + - test code + - **Answer**: + - test results summary + - **Answer**: + - attestations + - **Answer**: + +- list of components where source code is not available + - risk analysis + - **Answer**: + - attestations + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-ITERATIONS based on + +- number and importance of source components +- number and importance of non-source components +- assessment of attestations + +**Checklist** + +- How much of the software is provided as binary only, expressed as a + fraction of the BoM list? + - **Answer**: +- How much is binary, expressed as a fraction of the total storage footprint? + - **Answer**: +- For binaries, what claims are being made and how confident are we in the + people/organisations making the claims? + - **Answer**: +- For third-party source code, what claims are we making, and how confident + are we about these claims? + - **Answer**: +- For software developed by us, what claims are we making, and how confident + are we about these claims? + - **Answer**: + diff --git a/TSF/trustable/assertions/TA-METHODOLOGIES.md b/TSF/trustable/assertions/TA-METHODOLOGIES.md index 4773814954..cc3e2e5dbf 100644 --- a/TSF/trustable/assertions/TA-METHODOLOGIES.md +++ b/TSF/trustable/assertions/TA-METHODOLOGIES.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-METHODOLOGIES-CHECKLIST.md --- Manual methodologies applied for the nlohmann/json library by contributors, and their results, are managed according to specified objectives. diff --git a/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md b/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md new file mode 100644 index 0000000000..c1c1f4bb7b --- /dev/null +++ b/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md @@ -0,0 +1,72 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +To satisfy this assertion, all manual processes used in the verification of XYZ must be documented, including the methodologies applied, the results for specific aspects and iterations, and evidence that these processes were reviewed against documented criteria. + +Most analysis (e.g., data analysis for TA-ANALYSIS) should be automated to enable continuous feedback. +However, the quality of any remaining manual processes (whether from first parties or external third parties) must be considered and how they are documented and reviewed. +Considerations should be made about how manual processes may impact identifying and addressing Misbehaviours (TA-MISBEHAVIOURS). + +Assignment of responsibilities for any manual work must follow a documented process that verifies competence and grants appropriate access, with automation applied where possible. +Resulting assigned responsibilities must ensure organisational robustness (e.g., avoidance of conflicts of interest) together with appropriate independent verification and validation. +Manual reviews involving source inspections must follow documented guidelines, with exceptions recorded and illustrated through examples. +These guidelines should evolve over time and cover: + +- coding patterns (e.g., good patterns, anti-patterns, defensive coding) +- structured design practices (e.g., control flow constraints) +- complexity management (e.g., limiting feature creep) +- documentation (e.g., clear, formal figures and diagrams) +- feature subset restrictions (e.g., programming language subsets) +- code of conduct guidelines (e.g., review etiquette, handling disagreements) + +Nevertheless, specific coding rules (e.g., memory allocation, typing, concurrency) should be integrated into automatic linting and static analysis tools where appropriate. + +All processes and checks must themselves be reviewed to drive continuous improvement following specified guidelines. +Any resulting changes from reviews must follow change control, regardless of who initiates them or under what circumstances. + +**Evidence** + +- Manual process documentation + - **Answer**: +- References to methodologies applied as part of these processes + - **Answer**: +- Results of applying the processes + - **Answer**: +- Criteria used to confirm that the processes were applied correctly + - **Answer**: +- Review records for results + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-METHODOLOGIES is based on identifying areas of need for +manual processes, assessing the clarity of proposed processes, analysing the +results of their implementation, and evaluating the evidence of effectiveness +in comparison to the analysed results + +**Checklist** + +- Are the identified gaps documented clearly to justify using a manual process? + - **Answer**: +- Are the goals for each process clearly defined? + - **Answer**: +- Is the sequence of procedures documented in an unambiguous manner? + - **Answer**: +- Can improvements to the processes be suggested and implemented? + - **Answer**: +- How frequently are processes changed? + - **Answer**: +- How are changes to manual processes communicated? + - **Answer**: +- Are there any exceptions to the processes? + - **Answer**: +- How is evidence of process adherence recorded? + - **Answer**: +- How is the effectiveness of the process evaluated? + - **Answer**: +- Is ongoing training required to follow these processes? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-MISBEHAVIOURS.md b/TSF/trustable/assertions/TA-MISBEHAVIOURS.md index e052a440bc..ff635ded93 100644 --- a/TSF/trustable/assertions/TA-MISBEHAVIOURS.md +++ b/TSF/trustable/assertions/TA-MISBEHAVIOURS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-MISBEHAVIOURS-CHECKLIST.md --- Prohibited misbehaviours for the nlohmann/json library are identified, and mitigations are specified, verified and validated based on analysis. diff --git a/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md b/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md new file mode 100644 index 0000000000..20fb520872 --- /dev/null +++ b/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md @@ -0,0 +1,138 @@ +--- +level: 1.1 +normative: false +--- + +The goal of TA-MISBEHAVIOURS is to force engineers to think critically about their work. +This means understanding and mitigating as many of the situations that cause the +software to deviate from Expected Behaviours as possible. This is not limited to +the contents of the final binary. + +**Guidance** + +This assertion is satisfied to the extent that we can: + +- Show we have identified all of the ways in which XYZ could deviate from its + Expected Behaviours. +- Demonstrate that mitigations have been specified, verified and validated for + all Misbehaviours. + +Once Expected Behaviours have been identified in TA-BEHAVIOURS, there are at +least four classes of Misbehaviour that can be identified: + +- Reachable vulnerable system states that cause deviations from Expected + Behaviour. These can be identified by stress testing, failures in functional + and soak testing in TA-BEHAVIOURS and reporting in TA-FIXES. Long run trends + in both test and production data should also be used to identify these states. +- Potentially unreachable vulnerable system states that could lead to deviations + from Expected Behaviour. These can be identified using risk/hazard analysis + techniques including HAZOP, FMEDA and STPA. +- Vulnerabilities in the development process that could lead to deviations from + Expected Behaviour. This includes those that occur as a result of misuse, + negligence or malicious intent. These can be identified by incident + investigation, random sampling of process artifacts and STPA of processes. +- Configurations in integrating projects (including the computer or embedded + system that is the final product) that could lead to deviations from Expected + Behaviour. + +Identified Misbehaviours must be mitigated. Mitigations include patching, +re-designing components, re-designing architectures, removing components, +testing, static analysis etc. They explicitly _do not_ include the use of AWIs +to return to a known-good state. These are treated specifically and in detail in +TA-INDICATORS. + +Mitigations could be verified by: + +- Specifying and repeatedly executing false negative tests to confirm that + functional tests detect known classes of misbehaviour. +- Specifying fault induction tests or stress tests to demonstrate that the + system continues providing the Expected Behaviour after entering a vulnerable + system state. +- Performing statistical analysis of test data, including using statistical path + coverage to demonstrate that the vulnerable system state is never reached. +- Conducting fault injections in development processes to demonstrate that + vulnerabilities cannot be exploited (knowingly or otherwise) to affect either + output binaries or our analysis of it, whether this is by manipulating the + source code, build environment, test cases or any other means. +- Stress testing of assumptions of use. That is, confirming assumptions of use + are actually consistent with the system and its Expected Behaviours by + intentionally misinterpreting or liberally interpreting them in a test + environment. For example, we could consider testing XYZ on different pieces of + hardware that satisfy its assumptions of use. + +Remember that a Misbehaviour is _anything_ that could lead to a deviation from +Expected Behaviour. The specific technologies in and applications of XYZ should +always be considered in addition to the guidance above. + +At the core, a faulty design is inherently difficult to mitigate. The first +priority, therefore, is to ensure a fault-tolerant and fault-avoidant design +that minimises fault impact and maximises fault control across all modes and +states. All design considerations should be traceable to analyses at the correct +abstraction level, with appropriate partitioning and scoping, which address +prevalent aspects in complex systems, such as: + +- Spatial constraints (e.g., memory corruption) +- Temporal constraints (e.g., timing violations) +- Concurrency constraints (e.g., interference) +- Computational constraints (e.g., precision limits) +- Performance constraints (e.g., latency spikes under load) +- Environmental constraints (e.g., hardware non-determinism) +- Usability constraints (e.g., human interaction errors) + +Finally, each new Expectation, whether a required behaviour or a misbehaviour +mitigation, introduces the potential for unexpected emergent properties, +highlighting the importance of simple, understandable designs that build on +established and reusable solutions. + +**Suggested evidence** + +- List of identified Misbehaviours + - **Answer**: +- List of Expectations for mitigations addressing identified Misbehaviours + - **Answer**: +- Risk analysis + - **Answer**: +- Test analysis, including: + - False negative tests + - **Answer**: + - Exception handling tests + - **Answer**: + - Stress tests + - **Answer**: + - Soak tests + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-MISBEHAVIOURS is based on confidence that +identification and coverage of misbehaviours by tests is complete when +considered against the list of Expectations. + +**Checklist** + +- How has the list of misbehaviours varied over time? + - **Answer**: +- How confident can we be that this list is comprehensive? + - **Answer**: +- How well do the misbehaviours map to the expectations? + - **Answer**: +- Could some participants have incentives to manipulate information? + - **Answer**: +- Could there be whole categories of misbehaviours still undiscovered? + - **Answer**: +- Can we identify misbehaviours that have been understood but not specified? + - **Answer**: +- Can we identify some new misbehaviours, right now? + - **Answer**: +- Is every misbehaviour represented by at least one fault induction test? + - **Answer**: +- Are fault inductions used to demonstrate that tests which usually pass can + and do fail appropriately? + - **Answer**: +- Are all the fault induction results actually collected? + - **Answer**: +- Are the results evaluated? + - **Answer**: +- Do input analysis findings on verifiable tool or component claims and features +identify additional misbehaviours or support existing mitigations? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-RELEASES.md b/TSF/trustable/assertions/TA-RELEASES.md index 5cf9412fe9..85e6f71225 100644 --- a/TSF/trustable/assertions/TA-RELEASES.md +++ b/TSF/trustable/assertions/TA-RELEASES.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-RELEASES-CHECKLIST.md --- Construction of releases for the nlohmann/json library is fully repeatable and the results are fully reproducible, with any exceptions documented and justified. diff --git a/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md b/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md new file mode 100644 index 0000000000..8e2a528595 --- /dev/null +++ b/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md @@ -0,0 +1,79 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied if each iteration of XYZ is *repeatable*, with all +required inputs controlled, and *reproducible* (covering both XYZ and the +construction toolchain/environment, as described in TA-TESTS). + +This assertion can be most effectively satisfied in a Continuous Integration +environment with mirrored projects (see TA-SUPPLY_CHAIN) and build servers +without internet access. The aim is to show that all build tools, XYZ +components, and dependencies are built from controlled inputs, that rebuilding +produces the same binary fileset, and that builds can be repeated on any +suitably configured server, with server differences shown not to affect +reproducibility. + +For releases in particular, builds from source must be shown to produce +identical outputs both with and without cache access. + +Again this will not be achievable for components/tools provided in binary form, +or accessed via an external service - we must consider our confidence in +attestations made by/for the supply chain. + +All non-reproducible elements, such as timestamps or embedded random values from +build metadata, are clearly identified and considered when evaluating +reproducibility. + +As a result, we gain increased confidence that the toolchain behaves correctly +during version upgrades: unintended changes to the project are avoided, intended +fixes produce the expected effects, and the constructed output of XYZ shows the +correct behavioural changes, verified and validated with test results according +to TT-RESULTS analysis. + +**Evidence** + +- list of reproducible SHAs + - **Answer**: +- list of non-reproducible elements with: + - explanation and justification + - **Answer**: + - details of what is not reproducible + - **Answer**: +- evidence of configuration management for build instructions and infrastructure + - **Answer**: +- evidence of repeatable builds + - **Answer**: + +**Confidence scoring** + +Calculate: + +R = number of reproducible components (including sources which have no build stage) +N = number of non-reproducible +B = number of binaries +M = number of mirrored +X = number of things not mirrored + +Confidence scoring for TA-RELEASES could possibly be calculated as +R / (R + N + B + M / (M + X)) + +**Checklist** + +- How confident are we that all components are taken from within our + controlled environment? + - **Answer**: +- How confident are we that all of the tools we are using are also under our + control? + - **Answer**: +- Are our builds repeatable on a different server, or in a different context? + - **Answer**: +- How sure are we that our builds don't access the internet? + - **Answer**: +- How many of our components are non-reproducible? + - **Answer**: +- How confident are we that our reproducibility check is correct? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-SUPPLY_CHAIN.md b/TSF/trustable/assertions/TA-SUPPLY_CHAIN.md index e585a17752..62f675ec57 100644 --- a/TSF/trustable/assertions/TA-SUPPLY_CHAIN.md +++ b/TSF/trustable/assertions/TA-SUPPLY_CHAIN.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-SUPPLY_CHAIN-CHECKLIST.md --- All sources and tools for the nlohmann/json library are mirrored in our controlled environment. diff --git a/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md b/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md new file mode 100644 index 0000000000..48388f0cb6 --- /dev/null +++ b/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md @@ -0,0 +1,67 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied to the extent that we have traced and captured +source code for XYZ and all of its dependencies (including transitive +dependencies, all the way down), and for all of the tools used to construct +XYZ from source, and have mirrored versions of these inputs under our control. +Any associated data and documentation dependencies must also be considered. + +'Mirrored' in this context means that we have a version of the upstream project +that we keep up-to-date with additions and changes to the upstream project, +but which is protected from changes that would delete the project, or remove +parts of its history. + +Clearly this is not possible for components or tools (or data) that are provided only in +binary form, or accessed via online services - in these circumstances we can +only assess confidence based on attestations made by the suppliers, and on our +experience with the suppliers' people and processes. + +Keep in mind that even if repositories with source code for a particular +component or tool are available, not all of it may be stored in Git as +plaintext. A deeper analysis is required in TA-INPUTS to assess the impact of any +binaries present within the repositories of the components and tools used. + +**Evidence** + +- list of all XYZ components including + - URL of mirrored projects in controlled environment + - **Answer**: + - URL of upstream projects + - **Answer**: +- successful build of XYZ from source + - without access to external source projects + - **Answer**: + - without access to cached data + - **Answer**: +- update logs for mirrored projects + - **Answer**: +- mirrors reject history rewrites + - **Answer**: +- mirroring is configured via infrastructure under direct control + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-SUPPLY_CHAIN is based on confidence that all inputs and +dependencies are identified and mirrored, and that mirrored projects cannot be +compromised. + +**Checklist** + +- Could there be other components, missed from the list? + - **Answer**: +- Does the list include all toolchain components? + - **Answer**: +- Does the toolchain include a bootstrap? + - **Answer**: +- Could the content of a mirrored project be compromised by an upstream change? + - **Answer**: +- Are mirrored projects up to date with the upstream project? + - **Answer**: +- Are mirrored projects based on the correct upstream? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-TESTS.md b/TSF/trustable/assertions/TA-TESTS.md index 7269be53e3..f1b8497595 100644 --- a/TSF/trustable/assertions/TA-TESTS.md +++ b/TSF/trustable/assertions/TA-TESTS.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-TESTS-CHECKLIST.md --- All tests for the nlohmann/json library, and its build and test environments, are constructed from controlled/mirrored sources and are reproducible, with any exceptions documented. diff --git a/TSF/trustable/assertions/TA-TESTS_CONTEXT.md b/TSF/trustable/assertions/TA-TESTS_CONTEXT.md new file mode 100644 index 0000000000..2066e8addd --- /dev/null +++ b/TSF/trustable/assertions/TA-TESTS_CONTEXT.md @@ -0,0 +1,52 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied if all build and test environments and tools used to support Expectations are shown to be +*reproducible*, all build and test steps are *repeatable*, and all required inputs are controlled. +TA-TESTS does not include reproducibility of XYZ itself, this is instead included in TA-RELEASES. + +All tools and test environments should be constructed from change-managed sources (see TA-UPDATES) and mirrored +sources (see TA-SUPPLY_CHAIN). +Additional evidence needs to demonstrate that construction of tools and environments produces the same binary fileset +used for testing and that builds can be repeated on any suitably configured server +(similar to how the XYZ is evaluated for TA-RELEASES). + +Test environment repeatability should be ensured to enable effective Misbehaviour investigations, and enable additional +data generations (including those by third parties). +To achieve repeatability, all infrastructure, hardware, and configurations must be identified and documented for all +test environments. Storage of this information is evaluated in TA-DATA, and its availability is considered in +TA-ITERATIONS. + +**Evidence** + +- Test build environment reproducibility + - **Answer**: +- Test build configuration + - **Answer**: +- Test build reproducibility + - **Answer**: +- Test environment configuration + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-TESTS is based on confidence that the construction and deployment of test environments, +tooling and their build environments are repeatable and reproducible. + +**CHECKLIST** + +- How confident are we that our test tooling and environment setups used for tests, fault inductions, and analyses are reproducible? + - Are any exceptions identified, documented and justified? + - **Answer**: +- How confident are we that all test components are taken from within our controlled environment? + - **Answer**: +- How confident are we that all of the test environments we are using are also under our control? + - **Answer**: +- Do we record all test environment components, including hardware and infrastructure used for exercising tests and processing input/output data? + - **Answer**: +- How confident are we that all tests scenarios are repeatable? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-UPDATES.md b/TSF/trustable/assertions/TA-UPDATES.md index 0113c23a4f..c54015ba90 100644 --- a/TSF/trustable/assertions/TA-UPDATES.md +++ b/TSF/trustable/assertions/TA-UPDATES.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-UPDATES-CHECKLIST.md --- nlohmann/json library components, configurations and tools are updated under specified change and configuration management controls. diff --git a/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md b/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md new file mode 100644 index 0000000000..b3f2612fed --- /dev/null +++ b/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md @@ -0,0 +1,65 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion requires control over all changes to XYZ, including configurations, components, tools, data, documentation, and dependency versions used to build, verify, and validate it. + +As part of change control, all automated checks must run and pass (e.g., tests, static analysis, lint checks) before accepting proposed changes. +These checks must be configured against appropriate claims and coding guidelines. +Where a change affects tracked claims, the impact must be identified, reasoned, and verified, with linked analysis performed (e.g., input analysis for new dependencies as per TA-INPUTS). +Even changes with no direct impact on project claims must be justified. + +Multiple roles (assigned to appropriate parties under suitable guidelines) should be involved in assessing changes. +Reviews must focus on the integrity and consistency of claims, the software, and its tests. +What each reviewer did or did not examine must be recorded, and this information (together with all checks) made available for every change throughout the project lifecycle (see TA-DATA). +Details of manual quality management aspects are addressed in TA-METHODOLOGIES. + +As a result, all changes must be regression-free (blocking problematic changes until resolved) and aim to exhibit the following properties: + +- simple +- atomic +- modular +- understandable +- testable +- maintainable +- sustainable + +Practices that enforce these properties help identify and resolve inconsistent changes early in development. + +Change control itself must not be subverted, whether accidentally or maliciously. +Process documentation, guidance, and automated checks must also be under change control, approved by appropriate parties, and protected with suitable security controls. + +To prevent regressions and reduce the rate of bugs and vulnerabilities, consistent dependency updates must be applied and new issues promptly addressed (TA-FIXES). +Evidence for each iteration must demonstrate that change control requirements are applied consistently and evolve as improvements are identified, ensuring the process remains repeatable and reproducible. +Timeliness must be monitored across detection, resolution, and deployment, with automation and process improvements introduced where delays are found. + +Ultimately, the trustable controlled process is the only path to production for the constructed target software. + +**Evidence** + +- change management process and configuration artifacts + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-UPDATES is based on confidence that we have +control over the changes that we make to XYZ, including its configuration and +dependencies. + +**Checklist** + +- Where are the change and configuration management controls specified? + - **Answer**: +- Are these controls enforced for all of components, tools, data, documentation and configurations? + - **Answer**: +- Are there any ways in which these controls can be subverted, and have we mitigated them? + - **Answer**: +- Does change control capture all potential regressions? + - **Answer**: +- Is change control timely enough? + - **Answer**: +- Are all guidance and checks understandable and consistently followed? + - **Answer**: diff --git a/TSF/trustable/assertions/TA-VALIDATION.md b/TSF/trustable/assertions/TA-VALIDATION.md index e10fc5302b..bffb735373 100644 --- a/TSF/trustable/assertions/TA-VALIDATION.md +++ b/TSF/trustable/assertions/TA-VALIDATION.md @@ -1,9 +1,6 @@ --- level: 1.1 normative: true -references: - - type: checklist - path: ./TSF/trustable/assertions/TA-VALIDATION-CHECKLIST.md --- All specified tests are executed repeatedly, under defined conditions in controlled environments, according to specified objectives. (To revisit) \ No newline at end of file diff --git a/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md b/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md new file mode 100644 index 0000000000..e06f821d8a --- /dev/null +++ b/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md @@ -0,0 +1,69 @@ +--- +level: 1.1 +normative: false +--- + +**Guidance** + +This assertion is satisfied when tests demonstrate that the features specified to meet project Expectations (TT-EXPECTATIONS) are present and function as intended. +These tests run repeatedly in a controlled environment (TA-TESTs) on a defined schedule (e.g., daily, per change, or per candidate release of XYZ). + +Confidence grows when tests not only verify Expectations but also validate (continuously) that they meet stakeholder and user needs. +Robust validation depends on three aspects: + +- TA-VALIDATION – a strategy that produces representative and stressing data. +- TA-DATA – appropriate handling of collected data. +- TA-ANALYSIS – analysis methods that remain dependable as the project evolves. + +This structure enables iterative convergence toward required behaviours, even when early validation results are unsatisfactory. + +A strategy to generate appropriate data addresses quantity, quality, and selection: + +- Selection: Testing remains exploratory, combining monitoring with verified and new indicators (supporting TA-INDICATORS). + Coverage spans input, design, and output analysis with traceable specifications and results (considering TA-BEHAVIOURS). + Tests also support calibration of capacity, scalability, response time, latency, and throughput, executed in targetted conditions and under stress (e.g., equivalence class and boundary-value testing). +- Quantity: Automation scheduling provides sufficient repetition and covers diverse environments (e.g., multiple hardware platforms). + Failures block merge requests, with pre- and post-merge tests giving fast feedback. + Adequacy of data is assessed through TA-ANALYSIS. +- Quality: Test suites include fault induction (considering TA-MISBEHAVIOURS) and checks that good data yields good results while bad data yields bad results. + +**Evidence** + +- Test results from per-change tests + - **Answer**: +- Test results from scheduled tests as time series + - **Answer**: + +**Confidence scoring** + +Confidence scoring for TA-VALIDATION is based on verification that we have +results for all expected tests (both pass / fail and performance). + +**Checklist** + +- Is the selection of tests correct? + - **Answer**: +- Are the tests executed enough times? + - **Answer**: +- How confident are we that all test results are being captured? + - **Answer**: +- Can we look at any individual test result, and establish what it relates to? + - **Answer**: +- Can we trace from any test result to the expectation it relates to? + - **Answer**: +- Can we identify precisely which environment (software and hardware) were used? + - **Answer**: +- How many pass/fail results would be expected, based on the scheduled tests? + - **Answer**: +- Do we have all of the expected results? + - **Answer**: +- Do we have time-series data for all of those results? + - **Answer**: +- If there are any gaps, do we understand why? + - **Answer**: +- Are the test validation strategies credible and appropriate? + - **Answer**: +- What proportion of the implemented tests are validated? + - **Answer**: +- Have the tests been verified using known good and bad data? + - **Answer**: From 14e403a860cc6f8c49a62537fa11ba5fe6dae696 Mon Sep 17 00:00:00 2001 From: Erik Hu Date: Wed, 19 Nov 2025 14:45:36 +0100 Subject: [PATCH 3/9] update to trudag v2025.10.22 (#4) * update to trudag v2025.10.22 * upgrade pip * upgrade pip in test_publication workflow * pip install requests --- .devcontainer/S-CORE/Dockerfile | 2 +- .devcontainer/S-CORE/post_create_script.sh | 2 +- .dotstop_extensions/README.md | 4 +++- .dotstop_extensions/data_store.py | 21 +++++++++++++-------- .github/workflows/publish_documentation.yml | 6 ++++-- .github/workflows/test_publication.yml | 6 ++++-- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.devcontainer/S-CORE/Dockerfile b/.devcontainer/S-CORE/Dockerfile index ec8d23cf68..788a65b672 100644 --- a/.devcontainer/S-CORE/Dockerfile +++ b/.devcontainer/S-CORE/Dockerfile @@ -87,5 +87,5 @@ RUN groupadd --gid $USER_GID $USERNAME \ USER $USERNAME # Install trudag using pipx -RUN pipx install git+https://gitlab.com/CodethinkLabs/trustable/trustable@9957f12171cb898d83df5ae708fdba0a38fece2e && \ +RUN pipx install git+https://gitlab.com/CodethinkLabs/trustable/trustable@v2025.10.22 && \ pipx ensurepath diff --git a/.devcontainer/S-CORE/post_create_script.sh b/.devcontainer/S-CORE/post_create_script.sh index 4c3b6da47a..8b3fd93b12 100644 --- a/.devcontainer/S-CORE/post_create_script.sh +++ b/.devcontainer/S-CORE/post_create_script.sh @@ -5,4 +5,4 @@ source .venv/bin/activate # Install trustable pip install --require-hashes -r .devcontainer/S-CORE/requirements.txt -pip install git+https://gitlab.com/CodethinkLabs/trustable/trustable@9957f12171cb898d83df5ae708fdba0a38fece2e +pip install git+https://gitlab.com/CodethinkLabs/trustable/trustable@v2025.10.22 diff --git a/.dotstop_extensions/README.md b/.dotstop_extensions/README.md index 586ad18bd0..eceaafb271 100644 --- a/.dotstop_extensions/README.md +++ b/.dotstop_extensions/README.md @@ -450,9 +450,11 @@ Since no persistent data store is established as of now, the current implementat The input of the data store are the data generated by the trudag tool during the `score` or `publish` operation. These data have the format: ``` -[{"scores": [{id: "ID-1", "score": score}, ...], "info": {"Repository root": "my_repository", "Commit SHA": "sha_123", "Commit date/time": "%a %b %d %H:%M:%S %Y", "Commit tag": "my_tag", "CI job id": 123, "Schema version": 123, "Branch name": "my_branch"}}] +[{"scores": [{id: "ID-1", "score": score}, ...], "info": {"Repository root": "my_repository", "Commit SHA": "sha_123", "Commit date/time": , "Commit tag": "my_tag", "CI job id": 123, "Schema version": 123, "Branch name": "my_branch"}}] ``` +Note: Starting with trudag v2025.09.16, "Commit date/time" is a unix timestamp (integer) instead of a formatted string. The values for "Commit SHA", "Commit tag", "CI job id", and "Branch name" can also be `None`. + ## push This functionality writes the generated data into an sqlite database `TrustableScoring.db` located in the folder `TSF`. This database contains two tables, `commit_info`, where the metadata of "info" are stored, and `scores`, where the scores are stored, and which references `commit_info` via the date as foreign key. diff --git a/.dotstop_extensions/data_store.py b/.dotstop_extensions/data_store.py index e3e8b25d94..335ac3a707 100644 --- a/.dotstop_extensions/data_store.py +++ b/.dotstop_extensions/data_store.py @@ -32,15 +32,16 @@ def get_my_data() -> list[dict]: command = f"SELECT * FROM scores WHERE date=={info[0]}" cursor.execute(command) scores = cursor.fetchall() - date = datetime.fromtimestamp(info[0]) - date_as_string = date.strftime("%a %b %d %H:%M:%S %Y") + # Return unix timestamp directly for trudag v2025.09.16+ compatibility + # (older versions expected formatted string, newer versions expect int) + date_timestamp = info[0] if len(info) == 6: branch_name = "" else: branch_name = info[6] if info[6]!=None else "" commit = {"Repository root": info[1], "Commit SHA": info[2], - "Commit date/time": date_as_string, + "Commit date/time": date_timestamp, "Commit tag": info[3], "CI job id": info[4], "Schema version": info[5], @@ -71,11 +72,15 @@ def push_my_data(data: list[dict]): # extract data from data info = data[0].get("info") scores = data[0].get("scores") - # Currently, the commit date is stored as string. - # Since the local timezone is used and for comparison, - # it would be better to have it as a unix-timestamp. - datum_string = info.get("Commit date/time") - datum = int(datetime.strptime(datum_string, "%a %b %d %H:%M:%S %Y").timestamp()) + # Starting with trudag v2025.09.16, the commit date is already a unix timestamp (int). + # For backward compatibility, handle both string and int formats. + datum_value = info.get("Commit date/time") + if isinstance(datum_value, str): + # Old format: string date, convert to timestamp + datum = int(datetime.strptime(datum_value, "%a %b %d %H:%M:%S %Y").timestamp()) + else: + # New format: already a unix timestamp + datum = datum_value # check if current commit coincides with existing commit cursor.execute("SELECT MAX(date) AS recent_commit FROM commit_info") if datum == cursor.fetchone()[0]: diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 2fa89d4ac8..0f4f538d99 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -48,12 +48,14 @@ jobs: with: python-version: '3.11.2' - # trudag version 2025.8.5 + # trudag version v2025.10.22 - name: Install trudag and dependencies run: | sudo apt-get update sudo apt-get install -y graphviz - pip install git+https://gitlab.com/CodethinkLabs/trustable/trustable@9957f12171cb898d83df5ae708fdba0a38fece2e + pip install --upgrade pip + pip install PyYAML requests + pip install git+https://gitlab.com/CodethinkLabs/trustable/trustable@v2025.10.22 - name: Install tools run: | diff --git a/.github/workflows/test_publication.yml b/.github/workflows/test_publication.yml index 13da6bd616..c01ee01ae1 100644 --- a/.github/workflows/test_publication.yml +++ b/.github/workflows/test_publication.yml @@ -48,12 +48,14 @@ jobs: with: python-version: '3.11.2' - # trudag version 2025.8.5 + # trudag version v2025.10.22 - name: Install trudag and dependencies run: | sudo apt-get update sudo apt-get install -y graphviz - pip install git+https://gitlab.com/CodethinkLabs/trustable/trustable@9957f12171cb898d83df5ae708fdba0a38fece2e + pip install --upgrade pip + pip install PyYAML requests + pip install git+https://gitlab.com/CodethinkLabs/trustable/trustable@v2025.10.22 - name: Generate trudag report run: | From e13078aa7c3548c731c1a81ec6a59708498ebc8b Mon Sep 17 00:00:00 2001 From: Erik Hu Date: Thu, 20 Nov 2025 11:55:40 +0100 Subject: [PATCH 4/9] TT changes documentation (#7) * answer checklist TT-CHANGES * update TSF/README * add AOU-30 * steps 2-12 * mean of the scores * library -> project * fix typo * additional SMEs * fix comprise typo * nlohmann/json triage process * add JLS-34 to TA_UPDATES context --- .dotstop.dot | 4 +- TSF/README.md | 74 ++++++++++--------- TSF/trustable/assertions/TA-FIXES_CONTEXT.md | 30 ++++---- .../assertions/TA-UPDATES_CONTEXT.md | 14 ++-- TSF/trustable/assumptions-of-use/AOU-30.md | 6 ++ 5 files changed, 70 insertions(+), 58 deletions(-) create mode 100644 TSF/trustable/assumptions-of-use/AOU-30.md diff --git a/.dotstop.dot b/.dotstop.dot index 7e15da1524..c7eb3e288c 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -1,5 +1,5 @@ # This file is automatically generated by dotstop and should not be edited manually. -# Generated using trustable 2025.8.5. +# Generated using trustable 2025.10.22. digraph G { "TT-CHANGES" [sha=e276949659e77f8f453a9b32798f607abdfda44450bb10bfbca7a20d68835f7a]; @@ -31,6 +31,7 @@ digraph G { "AOU-27" [sha="43da5040f2cab643f2bbb4a0abcf67fba91d68108786604f44f6ef6a91f6681a"]; "AOU-28" [sha="76887b683a1e5a2f36a029bf80c7489226d1728e6f148f69f7b2167211f1f443"]; "AOU-29" [sha="cb3302e313dae237d67bf249c61419232a83aa6245af30a0f33c795ab1b3a8f5"]; +"AOU-30" [sha="ee747671350441b0ca2438c651e0c30a90ac9b126540308415e1645eeea2707a"]; "JLEX-01" [sha="1cd16efee6db392909248a77c773e0c5aa76b4c05001ab0b981201004f2162b0"]; "JLEX-02" [sha=f40046dae47252d62f28c935db90460772263ced0fa9ed1a5e1383436b89daa9]; "JLS-01" [sha=dce41089598ceff4a3ce3df7a8d3d16183346b785d791d60c8292f51c055946d]; @@ -405,6 +406,7 @@ digraph G { "TA-CONSTRAINTS" -> "AOU-29" [sha="278f1a8cacd32b255fafdbac1db71a692029a838859bb7c7227435fd85298ab2"]; "TA-CONSTRAINTS" -> "AOU-10" [sha="54e2cf65bcc7544eda66fd5aca9763008a094dd7389db2463b28c81e1c586375"]; "TA-CONSTRAINTS" -> "AOU-11" [sha="376c75481202bdc4c6d8f9073f5173ab307859b0495735b204de05e58ef2742c"]; +"TA-CONSTRAINTS" -> "AOU-30" [sha="28a722163059e712168605d165311d28bbb6e0d88f34e4b8b317f18d47d02843"]; "TA-DATA" -> "JLS-18" [sha=af2567d28552ab32643e995a9c9c9fd73b2208ce3d5255fba333c8ddd28f72a6]; "TA-FIXES" -> "JLS-05" [sha=ed0acae690ee7d09884c0e7516c9d2306c646c7d14423d4b69719e9dfa49dff6]; "TA-FIXES" -> "JLS-04" [sha="873b8bbdac8d45772d9b2d46adbafe7ab75b96a45ca12a15b34bdf813b0bb1df"]; diff --git a/TSF/README.md b/TSF/README.md index 0dc6967d66..6fbc05c1e5 100644 --- a/TSF/README.md +++ b/TSF/README.md @@ -49,11 +49,9 @@ To update either the version of nlohmann/json within S-CORE or TSF documentation ## Assumptions of use -This description of an update process is based on the following structure of the repository WHICH IS NOT THE CASE YET. -It is assumed that the repository possesses a default branch called ``main`` containing the most recent documented version of ``nlohmann/json`` together with its documentation. -Additionally, there is a branch ``develop``, which is **not** intended to mirror the branch of ``nlohmann/json`` with the same name, but instead serves as an in-repository testing ground for changes to either the library or its documentation. -The releases of the documented version are identified by tags on ``main``. -Moreover, the branch protection rules for ``main`` are set as described in the description of the forking process in ``TSF/README.md`` (WIP). +This description of the update process is based on the following structure of the repository. +It is assumed that the repository has a default branch called ``main`` containing the most recent documented version of ``nlohmann/json`` together with its documentation. +The releases of the documented nlohmann/json version are identified by release notes on ``main``. Note that there is **no automatic information** on the existence of a new release in the original ``nlohmann/json``; instead the possibility to update is detected **manually**. Note further that, due to the currently relatively limited use of nlohmann/json within S-CORE, there appears currently no inherent need to keep the version up to date. @@ -62,7 +60,7 @@ Note further that, due to the currently relatively limited use of nlohmann/json The releases of ``nlohmann/json`` are collected on the [Release site](https://github.com/nlohmann/json/releases) of the repository ``nlohmann/json``. Each release announcement is expected to contain the release date, SHA-256 values for json.hpp, include.zip and json.tar.xz, and a brief list containing bug fixes, improvements, further changes and deprecated functions. -The new release is expected to be located within the branch **master**, from where the most recent version can be drawn. +The new release is expected to be located within the branch **master** in nlohmann/json, from where the most recent version can be drawn. ## Update process of the S-CORE version @@ -100,12 +98,11 @@ Moreover, some parts of the documentation must be adapted to the new version. * ``.github/workflows/parent-workflow.yml`` To ensure a specific execution order for the individual github workflows, their execution is orchestrated by the parent-workflow. - To guarantee that this order is respected, it must be ensured that every other workflow except for ``docs-cleanup.yml``, ``scorecards.yml`` and ``stale.yml`` runs ``on workflow_call``, only. - For the three exceptions, it is recommended to keep the execution scheduled as currently the case. + To guarantee that this order is respected, it must be ensured that every workflow except for ``comment_check_amalgamation.yml``, ``docs-cleanup.yml``, ``parent-workflow.yml``, ``scorecards.yml`` and ``stale.yml`` runs ``on workflow_call``, only. * ``.github/workflows/ubuntu.yml`` The ubuntu workflow orchestrates the parallel execution of various cmake targets with varying configurations running on the latest version of ubuntu. - The first adaptation is that every step, in which a junit-report is generated, generates an artifact. + The first TSF related adaptation of this workflow is that every step, in which a junit-report is generated, generates an artifact. It must be ensured, that these artifacts are still generated after the update. The second adaptation is that the test-results are captured, processed and persistently stored or stored in the ubuntu-artifact. Therefore, it must be ensured that the jobs ``publish_test_data_success``, ``publish_test_data_failure``, ``publish_test_data_cancellation`` and ``ubuntu_artifact`` are executed. @@ -118,7 +115,7 @@ Moreover, some parts of the documentation must be adapted to the new version. Currently, this is done by removing it altogether, which we recommend to do so that no confusion as to why this workflow is not executed arises. * ``.github/workflows/publish_documentation.yml`` - This workflow is replaced with a completely customised version, which reflects the use of trudag and the integration into the Eclipse S-CORE organisation. + The original version of this workflow is replaced with a completely customised version, which reflects the use of trudag and the integration into the Eclipse S-CORE organisation. Therefore, it is recommended to not change this workflow. In particular, the version of publish_documentation.yml in the original repository nlohmann/json must not replace the publish_documentation.yml of the present repository. @@ -135,7 +132,7 @@ Moreover, some parts of the documentation must be adapted to the new version. In case that it is determined that these workflows should be deleted also in the documented copy of ``nlohmann/json``, then the validator ``check_artifact_exists`` and all its occurrences must be adapted accordingly. * ``ChangeLog.md`` - It must be ensured that the changes of the update are properly described in the file ``ChangeLog.md``. + It must be ensured that the changes of the new release of nlohmann/json are properly described in the file ``ChangeLog.md``. ### Necessary adaptations @@ -164,46 +161,43 @@ For the error-free execution is it necessary, however, to adhere to the naming s This script contains version and release date hard-coded. Both must be updated. -### Recommended procedure VERY MUCH WIP +### Recommended procedure -Based on the above observations, the following recommendations are derived. +Based on the above observations, the following steps are recommended for each update to the library. -1. Ensure that the content of the branch ``develop`` is identical to the branch ``main``. - Since it is intended to not change the library itself, in particular the folders ``include`` and ``single_include``, this should be possible by updating the documentation. -2. Merge branch master from the original nlohmann/json into ``develop``, e.g. ``git checkout -b json_version_X_XX_X && git merge --no-commit nlohmann/master`` -3. Confirm the deletion of cifuzz.yml, macos.yml and windows.yml. -4. Resolve the potential merge conflict in publish-documentation.yml by rejecting the incoming changes. - Update the versions of the github actions, if necessary. -5. Resolve the potential merge conflicts in check_amalgamation.yml, codeql.yml, dependency_review.yml, labeler.yml, ``test_trudag_extensions.yml`` to ensure that the artifacts are generated, i.e. the jobs ``Generate XXX artifact`` and ``Upload XXX artifact`` are retained. +1. Merge branch master from the original nlohmann/json into an external fork of the eclipse-score/inc_nlohmann_json repository, where steps 2-12 shall be performed. +2. Confirm the deletion of cifuzz.yml, macos.yml and windows.yml. +3. Resolve the potential merge conflict in publish-documentation.yml by rejecting the incoming changes. +4. Update the versions of the github actions, if necessary. +5. Resolve the potential merge conflicts in check_amalgamation.yml, codeql.yml, dependency_review.yml, labeler.yml and test_trudag_extensions.yml to ensure that the artifacts are generated, i.e. the jobs ``Generate XXX artifact`` and ``Upload XXX artifact`` are retained. 6. Resolve the potential merge conflict in ubuntu.yml following the above instructions. 7. Resolve the potential merge conflicts in cmake/download_test_data.cmake and cmake/ci.cmake following the above instructions. 8. Carefully examine the automatically merged changes. If no interference is to be expected, complete the merge. 9. In case any additional workflow has been added, carefully examine and integrate into the parent-workflow or schedule appropriately. 10. Adapt the documentation as described above. -11. Generate the documentation locally and carefully investigate any change in the trustable score(s). +11. Generate the TSF report locally and carefully investigate any change in the trustable score(s). If any relevant behaviour of the library changes, adapt the documentation. Additionally, if any additional tests were added, or existing tests were changed, carefully investigate whether these warrant an amendment of the documentation. -12. Merge into the ``main``. -13. Create a new release under the tag FIXME +12. Merge into the ``main`` branch of eclipse-score/inc_nlohmann_json. +13. Create a new release tag in eclipse-score/inc_nlohmann_json, following semantic versioning. # Update concept for the TSF documentation ## Assumptions of use The documentation follows the Trustable Software Framework (TSF), which is documented [here](https://codethinklabs.gitlab.io/trustable/trustable/print_page.html). -Furthermore, the automatic compilation of the documentation and the tracking of changes to the core functionalities of the library uses _trudag_, which is developed by Codethink and located [here](https://gitlab.com/CodethinkLabs/trustable/trustable). +Furthermore, the automatic generation of the TSF report and the tracking of changes to the core functionalities of the library uses _trudag_, which is developed by Codethink and located [here](https://gitlab.com/CodethinkLabs/trustable/trustable). ## Version of trudag -The documentation is currently built using trudag version 2025.8.5. -In case a major change of the trudag happens in the future, this might break some features of the documentation, or change some intended behaviours. -Thus, it is recommended to not change the version of trudag. -In case that it appears wise or necessary to change the version of trudag (e.g. when trudag is eventually certified), the following should be considered: +The documentation is currently built using trudag version v2025.10.22 +In case of new releases of trudag in the future, it is recommended to carefully review the introduced changes and rigorously test it before merging it into eclipse-score/inc_nlohmann_json. +The following should be considered: -* How has the algorithm for the accumulation of the trustable score changed? Ideally, it does not change, otherwise the necessity for a new review arises. -* How has the data store interface changed? Ideally, it has not changed, but historical data and the documentation indicate that a change of the data store interface happened at some time. -* How has the the expected configuration for the items changed? It is known that this configuration changed (at least) once before. What does the potential change mean? +* How has the algorithm for the accumulation of the trustable score changed? Ideally, it is not changed, otherwise the necessity for a new review arises. +* How has the data store interface changed? Ideally, it is not changed, but if it does and the expected schema format changes, data_store.py needs to be updated accordingly. +* How has the the expected configuration for the TSF items changed? It is known that this configuration changed (at least) once before. What does the potential change mean? * Do all custom references and validators as well as the data store interface work as before? * Has the algorithm for the hashing changed, or are there any changes to the trustable scores? If so, investigate carefully! @@ -211,9 +205,8 @@ In case that it appears wise or necessary to change the version of trudag (e.g. ## Subject-Matter-Expert-scores The intention with the SME scores is to find the _true_ trustable score by means of a heuristic law-of-large-numbers argument. -Therefore, it is very much welcome if contributors add their SME scores to statements for which they feel confident to do so. +Therefore, it is very much welcome for contributors to add their SME scores to statements for which they feel confident to do so. While the committer may check SME scores for plausibility, it is recommended to not question SME scores as this interferes with the assumed independence of the SME! -It is recommended that changes to SME scores are accumulated in the branch ``develop`` before the release of a new version of the documentation as to not clutter the release history. It is highly recommended to not delete SME scores under usual circumstances; most certainly, the SME scores should never be changed by anybody except the original SME. The following unusual circumstances can, after careful consideration, justify the removal or (much preferably!) the request for re-evaluation by the original SMEs: @@ -225,15 +218,26 @@ The following unusual circumstances can, after careful consideration, justify th In the absence of a validator, the SME shall assess their confidence in the statement based on linked artifacts (references) and their own knowledge. In the presence of a validator, the SME shall assess only their confidence in the validator as an accurate measure of the truth of the statement. +Some important remarks regarding the process of performing an SME review include: + +* Assumptions-of-Use (AoU) items are not to be scored by an SME. These should at some point in time be converted to a regular statement by the integrator, but shall until then remain without SME scores, evidence or references. These automatically contribute with a score of 0 for the time being. +* Any statement that has supporting statements (child nodes) shall not be scored by an SME. The scores for these shall only comprise of the automatic calculation of the mean of the scores of all linked child nodes. +* The TSF report is only published for the default branch of a repository. In order to to see TSF documentation for a non-default branch, the reviewer can checkout the branch in a clone and locally generate the report by running the ``TSF/scripts/generate_documentation.sh`` script. +* Any TSF item that has the ``normative`` parameter set to false shall not be scored by the SME. These are just informational and do not make a statement about the project. Note that the parameter itself shall not be set or changed by the SME, but is defined by the contributor who created the TSF item. + +For documentation on how to actually perform a SME review, please refer to the following: +* https://codethinklabs.gitlab.io/trustable/trustable/trudag/usage.html#reviewing-items +* https://codethinklabs.gitlab.io/trustable/trustable/trudag/usage.html#sme-review + ## Validators The automatic validators are intended to calculate a trustable score based on quantifiable data. In particular the introduction of a validator changes the meaning of the (potential) SME scores associated to a statement. Therefore, the change or introduction of an automatic validator is most critical. -It is highly recommended to urge the original SME to re-review the statement and adapt their scores, or (at the least) to enlist additional SME to judge the changed statement. +In case the validator used in a statement has been changed, it is highly recommended to urge the original SME to re-review the statement and adapt their scores, or (at the very least) to enlist additional SMEs to judge the changed statement. After careful consideration the highly critical decision to remove some SME scores no longer reflecting the statement could be made. ## References -References should be treated as validators, i.e. any update of a reference should trigger a re-review by the SME. +References should be treated in the same way as validators, i.e. any update of a reference should trigger a re-review by the SME. For references, however, the decision to remove a stale SME score is even more critical unless the reference reveals critical new information, which is highly unlikely, or the change of the reference is triggered by a significant change in the behaviour of the library, which heavily affected the statement. \ No newline at end of file diff --git a/TSF/trustable/assertions/TA-FIXES_CONTEXT.md b/TSF/trustable/assertions/TA-FIXES_CONTEXT.md index 2ea376dfb8..4d36257036 100644 --- a/TSF/trustable/assertions/TA-FIXES_CONTEXT.md +++ b/TSF/trustable/assertions/TA-FIXES_CONTEXT.md @@ -71,34 +71,34 @@ Each iteration, we should improve the algorithm based on measurements **Checklist** - How many faults have we identified in XYZ? - - **Answer**: + - **Answer**: None that are relevant for S-CORE's use case of the library. - How many unknown faults remain to be found, based on the number that have been processed so far? - - **Answer**: + - **Answer**: It is unlikely that there are unknown faults relevant to S-CORE. - Is there any possibility that people could be motivated to manipulate the lists (e.g. bug bonus or pressure to close). - - **Answer**: + - **Answer**: Unlikely, since the project is entirely open source. - How many faults may be unrecorded (or incorrectly closed, or downplayed)? - - **Answer**: + - **Answer**: Few or none, considering the wide use of the nlohmann/json library. - How do we collect lists of bugs and known vulnerabilities from components? - - **Answer**: + - **Answer**: We pull the list from the issues reported to nlohmann/json labelled as bug and are currently open or were opened since the last release. This list is then stored using GitHub, thereby enabling a traceability of the list. - How (and how often) do we check these lists for relevant bugs and known vulnerabilities? - - **Answer**: + - **Answer**: Whenever we generate the documentation, the list is pulled. If there is an issue previously unrecorded, then the maintainer is encouraged by the change of the trustable score to check the relevance of the issue. - How confident can we be that the lists are honestly maintained? - - **Answer**: + - **Answer**: Very confident, since the authors of the issues in the list mainly comprise of independent downstream users. - Could some participants have incentives to manipulate information? - - **Answer**: + - **Answer**: No such incentives have been identified. - How confident are we that the lists are comprehensive? - - **Answer**: + - **Answer**: Fairly confident, considering the wide use of the library and that downstream users are likely to report discovered bugs. - Could there be whole categories of bugs/vulnerabilities still undiscovered? - - **Answer**: + - **Answer**: Unlikely, considering the wide use of the library and that downstream users are likely to report discovered bugs. - How effective is our triage/prioritisation? - - **Answer**: + - **Answer**: There is no development of the json library within S-CORE, and therefore no triage/prioritisation. Any identified bugs/vulnerabilities are reported to nlohmann/json. Within nlohmann/json, no formal triage process has been identified. Nevertheless, reported bugs and vulnerabilities seem to be handled in a timely manner. - How many components have never been updated? - - **Answer**: + - **Answer**: None, the nlohmann/json library consists of a single header file, which the only component. This component is up to date. - How confident are we that we could update them? - - **Answer**: + - **Answer**: If a new version of the nlohmann/json library is released, we are very confident that we can update to that version. - How confident are we that outstanding fixes do not impact our Expectations? - - **Answer**: + - **Answer**: No outstanding fixes that impact the Expectation have been identified. - How confident are we that outstanding fixes do not address Misbehaviours? - - **Answer**: + - **Answer**: Very confident, as no Misbehaviours have been identified. diff --git a/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md b/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md index b3f2612fed..4d39aad5e7 100644 --- a/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md +++ b/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md @@ -41,7 +41,7 @@ Ultimately, the trustable controlled process is the only path to production for **Evidence** - change management process and configuration artifacts - - **Answer**: + - **Answer**: Provided in JLS-06, JLS-07, JLS-12, JLS-32, JLS-34 and JLS-35. **Confidence scoring** @@ -52,14 +52,14 @@ dependencies. **Checklist** - Where are the change and configuration management controls specified? - - **Answer**: + - **Answer**: In the contribution guidelines and security policy. - Are these controls enforced for all of components, tools, data, documentation and configurations? - - **Answer**: + - **Answer**: Yes. Any proposed change is subject to the same change controls. - Are there any ways in which these controls can be subverted, and have we mitigated them? - - **Answer**: + - **Answer**: No. The controls are enforced using branch protection rules and are mostly automated. - Does change control capture all potential regressions? - - **Answer**: + - **Answer**: Yes. All changes are rigorously tested in a separate branch before being deployed to the master branch. - Is change control timely enough? - - **Answer**: + - **Answer**: The change control is indeed timely enough, mainly due to the fact that the nlohmann/json library is open source and has many contributors. - Are all guidance and checks understandable and consistently followed? - - **Answer**: + - **Answer**: Yes. The contribution guidelines and CI workflows are clear and well-documented, as well as automatically enforced. diff --git a/TSF/trustable/assumptions-of-use/AOU-30.md b/TSF/trustable/assumptions-of-use/AOU-30.md new file mode 100644 index 0000000000..27df9d2ea6 --- /dev/null +++ b/TSF/trustable/assumptions-of-use/AOU-30.md @@ -0,0 +1,6 @@ +--- +level: 1.1 +normative: true +--- + +The integrator shall review the answers to each of the TSF evidence lists in the TA_CONTEXT files (see e.g., TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md). For each point that has not already been fulfilled, the integrator shall evaluate it and provide the relevant evidence if possible. From 46ed93c59b95548539f1079e563d40873e370d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luca=20F=C3=BCger?= Date: Thu, 20 Nov 2025 13:36:07 +0100 Subject: [PATCH 5/9] Reworked the README file for references and validators (#6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * extended TimeVaryingWebReference doc * added validator info to website and project_website reference * Updated readme for all references up to ListOfTestCases * Again reworked the references readme * extended documentation for validators * cosmetic change * cosmetic changes * small fix in TimeVaryingWebReference * Update .dotstop_extensions/README.md Co-authored-by: halnasri Signed-off-by: Luca Füger * Added : to expected configuration * Update .dotstop_extensions/README.md Co-authored-by: halnasri Signed-off-by: Luca Füger * validator addition * Added combinator trudag remark * added database to README * removed : * Update .dotstop_extensions/README.md Co-authored-by: Erik Hu Signed-off-by: Luca Füger --------- Signed-off-by: Luca Füger Co-authored-by: halnasri Co-authored-by: Erik Hu --- .dotstop_extensions/README.md | 158 ++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 75 deletions(-) diff --git a/.dotstop_extensions/README.md b/.dotstop_extensions/README.md index eceaafb271..4677ed31f2 100644 --- a/.dotstop_extensions/README.md +++ b/.dotstop_extensions/README.md @@ -10,32 +10,33 @@ For each item of the trustable graph, the hash is calculated by trudag using: * for every of its references, the *content* of that reference * for every of its fallacies, the description and content of the corresponding reference -Custom references are defined in `references.py`. A (custom) reference is used by adding an object into the list `references` in the header of the item file. The `type` corresponds to the classmethod `type` of a reference class of `references.py`, and the remaining object correspond to the arguments of the constructor. +Custom references are defined and implemented in `references.py`. A (custom) reference is used by adding a corresponding entry into the references list in the header of an item file. The type of this entry corresponds to the `classmethod` type of a reference class in `references.py`, and the remaining fields must map to the constructor arguments of the reference class. ## CPPTestReference -The content of a `CPPTestReference` is given by the lines of code corresponding to a test-case or a section of a test-case in a specified unit-test-file. The sections are identified in the value of "name", where the nested sections are separated by semicolons. +A `CPPTestReference` points to a specific C++ test-case or a nested section within a unit test file and includes the exact lines of code for that section in the reference content. The section path is specified using semicolons to denote nesting in the name field (for example, “TEST_CASE_NAME;SECTION_A;SECTION_B”), and the referenced text is presented as a C++ code block in the rendered documentation. -For the `CPPTestReference` the expected configuration is: +The reference locates the requested `TEST_CASE` or `SECTION` by name and follows any nested sections indicated by semicolons in the order they appear. It assumes the conventional brace layout used in the `nlohmann/json` tests, where the opening brace follows immediately after the declaration line and the closing brace has matching indentation. The content returned for hashing is the section’s exact source text in UTF-8, and the documentation renders the section cleanly as a C++ code block with indentation normalized when appropriate. + +For the `CPPTestReference` The expected configuration is: ``` --- ... references: - type: cpp_test - name: "compliance tests from json.org;expected failures" + name: "compliance tests from json.org;expected failures" # Uses semicolon-separated section names to identify nested sections (e.g., "TEST_CASE_NAME;SECTION_A;SECTION_B") path: "tests/src/unit-testsuites.cpp" --- ``` ## JSONTestsuiteReference -The `JSONTestsuiteReference` is a variant of the function reference, which is augmented by an external file containing test-data in the form of well- or ill-formed JSON candidate data. -A `JSONTestsuiteReference` is therefore given by the data of a `CPPTestReference` together with a list containing the paths to these external files. -The external files are stored in a separate branch of the repository, and their text is loaded via call to github. -The content of a `JSONTestsuiteReference` is given by the content of the underlying `CPPTestReference` together with the sum of the contents of the external test-suite files. +A `JSONTestsuiteReference` bundles a selected C++ test section together with one or more external JSON test files and uses both as the reference content. It is designed for tests that read JSON samples from files, allowing the documentation to present the JSON data alongside the test code. The external JSON files are fetched from a known test-data branch and must be referenced by the selected C++ section. + +The reference extends `CPPTestReference` by supplementing the identified C++ section with the content of the listed JSON files. It combines the test code and JSON data into a single payload for hashing, ensuring that the evidence reflects both the test harness and its inputs. In the documentation, each JSON file is shown either in full or replaced by a link if the file is very large, and the relevant C++ section is rendered below; an optional description appears above the content. If enabled, the reference filters lines in the displayed C++ section that mention other JSON files not included in the selection, while the underlying content still includes the full section text used for hashing. -For the `JSONTestsuiteReference` the expected configuration is: +For the `JSONTestsuiteReference` The expected configuration is: ``` --- ... @@ -44,17 +45,19 @@ references: - type: JSON_testsuite name: "compliance tests from json.org;expected failures" path: "tests/src/unit-testsuites.cpp" - test_suite_paths: + test_suite_paths: # List of JSON file paths (strings) to load from the test data branch - "/json_tests/fail2.json" - "/json_tests/fail3.json" description: "invalid json" - remove_other_test_data_lines: False # optional, the default value is True + remove_other_test_data_lines: False # optional, the default value is True; Removes all other test data lines in the test case or in a section of the test case, as given by the value of "name" --- ``` ## FunctionReference -The content of a `FunctionReference` is given by the code inclusive all comments of a C++ function within a class in a specified file in the repository. The specific position, i.e. start- and end-line, of the code within that file is not part of the content. +A `FunctionReference` identifies and extracts the full source code of a single C++ member function that is defined inside a class in a given header file. The content includes all comments and spans from the function’s signature line through its closing brace. If multiple definitions with the same name exist inside the class, an overload index can be used to select the nth one from top to bottom. + +The reference uses a name in the format `ClassName::FunctionName` together with the file path to find the class and the matching function inside the class body. When an overload index is provided, it selects the nth function definition encountered within the class; if no index is given, the first definition is used. The content returned for hashing is the function’s exact source code, and the documentation renders it as a C++ code block with an optional description shown above. This reference is intended for inline method definitions within headers and does not cover out-of-class implementations. For the `FunctionReference` an example is: ``` @@ -68,8 +71,7 @@ references: --- ``` -Since functions may be overloaded, a `FunctionReference` can be initialised with an optional overload-parameter. -The overload-parameter specifies which implementation of the function is referred to, i.e. if the overload-parameter for the function ``class::function()`` is set to _n_, then the _n_-th implementation when counting the occurrences from top to bottom of ``function()`` within the class ``class`` is used, if it exists; otherwise, an error is thrown. Additionally, it is possible, but not mandatory, to give a description. The full example is: +Additionally, it is possible, but not mandatory, to provide a `description` and an `overload` parameter. The full example including all parameters is: ``` --- ... @@ -85,7 +87,9 @@ references: ## WebReference -The content of a `WebReference` is its url. This reference is intended to be utilised in case that the content of the web-site is constantly changing (e.g. due to a clock being implemented somewhere on the site), but the reviewer is certain that the type of the content and it being supportive of the statement is fulfilled as long a the website is reachable. An example is `https://introspector.oss-fuzz.com/project-profile?project=json`, where the most recent fuzz-testing report for nlohmann/json is published. +The content of a `WebReference` is its `url` string. This is suitable when the page is expected to change continuously (e.g., dashboards or status pages), but the type of content and its supportive role are considered sufficient if the URL is reachable. An example is `https://introspector.oss-fuzz.com/project-profile?project=json`, where the most recent fuzz-testing report for nlohmann/json is published. Additionally, an optional `description` can be provided. + +The reference stores the `url` string as its content and presents it directly in the report. If a description is provided, it appears beneath the URL to clarify the relevance of the link. This reference pairs well with availability validators (like `https_response_time`), which can confirm that the page is reachable without relying on the page’s changing text. For the `WebReference`, an example is: ``` @@ -97,7 +101,7 @@ references: url: "https://math.stackexchange.com/" --- ``` -An example of `WebReference` with non-empty description is +An example of `WebReference` with non-empty description is: ``` --- ... @@ -111,11 +115,15 @@ references: ## WebContentReference -The content of a `WebContentReference` is its content. This reference is intended to be utilised in case of *static* references, that should not vary in a short time-frame, and whose content is most important for the trustability of the statement. An example is a file located on a github repository, e.g. `https://raw.githubusercontent.com/nlohmann/json/refs/heads/develop/.github/workflows/cifuzz.yml` +A `WebContentReference` captures the fetched body of a URL as its content and is intended for relatively static resources where the actual text on the page matters for the documentation. The URL is rendered in the report, optionally followed by a description. An example is a file located on a github repository, e.g. `https://raw.githubusercontent.com/nlohmann/json/refs/heads/develop/.github/workflows/cifuzz.yml`. + +The reference downloads the page content from the given URL and uses it directly for hashing so that the evidence reflects the text as it existed at the time of retrieval. In the documentation, the URL and optional description are shown, while the full downloaded text is included in the content used for trustability, which allows the reference to anchor the documentation to a specific, stable version of the page or file. + + A `WebContentReference` looks identical to a `WebReference` with `type: web_content` instead of `type: website`. -For the `TimeVaryingWebReference`, examples of the possible configurations are: +For the `WebContentReference`, examples of the possible configurations are: ``` --- ... @@ -140,11 +148,18 @@ in case of a custom description. ## TimeVaryingWebReference -The content of a `TimeVaryingWebReference` is given by the content of a changelog, whose default value is `ChangeLog.md`, which mirrors the changelog of nlohmann/json. This reference is intended for websites whose content is constantly changing, so that a `WebContentReference` makes the item un-reviewable, but whose content at the time of an update influences the trustability. An example is `https://github.com/nlohmann/json/pulse/monthly`, which can be used to demonstrate that nlohmann/json is *up to the most recent version* under active development. +Use this reference when the content of a site changes continuously, but the mere existence and reachability of the site is not sufficient to support the statement. In other words, use this reference if the site requires regular re-reviews. The content for hashing is the changelog file from this repository prefixed with the URL. For example, https://github.com/nlohmann/json/pulse/monthly can be used to demonstrate that `nlohmann/json` is up to the most recent version under active development. The content of a `TimeVaryingWebReference` is determined by a changelog file in this repository. By default, this is `ChangeLog.md`, which mirrors the upstream changelog of `nlohmann/json`. + +As with WebReference, consider an https_response_time validator to check reachability of the URL if needed. + +Important clarifications: + +The system checks the `ChangeLog.md` within our own repository. It does not read or rely on external changelogs or any files in external repositories. +Whenever `nlohmann/json` publishes a new release/patch and we integrate it into our repository, our `ChangeLog.md` will be updated accordingly (since we mirror upstream changes). Any change to this changelog will automatically set the review_status of all statements with TimeVaryingWebReferences to unreviewed, meaning they are invalidated and must be re-reviewed. +The changelog argument defaults to `ChangeLog.md`, which is the correct path to the changelog in this repo. You only need to specify the changelog argument if the file is moved or renamed. +An example of the complete configuration for `TimeVaryingWebReference` (overriding the changelog path) is: -An example of the complete configuration for `TimeVaryingWebReference` is -``` --- ... references: @@ -153,13 +168,26 @@ references: description: "Wiki article on the smooth Serre-Swan theorem" changelog: "ideas/graded/graded_Serre_Swan.tex" --- -``` -where `description` and `changelog` are optional arguments. + +In the common case (using the default ChangeLog.md in this repository), you can omit the `changelog` argument: + + +--- +... +references: +- type: project_website + url: "https://github.com/nlohmann/json/pulse/monthly" + description: "Development activity for nlohmann/json" +--- +Both `description` and `changelog` are optional arguments. ## ListOfTestCases -The content of a `ListOfTestCases` is given by the list of test-cases extracted from the unit-tests given in the files in the provided directories. -It is assumed that a unit-test is saved in a file with the name unit-xxx.cpp, and only those files are used to compile the list. +A `ListOfTestCases` reference produces a markdown overview of all unit tests and nested sections found in the provided directories and files and augments this structure with recent execution environments from a test-results database (currently "artifacts/MemoryEfficientTestResults.db"). This provides a concise map of the test suite together with information on where it has been executed successfully or with skips. + +The reference scans files matching the pattern `unit-*.cpp` to extract `TEST_CASE` and `SECTION` names and their nesting, and it formats the results as nested lists to mirror the test structure. It then queries the database of recent test runs to list the compiler and standard combinations that executed all tests without skipping, as well as those where some tests were skipped, and it incorporates these details into the same markdown report. The content returned for hashing is the complete markdown report, which the documentation also renders directly. + + Further, it is assumed that a unit-test-file is structured as ``` @@ -175,7 +203,9 @@ TEST_CASE("my test case") } ``` -and the structure regarding test-cases and (nested) sections of test-cases is extracted. The expected configuration is +and the structure regarding test-cases and (nested) sections of test-cases is extracted. + +The expected configuration is: ``` --- @@ -190,12 +220,11 @@ references: ## workflow_failures -This reference queries `https://github.com/{self._owner}/{self._repo}/actions?query=is%3Afailure+branch%3A{self._branch}` and collects the number of failed workflow runs as its content. -Here, owner, repo and branch are the arguments given to the constructor of the reference. -If no branch is specified, then all failures are collected, i.e. `https://github.com/{self._owner}/{self._repo}/actions?query=is%3Afailure` is queried. -In case the website is un-reachable, or the github layout changes drastically so that the number of failed workflow runs does not exist at the expected location, an error is thrown. +This reference queries Github Actions (`https://github.com/{self._owner}/{self._repo}/actions?query=is%3Afailure+branch%3A{self._branch}`) for failed workflow runs and records the number as content. This allows the documentation to include the current failure count as evidence without embedding the page itself. -The expected configuration is +The reference constructs a URL that filters workflow runs by failure status, optionally restricting the query to a specific branch, and it extracts the number displayed at the top of the results table. It returns this number as the content used for hashing, and in the documentation it presents a short sentence indicating the count and, when applicable, the branch to which the count refers. + +The expected configuration is: ``` --- @@ -210,16 +239,11 @@ references: ## ItemReference -Some references support every (directly or indirectly) supporting item of an item. -Instead of repeating these references in each supporting item, these references are listed in the supported item. -The inheritance of the references is then clarified in the documentation by an `ItemReference`. -In the final documentation in human-readable form, an ItemReference simply lists all items of which the references are inherited with hyperlinks. +An `ItemReference` documents inheritance of references from other items so that shared evidence does not need to be repeated across multiple supporting items. In the rendered documentation, it lists links to the referenced items so readers can navigate to the source evidence. + +The reference reads entries from the local trustable graph file and combines the signatures of the referenced items into its content, thereby tying the current item’s trustability to those items’ evidence. In the documentation, it prints a list of hyperlinks pointing to the target items, and any change in the referenced items’ evidence automatically propagates to the dependent item through this linkage. If any reference of any of the listed supported items changes, then its sha changes and the review-status of the item becomes false. After successful re-review, the review-status of the supported items is re-set to true, so that the new sha is stored in the `.dotstop.dot` file. This automatically sets the review-status of the supporting items, which inherit the references, to false, thereby triggering a re-review of these. -To detect the inheritance of references in the content of the supporting items, the content of an ItemReference is the combination of the sha's stored in the .dotstop.dot file of the listed supported items. -If any reference of any of the listed supported items changes, then its sha changes and the review-status of the item becomes false. -After successful re-review, the review-status of the supported items is re-set to true, so that the new sha is stored in the .dotstop.dot file. -This automatically sets the review-status of the supporting items, which inherit the references, to false, thereby triggering a re-review of these. -The expected configuration is as follows +The expected configuration is: ``` --- @@ -237,14 +261,11 @@ Here, the elements of the list `items` must be normative nodes of the trustable ## IncludeListReference -The content of an `IncludeListReference` is given by the list of `#include` lines extracted from a specified source/header file in the repository (for example `single_include/nlohmann/json.hpp`). This reference is useful to document which libraries a file depends on without embedding the full file content into the report. +An `IncludeListReference` extracts all `#include` lines from a specified source or header file (e.g., single_include/nlohmann/json.hpp) and uses them as the content. It is useful for showing direct dependencies of a file without embedding its full text. -Behaviour: -- content: returns the concatenation of all lines that begin with `#include` in the target file as UTF-8 encoded bytes. If no includes are found, the content is `b"No includes found"`. -- as_markdown: renders the found `#include` lines as a C++ code block (```cpp ... ```). If a `description` was provided when constructing the reference, the description is shown as an indented bullet above the code block. -- If the referenced file does not exist or is not a regular file, accessing `content` raises a ReferenceError. +The reference reads the target file and collects only those lines whose first non-whitespace characters form a `#include` directive, removing inline comments where necessary to present a clean list. It returns the list of includes as the content used for hashing, or a placeholder message when the file contains no includes, and it renders the includes as a C++ code block in the documentation with an optional `description` above the block. -Usage example: +The expected configuration is: ``` --- @@ -257,17 +278,13 @@ references: --- ``` -Notes: -- `description` is optional. -- The reference only extracts lines whose first non-whitespace characters are `#include`. - # Validators -Validators are extensions of trudag, used to validate any data that can be reduced to a floating point metric. The resulting scores are used as evidence for the trustability of items in the trustable graph. +Validators are extensions of trudag that assess evidence by computing floating-point scores from measurable data. These scores provide quantitative support for the trustability of items in the trustable graph and are combined with the items’ references and statements during hashing and reporting. The implementation of the validators can be found in validators.py. ## check_artifact_exists -The check_artifact_exists script validates the presence of artifacts from GitHub Actions workflows for the current SHA. The score is given based on the number of artifacts found vs the number of artifacts expected. +The `check_artifact_exists` validator confirms whether the expected GitHub Actions artifacts exist for the current commit SHA and assigns a score based on how many of the requested artifacts were found relative to how many were expected. The configuration consists of keys that correspond to known artifact names (namely `check_amalgamation`, `codeql`, `dependency_review`, `labeler`, `test_trudag_extensions`, and `ubuntu`) and values that specify whether each artifact should be included as part of the evidence (“include”) or excluded from consideration (“exclude”). The validator interprets this mapping to determine the expected set and then compares it against the artifacts available for the current build to derive the score. The available configuration dict keys for check_artifact_names are: - `check_amalgamation` @@ -281,11 +298,11 @@ The available configuration dict values for check_artifact_names are: - 'include' - 'exclude' -These indicate whether a certain artifact should be included as evidence for a Trustable graph item. - ## https_response_time -The automatic validator https_response_time checks the responsiveness of a given website. The expected configuration is as in the example: +The `https_response_time` validator measures how quickly a list of websites responds and derives a score based on the responsiveness and the HTTP status code. The configuration specifies an acceptable response time threshold in seconds and a list of URLs to check. Any URL that responds in at least five times the acceptable threshold, or that returns a non-200 response code, receives an individual score of zero. The final score is calculated as the mean of all individual scores across the list of URLs, which encourages fast and reliable endpoints and flags unresponsive or failing sites. + +The expected configuration is: ``` evidence: type: https_response_time @@ -296,16 +313,12 @@ evidence: - "https://github.com/nlohmann/json/graphs/commit-activity" - "https://github.com/nlohmann/json/forks?include=active&page=1&period=&sort_by=last_updated" ``` -A response time of at least the five-fold of the acceptable response time is deemed unacceptable and gives an individual score of zero. -Likewise unacceptable is a response code other than `200`, which gives an individual score of zero. - -The total score is the mean of the individual scores. ## check_test_results -The automatic validator `check_test_results` is intended to evaluate the database `MemoryEfficientTestResults.db` which is generated in the ubuntu-Workflow, and which contains the test-report of the most recent workflow run. This database is temporary, and, contrary to `TSF/MemoryEfficientTestResultData.db`, which is persistently stored on the branch `save_historical_data`, not persistently stored. +The `check_test_results` validator analyzes the most recent unit test outcomes stored in a SQLite database generated by the ubuntu workflow and calculates a score that reflects the proportion of passed test cases among those that were executed. The `configuration` lists the test files of interest (by their base names without the “unit-” prefix or file extension) and optionally overrides the `database` file path and `table` name. For each specified test, the validator counts the passed and failed cases and ignores skipped cases to compute an individual pass ratio. The overall score is the mean of these ratios, which provides a balanced measure of recent test success across the selected test files. -The expected configuration is given as follows: +The expected configuration is: ``` evidence: @@ -321,8 +334,6 @@ evidence: The test-files are called unit-FILE_NAME.cpp. In the configuration, FILE_NAME is expected only, i.e. without the leading unit- and without the file-extension. -For each test specified in test-files, the number of passed and failed test-cases is calculated, while the number of skipped test-cases is ignored. The score of each test is then the ratio of passed test-cases compared to all non-skipped test-cases; the total score is the mean of the individual scores. - ## check_issues The automatic validator `check_issues` is intended to evaluate the json-lists `raw_open_issues.json` and `raw_closed_issues.json` and compare with the list of known issues of nlohmann/json labelled as bug opened since the release of the version of nlohmann/json that is documented. The json lists are generated in the publish_documentation-Workflow, and not persistently stored. @@ -348,10 +359,9 @@ Otherwise, the score 1.0 is assigned. ## did_workflows_fail -The automatic validator `did_workflows_fail` queries the web-site `https://github.com/{owner}/{repo}/actions?query=event%3A{event}+is%3Afailure+branch%3A{branch}` and looks on the number of workflow run results which is printed at the head of the table. -In case that this number is not zero, a score of 0.0 is returned, and 1.0 otherwise. +The `did_workflows_fail` validator checks the results of GitHub Actions runs for a repository, optionally filtered by event and branch, and produces a binary score indicating whether any runs failed. It constructs a query that lists workflow results matching failure status and then reads the number displayed at the top of the results table. If the number is zero, it returns a score of 1.0 to indicate no failures; otherwise, it returns 0.0 with a warning. If the GitHub page cannot be reached or the number of failed runs cannot be parsed, the validator returns 0.0 with an error. The configuration requires the repository owner and name and may include the branch and event, with “push” as the default event. It is important to enclose all configuration values in quotation marks to ensure the update helper processes them correctly. -The expected configuration is given as follows: +The expected configuration is: ``` evidence: @@ -363,18 +373,14 @@ evidence: action: "push" # optional, default is push ``` -It is of utmost importance that the arguments come with quotation marks. Otherwise, the update helper does not work as intended. ## coveralls_reporter -The automatic validator `coveralls_reporter` queries the [coveralls](https://coveralls.io/) api to get the line and branch coverages calculated by the service, which is running on the repository. +The coveralls_reporter validator queries the Coveralls API [coveralls](https://coveralls.io/) for the current line and branch coverage figures and compares them with expected values specified in the configuration. It returns a score of 1.0 if both fetched coverage numbers, rounded to a configurable number of decimal digits, match the given expectations and returns 0.0 otherwise. The configuration includes the Coveralls project coordinates (owner, repo, and optionally branch), the expected coverage values, and the rounding precision, which defaults to three decimal digits. When no branch is provided, the validator uses the most recent coverage values, although specifying a branch is recommended for consistent comparisons. + Unless the version of `nlohmann/json` documented in this repository changes, it is expected that both coverage numbers remain constant. -When initialising the reference, the current code coverage is given as a parameter, to which the fetched coverages are compared. -If no branch is specified, then the most recently calculated coverage is fetched, so that it is generally recommended to specify a branch. -Moreover, it is possible to specify the number of decimal digits, which is defaulted to three, when not specified. -The validator returns a score of 1.0 if both fetched coverages rounded to the specified number of decimal digits coincide with the specified ones, and a score of 0.0 otherwise. -The expected configuration is the following: +The expected configuration is: ``` evidence: @@ -390,7 +396,9 @@ evidence: ## combinator -The trudag tool does currently not support the use of multiple custom validators for one single TSF item. To work around this, the validator `combinator` is implemented as a meta-validator that executes multiple validators and combines their scores using a weighted average. This enables the validation of complex trustable items that require evidence from multiple sources or validation methods. +The combinator validator serves as a meta-validator that enables the evaluation of complex items by running multiple validators and combining their scores into a single result using a weighted average. It accepts a list of validator configurations and optional weights, executes each validator independently, and then computes the weighted mean of their scores. If no weights are specified, all validators are treated equally. The validator supports a predefined set of underlying validator types and aggregates their exceptions and warnings in its output. All weights must be non-negative, and if the sum of weights is zero, the combinator returns a score of 0.0. + +The trudag tool does currently not support the use of multiple custom validators for one single TSF statement. The combinator accepts a list of validators, each with its own configuration and optional weight. Each validator is executed independently, and their scores are combined using the formula: `(score1 * weight1 + score2 * weight2 + ...) / (weight1 + weight2 + ...)`. If no weights are specified, all validators are treated with equal weight (weight = 1.0). @@ -463,4 +471,4 @@ It is intended to store data only once per commit. If, for any reason, the same ## pull -This functionality parses the information stored in `TrustableScoring.db` into the format which is expected by trudag. In case that no data are found, the empty history is returned. \ No newline at end of file +This functionality parses the information stored in `TrustableScoring.db` into the format which is expected by trudag. In case that no data is found, the empty history is returned. \ No newline at end of file From 97d5458303b1557c18b117a1fd9e72323e5e4f7d Mon Sep 17 00:00:00 2001 From: Erik Hu Date: Mon, 24 Nov 2025 09:25:25 +0100 Subject: [PATCH 6/9] link TA-BEHAVIOURS to JLS-27 (#9) --- .dotstop.dot | 1 + 1 file changed, 1 insertion(+) diff --git a/.dotstop.dot b/.dotstop.dot index c7eb3e288c..6ef07d7dce 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -376,6 +376,7 @@ digraph G { "TA-BEHAVIOURS" -> "JLEX-01" [sha="8cd931ef61b7012140344adf54469e943bfc690ee54f12db12777464880061db"]; "TA-BEHAVIOURS" -> "JLEX-02" [sha=cb26451e31a56b1eb51a4d45283ba4a7c6e898efbd045b59cba10d3c6aa093de]; "TA-BEHAVIOURS" -> "JLS-03" [sha=cf9211c07452914cb2d0b455f859b26cb2724423eae5187e8cbfdff06d1b5ba3]; +"TA-BEHAVIOURS" -> "JLS-27" [sha="880ec996ed026258b58299c356aab7d02652ae55cbf1f98494e2a7770fd96275"]; "TA-CONFIDENCE" -> "JLS-08" [sha="506164051180023c8533ea1f6dedf1bad894c3ee6020ff16b002e33b109c2791"]; "TA-CONFIDENCE" -> "JLS-09" [sha="80bbde95fc14f89acf3dad10b3831bc751943fe4a1d79d5cbf4702416c27530f"]; "TA-CONFIDENCE" -> "JLS-20" [sha="1bfd214ab8186a3c095262ae503451b8d71ada8db5b13ecc7b906739a05bc102"]; From 7233c865e98f03519328db1a04349d8a7b0ef1a5 Mon Sep 17 00:00:00 2001 From: Erik Hu Date: Mon, 24 Nov 2025 16:01:45 +0100 Subject: [PATCH 7/9] Erikhu1 tt changes documentation (#10) * answer checklist TT-CHANGES * update TSF/README * add AOU-30 * steps 2-12 * mean of the scores * library -> project * fix typo * additional SMEs * fix comprise typo * nlohmann/json triage process * add JLS-34 to TA_UPDATES context * replace XYZ and add disclaimer * update TT_CHANGES context files * update TA_UPDATES context * clean up TA_FIXES context --------- Signed-off-by: Erik Hu --- .../assertions/TA-ANALYSIS_CONTEXT.md | 4 ++- .../assertions/TA-BEHAVIOURS_CONTEXT.md | 6 ++-- .../assertions/TA-CONFIDENCE_CONTEXT.md | 2 ++ .../assertions/TA-CONSTRAINTS_CONTEXT.md | 2 ++ TSF/trustable/assertions/TA-DATA_CONTEXT.md | 2 ++ TSF/trustable/assertions/TA-FIXES_CONTEXT.md | 36 ++++++++++--------- .../assertions/TA-INDICATORS_CONTEXT.md | 2 ++ TSF/trustable/assertions/TA-INPUTS_CONTEXT.md | 20 ++++++----- .../assertions/TA-ITERATIONS_CONTEXT.md | 2 ++ .../assertions/TA-METHODOLOGIES_CONTEXT.md | 4 ++- .../assertions/TA-MISBEHAVIOURS_CONTEXT.md | 8 +++-- .../assertions/TA-RELEASES_CONTEXT.md | 10 +++--- .../assertions/TA-SUPPLY_CHAIN_CONTEXT.md | 10 +++--- TSF/trustable/assertions/TA-TESTS_CONTEXT.md | 6 ++-- .../assertions/TA-UPDATES_CONTEXT.md | 16 +++++---- .../assertions/TA-VALIDATION_CONTEXT.md | 4 ++- 16 files changed, 83 insertions(+), 51 deletions(-) diff --git a/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md b/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md index e6dc6ee310..1a4e46d6d3 100644 --- a/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md @@ -3,10 +3,12 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** This assertion is satisfied to the extent that test data, and data collected -from monitoring of deployed versions of XYZ, has been analysed, and the results +from monitoring of deployed versions of nlohmann/json, has been analysed, and the results used to inform the refinement of Expectations and risk analysis. The extent of the analysis is with sufficient precision to confirm that: diff --git a/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md b/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md index c85654a6ce..2aea1e70c8 100644 --- a/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md @@ -3,16 +3,18 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + Although it is practically impossible to specify all of the necessary behaviours and required properties for complex software, we must clearly specify the most important of these (e.g. where harm could result if given criteria are not met), -and verify that these are correctly provided by XYZ. +and verify that these are correctly provided by nlohmann/json. **Guidance** This assertion is satisfied to the extent that we have: -- Determined which Behaviours are critical for consumers of XYZ and recorded +- Determined which Behaviours are critical for consumers of nlohmann/json and recorded them as Expectations. - Verified these Behaviours are achieved. diff --git a/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md b/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md index efa1d013e1..3a59e34ab2 100644 --- a/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md +++ b/TSF/trustable/assertions/TA-CONFIDENCE_CONTEXT.md @@ -3,6 +3,8 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** To quantify confidence, either a subjective assessment or a statistical argument must be presented for each statement and then systematically and repeatably aggregated to assess whether the final deliverable is fit for purpose. diff --git a/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md b/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md index c57b098a9d..c484995645 100644 --- a/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md @@ -3,6 +3,8 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** Constraints on reuse, reconfiguration, modification, and deployment are diff --git a/TSF/trustable/assertions/TA-DATA_CONTEXT.md b/TSF/trustable/assertions/TA-DATA_CONTEXT.md index b918240144..7a5a549b2a 100644 --- a/TSF/trustable/assertions/TA-DATA_CONTEXT.md +++ b/TSF/trustable/assertions/TA-DATA_CONTEXT.md @@ -3,6 +3,8 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** This assertion is satisfied if results from all tests and monitored deployments are captured accurately, ensuring: diff --git a/TSF/trustable/assertions/TA-FIXES_CONTEXT.md b/TSF/trustable/assertions/TA-FIXES_CONTEXT.md index 4d36257036..10f561d79e 100644 --- a/TSF/trustable/assertions/TA-FIXES_CONTEXT.md +++ b/TSF/trustable/assertions/TA-FIXES_CONTEXT.md @@ -3,14 +3,16 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** -This assertion is satisfied to the extent that we have identified, triaged, and applied fixes or mitigations to faults in XYZ, as well as to bugs and publicly disclosed vulnerabilities identified in upstream dependencies. +This assertion is satisfied to the extent that we have identified, triaged, and applied fixes or mitigations to faults in nlohmann/json, as well as to bugs and publicly disclosed vulnerabilities identified in upstream dependencies. -Confidence can be improved by assessing known faults, bugs, and vulnerabilities to establish their relevance and impact for XYZ. +Confidence can be improved by assessing known faults, bugs, and vulnerabilities to establish their relevance and impact for nlohmann/json. An important aspect is documenting how issues are discovered and tracked, including identifying additional Misbehaviours (TA-MISBEHAVIOURS) that may require immediate mitigation measures (including recalls), and how such issues are communicated to users. -In principle, this analysis should include not only the code in XYZ but also its dependencies (all the way down) and the tools and data used to construct the release. +In principle, this analysis should include not only the code in nlohmann/json but also its dependencies (all the way down) and the tools and data used to construct the release. In practice, however, the cost/benefit of this work must be weighed against: - the volume and quality of available bug and vulnerability reports @@ -38,7 +40,7 @@ As part of ongoing monitoring, the rate of incoming, resolved, and rejected issu - List of outstanding known vulnerabilities still not fixed, with triage/prioritisation based on severity/relevance/impact - **Answer**: Provided in JLS-30, JLS-33 and AOU-29 -- List of XYZ component versions, showing where a newer version exists upstream +- List of nlohmann/json component versions, showing where a newer version exists upstream - **Answer**: Not relevant since nlohmann/json has no external components, as stated in JLS-34 - List of component version updates since last release - **Answer**: Not relevant as nlohmann/json has no external components, as stated in JLS-34 @@ -46,18 +48,18 @@ As part of ongoing monitoring, the rate of incoming, resolved, and rejected issu - **Answer**: Provided in JLS-29 - List of fixes for developed code that are outstanding, not applied yet - **Answer**: Provided in JLS-11 -- List of XYZ faults outstanding (O) +- List of nlohmann/json faults outstanding (O) - **Answer**: Provided in JLS-11 -- List of XYZ faults fixed since last release (F) +- List of nlohmann/json faults fixed since last release (F) - **Answer**: Provided in JLS-29 -- List of XYZ faults mitigated since last release (M) +- List of nlohmann/json faults mitigated since last release (M) - **Answer**: Provided in JLS-29 **Confidence scoring** Confidence scoring for TA-FIXES can be based on -- some function of [O, F, M] for XYZ +- some function of [O, F, M] for nlohmann/json - number of outstanding relevant bugs from components - bug triage results, accounting for undiscovered bugs - number of outstanding known vulnerabilities @@ -70,35 +72,35 @@ Each iteration, we should improve the algorithm based on measurements **Checklist** -- How many faults have we identified in XYZ? - - **Answer**: None that are relevant for S-CORE's use case of the library. +- How many faults have we identified in nlohmann/json? + - **Answer**: 58, but none are relevant for S-CORE's use case of the library (see JLS-11). - How many unknown faults remain to be found, based on the number that have been processed so far? - **Answer**: It is unlikely that there are unknown faults relevant to S-CORE. - Is there any possibility that people could be motivated to manipulate the lists (e.g. bug bonus or pressure to close). - - **Answer**: Unlikely, since the project is entirely open source. + - **Answer**: It is unlikely that people would be motivated to manipulate the lists in nlohmann/json. The nlohmann/json project has no bug bounties, and since it is open source, third party individuals suggest fixes with no pressure/incentive to manipulate unfixed issues. - How many faults may be unrecorded (or incorrectly closed, or downplayed)? - - **Answer**: Few or none, considering the wide use of the nlohmann/json library. + - **Answer**: Few or none, considering the wide use of the nlohmann/json library (see JLS-05). - How do we collect lists of bugs and known vulnerabilities from components? - **Answer**: We pull the list from the issues reported to nlohmann/json labelled as bug and are currently open or were opened since the last release. This list is then stored using GitHub, thereby enabling a traceability of the list. - How (and how often) do we check these lists for relevant bugs and known vulnerabilities? - **Answer**: Whenever we generate the documentation, the list is pulled. If there is an issue previously unrecorded, then the maintainer is encouraged by the change of the trustable score to check the relevance of the issue. - How confident can we be that the lists are honestly maintained? - - **Answer**: Very confident, since the authors of the issues in the list mainly comprise of independent downstream users. + - **Answer**: Very confident, since the authors of the issues in the list mainly comprise independent downstream users. - Could some participants have incentives to manipulate information? - **Answer**: No such incentives have been identified. - How confident are we that the lists are comprehensive? - - **Answer**: Fairly confident, considering the wide use of the library and that downstream users are likely to report discovered bugs. + - **Answer**: Fairly confident, considering the wide use of the library (see JLS-05) and that downstream users are likely to report discovered bugs. - Could there be whole categories of bugs/vulnerabilities still undiscovered? - - **Answer**: Unlikely, considering the wide use of the library and that downstream users are likely to report discovered bugs. + - **Answer**: Unlikely, considering the wide use of the library (see JLS-05) and that downstream users are likely to report discovered bugs. - How effective is our triage/prioritisation? - **Answer**: There is no development of the json library within S-CORE, and therefore no triage/prioritisation. Any identified bugs/vulnerabilities are reported to nlohmann/json. Within nlohmann/json, no formal triage process has been identified. Nevertheless, reported bugs and vulnerabilities seem to be handled in a timely manner. - How many components have never been updated? - **Answer**: None, the nlohmann/json library consists of a single header file, which the only component. This component is up to date. - How confident are we that we could update them? - - **Answer**: If a new version of the nlohmann/json library is released, we are very confident that we can update to that version. + - **Answer**: Within nlohmann/json, there are no external components to update. Within S-CORE, if a new version of the nlohmann/json library is released, we are very confident that we can update to that version. (See the update process in TSF/README.md) - How confident are we that outstanding fixes do not impact our Expectations? - - **Answer**: No outstanding fixes that impact the Expectation have been identified. + - **Answer**: No outstanding fixes that impact the Expectations have been identified. - How confident are we that outstanding fixes do not address Misbehaviours? - **Answer**: Very confident, as no Misbehaviours have been identified. diff --git a/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md b/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md index 20ed4d193e..6cf6af6b46 100644 --- a/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md @@ -3,6 +3,8 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + Not all deviations from Expected Behaviour can be associated with a specific condition. Therefore, we must have a strategy for managing deviations that arise from unknown system states, process vulnerabilities or configurations. diff --git a/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md b/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md index 7721a6a705..c2d04fa074 100644 --- a/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-INPUTS_CONTEXT.md @@ -3,25 +3,27 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** -Anything that can influence the output of the XYZ project is considered an input. +Anything that can influence the output of the nlohmann/json project is considered an input. This includes: - Software components used to implement specified features and meet defined Expectations - Software tools, and their outputs, used for design, construction and verification - Infrastructure that supports development and release processes -All inputs (components, tools, data) and their dependencies (recursively) used to build and verify XYZ releases must be identified and assessed, since they are untrusted by default. +All inputs (components, tools, data) and their dependencies (recursively) used to build and verify nlohmann/json releases must be identified and assessed, since they are untrusted by default. Each input should be evaluated on verifiable merits, regardless of any claims it makes (including adherence to standards or guidance). Evaluation must include the project's defined Expectations to ensure that inputs meet requirements, and that risks are recorded and addressed appropriately. -For components, we need to consider how their misbehaviour might impact achieving project XYZ's Expectations. +For components, we need to consider how their misbehaviour might impact achieving project nlohmann/json's Expectations. Sources (e.g. bug databases, advisories) for known risks should be identified, their update frequency recorded, and tests defined for detecting them. These form the inputs to TA-FIXES. -For the tools used to construct and verify XYZ, we need to consider how their misbehaviour could: +For the tools used to construct and verify nlohmann/json, we need to consider how their misbehaviour could: - Introduce unintended changes - Fail to detect Misbehaviours during testing @@ -45,7 +47,7 @@ As a result, for example, any binary inputs without reproducible build steps or **Evidence** -- List of components used to build XYZ, including: +- List of components used to build nlohmann/json, including: - Whether content is provided as source or binary - **Answer**: - Record of component assessments: @@ -53,20 +55,20 @@ As a result, for example, any binary inputs without reproducible build steps or - **Answer**: - Date of assessments and identity of assessors - **Answer**: - - Role of component in XYZ + - Role of component in nlohmann/json - **Answer**: - Sources of bug and risk data - **Answer**: - Potential misbehaviours and risks identified and assessed - **Answer**: -- List of tools used to build and verify XYZ +- List of tools used to build and verify nlohmann/json - **Answer**: - Record of tool assessments: - Originating project and tool version - **Answer**: - Date of assessments and identity of assessors - **Answer**: - - Role of the tool in XYZ releases + - Role of the tool in nlohmann/json releases - **Answer**: - Potential misbehaviours and impacts - **Answer**: @@ -79,7 +81,7 @@ As a result, for example, any binary inputs without reproducible build steps or Confidence scoring for TA-INPUTS is based on the set of components and tools identified, how many of (and how often) these have been assessed for their risk -and impact for XYZ, and the sources of risk and issue data identified. +and impact for nlohmann/json, and the sources of risk and issue data identified. **Checklist** diff --git a/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md b/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md index fab30edf85..28a622414d 100644 --- a/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-ITERATIONS_CONTEXT.md @@ -3,6 +3,8 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** This assertion is best satisfied by checking generated documentation to confirm that: diff --git a/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md b/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md index c1c1f4bb7b..d50c46f4c0 100644 --- a/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md +++ b/TSF/trustable/assertions/TA-METHODOLOGIES_CONTEXT.md @@ -3,9 +3,11 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** -To satisfy this assertion, all manual processes used in the verification of XYZ must be documented, including the methodologies applied, the results for specific aspects and iterations, and evidence that these processes were reviewed against documented criteria. +To satisfy this assertion, all manual processes used in the verification of nlohmann/json must be documented, including the methodologies applied, the results for specific aspects and iterations, and evidence that these processes were reviewed against documented criteria. Most analysis (e.g., data analysis for TA-ANALYSIS) should be automated to enable continuous feedback. However, the quality of any remaining manual processes (whether from first parties or external third parties) must be considered and how they are documented and reviewed. diff --git a/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md b/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md index 20fb520872..754b9c3e0c 100644 --- a/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md @@ -3,6 +3,8 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + The goal of TA-MISBEHAVIOURS is to force engineers to think critically about their work. This means understanding and mitigating as many of the situations that cause the software to deviate from Expected Behaviours as possible. This is not limited to @@ -12,7 +14,7 @@ the contents of the final binary. This assertion is satisfied to the extent that we can: -- Show we have identified all of the ways in which XYZ could deviate from its +- Show we have identified all of the ways in which nlohmann/json could deviate from its Expected Behaviours. - Demonstrate that mitigations have been specified, verified and validated for all Misbehaviours. @@ -57,11 +59,11 @@ Mitigations could be verified by: - Stress testing of assumptions of use. That is, confirming assumptions of use are actually consistent with the system and its Expected Behaviours by intentionally misinterpreting or liberally interpreting them in a test - environment. For example, we could consider testing XYZ on different pieces of + environment. For example, we could consider testing nlohmann/json on different pieces of hardware that satisfy its assumptions of use. Remember that a Misbehaviour is _anything_ that could lead to a deviation from -Expected Behaviour. The specific technologies in and applications of XYZ should +Expected Behaviour. The specific technologies in and applications of nlohmann/json should always be considered in addition to the guidance above. At the core, a faulty design is inherently difficult to mitigate. The first diff --git a/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md b/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md index 8e2a528595..6e31d11d95 100644 --- a/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md +++ b/TSF/trustable/assertions/TA-RELEASES_CONTEXT.md @@ -3,15 +3,17 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** -This assertion is satisfied if each iteration of XYZ is *repeatable*, with all -required inputs controlled, and *reproducible* (covering both XYZ and the +This assertion is satisfied if each iteration of nlohmann/json is *repeatable*, with all +required inputs controlled, and *reproducible* (covering both nlohmann/json and the construction toolchain/environment, as described in TA-TESTS). This assertion can be most effectively satisfied in a Continuous Integration environment with mirrored projects (see TA-SUPPLY_CHAIN) and build servers -without internet access. The aim is to show that all build tools, XYZ +without internet access. The aim is to show that all build tools, nlohmann/json components, and dependencies are built from controlled inputs, that rebuilding produces the same binary fileset, and that builds can be repeated on any suitably configured server, with server differences shown not to affect @@ -30,7 +32,7 @@ reproducibility. As a result, we gain increased confidence that the toolchain behaves correctly during version upgrades: unintended changes to the project are avoided, intended -fixes produce the expected effects, and the constructed output of XYZ shows the +fixes produce the expected effects, and the constructed output of nlohmann/json shows the correct behavioural changes, verified and validated with test results according to TT-RESULTS analysis. diff --git a/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md b/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md index 48388f0cb6..22927dd1be 100644 --- a/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md +++ b/TSF/trustable/assertions/TA-SUPPLY_CHAIN_CONTEXT.md @@ -3,12 +3,14 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** This assertion is satisfied to the extent that we have traced and captured -source code for XYZ and all of its dependencies (including transitive +source code for nlohmann/json and all of its dependencies (including transitive dependencies, all the way down), and for all of the tools used to construct -XYZ from source, and have mirrored versions of these inputs under our control. +nlohmann/json from source, and have mirrored versions of these inputs under our control. Any associated data and documentation dependencies must also be considered. 'Mirrored' in this context means that we have a version of the upstream project @@ -28,12 +30,12 @@ binaries present within the repositories of the components and tools used. **Evidence** -- list of all XYZ components including +- list of all nlohmann/json components including - URL of mirrored projects in controlled environment - **Answer**: - URL of upstream projects - **Answer**: -- successful build of XYZ from source +- successful build of nlohmann/json from source - without access to external source projects - **Answer**: - without access to cached data diff --git a/TSF/trustable/assertions/TA-TESTS_CONTEXT.md b/TSF/trustable/assertions/TA-TESTS_CONTEXT.md index 2066e8addd..c42845fed5 100644 --- a/TSF/trustable/assertions/TA-TESTS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-TESTS_CONTEXT.md @@ -3,17 +3,19 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** This assertion is satisfied if all build and test environments and tools used to support Expectations are shown to be *reproducible*, all build and test steps are *repeatable*, and all required inputs are controlled. -TA-TESTS does not include reproducibility of XYZ itself, this is instead included in TA-RELEASES. +TA-TESTS does not include reproducibility of nlohmann/json itself, this is instead included in TA-RELEASES. All tools and test environments should be constructed from change-managed sources (see TA-UPDATES) and mirrored sources (see TA-SUPPLY_CHAIN). Additional evidence needs to demonstrate that construction of tools and environments produces the same binary fileset used for testing and that builds can be repeated on any suitably configured server -(similar to how the XYZ is evaluated for TA-RELEASES). +(similar to how the nlohmann/json is evaluated for TA-RELEASES). Test environment repeatability should be ensured to enable effective Misbehaviour investigations, and enable additional data generations (including those by third parties). diff --git a/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md b/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md index 4d39aad5e7..315573df30 100644 --- a/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md +++ b/TSF/trustable/assertions/TA-UPDATES_CONTEXT.md @@ -3,9 +3,11 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** -This assertion requires control over all changes to XYZ, including configurations, components, tools, data, documentation, and dependency versions used to build, verify, and validate it. +This assertion requires control over all changes to nlohmann/json, including configurations, components, tools, data, documentation, and dependency versions used to build, verify, and validate it. As part of change control, all automated checks must run and pass (e.g., tests, static analysis, lint checks) before accepting proposed changes. These checks must be configured against appropriate claims and coding guidelines. @@ -41,25 +43,25 @@ Ultimately, the trustable controlled process is the only path to production for **Evidence** - change management process and configuration artifacts - - **Answer**: Provided in JLS-06, JLS-07, JLS-12, JLS-32, JLS-34 and JLS-35. + - **Answer**: Provided in JLS-06, JLS-07, JLS-12, JLS-32, JLS-34, JLS-35 and AOU-27. **Confidence scoring** Confidence scoring for TA-UPDATES is based on confidence that we have -control over the changes that we make to XYZ, including its configuration and +control over the changes that we make to nlohmann/json, including its configuration and dependencies. **Checklist** - Where are the change and configuration management controls specified? - - **Answer**: In the contribution guidelines and security policy. + - **Answer**: In the [contribution guidelines](https://github.com/nlohmann/json?tab=contributing-ov-file#readme) and [security policy](https://github.com/nlohmann/json?tab=security-ov-file#readme). - Are these controls enforced for all of components, tools, data, documentation and configurations? - **Answer**: Yes. Any proposed change is subject to the same change controls. - Are there any ways in which these controls can be subverted, and have we mitigated them? - **Answer**: No. The controls are enforced using branch protection rules and are mostly automated. - Does change control capture all potential regressions? - - **Answer**: Yes. All changes are rigorously tested in a separate branch before being deployed to the master branch. + - **Answer**: Yes. All changes are tested in the branch '[develop](https://github.com/nlohmann/json/actions?query=branch%3Adevelop)' before being deployed to the master branch. - Is change control timely enough? - - **Answer**: The change control is indeed timely enough, mainly due to the fact that the nlohmann/json library is open source and has many contributors. + - **Answer**: The change control is indeed timely enough. Any [issues](https://github.com/nlohmann/json/issues) or [discussions](https://github.com/nlohmann/json/discussions) opened are addressed within a reasonable time frame. - Are all guidance and checks understandable and consistently followed? - - **Answer**: Yes. The contribution guidelines and CI workflows are clear and well-documented, as well as automatically enforced. + - **Answer**: Yes. The [contribution guideline](https://github.com/nlohmann/json?tab=contributing-ov-file#readme) is clear and well-documented, and checks are enforced through automatic [CI workflows](https://github.com/nlohmann/json/tree/develop/.github/workflows). diff --git a/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md b/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md index e06f821d8a..54de3dcecb 100644 --- a/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md +++ b/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md @@ -3,10 +3,12 @@ level: 1.1 normative: false --- +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) + **Guidance** This assertion is satisfied when tests demonstrate that the features specified to meet project Expectations (TT-EXPECTATIONS) are present and function as intended. -These tests run repeatedly in a controlled environment (TA-TESTs) on a defined schedule (e.g., daily, per change, or per candidate release of XYZ). +These tests run repeatedly in a controlled environment (TA-TESTs) on a defined schedule (e.g., daily, per change, or per candidate release of nlohmann/json). Confidence grows when tests not only verify Expectations but also validate (continuously) that they meet stakeholder and user needs. Robust validation depends on three aspects: From fc22a5f07aabb4f2382c48d1936c9a8b02295944 Mon Sep 17 00:00:00 2001 From: LucaFgr Date: Mon, 17 Nov 2025 07:56:50 +0000 Subject: [PATCH 8/9] Moving changes from json to inc_nlohmann_json --- .dotstop.dot | 3 +++ TSF/trustable/statements/JLS-02.md | 2 +- TSF/trustable/statements/JLS-10.md | 8 +++++++- TSF/trustable/statements/JLS-16.md | 10 +++++++++- TSF/trustable/statements/JLS-38.md | 20 ++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 TSF/trustable/statements/JLS-38.md diff --git a/.dotstop.dot b/.dotstop.dot index 6ef07d7dce..f1e46ae15f 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -68,6 +68,7 @@ digraph G { "JLS-33" [sha="16ebc7717e389ac1ca349ead591b4dc5b65997e8c5f78d58d6293cd75bbe0d28"]; "JLS-34" [sha="3484d9766deace45ecbc0d6892c7114ce7d97a51836399887500a318b3a88dc6"]; "JLS-35" [sha="b11006d1d5708c3aba84d4f06834ad965d6aebde8619306389a4f8fa655b2dcf"]; +"JLS-38" [sha="a7ab6cc546e4c9c02c6dc479b8fff29ef6f5be5459185daac4ad1117b2845115"]; "NJF-01" [sha="548dc86014e093974f68660942daa231271496a471885bbed092a375b3079bd8"]; "NJF-02" [sha="6ea015646d696e3f014390ff41612eab66ac940f20cf27ce933cbadf8482d526"]; "NJF-03" [sha="4bd1f8210b7bba9a248055a437f377d9da0b7576c5e3ed053606cf8b5b2febe3"]; @@ -426,8 +427,10 @@ digraph G { "TA-MISBEHAVIOURS" -> "JLS-31" [sha="ff3352e20146a81904da6d8d94b003b4e0acbc2a8a63a73ea017ea0535e45e79"]; "TA-RELEASES" -> "JLS-14" [sha="1202b9934353436fba927de6762cf62a8fc23ab0815a3c06f9d0a77b55830720"]; "TA-RELEASES" -> "JLS-21" [sha="5d57d2b547a841bb31f29034b785d9bec1ffb0e495d80e0e356a54391aa22e1b"]; +"TA-RELEASES" -> "JLS-38" [sha="ed0d250c8c191ac4fc03712a321bf399d26e932edabb13067fbcffae1640cb90"]; "TA-SUPPLY_CHAIN" -> "JLS-23" [sha=fe2b810e22c4da9911266183bc8679a56d8dd2d5a76624cd1f3ee329d9b93a08]; "TA-TESTS" -> "JLS-16" [sha=a4143b13d9ae2553534457603bdca9beb6cca0ee5b8b9bae50cefa97d2519702]; +"TA-TESTS" -> "JLS-02" [sha="e99cf5b009b3cdc149edc81b3454dddfaf69ab10f80e70ce698bcfb823b5fbd1"]; "TA-UPDATES" -> "JLS-06" [sha="7386ba4dfdca14a2b0c73b6b759ddeee66e0196f164322d552c2867e5c7a4b96"]; "TA-UPDATES" -> "JLS-07" [sha="9256bec79e828b44dd12d4298483bbab7ab24a1eb542c133ee5392ee5829cb7f"]; "TA-UPDATES" -> "JLS-12" [sha="45b7cf8eebee7a35ba39b3f990cefe3cbfd79c5f74415c5321026d64d89f5703"]; diff --git a/TSF/trustable/statements/JLS-02.md b/TSF/trustable/statements/JLS-02.md index 814585ea16..62312fbf92 100644 --- a/TSF/trustable/statements/JLS-02.md +++ b/TSF/trustable/statements/JLS-02.md @@ -25,4 +25,4 @@ score: Erikhu1: 1.0 --- -Fuzz testing is used in the original nlohmann/json repository (https://github.com/nlohmann/json) to uncover edge cases and failure modes throughout development. (https://github.com/nlohmann/json/blob/develop/tests/fuzzing.md) \ No newline at end of file +Fuzz testing is used in the nlohmann/json repository to uncover edge cases and failure modes throughout development. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-10.md b/TSF/trustable/statements/JLS-10.md index e6c0555d38..446c5bb226 100644 --- a/TSF/trustable/statements/JLS-10.md +++ b/TSF/trustable/statements/JLS-10.md @@ -1,6 +1,12 @@ --- level: 1.1 normative: true +references: + - type: website + url: "https://github.com/nlohmann/json/releases" + description: "List of nlohmann/json releases consisting of source code, build instructions, test code and test result summaries." --- -Every release of nlohmann/json includes source code, build instructions, tests and attestations. (TODO: Test result summary) \ No newline at end of file +Every release of the nlohmann/json library includes source code, build instructions, test code, test results summaries and attestations. + +NOTE: Currently, I cannot find any attestations for nlohmann/json. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-16.md b/TSF/trustable/statements/JLS-16.md index 9fee7fc698..daf78c64e8 100644 --- a/TSF/trustable/statements/JLS-16.md +++ b/TSF/trustable/statements/JLS-16.md @@ -3,14 +3,22 @@ references: - type: verbose_file path: "./TSF/docs/list_of_test_environments.md" comment: "The list of all test-cases together with their execution environments" + - type: website + url: "https://github.com/score-json/json/actions" + description: "Github actions page showing that score-json is using Github host environment." evidence: type: check_list_of_tests configuration: sources: - "./tests/src" - "./TSF/tests" + type: https_response_time + configuration: + target: 2.0 + urls: + - https://github.com/score-json/json/actions level: 1.1 normative: true --- -A list of tests, which is extracted from the test execution, is provided, along with a list of test environments. \ No newline at end of file +A list of tests, which is extracted from the test execution, is provided, along with a list of test environments, a list of fault induction tests and test construction configurations and results. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-38.md b/TSF/trustable/statements/JLS-38.md new file mode 100644 index 0000000000..d4a8b1046a --- /dev/null +++ b/TSF/trustable/statements/JLS-38.md @@ -0,0 +1,20 @@ +--- +level: 1.1 +normative: true +references: + - type: website + url: "https://json.nlohmann.me/integration/cmake/" + description: "cmake build management documentation for nlohmann/json" + - type: website + url: "https://json.nlohmann.me/integration/package_managers/" + description: "package manager documentation for nlohmann/json" +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/integration/cmake/" + - "https://json.nlohmann.me/integration/package_managers/" +--- + +Every release of the nlohmann/json library shall provide configuration management for build instructions and infrastructure. \ No newline at end of file From 4ffa14022510deada3c093710299a10c50c755fb Mon Sep 17 00:00:00 2001 From: LucaFgr Date: Mon, 17 Nov 2025 07:56:50 +0000 Subject: [PATCH 9/9] Moving changes from json to inc_nlohmann_json --- .dotstop.dot | 3 +++ TSF/trustable/statements/JLS-02.md | 2 +- TSF/trustable/statements/JLS-10.md | 8 +++++++- TSF/trustable/statements/JLS-16.md | 10 +++++++++- TSF/trustable/statements/JLS-38.md | 20 ++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 TSF/trustable/statements/JLS-38.md diff --git a/.dotstop.dot b/.dotstop.dot index 6ef07d7dce..f1e46ae15f 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -68,6 +68,7 @@ digraph G { "JLS-33" [sha="16ebc7717e389ac1ca349ead591b4dc5b65997e8c5f78d58d6293cd75bbe0d28"]; "JLS-34" [sha="3484d9766deace45ecbc0d6892c7114ce7d97a51836399887500a318b3a88dc6"]; "JLS-35" [sha="b11006d1d5708c3aba84d4f06834ad965d6aebde8619306389a4f8fa655b2dcf"]; +"JLS-38" [sha="a7ab6cc546e4c9c02c6dc479b8fff29ef6f5be5459185daac4ad1117b2845115"]; "NJF-01" [sha="548dc86014e093974f68660942daa231271496a471885bbed092a375b3079bd8"]; "NJF-02" [sha="6ea015646d696e3f014390ff41612eab66ac940f20cf27ce933cbadf8482d526"]; "NJF-03" [sha="4bd1f8210b7bba9a248055a437f377d9da0b7576c5e3ed053606cf8b5b2febe3"]; @@ -426,8 +427,10 @@ digraph G { "TA-MISBEHAVIOURS" -> "JLS-31" [sha="ff3352e20146a81904da6d8d94b003b4e0acbc2a8a63a73ea017ea0535e45e79"]; "TA-RELEASES" -> "JLS-14" [sha="1202b9934353436fba927de6762cf62a8fc23ab0815a3c06f9d0a77b55830720"]; "TA-RELEASES" -> "JLS-21" [sha="5d57d2b547a841bb31f29034b785d9bec1ffb0e495d80e0e356a54391aa22e1b"]; +"TA-RELEASES" -> "JLS-38" [sha="ed0d250c8c191ac4fc03712a321bf399d26e932edabb13067fbcffae1640cb90"]; "TA-SUPPLY_CHAIN" -> "JLS-23" [sha=fe2b810e22c4da9911266183bc8679a56d8dd2d5a76624cd1f3ee329d9b93a08]; "TA-TESTS" -> "JLS-16" [sha=a4143b13d9ae2553534457603bdca9beb6cca0ee5b8b9bae50cefa97d2519702]; +"TA-TESTS" -> "JLS-02" [sha="e99cf5b009b3cdc149edc81b3454dddfaf69ab10f80e70ce698bcfb823b5fbd1"]; "TA-UPDATES" -> "JLS-06" [sha="7386ba4dfdca14a2b0c73b6b759ddeee66e0196f164322d552c2867e5c7a4b96"]; "TA-UPDATES" -> "JLS-07" [sha="9256bec79e828b44dd12d4298483bbab7ab24a1eb542c133ee5392ee5829cb7f"]; "TA-UPDATES" -> "JLS-12" [sha="45b7cf8eebee7a35ba39b3f990cefe3cbfd79c5f74415c5321026d64d89f5703"]; diff --git a/TSF/trustable/statements/JLS-02.md b/TSF/trustable/statements/JLS-02.md index 814585ea16..62312fbf92 100644 --- a/TSF/trustable/statements/JLS-02.md +++ b/TSF/trustable/statements/JLS-02.md @@ -25,4 +25,4 @@ score: Erikhu1: 1.0 --- -Fuzz testing is used in the original nlohmann/json repository (https://github.com/nlohmann/json) to uncover edge cases and failure modes throughout development. (https://github.com/nlohmann/json/blob/develop/tests/fuzzing.md) \ No newline at end of file +Fuzz testing is used in the nlohmann/json repository to uncover edge cases and failure modes throughout development. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-10.md b/TSF/trustable/statements/JLS-10.md index e6c0555d38..446c5bb226 100644 --- a/TSF/trustable/statements/JLS-10.md +++ b/TSF/trustable/statements/JLS-10.md @@ -1,6 +1,12 @@ --- level: 1.1 normative: true +references: + - type: website + url: "https://github.com/nlohmann/json/releases" + description: "List of nlohmann/json releases consisting of source code, build instructions, test code and test result summaries." --- -Every release of nlohmann/json includes source code, build instructions, tests and attestations. (TODO: Test result summary) \ No newline at end of file +Every release of the nlohmann/json library includes source code, build instructions, test code, test results summaries and attestations. + +NOTE: Currently, I cannot find any attestations for nlohmann/json. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-16.md b/TSF/trustable/statements/JLS-16.md index 9fee7fc698..daf78c64e8 100644 --- a/TSF/trustable/statements/JLS-16.md +++ b/TSF/trustable/statements/JLS-16.md @@ -3,14 +3,22 @@ references: - type: verbose_file path: "./TSF/docs/list_of_test_environments.md" comment: "The list of all test-cases together with their execution environments" + - type: website + url: "https://github.com/score-json/json/actions" + description: "Github actions page showing that score-json is using Github host environment." evidence: type: check_list_of_tests configuration: sources: - "./tests/src" - "./TSF/tests" + type: https_response_time + configuration: + target: 2.0 + urls: + - https://github.com/score-json/json/actions level: 1.1 normative: true --- -A list of tests, which is extracted from the test execution, is provided, along with a list of test environments. \ No newline at end of file +A list of tests, which is extracted from the test execution, is provided, along with a list of test environments, a list of fault induction tests and test construction configurations and results. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-38.md b/TSF/trustable/statements/JLS-38.md new file mode 100644 index 0000000000..d4a8b1046a --- /dev/null +++ b/TSF/trustable/statements/JLS-38.md @@ -0,0 +1,20 @@ +--- +level: 1.1 +normative: true +references: + - type: website + url: "https://json.nlohmann.me/integration/cmake/" + description: "cmake build management documentation for nlohmann/json" + - type: website + url: "https://json.nlohmann.me/integration/package_managers/" + description: "package manager documentation for nlohmann/json" +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/integration/cmake/" + - "https://json.nlohmann.me/integration/package_managers/" +--- + +Every release of the nlohmann/json library shall provide configuration management for build instructions and infrastructure. \ No newline at end of file