Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
271 commits
Select commit Hold shift + click to select a range
b91fff5
Pylint (#32)
jhughesoti May 30, 2023
b84a026
Add license header (#36)
jhughesoti May 30, 2023
622b12d
merge main
jhughesoti May 31, 2023
38d71aa
Ovs (#35)
jhughesoti May 31, 2023
00be9cb
remove ovs files added back in during merge
jhughesoti May 31, 2023
f331239
Nmap (#38)
jhughesoti Jun 5, 2023
2a68fba
Create startup capture (#37)
jboddey Jun 5, 2023
8e8e154
Connection (#40)
jhughesoti Jun 7, 2023
6ff220b
Conn mac oui (#42)
jhughesoti Jun 8, 2023
4ca8f44
Con mac address (#43)
jhughesoti Jun 8, 2023
ff04f43
Dns (#44)
jhughesoti Jun 8, 2023
752f701
File permissions (#45)
jhughesoti Jun 9, 2023
f6e4e93
Add connection single ip test (#47)
jhughesoti Jun 12, 2023
bca0db8
Nmap results (#49)
jhughesoti Jun 15, 2023
5b56a79
Framework restructure (#50)
jboddey Jun 15, 2023
7bb9366
Ip control (#51)
jhughesoti Jun 20, 2023
b0d14c2
Move config to /local (#52)
jboddey Jun 23, 2023
94e937f
Add documentation (#53)
jboddey Jun 23, 2023
098de20
Sync dev to main (#56)
jboddey Jun 28, 2023
f185bb1
Fix missing results on udp tests when tcp ports are also defined (#59)
jhughesoti Jun 29, 2023
355c838
Add licence header (#61)
jboddey Jul 3, 2023
d374640
Resolve merge conflicts
jboddey Jul 3, 2023
8d65386
Resolve merge conflict
jboddey Jul 3, 2023
26f8c5b
Add network docs (#63)
jboddey Jul 4, 2023
4a5c1ea
Dhcp (#64)
jhughesoti Jul 5, 2023
af8367c
Dhcp (#67)
jhughesoti Jul 6, 2023
7dd5772
Add connection.dhcp_address test (#68)
jhughesoti Jul 6, 2023
9ef0d4f
Add NTP tests (#60)
jboddey Jul 12, 2023
2ae337d
Add ipv6 tests (#65)
jboddey Jul 12, 2023
59ab65f
Merge branch 'main' into dev
jboddey Jul 12, 2023
fe4bf43
Connection private address (#71)
jhughesoti Jul 13, 2023
0c550c8
fix windows line ending
jhughesoti Jul 13, 2023
2df6b4a
Fix python import
jboddey Jul 13, 2023
bfae1e0
move isc-dhcp service commands to their own class
jhughesoti Jul 13, 2023
4bdacf3
fix dhcp1
jhughesoti Jul 13, 2023
a57393b
Merge branch 'main' into dev
jboddey Jul 14, 2023
d376655
Initial CI testing for tests (#72)
noursaidi Jul 14, 2023
01761dd
Fix radvd conf
jboddey Jul 14, 2023
c23f258
Fix individual test disable
jhughesoti Jul 14, 2023
97c4c65
Merge branch 'dev' of https://github.com/auto-iot/test-run into dev
jhughesoti Jul 14, 2023
5d24497
Add NTP Pass CI test (#76)
noursaidi Jul 17, 2023
006aa11
add shared address test (#75)
jhughesoti Jul 17, 2023
153de19
Fix single ip test (#58)
jhughesoti Jul 17, 2023
e2c934e
Merge API into dev (#70)
jboddey Jul 24, 2023
0023003
Dhcp tests (#81)
jhughesoti Aug 2, 2023
4b1d69c
Merge tls tests into dev (#80)
jboddey Aug 2, 2023
4ce26a1
Test output restructure (#79)
jhughesoti Aug 3, 2023
c78f2de
Keep test results in memory (#82)
jboddey Aug 3, 2023
11c9740
Result descriptions (#92)
jhughesoti Aug 11, 2023
9046ffd
Misc cleanup (#93)
jhughesoti Aug 11, 2023
fca08ce
Resolve merge conflicts
jboddey Aug 15, 2023
5c9c7cc
Allow CORS (#91)
jboddey Aug 16, 2023
29242c4
Add /history and device config endpoints (#88)
jboddey Aug 16, 2023
290ba50
Re-enable actions, fix conn module (#89)
jboddey Aug 16, 2023
38792e8
Add required result to module configs (#95)
jboddey Aug 16, 2023
6afd9ea
Fix DNS test name (#110)
jboddey Aug 31, 2023
2cbc343
Clear runtime folder on start (#111)
jboddey Aug 31, 2023
f98ea18
Initial work on pdf report output and format via html (#103)
jhughesoti Sep 1, 2023
8d7c6f7
Add user interface (#98)
jboddey Sep 1, 2023
13ec5b8
Add some documentation
jboddey Sep 4, 2023
702147c
Save test modules (#115)
jboddey Sep 9, 2023
5d41edb
Expand testing to include API and more testing (#96)
noursaidi Sep 11, 2023
b4dd440
Create Testrun package (#114)
jboddey Sep 12, 2023
87367ab
Merge conflicts
jboddey Sep 12, 2023
78e2d92
Remove merge conflicts
jboddey Sep 12, 2023
30baa40
Add extra validation (#118)
jboddey Sep 14, 2023
e7c2906
Refactor nmap module (#102)
jboddey Sep 14, 2023
a57080e
Add protocol module (#107)
jhughesoti Sep 15, 2023
3c10ba1
Hotfix
jboddey Sep 15, 2023
1d1719f
bug fixes (#119)
jboddey Sep 18, 2023
b42b31d
Chronyd (#116)
jhughesoti Sep 18, 2023
6bad0dd
Test fixes (#123)
jhughesoti Sep 21, 2023
62c95e1
Update README.md
jboddey Sep 21, 2023
0472605
Resolve 2 bugs (#121)
jboddey Sep 26, 2023
9975341
Bug/test count nmap check (#124)
jboddey Sep 26, 2023
9746a13
Report styling (#122)
jhughesoti Sep 26, 2023
129870b
Update UI (#125)
jboddey Sep 26, 2023
fe4275d
Test fixes (#127)
jhughesoti Sep 27, 2023
1be60c3
Add docs, disable network validation by default (#126)
jboddey Sep 28, 2023
61a184d
Implement delete device endpoint (#129)
jboddey Oct 2, 2023
b3242d9
Sync dev with main (#156)
jboddey Oct 16, 2023
b4afc7d
Delete report (#157)
jboddey Oct 16, 2023
da9f771
Ipv6 updates (#131)
jhughesoti Oct 20, 2023
b652479
Update UI
jboddey Oct 24, 2023
d5888a2
Bug fixes (#166)
jhughesoti Oct 27, 2023
74cf624
Merge main hotfix into dev (#165)
jboddey Oct 31, 2023
9b38941
Conn test fixes (#167)
jhughesoti Nov 6, 2023
982c1b9
Add workflow for UI test (#177)
jboddey Nov 9, 2023
8445bbb
Fix duplicate port number (#175)
jboddey Nov 10, 2023
0104bb5
Check all packet captures for SLAAC address (#173)
jboddey Nov 13, 2023
c6bcda4
Remove interfaces which should not be chosen (#161)
jboddey Nov 13, 2023
31b2c36
Fix device search issue (#176)
jboddey Nov 13, 2023
86a1d1f
Add hotfix to dev (#181)
jboddey Nov 13, 2023
9fa6f03
Bug fixes
jboddey Nov 13, 2023
811a623
Increase connection timeout to 30 minutes (#185)
jhughesoti Dec 1, 2023
9d327bc
Add new testrun statuses (#195)
jboddey Dec 3, 2023
aa5edc5
Resolve some bugs (#193)
jboddey Dec 3, 2023
d463e5e
Check that Docker is installed (#178)
jboddey Dec 3, 2023
a1d5e8c
Add version endpoint (#196)
jboddey Dec 3, 2023
357fc49
Fix bugs and update UI (#198)
jboddey Dec 8, 2023
905be03
Update UI and API to allow for remote connections (#202)
jhughesoti Dec 8, 2023
4de04da
Add device configuration to PDF report (#203)
jboddey Dec 9, 2023
c8a78ba
Add separate error message if config not completed (#197)
jboddey Dec 9, 2023
0ffef4a
Resolve merge conflicts with main (#208)
jboddey Dec 14, 2023
bf14ce8
Add instruction for local CA cert (#204)
jboddey Dec 14, 2023
d36c0d0
Add UI and update README (#207)
jboddey Dec 14, 2023
19d044a
Fix remote ui (#205)
jhughesoti Dec 14, 2023
bc1830a
Resolve merge conflicts
jboddey Dec 15, 2023
5ef25aa
Remove unused docker file
jboddey Dec 15, 2023
5526c9f
Change buffer option in network and test module capture files so pack…
jhughesoti Jan 2, 2024
fe4769c
Remove unnecessary OUI.txt downloads (#217)
jboddey Jan 4, 2024
e16d6bb
Set permissions of workflows (#218)
jboddey Jan 4, 2024
e876e03
Add check before launching UI (#216)
jboddey Jan 4, 2024
0eefd21
Set owner of config file during install (#211)
jboddey Jan 5, 2024
0ec1036
Add NTP server network captures to NTP traffic scan (#222)
jhughesoti Jan 8, 2024
021b4ba
Add edit device endpoint (#213)
jboddey Jan 9, 2024
1868a06
Module reports (#231)
jhughesoti Jan 22, 2024
3f29161
Nmap report (#239)
jhughesoti Feb 5, 2024
ead1f08
Add zip export (#228)
jboddey Feb 5, 2024
fda3745
Merge release v1.1.1 into dev (#237)
jboddey Feb 7, 2024
6f17edb
Feature/dev UI (#260)
sofyakurilova Feb 8, 2024
7c5e0e3
Refactoring: (#263)
sofyakurilova Feb 9, 2024
98c39be
Add log level and monitoring fields (#264)
sofyakurilova Feb 12, 2024
81ad642
323348639: (feat) add expanded panels with steps to resolve (#267)
OlgaMardvilko Feb 13, 2024
45eb58f
324383967: (feat) display in progress test status (#268)
OlgaMardvilko Feb 13, 2024
84ad364
Update gateway build (#272)
jboddey Feb 14, 2024
825ce9f
322355032: (feat) add high and limited compliant test statuses (#273)
OlgaMardvilko Feb 15, 2024
bf7ac81
315288666: (fix) increase the request wait limit up to 10 seconds whe…
OlgaMardvilko Feb 15, 2024
2a85777
Adds component state for reports page (#276)
sofyakurilova Feb 16, 2024
57ecc56
325419639: (fix) [a11y] change styles for meet GAR 2.11 for the arrow…
OlgaMardvilko Feb 16, 2024
d9130a2
Ntp report (#279)
jhughesoti Feb 19, 2024
f12965d
324202752: (fix) device name and device model name cutting (#282)
OlgaMardvilko Feb 19, 2024
529778c
Save button is disabled when port is no longer available (#281)
sofyakurilova Feb 19, 2024
fc87da7
324061101: (fix) [a11y] GAR 1.3 the focus is not placed to the snackb…
OlgaMardvilko Feb 21, 2024
b810113
Add steps to resolve to dev (#257)
jboddey Feb 21, 2024
fc533f8
Add in progress tests to session (#236)
jboddey Feb 22, 2024
9033e38
324864346: (fix) [a11Y] change color for field label to fix GAR 1.14 …
OlgaMardvilko Feb 22, 2024
87f79a4
Adds consent window to grant/deny GA (#285)
sofyakurilova Feb 22, 2024
dca5892
Merge release v1.1.2 into dev (#274)
jboddey Feb 22, 2024
e56c07a
Update interfaces on settings open (#288)
sofyakurilova Feb 22, 2024
e2a2007
Show consent dialog only once per session (#289)
sofyakurilova Feb 23, 2024
b23163b
Bump version
jboddey Feb 23, 2024
3231f89
Remove duplicate device config (#290)
jboddey Feb 26, 2024
ed7ce71
Report cleanup (#297)
jhughesoti Feb 27, 2024
e4cea78
Fix text; fix gaps (#299)
sofyakurilova Mar 1, 2024
37c1eaa
Re-enable protocol tests (#277)
jboddey Mar 1, 2024
139e212
326951948: (fix) [a11y] add accessible names for welcome modal and bu…
OlgaMardvilko Mar 1, 2024
739f0ff
Feature/add bacnet version (#300)
jhughesoti Mar 1, 2024
6052f88
Fix focus style when consent pop-up closed (#301)
sofyakurilova Mar 4, 2024
9fd97c9
Adds state for devices component; devices added to shared state (#298)
sofyakurilova Mar 4, 2024
e753023
Return old version if request fails (#303)
sofyakurilova Mar 4, 2024
5edec94
327410289: (feat) add protocol module (#307)
OlgaMardvilko Mar 4, 2024
ff373c8
327607397: (fix) update packages to fix security issue from dependabo…
OlgaMardvilko Mar 5, 2024
df04bd4
327544271: (feat) update texts in app (#310)
OlgaMardvilko Mar 5, 2024
d1ed30a
327324282: (fix) remove compliant high and limited statuses from filt…
OlgaMardvilko Mar 6, 2024
0b75c05
325424134: (fix) change borders for results table to be visible on sc…
OlgaMardvilko Mar 6, 2024
8892e41
Remove steps to resolve from in progress tests (#302)
jboddey Mar 7, 2024
73600b1
Improve dev process (#286)
jboddey Mar 7, 2024
41349a6
326193923: (fix) change to prevent the callout for step 1 blinks on l…
OlgaMardvilko Mar 7, 2024
6a7fa0c
327722371 (fix) make the Welcome modal appears in case the version re…
OlgaMardvilko Mar 8, 2024
a2c920e
Change test status when cancelling test (#305)
jboddey Mar 8, 2024
405c098
327161779: (fix) change get status with finished data to display a pr…
OlgaMardvilko Mar 11, 2024
7b0ef54
327324281: (feat) change callout messages for missing setting ports (…
OlgaMardvilko Mar 12, 2024
1f2bdaa
Remove consent dialog close on esc (#322)
sofyakurilova Mar 12, 2024
6692791
Update documentation (#319)
jboddey Mar 12, 2024
ea735f8
Fix padding in menu item (#323)
sofyakurilova Mar 12, 2024
8eb5c40
327352736: (fix) add notification about validation on the create devi…
OlgaMardvilko Mar 13, 2024
2c8522b
Update README.md
jboddey Mar 13, 2024
e199f7a
Change progress page spinner size to small (#327)
sofyakurilova Mar 13, 2024
1f42928
Adds whole row highlight on hover (steps to resolve) (#328)
sofyakurilova Mar 13, 2024
d910b7c
Save expand/collapse state (#330)
sofyakurilova Mar 13, 2024
8c5864f
Adds zip report link (#333)
sofyakurilova Mar 15, 2024
f779f5a
Tls module report (#331)
jhughesoti Mar 18, 2024
2ae774f
329086273: (fix) [a11Y] change the version button accessible name (#336)
OlgaMardvilko Mar 18, 2024
152711a
328175839: (fix) [a11Y] change to set focus on opening the settings p…
OlgaMardvilko Mar 18, 2024
b87b32b
Disable test_tests temporarily (#338)
jboddey Mar 19, 2024
3ddf4c7
Fix edge cases in listener (#341)
jboddey Mar 19, 2024
ebcae1e
Add hotfix changes (#332)
jboddey Mar 19, 2024
35e5526
Updates error message for version endpoint (#340)
sofyakurilova Mar 20, 2024
34962bb
Adds empty table for cancelled state (if no tests are available) (#337)
sofyakurilova Mar 20, 2024
f1e5e54
Remove cancel on download click (#334)
sofyakurilova Mar 20, 2024
3a1bba1
Run package on every dev commit and every PR (#296)
jboddey Mar 20, 2024
8427c4f
Bug/tls server cert chain (#344)
jhughesoti Mar 21, 2024
2f8d6f8
329834205: (fix) apply Angular tooltip to fix the long text issue on …
OlgaMardvilko Mar 21, 2024
fefaceb
Fix sort and filtering on data source update (#343)
sofyakurilova Mar 21, 2024
58e79b2
Fix report removal from session (#320)
jboddey Mar 22, 2024
af86ac0
Zip report updates (#348)
jhughesoti Mar 25, 2024
ff9d660
Add virtual machine install guide (#174)
jboddey Mar 25, 2024
12c4fff
Improve design for module pages (#314)
jboddey Mar 25, 2024
0e6c254
Resolve merge conflicts (#342)
jboddey Mar 25, 2024
51930e6
Resolve merge conflicts
jboddey Mar 25, 2024
88c0737
Remove postinst file
jboddey Mar 25, 2024
80aede7
328405692: (fix) [a11y] add validation, error messages, hints for fil…
OlgaMardvilko Mar 25, 2024
545c225
330699330: (fix) change styles to prevent unexpected space on pages w…
OlgaMardvilko Mar 25, 2024
5e66392
improve pylint score (#353)
jhughesoti Mar 26, 2024
dbd272b
Bump dependabot alerts (#349)
jboddey Mar 26, 2024
4ab6ccb
fix: update outdated packages to fix security issues (#354)
OlgaMardvilko Mar 26, 2024
fe6cfca
Cache oui.txt to prevent install fail (#355)
jboddey Mar 26, 2024
99d4159
330700777: (fix) update styles for fixing ui focus border issues (#356)
OlgaMardvilko Mar 26, 2024
1fc37ca
Add private ip subnet filter for non-tls traffic (#352)
jhughesoti Mar 26, 2024
d619780
331364084: (fix) update label and placeholder for started modal filte…
OlgaMardvilko Mar 27, 2024
feda0da
328745809: (fix) [a11y] change styles to status card and device item …
OlgaMardvilko Mar 27, 2024
7ab6e59
331572594: (fix) [a11Y] GAR 1.13 The firmware field validation (#359)
OlgaMardvilko Mar 27, 2024
a275b08
Add arp inspection and dhcp snooping tests (#280)
jboddey Mar 27, 2024
b0a858d
Remove all pylint errors in framework (#360)
jboddey Mar 28, 2024
533cdd2
Update oui cache file (#365)
jhughesoti Apr 2, 2024
cbf9328
pin python requirement versions (#366)
jhughesoti Apr 2, 2024
83a678e
Fix bug with ARP inspection test (#369)
jboddey Apr 4, 2024
61e69f1
Bug/docker network dns (#368)
jhughesoti Apr 5, 2024
f51aaf5
Fix bug with deleting reports (#372)
jboddey Apr 5, 2024
66f2544
Add a SYN/ACK filter to unencrypted TLS check (#373)
jhughesoti Apr 8, 2024
3b6861d
Add test status docs (#382)
jboddey Apr 8, 2024
4495e5e
Render error result in PDF report (#381)
jboddey Apr 8, 2024
f11d63b
Rename nmap module to services (#374)
jhughesoti Apr 8, 2024
f3009b7
Bump version to v1.2.1-alpha (#380)
jboddey Apr 8, 2024
8f98b3c
monitor device disconnect (#376)
jhughesoti Apr 9, 2024
f0f9843
330867613: (feat) change start testrun icon on device tile (#367)
OlgaMardvilko Apr 9, 2024
ec3f908
331378849: (feat) add log level explanation (#362)
OlgaMardvilko Apr 9, 2024
7062d37
Extend timeout when testrun is stopping (#375)
sofyakurilova Apr 10, 2024
ad4ad61
Fix modbus test and bump version (#394)
jboddey Apr 15, 2024
dc8fad2
Upgrade dependencies in TLS module (#395)
jboddey Apr 16, 2024
d7fb86b
Fix AsyncSniffer not storing packets during monitor period (#384)
jhughesoti Apr 16, 2024
1b6eb8c
Resolve merge conflicts
jboddey Apr 16, 2024
6ee5dfd
332883321: (feat) add support for Feature Not Present and Error statu…
OlgaMardvilko Apr 16, 2024
b7a30e5
Expand pylint to whole project and fix simple pylint issues (#401)
jhughesoti Apr 17, 2024
21447fa
333723882: (feat) add download options on testing page (#406)
OlgaMardvilko Apr 19, 2024
ca45d70
Fix scrolls for mozilla (#405)
sofyakurilova Apr 23, 2024
c2de937
333725552: (feat) ability to shutdown testrun (#410)
OlgaMardvilko Apr 25, 2024
2b8b2db
336737453: (fix) the reports table is not fitting the content (#416)
OlgaMardvilko Apr 25, 2024
1b7a1a8
336268076: (fix) change borders for the button and options in downloa…
OlgaMardvilko Apr 26, 2024
92e425b
Fix mac_address validation (#403)
sofyakurilova Apr 26, 2024
8fec366
Returns the currently installed version if Testrun fails to fetch the…
jboddey Apr 26, 2024
c350de7
Change model and manufacturer length (#417)
sofyakurilova Apr 29, 2024
ea354a3
Change paddings depends on table width (#408)
sofyakurilova Apr 29, 2024
a1e26ef
Change paddings depends on table width (#407)
sofyakurilova Apr 29, 2024
7d4d8c2
In progress test result (#404)
sofyakurilova Apr 29, 2024
671c513
Fixes defects in the PDF report if a long device name is specified (#…
jboddey Apr 29, 2024
4c83eca
Change report limit default behavior (#423)
jhughesoti Apr 29, 2024
f782bed
Bug/test results too long (#426)
jhughesoti May 1, 2024
5f811bd
Support certificate names with spaces in TLS test (#390)
gabrielandrade2 May 6, 2024
25da8f8
Reset module reports after testrun (#427)
jboddey May 7, 2024
2ad32b0
Update requirements.txt (#429)
jboddey May 7, 2024
c860757
Remove in progress status after discovery (#422)
jboddey May 7, 2024
b8281c6
Add steps to resolve to PDF report (#411)
jboddey May 7, 2024
87ef9ae
Bump ejs in /modules/ui in the npm_and_yarn group across 1 directory …
dependabot[bot] May 7, 2024
9ae8411
For delete report take mac_addr from upper level (#436)
sofyakurilova May 7, 2024
4bfe420
Update dependencies (#435)
sofyakurilova May 7, 2024
7820ded
Add new mac addr field for report deleting (#432)
jboddey May 7, 2024
9809ada
Bump version to v1.2.2 (#433)
jboddey May 7, 2024
ae0bae8
Pass log level to test module
jboddey May 9, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ testing/unit/dns/output/
testing/unit/nmap/output/
testing/unit/ntp/output/
testing/unit/tls/output/
testing/unit/tls/tmp/
testing/unit/report/output/

*.deb
Expand Down
9 changes: 8 additions & 1 deletion cmd/package
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
# Creates a package for Testrun

MAKE_SRC_DIR=make
TESTRUN_VER="1-2-2"

# Delete existing make files
rm -rf $MAKE_SRC_DIR/usr

# Delete existing make files
rm -rf $MAKE_SRC_DIR/usr
Expand Down Expand Up @@ -56,4 +60,7 @@ cp -r {framework,modules} $MAKE_SRC_DIR/usr/local/testrun
dpkg-deb --build --root-owner-group make

# Rename the .deb file
mv make.deb testrun_1-2-1_amd64.deb
mv make.deb testrun_${TESTRUN_VER}_amd64.deb

# Echo package version
echo Created installation package at testrun_${TESTRUN_VER}_amd64.deb
5 changes: 4 additions & 1 deletion docs/get_started.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,5 +109,8 @@ If you encounter any issues or need assistance, consider the following:
- Check the configuration settings.
- Refer to the Testrun documentation or ask for assistance in the issues page: https://github.com/google/testrun/issues

# Reviewing
Once you have completed a test attempt, you may want to review the test report provided by Testrun. For more information about what Testrun looks for when testing, and what the output means, take a look at the testing documentation: [Testing](/docs/test/index.md).

# Uninstall
To uninstall Testrun, use the built-in dpkg uninstall command to remove Testrun correctly. For Testrun, this would be: ```sudo apt-get remove testrun```
To uninstall Testrun, use the built-in dpkg uninstall command to remove Testrun correctly. For Testrun, this would be: ```sudo apt-get remove testrun```. Note that this
5 changes: 5 additions & 0 deletions docs/test/index.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Testing

The test requirements that are investigated by Testrun can be found in the [test modules documentation](/docs/test/modules.md).

To understand the testing results, various definitions of test results and requirements are specified in the [statuses documentation](/docs/test/statuses.md).
32 changes: 32 additions & 0 deletions docs/test/statuses.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Test Statuses
Testrun will output the result and description of each automated test. The test results will be one of the following:

| Name | Description | What next? |
|---|---|---|
| Compliant | The device implements the required feature correctly | Nothing |
| Non-Compliant | The device does not support the specified requirements for the test | Modify or implement the required functionality on the device |
| Feature Not Present | The device does not implement a feature covered by the test | You may implement the functionality (not required) |
| Error | An error occured whilst running the test | Create a bug report requesting additional support to diagnose the issue |
| Skipped | The test has not been executed because a linked test did not produce a compliant result | You may implement the functionality (not required) |

## Test requirement
Testrun also determines whether each test is required for the device to receive an overall compliant result. These rules are:

| Name | Description |
|---|---|
| Required | The device must implement the feature |
| Recommended | The device should implement the feature, but will not receive an overall Non-Compliant if not implemented |
| Roadmap | The device should implement this feature in the future, but is not required at the moment |
| Required If Applicable | If the device implements this feature, it must be implemented correctly (as per the test requirements) |

## Testrun Statuses
Once testing is completed, an overall result for the test attempt will be produced. This is calculated by comparing the result of all tests, and whether they are required or not required.

### Compliant
All required tests are implemented correctly, and all required if applicable tests are implemented correctly (where the feature has been implemented).

### Non-Compliant
One or more of the required tests (or required if applicable tests) have produced a non-compliant result.

### Error
One of more of the required tests (or required if applicable tests) have not executed correctly. This does not necessarily indicate that the device is compliant or non-compliant.
48 changes: 39 additions & 9 deletions framework/python/src/api/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import os
import psutil
import requests
import signal
import threading
import uvicorn
from urllib.parse import urlparse
Expand All @@ -46,7 +47,7 @@ class Api:
def __init__(self, test_run):

self._test_run = test_run
self._name = "TestRun API"
self._name = "Testrun API"
self._router = APIRouter()

self._session = self._test_run.get_session()
Expand All @@ -66,6 +67,9 @@ def __init__(self, test_run):
methods=["POST"])
self._router.add_api_route("/system/status",
self.get_status)
self._router.add_api_route("/system/shutdown",
self.shutdown,
methods=["POST"])

self._router.add_api_route("/system/version",
self.get_version)
Expand Down Expand Up @@ -243,8 +247,35 @@ async def stop_test_run(self):
async def get_status(self):
return self._test_run.get_session().to_json()

def shutdown(self, response: Response):

LOGGER.debug("Received request to shutdown Testrun")

# Check that Testrun is not currently running
if (self._session.get_status() not in [
"Cancelled",
"Compliant",
"Non-Compliant",
"Idle"]):
LOGGER.debug("Unable to shutdown Testrun as Testrun is in progress")
response.status_code = 400
return self._generate_msg(
False,
"Unable to shutdown. A test is currently in progress."
)

self._test_run.shutdown()
os.kill(os.getpid(), signal.SIGTERM)

async def get_version(self, response: Response):

# Add defaults
json_response = {}
json_response["installed_version"] = "v" + self._test_run.get_version()
json_response["update_available"] = False
json_response["latest_version"] = None
json_response["latest_version_url"] = (
"https://github.com/google/testrun/releases")

# Obtain the current version
current_version = self._session.get_version()
Expand All @@ -264,9 +295,9 @@ async def get_version(self, response: Response):
# Check OK response was received
if version_check.status_code != 200:
response.status_code = 500
LOGGER.error(version_check.content)
return self._generate_msg(False, "Failed to fetch latest version. " +
"Please, check the internet connection")
LOGGER.debug(version_check.content)
LOGGER.error("Failed to fetch latest version")
return json_response

# Extract version number from response, removing the leading 'v'
latest_version_no = version_check.json()["name"].strip("v")
Expand All @@ -289,8 +320,7 @@ async def get_version(self, response: Response):
response.status_code = 500
LOGGER.error("Failed to fetch latest version")
LOGGER.debug(e)
return self._generate_msg(False, "Failed to fetch latest version. " +
"Please, check the internet connection")
return json_response

async def get_reports(self, request: Request):
LOGGER.debug("Received reports list request")
Expand All @@ -304,7 +334,7 @@ async def get_reports(self, request: Request):
for report in reports:
# report URL is currently hard coded as localhost so we can
# replace that to fix the IP dynamically from the requester
report["report"] = report["report"].replace("localhost",server_ip)
report["report"] = report["report"].replace("localhost", server_ip)
return reports

async def delete_report(self, request: Request, response: Response):
Expand Down Expand Up @@ -531,8 +561,8 @@ def _validate_device_json(self, json_obj):
return False

# Check length of strings
if len(json_obj.get(DEVICE_MANUFACTURER_KEY)) > 64 or len(
json_obj.get(DEVICE_MODEL_KEY)) > 64:
if len(json_obj.get(DEVICE_MANUFACTURER_KEY)) > 28 or len(
json_obj.get(DEVICE_MODEL_KEY)) > 28:
return False

disallowed_chars = ["/", "\\", "\'", "\"", ";"]
Expand Down
3 changes: 3 additions & 0 deletions framework/python/src/common/device.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ def add_report(self, report):
def get_reports(self):
return self.reports

def clear_reports(self):
self.reports = []

def remove_report(self, timestamp: datetime):
for report in self.reports:
if report.get_started().strftime('%Y-%m-%dT%H:%M:%S') == timestamp:
Expand Down
24 changes: 14 additions & 10 deletions framework/python/src/common/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@

LOGGER = logger.get_logger('session')

class TestRunSession():
class TestrunSession():
"""Represents the current session of Test Run."""

def __init__(self, config_file):
def __init__(self, config_file, version):
self._status = 'Idle'
self._device = None
self._started = None
Expand All @@ -46,8 +46,7 @@ def __init__(self, config_file):
self._total_tests = 0
self._report_url = None

self._version = None
self._load_version()
self._load_version(default_version=version)

self._config_file = config_file
self._config = self._get_default_config()
Expand Down Expand Up @@ -90,7 +89,7 @@ def _get_default_config(self):
'log_level': 'INFO',
'startup_timeout': 60,
'monitor_period': 30,
'max_device_reports': 5,
'max_device_reports': 0,
'api_url': 'http://localhost',
'api_port': 8000
}
Expand Down Expand Up @@ -141,14 +140,18 @@ def _load_config(self):

LOGGER.debug(self._config)

def _load_version(self):
def _load_version(self, default_version):
version_cmd = util.run_command(
'dpkg-query --showformat=\'${Version}\' --show testrun')

if version_cmd:
# index 1 of response is the stderr byte stream so if
# it has any data in it, there was an error and we
# did not resolve the version and we'll use the fallback
if len(version_cmd[1]) == 0:
version = version_cmd[0]
LOGGER.info(f'Running Testrun version {version}')
self._version = version
self._version = version
else:
self._version = default_version
LOGGER.info(f'Running Testrun version {self._version}')

def get_version(self):
return self._version
Expand Down Expand Up @@ -294,6 +297,7 @@ def reset(self):
self.set_target_device(None)
self._report_url = None
self._total_tests = 0
self._module_reports = []
self._results = []
self._started = None
self._finished = None
Expand Down
Loading