From 002217df10723032f428223480be61033da78595 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Fri, 14 Jul 2023 18:42:03 -0400 Subject: [PATCH 1/4] dynamic cutoff --- agbenchmark/config.json | 3 +-- agbenchmark/reports/1.json | 31 +++++++++++++++---------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/agbenchmark/config.json b/agbenchmark/config.json index bc89d5ef4..5fd051ecc 100644 --- a/agbenchmark/config.json +++ b/agbenchmark/config.json @@ -1,5 +1,4 @@ { "workspace": "generated", - "entry_path": "agbenchmark/benchmarks.py", - "cutoff": 60 + "entry_path": "agbenchmark.benchmarks" } diff --git a/agbenchmark/reports/1.json b/agbenchmark/reports/1.json index 48a631c5b..a37bb9c13 100644 --- a/agbenchmark/reports/1.json +++ b/agbenchmark/reports/1.json @@ -1,18 +1,17 @@ { - "command": "agbenchmark start --test TestWriteFile --mock", - "completion_time": "2023-07-10-21:19", - "time_elapsed": "8.34 seconds", - "tests": { - "TestWriteFile": { - "difficulty": "basic", - "dependencies": [], - "test": "agbenchmark/challenges/interface/write_file", - "success": true - } - }, - "config": { - "workspace": "generated", - "entry_path": "agbenchmark/benchmarks.py", - "cutoff": 60 + "command": "agbenchmark start --test TestWriteFile --mock", + "completion_time": "2023-07-10-21:19", + "time_elapsed": "8.34 seconds", + "tests": { + "TestWriteFile": { + "difficulty": "basic", + "dependencies": [], + "test": "agbenchmark/challenges/interface/write_file", + "success": true } -} \ No newline at end of file + }, + "config": { + "workspace": "generated", + "entry_path": "agbenchmark/benchmarks.py" + } +} From bec01917a9fa6e7bd73e4d14b328dba468cae495 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Sun, 16 Jul 2023 13:10:11 -0400 Subject: [PATCH 2/4] removing entry path from config --- agbenchmark/config.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/agbenchmark/config.json b/agbenchmark/config.json index 5fd051ecc..c197cee7d 100644 --- a/agbenchmark/config.json +++ b/agbenchmark/config.json @@ -1,4 +1,3 @@ { - "workspace": "generated", - "entry_path": "agbenchmark.benchmarks" + "workspace": "generated" } From bac8549629cfc9e1766350a599fdf3a653d2dd45 Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Mon, 24 Jul 2023 18:08:40 +0100 Subject: [PATCH 3/4] fixing regression file --- agbenchmark/regression_tests.json | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/agbenchmark/regression_tests.json b/agbenchmark/regression_tests.json index b66a16cf3..5e5293571 100644 --- a/agbenchmark/regression_tests.json +++ b/agbenchmark/regression_tests.json @@ -1,10 +1,7 @@ { - "TestWriteFile": { - "difficulty": "interface", - "dependencies": [], - "data_path": "agbenchmark/challenges/interface/write_file" - }, - "TestBasicCodeGeneration": { - "data_path": "agbenchmark/challenges/code/d4", - } + "TestWriteFile": { + "difficulty": "interface", + "dependencies": [], + "data_path": "agbenchmark/challenges/interface/write_file" + } } From 2bdb7f24a8d28c2e8eac402cfd4fbda7dbc1ba8f Mon Sep 17 00:00:00 2001 From: Silen Naihin Date: Mon, 31 Jul 2023 16:09:42 +0100 Subject: [PATCH 4/4] new agbenchmark layout --- agbenchmark/regression_tests.json | 7 ------- agbenchmark/reports/1.json | 17 ----------------- agbenchmark/reports/regression_tests.json | 10 ++++++++++ 3 files changed, 10 insertions(+), 24 deletions(-) delete mode 100644 agbenchmark/regression_tests.json delete mode 100644 agbenchmark/reports/1.json create mode 100644 agbenchmark/reports/regression_tests.json diff --git a/agbenchmark/regression_tests.json b/agbenchmark/regression_tests.json deleted file mode 100644 index 5e5293571..000000000 --- a/agbenchmark/regression_tests.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "TestWriteFile": { - "difficulty": "interface", - "dependencies": [], - "data_path": "agbenchmark/challenges/interface/write_file" - } -} diff --git a/agbenchmark/reports/1.json b/agbenchmark/reports/1.json deleted file mode 100644 index a37bb9c13..000000000 --- a/agbenchmark/reports/1.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "command": "agbenchmark start --test TestWriteFile --mock", - "completion_time": "2023-07-10-21:19", - "time_elapsed": "8.34 seconds", - "tests": { - "TestWriteFile": { - "difficulty": "basic", - "dependencies": [], - "test": "agbenchmark/challenges/interface/write_file", - "success": true - } - }, - "config": { - "workspace": "generated", - "entry_path": "agbenchmark/benchmarks.py" - } -} diff --git a/agbenchmark/reports/regression_tests.json b/agbenchmark/reports/regression_tests.json new file mode 100644 index 000000000..b66a16cf3 --- /dev/null +++ b/agbenchmark/reports/regression_tests.json @@ -0,0 +1,10 @@ +{ + "TestWriteFile": { + "difficulty": "interface", + "dependencies": [], + "data_path": "agbenchmark/challenges/interface/write_file" + }, + "TestBasicCodeGeneration": { + "data_path": "agbenchmark/challenges/code/d4", + } +}