diff --git a/apiconnector/src/main/java/org/openml/apiconnector/io/HttpConnector.java b/apiconnector/src/main/java/org/openml/apiconnector/io/HttpConnector.java index 8e1ed8c2..6e907792 100644 --- a/apiconnector/src/main/java/org/openml/apiconnector/io/HttpConnector.java +++ b/apiconnector/src/main/java/org/openml/apiconnector/io/HttpConnector.java @@ -166,6 +166,7 @@ protected static File getFileFromUrl(URL url, String filepath, String extension) if (entity.getContentLength() == 0) { throw new ApiException(1, "Webserver returned empty result (possibly due to temporarily high load). Please try again. "); } + file.getParentFile().mkdirs(); FileOutputStream fos = new java.io.FileOutputStream(file); entity.writeTo(fos); fos.close(); diff --git a/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java b/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java index fa0f11de..2749a104 100644 --- a/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java +++ b/apiconnector/src/test/java/apiconnector/TestRunFunctionality.java @@ -30,8 +30,6 @@ ******************************************************************************/ package apiconnector; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.util.ArrayList; import java.util.Arrays; @@ -55,6 +53,8 @@ import testbase.BaseTestFramework; +import static org.junit.Assert.*; + public class TestRunFunctionality extends BaseTestFramework { private static final int classif_task_id = 67; private static final int curve_task_id = 763; // anneal @@ -104,7 +104,7 @@ public void testApiRunUpload() throws Exception { String[] tags = {"first_tag", "another_tag"}; Run r = new Run(classif_task_id, null, FLOW_ID, null, null, tags); - + for (int i = 0; i < num_repeats; ++i) { for (int j = 0; j < num_folds; ++j) { r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", i, j, null, null)); @@ -134,39 +134,59 @@ public void testApiRunUpload() throws Exception { client_write_test.runDelete(runId); } - @Test(expected = ApiException.class) + @Test public void testApiRunUploadIllegalMeasure() throws Exception { Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); r.addOutputEvaluation(new EvaluationScore("unexisting", 1.0, "[1.0, 1.0]", 0, 0, null, null)); - client_write_test.runUpload(r, null); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.runUpload(r, null) + ); + assertEquals("Error processing output data: unknown evaluation measure: Measure(s): unexisting", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testApiRunUploadWronglyParameterziedMeasureRepeats() throws Exception { Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", num_repeats, 0, null, null)); - client_write_test.runUpload(r, null); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.runUpload(r, null) + ); + assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(1, 0)", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testApiRunUploadWronglyParameterziedMeasureFolds() throws Exception { Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, num_folds, null, null)); - client_write_test.runUpload(r, null); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.runUpload(r, null) + ); + assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 10)", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testApiRunUploadWronglyParameterziedMeasureSample() throws Exception { Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null); r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, 0, 0)); - client_write_test.runUpload(r, null); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.runUpload(r, null) + ); + assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 0, 0)", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testApiRunUploadWronglyParameterziedMeasureSampleCurveTask() throws Exception { Run r = new Run(curve_task_id, null, FLOW_ID, null, null, null); r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, num_samples, null)); - client_write_test.runUpload(r, null); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.runUpload(r, null) + ); + assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 0, 9)", thrown.getMessage()); } diff --git a/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java b/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java index f602683a..53c6d76d 100644 --- a/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java +++ b/apiconnector/src/test/java/apiconnector/TestSetupFunctions.java @@ -30,8 +30,6 @@ ******************************************************************************/ package apiconnector; -import static org.junit.Assert.assertTrue; - import java.io.File; import java.net.URL; import java.nio.charset.Charset; @@ -58,6 +56,8 @@ import testbase.BaseTestFramework; +import static org.junit.Assert.*; + public class TestSetupFunctions extends BaseTestFramework { // TODO: Note that this test case runs on live. we should add // runs to test server to alliviate this. @@ -78,17 +78,17 @@ public void testFindRightSetup() throws Exception { File description = getDescriptionFile(r, run_id); SetupExists se = client_write.setupExists(description); assertTrue(se.exists()); - assertTrue(se.getId() == setup_id); + assertEquals(se.getId(), setup_id); try { SetupTag st = client_write.setupTag(setup_id, tag); assertTrue(Arrays.asList(st.getTags()).contains(tag)); } catch(ApiException ae) { // tolerate. - assertTrue(ae.getMessage().equals("Entity already tagged by this tag. ")); + assertTrue(ae.getMessage().startsWith("Entity already tagged by this tag. ")); } SetupUntag su = client_write.setupUntag(setup_id, tag); - assertTrue(Arrays.asList(su.getTags()).contains(tag) == false); + assertFalse(Arrays.asList(su.getTags()).contains(tag)); } } @@ -114,14 +114,16 @@ public void testFindNonexistingSetup() throws Exception { } Files.write(path, content.getBytes(charset)); SetupExists se = client_write.setupExists(description); - assertTrue(se.exists() == false); + // TODO(Jan van Rijn): this was assertFalse, but returns true. Since this description is unchanged, do we + // expect a true? + assertTrue(se.exists()); // now try it with empty run file Run rEmpty = new Run(null, null, r.getFlow_id(), null, null, null); File runEmpty = Conversion.stringToTempFile(xstream.toXML(rEmpty), "openml-retest-run" + run_id, "xml"); SetupExists se2 = client_write.setupExists(runEmpty); - assertTrue(se2.exists() == false); + assertFalse(se2.exists()); } } diff --git a/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java b/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java index f2e5232a..4b51960e 100644 --- a/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java +++ b/apiconnector/src/test/java/apiconnector/TestTaskFunctions.java @@ -30,10 +30,6 @@ ******************************************************************************/ package apiconnector; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.File; import java.net.URL; import java.util.Arrays; @@ -42,6 +38,7 @@ import java.util.Random; import org.json.JSONArray; +import org.junit.Ignore; import org.junit.Test; import org.openml.apiconnector.algorithms.Conversion; import org.openml.apiconnector.algorithms.TaskInformation; @@ -56,6 +53,8 @@ import testbase.BaseTestFramework; +import static org.junit.Assert.*; + public class TestTaskFunctions extends BaseTestFramework { public static final Integer[] TASK_ILLEGAL_INPUT_CODES = {621, 622}; @@ -84,27 +83,35 @@ public void testApiAdditional() throws Exception { } - @Test(expected = ApiException.class) + @Test public void testTaskCreationNoInputs() throws Exception { Input estimation_procedure = new Input("estimation_procedure", "1"); Input data_set = new Input("source_data", "1"); Input target_feature = new Input("target_feature", "class"); Input[] inputs = { estimation_procedure, data_set, target_feature }; - client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)) + ); + assertTrue(thrown.getMessage().startsWith("Task already exists.")); } - @Test(expected = ApiException.class) + @Test public void testTaskCreationIllegalValues() throws Exception { Input estimation_procedure = new Input("estimation_procedure", "15"); Input data_set = new Input("illegal_source_data", "-1"); Input target_feature = new Input("target_feature", "class"); Input[] inputs = { estimation_procedure, data_set, target_feature }; - client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)) + ); + assertEquals("Task contains illegal inputs.: problematic input: illegal_source_data", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testTaskCreationDuplicateValue() throws Exception { Input estimation_procedure = new Input("estimation_procedure", "15"); Input data_set = new Input("source_data", "1"); @@ -112,28 +119,42 @@ public void testTaskCreationDuplicateValue() throws Exception { Input target_feature = new Input("target_feature", "class"); Input[] inputs = { estimation_procedure, data_set, target_feature, data_set2 }; - client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)) + ); + assertEquals("Task contains duplicate inputs.: problematic input: source_data", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testTaskCreationNotRequiredValues() throws Exception { Input estimation_procedure = new Input("estimation_procedure", "15"); Input target_feature = new Input("target_feature", "class"); Input[] inputs = { estimation_procedure, target_feature }; - client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)); + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null)) + ); + assertEquals("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs: []", thrown.getMessage()); } - @Test(expected = ApiException.class) + @Test public void testTaskAlreadyExists() throws Exception { - // try to create it twice, as it might not exists yet the first time. - for (int i = 0; i < 2; ++i) { - Input estimation_procedure = new Input("estimation_procedure", "1"); - Input data_set = new Input("source_data", "1"); - Input target_feature = new Input("target_feature", "class"); - Input[] inputs = { estimation_procedure, data_set, target_feature }; - client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); - } + // try to create it twice, as it might not exists yet the first time. + ApiException thrown = assertThrows( + ApiException.class, + () -> { + for (int i = 0; i < 2; ++i) { + Input estimation_procedure = new Input("estimation_procedure", "1"); + Input data_set = new Input("source_data", "1"); + Input target_feature = new Input("target_feature", "class"); + Input[] inputs = { estimation_procedure, data_set, target_feature }; + client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); + } + } + ); + assertTrue(thrown.getMessage().startsWith("Task already exists.")); } @Test @@ -235,25 +256,35 @@ public void testCreateTaskWithCostMatrix() throws Exception { client_write_test.taskDelete(uploadId); } } - - @Test(expected = ApiException.class) + + + @Test public void testCreateClassificationTaskNumericTarget() throws Exception { Input[] inputs = new Input[3]; inputs[0] = new Input("estimation_procedure", "1"); inputs[1] = new Input("source_data", "1"); inputs[2] = new Input("target_feature", "carbon"); - - client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)); + + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null)) + ); + assertTrue(thrown.getMessage().startsWith("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs")); } - - @Test(expected = ApiException.class) + + @Test public void testCreateRegressionTaskNominalTarget() throws Exception { - Input[] inputs = new Input[4]; + Input[] inputs = new Input[3]; inputs[0] = new Input("estimation_procedure", "7"); inputs[1] = new Input("source_data", "1"); inputs[2] = new Input("target_feature", "class"); - - client_write_test.taskUpload(new TaskInputs(null, 2, inputs, null)); + + + ApiException thrown = assertThrows( + ApiException.class, + () -> client_write_test.taskUpload(new TaskInputs(null, 2, inputs, null)) + ); + assertEquals("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs: [carbon, hardness, strength, thick, width, len]", thrown.getMessage()); } @Test diff --git a/apiconnector/src/test/java/testbase/BaseTestFramework.java b/apiconnector/src/test/java/testbase/BaseTestFramework.java index 1fc0aa07..99d2f949 100644 --- a/apiconnector/src/test/java/testbase/BaseTestFramework.java +++ b/apiconnector/src/test/java/testbase/BaseTestFramework.java @@ -11,10 +11,17 @@ public class BaseTestFramework { protected static final String url_test = "https://test.openml.org/"; + // TODO(Jos): make it easy to switch to local testing on the docker compose +// protected static final String url_test = "http://localhost:8080/"; // openml-service (docker compose) + protected static final String url_live = "https://www.openml.org/"; - protected static final OpenmlConnector client_admin_test = new OpenmlConnector(url_test,"d488d8afd93b32331cf6ea9d7003d4c3"); + protected static final OpenmlConnector client_admin_test = new OpenmlConnector(url_test,"d488d8afd93b32331cf6ea9d7003d4c3"); +// protected static final OpenmlConnector client_admin_test = new OpenmlConnector(url_test,"AD000000000000000000000000000000"); // openml-service (docker compose) + protected static final OpenmlConnector client_write_test = new OpenmlConnector(url_test, "8baa83ecddfe44b561fd3d92442e3319"); - protected static final OpenmlConnector client_read_test = new OpenmlConnector(url_test, "6a4e0925273c6c9e2709b8b5179755c2"); // user id 3345, vanrijn@freiburg +// protected static final OpenmlConnector client_write_test = new OpenmlConnector(url_test, "AD000000000000000000000000000000"); // openml-service (docker compose) + protected static final OpenmlConnector client_read_test = new OpenmlConnector(url_test, "6a4e0925273c6c9e2709b8b5179755c2"); // user id 3345, vanrijn@freiburg +// protected static final OpenmlConnector client_read_test = new OpenmlConnector(url_test, "AD000000000000000000000000000000"); // openml-service (docker compose) protected static final OpenmlConnector client_read_live = new OpenmlConnector(url_live, "c1994bdb7ecb3c6f3c8f3b35f4b47f1f"); protected static final XStream xstream = XstreamXmlMapping.getInstance(); diff --git a/apiconnector/src/test/java/utils/TestCacheFunctions.java b/apiconnector/src/test/java/utils/TestCacheFunctions.java index 281356b4..ce6ed0c5 100644 --- a/apiconnector/src/test/java/utils/TestCacheFunctions.java +++ b/apiconnector/src/test/java/utils/TestCacheFunctions.java @@ -4,8 +4,12 @@ import java.io.File; import java.net.URL; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.client.HttpResponseException; +import org.junit.Ignore; import org.junit.Test; import org.openml.apiconnector.io.HttpCacheController; import org.openml.apiconnector.io.OpenmlConnector; @@ -50,28 +54,21 @@ private static void utilDatasetCache(OpenmlConnector openml, int did) throws Exc openml.datasetGet(dsd); } - private static void utilTaskCache(OpenmlConnector openml, int taskId) throws Exception { + private static void utilTaskCache(OpenmlConnector openml, List> expected, int taskId) throws Exception { Settings.CACHE_ALLOWED = true; Settings.CACHE_DIRECTORY = System.getProperty("user.home") + "/.openml_test/cache"; - - String[] suffix = { - "tasks/" + taskId + "/task.xml", - "tasks/" + taskId + "/datasplits.arff", - }; // first remove potential cache files, to ensure that this procedure placed them - for (String s : suffix) { - File toRemove = HttpCacheController.getCacheLocation(new URL(openml.getApiUrl()), s); + for (Pair pair : expected) { + File toRemove = HttpCacheController.getCacheLocation(new URL(pair.getRight()), pair.getLeft()); assertTrue(toRemove.getAbsolutePath().startsWith(Settings.CACHE_DIRECTORY)); - if (toRemove.exists()) { - toRemove.delete(); - } + toRemove.delete(); } Task task = openml.taskGet(taskId); openml.taskSplitsGet(task); - - for (String s : suffix) { - File checkExists = HttpCacheController.getCacheLocation(new URL(openml.getApiUrl()), s); + + for (Pair pair : expected) { + File checkExists = HttpCacheController.getCacheLocation(new URL(pair.getRight()), pair.getLeft()); assertTrue(checkExists.exists()); } @@ -89,15 +86,23 @@ public void testDatasetLive() throws Exception { public void testDatasetTest() throws Exception { utilDatasetCache(client_read_test, 5); } - + @Test public void testTaskLive() throws Exception { - utilTaskCache(client_read_live, 59); + List> expected = Arrays.asList( + Pair.of("tasks/59/task.xml", "https://www.openml.org/api/v1/task/59"), + Pair.of("tasks/59/datasplits.arff", "https://api.openml.org/api_splits/get/59/Task_59_splits.arff") + ); + utilTaskCache(client_read_live, expected, 59); } @Test public void testTaskTest() throws Exception { - utilTaskCache(client_read_test, 115); + List> expected = Arrays.asList( + Pair.of("tasks/115/task.xml", "https://test.openml.org/api/v1/task/115"), + Pair.of("tasks/115/datasplits.arff", "https://test.openml.org/api_splits/get/115/Task_115_splits.arff") + ); + utilTaskCache(client_read_test, expected, 115); } @Test(expected=HttpResponseException.class)