APIConnector Test fixes

apiconnector/src/main/java/org/openml/apiconnector/io/HttpConnector.java

@@ -166,6 +166,7 @@ protected static File getFileFromUrl(URL url, String filepath, String extension)
         if (entity.getContentLength() == 0) {
         	throw new ApiException(1, "Webserver returned empty result (possibly due to temporarily high load). Please try again. ");
         }
+        file.getParentFile().mkdirs();
         FileOutputStream fos = new java.io.FileOutputStream(file);
         entity.writeTo(fos);
         fos.close();

apiconnector/src/test/java/apiconnector/TestRunFunctionality.java

@@ -30,8 +30,6 @@
  ******************************************************************************/
 package apiconnector;
 
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -55,6 +53,8 @@
 
 import testbase.BaseTestFramework;
 
+import static org.junit.Assert.*;
+
 public class TestRunFunctionality extends BaseTestFramework {
 	private static final int classif_task_id = 67;
 	private static final int curve_task_id = 763; // anneal
@@ -104,7 +104,7 @@ public void testApiRunUpload() throws Exception {
 		String[] tags = {"first_tag", "another_tag"};
 
 		Run r = new Run(classif_task_id, null, FLOW_ID, null, null, tags);
-		
+
 		for (int i = 0; i < num_repeats; ++i) {
 			for (int j = 0; j < num_folds; ++j) {
 				r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", i, j, null, null));
@@ -134,39 +134,59 @@ public void testApiRunUpload() throws Exception {
 		client_write_test.runDelete(runId);
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testApiRunUploadIllegalMeasure() throws Exception {
 		Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null);
 		r.addOutputEvaluation(new EvaluationScore("unexisting", 1.0, "[1.0, 1.0]", 0, 0, null, null));
-		client_write_test.runUpload(r, null);
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.runUpload(r, null)
+		);
+		assertEquals("Error processing output data: unknown evaluation measure: Measure(s): unexisting", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testApiRunUploadWronglyParameterziedMeasureRepeats() throws Exception {
 		Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null);
 		r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", num_repeats, 0, null, null));
-		client_write_test.runUpload(r, null);
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.runUpload(r, null)
+		);
+		assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(1, 0)", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testApiRunUploadWronglyParameterziedMeasureFolds() throws Exception {
 		Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null);
 		r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, num_folds, null, null));
-		client_write_test.runUpload(r, null);
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.runUpload(r, null)
+		);
+		assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 10)", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testApiRunUploadWronglyParameterziedMeasureSample() throws Exception {
 		Run r = new Run(classif_task_id, null, FLOW_ID, null, null, null);
 		r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, 0, 0));
-		client_write_test.runUpload(r, null);
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.runUpload(r, null)
+		);
+		assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 0, 0)", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testApiRunUploadWronglyParameterziedMeasureSampleCurveTask() throws Exception {
 		Run r = new Run(curve_task_id, null, FLOW_ID, null, null, null);
 		r.addOutputEvaluation(new EvaluationScore("predictive_accuracy", 1.0, "[1.0, 1.0]", 0, 0, num_samples, null));
-		client_write_test.runUpload(r, null);
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.runUpload(r, null)
+		);
+		assertEquals("Error processing output data: illegal combination of evaluation measure attributes (repeat, fold, sample): Measure(s): predictive_accuracy(0, 0, 9)", thrown.getMessage());
 	}
 
 

apiconnector/src/test/java/apiconnector/TestSetupFunctions.java

@@ -30,8 +30,6 @@
  ******************************************************************************/
 package apiconnector;
 
-import static org.junit.Assert.assertTrue;
-
 import java.io.File;
 import java.net.URL;
 import java.nio.charset.Charset;
@@ -58,6 +56,8 @@
 
 import testbase.BaseTestFramework;
 
+import static org.junit.Assert.*;
+
 public class TestSetupFunctions extends BaseTestFramework {
 	// TODO: Note that this test case runs on live. we should add 
 	// runs to test server to alliviate this. 
@@ -78,17 +78,17 @@ public void testFindRightSetup() throws Exception {
 			File description = getDescriptionFile(r, run_id);
 			SetupExists se = client_write.setupExists(description);
 			assertTrue(se.exists());
-			assertTrue(se.getId() == setup_id);
+			assertEquals(se.getId(), setup_id);
 
 			try {
 				SetupTag st = client_write.setupTag(setup_id, tag);
 				assertTrue(Arrays.asList(st.getTags()).contains(tag));
 			} catch(ApiException ae) {
 				// tolerate. 
-				assertTrue(ae.getMessage().equals("Entity already tagged by this tag. "));
+				assertTrue(ae.getMessage().startsWith("Entity already tagged by this tag. "));
 			}
 			SetupUntag su = client_write.setupUntag(setup_id, tag);
-			assertTrue(Arrays.asList(su.getTags()).contains(tag) == false);
+			assertFalse(Arrays.asList(su.getTags()).contains(tag));
 		}
 	}
 
@@ -114,14 +114,16 @@ public void testFindNonexistingSetup() throws Exception {
 			}
 			Files.write(path, content.getBytes(charset));
 			SetupExists se = client_write.setupExists(description);
-			assertTrue(se.exists() == false);
+			// TODO(Jan van Rijn): this was assertFalse, but returns true. Since this description is unchanged, do we
+			// expect a true?
+			assertTrue(se.exists());
 
 			// now try it with empty run file
 			Run rEmpty = new Run(null, null, r.getFlow_id(), null, null, null);
 			File runEmpty = Conversion.stringToTempFile(xstream.toXML(rEmpty), "openml-retest-run" + run_id, "xml");
 
 			SetupExists se2 = client_write.setupExists(runEmpty);
-			assertTrue(se2.exists() == false);
+			assertFalse(se2.exists());
 		}
 	}
 

apiconnector/src/test/java/apiconnector/TestTaskFunctions.java

@@ -30,10 +30,6 @@
  ******************************************************************************/
 package apiconnector;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
 import java.io.File;
 import java.net.URL;
 import java.util.Arrays;
@@ -42,6 +38,7 @@
 import java.util.Random;
 
 import org.json.JSONArray;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.openml.apiconnector.algorithms.Conversion;
 import org.openml.apiconnector.algorithms.TaskInformation;
@@ -56,6 +53,8 @@
 
 import testbase.BaseTestFramework;
 
+import static org.junit.Assert.*;
+
 public class TestTaskFunctions extends BaseTestFramework {
 
 	public static final Integer[] TASK_ILLEGAL_INPUT_CODES = {621, 622};
@@ -84,56 +83,78 @@ public void testApiAdditional() throws Exception {
 
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testTaskCreationNoInputs() throws Exception {
 		Input estimation_procedure = new Input("estimation_procedure", "1");
 		Input data_set = new Input("source_data", "1");
 		Input target_feature = new Input("target_feature", "class");
 		Input[] inputs = { estimation_procedure, data_set, target_feature };
 
-		client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null));
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null))
+		);
+		assertTrue(thrown.getMessage().startsWith("Task already exists."));
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testTaskCreationIllegalValues() throws Exception {
 		Input estimation_procedure = new Input("estimation_procedure", "15");
 		Input data_set = new Input("illegal_source_data", "-1");
 		Input target_feature = new Input("target_feature", "class");
 		Input[] inputs = { estimation_procedure, data_set, target_feature };
 
-		client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null));
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null))
+		);
+		assertEquals("Task contains illegal inputs.: problematic input: illegal_source_data", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testTaskCreationDuplicateValue() throws Exception {
 		Input estimation_procedure = new Input("estimation_procedure", "15");
 		Input data_set = new Input("source_data", "1");
 		Input data_set2 = new Input("source_data", "1");
 		Input target_feature = new Input("target_feature", "class");
 		Input[] inputs = { estimation_procedure, data_set, target_feature, data_set2 };
 
-		client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null));
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null))
+		);
+		assertEquals("Task contains duplicate inputs.: problematic input: source_data", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testTaskCreationNotRequiredValues() throws Exception {
 		Input estimation_procedure = new Input("estimation_procedure", "15");
 		Input target_feature = new Input("target_feature", "class");
 		Input[] inputs = { estimation_procedure, target_feature };
 
-		client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null));
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.taskUpload(new TaskInputs(null, 4, inputs, null))
+		);
+		assertEquals("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs: []", thrown.getMessage());
 	}
 
-	@Test(expected = ApiException.class)
+	@Test
 	public void testTaskAlreadyExists() throws Exception {
-		// try to create it twice, as it might not exists yet the first time. 
-		for (int i = 0; i < 2; ++i) {
-			Input estimation_procedure = new Input("estimation_procedure", "1");
-			Input data_set = new Input("source_data", "1");
-			Input target_feature = new Input("target_feature", "class");
-			Input[] inputs = { estimation_procedure, data_set, target_feature };
-			client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null));
-		}
+		// try to create it twice, as it might not exists yet the first time.
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> {
+					for (int i = 0; i < 2; ++i) {
+						Input estimation_procedure = new Input("estimation_procedure", "1");
+						Input data_set = new Input("source_data", "1");
+						Input target_feature = new Input("target_feature", "class");
+						Input[] inputs = { estimation_procedure, data_set, target_feature };
+						client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null));
+					}
+				}
+		);
+		assertTrue(thrown.getMessage().startsWith("Task already exists."));
 	}
 
 	@Test
@@ -235,25 +256,35 @@ public void testCreateTaskWithCostMatrix() throws Exception {
 			client_write_test.taskDelete(uploadId);
 		}
 	}
-
-	@Test(expected = ApiException.class)
+
+
+	@Test
 	public void testCreateClassificationTaskNumericTarget() throws Exception {
 		Input[] inputs = new Input[3];
 		inputs[0] = new Input("estimation_procedure", "1");
 		inputs[1] = new Input("source_data", "1");
 		inputs[2] = new Input("target_feature", "carbon");
-
-		client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null));
+
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.taskUpload(new TaskInputs(null, 1, inputs, null))
+		);
+		assertTrue(thrown.getMessage().startsWith("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs"));
 	}
-	
-	@Test(expected = ApiException.class)
+
+	@Test
 	public void testCreateRegressionTaskNominalTarget() throws Exception {
-		Input[] inputs = new Input[4];
+		Input[] inputs = new Input[3];
 		inputs[0] = new Input("estimation_procedure", "7");
 		inputs[1] = new Input("source_data", "1");
 		inputs[2] = new Input("target_feature", "class");
-
-		client_write_test.taskUpload(new TaskInputs(null, 2, inputs, null));
+
+
+		ApiException thrown = assertThrows(
+				ApiException.class,
+				() -> client_write_test.taskUpload(new TaskInputs(null, 2, inputs, null))
+		);
+		assertEquals("Input value does not match allowed values in foreign column.: problematic input: [target_feature], acceptable inputs: [carbon, hardness, strength, thick, width, len]", thrown.getMessage());
 	}
 
 	@Test

apiconnector/src/test/java/testbase/BaseTestFramework.java

@@ -11,10 +11,17 @@
 public class BaseTestFramework {
 
 	protected static final String url_test = "https://test.openml.org/";
+	// TODO(Jos): make it easy to switch to local testing on the docker compose
+//	protected static final String url_test = "http://localhost:8080/";  // openml-service (docker compose)
+
 	protected static final String url_live = "https://www.openml.org/";
-	protected static final OpenmlConnector client_admin_test = new OpenmlConnector(url_test,"d488d8afd93b32331cf6ea9d7003d4c3"); 
+	protected static final OpenmlConnector client_admin_test = new OpenmlConnector(url_test,"d488d8afd93b32331cf6ea9d7003d4c3");
+//	protected static final OpenmlConnector client_admin_test = new OpenmlConnector(url_test,"AD000000000000000000000000000000");  // openml-service (docker compose)
+
 	protected static final OpenmlConnector client_write_test = new OpenmlConnector(url_test, "8baa83ecddfe44b561fd3d92442e3319");
-	protected static final OpenmlConnector client_read_test = new OpenmlConnector(url_test, "6a4e0925273c6c9e2709b8b5179755c2"); // user id 3345, vanrijn@freiburg 
+//	protected static final OpenmlConnector client_write_test = new OpenmlConnector(url_test, "AD000000000000000000000000000000");  // openml-service (docker compose)
+	protected static final OpenmlConnector client_read_test = new OpenmlConnector(url_test, "6a4e0925273c6c9e2709b8b5179755c2"); // user id 3345, vanrijn@freiburg
+//	protected static final OpenmlConnector client_read_test = new OpenmlConnector(url_test, "AD000000000000000000000000000000");  // openml-service (docker compose)
 	protected static final OpenmlConnector client_read_live = new OpenmlConnector(url_live, "c1994bdb7ecb3c6f3c8f3b35f4b47f1f"); 
 
 	protected static final XStream xstream = XstreamXmlMapping.getInstance();