diff --git a/labkey-client-api/src/org/labkey/remoteapi/Connection.java b/labkey-client-api/src/org/labkey/remoteapi/Connection.java index 5092811b..15c7b63f 100644 --- a/labkey-client-api/src/org/labkey/remoteapi/Connection.java +++ b/labkey-client-api/src/org/labkey/remoteapi/Connection.java @@ -173,12 +173,11 @@ public Connection(String baseUrl, CredentialsProvider credentialsProvider) * Constructs a new Connection object with a base URL that attempts authentication via .netrc/_netrc entry, if present. * If not present, connects as guest. * @param baseUrl The base URL - * @throws URISyntaxException if the given url is not a valid URI * @throws IOException if there are problems reading the credentials * @see NetrcCredentialsProvider * @see #Connection(URI, CredentialsProvider) */ - public Connection(String baseUrl) throws URISyntaxException, IOException + public Connection(String baseUrl) throws IOException { this(toURI(baseUrl), new NetrcCredentialsProvider(toURI(baseUrl))); } diff --git a/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataCommand.java b/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataCommand.java index 7f493733..6a6462d0 100644 --- a/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataCommand.java +++ b/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataCommand.java @@ -20,11 +20,21 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.json.simple.JSONObject; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.Connection; import org.labkey.remoteapi.PostCommand; +import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringWriter; import java.net.URI; +import java.util.Arrays; import java.util.Objects; +import java.util.stream.Collectors; /** * Import data in bulk from a text, local file, or a file @@ -35,16 +45,42 @@ */ public class ImportDataCommand extends PostCommand { + public enum ImportDataType + { + text, path, moduleResource, file + } + + public enum InsertOption + { + /** + * Bulk insert. + */ + IMPORT, + + /** + * Bulk insert or update. + *
+ * NOTE: Not supported for all tables -- tables with auto-increment primary keys in particular. + * See Issue 42788 for details. + */ + MERGE, + } + private final String _schemaName; private final String _queryName; - // Data type is one of 'text', 'path', 'moduleResource', or 'file' and is required. - private String _dataType; + private ImportDataType _dataType; + // Data file format. e.g. "tsv" or "csv" + private String _format; private Object _dataValue; private String _module; + private InsertOption _insertOption; + + private Boolean _useAsync; + private Boolean _saveToPipeline; - public boolean _importIdentity; - public boolean _importLookupByAlternateKey; + public Boolean _importIdentity; + public Boolean _importLookupByAlternateKey; /** @@ -84,55 +120,122 @@ public String getQueryName() return _queryName; } + /** + * Submits tsv or csv data as text to the server for import. + * @see #setFormat(String) + */ public void setText(String text) { Objects.requireNonNull(text); - _dataType = "text"; + _dataType = ImportDataType.text; _dataValue = text; } + /** + * Import data from a file relative from the server's webdav root. + */ public void setPath(String path) { Objects.requireNonNull(path); - _dataType = "path"; + _dataType = ImportDataType.path; _dataValue = path; } + /** + * Import data from a resource file embedded in a module. + * @param module module name + * @param moduleResource path to the file from the exploded module's directory + */ public void setModuleResource(String module, String moduleResource) { Objects.requireNonNull(moduleResource); - _dataType = "moduleResource"; + _dataType = ImportDataType.moduleResource; _dataValue = moduleResource; _module = module; } + /** + * Uploads a file for import. + * @see #setUseAsync(boolean) + * @see #setSaveToPipeline(boolean) + */ public void setFile(File file) { Objects.requireNonNull(file); - _dataType = "file"; + _dataType = ImportDataType.file; _dataValue = file; } - public boolean isImportIdentity() + public Boolean isImportIdentity() { return _importIdentity; } - public void setImportIdentity(boolean importIdentity) + public void setImportIdentity(Boolean importIdentity) { _importIdentity = importIdentity; } - public boolean isImportLookupByAlternateKey() + public Boolean isImportLookupByAlternateKey() { return _importLookupByAlternateKey; } - public void setImportLookupByAlternateKey(boolean importLookupByAlternateKey) + public void setImportLookupByAlternateKey(Boolean importLookupByAlternateKey) { _importLookupByAlternateKey = importLookupByAlternateKey; } + public String getFormat() + { + return _format; + } + + public void setFormat(String format) + { + _format = format; + } + + public InsertOption getInsertOption() + { + return _insertOption; + } + + public void setInsertOption(InsertOption insertOption) + { + _insertOption = insertOption; + } + + public boolean isUseAsync() + { + return _useAsync; + } + + /** + * Import the data in a pipeline job. + * Currently only supported when submitting data as a file. + * @see #setFile(File) + */ + public void setUseAsync(boolean useAsync) + { + _useAsync = useAsync; + } + + public boolean isSaveToPipeline() + { + return _saveToPipeline; + } + + /** + * Save the uploaded file to the pipeline root under the "QueryImportFiles" directory. + * Currently only supported when submitting data as a file. + * @see #setFile(File) + */ + public void setSaveToPipeline(boolean saveToPipeline) + { + _saveToPipeline = saveToPipeline; + } + @Override public JSONObject getJsonObject() { @@ -145,22 +248,35 @@ protected HttpUriRequest createRequest(URI uri) Objects.requireNonNull(_schemaName, "schemaName required"); Objects.requireNonNull(_queryName, "queryName required"); - if (!"text".equals(_dataType) && !"path".equals(_dataType) && !"moduleResource".equals(_dataType) && !"file".equals(_dataType)) - throw new IllegalArgumentException("One of 'text', 'path', 'moduleResource', or 'file' is required"); + Objects.requireNonNull(_dataType, "Data type required and may be one of 'text', 'path', 'moduleResource', or 'file'"); Objects.requireNonNull(_dataValue, "Value for '" + _dataType + "' must not be null"); MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.addTextBody("schemaName", _schemaName); builder.addTextBody("queryName", _queryName); - builder.addTextBody("importIdentity", Boolean.toString(_importIdentity)); - builder.addTextBody("importLookupByAlternateKey", Boolean.toString(_importLookupByAlternateKey)); + if (_importIdentity != null) + builder.addTextBody("importIdentity", Boolean.toString(_importIdentity)); + if (_importLookupByAlternateKey != null) + builder.addTextBody("importLookupByAlternateKey", Boolean.toString(_importLookupByAlternateKey)); if (_module != null) builder.addTextBody("module", _module); - if (_dataType.equals("file")) - builder.addBinaryBody(_dataType, (File)_dataValue, ContentType.APPLICATION_OCTET_STREAM, ((File)_dataValue).getName()); + if (_dataType == ImportDataType.file) + builder.addBinaryBody(_dataType.name(), (File)_dataValue, ContentType.APPLICATION_OCTET_STREAM, ((File)_dataValue).getName()); else - builder.addTextBody(_dataType, (String)_dataValue); + builder.addTextBody(_dataType.name(), (String)_dataValue); + + if (_format != null) + builder.addTextBody("format", _format); + + if (_insertOption != null) + builder.addTextBody("insertOption", _insertOption.toString()); + + if (_saveToPipeline != null) + builder.addTextBody("saveToPipeline", _saveToPipeline.toString()); + + if (_useAsync != null) + builder.addTextBody("useAsync", _useAsync.toString()); HttpPost post = new HttpPost(uri); // Setting this header forces the ExtFormResponseWriter to return application/json contentType instead of text/html. @@ -180,4 +296,302 @@ public ImportDataCommand copy() { return new ImportDataCommand(this); } + + public static void main(String[] args) throws IOException + { + // required + String baseServerUrl = null; + String folderPath = null; + String schemaName = null; + String queryName = null; + + // data + String localPath = null; + String remotePath = null; + String moduleResource = null; + String text = null; + + // optional + String user = null; + String password = null; + String format = null; + Boolean useAsync = null; + Boolean saveToPipeline = null; + InsertOption insertOption = null; + Boolean importIdentity = null; + Boolean importLookupByAlternateKey = null; + + if (args.length < 5) + { + printUsage(); + return; + } + + OUTER: for (int i = 0; i < args.length; i++) + { + String arg = args[i]; + + // + // optional flag arguments + // + + switch (arg) + { + case "-h": + case "--help": + printUsage(); + return; + + case "-d": + case "--importIdentity": + importIdentity = true; + continue OUTER; + + case "-k": + case "--importLookupByAlternateKey": + importLookupByAlternateKey = true; + continue OUTER; + + case "-a": + case "--async": + useAsync = true; + continue OUTER; + + case "-s": + case "--saveToPipeline": + saveToPipeline = true; + continue OUTER; + + case "-F": + case "--format": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument: 'csv' or 'tsv'"); + format = args[++i]; + continue OUTER; + + case "-x": + case "--insertOption": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument: " + Arrays.stream(InsertOption.values()).map(Objects::toString).collect(Collectors.joining(", "))); + insertOption = InsertOption.valueOf(args[++i]); + continue OUTER; + + case "-U": + case "--username": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument"); + user = args[++i]; + continue OUTER; + + case "-P": + case "--password": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument"); + password = args[++i]; + continue OUTER; + + case "-f": + case "--file": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument"); + localPath = args[++i]; + continue OUTER; + + case "-p": + case "--path": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument"); + remotePath = args[++i]; + continue OUTER; + + case "-r": + case "--resource": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument"); + moduleResource = args[++i]; + continue OUTER; + + case "-t": + case "--text": + if (i == args.length-1) + throw new IllegalArgumentException(arg + " requires argument"); + text = args[++i]; + continue OUTER; + } + + // + // positional arguments + // + + if (baseServerUrl == null) + baseServerUrl = arg; + else if (folderPath == null) + folderPath = arg; + else if (schemaName == null) + schemaName = arg; + else if (queryName == null) + queryName = arg; + else + { + System.err.println("Unexpected argument '" + arg + "'. See --help for more."); + return; + } + } + + if (baseServerUrl == null) + { + System.err.println("Server base URL required. See --help for more"); + return; + } + + if (folderPath == null) + { + System.err.println("Server folder required. See --help for more"); + return; + } + + if (schemaName == null) + { + System.err.println("Schema name required. See --help for more"); + return; + } + + if (queryName == null) + { + System.err.println("Query name required. See --help for more"); + return; + } + + if (localPath == null && remotePath == null && moduleResource == null && text == null) + { + System.err.println("One of --file, --path, --resource, --text is required. See --help for more"); + return; + } + + ImportDataCommand cmd = new ImportDataCommand(schemaName, queryName); + + if (localPath != null) + { + cmd.setFile(new File(localPath)); + } + else if (remotePath != null) + { + cmd.setPath(remotePath); + } + else if (moduleResource != null) + { + int colon = moduleResource.indexOf(":"); + if (colon == -1) + { + System.err.println("Expected module resource: :"); + return; + } + String module = moduleResource.substring(0, colon); + String resourcePath = moduleResource.substring(colon+1); + cmd.setModuleResource(module, resourcePath); + } + else if (text != null) + { + if (text.equals("-")) + { + text = readFully(System.in); + } + else + { + text = readFully(new FileInputStream(text)); + } + cmd.setText(text); + } + + if (importIdentity != null) + cmd.setImportIdentity(importIdentity); + + if (importLookupByAlternateKey != null) + cmd.setImportLookupByAlternateKey(importLookupByAlternateKey); + + if (format != null) + cmd.setFormat(format); + + if (insertOption != null) + cmd.setInsertOption(insertOption); + + if (useAsync != null) + cmd.setUseAsync(useAsync); + + if (saveToPipeline != null) + cmd.setSaveToPipeline(saveToPipeline); + + Connection conn; + if (user != null && password != null) + conn = new Connection(baseServerUrl, user, password); + else + conn = new Connection(baseServerUrl); + + try + { + ImportDataResponse resp = cmd.execute(conn, folderPath); + if (resp.getSuccess()) + System.out.println("Successfully imported " + resp.getRowCount() + " rows"); + else + System.out.println("Failed to import data");// error message? {error: {_form: '...'}} + } + catch (CommandException e) + { + System.err.println("Failure! Response code: " + e.getStatusCode()); + e.printStackTrace(); + System.err.println(); + String responseText = e.getResponseText(); + if (responseText != null) + { + System.err.println("Response text: "); + System.err.println(responseText); + } + } + } + + private static void printUsage() + { + System.err.println("Usage:"); + System.err.println(" java " + ImportDataCommand.class.getName() + " [OPTIONS] SERVER_URL FOLDER_PATH SCHEMA QUERY"); + System.err.println(); + System.err.println("Requires one of:"); + System.err.println(" -f, --file LOCAL-PATH"); + System.err.println(" -p, --path SERVER-FILE-PATH"); + System.err.println(" -r, --resource MODULE:MODULE-RESOURCE-PATH"); + System.err.println(" -t, --text LOCAL-PATH (use '-' to read from stdin)"); + System.err.println(); + System.err.println("Other options:"); + System.err.println(" -U, --username USERNAME"); + System.err.println(" -P, --password PASSWORD"); + System.err.println(" -d, --importIdentity"); + System.err.println(" -k, --importLookupByAlternateKey"); + System.err.println(" -x, --insertOption [" + Arrays.stream(InsertOption.values()).map(Objects::toString).collect(Collectors.joining("|")) + "]"); + System.err.println(" -F, --format [tsv|csv] (may be used with --text))"); + System.err.println(" -a, --async (may be used with --file)"); + System.err.println(" -s, --saveToPipeline (may be used with --file)"); + System.err.println(); + System.err.println("Examples"); + System.err.println(" java " + ImportDataCommand.class.getName() + " -a -p --file file.tsv http://localhost:8080/labkey /foldername lists MyList"); + System.err.println(); + System.err.println(" java " + ImportDataCommand.class.getName() + " --resource biologics:data/test/lists/Vessel.tsv http://localhost:8080/labkey /foldername lists Vessels"); + System.err.println(); + } + + private static String readFully(InputStream in) throws IOException + { + StringWriter sw = new StringWriter(); + try (BufferedReader buf = new BufferedReader(new InputStreamReader(in))) + { + String line; + do + { + line = buf.readLine(); + if (line != null) + sw.append(line).append(System.lineSeparator()); + } + while (line != null); + } + + return sw.toString(); + } } diff --git a/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataResponse.java b/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataResponse.java index 88bb0a73..983983cb 100644 --- a/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataResponse.java +++ b/labkey-client-api/src/org/labkey/remoteapi/query/ImportDataResponse.java @@ -27,12 +27,14 @@ public class ImportDataResponse extends CommandResponse { private final Boolean _success; private final int _rowCount; + private final String _jobId; public ImportDataResponse(String text, int statusCode, String contentType, JSONObject json, Command sourceCommand) { super(text, statusCode, contentType, json, sourceCommand); _success = json.containsKey("success") ? (Boolean)json.get("success") : Boolean.FALSE; _rowCount = json.containsKey("rowCount") ? ((Number)json.get("rowCount")).intValue() : 0; + _jobId = (String)json.getOrDefault("jobId", null); } public Boolean getSuccess() @@ -45,4 +47,12 @@ public int getRowCount() return _rowCount; } + /** + * When importing a file asynchronously, the jobId of the queued job is returned. + * @see ImportDataCommand#setUseAsync(boolean) + */ + public String getJobId() + { + return _jobId; + } }