Skip to content

disables ingest of filetype text/tab-separated-values #6517

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kcondon merged 2 commits into from
Jan 14, 2020
Merged

disables ingest of filetype text/tab-separated-values #6517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aaf7d9a
disables ingest of filetype text/tab-separated-values
landreev Jan 14, 2020
4898066
(a few more lines of code to protect against attempted ingests of alr…
landreev Jan 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,10 @@ from a local InputStream (or a readChannel) into the
} else {
logger.fine("Failed to extract indexable metadata from file " + fileName);
}
} else if (FileUtil.MIME_TYPE_INGESTED_FILE.equals(dataFile.getContentType())) {
// Make sure no *uningested* tab-delimited files are saved with the type "text/tab-separated-values"!
// "text/tsv" should be used instead:
dataFile.setContentType(FileUtil.MIME_TYPE_TSV);
}
// temp dbug line
//System.out.println("ADDING FILE: " + fileName + "; for dataset: " + dataset.getGlobalId());
Expand Down Expand Up @@ -423,12 +427,12 @@ public String startIngestJobs(List<DataFile> dataFiles, AuthenticatedUser user)

List<DataFile> scheduledFiles = new ArrayList<>();
for (DataFile dataFile : dataFiles) {
// refresh the copy of the DataFile:
dataFile = fileService.find(dataFile.getId());

if (dataFile.isIngestScheduled()) {

// refresh the copy of the DataFile:
dataFile = fileService.find(dataFile.getId());

long ingestSizeLimit = -1;
long ingestSizeLimit = 0;
try {
ingestSizeLimit = systemConfig.getTabularIngestSizeLimit(getTabDataReaderByMimeType(dataFile.getContentType()).getFormatName());
} catch (IOException ioex) {
Expand Down Expand Up @@ -731,6 +735,15 @@ public boolean ingestAsTabular(Long datafile_id) {
boolean ingestSuccessful = false;
boolean forceTypeCheck = false;

// Never attempt to ingest a file that's already ingested!
if (dataFile.isTabularData()) {
FileUtil.createIngestFailureReport(dataFile, "Repeated ingest attempted on a tabular data file! (status flag was: "+dataFile.getIngestStatus());
dataFile.setIngestDone();
dataFile = fileService.save(dataFile);
logger.warning("Repeated ingest attempted on a tabular data file (datafile id "+datafile_id+"); exiting.");
return false;
}

IngestRequest ingestRequest = dataFile.getIngestRequest();
if (ingestRequest != null) {
forceTypeCheck = ingestRequest.isForceTypeCheck();
Expand Down Expand Up @@ -1068,7 +1081,7 @@ public static TabularDataFileReader getTabDataReaderByMimeType(String mimeType)
ingestPlugin = new RDATAFileReader(new RDATAFileReaderSpi());
} else if (mimeType.equals(FileUtil.MIME_TYPE_CSV) || mimeType.equals(FileUtil.MIME_TYPE_CSV_ALT)) {
ingestPlugin = new CSVFileReader(new CSVFileReaderSpi(), ',');
} else if (mimeType.equals(FileUtil.MIME_TYPE_TSV) || mimeType.equals(FileUtil.MIME_TYPE_TSV_ALT)) {
} else if (mimeType.equals(FileUtil.MIME_TYPE_TSV) /*|| mimeType.equals(FileUtil.MIME_TYPE_TSV_ALT)*/) {
ingestPlugin = new CSVFileReader(new CSVFileReaderSpi(), '\t');
} else if (mimeType.equals(FileUtil.MIME_TYPE_XLSX)) {
ingestPlugin = new XLSXFileReader(new XLSXFileReaderSpi());
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,7 +1246,7 @@ public static boolean canIngestAsTabular(String mimeType) {
case MIME_TYPE_CSV:
case MIME_TYPE_CSV_ALT:
case MIME_TYPE_TSV:
case MIME_TYPE_TSV_ALT:
//case MIME_TYPE_TSV_ALT:
case MIME_TYPE_XLSX:
case MIME_TYPE_SPSS_SAV:
case MIME_TYPE_SPSS_POR:
Expand Down