diff --git a/api/src/org/labkey/api/data/ArrayExcelWriter.java b/api/src/org/labkey/api/data/ArrayExcelWriter.java index a3a05410c48..8a0356fdcfa 100644 --- a/api/src/org/labkey/api/data/ArrayExcelWriter.java +++ b/api/src/org/labkey/api/data/ArrayExcelWriter.java @@ -17,19 +17,19 @@ public class ArrayExcelWriter extends ExcelWriter * @param data The data rows, in which index position of a value corresponds to the desired respective column index * @param cols The columns, in which ordering determines the left-to-right column ordering in the generated Excel */ - public ArrayExcelWriter(List data, ColumnDescriptor[] cols) + public ArrayExcelWriter(List data, List cols) { super(ExcelDocumentType.xlsx); this.data = data; - List xlcols = new ArrayList<>(); + List displayColumns = new ArrayList<>(); - for (int i = 0; i < cols.length; i++) + for (int i = 0; i < cols.size(); i++) { - ColumnDescriptor col = cols[i]; - xlcols.add(new ArrayDisplayColumn(col.name, col.clazz, i)); + ColumnDescriptor col = cols.get(i); + displayColumns.add(new ArrayDisplayColumn(col.name, col.clazz, i)); } - setDisplayColumns(xlcols); + setDisplayColumns(displayColumns); } @Override diff --git a/api/src/org/labkey/api/data/TSVArrayWriter.java b/api/src/org/labkey/api/data/TSVArrayWriter.java index be40c90480a..edcc868eaee 100644 --- a/api/src/org/labkey/api/data/TSVArrayWriter.java +++ b/api/src/org/labkey/api/data/TSVArrayWriter.java @@ -5,7 +5,6 @@ import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; // This class supports generating files with duplicate column names. Consider using TSVMapWriter if // multiple identical column names is not an implementation concern. @@ -15,17 +14,17 @@ public class TSVArrayWriter extends TSVWriter private final List> _rows; private final String _fileName; - public TSVArrayWriter(String fileName, ColumnDescriptor[] columns, List rows) + public TSVArrayWriter(String fileName, List columns, List rows) { _fileName = fileName; - _columns = Arrays.stream(columns) + _columns = columns.stream() .map(ColumnDescriptor::getColumnName) - .collect(Collectors.toList()); + .toList(); _rows = rows.stream() .map(array -> Arrays.stream(array) .map(obj -> (obj == null) ? "" : String.valueOf(obj)) - .collect(Collectors.toList())) - .collect(Collectors.toList()); + .toList()) + .toList(); } @Override diff --git a/api/src/org/labkey/api/data/TSVColumnWriter.java b/api/src/org/labkey/api/data/TSVColumnWriter.java index ba19978dcbc..cbd2550923d 100644 --- a/api/src/org/labkey/api/data/TSVColumnWriter.java +++ b/api/src/org/labkey/api/data/TSVColumnWriter.java @@ -26,9 +26,6 @@ * Extracted DisplayColumn handling out of TSVGridWriter so rendering DisplayColumns * may be used without a ResultSet. You will still need to set up a RenderContext * for DisplayColumn to render values. - * - * User: kevink - * Date: 9/9/11 */ public abstract class TSVColumnWriter extends TSVWriter { @@ -72,11 +69,11 @@ protected Iterable getColumnHeaders(RenderContext ctx, Iterable getValues(RenderContext ctx, Iterable displayColumns) { diff --git a/api/src/org/labkey/api/data/TSVMapWriter.java b/api/src/org/labkey/api/data/TSVMapWriter.java index 52d6e5519b2..fa78e854a48 100644 --- a/api/src/org/labkey/api/data/TSVMapWriter.java +++ b/api/src/org/labkey/api/data/TSVMapWriter.java @@ -31,10 +31,6 @@ import java.util.List; import java.util.Map; -/** - * User: kevink - * Date: 9/11/11 - */ public class TSVMapWriter extends TSVWriter { private final Collection _columns; diff --git a/api/src/org/labkey/api/data/TSVWriter.java b/api/src/org/labkey/api/data/TSVWriter.java index 249ed2c73ed..396cebd9326 100644 --- a/api/src/org/labkey/api/data/TSVWriter.java +++ b/api/src/org/labkey/api/data/TSVWriter.java @@ -40,6 +40,7 @@ public abstract class TSVWriter extends TextWriter protected char _chQuote = '"'; protected String _rowSeparator = "\n"; public static final String BACKSLASH_CHAR_STRING = "\\"; + private static final char COMMENT_CHAR = '#'; protected List _fileHeader = null; protected boolean _headerRowVisible = true; @@ -96,7 +97,6 @@ public TSVWriter() { } - public String getFilenamePrefix() { return _filenamePrefix; @@ -210,7 +210,6 @@ protected boolean shouldQuote(String value) _escapedCharsString = "\r\n" + _rowSeparator + _chDelimiter + _chQuote; } - int len = value.length(); if (len == 0) return _preserveEmptyString; @@ -218,6 +217,8 @@ protected boolean shouldQuote(String value) char lastCh = value.charAt(len-1); if (Character.isSpaceChar(firstCh) || Character.isSpaceChar(lastCh)) return true; + if (firstCh == COMMENT_CHAR) // Issue 50719, Issue 53302 + return true; if (StringUtils.containsAny(value, _additionalQuotedChars)) return true; return StringUtils.containsAny(value,_escapedCharsString); @@ -236,7 +237,7 @@ protected String getContentType() return delim.contentType; } - return "text/tab-separated-values"; + return DELIM.TAB.contentType; } /** @@ -279,7 +280,6 @@ public boolean isHeaderRowVisible() return _headerRowVisible; } - public void setHeaderRowVisible(boolean headerRowVisible) { _headerRowVisible = headerRowVisible; diff --git a/api/src/org/labkey/api/reader/TabLoader.java b/api/src/org/labkey/api/reader/TabLoader.java index 4631b33f02d..bb032c90537 100644 --- a/api/src/org/labkey/api/reader/TabLoader.java +++ b/api/src/org/labkey/api/reader/TabLoader.java @@ -56,7 +56,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; - /** * Parses rows of tab-delimited text, returning a CloseableIterator of Map. The iterator must be closed * (typically via try-with-resources or a finally block) to close the underlying input source. The iterator can be wrapped @@ -91,7 +90,10 @@ public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Contain } @NotNull @Override - public FileType getFileType() { return TSV_FILE_TYPE; } + public FileType getFileType() + { + return TSV_FILE_TYPE; + } } public static class CsvFactory extends AbstractDataLoaderFactory @@ -114,7 +116,10 @@ public TabLoader createLoader(InputStream is, boolean hasColumnHeaders, Containe } @NotNull @Override - public FileType getFileType() { return CSV_FILE_TYPE; } + public FileType getFileType() + { + return CSV_FILE_TYPE; + } } public static class CsvFactoryNoConversions extends CsvFactory @@ -141,9 +146,6 @@ public TabLoader createLoader(InputStream is, boolean hasColumnHeaders, Containe TabLoader loader = super.createLoader(is, hasColumnHeaders, mvIndicatorContainer); return configParsing(loader); } - - @NotNull @Override - public FileType getFileType() { return CSV_FILE_TYPE; } } public static class MysqlFactory extends AbstractDataLoaderFactory @@ -172,7 +174,10 @@ public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Contain } @NotNull @Override - public FileType getFileType() { return CSV_FILE_TYPE; } + public FileType getFileType() + { + return CSV_FILE_TYPE; + } } protected static char COMMENT_CHAR = '#'; diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 333c17dc567..6a8a5ea771c 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -1309,18 +1309,17 @@ public Object execute(WorklistForm form, BindException errors) throws Exception List sourceIncludedMetadataCols = PlateManager.get().getMetadataColumns(plateSetSource, getContainer(), getUser(), cf); List destinationIncludedMetadataCols = PlateManager.get().getMetadataColumns(plateSetDestination, getContainer(), getUser(), cf); - ColumnDescriptor[] sourceXlCols = PlateSetExport.getColumnDescriptors(PlateSetExport.SOURCE, sourceIncludedMetadataCols); - ColumnDescriptor[] destinationXlCols = PlateSetExport.getColumnDescriptors(PlateSetExport.DESTINATION, destinationIncludedMetadataCols); - ColumnDescriptor[] xlCols = ArrayUtils.addAll(sourceXlCols, destinationXlCols); + List sourceColumns = PlateSetExport.getColumnDescriptors(PlateSetExport.SOURCE, sourceIncludedMetadataCols); + List destinationColumns = PlateSetExport.getColumnDescriptors(PlateSetExport.DESTINATION, destinationIncludedMetadataCols); + List exportColumns = new ArrayList<>(sourceColumns); + exportColumns.addAll(destinationColumns); List plateDataRows = PlateManager.get().getWorklist(form.getSourcePlateSetId(), form.getDestinationPlateSetId(), sourceIncludedMetadataCols, destinationIncludedMetadataCols, getContainer(), getUser()); String fullFileName = plateSetSource.getName() + " - " + plateSetDestination.getName(); - PlateManager.get().getPlateSetExportFile(fullFileName, xlCols, plateDataRows, form.getFileType(), getViewContext().getResponse()); + PlateManager.get().getPlateSetExportFile(fullFileName, exportColumns, plateDataRows, form.getFileType(), getViewContext().getResponse()); SimpleMetricsService.get().increment(AssayModule.NAME, "plateSet", "exportWorklist"); - - return null; // Returning anything here will cause error as excel writer will close the response stream } catch (Exception e) { @@ -1387,12 +1386,10 @@ public Object execute(InstrumentInstructionForm form, BindException errors) thro cf = form.getContainerFilter().create(getViewContext()); List includedMetadataCols = PlateManager.get().getMetadataColumns(plateSet, getContainer(), getUser(), cf); - ColumnDescriptor[] xlCols = PlateSetExport.getColumnDescriptors("", includedMetadataCols); + List exportColumns = PlateSetExport.getColumnDescriptors("", includedMetadataCols); List plateDataRows = PlateManager.get().getInstrumentInstructions(form.getPlateSetId(), includedMetadataCols, getContainer(), getUser()); - PlateManager.get().getPlateSetExportFile(plateSet.getName() + "-instructions", xlCols, plateDataRows, form.getFileType(), getViewContext().getResponse()); - - return null; // Returning anything here will cause error as excel writer will close the response stream + PlateManager.get().getPlateSetExportFile(plateSet.getName() + "-instructions", exportColumns, plateDataRows, form.getFileType(), getViewContext().getResponse()); } catch (Exception e) { diff --git a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java index 881ec528aaa..7d10f5f1291 100644 --- a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java +++ b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java @@ -279,7 +279,6 @@ else if (rawData != null && !rawData.isEmpty()) try (TSVMapWriter tsvWriter = saveMatchingColumnDataOnly ? new TSVMapWriter(columns, rawData) : new TSVMapWriter(columns, rawData, true)) { - tsvWriter.setAdditionalQuotedChars("#"); //Issue 50719: If the first column name starts with a #, the data loader will treat the header row as a comment tsvWriter.write(fileObject.toNioPathForWrite().toFile()); factory.setRawData(null); factory.setUploadedData(Collections.singletonMap(PRIMARY_FILE, fileObject)); diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 8a6441440ee..d423c69a8ee 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -3654,10 +3654,10 @@ else if (isSampleOrReplicate) return counter; } - public void getPlateSetExportFile(String fileName, ColumnDescriptor[] cols, List rows, PlateController.FileType fileType, HttpServletResponse response) throws IOException + public void getPlateSetExportFile(String fileName, List cols, List rows, PlateController.FileType fileType, HttpServletResponse response) throws IOException { - boolean isCSV = fileType.equals(PlateController.FileType.CSV); - boolean isTSV = fileType.equals(PlateController.FileType.TSV); + boolean isCSV = PlateController.FileType.CSV.equals(fileType); + boolean isTSV = PlateController.FileType.TSV.equals(fileType); if (isCSV || isTSV) { try (TSVArrayWriter writer = new TSVArrayWriter(fileName, cols, rows)) @@ -3754,7 +3754,7 @@ private List getPlateDisplayColumns(QueryView queryView) // Filter on isQueryColumn, so we don't get the details or update columns return dataRegion.getDisplayColumns().stream() .filter(DisplayColumn::isQueryColumn) - .filter(col -> !col.getName().equals("sampleID")) + .filter(col -> !col.getName().equalsIgnoreCase(WellTable.Column.SampleID.name())) .toList(); } diff --git a/assay/src/org/labkey/assay/plate/PlateSetExport.java b/assay/src/org/labkey/assay/plate/PlateSetExport.java index 682b5df91fa..bdd7f87400c 100644 --- a/assay/src/org/labkey/assay/plate/PlateSetExport.java +++ b/assay/src/org/labkey/assay/plate/PlateSetExport.java @@ -67,7 +67,7 @@ private Object[] getDataRow(String prefix, Results rs, List includedMe ) ); - if (!prefix.equals(PlateSetExport.DESTINATION)) + if (!PlateSetExport.DESTINATION.equals(prefix)) baseColumns.add(rs.getString(FKMap.get(SAMPLE_ID_COL))); for (FieldKey col : includedMetaDataCols) @@ -77,9 +77,9 @@ private Object[] getDataRow(String prefix, Results rs, List includedMe } // Returns array of ColumnDescriptors used as column layout once fed to an ArrayExcelWriter - public static ColumnDescriptor[] getColumnDescriptors(String prefix, List includedMetadataCols) + public static List getColumnDescriptors(String prefix, List includedMetadataCols) { - List baseColumns = new ArrayList<>( + List columnDescriptors = new ArrayList<>( Arrays.asList( new ColumnDescriptor(prefix + "Plate ID"), new ColumnDescriptor(prefix + "Barcode"), @@ -89,21 +89,22 @@ public static ColumnDescriptor[] getColumnDescriptors(String prefix, List metadataColumns = includedMetadataCols .stream() .map(fk -> new ColumnDescriptor(fk.getParts().size() > 1 ? fk.getParent().getCaption() : fk.getCaption())) .toList(); - baseColumns.addAll(metadataColumns); - return baseColumns.toArray(new ColumnDescriptor[0]); + columnDescriptors.addAll(metadataColumns); + return columnDescriptors; } // Create sampleIdToRow of the following form: // {: [{dataRow1}, {dataRow2}, ... ], ... } // Where the data rows contain the key's sample - private Map> getSampleIdToRows(TableInfo wellTable, List includedMetadataCols, int plateSetId, String plateSetExport) { + private Map> getSampleIdToRows(TableInfo wellTable, List includedMetadataCols, int plateSetId, String plateSetExport) + { Map> sampleIdToRow = new LinkedHashMap<>(); try (Results rs = QueryService.get().select(wellTable, getWellColumns(wellTable, includedMetadataCols), new SimpleFilter(FKMap.get(PLATE_SET_ID_COL), plateSetId), new Sort(ROW_ID_COL))) { diff --git a/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java b/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java index 2d02a737bba..f33040feffe 100644 --- a/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java +++ b/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java @@ -10,6 +10,7 @@ import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.api.assay.plate.PositionImpl; +import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.ResultSetRowMapFactory; import org.labkey.api.collections.RowMap; import org.labkey.api.data.ColumnInfo; @@ -22,7 +23,7 @@ import org.labkey.api.query.QueryView; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.HttpView; -import org.labkey.assay.plate.query.WellTable; +import org.labkey.assay.plate.query.WellTable.Column; import java.io.IOException; import java.sql.SQLException; @@ -34,7 +35,7 @@ public class PlateMapExcelWriter extends ExcelWriter { private static final Logger logger = LogHelper.getLogger(PlateMapExcelWriter.class, "Plate map export"); - private static final Set excludedFields = Set.of("sampleid", "type", "wellgroup"); + private static final Set excludedFields = CaseInsensitiveHashSet.of(Column.SampleID.name(), Column.Type.name(), Column.WellGroup.name()); private final Plate _plate; private final QueryView _queryView; @@ -66,8 +67,8 @@ private void initializeWellData() throws SQLException, IOException while (results.next()) { RowMap well = factory.getRowMap(results); - Integer row = (Integer) well.get(WellTable.Column.Row.name()); - Integer col = (Integer) well.get(WellTable.Column.Col.name()); + Integer row = (Integer) well.get(Column.Row.name()); + Integer col = (Integer) well.get(Column.Col.name()); Map> rowMap = _wellData.computeIfAbsent(row, k -> new HashMap<>()); @@ -190,7 +191,7 @@ protected void renderGrid(Sheet sheet, List displayColumns) throw // Removes fields explicitly excluded for Map export protected List getDisplayColumns() { - return _displayColumns.stream().filter(col -> !excludedFields.contains(col.getName().toLowerCase())).toList(); + return _displayColumns.stream().filter(col -> !excludedFields.contains(col.getName())).toList(); } protected List getCustomFields() @@ -212,7 +213,10 @@ protected void renderSheet(Workbook workbook, int sheetNumber) List displayColumns; if (sheetNumber == 0) // Summary view, render all values in each cell - displayColumns = displayCols.stream().filter(dc -> !dc.getName().equals("row") && !dc.getName().equals("col")).toList(); + { + Set excludeFromSummary = Set.of(Column.Col.fieldKey(), Column.Row.fieldKey()); + displayColumns = displayCols.stream().filter(dc -> !excludeFromSummary.contains(dc.getColumnInfo().getFieldKey())).toList(); + } else if (sheetNumber == 1) // Sample ID view displayColumns = List.of(displayCols.get(0)); else // CustomField view