From d7c0672fab492e3b6d8e5e6aeb8ec993b47ed312 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Sat, 19 Jul 2025 09:43:08 -0700 Subject: [PATCH 1/4] PlateMapExcelWriter: case-insensitive column exclusion --- .../src/org/labkey/assay/plate/PlateManager.java | 2 +- .../assay/plate/data/PlateMapExcelWriter.java | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 8a6441440ee..8a723b7ba45 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -3754,7 +3754,7 @@ private List getPlateDisplayColumns(QueryView queryView) // Filter on isQueryColumn, so we don't get the details or update columns return dataRegion.getDisplayColumns().stream() .filter(DisplayColumn::isQueryColumn) - .filter(col -> !col.getName().equals("sampleID")) + .filter(col -> !col.getName().equalsIgnoreCase(WellTable.Column.SampleID.name())) .toList(); } diff --git a/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java b/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java index 2d02a737bba..f33040feffe 100644 --- a/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java +++ b/assay/src/org/labkey/assay/plate/data/PlateMapExcelWriter.java @@ -10,6 +10,7 @@ import org.labkey.api.assay.plate.Plate; import org.labkey.api.assay.plate.PlateCustomField; import org.labkey.api.assay.plate.PositionImpl; +import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.collections.ResultSetRowMapFactory; import org.labkey.api.collections.RowMap; import org.labkey.api.data.ColumnInfo; @@ -22,7 +23,7 @@ import org.labkey.api.query.QueryView; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.HttpView; -import org.labkey.assay.plate.query.WellTable; +import org.labkey.assay.plate.query.WellTable.Column; import java.io.IOException; import java.sql.SQLException; @@ -34,7 +35,7 @@ public class PlateMapExcelWriter extends ExcelWriter { private static final Logger logger = LogHelper.getLogger(PlateMapExcelWriter.class, "Plate map export"); - private static final Set excludedFields = Set.of("sampleid", "type", "wellgroup"); + private static final Set excludedFields = CaseInsensitiveHashSet.of(Column.SampleID.name(), Column.Type.name(), Column.WellGroup.name()); private final Plate _plate; private final QueryView _queryView; @@ -66,8 +67,8 @@ private void initializeWellData() throws SQLException, IOException while (results.next()) { RowMap well = factory.getRowMap(results); - Integer row = (Integer) well.get(WellTable.Column.Row.name()); - Integer col = (Integer) well.get(WellTable.Column.Col.name()); + Integer row = (Integer) well.get(Column.Row.name()); + Integer col = (Integer) well.get(Column.Col.name()); Map> rowMap = _wellData.computeIfAbsent(row, k -> new HashMap<>()); @@ -190,7 +191,7 @@ protected void renderGrid(Sheet sheet, List displayColumns) throw // Removes fields explicitly excluded for Map export protected List getDisplayColumns() { - return _displayColumns.stream().filter(col -> !excludedFields.contains(col.getName().toLowerCase())).toList(); + return _displayColumns.stream().filter(col -> !excludedFields.contains(col.getName())).toList(); } protected List getCustomFields() @@ -212,7 +213,10 @@ protected void renderSheet(Workbook workbook, int sheetNumber) List displayColumns; if (sheetNumber == 0) // Summary view, render all values in each cell - displayColumns = displayCols.stream().filter(dc -> !dc.getName().equals("row") && !dc.getName().equals("col")).toList(); + { + Set excludeFromSummary = Set.of(Column.Col.fieldKey(), Column.Row.fieldKey()); + displayColumns = displayCols.stream().filter(dc -> !excludeFromSummary.contains(dc.getColumnInfo().getFieldKey())).toList(); + } else if (sheetNumber == 1) // Sample ID view displayColumns = List.of(displayCols.get(0)); else // CustomField view From 74ef1a05917d1913c18f49bcf8265be68dc027b2 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 21 Jul 2025 06:38:07 -0700 Subject: [PATCH 2/4] Initial updates --- .../org/labkey/api/data/ArrayExcelWriter.java | 12 ++++++------ .../org/labkey/api/data/TSVArrayWriter.java | 11 +++++------ .../org/labkey/api/data/TSVColumnWriter.java | 10 +++------- api/src/org/labkey/api/data/TSVWriter.java | 3 --- api/src/org/labkey/api/reader/TabLoader.java | 19 ++++++++++++------- .../src/org/labkey/assay/PlateController.java | 17 +++++++---------- .../org/labkey/assay/plate/PlateManager.java | 6 +++--- .../labkey/assay/plate/PlateSetExport.java | 10 +++++----- 8 files changed, 41 insertions(+), 47 deletions(-) diff --git a/api/src/org/labkey/api/data/ArrayExcelWriter.java b/api/src/org/labkey/api/data/ArrayExcelWriter.java index a3a05410c48..8a0356fdcfa 100644 --- a/api/src/org/labkey/api/data/ArrayExcelWriter.java +++ b/api/src/org/labkey/api/data/ArrayExcelWriter.java @@ -17,19 +17,19 @@ public class ArrayExcelWriter extends ExcelWriter * @param data The data rows, in which index position of a value corresponds to the desired respective column index * @param cols The columns, in which ordering determines the left-to-right column ordering in the generated Excel */ - public ArrayExcelWriter(List data, ColumnDescriptor[] cols) + public ArrayExcelWriter(List data, List cols) { super(ExcelDocumentType.xlsx); this.data = data; - List xlcols = new ArrayList<>(); + List displayColumns = new ArrayList<>(); - for (int i = 0; i < cols.length; i++) + for (int i = 0; i < cols.size(); i++) { - ColumnDescriptor col = cols[i]; - xlcols.add(new ArrayDisplayColumn(col.name, col.clazz, i)); + ColumnDescriptor col = cols.get(i); + displayColumns.add(new ArrayDisplayColumn(col.name, col.clazz, i)); } - setDisplayColumns(xlcols); + setDisplayColumns(displayColumns); } @Override diff --git a/api/src/org/labkey/api/data/TSVArrayWriter.java b/api/src/org/labkey/api/data/TSVArrayWriter.java index be40c90480a..edcc868eaee 100644 --- a/api/src/org/labkey/api/data/TSVArrayWriter.java +++ b/api/src/org/labkey/api/data/TSVArrayWriter.java @@ -5,7 +5,6 @@ import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; // This class supports generating files with duplicate column names. Consider using TSVMapWriter if // multiple identical column names is not an implementation concern. @@ -15,17 +14,17 @@ public class TSVArrayWriter extends TSVWriter private final List> _rows; private final String _fileName; - public TSVArrayWriter(String fileName, ColumnDescriptor[] columns, List rows) + public TSVArrayWriter(String fileName, List columns, List rows) { _fileName = fileName; - _columns = Arrays.stream(columns) + _columns = columns.stream() .map(ColumnDescriptor::getColumnName) - .collect(Collectors.toList()); + .toList(); _rows = rows.stream() .map(array -> Arrays.stream(array) .map(obj -> (obj == null) ? "" : String.valueOf(obj)) - .collect(Collectors.toList())) - .collect(Collectors.toList()); + .toList()) + .toList(); } @Override diff --git a/api/src/org/labkey/api/data/TSVColumnWriter.java b/api/src/org/labkey/api/data/TSVColumnWriter.java index ba19978dcbc..cbd2550923d 100644 --- a/api/src/org/labkey/api/data/TSVColumnWriter.java +++ b/api/src/org/labkey/api/data/TSVColumnWriter.java @@ -26,9 +26,6 @@ * Extracted DisplayColumn handling out of TSVGridWriter so rendering DisplayColumns * may be used without a ResultSet. You will still need to set up a RenderContext * for DisplayColumn to render values. - * - * User: kevink - * Date: 9/9/11 */ public abstract class TSVColumnWriter extends TSVWriter { @@ -72,11 +69,11 @@ protected Iterable getColumnHeaders(RenderContext ctx, Iterable getValues(RenderContext ctx, Iterable displayColumns) { diff --git a/api/src/org/labkey/api/data/TSVWriter.java b/api/src/org/labkey/api/data/TSVWriter.java index 249ed2c73ed..6189b480992 100644 --- a/api/src/org/labkey/api/data/TSVWriter.java +++ b/api/src/org/labkey/api/data/TSVWriter.java @@ -96,7 +96,6 @@ public TSVWriter() { } - public String getFilenamePrefix() { return _filenamePrefix; @@ -210,7 +209,6 @@ protected boolean shouldQuote(String value) _escapedCharsString = "\r\n" + _rowSeparator + _chDelimiter + _chQuote; } - int len = value.length(); if (len == 0) return _preserveEmptyString; @@ -279,7 +277,6 @@ public boolean isHeaderRowVisible() return _headerRowVisible; } - public void setHeaderRowVisible(boolean headerRowVisible) { _headerRowVisible = headerRowVisible; diff --git a/api/src/org/labkey/api/reader/TabLoader.java b/api/src/org/labkey/api/reader/TabLoader.java index 4631b33f02d..bb032c90537 100644 --- a/api/src/org/labkey/api/reader/TabLoader.java +++ b/api/src/org/labkey/api/reader/TabLoader.java @@ -56,7 +56,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; - /** * Parses rows of tab-delimited text, returning a CloseableIterator of Map. The iterator must be closed * (typically via try-with-resources or a finally block) to close the underlying input source. The iterator can be wrapped @@ -91,7 +90,10 @@ public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Contain } @NotNull @Override - public FileType getFileType() { return TSV_FILE_TYPE; } + public FileType getFileType() + { + return TSV_FILE_TYPE; + } } public static class CsvFactory extends AbstractDataLoaderFactory @@ -114,7 +116,10 @@ public TabLoader createLoader(InputStream is, boolean hasColumnHeaders, Containe } @NotNull @Override - public FileType getFileType() { return CSV_FILE_TYPE; } + public FileType getFileType() + { + return CSV_FILE_TYPE; + } } public static class CsvFactoryNoConversions extends CsvFactory @@ -141,9 +146,6 @@ public TabLoader createLoader(InputStream is, boolean hasColumnHeaders, Containe TabLoader loader = super.createLoader(is, hasColumnHeaders, mvIndicatorContainer); return configParsing(loader); } - - @NotNull @Override - public FileType getFileType() { return CSV_FILE_TYPE; } } public static class MysqlFactory extends AbstractDataLoaderFactory @@ -172,7 +174,10 @@ public DataLoader createLoader(InputStream is, boolean hasColumnHeaders, Contain } @NotNull @Override - public FileType getFileType() { return CSV_FILE_TYPE; } + public FileType getFileType() + { + return CSV_FILE_TYPE; + } } protected static char COMMENT_CHAR = '#'; diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 333c17dc567..6a8a5ea771c 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -1309,18 +1309,17 @@ public Object execute(WorklistForm form, BindException errors) throws Exception List sourceIncludedMetadataCols = PlateManager.get().getMetadataColumns(plateSetSource, getContainer(), getUser(), cf); List destinationIncludedMetadataCols = PlateManager.get().getMetadataColumns(plateSetDestination, getContainer(), getUser(), cf); - ColumnDescriptor[] sourceXlCols = PlateSetExport.getColumnDescriptors(PlateSetExport.SOURCE, sourceIncludedMetadataCols); - ColumnDescriptor[] destinationXlCols = PlateSetExport.getColumnDescriptors(PlateSetExport.DESTINATION, destinationIncludedMetadataCols); - ColumnDescriptor[] xlCols = ArrayUtils.addAll(sourceXlCols, destinationXlCols); + List sourceColumns = PlateSetExport.getColumnDescriptors(PlateSetExport.SOURCE, sourceIncludedMetadataCols); + List destinationColumns = PlateSetExport.getColumnDescriptors(PlateSetExport.DESTINATION, destinationIncludedMetadataCols); + List exportColumns = new ArrayList<>(sourceColumns); + exportColumns.addAll(destinationColumns); List plateDataRows = PlateManager.get().getWorklist(form.getSourcePlateSetId(), form.getDestinationPlateSetId(), sourceIncludedMetadataCols, destinationIncludedMetadataCols, getContainer(), getUser()); String fullFileName = plateSetSource.getName() + " - " + plateSetDestination.getName(); - PlateManager.get().getPlateSetExportFile(fullFileName, xlCols, plateDataRows, form.getFileType(), getViewContext().getResponse()); + PlateManager.get().getPlateSetExportFile(fullFileName, exportColumns, plateDataRows, form.getFileType(), getViewContext().getResponse()); SimpleMetricsService.get().increment(AssayModule.NAME, "plateSet", "exportWorklist"); - - return null; // Returning anything here will cause error as excel writer will close the response stream } catch (Exception e) { @@ -1387,12 +1386,10 @@ public Object execute(InstrumentInstructionForm form, BindException errors) thro cf = form.getContainerFilter().create(getViewContext()); List includedMetadataCols = PlateManager.get().getMetadataColumns(plateSet, getContainer(), getUser(), cf); - ColumnDescriptor[] xlCols = PlateSetExport.getColumnDescriptors("", includedMetadataCols); + List exportColumns = PlateSetExport.getColumnDescriptors("", includedMetadataCols); List plateDataRows = PlateManager.get().getInstrumentInstructions(form.getPlateSetId(), includedMetadataCols, getContainer(), getUser()); - PlateManager.get().getPlateSetExportFile(plateSet.getName() + "-instructions", xlCols, plateDataRows, form.getFileType(), getViewContext().getResponse()); - - return null; // Returning anything here will cause error as excel writer will close the response stream + PlateManager.get().getPlateSetExportFile(plateSet.getName() + "-instructions", exportColumns, plateDataRows, form.getFileType(), getViewContext().getResponse()); } catch (Exception e) { diff --git a/assay/src/org/labkey/assay/plate/PlateManager.java b/assay/src/org/labkey/assay/plate/PlateManager.java index 8a723b7ba45..d423c69a8ee 100644 --- a/assay/src/org/labkey/assay/plate/PlateManager.java +++ b/assay/src/org/labkey/assay/plate/PlateManager.java @@ -3654,10 +3654,10 @@ else if (isSampleOrReplicate) return counter; } - public void getPlateSetExportFile(String fileName, ColumnDescriptor[] cols, List rows, PlateController.FileType fileType, HttpServletResponse response) throws IOException + public void getPlateSetExportFile(String fileName, List cols, List rows, PlateController.FileType fileType, HttpServletResponse response) throws IOException { - boolean isCSV = fileType.equals(PlateController.FileType.CSV); - boolean isTSV = fileType.equals(PlateController.FileType.TSV); + boolean isCSV = PlateController.FileType.CSV.equals(fileType); + boolean isTSV = PlateController.FileType.TSV.equals(fileType); if (isCSV || isTSV) { try (TSVArrayWriter writer = new TSVArrayWriter(fileName, cols, rows)) diff --git a/assay/src/org/labkey/assay/plate/PlateSetExport.java b/assay/src/org/labkey/assay/plate/PlateSetExport.java index 682b5df91fa..de29d8c4587 100644 --- a/assay/src/org/labkey/assay/plate/PlateSetExport.java +++ b/assay/src/org/labkey/assay/plate/PlateSetExport.java @@ -77,9 +77,9 @@ private Object[] getDataRow(String prefix, Results rs, List includedMe } // Returns array of ColumnDescriptors used as column layout once fed to an ArrayExcelWriter - public static ColumnDescriptor[] getColumnDescriptors(String prefix, List includedMetadataCols) + public static List getColumnDescriptors(String prefix, List includedMetadataCols) { - List baseColumns = new ArrayList<>( + List columnDescriptors = new ArrayList<>( Arrays.asList( new ColumnDescriptor(prefix + "Plate ID"), new ColumnDescriptor(prefix + "Barcode"), @@ -89,15 +89,15 @@ public static ColumnDescriptor[] getColumnDescriptors(String prefix, List metadataColumns = includedMetadataCols .stream() .map(fk -> new ColumnDescriptor(fk.getParts().size() > 1 ? fk.getParent().getCaption() : fk.getCaption())) .toList(); - baseColumns.addAll(metadataColumns); - return baseColumns.toArray(new ColumnDescriptor[0]); + columnDescriptors.addAll(metadataColumns); + return columnDescriptors; } // Create sampleIdToRow of the following form: From 29b96e72021907b14856109195b4737b9fcaf191 Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 21 Jul 2025 06:54:29 -0700 Subject: [PATCH 3/4] nits --- api/src/org/labkey/api/data/TSVMapWriter.java | 4 ---- assay/src/org/labkey/assay/plate/PlateSetExport.java | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/api/src/org/labkey/api/data/TSVMapWriter.java b/api/src/org/labkey/api/data/TSVMapWriter.java index 52d6e5519b2..fa78e854a48 100644 --- a/api/src/org/labkey/api/data/TSVMapWriter.java +++ b/api/src/org/labkey/api/data/TSVMapWriter.java @@ -31,10 +31,6 @@ import java.util.List; import java.util.Map; -/** - * User: kevink - * Date: 9/11/11 - */ public class TSVMapWriter extends TSVWriter { private final Collection _columns; diff --git a/assay/src/org/labkey/assay/plate/PlateSetExport.java b/assay/src/org/labkey/assay/plate/PlateSetExport.java index de29d8c4587..bdd7f87400c 100644 --- a/assay/src/org/labkey/assay/plate/PlateSetExport.java +++ b/assay/src/org/labkey/assay/plate/PlateSetExport.java @@ -67,7 +67,7 @@ private Object[] getDataRow(String prefix, Results rs, List includedMe ) ); - if (!prefix.equals(PlateSetExport.DESTINATION)) + if (!PlateSetExport.DESTINATION.equals(prefix)) baseColumns.add(rs.getString(FKMap.get(SAMPLE_ID_COL))); for (FieldKey col : includedMetaDataCols) @@ -103,7 +103,8 @@ public static List getColumnDescriptors(String prefix, List: [{dataRow1}, {dataRow2}, ... ], ... } // Where the data rows contain the key's sample - private Map> getSampleIdToRows(TableInfo wellTable, List includedMetadataCols, int plateSetId, String plateSetExport) { + private Map> getSampleIdToRows(TableInfo wellTable, List includedMetadataCols, int plateSetId, String plateSetExport) + { Map> sampleIdToRow = new LinkedHashMap<>(); try (Results rs = QueryService.get().select(wellTable, getWellColumns(wellTable, includedMetadataCols), new SimpleFilter(FKMap.get(PLATE_SET_ID_COL), plateSetId), new Sort(ROW_ID_COL))) { From 4aaf0fc56cbb7b7526de6af653eb80ed79b77a1a Mon Sep 17 00:00:00 2001 From: labkey-nicka Date: Mon, 21 Jul 2025 07:34:10 -0700 Subject: [PATCH 4/4] Issue 53302: quote comment-like values in TSVWriter --- api/src/org/labkey/api/data/TSVWriter.java | 5 ++++- assay/src/org/labkey/assay/actions/ImportRunApiAction.java | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/data/TSVWriter.java b/api/src/org/labkey/api/data/TSVWriter.java index 6189b480992..396cebd9326 100644 --- a/api/src/org/labkey/api/data/TSVWriter.java +++ b/api/src/org/labkey/api/data/TSVWriter.java @@ -40,6 +40,7 @@ public abstract class TSVWriter extends TextWriter protected char _chQuote = '"'; protected String _rowSeparator = "\n"; public static final String BACKSLASH_CHAR_STRING = "\\"; + private static final char COMMENT_CHAR = '#'; protected List _fileHeader = null; protected boolean _headerRowVisible = true; @@ -216,6 +217,8 @@ protected boolean shouldQuote(String value) char lastCh = value.charAt(len-1); if (Character.isSpaceChar(firstCh) || Character.isSpaceChar(lastCh)) return true; + if (firstCh == COMMENT_CHAR) // Issue 50719, Issue 53302 + return true; if (StringUtils.containsAny(value, _additionalQuotedChars)) return true; return StringUtils.containsAny(value,_escapedCharsString); @@ -234,7 +237,7 @@ protected String getContentType() return delim.contentType; } - return "text/tab-separated-values"; + return DELIM.TAB.contentType; } /** diff --git a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java index 881ec528aaa..7d10f5f1291 100644 --- a/assay/src/org/labkey/assay/actions/ImportRunApiAction.java +++ b/assay/src/org/labkey/assay/actions/ImportRunApiAction.java @@ -279,7 +279,6 @@ else if (rawData != null && !rawData.isEmpty()) try (TSVMapWriter tsvWriter = saveMatchingColumnDataOnly ? new TSVMapWriter(columns, rawData) : new TSVMapWriter(columns, rawData, true)) { - tsvWriter.setAdditionalQuotedChars("#"); //Issue 50719: If the first column name starts with a #, the data loader will treat the header row as a comment tsvWriter.write(fileObject.toNioPathForWrite().toFile()); factory.setRawData(null); factory.setUploadedData(Collections.singletonMap(PRIMARY_FILE, fileObject));