From fd6fa55c95b8d08e00ac07a16cb5687eea4d376a Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 29 Jan 2024 09:21:25 +0530 Subject: [PATCH 01/11] long frame column writer --- .../columnar/LongArrayFrameColumnWriter.java | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java new file mode 100644 index 000000000000..fb000871ac4d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java @@ -0,0 +1,139 @@ +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.AppendableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.frame.allocation.MemoryRange; +import org.apache.druid.frame.write.FrameWriterUtils; +import org.apache.druid.segment.ColumnValueSelector; + +import java.util.List; + +public class LongArrayFrameColumnWriter implements FrameColumnWriter +{ + // TODO(laksh): Copy pasted after following the logic from StringFrameColumnWriter, since numeric writers have 3 + // regions as well. The type size is fixed, however there's no special nullable marker + private static final int INITIAL_ALLOCATION_SIZE = 180; + + private static final int ELEMENT_SIZE = Long.BYTES; + + private static final byte NULL_ELEMENT_MARKER = 0x00; + private static final byte NON_NULL_ELEMENT_MARKER = 0x01; + + /** + * A byte required at the beginning for type code + */ + public static final long DATA_OFFSET = 1; + + final ColumnValueSelector selector; + final MemoryAllocator allocator; + final byte typeCode; + + + + /** + * Row lengths: one int per row with the number of values contained by that row and all previous rows. + * Only written for multi-value and array columns. When the corresponding row is null itself, the length is + * written as -(actual length) - 1. (Guaranteed to be a negative number even if "actual length" is zero.) + */ + private final AppendableMemory cumulativeRowLengths; + + /** + * Denotes if the element of the row is null or not + */ + private final AppendableMemory rowNullityData; + + /** + * Row data. + */ + private final AppendableMemory rowData; + + private int lastCumulativeRowLength = 0; + private int lastRowLength = 0; + + + public LongArrayFrameColumnWriter( + final ColumnValueSelector selector, + final MemoryAllocator allocator, + final byte typeCode + ) + { + this.selector = selector; + this.allocator = allocator; + this.typeCode = typeCode; + this.cumulativeRowLengths = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + this.rowNullityData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + this.rowData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + } + + @Override + public boolean addSelection() + { + List numericArray = FrameWriterUtils.getNumericArrayFromObject(selector.getObject()); + int rowLength = numericArray == null ? 0 : numericArray.size(); + + if ((long) lastCumulativeRowLength + rowLength > Integer.MAX_VALUE) { + return false; + } + + if (!cumulativeRowLengths.reserveAdditional(Integer.BYTES)) { + return false; + } + + if (!rowNullityData.reserveAdditional(rowLength * Byte.BYTES)) { + return false; + } + + if (!rowData.reserveAdditional(rowLength * ELEMENT_SIZE)) { + return false; + } + + final MemoryRange rowLengthsCursor = cumulativeRowLengths.cursor(); + + if (numericArray == null) { + rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), -(lastCumulativeRowLength + rowLength) - 1); + } + cumulativeRowLengths.advanceCursor(Integer.BYTES); + lastRowLength = rowLength; + lastCumulativeRowLength += rowLength; + + final MemoryRange rowNullityDataCursor = rowLength > 0 ? rowData.cursor() : null; + final MemoryRange rowDataCursor = rowLength > 0 ? rowData.cursor() : null; + for (int i = 0; i < rowLength; ++i) { + final Number element = numericArray.get(i); + if (element == null) { + rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NULL_ELEMENT_MARKER); + rowDataCursor.memory().putLong(rowDataCursor.start() + (long) Long.BYTES * i, 0); + } else { + rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NON_NULL_ELEMENT_MARKER); + // Cast should be safe, as we have a long array column value selector, and we have checked that it is not null + rowDataCursor.memory().putLong(rowDataCursor.start() + (long) Long.BYTES * i, (long) element); + } + } + + } + + @Override + public void undo() + { + + } + + @Override + public long size() + { + return 0; + } + + @Override + public long writeTo(WritableMemory memory, long position) + { + return 0; + } + + @Override + public void close() + { + + } +} From 0421e2d2a6bdb900348f21ec8d11b41118c8c39a Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 15 Feb 2024 16:20:45 +0530 Subject: [PATCH 02/11] frame reader beginning --- docs/release-info/release-notes.md | 106 --- docs/release-info/upgrade-notes.md | 634 ------------------ .../columnar/LongArrayFrameColumnReader.java | 96 +++ .../columnar/LongArrayFrameColumnWriter.java | 45 +- 4 files changed, 133 insertions(+), 748 deletions(-) delete mode 100644 docs/release-info/release-notes.md delete mode 100644 docs/release-info/upgrade-notes.md create mode 100644 processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java diff --git a/docs/release-info/release-notes.md b/docs/release-info/release-notes.md deleted file mode 100644 index 768ceef697bc..000000000000 --- a/docs/release-info/release-notes.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -id: release-notes -title: "Release notes" ---- - - - - - -Apache Druid {{DRUIDVERSION}} contains over $NUMBER_FEATURES new features, bug fixes, performance enhancements, documentation improvements, and additional test coverage from $NUMBER_OF_CONTRIBUTORS contributors. - - - -See the [complete set of changes](https://github.com/apache/druid/issues?q=is%3Aclosed+milestone%3A{{MILESTONE}}+sort%3Aupdated-desc+) for additional details, including bug fixes. - -Review the [upgrade notes](#upgrade-notes) and [incompatible changes](#incompatible-changes) before you upgrade to Druid {{DRUIDVERSION}}. -If you are upgrading across multiple versions, see the [Upgrade notes](upgrade-notes.md) page, which lists upgrade notes for the most recent Druid versions. - - - -## Important features, changes, and deprecations - -This section contains important information about new and existing features. - -## Functional area and related changes - -This section contains detailed release notes separated by areas. - -### Web console - -#### Other web console improvements - -### Ingestion - -#### SQL-based ingestion - -##### Other SQL-based ingestion improvements - -#### Streaming ingestion - -##### Other streaming ingestion improvements - -### Querying - -#### Other querying improvements - -### Cluster management - -#### Other cluster management improvements - -### Data management - -#### Other data management improvements - -### Metrics and monitoring - -### Extensions - -### Documentation improvements - -## Upgrade notes and incompatible changes - -### Upgrade notes - -### Incompatible changes - -### Developer notes - -#### Dependency updates - -The following dependencies have had their versions bumped: \ No newline at end of file diff --git a/docs/release-info/upgrade-notes.md b/docs/release-info/upgrade-notes.md deleted file mode 100644 index 46e5ed6fc1a1..000000000000 --- a/docs/release-info/upgrade-notes.md +++ /dev/null @@ -1,634 +0,0 @@ ---- -id: upgrade-notes -title: "Upgrade notes" ---- - - - -The upgrade notes assume that you are upgrading from the Druid version that immediately precedes your target version. If you are upgrading across multiple versions, make sure you read the upgrade notes for all the intermediate versions. - -For the full release notes for a specific version, see the [releases page](https://github.com/apache/druid/releases). - -## 28.0.0 - -### Upgrade notes - -#### Upgrade Druid segments table - -Druid 28.0.0 adds a new column to the Druid metadata table that requires an update to the table. - -If `druid.metadata.storage.connector.createTables` is set to `true` and the metadata store user has DDL privileges, the segments table gets automatically updated at startup to include the new `used_status_last_updated` column. No additional work is needed for the upgrade. - -If either of those requirements are not met, pre-upgrade steps are required. You must make these updates before you upgrade to Druid 28.0.0, or the Coordinator and Overlord processes fail. - -Although you can manually alter your table to add the new `used_status_last_updated` column, Druid also provides a [CLI tool](https://druid.apache.org/docs/latest/operations/metadata-migration/#create-druid-tables) to do it. - -[#12599](https://github.com/apache/druid/pull/12599) [#14868](https://github.com/apache/druid/pull/14868) - -In the example commands below: - -- `lib` is the Druid lib directory -- `extensions` is the Druid extensions directory -- `base` corresponds to the value of `druid.metadata.storage.tables.base` in the configuration, `druid` by default. -- The `--connectURI` parameter corresponds to the value of `druid.metadata.storage.connector.connectURI`. -- The `--user` parameter corresponds to the value of `druid.metadata.storage.connector.user`. -- The `--password` parameter corresponds to the value of `druid.metadata.storage.connector.password`. -- The `--action` parameter corresponds to the update action you are executing. In this case, it is `add-last-used-to-segments` - -##### Upgrade step for MySQL - -```bash -cd ${DRUID_ROOT} -java -classpath "lib/*" -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory="extensions" -Ddruid.extensions.loadList=[\"mysql-metadata-storage\"] -Ddruid.metadata.storage.type=mysql org.apache.druid.cli.Main tools metadata-update --connectURI="" --user USER --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments -``` - -##### Upgrade step for PostgreSQL - -```bash -cd ${DRUID_ROOT} -java -classpath "lib/*" -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory="extensions" -Ddruid.extensions.loadList=[\"postgresql-metadata-storage\"] -Ddruid.metadata.storage.type=postgresql org.apache.druid.cli.Main tools metadata-update --connectURI="" --user USER --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments -``` - -##### Manual upgrade step - -```SQL -ALTER TABLE druid_segments -ADD used_status_last_updated varchar(255); -``` - -#### Recommended syntax for SQL UNNEST - -The recommended syntax for SQL UNNEST has changed. We recommend using CROSS JOIN instead of commas for most queries to prevent issues with precedence. For example, use: - -```sql -SELECT column_alias_name1 FROM datasource CROSS JOIN UNNEST(source_expression1) AS table_alias_name1(column_alias_name1) CROSS JOIN UNNEST(source_expression2) AS table_alias_name2(column_alias_name2), ... -``` - -Do not use: - -```sql -SELECT column_alias_name FROM datasource, UNNEST(source_expression1) AS table_alias_name1(column_alias_name1), UNNEST(source_expression2) AS table_alias_name2(column_alias_name2), ... -``` - -#### Dynamic parameters - -The Apache Calcite version has been upgraded from 1.21 to 1.35. As part of the Calcite upgrade, the behavior of type inference for dynamic parameters has changed. To avoid any type interference issues, explicitly `CAST` all dynamic parameters as a specific data type in SQL queries. For example, use: - -```sql -SELECT (1 * CAST (? as DOUBLE))/2 as tmp -``` - -Do not use: - -```sql -SELECT (1 * ?)/2 as tmp -``` - -#### Nested column format - -`json` type columns created with Druid 28.0.0 are not backwards compatible with Druid versions older than 26.0.0. -If you are upgrading from a version prior to Druid 26.0.0 and you use `json` columns, upgrade to Druid 26.0.0 before you upgrade to Druid 28.0.0. -Additionally, to downgrade to a version older than Druid 26.0.0, any new segments created in Druid 28.0.0 should be re-ingested using Druid 26.0.0 or 27.0.0 prior to further downgrading. - -When upgrading from a previous version, you can continue to write nested columns in a backwards compatible format (version 4). - -In a classic batch ingestion job, include `formatVersion` in the `dimensions` list of the `dimensionsSpec` property. For example: - -```json - "dimensionsSpec": { - "dimensions": [ - "product", - "department", - { - "type": "json", - "name": "shipTo", - "formatVersion": 4 - } - ] - }, -``` - -To set the default nested column version, set the desired format version in the common runtime properties. For example: - -```java -druid.indexing.formats.nestedColumnFormatVersion=4 -``` - -#### SQL compatibility - -Starting with Druid 28.0.0, the default way Druid treats nulls and booleans has changed. - -For nulls, Druid now differentiates between an empty string and a record with no data as well as between an empty numerical record and `0`. -You can revert to the previous behavior by setting `druid.generic.useDefaultValueForNull` to `true`. - -This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion. - -For booleans, Druid now strictly uses `1` (true) or `0` (false). Previously, true and false could be represented either as `true` and `false` as well as `1` and `0`, respectively. In addition, Druid now returns a null value for boolean comparisons like `True && NULL`. - -You can revert to the previous behavior by setting `druid.expressions.useStrictBooleans` to `false`. -This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion. - -The following table illustrates some example scenarios and the impact of the changes. - -
Show the table - -| Query| Druid 27.0.0 and earlier| Druid 28.0.0 and later| -|------|------------------------|----------------------| -| Query empty string| Empty string (`''`) or null| Empty string (`''`)| -| Query null string| Null or empty| Null| -| COUNT(*)| All rows, including nulls| All rows, including nulls| -| COUNT(column)| All rows excluding empty strings| All rows including empty strings but excluding nulls| -| Expression 100 && 11| 11| 1| -| Expression 100 || 11| 100| 1| -| Null FLOAT/DOUBLE column| 0.0| Null| -| Null LONG column| 0| Null| -| Null `__time` column| 0, meaning 1970-01-01 00:00:00 UTC| 1970-01-01 00:00:00 UTC| -| Null MVD column| `''`| Null| -| ARRAY| Null| Null| -| COMPLEX| none| Null| -
- -Before upgrading to Druid 28.0.0, update your queries to account for the changed behavior as described in the following sections. - -##### NULL filters - -If your queries use NULL in the filter condition to match both nulls and empty strings, you should add an explicit filter clause for empty strings. For example, update `s IS NULL` to `s IS NULL OR s = ''`. - -##### COUNT functions - -`COUNT(column)` now counts empty strings. If you want to continue excluding empty strings from the count, replace `COUNT(column)` with `COUNT(column) FILTER(WHERE column <> '')`. - -##### GroupBy queries - -GroupBy queries on columns containing null values can now have additional entries as nulls can co-exist with empty strings. - -#### Stop Supervisors that ingest from multiple Kafka topics before downgrading - -If you have added supervisors that ingest from multiple Kafka topics in Druid 28.0.0 or later, stop those supervisors before downgrading to a version prior to Druid 28.0.0 because the supervisors will fail in versions prior to Druid 28.0.0. - -#### `lenientAggregatorMerge` deprecated - -`lenientAggregatorMerge` property in segment metadata queries has been deprecated. It will be removed in future releases. -Use `aggregatorMergeStrategy` instead. `aggregatorMergeStrategy` also supports the `latest` and `earliest` strategies in addition to `strict` and `lenient` strategies from `lenientAggregatorMerge`. - -[#14560](https://github.com/apache/druid/pull/14560) -[#14598](https://github.com/apache/druid/pull/14598) - -#### Broker parallel merge config options - -The paths for `druid.processing.merge.pool.*` and `druid.processing.merge.task.*` have been flattened to use `druid.processing.merge.*` instead. The legacy paths for the configs are now deprecated and will be removed in a future release. Migrate your settings to use the new paths because the old paths will be ignored in the future. - -[#14695](https://github.com/apache/druid/pull/14695) - -#### Ingestion options for ARRAY typed columns - -Starting with Druid 28.0.0, the MSQ task engine can detect and ingest arrays as ARRAY typed columns when you set the query context parameter `arrayIngestMode` to `array`. -The `arrayIngestMode` context parameter controls how ARRAY type values are stored in Druid segments. - -When you set `arrayIngestMode` to `array` (recommended for SQL compliance), the MSQ task engine stores all ARRAY typed values in [ARRAY typed columns](https://druid.apache.org/docs/latest/querying/arrays) and supports storing both VARCHAR and numeric typed arrays. - -For backwards compatibility, `arrayIngestMode` defaults to `mvd`. When `"arrayIngestMode":"mvd"`, Druid only supports VARCHAR typed arrays and stores them as [multi-value string columns](https://druid.apache.org/docs/latest/querying/multi-value-dimensions). - -When you set `arrayIngestMode` to `none`, Druid throws an exception when trying to store any type of arrays. - -For more information on how to ingest `ARRAY` typed columns with SQL-based ingestion, see [SQL data types](https://druid.apache.org/docs/latest/querying/sql-data-types#arrays) and [Array columns](https://druid.apache.org/docs/latest/querying/arrays). - -### Incompatible changes - -#### Removed Hadoop 2 - -Support for Hadoop 2 has been removed. -Migrate to SQL-based ingestion or JSON-based batch ingestion if you are using Hadoop 2.x for ingestion today. -If migrating to Druid's built-in ingestion is not possible, you must upgrade your Hadoop infrastructure to 3.x+ before upgrading to Druid 28.0.0. - -[#14763](https://github.com/apache/druid/pull/14763) - -#### Removed GroupBy v1 - -The GroupBy v1 engine has been removed. Use the GroupBy v2 engine instead, which has been the default GroupBy engine for several releases. -There should be no impact on your queries. - -Additionally, `AggregatorFactory.getRequiredColumns` has been deprecated and will be removed in a future release. If you have an extension that implements `AggregatorFactory`, then this method should be removed from your implementation. - -[#14866](https://github.com/apache/druid/pull/14866) - -#### Removed Coordinator dynamic configs - -The `decommissioningMaxPercentOfMaxSegmentsToMove` config has been removed. -The use case for this config is handled by smart segment loading now, which is enabled by default. - -[#14923](https://github.com/apache/druid/pull/14923) - -#### Removed `cachingCost` strategy - -The `cachingCost` strategy for segment loading has been removed. -Use `cost` instead, which has the same benefits as `cachingCost`. - -If you have `cachingCost` set, the system ignores this setting and automatically uses `cost`. - -[#14798](https://github.com/apache/druid/pull/14798) - -#### Removed `InsertCannotOrderByDescending` - -The deprecated MSQ fault `InsertCannotOrderByDescending` has been removed. - -[#14588](https://github.com/apache/druid/pull/14588) - -#### Removed the backward compatibility code for the Handoff API - -The backward compatibility code for the Handoff API in `CoordinatorBasedSegmentHandoffNotifier` has been removed. -If you are upgrading from a Druid version older than 0.14.0, upgrade to a newer version of Druid before upgrading to Druid 28.0.0. - -[#14652](https://github.com/apache/druid/pull/14652) - -## 27.0.0 - -### Upgrade notes - -#### Worker input bytes for SQL-based ingestion - -The maximum input bytes for each worker for SQL-based ingestion is now 512 MiB (previously 10 GiB). - -[#14307](https://github.com/apache/druid/pull/14307) - -#### Parameter execution changes for Kafka - -When using the built-in `FileConfigProvider` for Kafka, interpolations are now intercepted by the JsonConfigurator instead of being passed down to the Kafka provider. This breaks existing deployments. - -For more information, see [KIP-297](https://cwiki.apache.org/confluence/display/KAFKA/KIP-297%3A+Externalizing+Secrets+for+Connect+Configurations). - -[#13023](https://github.com/apache/druid/pull/13023) - -#### Hadoop 2 deprecated - -Many of the important dependent libraries that Druid uses no longer support Hadoop 2. In order for Druid to stay current and have pathways to mitigate security vulnerabilities, the community has decided to deprecate support for Hadoop 2.x releases starting this release. Starting with Druid 28.x, Hadoop 3.x is the only supported Hadoop version. - -Consider migrating to SQL-based ingestion or native ingestion if you are using Hadoop 2.x for ingestion today. If migrating to Druid ingestion is not possible, plan to upgrade your Hadoop infrastructure before upgrading to the next Druid release. - -#### GroupBy v1 deprecated - -GroupBy queries using the v1 legacy engine has been deprecated. It will be removed in future releases. Use v2 instead. Note that v2 has been the default GroupBy engine. - -For more information, see [GroupBy queries](https://druid.apache.org/docs/latest/querying/groupbyquery.html). - -#### Push-based real-time ingestion deprecated - -Support for push-based real-time ingestion has been deprecated. It will be removed in future releases. - -#### `cachingCost` segment balancing strategy deprecated - -The `cachingCost` strategy has been deprecated and will be removed in future releases. Use an alternate segment balancing strategy instead, such as `cost`. - -#### Segment loading config changes - -The following segment related configs are now deprecated and will be removed in future releases: - -* `maxSegmentsInNodeLoadingQueue` -* `maxSegmentsToMove` -* `replicationThrottleLimit` -* `useRoundRobinSegmentAssignment` -* `replicantLifetime` -* `maxNonPrimaryReplicantsToLoad` -* `decommissioningMaxPercentOfMaxSegmentsToMove` - -Use `smartSegmentLoading` mode instead, which calculates values for these variables automatically. - -Additionally, the defaults for the following Coordinator dynamic configs have changed: - -* `maxsegmentsInNodeLoadingQueue` : 500, previously 100 -* `maxSegmentsToMove`: 100, previously 5 -* `replicationThrottleLimit`: 500, previously 10 - -These new defaults can improve performance for most use cases. - -[#13197](https://github.com/apache/druid/pull/13197) -[#14269](https://github.com/apache/druid/pull/14269) - -#### `SysMonitor` support deprecated - -Switch to `OshiSysMonitor` as `SysMonitor` is now deprecated and will be removed in future releases. - -### Incompatible changes - -#### Removed property for setting max bytes for dimension lookup cache - -`druid.processing.columnCache.sizeBytes` has been removed since it provided limited utility after a number of internal changes. Leaving this config is harmless, but it does nothing. - -[#14500](https://github.com/apache/druid/pull/14500) - -#### Removed Coordinator dynamic configs - -The following Coordinator dynamic configs have been removed: - -* `emitBalancingStats`: Stats for errors encountered while balancing will always be emitted. Other debugging stats will not be emitted but can be logged by setting the appropriate `debugDimensions`. -* `useBatchedSegmentSampler` and `percentOfSegmentsToConsiderPerMove`: Batched segment sampling is now the standard and will always be on. - -Use the new [smart segment loading](https://druid.apache.org/docs/latest/configuration/#smart-segment-loading) mode instead. - -[#14524](https://github.com/apache/druid/pull/14524) - -## 26.0.0 - -### Upgrade notes - -#### Real-time tasks - -Optimized query performance by lowering the default maxRowsInMemory for real-time ingestion, which might lower overall ingestion throughput. - -[#13939](https://github.com/apache/druid/pull/13939) - -### Incompatible changes - -#### Firehose ingestion removed - -The firehose/parser specification used by legacy Druid streaming formats is removed. -Firehose ingestion was deprecated in version 0.17, and support for this ingestion was removed in version 24.0.0. - -[#12852](https://github.com/apache/druid/pull/12852) - -#### Information schema now uses numeric column types - -The Druid system table (`INFORMATION_SCHEMA`) now uses SQL types instead of Druid types for columns. This change makes the `INFORMATION_SCHEMA` table behave more like standard SQL. You may need to update your queries in the following scenarios in order to avoid unexpected results if you depend either of the following: - -* Numeric fields being treated as strings. -* Column numbering starting at 0. Column numbering is now 1-based. - -[#13777](https://github.com/apache/druid/pull/13777) - -#### `frontCoded` segment format change - -The `frontCoded` type of `stringEncodingStrategy` on `indexSpec` with a new segment format version, which typically has faster read speeds and reduced segment size. This improvement is backwards incompatible with Druid 25.0.0. - -## 25.0.0 - -### Upgrade notes - -#### Default HTTP-based segment discovery and task management - -The default segment discovery method now uses HTTP instead of ZooKeeper. - -This update changes the defaults for the following properties: - -|Property|New default|Previous default| -|--------|-----------|----------------| -|`druid.serverview.type` for segment management|http|batch| -|`druid.coordinator.loadqueuepeon.type` for segment management|http| curator| -|`druid.indexer.runner.type` for the Overlord|httpRemote|local| - -To use ZooKeeper instead of HTTP, change the values for the properties back to the previous defaults. ZooKeeper-based implementations for these properties are deprecated and will be removed in a subsequent release. - -[#13092](https://github.com/apache/druid/pull/13092) - -#### Finalizing HLL and quantiles sketch aggregates - -The aggregation functions for HLL and quantiles sketches returned sketches or numbers when they are finalized depending on where they were in the native query plan. - -Druid no longer finalizes aggregators in the following two cases: - -* aggregators appear in the outer level of a query -* aggregators are used as input to an expression or finalizing-field-access post-aggregator - -This change aligns the behavior of HLL and quantiles sketches with theta sketches. - -To restore old behavior, you can set `sqlFinalizeOuterSketches=true` in the query context. - -[#13247](https://github.com/apache/druid/pull/13247) - -#### Kill tasks mark segments as unused only if specified - -When you issue a kill task, Druid marks the underlying segments as unused only if explicitly specified. For more information, see the [API reference](https://druid.apache.org/docs/latest/api-reference/data-management-api). - -[#13104](https://github.com/apache/druid/pull/13104) - -### Incompatible changes - -#### Upgrade curator to 5.3.0 - -Apache Curator upgraded to the latest version, 5.3.0. This version drops support for ZooKeeper 3.4 but Druid has already officially dropped support in 0.22. In 5.3.0, Curator has removed support for Exhibitor so all related configurations and tests have been removed. - -[#12939](https://github.com/apache/druid/pull/12939) - -#### Fixed Parquet list conversion - -The behavior of the parquet reader for lists of structured objects has been changed to be consistent with other parquet logical list conversions. The data is now fetched directly, more closely matching its expected structure. - -[#13294](https://github.com/apache/druid/pull/13294) - -## 24.0.0 - -### Upgrade notes - -#### Permissions for multi-stage query engine - -To read external data using the multi-stage query task engine, you must have READ permissions for the EXTERNAL resource type. Users without the correct permission encounter a 403 error when trying to run SQL queries that include EXTERN. - -The way you assign the permission depends on your authorizer. For example, with [basic security](https://github.com/apache/druid/blob/druid-24.0.0/docs/operations/security-user-auth.md) in Druid, add the EXTERNAL READ permission by sending a POST request to the [roles API](https://github.com/apache/druid/blob/druid-24.0.0/docs/development/extensions-core/druid-basic-security.md#permissions). - -The example adds permissions for users with the admin role using a basic authorizer named MyBasicMetadataAuthorizer. The following permissions are granted: - -* DATASOURCE READ -* DATASOURCE WRITE -* CONFIG READ -* CONFIG WRITE -* STATE READ -* STATE WRITE -* EXTERNAL READ - -``` -curl --location --request POST 'http://localhost:8081/druid-ext/basic-security/authorization/db/MyBasicMetadataAuthorizer/roles/admin/permissions' \ ---header 'Content-Type: application/json' \ ---data-raw '[ -{ - "resource": { - "name": ".*", - "type": "DATASOURCE" - }, - "action": "READ" -}, -{ - "resource": { - "name": ".*", - "type": "DATASOURCE" - }, - "action": "WRITE" -}, -{ - "resource": { - "name": ".*", - "type": "CONFIG" - }, - "action": "READ" -}, -{ - "resource": { - "name": ".*", - "type": "CONFIG" - }, - "action": "WRITE" -}, -{ - "resource": { - "name": ".*", - "type": "STATE" - }, - "action": "READ" -}, -{ - "resource": { - "name": ".*", - "type": "STATE" - }, - "action": "WRITE" -}, -{ - "resource": { - "name": "EXTERNAL", - "type": "EXTERNAL" - }, - "action": "READ" -} -]' -``` - -#### Behavior for unused segments - -Druid automatically retains any segments marked as unused. Previously, Druid permanently deleted unused segments from metadata store and deep storage after their duration to retain passed. This behavior was reverted from 0.23.0. - -[#12693](https://github.com/apache/druid/pull/12693) - -#### Default for `druid.processing.fifo` - -The default for `druid.processing.fifo` is now true. This means that tasks of equal priority are treated in a FIFO manner. For most use cases, this change can improve performance on heavily loaded clusters. - -[#12571](https://github.com/apache/druid/pull/12571) - -#### Update to JDBC statement closure - -In previous releases, Druid automatically closed the JDBC Statement when the ResultSet was closed. Druid closed the ResultSet on EOF. Druid closed the statement on any exception. This behavior is, however, non-standard. -In this release, Druid's JDBC driver follows the JDBC standards more closely: -The ResultSet closes automatically on EOF, but does not close the Statement or PreparedStatement. Your code must close these statements, perhaps by using a try-with-resources block. -The PreparedStatement can now be used multiple times with different parameters. (Previously this was not true since closing the ResultSet closed the PreparedStatement.) -If any call to a Statement or PreparedStatement raises an error, the client code must still explicitly close the statement. According to the JDBC standards, statements are not closed automatically on errors. This allows you to obtain information about a failed statement before closing it. -If you have code that depended on the old behavior, you may have to change your code to add the required close statement. - -[#12709](https://github.com/apache/druid/pull/12709) - -## 0.23.0 - -### Upgrade notes - -#### Auto-killing of segments - -In 0.23.0, Auto killing of segments is now enabled by default [(#12187)](https://github.com/apache/druid/pull/12187). The new defaults should kill all unused segments older than 90 days. If users do not want this behavior on an upgrade, they should explicitly disable the behavior. This is a risky change since depending on the interval, segments will be killed immediately after being marked unused. this behavior will be reverted or changed in the next druid release. Please see [(#12693)](https://github.com/apache/druid/pull/12693) for more details. - -#### Other changes - -# Other changes - -- Kinesis ingestion requires `listShards` API access on the stream. -- Kafka clients libraries have been upgraded to 3.0.0 [(#11735)](https://github.com/apache/druid/pull/11735) -- The dynamic coordinator config, `percentOfSegmentsToConsiderPerMove` has been deprecated and will be removed in a future release of Druid. It is being replaced by a new segment picking strategy introduced in [(#11257)](https://github.com/apache/druid/pull/11257). This new strategy is currently toggled off by default, but can be toggled on if you set the dynamic coordinator config `useBatchedSegmentSampler` to true. Setting this as such, will disable the use of the deprecated `percentOfSegmentsToConsiderPerMove`. In a future release, `useBatchedSegmentSampler` will become permanently true. [(#11960)](https://github.com/apache/druid/pull/11960) - -## 0.22.0 - -### Upgrade notes - -#### Dropped support for Apache ZooKeeper 3.4 - -Following up to 0.21, which officially deprecated support for ZooKeeper 3.4, [which has been end-of-life for a while](https://lists.apache.org/thread/xckr6nnsg9rxchkbvltkvt7hr2d0mhbo), support for ZooKeeper 3.4 is now removed in 0.22.0. Be sure to upgrade your ZooKeeper cluster prior to upgrading your Druid cluster to 0.22.0. - -[#10780](https://github.com/apache/druid/issues/10780) -[#11073](https://github.com/apache/druid/pull/11073) - -#### Native batch ingestion segment allocation fix - -Druid 0.22.0 includes an important bug-fix in native batch indexing where transient failures of indexing sub-tasks can result in non-contiguous partitions in the result segments, which will never become queryable due to logic which checks for the 'complete' set. This issue has been resolved in the latest version of Druid, but required a change in the protocol which batch tasks use to allocate segments, and this change can cause issues during rolling downgrades if you decide to roll back from Druid 0.22.0 to an earlier version. - -To avoid task failure during a rolling-downgrade, set - -``` -druid.indexer.task.default.context={ "useLineageBasedSegmentAllocation" : false } -``` - -in the overlord runtime properties, and wait for all tasks which have `useLineageBasedSegmentAllocation` set to true to complete before initiating the downgrade. After these tasks have all completed the downgrade shouldn't have any further issue and the setting can be removed from the overlord configuration (recommended, as you will want this setting enabled if you are running Druid 0.22.0 or newer). - -[#11189](https://github.com/apache/druid/pull/11189) - -#### SQL timeseries no longer skip empty buckets with all granularity - -Prior to Druid 0.22, an SQL group by query which is using a single universal grouping key (e.g. only aggregators) such as `SELECT COUNT(*), SUM(x) FROM y WHERE z = 'someval'` would produce an empty result set instead of `[0, null]` that might be expected from this query matching no results. This was because underneath this would plan into a timeseries query with 'ALL' granularity, and skipEmptyBuckets set to true in the query context. This latter option caused the results of such a query to return no results, as there are no buckets with values to aggregate and so they are skipped, making an empty result set instead of a 'nil' result set. This behavior has been changed to behave in line with other SQL implementations, but the previous behavior can be obtained by explicitly setting `skipEmptyBuckets` on the query context. - -[#11188](https://github.com/apache/druid/pull/11188) - -#### Druid reingestion incompatible changes - -Batch tasks using a 'Druid' input source to reingest segment data will no longer accept the 'dimensions' and 'metrics' sections of their task spec, and now will internally use a new columns filter to specify which columns from the original segment should be retained. Additionally, timestampSpec is no longer ignored, allowing the __time column to be modified or replaced with a different column. These changes additionally fix a bug where transformed columns would be ignored and unavailable on the new segments. - -[#10267](https://github.com/apache/druid/pull/10267) - -#### Druid web-console no longer supports IE11 and other older browsers - -Some things might still work, but it is no longer officially supported so that newer Javascript features can be used to develop the web-console. - -[#11357](https://github.com/apache/druid/pull/11357) - -#### Changed default maximum segment loading queue size - -Druid coordinator `maxSegmentsInNodeLoadingQueue` dynamic configuration has been changed from unlimited (`0`) to `100`. This should make the coordinator behave in a much more relaxed manner during periods of cluster volatility, such as a rolling upgrade, but caps the total number of segments that will be loaded in any given coordinator cycle to 100 per server, which can slow down the speed at which a completely stopped cluster is started and loaded from deep storage. - -[#11540](https://github.com/apache/druid/pull/11540) - -## 0.21.0 - -#### Improved HTTP status codes for query errors - -Before this release, Druid returned the "internal error (500)" for most of the query errors. Now Druid returns different error codes based on their cause. The following table lists the errors and their corresponding codes that has changed: - -| Exception | Description| Old code | New code | -|-----|-|--|-----| -| SqlParseException and ValidationException from Calcite | Query planning failed | 500 | 400 | -| QueryTimeoutException | Query execution didn't finish in timeout | 500 | 504 | -| ResourceLimitExceededException | Query asked more resources than configured threshold | 500 | 400 | -| InsufficientResourceException | Query failed to schedule because of lack of merge buffers available at the time when it was submitted | 500 | 429, merged to QueryCapacityExceededException | -| QueryUnsupportedException | Unsupported functionality | 400 | 501 | - -[#10464](https://github.com/apache/druid/pull/10464) -[#10746](https://github.com/apache/druid/pull/10746) - -#### Query interrupted metric -`query/interrupted/count` no longer counts the queries that timed out. These queries are counted by `query/timeout/count`. - -#### context dimension in query metrics - -`context` is now a default dimension emitted for all query metrics. `context` is a JSON-formatted string containing the query context for the query that the emitted metric refers to. The addition of a dimension that was not previously alters some metrics emitted by Druid. You should plan to handle this new `context` dimension in your metrics pipeline. Since the dimension is a JSON-formatted string, a common solution is to parse the dimension and either flatten it or extract the bits you want and discard the full JSON-formatted string blob. - -[#10578](https://github.com/apache/druid/pull/10578) - -#### Deprecated support for Apache ZooKeeper 3.4 - -As [ZooKeeper 3.4 has been end-of-life for a while](https://mail-archives.apache.org/mod_mbox/zookeeper-user/202004.mbox/%3C41A7EC67-D8F4-4C3A-B2DB-C2741C2EECA3%40apache.org%3E), support for ZooKeeper 3.4 is deprecated in 0.21.0 and will be removed in the near future. - -[#10780](https://github.com/apache/druid/issues/10780) - -#### Consistent serialization format and column naming convention for the sys.segments table - -All columns in the `sys.segments` table are now serialized in the JSON format to make them consistent with other system tables. Column names now use the same "snake case" convention. - -[#10481](https://github.com/apache/druid/pull/10481) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java new file mode 100644 index 000000000000..0895da230865 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java @@ -0,0 +1,96 @@ +package org.apache.druid.frame.read.columnar; + +import org.apache.datasketches.memory.Memory; +import org.apache.druid.error.DruidException; +import org.apache.druid.frame.Frame; +import org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.accessor.ObjectColumnAccessorBase; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.column.BaseColumn; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.data.ReadableOffset; + +import java.io.IOException; +import java.util.Comparator; + +public class LongArrayFrameColumnReader implements FrameColumnReader +{ + private final byte typeCode; + private final int columnNumber; + + public LongArrayFrameColumnReader(byte typeCode, int columnNumber) + { + this.columnNumber = columnNumber; + this.typeCode = typeCode; + } + + @Override + public Column readRACColumn(Frame frame) + { + final Memory memory = frame.region(columnNumber); + validate(memory); + throw DruidException.defensive("Multivalue not yet handled by RAC"); + } + + @Override + public ColumnPlus readColumn(Frame frame) + { + final Memory memory = frame.region(columnNumber); + validate(memory); + } + + private void validate(final Memory region) { + if (region.getCapacity() < LongArrayFrameColumnWriter.DATA_OFFSET) { + throw DruidException.defensive("Column[%s] is not big enough for a header", columnNumber); + } + final byte typeCode = region.getByte(0); + if (typeCode != this.typeCode) { + throw DruidException.defensive( + "Column[%s] does not have the correct type code; expected[%s], got[%s]", + columnNumber, + this.typeCode, + typeCode + ); + } + } + + private static class LongArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn + { + @Override + public void close() throws IOException + { + + } + + @Override + public ColumnType getType() + { + return null; + } + + @Override + public int numRows() + { + return 0; + } + + @Override + protected Object getVal(int rowNum) + { + return null; + } + + @Override + protected Comparator getComparator() + { + return null; + } + + @Override + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + { + return null; + } + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java index fb000871ac4d..ce2412a1cd00 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java @@ -1,6 +1,7 @@ package org.apache.druid.frame.write.columnar; import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.AppendableMemory; import org.apache.druid.frame.allocation.MemoryAllocator; import org.apache.druid.frame.allocation.MemoryRange; @@ -13,7 +14,7 @@ public class LongArrayFrameColumnWriter implements FrameColumnWriter { // TODO(laksh): Copy pasted after following the logic from StringFrameColumnWriter, since numeric writers have 3 // regions as well. The type size is fixed, however there's no special nullable marker - private static final int INITIAL_ALLOCATION_SIZE = 180; + private static final int INITIAL_ALLOCATION_SIZE = 120; private static final int ELEMENT_SIZE = Long.BYTES; @@ -29,8 +30,6 @@ public class LongArrayFrameColumnWriter implements FrameColumnWriter final MemoryAllocator allocator; final byte typeCode; - - /** * Row lengths: one int per row with the number of values contained by that row and all previous rows. * Only written for multi-value and array columns. When the corresponding row is null itself, the length is @@ -49,7 +48,7 @@ public class LongArrayFrameColumnWriter implements FrameColumnWriter private final AppendableMemory rowData; private int lastCumulativeRowLength = 0; - private int lastRowLength = 0; + private int lastRowLength = -1; public LongArrayFrameColumnWriter( @@ -92,6 +91,8 @@ public boolean addSelection() if (numericArray == null) { rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), -(lastCumulativeRowLength + rowLength) - 1); + } else { + rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), lastCumulativeRowLength + rowLength); } cumulativeRowLengths.advanceCursor(Integer.BYTES); lastRowLength = rowLength; @@ -99,6 +100,7 @@ public boolean addSelection() final MemoryRange rowNullityDataCursor = rowLength > 0 ? rowData.cursor() : null; final MemoryRange rowDataCursor = rowLength > 0 ? rowData.cursor() : null; + for (int i = 0; i < rowLength; ++i) { final Number element = numericArray.get(i); if (element == null) { @@ -111,29 +113,56 @@ public boolean addSelection() } } + if (rowLength > 0) { + rowNullityData.advanceCursor(Byte.BYTES * rowLength); + rowData.advanceCursor(ELEMENT_SIZE * rowLength); + } + + return true; } @Override public void undo() { + if (lastRowLength == -1) { + throw DruidException.defensive("Nothing written to undo()"); + } + cumulativeRowLengths.rewindCursor(Integer.BYTES); + rowNullityData.rewindCursor(lastRowLength * Byte.BYTES); + rowData.rewindCursor(lastRowLength * ELEMENT_SIZE); + + lastCumulativeRowLength -= lastRowLength; + // Multiple undo calls cannot be chained together + lastRowLength = -1; } @Override public long size() { - return 0; + return DATA_OFFSET + cumulativeRowLengths.size() + rowNullityData.size() + rowData.size(); } @Override - public long writeTo(WritableMemory memory, long position) + public long writeTo(final WritableMemory memory, final long startPosition) { - return 0; + long currentPosition = startPosition; + + memory.putByte(currentPosition, typeCode); + ++currentPosition; + + currentPosition += cumulativeRowLengths.writeTo(memory, currentPosition); + currentPosition += rowNullityData.writeTo(memory, currentPosition); + currentPosition += rowData.writeTo(memory, currentPosition); + + return currentPosition - startPosition; } @Override public void close() { - + cumulativeRowLengths.close(); + rowNullityData.close(); + rowData.close(); } } From c1af54d1005635a75d9e7584143f5efa934f636b Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Thu, 15 Feb 2024 16:43:08 +0530 Subject: [PATCH 03/11] revert releasenotes, upgradenotes --- docs/release-info/release-notes.md | 106 +++++ docs/release-info/upgrade-notes.md | 634 +++++++++++++++++++++++++++++ 2 files changed, 740 insertions(+) create mode 100644 docs/release-info/release-notes.md create mode 100644 docs/release-info/upgrade-notes.md diff --git a/docs/release-info/release-notes.md b/docs/release-info/release-notes.md new file mode 100644 index 000000000000..768ceef697bc --- /dev/null +++ b/docs/release-info/release-notes.md @@ -0,0 +1,106 @@ +--- +id: release-notes +title: "Release notes" +--- + + + + + +Apache Druid {{DRUIDVERSION}} contains over $NUMBER_FEATURES new features, bug fixes, performance enhancements, documentation improvements, and additional test coverage from $NUMBER_OF_CONTRIBUTORS contributors. + + + +See the [complete set of changes](https://github.com/apache/druid/issues?q=is%3Aclosed+milestone%3A{{MILESTONE}}+sort%3Aupdated-desc+) for additional details, including bug fixes. + +Review the [upgrade notes](#upgrade-notes) and [incompatible changes](#incompatible-changes) before you upgrade to Druid {{DRUIDVERSION}}. +If you are upgrading across multiple versions, see the [Upgrade notes](upgrade-notes.md) page, which lists upgrade notes for the most recent Druid versions. + + + +## Important features, changes, and deprecations + +This section contains important information about new and existing features. + +## Functional area and related changes + +This section contains detailed release notes separated by areas. + +### Web console + +#### Other web console improvements + +### Ingestion + +#### SQL-based ingestion + +##### Other SQL-based ingestion improvements + +#### Streaming ingestion + +##### Other streaming ingestion improvements + +### Querying + +#### Other querying improvements + +### Cluster management + +#### Other cluster management improvements + +### Data management + +#### Other data management improvements + +### Metrics and monitoring + +### Extensions + +### Documentation improvements + +## Upgrade notes and incompatible changes + +### Upgrade notes + +### Incompatible changes + +### Developer notes + +#### Dependency updates + +The following dependencies have had their versions bumped: \ No newline at end of file diff --git a/docs/release-info/upgrade-notes.md b/docs/release-info/upgrade-notes.md new file mode 100644 index 000000000000..46e5ed6fc1a1 --- /dev/null +++ b/docs/release-info/upgrade-notes.md @@ -0,0 +1,634 @@ +--- +id: upgrade-notes +title: "Upgrade notes" +--- + + + +The upgrade notes assume that you are upgrading from the Druid version that immediately precedes your target version. If you are upgrading across multiple versions, make sure you read the upgrade notes for all the intermediate versions. + +For the full release notes for a specific version, see the [releases page](https://github.com/apache/druid/releases). + +## 28.0.0 + +### Upgrade notes + +#### Upgrade Druid segments table + +Druid 28.0.0 adds a new column to the Druid metadata table that requires an update to the table. + +If `druid.metadata.storage.connector.createTables` is set to `true` and the metadata store user has DDL privileges, the segments table gets automatically updated at startup to include the new `used_status_last_updated` column. No additional work is needed for the upgrade. + +If either of those requirements are not met, pre-upgrade steps are required. You must make these updates before you upgrade to Druid 28.0.0, or the Coordinator and Overlord processes fail. + +Although you can manually alter your table to add the new `used_status_last_updated` column, Druid also provides a [CLI tool](https://druid.apache.org/docs/latest/operations/metadata-migration/#create-druid-tables) to do it. + +[#12599](https://github.com/apache/druid/pull/12599) [#14868](https://github.com/apache/druid/pull/14868) + +In the example commands below: + +- `lib` is the Druid lib directory +- `extensions` is the Druid extensions directory +- `base` corresponds to the value of `druid.metadata.storage.tables.base` in the configuration, `druid` by default. +- The `--connectURI` parameter corresponds to the value of `druid.metadata.storage.connector.connectURI`. +- The `--user` parameter corresponds to the value of `druid.metadata.storage.connector.user`. +- The `--password` parameter corresponds to the value of `druid.metadata.storage.connector.password`. +- The `--action` parameter corresponds to the update action you are executing. In this case, it is `add-last-used-to-segments` + +##### Upgrade step for MySQL + +```bash +cd ${DRUID_ROOT} +java -classpath "lib/*" -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory="extensions" -Ddruid.extensions.loadList=[\"mysql-metadata-storage\"] -Ddruid.metadata.storage.type=mysql org.apache.druid.cli.Main tools metadata-update --connectURI="" --user USER --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments +``` + +##### Upgrade step for PostgreSQL + +```bash +cd ${DRUID_ROOT} +java -classpath "lib/*" -Dlog4j.configurationFile=conf/druid/cluster/_common/log4j2.xml -Ddruid.extensions.directory="extensions" -Ddruid.extensions.loadList=[\"postgresql-metadata-storage\"] -Ddruid.metadata.storage.type=postgresql org.apache.druid.cli.Main tools metadata-update --connectURI="" --user USER --password PASSWORD --base druid --action add-used-flag-last-updated-to-segments +``` + +##### Manual upgrade step + +```SQL +ALTER TABLE druid_segments +ADD used_status_last_updated varchar(255); +``` + +#### Recommended syntax for SQL UNNEST + +The recommended syntax for SQL UNNEST has changed. We recommend using CROSS JOIN instead of commas for most queries to prevent issues with precedence. For example, use: + +```sql +SELECT column_alias_name1 FROM datasource CROSS JOIN UNNEST(source_expression1) AS table_alias_name1(column_alias_name1) CROSS JOIN UNNEST(source_expression2) AS table_alias_name2(column_alias_name2), ... +``` + +Do not use: + +```sql +SELECT column_alias_name FROM datasource, UNNEST(source_expression1) AS table_alias_name1(column_alias_name1), UNNEST(source_expression2) AS table_alias_name2(column_alias_name2), ... +``` + +#### Dynamic parameters + +The Apache Calcite version has been upgraded from 1.21 to 1.35. As part of the Calcite upgrade, the behavior of type inference for dynamic parameters has changed. To avoid any type interference issues, explicitly `CAST` all dynamic parameters as a specific data type in SQL queries. For example, use: + +```sql +SELECT (1 * CAST (? as DOUBLE))/2 as tmp +``` + +Do not use: + +```sql +SELECT (1 * ?)/2 as tmp +``` + +#### Nested column format + +`json` type columns created with Druid 28.0.0 are not backwards compatible with Druid versions older than 26.0.0. +If you are upgrading from a version prior to Druid 26.0.0 and you use `json` columns, upgrade to Druid 26.0.0 before you upgrade to Druid 28.0.0. +Additionally, to downgrade to a version older than Druid 26.0.0, any new segments created in Druid 28.0.0 should be re-ingested using Druid 26.0.0 or 27.0.0 prior to further downgrading. + +When upgrading from a previous version, you can continue to write nested columns in a backwards compatible format (version 4). + +In a classic batch ingestion job, include `formatVersion` in the `dimensions` list of the `dimensionsSpec` property. For example: + +```json + "dimensionsSpec": { + "dimensions": [ + "product", + "department", + { + "type": "json", + "name": "shipTo", + "formatVersion": 4 + } + ] + }, +``` + +To set the default nested column version, set the desired format version in the common runtime properties. For example: + +```java +druid.indexing.formats.nestedColumnFormatVersion=4 +``` + +#### SQL compatibility + +Starting with Druid 28.0.0, the default way Druid treats nulls and booleans has changed. + +For nulls, Druid now differentiates between an empty string and a record with no data as well as between an empty numerical record and `0`. +You can revert to the previous behavior by setting `druid.generic.useDefaultValueForNull` to `true`. + +This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion. + +For booleans, Druid now strictly uses `1` (true) or `0` (false). Previously, true and false could be represented either as `true` and `false` as well as `1` and `0`, respectively. In addition, Druid now returns a null value for boolean comparisons like `True && NULL`. + +You can revert to the previous behavior by setting `druid.expressions.useStrictBooleans` to `false`. +This property affects both storage and querying, and must be set on all Druid service types to be available at both ingestion time and query time. Reverting this setting to the old value restores the previous behavior without reingestion. + +The following table illustrates some example scenarios and the impact of the changes. + +
Show the table + +| Query| Druid 27.0.0 and earlier| Druid 28.0.0 and later| +|------|------------------------|----------------------| +| Query empty string| Empty string (`''`) or null| Empty string (`''`)| +| Query null string| Null or empty| Null| +| COUNT(*)| All rows, including nulls| All rows, including nulls| +| COUNT(column)| All rows excluding empty strings| All rows including empty strings but excluding nulls| +| Expression 100 && 11| 11| 1| +| Expression 100 || 11| 100| 1| +| Null FLOAT/DOUBLE column| 0.0| Null| +| Null LONG column| 0| Null| +| Null `__time` column| 0, meaning 1970-01-01 00:00:00 UTC| 1970-01-01 00:00:00 UTC| +| Null MVD column| `''`| Null| +| ARRAY| Null| Null| +| COMPLEX| none| Null| +
+ +Before upgrading to Druid 28.0.0, update your queries to account for the changed behavior as described in the following sections. + +##### NULL filters + +If your queries use NULL in the filter condition to match both nulls and empty strings, you should add an explicit filter clause for empty strings. For example, update `s IS NULL` to `s IS NULL OR s = ''`. + +##### COUNT functions + +`COUNT(column)` now counts empty strings. If you want to continue excluding empty strings from the count, replace `COUNT(column)` with `COUNT(column) FILTER(WHERE column <> '')`. + +##### GroupBy queries + +GroupBy queries on columns containing null values can now have additional entries as nulls can co-exist with empty strings. + +#### Stop Supervisors that ingest from multiple Kafka topics before downgrading + +If you have added supervisors that ingest from multiple Kafka topics in Druid 28.0.0 or later, stop those supervisors before downgrading to a version prior to Druid 28.0.0 because the supervisors will fail in versions prior to Druid 28.0.0. + +#### `lenientAggregatorMerge` deprecated + +`lenientAggregatorMerge` property in segment metadata queries has been deprecated. It will be removed in future releases. +Use `aggregatorMergeStrategy` instead. `aggregatorMergeStrategy` also supports the `latest` and `earliest` strategies in addition to `strict` and `lenient` strategies from `lenientAggregatorMerge`. + +[#14560](https://github.com/apache/druid/pull/14560) +[#14598](https://github.com/apache/druid/pull/14598) + +#### Broker parallel merge config options + +The paths for `druid.processing.merge.pool.*` and `druid.processing.merge.task.*` have been flattened to use `druid.processing.merge.*` instead. The legacy paths for the configs are now deprecated and will be removed in a future release. Migrate your settings to use the new paths because the old paths will be ignored in the future. + +[#14695](https://github.com/apache/druid/pull/14695) + +#### Ingestion options for ARRAY typed columns + +Starting with Druid 28.0.0, the MSQ task engine can detect and ingest arrays as ARRAY typed columns when you set the query context parameter `arrayIngestMode` to `array`. +The `arrayIngestMode` context parameter controls how ARRAY type values are stored in Druid segments. + +When you set `arrayIngestMode` to `array` (recommended for SQL compliance), the MSQ task engine stores all ARRAY typed values in [ARRAY typed columns](https://druid.apache.org/docs/latest/querying/arrays) and supports storing both VARCHAR and numeric typed arrays. + +For backwards compatibility, `arrayIngestMode` defaults to `mvd`. When `"arrayIngestMode":"mvd"`, Druid only supports VARCHAR typed arrays and stores them as [multi-value string columns](https://druid.apache.org/docs/latest/querying/multi-value-dimensions). + +When you set `arrayIngestMode` to `none`, Druid throws an exception when trying to store any type of arrays. + +For more information on how to ingest `ARRAY` typed columns with SQL-based ingestion, see [SQL data types](https://druid.apache.org/docs/latest/querying/sql-data-types#arrays) and [Array columns](https://druid.apache.org/docs/latest/querying/arrays). + +### Incompatible changes + +#### Removed Hadoop 2 + +Support for Hadoop 2 has been removed. +Migrate to SQL-based ingestion or JSON-based batch ingestion if you are using Hadoop 2.x for ingestion today. +If migrating to Druid's built-in ingestion is not possible, you must upgrade your Hadoop infrastructure to 3.x+ before upgrading to Druid 28.0.0. + +[#14763](https://github.com/apache/druid/pull/14763) + +#### Removed GroupBy v1 + +The GroupBy v1 engine has been removed. Use the GroupBy v2 engine instead, which has been the default GroupBy engine for several releases. +There should be no impact on your queries. + +Additionally, `AggregatorFactory.getRequiredColumns` has been deprecated and will be removed in a future release. If you have an extension that implements `AggregatorFactory`, then this method should be removed from your implementation. + +[#14866](https://github.com/apache/druid/pull/14866) + +#### Removed Coordinator dynamic configs + +The `decommissioningMaxPercentOfMaxSegmentsToMove` config has been removed. +The use case for this config is handled by smart segment loading now, which is enabled by default. + +[#14923](https://github.com/apache/druid/pull/14923) + +#### Removed `cachingCost` strategy + +The `cachingCost` strategy for segment loading has been removed. +Use `cost` instead, which has the same benefits as `cachingCost`. + +If you have `cachingCost` set, the system ignores this setting and automatically uses `cost`. + +[#14798](https://github.com/apache/druid/pull/14798) + +#### Removed `InsertCannotOrderByDescending` + +The deprecated MSQ fault `InsertCannotOrderByDescending` has been removed. + +[#14588](https://github.com/apache/druid/pull/14588) + +#### Removed the backward compatibility code for the Handoff API + +The backward compatibility code for the Handoff API in `CoordinatorBasedSegmentHandoffNotifier` has been removed. +If you are upgrading from a Druid version older than 0.14.0, upgrade to a newer version of Druid before upgrading to Druid 28.0.0. + +[#14652](https://github.com/apache/druid/pull/14652) + +## 27.0.0 + +### Upgrade notes + +#### Worker input bytes for SQL-based ingestion + +The maximum input bytes for each worker for SQL-based ingestion is now 512 MiB (previously 10 GiB). + +[#14307](https://github.com/apache/druid/pull/14307) + +#### Parameter execution changes for Kafka + +When using the built-in `FileConfigProvider` for Kafka, interpolations are now intercepted by the JsonConfigurator instead of being passed down to the Kafka provider. This breaks existing deployments. + +For more information, see [KIP-297](https://cwiki.apache.org/confluence/display/KAFKA/KIP-297%3A+Externalizing+Secrets+for+Connect+Configurations). + +[#13023](https://github.com/apache/druid/pull/13023) + +#### Hadoop 2 deprecated + +Many of the important dependent libraries that Druid uses no longer support Hadoop 2. In order for Druid to stay current and have pathways to mitigate security vulnerabilities, the community has decided to deprecate support for Hadoop 2.x releases starting this release. Starting with Druid 28.x, Hadoop 3.x is the only supported Hadoop version. + +Consider migrating to SQL-based ingestion or native ingestion if you are using Hadoop 2.x for ingestion today. If migrating to Druid ingestion is not possible, plan to upgrade your Hadoop infrastructure before upgrading to the next Druid release. + +#### GroupBy v1 deprecated + +GroupBy queries using the v1 legacy engine has been deprecated. It will be removed in future releases. Use v2 instead. Note that v2 has been the default GroupBy engine. + +For more information, see [GroupBy queries](https://druid.apache.org/docs/latest/querying/groupbyquery.html). + +#### Push-based real-time ingestion deprecated + +Support for push-based real-time ingestion has been deprecated. It will be removed in future releases. + +#### `cachingCost` segment balancing strategy deprecated + +The `cachingCost` strategy has been deprecated and will be removed in future releases. Use an alternate segment balancing strategy instead, such as `cost`. + +#### Segment loading config changes + +The following segment related configs are now deprecated and will be removed in future releases: + +* `maxSegmentsInNodeLoadingQueue` +* `maxSegmentsToMove` +* `replicationThrottleLimit` +* `useRoundRobinSegmentAssignment` +* `replicantLifetime` +* `maxNonPrimaryReplicantsToLoad` +* `decommissioningMaxPercentOfMaxSegmentsToMove` + +Use `smartSegmentLoading` mode instead, which calculates values for these variables automatically. + +Additionally, the defaults for the following Coordinator dynamic configs have changed: + +* `maxsegmentsInNodeLoadingQueue` : 500, previously 100 +* `maxSegmentsToMove`: 100, previously 5 +* `replicationThrottleLimit`: 500, previously 10 + +These new defaults can improve performance for most use cases. + +[#13197](https://github.com/apache/druid/pull/13197) +[#14269](https://github.com/apache/druid/pull/14269) + +#### `SysMonitor` support deprecated + +Switch to `OshiSysMonitor` as `SysMonitor` is now deprecated and will be removed in future releases. + +### Incompatible changes + +#### Removed property for setting max bytes for dimension lookup cache + +`druid.processing.columnCache.sizeBytes` has been removed since it provided limited utility after a number of internal changes. Leaving this config is harmless, but it does nothing. + +[#14500](https://github.com/apache/druid/pull/14500) + +#### Removed Coordinator dynamic configs + +The following Coordinator dynamic configs have been removed: + +* `emitBalancingStats`: Stats for errors encountered while balancing will always be emitted. Other debugging stats will not be emitted but can be logged by setting the appropriate `debugDimensions`. +* `useBatchedSegmentSampler` and `percentOfSegmentsToConsiderPerMove`: Batched segment sampling is now the standard and will always be on. + +Use the new [smart segment loading](https://druid.apache.org/docs/latest/configuration/#smart-segment-loading) mode instead. + +[#14524](https://github.com/apache/druid/pull/14524) + +## 26.0.0 + +### Upgrade notes + +#### Real-time tasks + +Optimized query performance by lowering the default maxRowsInMemory for real-time ingestion, which might lower overall ingestion throughput. + +[#13939](https://github.com/apache/druid/pull/13939) + +### Incompatible changes + +#### Firehose ingestion removed + +The firehose/parser specification used by legacy Druid streaming formats is removed. +Firehose ingestion was deprecated in version 0.17, and support for this ingestion was removed in version 24.0.0. + +[#12852](https://github.com/apache/druid/pull/12852) + +#### Information schema now uses numeric column types + +The Druid system table (`INFORMATION_SCHEMA`) now uses SQL types instead of Druid types for columns. This change makes the `INFORMATION_SCHEMA` table behave more like standard SQL. You may need to update your queries in the following scenarios in order to avoid unexpected results if you depend either of the following: + +* Numeric fields being treated as strings. +* Column numbering starting at 0. Column numbering is now 1-based. + +[#13777](https://github.com/apache/druid/pull/13777) + +#### `frontCoded` segment format change + +The `frontCoded` type of `stringEncodingStrategy` on `indexSpec` with a new segment format version, which typically has faster read speeds and reduced segment size. This improvement is backwards incompatible with Druid 25.0.0. + +## 25.0.0 + +### Upgrade notes + +#### Default HTTP-based segment discovery and task management + +The default segment discovery method now uses HTTP instead of ZooKeeper. + +This update changes the defaults for the following properties: + +|Property|New default|Previous default| +|--------|-----------|----------------| +|`druid.serverview.type` for segment management|http|batch| +|`druid.coordinator.loadqueuepeon.type` for segment management|http| curator| +|`druid.indexer.runner.type` for the Overlord|httpRemote|local| + +To use ZooKeeper instead of HTTP, change the values for the properties back to the previous defaults. ZooKeeper-based implementations for these properties are deprecated and will be removed in a subsequent release. + +[#13092](https://github.com/apache/druid/pull/13092) + +#### Finalizing HLL and quantiles sketch aggregates + +The aggregation functions for HLL and quantiles sketches returned sketches or numbers when they are finalized depending on where they were in the native query plan. + +Druid no longer finalizes aggregators in the following two cases: + +* aggregators appear in the outer level of a query +* aggregators are used as input to an expression or finalizing-field-access post-aggregator + +This change aligns the behavior of HLL and quantiles sketches with theta sketches. + +To restore old behavior, you can set `sqlFinalizeOuterSketches=true` in the query context. + +[#13247](https://github.com/apache/druid/pull/13247) + +#### Kill tasks mark segments as unused only if specified + +When you issue a kill task, Druid marks the underlying segments as unused only if explicitly specified. For more information, see the [API reference](https://druid.apache.org/docs/latest/api-reference/data-management-api). + +[#13104](https://github.com/apache/druid/pull/13104) + +### Incompatible changes + +#### Upgrade curator to 5.3.0 + +Apache Curator upgraded to the latest version, 5.3.0. This version drops support for ZooKeeper 3.4 but Druid has already officially dropped support in 0.22. In 5.3.0, Curator has removed support for Exhibitor so all related configurations and tests have been removed. + +[#12939](https://github.com/apache/druid/pull/12939) + +#### Fixed Parquet list conversion + +The behavior of the parquet reader for lists of structured objects has been changed to be consistent with other parquet logical list conversions. The data is now fetched directly, more closely matching its expected structure. + +[#13294](https://github.com/apache/druid/pull/13294) + +## 24.0.0 + +### Upgrade notes + +#### Permissions for multi-stage query engine + +To read external data using the multi-stage query task engine, you must have READ permissions for the EXTERNAL resource type. Users without the correct permission encounter a 403 error when trying to run SQL queries that include EXTERN. + +The way you assign the permission depends on your authorizer. For example, with [basic security](https://github.com/apache/druid/blob/druid-24.0.0/docs/operations/security-user-auth.md) in Druid, add the EXTERNAL READ permission by sending a POST request to the [roles API](https://github.com/apache/druid/blob/druid-24.0.0/docs/development/extensions-core/druid-basic-security.md#permissions). + +The example adds permissions for users with the admin role using a basic authorizer named MyBasicMetadataAuthorizer. The following permissions are granted: + +* DATASOURCE READ +* DATASOURCE WRITE +* CONFIG READ +* CONFIG WRITE +* STATE READ +* STATE WRITE +* EXTERNAL READ + +``` +curl --location --request POST 'http://localhost:8081/druid-ext/basic-security/authorization/db/MyBasicMetadataAuthorizer/roles/admin/permissions' \ +--header 'Content-Type: application/json' \ +--data-raw '[ +{ + "resource": { + "name": ".*", + "type": "DATASOURCE" + }, + "action": "READ" +}, +{ + "resource": { + "name": ".*", + "type": "DATASOURCE" + }, + "action": "WRITE" +}, +{ + "resource": { + "name": ".*", + "type": "CONFIG" + }, + "action": "READ" +}, +{ + "resource": { + "name": ".*", + "type": "CONFIG" + }, + "action": "WRITE" +}, +{ + "resource": { + "name": ".*", + "type": "STATE" + }, + "action": "READ" +}, +{ + "resource": { + "name": ".*", + "type": "STATE" + }, + "action": "WRITE" +}, +{ + "resource": { + "name": "EXTERNAL", + "type": "EXTERNAL" + }, + "action": "READ" +} +]' +``` + +#### Behavior for unused segments + +Druid automatically retains any segments marked as unused. Previously, Druid permanently deleted unused segments from metadata store and deep storage after their duration to retain passed. This behavior was reverted from 0.23.0. + +[#12693](https://github.com/apache/druid/pull/12693) + +#### Default for `druid.processing.fifo` + +The default for `druid.processing.fifo` is now true. This means that tasks of equal priority are treated in a FIFO manner. For most use cases, this change can improve performance on heavily loaded clusters. + +[#12571](https://github.com/apache/druid/pull/12571) + +#### Update to JDBC statement closure + +In previous releases, Druid automatically closed the JDBC Statement when the ResultSet was closed. Druid closed the ResultSet on EOF. Druid closed the statement on any exception. This behavior is, however, non-standard. +In this release, Druid's JDBC driver follows the JDBC standards more closely: +The ResultSet closes automatically on EOF, but does not close the Statement or PreparedStatement. Your code must close these statements, perhaps by using a try-with-resources block. +The PreparedStatement can now be used multiple times with different parameters. (Previously this was not true since closing the ResultSet closed the PreparedStatement.) +If any call to a Statement or PreparedStatement raises an error, the client code must still explicitly close the statement. According to the JDBC standards, statements are not closed automatically on errors. This allows you to obtain information about a failed statement before closing it. +If you have code that depended on the old behavior, you may have to change your code to add the required close statement. + +[#12709](https://github.com/apache/druid/pull/12709) + +## 0.23.0 + +### Upgrade notes + +#### Auto-killing of segments + +In 0.23.0, Auto killing of segments is now enabled by default [(#12187)](https://github.com/apache/druid/pull/12187). The new defaults should kill all unused segments older than 90 days. If users do not want this behavior on an upgrade, they should explicitly disable the behavior. This is a risky change since depending on the interval, segments will be killed immediately after being marked unused. this behavior will be reverted or changed in the next druid release. Please see [(#12693)](https://github.com/apache/druid/pull/12693) for more details. + +#### Other changes + +# Other changes + +- Kinesis ingestion requires `listShards` API access on the stream. +- Kafka clients libraries have been upgraded to 3.0.0 [(#11735)](https://github.com/apache/druid/pull/11735) +- The dynamic coordinator config, `percentOfSegmentsToConsiderPerMove` has been deprecated and will be removed in a future release of Druid. It is being replaced by a new segment picking strategy introduced in [(#11257)](https://github.com/apache/druid/pull/11257). This new strategy is currently toggled off by default, but can be toggled on if you set the dynamic coordinator config `useBatchedSegmentSampler` to true. Setting this as such, will disable the use of the deprecated `percentOfSegmentsToConsiderPerMove`. In a future release, `useBatchedSegmentSampler` will become permanently true. [(#11960)](https://github.com/apache/druid/pull/11960) + +## 0.22.0 + +### Upgrade notes + +#### Dropped support for Apache ZooKeeper 3.4 + +Following up to 0.21, which officially deprecated support for ZooKeeper 3.4, [which has been end-of-life for a while](https://lists.apache.org/thread/xckr6nnsg9rxchkbvltkvt7hr2d0mhbo), support for ZooKeeper 3.4 is now removed in 0.22.0. Be sure to upgrade your ZooKeeper cluster prior to upgrading your Druid cluster to 0.22.0. + +[#10780](https://github.com/apache/druid/issues/10780) +[#11073](https://github.com/apache/druid/pull/11073) + +#### Native batch ingestion segment allocation fix + +Druid 0.22.0 includes an important bug-fix in native batch indexing where transient failures of indexing sub-tasks can result in non-contiguous partitions in the result segments, which will never become queryable due to logic which checks for the 'complete' set. This issue has been resolved in the latest version of Druid, but required a change in the protocol which batch tasks use to allocate segments, and this change can cause issues during rolling downgrades if you decide to roll back from Druid 0.22.0 to an earlier version. + +To avoid task failure during a rolling-downgrade, set + +``` +druid.indexer.task.default.context={ "useLineageBasedSegmentAllocation" : false } +``` + +in the overlord runtime properties, and wait for all tasks which have `useLineageBasedSegmentAllocation` set to true to complete before initiating the downgrade. After these tasks have all completed the downgrade shouldn't have any further issue and the setting can be removed from the overlord configuration (recommended, as you will want this setting enabled if you are running Druid 0.22.0 or newer). + +[#11189](https://github.com/apache/druid/pull/11189) + +#### SQL timeseries no longer skip empty buckets with all granularity + +Prior to Druid 0.22, an SQL group by query which is using a single universal grouping key (e.g. only aggregators) such as `SELECT COUNT(*), SUM(x) FROM y WHERE z = 'someval'` would produce an empty result set instead of `[0, null]` that might be expected from this query matching no results. This was because underneath this would plan into a timeseries query with 'ALL' granularity, and skipEmptyBuckets set to true in the query context. This latter option caused the results of such a query to return no results, as there are no buckets with values to aggregate and so they are skipped, making an empty result set instead of a 'nil' result set. This behavior has been changed to behave in line with other SQL implementations, but the previous behavior can be obtained by explicitly setting `skipEmptyBuckets` on the query context. + +[#11188](https://github.com/apache/druid/pull/11188) + +#### Druid reingestion incompatible changes + +Batch tasks using a 'Druid' input source to reingest segment data will no longer accept the 'dimensions' and 'metrics' sections of their task spec, and now will internally use a new columns filter to specify which columns from the original segment should be retained. Additionally, timestampSpec is no longer ignored, allowing the __time column to be modified or replaced with a different column. These changes additionally fix a bug where transformed columns would be ignored and unavailable on the new segments. + +[#10267](https://github.com/apache/druid/pull/10267) + +#### Druid web-console no longer supports IE11 and other older browsers + +Some things might still work, but it is no longer officially supported so that newer Javascript features can be used to develop the web-console. + +[#11357](https://github.com/apache/druid/pull/11357) + +#### Changed default maximum segment loading queue size + +Druid coordinator `maxSegmentsInNodeLoadingQueue` dynamic configuration has been changed from unlimited (`0`) to `100`. This should make the coordinator behave in a much more relaxed manner during periods of cluster volatility, such as a rolling upgrade, but caps the total number of segments that will be loaded in any given coordinator cycle to 100 per server, which can slow down the speed at which a completely stopped cluster is started and loaded from deep storage. + +[#11540](https://github.com/apache/druid/pull/11540) + +## 0.21.0 + +#### Improved HTTP status codes for query errors + +Before this release, Druid returned the "internal error (500)" for most of the query errors. Now Druid returns different error codes based on their cause. The following table lists the errors and their corresponding codes that has changed: + +| Exception | Description| Old code | New code | +|-----|-|--|-----| +| SqlParseException and ValidationException from Calcite | Query planning failed | 500 | 400 | +| QueryTimeoutException | Query execution didn't finish in timeout | 500 | 504 | +| ResourceLimitExceededException | Query asked more resources than configured threshold | 500 | 400 | +| InsufficientResourceException | Query failed to schedule because of lack of merge buffers available at the time when it was submitted | 500 | 429, merged to QueryCapacityExceededException | +| QueryUnsupportedException | Unsupported functionality | 400 | 501 | + +[#10464](https://github.com/apache/druid/pull/10464) +[#10746](https://github.com/apache/druid/pull/10746) + +#### Query interrupted metric +`query/interrupted/count` no longer counts the queries that timed out. These queries are counted by `query/timeout/count`. + +#### context dimension in query metrics + +`context` is now a default dimension emitted for all query metrics. `context` is a JSON-formatted string containing the query context for the query that the emitted metric refers to. The addition of a dimension that was not previously alters some metrics emitted by Druid. You should plan to handle this new `context` dimension in your metrics pipeline. Since the dimension is a JSON-formatted string, a common solution is to parse the dimension and either flatten it or extract the bits you want and discard the full JSON-formatted string blob. + +[#10578](https://github.com/apache/druid/pull/10578) + +#### Deprecated support for Apache ZooKeeper 3.4 + +As [ZooKeeper 3.4 has been end-of-life for a while](https://mail-archives.apache.org/mod_mbox/zookeeper-user/202004.mbox/%3C41A7EC67-D8F4-4C3A-B2DB-C2741C2EECA3%40apache.org%3E), support for ZooKeeper 3.4 is deprecated in 0.21.0 and will be removed in the near future. + +[#10780](https://github.com/apache/druid/issues/10780) + +#### Consistent serialization format and column naming convention for the sys.segments table + +All columns in the `sys.segments` table are now serialized in the JSON format to make them consistent with other system tables. Column names now use the same "snake case" convention. + +[#10481](https://github.com/apache/druid/pull/10481) From 49f221a7e3153860455fc3412ad0dcbc0fc304c5 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 16 Feb 2024 03:32:40 +0530 Subject: [PATCH 04/11] working --- .../read/columnar/FrameColumnReaderUtils.java | 66 +++++ .../read/columnar/FrameColumnReaders.java | 19 +- .../columnar/LongArrayFrameColumnReader.java | 240 +++++++++++++++++- .../columnar/StringFrameColumnReader.java | 54 ++-- .../write/columnar/FrameColumnWriters.java | 13 + .../columnar/LongArrayFrameColumnWriter.java | 25 +- .../druid/frame/write/FrameWriterTest.java | 2 +- 7 files changed, 361 insertions(+), 58 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaderUtils.java diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaderUtils.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaderUtils.java new file mode 100644 index 000000000000..c0dfdc6a76f1 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaderUtils.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.read.columnar; + +import org.apache.datasketches.memory.Memory; + +public class FrameColumnReaderUtils +{ + /** + * Adjusts a negative cumulative row length from {@link #getCumulativeRowLength(Memory, long, int)} to be the actual + * positive length. + */ + public static int adjustCumulativeRowLength(final int cumulativeRowLength) + { + if (cumulativeRowLength < 0) { + return -(cumulativeRowLength + 1); + } else { + return cumulativeRowLength; + } + } + + /** + * Returns cumulative row length, if the row is not null itself, or -(cumulative row length) - 1 if the row is + * null itself. + * + * To check if the return value from this function indicate a null row, use {@link #isNullRow(int)} + * + * To get the actual cumulative row length, use {@link FrameColumnReaderUtils#adjustCumulativeRowLength(int)}. + */ + public static int getCumulativeRowLength(final Memory memory, final long offset, final int physicalRow) + { + // Note: only valid to call this if multiValue = true. + return memory.getInt(offset + (long) Integer.BYTES * physicalRow); + } + + public static int getAdjustedCumulativeRowLength(final Memory memory, final long offset, final int physicalRow) + { + return adjustCumulativeRowLength(getCumulativeRowLength(memory, offset, physicalRow)); + } + + /** + * When given a return value from {@link FrameColumnReaderUtils#getCumulativeRowLength(Memory, long, int)}, returns whether the row is + * null itself (i.e. a null array). + */ + public static boolean isNullRow(final int cumulativeRowLength) + { + return cumulativeRowLength < 0; + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java index 98218819ce13..fc65256df522 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java @@ -19,8 +19,8 @@ package org.apache.druid.frame.read.columnar; +import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.ValueType; /** * Creates {@link FrameColumnReader} corresponding to a given column type and number. @@ -58,12 +58,19 @@ public static FrameColumnReader create( return new ComplexFrameColumnReader(columnNumber); case ARRAY: - if (columnType.getElementType().getType() == ValueType.STRING) { - return new StringFrameColumnReader(columnNumber, true); - } else { - return new UnsupportedColumnTypeFrameColumnReader(columnName, columnType); + switch (columnType.getElementType().getType()) + { + case STRING: + return new StringFrameColumnReader(columnNumber, true); + case LONG: + return new LongArrayFrameColumnReader( + FrameColumnWriters.TYPE_LONG_ARRAY, + ColumnType.LONG_ARRAY, + columnNumber + ); + default: + return new UnsupportedColumnTypeFrameColumnReader(columnName, columnType); } - default: return new UnsupportedColumnTypeFrameColumnReader(columnName, columnType); } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java index 0895da230865..874bfbf17cfd 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java @@ -1,28 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.frame.read.columnar; +import it.unimi.dsi.fastutil.objects.ObjectArrays; import org.apache.datasketches.memory.Memory; import org.apache.druid.error.DruidException; import org.apache.druid.frame.Frame; import org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessorBasedColumn; import org.apache.druid.query.rowsandcols.column.accessor.ObjectColumnAccessorBase; import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.ObjectColumnSelector; import org.apache.druid.segment.column.BaseColumn; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.data.ReadableOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.ReadableVectorOffset; +import org.apache.druid.segment.vector.VectorObjectSelector; +import javax.annotation.Nullable; import java.io.IOException; import java.util.Comparator; public class LongArrayFrameColumnReader implements FrameColumnReader { private final byte typeCode; + private final ColumnType columnType; private final int columnNumber; - public LongArrayFrameColumnReader(byte typeCode, int columnNumber) + public LongArrayFrameColumnReader(byte typeCode, ColumnType columnType, int columnNumber) { - this.columnNumber = columnNumber; this.typeCode = typeCode; + this.columnType = columnType; + this.columnNumber = columnNumber; } @Override @@ -30,7 +60,7 @@ public Column readRACColumn(Frame frame) { final Memory memory = frame.region(columnNumber); validate(memory); - throw DruidException.defensive("Multivalue not yet handled by RAC"); + return new ColumnAccessorBasedColumn(new LongArrayFrameColumn(frame, memory, columnType)); } @Override @@ -38,9 +68,15 @@ public ColumnPlus readColumn(Frame frame) { final Memory memory = frame.region(columnNumber); validate(memory); + return new ColumnPlus( + new LongArrayFrameColumn(frame, memory, columnType), + ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(columnType), + frame.numRows() + ); } - private void validate(final Memory region) { + private void validate(final Memory region) + { if (region.getCapacity() < LongArrayFrameColumnWriter.DATA_OFFSET) { throw DruidException.defensive("Column[%s] is not big enough for a header", columnNumber); } @@ -55,42 +91,224 @@ private void validate(final Memory region) { } } + private static long getStartOfCumulativeLengthSection() + { + return LongArrayFrameColumnWriter.DATA_OFFSET; + } + + private static long getStartOfRowNullityData(final int numRows) + { + return getStartOfCumulativeLengthSection() + ((long) numRows * Integer.BYTES); + } + + private static long getStartOfRowData(final Memory memory, final int numRows) + { + return getStartOfRowNullityData(numRows) + + (Byte.BYTES + * FrameColumnReaderUtils + .getAdjustedCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1)); + } + private static class LongArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn { - @Override - public void close() throws IOException + + private final Frame frame; + private final Memory memory; + private final ColumnType columnType; + + private final long rowNullityDataOffset; + private final long rowDataOffset; + + + public LongArrayFrameColumn(Frame frame, Memory memory, ColumnType columnType) { + this.frame = frame; + this.memory = memory; + this.columnType = columnType; + this.rowNullityDataOffset = getStartOfRowNullityData(frame.numRows()); + this.rowDataOffset = getStartOfRowData(memory, frame.numRows()); } @Override public ColumnType getType() { - return null; + return columnType; } @Override public int numRows() { - return 0; + return frame.numRows(); } @Override protected Object getVal(int rowNum) { - return null; + return getNumericArray(physicalRow(rowNum)); } @Override protected Comparator getComparator() { - return null; + return columnType.getNullableStrategy(); } @Override public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { - return null; + return new ObjectColumnSelector() + { + private int cachedLogicalRow = -1; + @Nullable + private Object[] cachedValue = null; + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + } + + @Nullable + @Override + public Object getObject() + { + compute(); + return cachedValue; + } + + @Override + public Class classOfObject() + { + return Object[].class; + } + + private void compute() + { + int currentLogicalRow = offset.getOffset(); + if (cachedLogicalRow == currentLogicalRow) { + return; + } + cachedValue = getNumericArray(physicalRow(currentLogicalRow)); + cachedLogicalRow = currentLogicalRow; + } + }; + } + + @Override + public VectorObjectSelector makeVectorObjectSelector(ReadableVectorOffset offset) + { + return new VectorObjectSelector() + { + private final Object[] vector = new Object[offset.getMaxVectorSize()]; + private int id = ReadableVectorInspector.NULL_ID; + + @Override + public Object[] getObjectVector() + { + computeVector(); + return vector; + } + + @Override + public int getMaxVectorSize() + { + return offset.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return offset.getCurrentVectorSize(); + } + + private void computeVector() + { + if (id == offset.getId()) { + return; + } + + if (offset.isContiguous()) { + // Contiguous offsets can have a cache optimized implementation if 'frame.isPermuted() == false', + // i.e. logicalRow == physicalRow. The implementation can separately fetch out the nullity data, and the + // element data continguously. + final int start = offset.getStartOffset(); + for (int i = 0; i < offset.getCurrentVectorSize(); ++i) { + vector[i] = getNumericArray(physicalRow(start + i)); + } + } else { + final int[] offsets = offset.getOffsets(); + for (int i = 0; i < offset.getCurrentVectorSize(); ++i) { + vector[i] = getNumericArray(physicalRow(offsets[i])); + } + + id = offset.getId(); + } + } + }; + } + + @Override + public void close() throws IOException + { + + } + + private int physicalRow(int logicalRow) + { + return frame.physicalRow(logicalRow); + } + + @Nullable + private Object[] getNumericArray(final int physicalRow) + { + final int cumulativeLength = FrameColumnReaderUtils.getCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow + ); + + final int rowLength; + if (FrameColumnReaderUtils.isNullRow(cumulativeLength)) { + return null; + } else if (physicalRow == 0) { + rowLength = cumulativeLength; + } else { + final int previousCumulativeLength = FrameColumnReaderUtils.adjustCumulativeRowLength( + FrameColumnReaderUtils.getCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow - 1 + ) + ); + rowLength = cumulativeLength - previousCumulativeLength; + } + + if (rowLength == 0) { + return ObjectArrays.EMPTY_ARRAY; + } + + final Object[] row = new Object[rowLength]; + for (int i = 0; i < rowLength; ++i) { + final int cumulativeIndex = cumulativeLength - rowLength + i; + row[i] = getElementNullity(cumulativeIndex) ? null : getElement(cumulativeIndex); + } + + return row; + } + + private boolean getElementNullity(final int cumulativeIndex) + { + byte b = memory.getByte(rowNullityDataOffset + cumulativeIndex * Byte.BYTES); + if (b == LongArrayFrameColumnWriter.NULL_ELEMENT_MARKER) { + return true; + } + assert b == LongArrayFrameColumnWriter.NON_NULL_ELEMENT_MARKER; + return false; + } + + private Number getElement(final int cumulativeIndex) + { + return memory.getLong(rowDataOffset + (long) cumulativeIndex * Long.BYTES); } } } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java index 119dd48a3f17..fb442ac10b2a 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java @@ -174,40 +174,9 @@ private static boolean isMultiValue(final Memory memory) return memory.getByte(1) == 1; } - /** - * Returns cumulative row length, if the row is not null itself, or -(cumulative row length) - 1 if the row is - * null itself. - * - * To check if the return value from this function indicate a null row, use {@link #isNullRow(int)} - * - * To get the actual cumulative row length, use {@link #adjustCumulativeRowLength(int)}. - */ - private static int getCumulativeRowLength(final Memory memory, final int physicalRow) - { - // Note: only valid to call this if multiValue = true. - return memory.getInt(StringFrameColumnWriter.DATA_OFFSET + (long) Integer.BYTES * physicalRow); - } - - /** - * When given a return value from {@link #getCumulativeRowLength(Memory, int)}, returns whether the row is - * null itself (i.e. a null array). - */ - private static boolean isNullRow(final int cumulativeRowLength) + private static long getStartOfCumulativeLengthSection() { - return cumulativeRowLength < 0; - } - - /** - * Adjusts a negative cumulative row length from {@link #getCumulativeRowLength(Memory, int)} to be the actual - * positive length. - */ - private static int adjustCumulativeRowLength(final int cumulativeRowLength) - { - if (cumulativeRowLength < 0) { - return -(cumulativeRowLength + 1); - } else { - return cumulativeRowLength; - } + return StringFrameColumnWriter.DATA_OFFSET; } private static long getStartOfStringLengthSection( @@ -231,7 +200,9 @@ private static long getStartOfStringDataSection( final int totalNumValues; if (multiValue) { - totalNumValues = adjustCumulativeRowLength(getCumulativeRowLength(memory, numRows - 1)); + totalNumValues = FrameColumnReaderUtils.adjustCumulativeRowLength( + FrameColumnReaderUtils.getCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), -1) + ); } else { totalNumValues = numRows; } @@ -489,15 +460,24 @@ private String getString(final int index) private Object getRowAsObject(final int physicalRow, final boolean decode) { if (multiValue) { - final int cumulativeRowLength = getCumulativeRowLength(memory, physicalRow); + final int cumulativeRowLength = FrameColumnReaderUtils.getCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow + ); final int rowLength; - if (isNullRow(cumulativeRowLength)) { + if (FrameColumnReaderUtils.isNullRow(cumulativeRowLength)) { return null; } else if (physicalRow == 0) { rowLength = cumulativeRowLength; } else { - rowLength = cumulativeRowLength - adjustCumulativeRowLength(getCumulativeRowLength(memory, physicalRow - 1)); + rowLength = cumulativeRowLength - FrameColumnReaderUtils.adjustCumulativeRowLength( + FrameColumnReaderUtils.getCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow - 1 + )); } if (rowLength == 0) { diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java index 8c5dbe758532..48fc0fe4f867 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java @@ -41,6 +41,7 @@ public class FrameColumnWriters public static final byte TYPE_STRING = 4; public static final byte TYPE_COMPLEX = 5; public static final byte TYPE_STRING_ARRAY = 6; + public static final byte TYPE_LONG_ARRAY = 7; private FrameColumnWriters() { @@ -76,6 +77,8 @@ static FrameColumnWriter create( switch (type.getElementType().getType()) { case STRING: return makeStringArrayWriter(columnSelectorFactory, allocator, column); + case LONG: + return makeLongArrayWriter(columnSelectorFactory, allocator, column); default: throw new UnsupportedColumnTypeException(column, type); } @@ -144,6 +147,16 @@ private static StringFrameColumnWriter makeStringArrayWriter( return new StringArrayFrameColumnWriterImpl(selector, allocator); } + private static LongArrayFrameColumnWriter makeLongArrayWriter( + final ColumnSelectorFactory selectorFactory, + final MemoryAllocator allocator, + final String columnName + ) + { + final ColumnValueSelector selector = selectorFactory.makeColumnValueSelector(columnName); + return new LongArrayFrameColumnWriter(selector, allocator, FrameColumnWriters.TYPE_LONG_ARRAY); + } + private static ComplexFrameColumnWriter makeComplexWriter( final ColumnSelectorFactory selectorFactory, final MemoryAllocator allocator, diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java index ce2412a1cd00..21c73fc9e044 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.frame.write.columnar; import org.apache.datasketches.memory.WritableMemory; @@ -18,8 +37,8 @@ public class LongArrayFrameColumnWriter implements FrameColumnWriter private static final int ELEMENT_SIZE = Long.BYTES; - private static final byte NULL_ELEMENT_MARKER = 0x00; - private static final byte NON_NULL_ELEMENT_MARKER = 0x01; + public static final byte NULL_ELEMENT_MARKER = 0x00; + public static final byte NON_NULL_ELEMENT_MARKER = 0x01; /** * A byte required at the beginning for type code @@ -98,7 +117,7 @@ public boolean addSelection() lastRowLength = rowLength; lastCumulativeRowLength += rowLength; - final MemoryRange rowNullityDataCursor = rowLength > 0 ? rowData.cursor() : null; + final MemoryRange rowNullityDataCursor = rowLength > 0 ? rowNullityData.cursor() : null; final MemoryRange rowDataCursor = rowLength > 0 ? rowData.cursor() : null; for (int i = 0; i < rowLength; ++i) { diff --git a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java index 16a1b6675567..abf6346aec81 100644 --- a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java @@ -246,7 +246,7 @@ public void test_arrayLong() { // ARRAY can't be read or written for columnar frames, therefore skip the check if it encounters those // parameters - Assume.assumeFalse(inputFrameType == FrameType.COLUMNAR || outputFrameType == FrameType.COLUMNAR); + // Assume.assumeFalse(inputFrameType == FrameType.COLUMNAR || outputFrameType == FrameType.COLUMNAR); testWithDataset(FrameWriterTestData.TEST_ARRAYS_LONG); } From aabb1f08758c13976cc9aa7a15c72979c730199e Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 16 Feb 2024 03:43:00 +0530 Subject: [PATCH 05/11] string array RAC working --- .../frame/read/columnar/FrameColumnReaders.java | 3 +-- .../read/columnar/StringFrameColumnReader.java | 15 ++++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java index fc65256df522..39ddd7d41f9f 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java @@ -58,8 +58,7 @@ public static FrameColumnReader create( return new ComplexFrameColumnReader(columnNumber); case ARRAY: - switch (columnType.getElementType().getType()) - { + switch (columnType.getElementType().getType()) { case STRING: return new StringFrameColumnReader(columnNumber, true); case LONG: diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java index fb442ac10b2a..9a9ffedf561f 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java @@ -92,17 +92,18 @@ public Column readRACColumn(Frame frame) final Memory memory = frame.region(columnNumber); validate(memory); - if (isMultiValue(memory)) { - // When we implement handling of multi-value, we should actually make this look like an Array of String instead - // of perpetuating the multi-value idea. Thus, when we add support for Arrays to the RAC stuff, that's when - // we can start supporting multi-value. - throw new ISE("Multivalue not yet handled by RAC"); - } final long positionOfLengths = getStartOfStringLengthSection(frame.numRows(), false); final long positionOfPayloads = getStartOfStringDataSection(memory, frame.numRows(), false); StringFrameColumn frameCol = - new StringFrameColumn(frame, false, memory, positionOfLengths, positionOfPayloads, false); + new StringFrameColumn( + frame, + false, + memory, + positionOfLengths, + positionOfPayloads, + asArray || isMultiValue(memory) // Read MVDs as String arrays + ); return new ColumnAccessorBasedColumn(frameCol); } From 5587ceb4134ee17313835f6f63d144cf0bb18aa0 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 16 Feb 2024 03:59:34 +0530 Subject: [PATCH 06/11] readers working --- .../DoubleArrayFrameColumnReader.java | 57 ++++ .../columnar/FloatArrayFrameColumnReader.java | 57 ++++ .../read/columnar/FrameColumnReaders.java | 11 +- .../columnar/LongArrayFrameColumnReader.java | 285 +--------------- .../NumericArrayFrameColumnReader.java | 318 ++++++++++++++++++ .../write/columnar/FrameColumnWriters.java | 6 + 6 files changed, 457 insertions(+), 277 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java create mode 100644 processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java create mode 100644 processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java new file mode 100644 index 000000000000..e4a401fcc8e8 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.read.columnar; + +import org.apache.datasketches.memory.Memory; +import org.apache.druid.frame.Frame; +import org.apache.druid.frame.write.columnar.FrameColumnWriters; +import org.apache.druid.segment.column.ColumnType; + +public class DoubleArrayFrameColumnReader extends NumericArrayFrameColumnReader +{ + public DoubleArrayFrameColumnReader(int columnNumber) + { + super(FrameColumnWriters.TYPE_DOUBLE_ARRAY, ColumnType.DOUBLE_ARRAY, columnNumber); + } + + @Override + NumericArrayFrameColumn column(Frame frame, Memory memory, ColumnType columnType) + { + return new DoubleArrayFrameColumn(frame, memory, columnType); + } + + private static class DoubleArrayFrameColumn extends NumericArrayFrameColumn + { + public DoubleArrayFrameColumn( + Frame frame, + Memory memory, + ColumnType columnType + ) + { + super(frame, memory, columnType); + } + + @Override + Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex) + { + return memory.getDouble(rowDataOffset + (long) cumulativeIndex * Double.BYTES); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java new file mode 100644 index 000000000000..83e065d0916e --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.read.columnar; + +import org.apache.datasketches.memory.Memory; +import org.apache.druid.frame.Frame; +import org.apache.druid.frame.write.columnar.FrameColumnWriters; +import org.apache.druid.segment.column.ColumnType; + +public class FloatArrayFrameColumnReader extends NumericArrayFrameColumnReader +{ + public FloatArrayFrameColumnReader(int columnNumber) + { + super(FrameColumnWriters.TYPE_FLOAT_ARRAY, ColumnType.FLOAT_ARRAY, columnNumber); + } + + @Override + NumericArrayFrameColumn column(Frame frame, Memory memory, ColumnType columnType) + { + return new FloatArrayFrameColumn(frame, memory, columnType); + } + + private static class FloatArrayFrameColumn extends NumericArrayFrameColumn + { + public FloatArrayFrameColumn( + Frame frame, + Memory memory, + ColumnType columnType + ) + { + super(frame, memory, columnType); + } + + @Override + Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex) + { + return memory.getFloat(rowDataOffset + (long) cumulativeIndex * Float.BYTES); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java index 39ddd7d41f9f..9b4dc85cb1e0 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FrameColumnReaders.java @@ -19,7 +19,6 @@ package org.apache.druid.frame.read.columnar; -import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.segment.column.ColumnType; /** @@ -62,11 +61,11 @@ public static FrameColumnReader create( case STRING: return new StringFrameColumnReader(columnNumber, true); case LONG: - return new LongArrayFrameColumnReader( - FrameColumnWriters.TYPE_LONG_ARRAY, - ColumnType.LONG_ARRAY, - columnNumber - ); + return new LongArrayFrameColumnReader(columnNumber); + case FLOAT: + return new FloatArrayFrameColumnReader(columnNumber); + case DOUBLE: + return new DoubleArrayFrameColumnReader(columnNumber); default: return new UnsupportedColumnTypeFrameColumnReader(columnName, columnType); } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java index 874bfbf17cfd..f5e1216e2f97 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java @@ -19,294 +19,37 @@ package org.apache.druid.frame.read.columnar; -import it.unimi.dsi.fastutil.objects.ObjectArrays; import org.apache.datasketches.memory.Memory; -import org.apache.druid.error.DruidException; import org.apache.druid.frame.Frame; -import org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.query.rowsandcols.column.Column; -import org.apache.druid.query.rowsandcols.column.ColumnAccessorBasedColumn; -import org.apache.druid.query.rowsandcols.column.accessor.ObjectColumnAccessorBase; -import org.apache.druid.segment.ColumnValueSelector; -import org.apache.druid.segment.ObjectColumnSelector; -import org.apache.druid.segment.column.BaseColumn; -import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.data.ReadableOffset; -import org.apache.druid.segment.vector.ReadableVectorInspector; -import org.apache.druid.segment.vector.ReadableVectorOffset; -import org.apache.druid.segment.vector.VectorObjectSelector; -import javax.annotation.Nullable; -import java.io.IOException; -import java.util.Comparator; - -public class LongArrayFrameColumnReader implements FrameColumnReader +public class LongArrayFrameColumnReader extends NumericArrayFrameColumnReader { - private final byte typeCode; - private final ColumnType columnType; - private final int columnNumber; - - public LongArrayFrameColumnReader(byte typeCode, ColumnType columnType, int columnNumber) - { - this.typeCode = typeCode; - this.columnType = columnType; - this.columnNumber = columnNumber; - } - - @Override - public Column readRACColumn(Frame frame) + public LongArrayFrameColumnReader(int columnNumber) { - final Memory memory = frame.region(columnNumber); - validate(memory); - return new ColumnAccessorBasedColumn(new LongArrayFrameColumn(frame, memory, columnType)); + super(FrameColumnWriters.TYPE_LONG_ARRAY, ColumnType.LONG_ARRAY, columnNumber); } @Override - public ColumnPlus readColumn(Frame frame) - { - final Memory memory = frame.region(columnNumber); - validate(memory); - return new ColumnPlus( - new LongArrayFrameColumn(frame, memory, columnType), - ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(columnType), - frame.numRows() - ); - } - - private void validate(final Memory region) - { - if (region.getCapacity() < LongArrayFrameColumnWriter.DATA_OFFSET) { - throw DruidException.defensive("Column[%s] is not big enough for a header", columnNumber); - } - final byte typeCode = region.getByte(0); - if (typeCode != this.typeCode) { - throw DruidException.defensive( - "Column[%s] does not have the correct type code; expected[%s], got[%s]", - columnNumber, - this.typeCode, - typeCode - ); - } - } - - private static long getStartOfCumulativeLengthSection() - { - return LongArrayFrameColumnWriter.DATA_OFFSET; - } - - private static long getStartOfRowNullityData(final int numRows) - { - return getStartOfCumulativeLengthSection() + ((long) numRows * Integer.BYTES); - } - - private static long getStartOfRowData(final Memory memory, final int numRows) + NumericArrayFrameColumn column(Frame frame, Memory memory, ColumnType columnType) { - return getStartOfRowNullityData(numRows) - + (Byte.BYTES - * FrameColumnReaderUtils - .getAdjustedCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1)); + return new LongArrayFrameColumn(frame, memory, columnType); } - private static class LongArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn + private static class LongArrayFrameColumn extends NumericArrayFrameColumn { - - private final Frame frame; - private final Memory memory; - private final ColumnType columnType; - - private final long rowNullityDataOffset; - private final long rowDataOffset; - - - public LongArrayFrameColumn(Frame frame, Memory memory, ColumnType columnType) - { - this.frame = frame; - this.memory = memory; - this.columnType = columnType; - - this.rowNullityDataOffset = getStartOfRowNullityData(frame.numRows()); - this.rowDataOffset = getStartOfRowData(memory, frame.numRows()); - } - - @Override - public ColumnType getType() - { - return columnType; - } - - @Override - public int numRows() - { - return frame.numRows(); - } - - @Override - protected Object getVal(int rowNum) - { - return getNumericArray(physicalRow(rowNum)); - } - - @Override - protected Comparator getComparator() - { - return columnType.getNullableStrategy(); - } - - @Override - public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + public LongArrayFrameColumn( + Frame frame, + Memory memory, + ColumnType columnType + ) { - return new ObjectColumnSelector() - { - private int cachedLogicalRow = -1; - @Nullable - private Object[] cachedValue = null; - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - } - - @Nullable - @Override - public Object getObject() - { - compute(); - return cachedValue; - } - - @Override - public Class classOfObject() - { - return Object[].class; - } - - private void compute() - { - int currentLogicalRow = offset.getOffset(); - if (cachedLogicalRow == currentLogicalRow) { - return; - } - cachedValue = getNumericArray(physicalRow(currentLogicalRow)); - cachedLogicalRow = currentLogicalRow; - } - }; - } - - @Override - public VectorObjectSelector makeVectorObjectSelector(ReadableVectorOffset offset) - { - return new VectorObjectSelector() - { - private final Object[] vector = new Object[offset.getMaxVectorSize()]; - private int id = ReadableVectorInspector.NULL_ID; - - @Override - public Object[] getObjectVector() - { - computeVector(); - return vector; - } - - @Override - public int getMaxVectorSize() - { - return offset.getMaxVectorSize(); - } - - @Override - public int getCurrentVectorSize() - { - return offset.getCurrentVectorSize(); - } - - private void computeVector() - { - if (id == offset.getId()) { - return; - } - - if (offset.isContiguous()) { - // Contiguous offsets can have a cache optimized implementation if 'frame.isPermuted() == false', - // i.e. logicalRow == physicalRow. The implementation can separately fetch out the nullity data, and the - // element data continguously. - final int start = offset.getStartOffset(); - for (int i = 0; i < offset.getCurrentVectorSize(); ++i) { - vector[i] = getNumericArray(physicalRow(start + i)); - } - } else { - final int[] offsets = offset.getOffsets(); - for (int i = 0; i < offset.getCurrentVectorSize(); ++i) { - vector[i] = getNumericArray(physicalRow(offsets[i])); - } - - id = offset.getId(); - } - } - }; + super(frame, memory, columnType); } @Override - public void close() throws IOException - { - - } - - private int physicalRow(int logicalRow) - { - return frame.physicalRow(logicalRow); - } - - @Nullable - private Object[] getNumericArray(final int physicalRow) - { - final int cumulativeLength = FrameColumnReaderUtils.getCumulativeRowLength( - memory, - getStartOfCumulativeLengthSection(), - physicalRow - ); - - final int rowLength; - if (FrameColumnReaderUtils.isNullRow(cumulativeLength)) { - return null; - } else if (physicalRow == 0) { - rowLength = cumulativeLength; - } else { - final int previousCumulativeLength = FrameColumnReaderUtils.adjustCumulativeRowLength( - FrameColumnReaderUtils.getCumulativeRowLength( - memory, - getStartOfCumulativeLengthSection(), - physicalRow - 1 - ) - ); - rowLength = cumulativeLength - previousCumulativeLength; - } - - if (rowLength == 0) { - return ObjectArrays.EMPTY_ARRAY; - } - - final Object[] row = new Object[rowLength]; - for (int i = 0; i < rowLength; ++i) { - final int cumulativeIndex = cumulativeLength - rowLength + i; - row[i] = getElementNullity(cumulativeIndex) ? null : getElement(cumulativeIndex); - } - - return row; - } - - private boolean getElementNullity(final int cumulativeIndex) - { - byte b = memory.getByte(rowNullityDataOffset + cumulativeIndex * Byte.BYTES); - if (b == LongArrayFrameColumnWriter.NULL_ELEMENT_MARKER) { - return true; - } - assert b == LongArrayFrameColumnWriter.NON_NULL_ELEMENT_MARKER; - return false; - } - - private Number getElement(final int cumulativeIndex) + Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex) { return memory.getLong(rowDataOffset + (long) cumulativeIndex * Long.BYTES); } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java new file mode 100644 index 000000000000..015e7241e590 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.read.columnar; + +import it.unimi.dsi.fastutil.objects.ObjectArrays; +import org.apache.datasketches.memory.Memory; +import org.apache.druid.error.DruidException; +import org.apache.druid.frame.Frame; +import org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessorBasedColumn; +import org.apache.druid.query.rowsandcols.column.accessor.ObjectColumnAccessorBase; +import org.apache.druid.segment.ColumnValueSelector; +import org.apache.druid.segment.ObjectColumnSelector; +import org.apache.druid.segment.column.BaseColumn; +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.data.ReadableOffset; +import org.apache.druid.segment.vector.ReadableVectorInspector; +import org.apache.druid.segment.vector.ReadableVectorOffset; +import org.apache.druid.segment.vector.VectorObjectSelector; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Comparator; + +public abstract class NumericArrayFrameColumnReader implements FrameColumnReader +{ + private final byte typeCode; + private final ColumnType columnType; + private final int columnNumber; + + public NumericArrayFrameColumnReader(byte typeCode, ColumnType columnType, int columnNumber) + { + this.typeCode = typeCode; + this.columnType = columnType; + this.columnNumber = columnNumber; + } + + @Override + public Column readRACColumn(Frame frame) + { + final Memory memory = frame.region(columnNumber); + validate(memory); + return new ColumnAccessorBasedColumn(column(frame, memory, columnType)); + } + + @Override + public ColumnPlus readColumn(Frame frame) + { + final Memory memory = frame.region(columnNumber); + validate(memory); + return new ColumnPlus( + column(frame, memory, columnType), + ColumnCapabilitiesImpl.createSimpleArrayColumnCapabilities(columnType), + frame.numRows() + ); + } + + abstract NumericArrayFrameColumn column(Frame frame, Memory memory, ColumnType columnType); + + private void validate(final Memory region) + { + if (region.getCapacity() < LongArrayFrameColumnWriter.DATA_OFFSET) { + throw DruidException.defensive("Column[%s] is not big enough for a header", columnNumber); + } + final byte typeCode = region.getByte(0); + if (typeCode != this.typeCode) { + throw DruidException.defensive( + "Column[%s] does not have the correct type code; expected[%s], got[%s]", + columnNumber, + this.typeCode, + typeCode + ); + } + } + + private static long getStartOfCumulativeLengthSection() + { + return LongArrayFrameColumnWriter.DATA_OFFSET; + } + + private static long getStartOfRowNullityData(final int numRows) + { + return getStartOfCumulativeLengthSection() + ((long) numRows * Integer.BYTES); + } + + private static long getStartOfRowData(final Memory memory, final int numRows) + { + return getStartOfRowNullityData(numRows) + + (Byte.BYTES + * FrameColumnReaderUtils + .getAdjustedCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1)); + } + + public static abstract class NumericArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn + { + + private final Frame frame; + private final Memory memory; + private final ColumnType columnType; + + private final long rowNullityDataOffset; + private final long rowDataOffset; + + + public NumericArrayFrameColumn(Frame frame, Memory memory, ColumnType columnType) + { + this.frame = frame; + this.memory = memory; + this.columnType = columnType; + + this.rowNullityDataOffset = getStartOfRowNullityData(frame.numRows()); + this.rowDataOffset = getStartOfRowData(memory, frame.numRows()); + } + + @Override + public ColumnType getType() + { + return columnType; + } + + @Override + public int numRows() + { + return frame.numRows(); + } + + @Override + protected Object getVal(int rowNum) + { + return getNumericArray(physicalRow(rowNum)); + } + + @Override + protected Comparator getComparator() + { + return columnType.getNullableStrategy(); + } + + @Override + public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) + { + return new ObjectColumnSelector() + { + private int cachedLogicalRow = -1; + @Nullable + private Object[] cachedValue = null; + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + } + + @Nullable + @Override + public Object getObject() + { + compute(); + return cachedValue; + } + + @Override + public Class classOfObject() + { + return Object[].class; + } + + private void compute() + { + int currentLogicalRow = offset.getOffset(); + if (cachedLogicalRow == currentLogicalRow) { + return; + } + cachedValue = getNumericArray(physicalRow(currentLogicalRow)); + cachedLogicalRow = currentLogicalRow; + } + }; + } + + @Override + public VectorObjectSelector makeVectorObjectSelector(ReadableVectorOffset offset) + { + return new VectorObjectSelector() + { + private final Object[] vector = new Object[offset.getMaxVectorSize()]; + private int id = ReadableVectorInspector.NULL_ID; + + @Override + public Object[] getObjectVector() + { + computeVector(); + return vector; + } + + @Override + public int getMaxVectorSize() + { + return offset.getMaxVectorSize(); + } + + @Override + public int getCurrentVectorSize() + { + return offset.getCurrentVectorSize(); + } + + private void computeVector() + { + if (id == offset.getId()) { + return; + } + + if (offset.isContiguous()) { + // Contiguous offsets can have a cache optimized implementation if 'frame.isPermuted() == false', + // i.e. logicalRow == physicalRow. The implementation can separately fetch out the nullity data, and the + // element data continguously. + final int start = offset.getStartOffset(); + for (int i = 0; i < offset.getCurrentVectorSize(); ++i) { + vector[i] = getNumericArray(physicalRow(start + i)); + } + } else { + final int[] offsets = offset.getOffsets(); + for (int i = 0; i < offset.getCurrentVectorSize(); ++i) { + vector[i] = getNumericArray(physicalRow(offsets[i])); + } + + id = offset.getId(); + } + } + }; + } + + @Override + public void close() throws IOException + { + + } + + private int physicalRow(int logicalRow) + { + return frame.physicalRow(logicalRow); + } + + @Nullable + private Object[] getNumericArray(final int physicalRow) + { + final int cumulativeLength = FrameColumnReaderUtils.getCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow + ); + + final int rowLength; + if (FrameColumnReaderUtils.isNullRow(cumulativeLength)) { + return null; + } else if (physicalRow == 0) { + rowLength = cumulativeLength; + } else { + final int previousCumulativeLength = FrameColumnReaderUtils.adjustCumulativeRowLength( + FrameColumnReaderUtils.getCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow - 1 + ) + ); + rowLength = cumulativeLength - previousCumulativeLength; + } + + if (rowLength == 0) { + return ObjectArrays.EMPTY_ARRAY; + } + + final Object[] row = new Object[rowLength]; + for (int i = 0; i < rowLength; ++i) { + final int cumulativeIndex = cumulativeLength - rowLength + i; + row[i] = getElementNullity(cumulativeIndex) ? null : getElement(memory, rowDataOffset, cumulativeIndex); + } + + return row; + } + + private boolean getElementNullity(final int cumulativeIndex) + { + byte b = memory.getByte(rowNullityDataOffset + cumulativeIndex * Byte.BYTES); + if (b == LongArrayFrameColumnWriter.NULL_ELEMENT_MARKER) { + return true; + } + assert b == LongArrayFrameColumnWriter.NON_NULL_ELEMENT_MARKER; + return false; + } + + abstract Number getElement(final Memory memory, final long rowDataOffset, final int cumulativeIndex); + +// private Number getElement(final int cumulativeIndex) +// { +// return memory.getLong(rowDataOffset + (long) cumulativeIndex * Long.BYTES); +// } + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java index 48fc0fe4f867..d3d9fdca68a0 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java @@ -42,6 +42,8 @@ public class FrameColumnWriters public static final byte TYPE_COMPLEX = 5; public static final byte TYPE_STRING_ARRAY = 6; public static final byte TYPE_LONG_ARRAY = 7; + public static final byte TYPE_FLOAT_ARRAY = 8; + public static final byte TYPE_DOUBLE_ARRAY = 9; private FrameColumnWriters() { @@ -79,6 +81,10 @@ static FrameColumnWriter create( return makeStringArrayWriter(columnSelectorFactory, allocator, column); case LONG: return makeLongArrayWriter(columnSelectorFactory, allocator, column); + case FLOAT: + return makeLongArrayWriter(columnSelectorFactory, allocator, column); + case DOUBLE: + return makeLongArrayWriter(columnSelectorFactory, allocator, column); default: throw new UnsupportedColumnTypeException(column, type); } From 698dde908596ddfce20cca22f95c61076b7ea230 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 16 Feb 2024 04:28:36 +0530 Subject: [PATCH 07/11] all readers and writers --- .../NumericArrayFrameColumnReader.java | 15 +- .../DoubleArrayFrameColumnWriter.java | 54 +++++ .../columnar/FloatArrayFrameColumnWriter.java | 54 +++++ .../write/columnar/FrameColumnWriters.java | 28 ++- .../columnar/LongArrayFrameColumnWriter.java | 155 +------------- .../NumericArrayFrameColumnWriter.java | 193 ++++++++++++++++++ .../druid/frame/write/FrameWriterTest.java | 9 - 7 files changed, 341 insertions(+), 167 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java create mode 100644 processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java create mode 100644 processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java index 015e7241e590..5eecfc0f1e27 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java @@ -23,7 +23,7 @@ import org.apache.datasketches.memory.Memory; import org.apache.druid.error.DruidException; import org.apache.druid.frame.Frame; -import org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter; +import org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.query.rowsandcols.column.Column; import org.apache.druid.query.rowsandcols.column.ColumnAccessorBasedColumn; @@ -79,7 +79,7 @@ public ColumnPlus readColumn(Frame frame) private void validate(final Memory region) { - if (region.getCapacity() < LongArrayFrameColumnWriter.DATA_OFFSET) { + if (region.getCapacity() < NumericArrayFrameColumnWriter.DATA_OFFSET) { throw DruidException.defensive("Column[%s] is not big enough for a header", columnNumber); } final byte typeCode = region.getByte(0); @@ -95,7 +95,7 @@ private void validate(final Memory region) private static long getStartOfCumulativeLengthSection() { - return LongArrayFrameColumnWriter.DATA_OFFSET; + return NumericArrayFrameColumnWriter.DATA_OFFSET; } private static long getStartOfRowNullityData(final int numRows) @@ -301,18 +301,13 @@ private Object[] getNumericArray(final int physicalRow) private boolean getElementNullity(final int cumulativeIndex) { byte b = memory.getByte(rowNullityDataOffset + cumulativeIndex * Byte.BYTES); - if (b == LongArrayFrameColumnWriter.NULL_ELEMENT_MARKER) { + if (b == NumericArrayFrameColumnWriter.NULL_ELEMENT_MARKER) { return true; } - assert b == LongArrayFrameColumnWriter.NON_NULL_ELEMENT_MARKER; + assert b == NumericArrayFrameColumnWriter.NON_NULL_ELEMENT_MARKER; return false; } abstract Number getElement(final Memory memory, final long rowDataOffset, final int cumulativeIndex); - -// private Number getElement(final int cumulativeIndex) -// { -// return memory.getLong(rowDataOffset + (long) cumulativeIndex * Long.BYTES); -// } } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java new file mode 100644 index 000000000000..7415887f4fc0 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.segment.ColumnValueSelector; + +public class DoubleArrayFrameColumnWriter extends NumericArrayFrameColumnWriter +{ + public DoubleArrayFrameColumnWriter( + ColumnValueSelector selector, + MemoryAllocator allocator + ) + { + super(selector, allocator, FrameColumnWriters.TYPE_DOUBLE_ARRAY); + } + + @Override + int elementSizeBytes() + { + return Double.BYTES; + } + + @Override + void putNull(WritableMemory memory, long offset) + { + memory.putDouble(offset, 0d); + } + + @Override + void putArrayElement(WritableMemory memory, long offset, Number element) + { + // The element is of type Double, and non-null, therefore it can be cast safely + memory.putDouble(offset, (double) element); + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java new file mode 100644 index 000000000000..166afc6fc0d9 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.segment.ColumnValueSelector; + +public class FloatArrayFrameColumnWriter extends NumericArrayFrameColumnWriter +{ + public FloatArrayFrameColumnWriter( + ColumnValueSelector selector, + MemoryAllocator allocator + ) + { + super(selector, allocator, FrameColumnWriters.TYPE_FLOAT_ARRAY); + } + + @Override + int elementSizeBytes() + { + return Float.BYTES; + } + + @Override + void putNull(WritableMemory memory, long offset) + { + memory.putFloat(offset, 0f); + } + + @Override + void putArrayElement(WritableMemory memory, long offset, Number element) + { + // The element is of type Float, and non-null, therefore it can be cast safely + memory.putFloat(offset, (float) element); + } +} diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java index d3d9fdca68a0..93f0c12bae6f 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriters.java @@ -82,9 +82,9 @@ static FrameColumnWriter create( case LONG: return makeLongArrayWriter(columnSelectorFactory, allocator, column); case FLOAT: - return makeLongArrayWriter(columnSelectorFactory, allocator, column); + return makeFloatArrayWriter(columnSelectorFactory, allocator, column); case DOUBLE: - return makeLongArrayWriter(columnSelectorFactory, allocator, column); + return makeDoubleArrayWriter(columnSelectorFactory, allocator, column); default: throw new UnsupportedColumnTypeException(column, type); } @@ -153,14 +153,34 @@ private static StringFrameColumnWriter makeStringArrayWriter( return new StringArrayFrameColumnWriterImpl(selector, allocator); } - private static LongArrayFrameColumnWriter makeLongArrayWriter( + private static NumericArrayFrameColumnWriter makeLongArrayWriter( + final ColumnSelectorFactory selectorFactory, + final MemoryAllocator allocator, + final String columnName + ) + { + final ColumnValueSelector selector = selectorFactory.makeColumnValueSelector(columnName); + return new LongArrayFrameColumnWriter(selector, allocator); + } + + private static NumericArrayFrameColumnWriter makeFloatArrayWriter( + final ColumnSelectorFactory selectorFactory, + final MemoryAllocator allocator, + final String columnName + ) + { + final ColumnValueSelector selector = selectorFactory.makeColumnValueSelector(columnName); + return new FloatArrayFrameColumnWriter(selector, allocator); + } + + private static NumericArrayFrameColumnWriter makeDoubleArrayWriter( final ColumnSelectorFactory selectorFactory, final MemoryAllocator allocator, final String columnName ) { final ColumnValueSelector selector = selectorFactory.makeColumnValueSelector(columnName); - return new LongArrayFrameColumnWriter(selector, allocator, FrameColumnWriters.TYPE_LONG_ARRAY); + return new DoubleArrayFrameColumnWriter(selector, allocator); } private static ComplexFrameColumnWriter makeComplexWriter( diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java index 21c73fc9e044..495de57f4778 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java @@ -20,168 +20,35 @@ package org.apache.druid.frame.write.columnar; import org.apache.datasketches.memory.WritableMemory; -import org.apache.druid.error.DruidException; -import org.apache.druid.frame.allocation.AppendableMemory; import org.apache.druid.frame.allocation.MemoryAllocator; -import org.apache.druid.frame.allocation.MemoryRange; -import org.apache.druid.frame.write.FrameWriterUtils; import org.apache.druid.segment.ColumnValueSelector; -import java.util.List; - -public class LongArrayFrameColumnWriter implements FrameColumnWriter +public class LongArrayFrameColumnWriter extends NumericArrayFrameColumnWriter { - // TODO(laksh): Copy pasted after following the logic from StringFrameColumnWriter, since numeric writers have 3 - // regions as well. The type size is fixed, however there's no special nullable marker - private static final int INITIAL_ALLOCATION_SIZE = 120; - - private static final int ELEMENT_SIZE = Long.BYTES; - - public static final byte NULL_ELEMENT_MARKER = 0x00; - public static final byte NON_NULL_ELEMENT_MARKER = 0x01; - - /** - * A byte required at the beginning for type code - */ - public static final long DATA_OFFSET = 1; - - final ColumnValueSelector selector; - final MemoryAllocator allocator; - final byte typeCode; - - /** - * Row lengths: one int per row with the number of values contained by that row and all previous rows. - * Only written for multi-value and array columns. When the corresponding row is null itself, the length is - * written as -(actual length) - 1. (Guaranteed to be a negative number even if "actual length" is zero.) - */ - private final AppendableMemory cumulativeRowLengths; - - /** - * Denotes if the element of the row is null or not - */ - private final AppendableMemory rowNullityData; - - /** - * Row data. - */ - private final AppendableMemory rowData; - - private int lastCumulativeRowLength = 0; - private int lastRowLength = -1; - - public LongArrayFrameColumnWriter( - final ColumnValueSelector selector, - final MemoryAllocator allocator, - final byte typeCode + ColumnValueSelector selector, + MemoryAllocator allocator ) { - this.selector = selector; - this.allocator = allocator; - this.typeCode = typeCode; - this.cumulativeRowLengths = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); - this.rowNullityData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); - this.rowData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); - } - - @Override - public boolean addSelection() - { - List numericArray = FrameWriterUtils.getNumericArrayFromObject(selector.getObject()); - int rowLength = numericArray == null ? 0 : numericArray.size(); - - if ((long) lastCumulativeRowLength + rowLength > Integer.MAX_VALUE) { - return false; - } - - if (!cumulativeRowLengths.reserveAdditional(Integer.BYTES)) { - return false; - } - - if (!rowNullityData.reserveAdditional(rowLength * Byte.BYTES)) { - return false; - } - - if (!rowData.reserveAdditional(rowLength * ELEMENT_SIZE)) { - return false; - } - - final MemoryRange rowLengthsCursor = cumulativeRowLengths.cursor(); - - if (numericArray == null) { - rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), -(lastCumulativeRowLength + rowLength) - 1); - } else { - rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), lastCumulativeRowLength + rowLength); - } - cumulativeRowLengths.advanceCursor(Integer.BYTES); - lastRowLength = rowLength; - lastCumulativeRowLength += rowLength; - - final MemoryRange rowNullityDataCursor = rowLength > 0 ? rowNullityData.cursor() : null; - final MemoryRange rowDataCursor = rowLength > 0 ? rowData.cursor() : null; - - for (int i = 0; i < rowLength; ++i) { - final Number element = numericArray.get(i); - if (element == null) { - rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NULL_ELEMENT_MARKER); - rowDataCursor.memory().putLong(rowDataCursor.start() + (long) Long.BYTES * i, 0); - } else { - rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NON_NULL_ELEMENT_MARKER); - // Cast should be safe, as we have a long array column value selector, and we have checked that it is not null - rowDataCursor.memory().putLong(rowDataCursor.start() + (long) Long.BYTES * i, (long) element); - } - } - - if (rowLength > 0) { - rowNullityData.advanceCursor(Byte.BYTES * rowLength); - rowData.advanceCursor(ELEMENT_SIZE * rowLength); - } - - return true; - } - - @Override - public void undo() - { - if (lastRowLength == -1) { - throw DruidException.defensive("Nothing written to undo()"); - } - - cumulativeRowLengths.rewindCursor(Integer.BYTES); - rowNullityData.rewindCursor(lastRowLength * Byte.BYTES); - rowData.rewindCursor(lastRowLength * ELEMENT_SIZE); - - lastCumulativeRowLength -= lastRowLength; - // Multiple undo calls cannot be chained together - lastRowLength = -1; + super(selector, allocator, FrameColumnWriters.TYPE_LONG_ARRAY); } @Override - public long size() + int elementSizeBytes() { - return DATA_OFFSET + cumulativeRowLengths.size() + rowNullityData.size() + rowData.size(); + return Long.BYTES; } @Override - public long writeTo(final WritableMemory memory, final long startPosition) + void putNull(WritableMemory memory, long offset) { - long currentPosition = startPosition; - - memory.putByte(currentPosition, typeCode); - ++currentPosition; - - currentPosition += cumulativeRowLengths.writeTo(memory, currentPosition); - currentPosition += rowNullityData.writeTo(memory, currentPosition); - currentPosition += rowData.writeTo(memory, currentPosition); - - return currentPosition - startPosition; + memory.putLong(offset, 0L); } @Override - public void close() + void putArrayElement(WritableMemory memory, long offset, Number element) { - cumulativeRowLengths.close(); - rowNullityData.close(); - rowData.close(); + // The element is of type Long, and non-null, therefore it can be casted safely + memory.putLong(offset, (long) element); } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java new file mode 100644 index 000000000000..2e01d9338673 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.frame.write.columnar; + +import org.apache.datasketches.memory.WritableMemory; +import org.apache.druid.error.DruidException; +import org.apache.druid.frame.allocation.AppendableMemory; +import org.apache.druid.frame.allocation.MemoryAllocator; +import org.apache.druid.frame.allocation.MemoryRange; +import org.apache.druid.frame.write.FrameWriterUtils; +import org.apache.druid.segment.ColumnValueSelector; + +import java.util.List; + +public abstract class NumericArrayFrameColumnWriter implements FrameColumnWriter +{ + /** + * Equivalent to {@link AppendableMemory#DEFAULT_INITIAL_ALLOCATION_SIZE} / 3, since the memory would be further split + * up into three regions + */ + private static final int INITIAL_ALLOCATION_SIZE = 120; + + public static final byte NULL_ELEMENT_MARKER = 0x00; + public static final byte NON_NULL_ELEMENT_MARKER = 0x01; + + /** + * A byte required at the beginning for type code + */ + public static final long DATA_OFFSET = 1; + + final ColumnValueSelector selector; + final MemoryAllocator allocator; + final byte typeCode; + + /** + * Row lengths: one int per row with the number of values contained by that row and all previous rows. + * Only written for multi-value and array columns. When the corresponding row is null itself, the length is + * written as -(actual length) - 1. (Guaranteed to be a negative number even if "actual length" is zero.) + */ + private final AppendableMemory cumulativeRowLengths; + + /** + * Denotes if the element of the row is null or not + */ + private final AppendableMemory rowNullityData; + + /** + * Row data. + */ + private final AppendableMemory rowData; + + private int lastCumulativeRowLength = 0; + private int lastRowLength = -1; + + + public NumericArrayFrameColumnWriter( + final ColumnValueSelector selector, + final MemoryAllocator allocator, + final byte typeCode + ) + { + this.selector = selector; + this.allocator = allocator; + this.typeCode = typeCode; + this.cumulativeRowLengths = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + this.rowNullityData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + this.rowData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); + } + + abstract int elementSizeBytes(); + + abstract void putNull(WritableMemory memory, long offset); + + abstract void putArrayElement(WritableMemory memory, long offset, Number element); + + @Override + public boolean addSelection() + { + List numericArray = FrameWriterUtils.getNumericArrayFromObject(selector.getObject()); + int rowLength = numericArray == null ? 0 : numericArray.size(); + + if ((long) lastCumulativeRowLength + rowLength > Integer.MAX_VALUE) { + return false; + } + + if (!cumulativeRowLengths.reserveAdditional(Integer.BYTES)) { + return false; + } + + if (!rowNullityData.reserveAdditional(rowLength * Byte.BYTES)) { + return false; + } + + if (!rowData.reserveAdditional(rowLength * elementSizeBytes())) { + return false; + } + + final MemoryRange rowLengthsCursor = cumulativeRowLengths.cursor(); + + if (numericArray == null) { + rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), -(lastCumulativeRowLength + rowLength) - 1); + } else { + rowLengthsCursor.memory().putInt(rowLengthsCursor.start(), lastCumulativeRowLength + rowLength); + } + cumulativeRowLengths.advanceCursor(Integer.BYTES); + lastRowLength = rowLength; + lastCumulativeRowLength += rowLength; + + final MemoryRange rowNullityDataCursor = rowLength > 0 ? rowNullityData.cursor() : null; + final MemoryRange rowDataCursor = rowLength > 0 ? rowData.cursor() : null; + + for (int i = 0; i < rowLength; ++i) { + final Number element = numericArray.get(i); + final long memoryOffset = rowDataCursor.start() + ((long) elementSizeBytes() * i); + if (element == null) { + rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NULL_ELEMENT_MARKER); + putNull(rowDataCursor.memory(), memoryOffset); + } else { + rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NON_NULL_ELEMENT_MARKER); + putArrayElement(rowDataCursor.memory(), memoryOffset, element); + } + } + + if (rowLength > 0) { + rowNullityData.advanceCursor(Byte.BYTES * rowLength); + rowData.advanceCursor(elementSizeBytes() * rowLength); + } + + return true; + } + + @Override + public void undo() + { + if (lastRowLength == -1) { + throw DruidException.defensive("Nothing written to undo()"); + } + + cumulativeRowLengths.rewindCursor(Integer.BYTES); + rowNullityData.rewindCursor(lastRowLength * Byte.BYTES); + rowData.rewindCursor(lastRowLength * elementSizeBytes()); + + lastCumulativeRowLength -= lastRowLength; + // Multiple undo calls cannot be chained together + lastRowLength = -1; + } + + @Override + public long size() + { + return DATA_OFFSET + cumulativeRowLengths.size() + rowNullityData.size() + rowData.size(); + } + + @Override + public long writeTo(final WritableMemory memory, final long startPosition) + { + long currentPosition = startPosition; + + memory.putByte(currentPosition, typeCode); + ++currentPosition; + + currentPosition += cumulativeRowLengths.writeTo(memory, currentPosition); + currentPosition += rowNullityData.writeTo(memory, currentPosition); + currentPosition += rowData.writeTo(memory, currentPosition); + + return currentPosition - startPosition; + } + + @Override + public void close() + { + cumulativeRowLengths.close(); + rowNullityData.close(); + rowData.close(); + } +} diff --git a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java index abf6346aec81..61b01ad0646e 100644 --- a/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java +++ b/processing/src/test/java/org/apache/druid/frame/write/FrameWriterTest.java @@ -244,27 +244,18 @@ public void test_long() @Test public void test_arrayLong() { - // ARRAY can't be read or written for columnar frames, therefore skip the check if it encounters those - // parameters - // Assume.assumeFalse(inputFrameType == FrameType.COLUMNAR || outputFrameType == FrameType.COLUMNAR); testWithDataset(FrameWriterTestData.TEST_ARRAYS_LONG); } @Test public void test_arrayFloat() { - // ARRAY can't be read or written for columnar frames, therefore skip the check if it encounters those - // parameters - Assume.assumeFalse(inputFrameType == FrameType.COLUMNAR || outputFrameType == FrameType.COLUMNAR); testWithDataset(FrameWriterTestData.TEST_ARRAYS_FLOAT); } @Test public void test_arrayDouble() { - // ARRAY can't be read or written for columnar frames, therefore skip the check if it encounters those - // parameters - Assume.assumeFalse(inputFrameType == FrameType.COLUMNAR || outputFrameType == FrameType.COLUMNAR); testWithDataset(FrameWriterTestData.TEST_ARRAYS_DOUBLE); } From 37868af705200c6a983091c9229d946689c0f872 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Fri, 16 Feb 2024 04:34:27 +0530 Subject: [PATCH 08/11] tests --- .../frame/read/columnar/NumericArrayFrameColumnReader.java | 4 ++-- .../druid/frame/read/columnar/StringFrameColumnReader.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java index 5eecfc0f1e27..6825715a18ea 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java @@ -111,7 +111,7 @@ private static long getStartOfRowData(final Memory memory, final int numRows) .getAdjustedCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1)); } - public static abstract class NumericArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn + public abstract static class NumericArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn { private final Frame frame; @@ -308,6 +308,6 @@ private boolean getElementNullity(final int cumulativeIndex) return false; } - abstract Number getElement(final Memory memory, final long rowDataOffset, final int cumulativeIndex); + abstract Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex); } } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java index 9a9ffedf561f..aafd1d411d34 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java @@ -202,7 +202,7 @@ private static long getStartOfStringDataSection( if (multiValue) { totalNumValues = FrameColumnReaderUtils.adjustCumulativeRowLength( - FrameColumnReaderUtils.getCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), -1) + FrameColumnReaderUtils.getCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1) ); } else { totalNumValues = numRows; From 07ee3218700f0404309a451af2fddae3cde451f6 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 19 Feb 2024 09:33:47 +0530 Subject: [PATCH 09/11] build fix --- .../DoubleArrayFrameColumnReader.java | 3 ++- .../columnar/FloatArrayFrameColumnReader.java | 3 ++- .../columnar/LongArrayFrameColumnReader.java | 3 ++- .../NumericArrayFrameColumnReader.java | 25 +++++++++++-------- .../NumericArrayFrameColumnWriter.java | 8 +++--- .../druid/frame/write/FrameWritersTest.java | 15 ++++++++--- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java index e4a401fcc8e8..086442f1d72c 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java @@ -19,6 +19,7 @@ package org.apache.druid.frame.read.columnar; +import com.google.common.math.LongMath; import org.apache.datasketches.memory.Memory; import org.apache.druid.frame.Frame; import org.apache.druid.frame.write.columnar.FrameColumnWriters; @@ -51,7 +52,7 @@ public DoubleArrayFrameColumn( @Override Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex) { - return memory.getDouble(rowDataOffset + (long) cumulativeIndex * Double.BYTES); + return memory.getDouble(LongMath.checkedAdd(rowDataOffset, (long) cumulativeIndex * Double.BYTES)); } } } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java index 83e065d0916e..c15e9bcfe477 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java @@ -19,6 +19,7 @@ package org.apache.druid.frame.read.columnar; +import com.google.common.math.LongMath; import org.apache.datasketches.memory.Memory; import org.apache.druid.frame.Frame; import org.apache.druid.frame.write.columnar.FrameColumnWriters; @@ -51,7 +52,7 @@ public FloatArrayFrameColumn( @Override Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex) { - return memory.getFloat(rowDataOffset + (long) cumulativeIndex * Float.BYTES); + return memory.getFloat(LongMath.checkedAdd(rowDataOffset, (long) cumulativeIndex * Float.BYTES)); } } } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java index f5e1216e2f97..81073f31e1d0 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java @@ -19,6 +19,7 @@ package org.apache.druid.frame.read.columnar; +import com.google.common.math.LongMath; import org.apache.datasketches.memory.Memory; import org.apache.druid.frame.Frame; import org.apache.druid.frame.write.columnar.FrameColumnWriters; @@ -51,7 +52,7 @@ public LongArrayFrameColumn( @Override Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex) { - return memory.getLong(rowDataOffset + (long) cumulativeIndex * Long.BYTES); + return memory.getLong(LongMath.checkedAdd(rowDataOffset, (long) cumulativeIndex * Long.BYTES)); } } } diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java index 6825715a18ea..02f421f1d441 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java @@ -19,6 +19,7 @@ package org.apache.druid.frame.read.columnar; +import com.google.common.math.LongMath; import it.unimi.dsi.fastutil.objects.ObjectArrays; import org.apache.datasketches.memory.Memory; import org.apache.druid.error.DruidException; @@ -39,7 +40,6 @@ import org.apache.druid.segment.vector.VectorObjectSelector; import javax.annotation.Nullable; -import java.io.IOException; import java.util.Comparator; public abstract class NumericArrayFrameColumnReader implements FrameColumnReader @@ -82,13 +82,13 @@ private void validate(final Memory region) if (region.getCapacity() < NumericArrayFrameColumnWriter.DATA_OFFSET) { throw DruidException.defensive("Column[%s] is not big enough for a header", columnNumber); } - final byte typeCode = region.getByte(0); - if (typeCode != this.typeCode) { + final byte actualTypeCode = region.getByte(0); + if (actualTypeCode != this.typeCode) { throw DruidException.defensive( "Column[%s] does not have the correct type code; expected[%s], got[%s]", columnNumber, this.typeCode, - typeCode + actualTypeCode ); } } @@ -105,10 +105,13 @@ private static long getStartOfRowNullityData(final int numRows) private static long getStartOfRowData(final Memory memory, final int numRows) { - return getStartOfRowNullityData(numRows) - + (Byte.BYTES - * FrameColumnReaderUtils - .getAdjustedCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1)); + long nullityDataOffset = + (long) Byte.BYTES * FrameColumnReaderUtils.getAdjustedCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + numRows - 1 + ); + return LongMath.checkedAdd(getStartOfRowNullityData(numRows), nullityDataOffset); } public abstract static class NumericArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn @@ -250,9 +253,9 @@ private void computeVector() } @Override - public void close() throws IOException + public void close() { - + // Do nothing } private int physicalRow(int logicalRow) @@ -300,7 +303,7 @@ private Object[] getNumericArray(final int physicalRow) private boolean getElementNullity(final int cumulativeIndex) { - byte b = memory.getByte(rowNullityDataOffset + cumulativeIndex * Byte.BYTES); + byte b = memory.getByte(LongMath.checkedAdd(rowNullityDataOffset, (long) cumulativeIndex * Byte.BYTES)); if (b == NumericArrayFrameColumnWriter.NULL_ELEMENT_MARKER) { return true; } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java index 2e01d9338673..98be3879b1c4 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java @@ -46,7 +46,6 @@ public abstract class NumericArrayFrameColumnWriter implements FrameColumnWriter public static final long DATA_OFFSET = 1; final ColumnValueSelector selector; - final MemoryAllocator allocator; final byte typeCode; /** @@ -77,7 +76,6 @@ public NumericArrayFrameColumnWriter( ) { this.selector = selector; - this.allocator = allocator; this.typeCode = typeCode; this.cumulativeRowLengths = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); this.rowNullityData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); @@ -130,10 +128,12 @@ public boolean addSelection() final Number element = numericArray.get(i); final long memoryOffset = rowDataCursor.start() + ((long) elementSizeBytes() * i); if (element == null) { - rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NULL_ELEMENT_MARKER); + rowNullityDataCursor.memory() + .putByte(rowNullityDataCursor.start() + (long) Byte.BYTES * i, NULL_ELEMENT_MARKER); putNull(rowDataCursor.memory(), memoryOffset); } else { - rowNullityDataCursor.memory().putByte(rowNullityDataCursor.start() + Byte.BYTES * i, NON_NULL_ELEMENT_MARKER); + rowNullityDataCursor.memory() + .putByte(rowNullityDataCursor.start() + (long) Byte.BYTES * i, NON_NULL_ELEMENT_MARKER); putArrayElement(rowDataCursor.memory(), memoryOffset, element); } } diff --git a/processing/src/test/java/org/apache/druid/frame/write/FrameWritersTest.java b/processing/src/test/java/org/apache/druid/frame/write/FrameWritersTest.java index 9d359eed05e0..71c67deadb02 100644 --- a/processing/src/test/java/org/apache/druid/frame/write/FrameWritersTest.java +++ b/processing/src/test/java/org/apache/druid/frame/write/FrameWritersTest.java @@ -67,7 +67,16 @@ public void test_columnar() final FrameWriterFactory factory = FrameWriters.makeFrameWriterFactory( FrameType.COLUMNAR, new ArenaMemoryAllocatorFactory(ALLOCATOR_CAPACITY), - RowSignature.builder().add("x", ColumnType.LONG).build(), + RowSignature.builder() + .add("a", ColumnType.LONG) + .add("b", ColumnType.FLOAT) + .add("c", ColumnType.DOUBLE) + .add("d", ColumnType.STRING) + .add("e", ColumnType.LONG_ARRAY) + .add("f", ColumnType.FLOAT_ARRAY) + .add("g", ColumnType.DOUBLE_ARRAY) + .add("h", ColumnType.STRING_ARRAY) + .build(), Collections.emptyList() ); @@ -81,7 +90,7 @@ public void test_columnar_unsupportedColumnType() final FrameWriterFactory factory = FrameWriters.makeFrameWriterFactory( FrameType.COLUMNAR, new ArenaMemoryAllocatorFactory(ALLOCATOR_CAPACITY), - RowSignature.builder().add("x", ColumnType.LONG_ARRAY).build(), + RowSignature.builder().add("x", ColumnType.ofArray(ColumnType.LONG_ARRAY)).build(), Collections.emptyList() ); @@ -91,7 +100,7 @@ public void test_columnar_unsupportedColumnType() ); Assert.assertEquals("x", e.getColumnName()); - Assert.assertEquals(ColumnType.LONG_ARRAY, e.getColumnType()); + Assert.assertEquals(ColumnType.ofArray(ColumnType.LONG_ARRAY), e.getColumnType()); } @Test From 75d933d06b73dc4bd492918503d97a231cb33593 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Mon, 19 Feb 2024 11:00:42 +0530 Subject: [PATCH 10/11] add comments --- .../DoubleArrayFrameColumnReader.java | 6 +++ .../columnar/FloatArrayFrameColumnReader.java | 6 +++ .../columnar/LongArrayFrameColumnReader.java | 6 +++ .../NumericArrayFrameColumnReader.java | 46 ++++++++++++++++++- .../DoubleArrayFrameColumnWriter.java | 3 ++ .../columnar/FloatArrayFrameColumnWriter.java | 3 ++ .../write/columnar/FrameColumnWriter.java | 20 ++++++++ .../columnar/LongArrayFrameColumnWriter.java | 3 ++ .../NumericArrayFrameColumnWriter.java | 30 ++++++++++++ 9 files changed, 122 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java index 086442f1d72c..67750616fe5f 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java @@ -25,6 +25,12 @@ import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.segment.column.ColumnType; +/** + * Reaaers for columns written by {@link org.apache.druid.frame.write.columnar.DoubleArrayFrameColumnWriter} + * + * @see NumericArrayFrameColumnReader + * @see org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter for column's layout in memory + */ public class DoubleArrayFrameColumnReader extends NumericArrayFrameColumnReader { public DoubleArrayFrameColumnReader(int columnNumber) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java index c15e9bcfe477..95d3dc5648dc 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java @@ -25,6 +25,12 @@ import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.segment.column.ColumnType; +/** + * Reaaers for columns written by {@link org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter} + * + * @see NumericArrayFrameColumnReader + * @see org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter for column's layout in memory + */ public class FloatArrayFrameColumnReader extends NumericArrayFrameColumnReader { public FloatArrayFrameColumnReader(int columnNumber) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java index 81073f31e1d0..0bcc50b86d0d 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java @@ -25,6 +25,12 @@ import org.apache.druid.frame.write.columnar.FrameColumnWriters; import org.apache.druid.segment.column.ColumnType; +/** + * Reaaers for columns written by {@link org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter} + * + * @see NumericArrayFrameColumnReader + * @see org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter for column's layout in memory + */ public class LongArrayFrameColumnReader extends NumericArrayFrameColumnReader { public LongArrayFrameColumnReader(int columnNumber) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java index 02f421f1d441..986baaa099d9 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/NumericArrayFrameColumnReader.java @@ -42,6 +42,11 @@ import javax.annotation.Nullable; import java.util.Comparator; +/** + * Implementations of this class reads columns written by the corresponding implementations of {@link NumericArrayFrameColumnWriter}. + * + * @see NumericArrayFrameColumnWriter for the column format read + */ public abstract class NumericArrayFrameColumnReader implements FrameColumnReader { private final byte typeCode; @@ -77,6 +82,10 @@ public ColumnPlus readColumn(Frame frame) abstract NumericArrayFrameColumn column(Frame frame, Memory memory, ColumnType columnType); + /** + * Validates that the written type code is the same as the provided type code. It's a defensive check that prevents + * unexpected results by reading columns of different types + */ private void validate(final Memory region) { if (region.getCapacity() < NumericArrayFrameColumnWriter.DATA_OFFSET) { @@ -93,16 +102,25 @@ private void validate(final Memory region) } } + /** + * Gets the start of the section where cumulative lengths of the array elements are stored (section 1) + */ private static long getStartOfCumulativeLengthSection() { return NumericArrayFrameColumnWriter.DATA_OFFSET; } + /** + * Gets the start of the section where information about element's nullity is stored (section 2) + */ private static long getStartOfRowNullityData(final int numRows) { return getStartOfCumulativeLengthSection() + ((long) numRows * Integer.BYTES); } + /** + * Gets the start of the section where elements are stored (section 3) + */ private static long getStartOfRowData(final Memory memory, final int numRows) { long nullityDataOffset = @@ -116,12 +134,18 @@ private static long getStartOfRowData(final Memory memory, final int numRows) public abstract static class NumericArrayFrameColumn extends ObjectColumnAccessorBase implements BaseColumn { - private final Frame frame; private final Memory memory; private final ColumnType columnType; + /** + * Cache start of rowNullityDataOffset, as it won't change + */ private final long rowNullityDataOffset; + + /** + * Cache start of rowDataOffset, as it won't change + */ private final long rowDataOffset; @@ -162,9 +186,14 @@ protected Comparator getComparator() @Override public ColumnValueSelector makeColumnValueSelector(ReadableOffset offset) { + // Cache's the row's value before returning return new ObjectColumnSelector() { + + // Cached row number private int cachedLogicalRow = -1; + + // Cached value @Nullable private Object[] cachedValue = null; @@ -187,6 +216,9 @@ public Class classOfObject() return Object[].class; } + /** + * Cache's the row value and the logical row number into the class variables + */ private void compute() { int currentLogicalRow = offset.getOffset(); @@ -263,6 +295,9 @@ private int physicalRow(int logicalRow) return frame.physicalRow(logicalRow); } + /** + * Given the physical row, it fetches the value from the memory + */ @Nullable private Object[] getNumericArray(final int physicalRow) { @@ -285,6 +320,8 @@ private Object[] getNumericArray(final int physicalRow) physicalRow - 1 ) ); + // cumulativeLength doesn't need to be adjusted, since its greater than 0 or else it would have been a null row, + // which we check for in the first if..else rowLength = cumulativeLength - previousCumulativeLength; } @@ -301,6 +338,9 @@ private Object[] getNumericArray(final int physicalRow) return row; } + /** + * Returns true if element is null, else false + */ private boolean getElementNullity(final int cumulativeIndex) { byte b = memory.getByte(LongMath.checkedAdd(rowNullityDataOffset, (long) cumulativeIndex * Byte.BYTES)); @@ -311,6 +351,10 @@ private boolean getElementNullity(final int cumulativeIndex) return false; } + /** + * Returns the value of the element of the array in the memory provided, given that the start of the array is + * {@code rowDataOffset} and the index of the element in the array is {@code cumulativeIndex} + */ abstract Number getElement(Memory memory, long rowDataOffset, int cumulativeIndex); } } diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java index 7415887f4fc0..80c1b5ebfdf6 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/DoubleArrayFrameColumnWriter.java @@ -23,6 +23,9 @@ import org.apache.druid.frame.allocation.MemoryAllocator; import org.apache.druid.segment.ColumnValueSelector; +/** + * Columnar frame writer for {@link org.apache.druid.segment.column.ColumnType#DOUBLE_ARRAY} columns + */ public class DoubleArrayFrameColumnWriter extends NumericArrayFrameColumnWriter { public DoubleArrayFrameColumnWriter( diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java index 166afc6fc0d9..47c492c28bba 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FloatArrayFrameColumnWriter.java @@ -23,6 +23,9 @@ import org.apache.druid.frame.allocation.MemoryAllocator; import org.apache.druid.segment.ColumnValueSelector; +/** + * Columnar frame writer for {@link org.apache.druid.segment.column.ColumnType#FLOAT_ARRAY} columns + */ public class FloatArrayFrameColumnWriter extends NumericArrayFrameColumnWriter { public FloatArrayFrameColumnWriter( diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriter.java index bb954d6c366a..0a91a58bd830 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/FrameColumnWriter.java @@ -23,14 +23,34 @@ import java.io.Closeable; +/** + * Represent writers for the columnar frames. + * + * The class objects must be provided with information on what values to write, usually provided as a + * {@link org.apache.druid.segment.ColumnValueSelector} and where to write to, usually temporary growable memory + * {@link #addSelection()} will be called repeatedly, as the current value to write gets updated. For the final write, + * call {@link #writeTo}, which will copy the values we have added so far to the destination memory. + */ public interface FrameColumnWriter extends Closeable { + /** + * Adds the current value to the writer + */ boolean addSelection(); + /** + * Reverts the last added value. Undo calls cannot be called in successsion + */ void undo(); + /** + * Size (in bytes) of the column data that will get written when {@link #writeTo} will be called + */ long size(); + /** + * Writes the value of the column to the provided memory at the given position + */ long writeTo(WritableMemory memory, long position); @Override diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java index 495de57f4778..cc26fd3a36a1 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/LongArrayFrameColumnWriter.java @@ -23,6 +23,9 @@ import org.apache.druid.frame.allocation.MemoryAllocator; import org.apache.druid.segment.ColumnValueSelector; +/** + * Columnar frame writer for {@link org.apache.druid.segment.column.ColumnType#LONG_ARRAY} columns + */ public class LongArrayFrameColumnWriter extends NumericArrayFrameColumnWriter { public LongArrayFrameColumnWriter( diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java index 98be3879b1c4..619bf53b8d3d 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/NumericArrayFrameColumnWriter.java @@ -29,6 +29,25 @@ import java.util.List; +/** + * Parent class for the family of writers writing numeric arrays in columnar frames. Since the numeric primitives are + * fixed width, we don't need to store the width of each element. The memory layout of a column written by this writer + * is as follows: + * + * n : Total number of rows + * k : Total number of elements in all the rows, cumulative + * + * | Section | Length of the section | Denotion | + * |---------|-----------------------|--------------------------------------------------------------------------------------| + * | 0 | 1 | typeCode | + * | 1 | n * Integer.BYTES | n integers, where i-th integer represents the cumulative length of the array | + * | 2 | k * Byte.BYTES | k bytes, where i-th byte represent whether the i-th value from the start is null | + * | 3 | k * ELEMENT_SIZE | k values, each representing the element, or null equivalent value (e.g 0 for double) | + * + * Note on cumulative lengths stored in section 1: Cumulative lengths are stored so that its fast to offset into the + * elements of the array. We also use negative cumulative length to denote that the array itself is null (as opposed to + * individual elements being null, which we store in section 2) + */ public abstract class NumericArrayFrameColumnWriter implements FrameColumnWriter { /** @@ -82,10 +101,19 @@ public NumericArrayFrameColumnWriter( this.rowData = AppendableMemory.create(allocator, INITIAL_ALLOCATION_SIZE); } + /** + * Returns the size of the elements of the array + */ abstract int elementSizeBytes(); + /** + * Inserts default null value in the given memory location at the provided offset. + */ abstract void putNull(WritableMemory memory, long offset); + /** + * Inserts the element value in the given memory location at the provided offset. + */ abstract void putArrayElement(WritableMemory memory, long offset, Number element); @Override @@ -94,6 +122,7 @@ public boolean addSelection() List numericArray = FrameWriterUtils.getNumericArrayFromObject(selector.getObject()); int rowLength = numericArray == null ? 0 : numericArray.size(); + // Begin memory allocations before writing if ((long) lastCumulativeRowLength + rowLength > Integer.MAX_VALUE) { return false; } @@ -109,6 +138,7 @@ public boolean addSelection() if (!rowData.reserveAdditional(rowLength * elementSizeBytes())) { return false; } + // Memory allocations completed final MemoryRange rowLengthsCursor = cumulativeRowLengths.cursor(); From cf293f4423351c84fe4392ad678136b78eea5a08 Mon Sep 17 00:00:00 2001 From: Laksh Singla Date: Tue, 20 Feb 2024 03:30:44 +0530 Subject: [PATCH 11/11] review comments --- .../columnar/DoubleArrayFrameColumnReader.java | 2 +- .../columnar/FloatArrayFrameColumnReader.java | 2 +- .../columnar/LongArrayFrameColumnReader.java | 2 +- .../read/columnar/StringFrameColumnReader.java | 17 +++++++++-------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java index 67750616fe5f..909f2c5b3641 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/DoubleArrayFrameColumnReader.java @@ -26,7 +26,7 @@ import org.apache.druid.segment.column.ColumnType; /** - * Reaaers for columns written by {@link org.apache.druid.frame.write.columnar.DoubleArrayFrameColumnWriter} + * Reader for columns written by {@link org.apache.druid.frame.write.columnar.DoubleArrayFrameColumnWriter} * * @see NumericArrayFrameColumnReader * @see org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter for column's layout in memory diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java index 95d3dc5648dc..ea1ffbdd060a 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/FloatArrayFrameColumnReader.java @@ -26,7 +26,7 @@ import org.apache.druid.segment.column.ColumnType; /** - * Reaaers for columns written by {@link org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter} + * Reader for columns written by {@link org.apache.druid.frame.write.columnar.FloatArrayFrameColumnWriter} * * @see NumericArrayFrameColumnReader * @see org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter for column's layout in memory diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java index 0bcc50b86d0d..898e1f1cebb1 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/LongArrayFrameColumnReader.java @@ -26,7 +26,7 @@ import org.apache.druid.segment.column.ColumnType; /** - * Reaaers for columns written by {@link org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter} + * Reader for columns written by {@link org.apache.druid.frame.write.columnar.LongArrayFrameColumnWriter} * * @see NumericArrayFrameColumnReader * @see org.apache.druid.frame.write.columnar.NumericArrayFrameColumnWriter for column's layout in memory diff --git a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java index aafd1d411d34..d9fb9d83a9f4 100644 --- a/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java +++ b/processing/src/main/java/org/apache/druid/frame/read/columnar/StringFrameColumnReader.java @@ -201,8 +201,10 @@ private static long getStartOfStringDataSection( final int totalNumValues; if (multiValue) { - totalNumValues = FrameColumnReaderUtils.adjustCumulativeRowLength( - FrameColumnReaderUtils.getCumulativeRowLength(memory, getStartOfCumulativeLengthSection(), numRows - 1) + totalNumValues = FrameColumnReaderUtils.getAdjustedCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + numRows - 1 ); } else { totalNumValues = numRows; @@ -473,12 +475,11 @@ private Object getRowAsObject(final int physicalRow, final boolean decode) } else if (physicalRow == 0) { rowLength = cumulativeRowLength; } else { - rowLength = cumulativeRowLength - FrameColumnReaderUtils.adjustCumulativeRowLength( - FrameColumnReaderUtils.getCumulativeRowLength( - memory, - getStartOfCumulativeLengthSection(), - physicalRow - 1 - )); + rowLength = cumulativeRowLength - FrameColumnReaderUtils.getAdjustedCumulativeRowLength( + memory, + getStartOfCumulativeLengthSection(), + physicalRow - 1 + ); } if (rowLength == 0) {