diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 30ae54224ee3..18c23da6be0c 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -844,26 +844,26 @@ A description of the compaction config is: |`taskPriority`|[Priority](../ingestion/tasks.md#priority) of compaction task.|no (default = 25)| |`inputSegmentSizeBytes`|Maximum number of total segment bytes processed per compaction task. Since a time chunk must be processed in its entirety, if the segments for a particular time chunk have a total size in bytes greater than this parameter, compaction will not run for that time chunk. Because each compaction task runs with a single thread, setting this value too far above 1–2GB will result in compaction tasks taking an excessive amount of time.|no (default = 419430400)| |`maxRowsPerSegment`|Max number of rows per segment after compaction.|no| -|`skipOffsetFromLatest`|The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources. |no (default = "P1D")| -|`tuningConfig`|Tuning config for compaction tasks. See below [Compaction Task TuningConfig](#compaction-tuningconfig).|no| +|`skipOffsetFromLatest`|The offset for searching segments to be compacted in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) duration format. Strongly recommended to set for realtime dataSources. See [Data handling with compaction](../ingestion/compaction.md#data-handling-with-compaction)|no (default = "P1D")| +|`tuningConfig`|Tuning config for compaction tasks. See below [Compaction Task TuningConfig](#automatic-compaction-tuningconfig).|no| |`taskContext`|[Task context](../ingestion/tasks.md#context) for compaction tasks.|no| +|`granularitySpec`|Custom `granularitySpec` to describe the `segmentGranularity` for the compacted segments. See [Automatic compaction granularitySpec](#automatic-compaction-granularityspec)|No| An example of compaction config is: ```json { - "dataSource": "wikiticker" + "dataSource": "wikiticker", + "granularitySpec" : { + "segmentGranularity : "none" + } } ``` -Note that compaction tasks can fail if their locks are revoked by other tasks of higher priorities. -Since realtime tasks have a higher priority than compaction task by default, -it can be problematic if there are frequent conflicts between compaction tasks and realtime tasks. -If this is the case, the coordinator's automatic compaction might get stuck because of frequent compaction task failures. -This kind of problem may happen especially in Kafka/Kinesis indexing systems which allow late data arrival. -If you see this problem, it's recommended to set `skipOffsetFromLatest` to some large enough value to avoid such conflicts between compaction tasks and realtime tasks. +Compaction tasks fail when higher priority tasks cause Druid to revoke their locks. By default, realtime tasks like ingestion have a higher priority than compaction tasks. Therefore frequent conflicts between compaction tasks and realtime tasks can cause the coordinator's automatic compaction to get stuck. +You may see this issue with streaming ingestion from Kafka and Kinesis, which ingest late-arriving data. To mitigate this problem, set `skipOffsetFromLatest` to a value large enough so that arriving data tends to fall outside the offset value from the current time. This way you can avoid conflicts between compaction tasks and realtime ingestion tasks. -###### Compaction TuningConfig +###### Automatic compaction TuningConfig Auto compaction supports a subset of the [tuningConfig for Parallel task](../ingestion/native-batch.md#tuningconfig). The below is a list of the supported configurations for auto compaction. @@ -888,6 +888,17 @@ The below is a list of the supported configurations for auto compaction. |`chatHandlerTimeout`|Timeout for reporting the pushed segments in worker tasks.|no (default = PT10S)| |`chatHandlerNumRetries`|Retries for reporting the pushed segments in worker tasks.|no (default = 5)| +###### Automatic compaction granularitySpec +You can optionally use the `granularitySpec` object to configure the segment granularity of the compacted segments. + +`granularitySpec` takes the following keys: + +|Field|Description|Required| +|-----|-----------|--------| +|`segmentGranularity`|Time chunking period for the segment granularity. Defaults to 'null', which preserves the original segment granularity. Accepts all [Query granularity](../querying/granularities.md) values.|No| + +> Unlike manual compaction, automatic compaction does not support query granularity. + ### Overlord For general Overlord Process information, see [here](../design/overlord.md). diff --git a/docs/ingestion/compaction.md b/docs/ingestion/compaction.md new file mode 100644 index 000000000000..e50d2625e536 --- /dev/null +++ b/docs/ingestion/compaction.md @@ -0,0 +1,227 @@ +--- +id: compaction +title: "Compaction" +description: "Defines compaction and automatic compaction (auto-compaction or autocompaction) for segment optimization. Use cases and strategies for compaction. Describes compaction task configuration." +--- + + +Query performance in Apache Druid depends on optimally sized segments. Compaction is one strategy you can use to optimize segment size for your Druid database. Compaction tasks read an existing set of segments for a given time interval and combine the data into a new "compacted" set of segments. In some cases the compacted segments are larger, but there are fewer of them. In other cases the compacted segments may be smaller. Compaction tends to increase performance because optimized segments require less per-segment processing and less memory overhead for ingestion and for querying paths. + +## Compaction strategies +There are several cases to consider compaction for segment optimization: +- With streaming ingestion, data can arrive out of chronological order creating lots of small segments. +- If you append data using `appendToExisting` for [native batch](native-batch.md) ingestion creating suboptimal segments. +- When you use `index_parallel` for parallel batch indexing and the parallel ingestion tasks create many small segments. +- When a misconfigured ingestion task creates oversized segments. + +By default, compaction does not modify the underlying data of the segments. However, there are cases when you may want to modify data during compaction to improve query performance: +- If, after ingestion, you realize that data for the time interval is sparse, you can use compaction to increase the segment granularity. +- Over time you don't need fine-grained granularity for older data so you want use compaction to change older segments to a coarser query granularity. This reduces the storage space required for older data. For example from `minute` to `hour`, or `hour` to `day`. You cannot go from coarser granularity to finer granularity. +- You can change the dimension order to improve sorting and reduce segment size. +- You can remove unused columns in compaction or implement an aggregation metric for older data. +- You can change segment rollup from dynamic partitioning with best-effort rollup to hash or range partitioning with perfect rollup. For more information on rollup, see [perfect vs best-effort rollup](index.md#perfect-rollup-vs-best-effort-rollup). + +Compaction does not improve performance in all situations. For example, if you rewrite your data with each ingestion task, you don't need to use compaction. See [Segment optimization](../operations/segment-optimization.md) for additional guidance to determine if compaction will help in your environment. + +## Types of compaction +You can configure the Druid Coordinator to perform automatic compaction, also called auto-compaction, for a datasource. Using a segment search policy, the coordinator periodically identifies segments for compaction starting with the newest to oldest. When it discovers segments that have not been compacted or segments that were compacted with a different or changed spec, it submits compaction task for those segments and only those segments. + +Automatic compaction works in most use cases and should be your first option. To learn more about automatic compaction, see [Compacting Segments](../design/coordinator.md#compacting-segments). + +In cases where you require more control over compaction, you can manually submit compaction tasks. For example: +- Automatic compaction is running into the limit of task slots available to it, so tasks are waiting for previous automatic compaction tasks to complete. Manual compaction can use all available task slots, therefore you can complete compaction more quickly by submitting more concurrent tasks for more intervals. +- You want to force compaction for a specific time range or you want to compact data out of chronological order. + +See [Setting up a manual compaction task](#setting-up-manual-compaction) for more about manual compaction tasks. + +## Data handling with compaction +During compaction, Druid overwrites the original set of segments with the compacted set. During compaction Druid locks the segments for the time interval being compacted to ensure data consistency. By default, compaction tasks do not modify the underlying data. You can configure the compaction task to change the query granularity or add or remove dimensions in the compaction task. This means that the only changes to query results should be the result of intentional, not automatic, changes. + +If an ingestion task needs to write data to a segment for a time interval locked for compaction, by default the ingestion task supersedes the compaction task and the compaction task fails without finishing. For manual compaction tasks you can adjust the input spec interval to avoid conflicts between ingestion and compaction. For automatic compaction, you can set the `skipOffsetFromLatest` key to adjustment the auto compaction starting point from the current time to reduce the chance of conflicts between ingestion and compaction. See [Compaction dynamic configuration](../configuration/index.md#compaction-dynamic-configuration) for more information. Another option is to set the compaction task to higher priority than the ingestion task. + +### Segment granularity handling + +Unless you modify the segment granularity in the [granularity spec](#compaction-granularity-spec), Druid attempts to retain the granularity for the compacted segments. When segments have different segment granularities with no overlap in interval Druid creates a separate compaction task for each to retain the segment granularity in the compacted segment. + +If segments have different segment granularities before compaction but there is some overlap in interval, Druid attempts find start and end of the overlapping interval and uses the closest segment granularity level for the compacted segment. For example consider two overlapping segments: segment "A" for the interval 01/01/2021-01/02/2021 with day granularity and segment "B" for the interval 01/01/2021-02/01/2021. Druid attempts to combine and compacted the overlapped segments. In this example, the earliest start time for the two segments above is 01/01/2020 and the latest end time of the two segments above is 02/01/2020. Druid compacts the segments together even though they have different segment granularity. Druid uses month segment granularity for the newly compacted segment even though segment A's original segment granularity was DAY. + +### Query granularity handling + +Unless you modify the query granularity in the [granularity spec](#compaction-granularity-spec), Druid retains the query granularity for the compacted segments. If segments have different query granularities before compaction, Druid chooses the finest level of granularity for the resulting compacted segment. For example if a compaction task combines two segments, one with day query granularity and one with minute query granularity, the resulting segment uses minute query granularity. + +> In Apache Druid 0.21.0 and prior, Druid sets the granularity for compacted segments to the default granularity of `NONE` regardless of the query granularity of the original segments. + +If you configure query granularity in compaction to go from a finer granularity like month to a coarser query granularity like year, then Druid overshadows the original segment with coarser granularity. Because the new segments have a coarser granularity, running a kill task to remove the overshadowed segments for those intervals will cause you to permanently lose the finer granularity data. + +### Dimension handling +Apache Druid supports schema changes. Therefore, dimensions can be different across segments even if they are a part of the same data source. See [Different schemas among segments](../design/segments.md#different-schemas-among-segments). If the input segments have different dimensions, the resulting compacted segment include all dimensions of the input segments. + +Even when the input segments have the same set of dimensions, the dimension order or the data type of dimensions can be different. The dimensions of recent segments precede that of old segments in terms of data types and the ordering because more recent segments are more likely to have the preferred order and data types. + +If you want to control dimension ordering or ensure specific values for dimension types, you can configure a custom `dimensionsSpec` in the compaction task spec. + +### Rollup +Druid only rolls up the output segment when `rollup` is set for all input segments. +See [Roll-up](../ingestion/index.md#rollup) for more details. +You can check that your segments are rolled up or not by using [Segment Metadata Queries](../querying/segmentmetadataquery.md#analysistypes). + +## Setting up manual compaction + +To perform a manual compaction, you submit a compaction task. Compaction tasks merge all segments for the defined interval according to the following syntax: + +```json +{ + "type": "compact", + "id": , + "dataSource": , + "ioConfig": , + "dimensionsSpec" , + "metricsSpec" , + "tuningConfig" , + "granularitySpec" , + "context": +} +``` + +|Field|Description|Required| +|-----|-----------|--------| +|`type`|Task type. Should be `compact`|Yes| +|`id`|Task id|No| +|`dataSource`|Data source name to compact|Yes| +|`ioConfig`|I/O configuration for compaction task. See [Compaction I/O configuration](#compaction-io-configuration) for details.|Yes| +|`dimensionsSpec`|Custom dimensions spec. The compaction task uses the specified dimensions spec if it exists instead of generating one.|No| +|`metricsSpec`|Custom metrics spec. The compaction task uses the specified metrics spec rather than generating one.|No| +|`segmentGranularity`|When set, the compaction task changes the segment granularity for the given interval. Deprecated. Use `granularitySpec`. |No.| +|`tuningConfig`|[Parallel indexing task tuningConfig](native-batch.md#tuningconfig)|No| +|`context`|[Task context](./tasks.md#context)|No| +|`granularitySpec`|Custom `granularitySpec` to describe the `segmentGranularity` and `queryGranularity` for the compacted segments. See [Compaction granularitySpec](#compaction-granularity-spec).|No| + +> Note: Use `granularitySpec` over `segmentGranularity` and only set one of these values. If you specify different values for these in the same compaction spec, the task fails. + +To control the number of result segments per time chunk, you can set [maxRowsPerSegment](../configuration/index.md#compaction-dynamic-configuration) or [numShards](../ingestion/native-batch.md#tuningconfig). + +> You can run multiple compaction tasks in parallel. For example, if you want to compact the data for a year, you are not limited to running a single task for the entire year. You can run 12 compaction tasks with month-long intervals. + +A compaction task internally generates an `index` task spec for performing compaction work with some fixed parameters. For example, its `inputSource` is always the [DruidInputSource](native-batch.md#druid-input-source), and `dimensionsSpec` and `metricsSpec` include all dimensions and metrics of the input segments by default. + +Compaction tasks would exit without doing anything and issue a failure status code: +- if the interval you specify has no data segments loaded
+OR +- if the interval you specify is empty. + +Note that the metadata between input segments and the resulting compacted segments may differ if the metadata among the input segments differs as well. If all input segments have the same metadata, however, the resulting output segment will have the same metadata as all input segments. + + +### Example compaction task +The following JSON illustrates a compaction task to compact _all segments_ within the interval `2017-01-01/2018-01-01` and create new segments: + +```json +{ + "type" : "compact", + "dataSource" : "wikipedia", + "ioConfig" : { + "type": "compact", + "inputSpec": { + "type": "interval", + "interval": "2020-01-01/2021-01-01", + } + }, + "granularitySpec": { + "segmentGranularity":"day", + "queryGranularity":"hour" + } +} +``` + +This task doesn't specify a `granularitySpec` so Druid retains the original segment granularity unchanged when compaction is complete. + +### Compaction I/O configuration + +The compaction `ioConfig` requires specifying `inputSpec` as follows: + +|Field|Description|Required| +|-----|-----------|--------| +|`type`|Task type. Should be `compact`|Yes| +|`inputSpec`|Input specification|Yes| + +There are two supported `inputSpec`s for now. + +The interval `inputSpec` is: + +|Field|Description|Required| +|-----|-----------|--------| +|`type`|Task type. Should be `interval`|Yes| +|`interval`|Interval to compact|Yes| + +The segments `inputSpec` is: + +|Field|Description|Required| +|-----|-----------|--------| +|`type`|Task type. Should be `segments`|Yes| +|`segments`|A list of segment IDs|Yes| + +### Compaction granularity spec + +You can optionally use the `granularitySpec` object to configure the segment granularity and the query granularity of the compacted segments. Their syntax is as follows: +```json + "type": "compact", + "id": , + "dataSource": , + ... + , + "granularitySpec": { + "segmentGranularity": , + "queryGranularity": + } + ... +``` + +`granularitySpec` takes the following keys: + +|Field|Description|Required| +|-----|-----------|--------| +|`segmentGranularity`|Time chunking period for the segment granularity. Defaults to 'null', which preserves the original segment granularity. Accepts all [Query granularity](../querying/granularities.md) values.|No| +|`queryGranularity`|Time chunking period for the query granularity. Defaults to 'null', which preserves the original query granularity. Accepts all [Query granularity](../querying/granularities.md) values. Not supported for automatic compaction.|No| + +For example, to set the segment granularity to "day" and the query granularity to "hour": +```json +{ + "type" : "compact", + "dataSource" : "wikipedia", + "ioConfig" : { + "type": "compact", + "inputSpec": { + "type": "interval", + "interval": "2017-01-01/2018-01-01", + }, + "granularitySpec": { + "segmentGranularity":"day", + "queryGranularity":"hour" + } + } +} +``` + +## Learn more +See the following topics for more information: +- [Segment optimization](../operations/segment-optimization.md) for guidance to determine if compaction will help in your case. +- [Compacting Segments](../design/coordinator.md#compacting-segments) for more on automatic compaction. +- See [Compaction Configuration API](../operations/api-reference.md#compaction-configuration) +and [Compaction Configuration](../configuration/index.md#compaction-dynamic-configuration) for automatic compaction configuration information. diff --git a/docs/ingestion/data-management.md b/docs/ingestion/data-management.md index f0fddda8d23d..c9e592f5b0a0 100644 --- a/docs/ingestion/data-management.md +++ b/docs/ingestion/data-management.md @@ -21,175 +21,11 @@ title: "Data management" ~ specific language governing permissions and limitations ~ under the License. --> +Within the context of this topic data management refers to Apache Druid's data maintenance capabilities for existing datasources. There are several options to help you keep your data relevant and to help your Druid cluster remain performant. For example updating, reingesting, adding lookups, reindexing, or deleting data. +In addition to the tasks covered on this page, you can also use segment compaction to improve the layout of your existing data. Refer to [Segment optimization](../operations/segment-optimization.md) to see if compaction will help in your environment. For an overview and steps to configure manual compaction tasks, see [Compaction](./compaction.md). - - -## Schema changes - -Schemas for datasources can change at any time and Apache Druid supports different schemas among segments. - -### Replacing segments - -Druid uniquely -identifies segments using the datasource, interval, version, and partition number. The partition number is only visible in the segment id if -there are multiple segments created for some granularity of time. For example, if you have hourly segments, but you -have more data in an hour than a single segment can hold, you can create multiple segments for the same hour. These segments will share -the same datasource, interval, and version, but have linearly increasing partition numbers. - -``` -foo_2015-01-01/2015-01-02_v1_0 -foo_2015-01-01/2015-01-02_v1_1 -foo_2015-01-01/2015-01-02_v1_2 -``` - -In the example segments above, the dataSource = foo, interval = 2015-01-01/2015-01-02, version = v1, partitionNum = 0. -If at some later point in time, you reindex the data with a new schema, the newly created segments will have a higher version id. - -``` -foo_2015-01-01/2015-01-02_v2_0 -foo_2015-01-01/2015-01-02_v2_1 -foo_2015-01-01/2015-01-02_v2_2 -``` - -Druid batch indexing (either Hadoop-based or IndexTask-based) guarantees atomic updates on an interval-by-interval basis. -In our example, until all `v2` segments for `2015-01-01/2015-01-02` are loaded in a Druid cluster, queries exclusively use `v1` segments. -Once all `v2` segments are loaded and queryable, all queries ignore `v1` segments and switch to the `v2` segments. -Shortly afterwards, the `v1` segments are unloaded from the cluster. - -Note that updates that span multiple segment intervals are only atomic within each interval. They are not atomic across the entire update. -For example, you have segments such as the following: - -``` -foo_2015-01-01/2015-01-02_v1_0 -foo_2015-01-02/2015-01-03_v1_1 -foo_2015-01-03/2015-01-04_v1_2 -``` - -`v2` segments will be loaded into the cluster as soon as they are built and replace `v1` segments for the period of time the -segments overlap. Before v2 segments are completely loaded, your cluster may have a mixture of `v1` and `v2` segments. - -``` -foo_2015-01-01/2015-01-02_v1_0 -foo_2015-01-02/2015-01-03_v2_1 -foo_2015-01-03/2015-01-04_v1_2 -``` - -In this case, queries may hit a mixture of `v1` and `v2` segments. - -### Different schemas among segments - -Druid segments for the same datasource may have different schemas. If a string column (dimension) exists in one segment but not -another, queries that involve both segments still work. Queries for the segment missing the dimension will behave as if the dimension has only null values. -Similarly, if one segment has a numeric column (metric) but another does not, queries on the segment missing the -metric will generally "do the right thing". Aggregations over this missing metric behave as if the metric were missing. - - - -## Compaction and reindexing - -Compaction is a type of overwrite operation, which reads an existing set of segments, combines them into a new set with larger but fewer segments, and overwrites the original set with the new compacted set, without changing the data that is stored. - -For performance reasons, it is sometimes beneficial to compact a set of segments into a set of larger but fewer segments, as there is some per-segment processing and memory overhead in both the ingestion and querying paths. - -Compaction tasks merge all segments of the given interval. The syntax is: - -```json -{ - "type": "compact", - "id": , - "dataSource": , - "ioConfig": , - "dimensionsSpec" , - "metricsSpec" , - "segmentGranularity": , - "tuningConfig" , - "context": -} -``` - -|Field|Description|Required| -|-----|-----------|--------| -|`type`|Task type. Should be `compact`|Yes| -|`id`|Task id|No| -|`dataSource`|DataSource name to be compacted|Yes| -|`ioConfig`|ioConfig for compaction task. See [Compaction IOConfig](#compaction-ioconfig) for details.|Yes| -|`dimensionsSpec`|Custom dimensionsSpec. Compaction task will use this dimensionsSpec if exist instead of generating one. See below for more details.|No| -|`metricsSpec`|Custom metricsSpec. Compaction task will use this metricsSpec if specified rather than generating one.|No| -|`segmentGranularity`|If this is set, compactionTask will change the segment granularity for the given interval. See `segmentGranularity` of [`granularitySpec`](index.md#granularityspec) for more details. See the below table for the behavior.|No| -|`tuningConfig`|[Parallel indexing task tuningConfig](../ingestion/native-batch.md#tuningconfig)|No| -|`context`|[Task context](../ingestion/tasks.md#context)|No| - - -An example of compaction task is - -```json -{ - "type" : "compact", - "dataSource" : "wikipedia", - "ioConfig" : { - "type": "compact", - "inputSpec": { - "type": "interval", - "interval": "2017-01-01/2018-01-01" - } - } -} -``` - -This compaction task reads _all segments_ of the interval `2017-01-01/2018-01-01` and results in new segments. -Since `segmentGranularity` is null, the original segment granularity will be remained and not changed after compaction. -To control the number of result segments per time chunk, you can set [maxRowsPerSegment](../configuration/index.md#compaction-dynamic-configuration) or [numShards](../ingestion/native-batch.md#tuningconfig). -Please note that you can run multiple compactionTasks at the same time. For example, you can run 12 compactionTasks per month instead of running a single task for the entire year. - -A compaction task internally generates an `index` task spec for performing compaction work with some fixed parameters. -For example, its `inputSource` is always the [DruidInputSource](native-batch.md#druid-input-source), and `dimensionsSpec` and `metricsSpec` -include all dimensions and metrics of the input segments by default. - -Compaction tasks will exit with a failure status code, without doing anything, if the interval you specify has no -data segments loaded in it (or if the interval you specify is empty). - -The output segment can have different metadata from the input segments unless all input segments have the same metadata. - -- Dimensions: since Apache Druid supports schema change, the dimensions can be different across segments even if they are a part of the same dataSource. -If the input segments have different dimensions, the output segment basically includes all dimensions of the input segments. -However, even if the input segments have the same set of dimensions, the dimension order or the data type of dimensions can be different. For example, the data type of some dimensions can be -changed from `string` to primitive types, or the order of dimensions can be changed for better locality. -In this case, the dimensions of recent segments precede that of old segments in terms of data types and the ordering. -This is because more recent segments are more likely to have the new desired order and data types. If you want to use -your own ordering and types, you can specify a custom `dimensionsSpec` in the compaction task spec. -- Roll-up: the output segment is rolled up only when `rollup` is set for all input segments. -See [Roll-up](../ingestion/index.md#rollup) for more details. -You can check that your segments are rolled up or not by using [Segment Metadata Queries](../querying/segmentmetadataquery.md#analysistypes). - - -### Compaction IOConfig - -The compaction IOConfig requires specifying `inputSpec` as seen below. - -|Field|Description|Required| -|-----|-----------|--------| -|`type`|Task type. Should be `compact`|Yes| -|`inputSpec`|Input specification|Yes| - -There are two supported `inputSpec`s for now. - -The interval `inputSpec` is: - -|Field|Description|Required| -|-----|-----------|--------| -|`type`|Task type. Should be `interval`|Yes| -|`interval`|Interval to compact|Yes| - -The segments `inputSpec` is: - -|Field|Description|Required| -|-----|-----------|--------| -|`type`|Task type. Should be `segments`|Yes| -|`segments`|A list of segment IDs|Yes| - - -## Adding new data +## Adding new data to existing datasources Druid can insert new data to an existing datasource by appending new segments to existing segment sets. It can also add new data by merging an existing set of segments with new data and overwriting the original set. @@ -280,3 +116,8 @@ Druid also supports separating Historical processes into tiers, and the retentio These features are useful for performance/cost management; a common use case is separating Historical processes into a "hot" tier and a "cold" tier. For more information, please see [Load rules](../operations/rule-configuration.md). + +## Learn more +See the following topics for more information: +- [Compaction](./compaction.md) for an overview and steps to configure manual compaction tasks. +- [Segments](../design/segments.md) for information on how Druid handles segment versioning. diff --git a/docs/ingestion/index.md b/docs/ingestion/index.md index 75ea7031dfb4..ccc9f0b29ac6 100644 --- a/docs/ingestion/index.md +++ b/docs/ingestion/index.md @@ -196,7 +196,7 @@ that datasource leads to much faster query times. This can often be done with ju footprint, since abbreviated datasources tend to be substantially smaller. - If you are using a [best-effort rollup](#perfect-rollup-vs-best-effort-rollup) ingestion configuration that does not guarantee perfect rollup, you can potentially improve your rollup ratio by switching to a guaranteed perfect rollup option, or by -[reindexing](data-management.md#compaction-and-reindexing) your data in the background after initial ingestion. +[reindexing](data-management.md#reingesting-data) or [compacting](compaction.md) your data in the background after initial ingestion. ### Perfect rollup vs Best-effort rollup @@ -258,7 +258,7 @@ storage size decreases - and it also tends to improve query performance as well. Not all ingestion methods support an explicit partitioning configuration, and not all have equivalent levels of flexibility. As of current Druid versions, If you are doing initial ingestion through a less-flexible method (like -Kafka) then you can use [reindexing techniques](data-management.md#compaction-and-reindexing) to repartition your data after it +Kafka) then you can use [reindexing](data-management.md#reingesting-data) or [compaction](compaction.md) to repartition your data after it is initially ingested. This is a powerful technique: you can use it to ensure that any data older than a certain threshold is optimally partitioned, even as you continuously add new data from a stream. @@ -268,8 +268,8 @@ The following table shows how each ingestion method handles partitioning: |------|------------| |[Native batch](native-batch.md)|Configured using [`partitionsSpec`](native-batch.md#partitionsspec) inside the `tuningConfig`.| |[Hadoop](hadoop.md)|Configured using [`partitionsSpec`](hadoop.md#partitionsspec) inside the `tuningConfig`.| -|[Kafka indexing service](../development/extensions-core/kafka-ingestion.md)|Partitioning in Druid is guided by how your Kafka topic is partitioned. You can also [reindex](data-management.md#compaction-and-reindexing) to repartition after initial ingestion.| -|[Kinesis indexing service](../development/extensions-core/kinesis-ingestion.md)|Partitioning in Druid is guided by how your Kinesis stream is sharded. You can also [reindex](data-management.md#compaction-and-reindexing) to repartition after initial ingestion.| +|[Kafka indexing service](../development/extensions-core/kafka-ingestion.md)|Partitioning in Druid is guided by how your Kafka topic is partitioned. You can also [reindex](data-management.md#reingesting-data) or [compact](compaction.md) to repartition after initial ingestion.| +|[Kinesis indexing service](../development/extensions-core/kinesis-ingestion.md)|Partitioning in Druid is guided by how your Kinesis stream is sharded. You can also [reindex](data-management.md#reingesting-data) or [compact](compaction.md) to repartition after initial ingestion.| > Note that, of course, one way to partition data is to load it into separate datasources. This is a perfectly viable > approach and works very well when the number of datasources does not lead to excessive per-datasource overheads. If diff --git a/docs/ingestion/tasks.md b/docs/ingestion/tasks.md index 4fc21d37ca79..3b96e759a35c 100644 --- a/docs/ingestion/tasks.md +++ b/docs/ingestion/tasks.md @@ -389,7 +389,7 @@ Submitted automatically, on your behalf, by [Tranquility](tranquility.md). ### `compact` Compaction tasks merge all segments of the given interval. See the documentation on -[compaction](data-management.md#compaction-and-reindexing) for details. +[compaction](compaction.md) for details. ### `kill` diff --git a/docs/operations/basic-cluster-tuning.md b/docs/operations/basic-cluster-tuning.md index 1f7253c23f21..9f9351aa2aac 100644 --- a/docs/operations/basic-cluster-tuning.md +++ b/docs/operations/basic-cluster-tuning.md @@ -262,7 +262,7 @@ The total memory usage of the MiddleManager + Tasks: If you use the [Kafka Indexing Service](../development/extensions-core/kafka-ingestion.md) or [Kinesis Indexing Service](../development/extensions-core/kinesis-ingestion.md), the number of tasks required will depend on the number of partitions and your taskCount/replica settings. On top of those requirements, allocating more task slots in your cluster is a good idea, so that you have free task -slots available for other tasks, such as [compaction tasks](../ingestion/data-management.md#compact). +slots available for other tasks, such as [compaction tasks](../ingestion/compaction.md). ###### Hadoop ingestion diff --git a/docs/operations/segment-optimization.md b/docs/operations/segment-optimization.md index e0e909efb240..73d1212939a5 100644 --- a/docs/operations/segment-optimization.md +++ b/docs/operations/segment-optimization.md @@ -38,7 +38,7 @@ In Apache Druid, it's important to optimize the segment size because It would be best if you can optimize the segment size at ingestion time, but sometimes it's not easy especially when it comes to stream ingestion because the amount of data ingested might vary over time. In this case, -you can create segments with a sub-optimized size first and optimize them later. +you can create segments with a sub-optimized size first and optimize them later using [compaction](../ingestion/compaction.md). You may need to consider the followings to optimize your segments. @@ -96,3 +96,6 @@ Once you find your segments need compaction, you can consider the below two opti inputSpec to read from the segments generated by the Kafka indexing tasks. This might be helpful if you want to compact a lot of segments in parallel. Details on how to do this can be found on the [Updating existing data](../ingestion/data-management.md#update) section of the data management page. + +## Learn more +For an overview of compaction and how to submit a manual compaction task, see [Compaction](../ingestion/compaction.md). diff --git a/docs/tutorials/tutorial-compaction.md b/docs/tutorials/tutorial-compaction.md index 05d5b724af9d..89950fd68fce 100644 --- a/docs/tutorials/tutorial-compaction.md +++ b/docs/tutorials/tutorial-compaction.md @@ -141,12 +141,15 @@ We have included a compaction task spec that will create DAY granularity segment "type": "compact", "dataSource": "compaction-tutorial", "interval": "2015-09-12/2015-09-13", - "segmentGranularity": "DAY", "tuningConfig" : { "type" : "index_parallel", "maxRowsPerSegment" : 5000000, "maxRowsInMemory" : 25000, "forceExtendableShardSpecs" : true + }, + "granularitySpec" : { + "segmentGranularity" : "DAY", + "queryGranularity" : "none" } } ``` diff --git a/examples/quickstart/tutorial/compaction-day-granularity.json b/examples/quickstart/tutorial/compaction-day-granularity.json index eb39276d254a..0314edbca3a9 100644 --- a/examples/quickstart/tutorial/compaction-day-granularity.json +++ b/examples/quickstart/tutorial/compaction-day-granularity.json @@ -2,11 +2,14 @@ "type": "compact", "dataSource": "compaction-tutorial", "interval": "2015-09-12/2015-09-13", - "segmentGranularity": "DAY", "tuningConfig" : { "type" : "index_parallel", "maxRowsPerSegment" : 5000000, "maxRowsInMemory" : 25000, "forceExtendableShardSpecs" : true + }, + "granularitySpec" : { + "segmentGranularity" : "DAY", + "queryGranularity" : "none" } } diff --git a/website/.spelling b/website/.spelling index 50a22a561fff..f89b29a578f6 100644 --- a/website/.spelling +++ b/website/.spelling @@ -218,6 +218,7 @@ codec colocated colocation compactable +compactionTask config configs consumerProperties @@ -258,6 +259,7 @@ firehoses fromPigAvroStorage frontends granularities +granularitySpec gzip gzipped hadoop @@ -300,6 +302,7 @@ mergeable metadata millis misconfiguration +misconfigured mostAvailableSize multitenancy multitenant @@ -312,6 +315,7 @@ netflow non-nullable noop numerics +numShards parameterized parseable partitioner diff --git a/website/i18n/en.json b/website/i18n/en.json index f67380992222..ae666f4c94c4 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -270,6 +270,9 @@ "development/versioning": { "title": "Versioning" }, + "ingestion/compaction": { + "title": "Compaction" + }, "ingestion/data-formats": { "title": "Data formats" }, diff --git a/website/sidebars.json b/website/sidebars.json index 20c9692f117d..86ade793f815 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -34,6 +34,7 @@ "ingestion/data-formats", "ingestion/schema-design", "ingestion/data-management", + "ingestion/compaction", { "type": "subcategory", "label": "Stream ingestion",