diff --git a/distribution/pom.xml b/distribution/pom.xml index 8845d18a31c5..ce163c01a7e2 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -79,8 +79,6 @@ -c io.druid.extensions:druid-avro-extensions -c - io.druid.extensions:druid-caffeine-cache - -c io.druid.extensions:druid-datasketches -c io.druid.extensions:druid-hdfs-storage diff --git a/docs/_redirects.json b/docs/_redirects.json index bafee4a5ad83..0e6ef6ff9868 100644 --- a/docs/_redirects.json +++ b/docs/_redirects.json @@ -113,5 +113,6 @@ {"source": "tutorials/tutorial-all-about-queries.html", "target": "quickstart.html"}, {"source": "tutorials/tutorial-loading-batch-data.html", "target": "tutorial-batch.html"}, {"source": "tutorials/tutorial-loading-streaming-data.html", "target": "tutorial-streams.html"}, - {"source": "tutorials/tutorial-the-druid-cluster.html", "target": "cluster.html"} + {"source": "tutorials/tutorial-the-druid-cluster.html", "target": "cluster.html"}, + {"source": "development/extensions-core/caffeine-cache.html", "target":"../../configuration/caching.html"} ] diff --git a/docs/content/configuration/caching.md b/docs/content/configuration/caching.md index 59093595c0da..806cc1e244cc 100644 --- a/docs/content/configuration/caching.md +++ b/docs/content/configuration/caching.md @@ -24,6 +24,12 @@ for both broker and historical nodes, when defined in the common properties file #### Local Cache +
+DEPRECATED: Use caffeine instead +
+ +The local cache is deprecated in favor of the Caffeine cache, and may be removed in a future version of Druid. The Caffeine cache affords significantly better performance and control over eviction behavior compared to `local` cache, and is recommended in any situation where you are using JRE 8u60 or higher. + A simple in-memory LRU cache. Local cache resides in JVM heap memory, so if you enable it, make sure you increase heap size accordingly. |Property|Description|Default| @@ -32,6 +38,39 @@ A simple in-memory LRU cache. Local cache resides in JVM heap memory, so if you |`druid.cache.initialSize`|Initial size of the hashtable backing the cache.|500000| |`druid.cache.logEvictionCount`|If non-zero, log cache eviction every `logEvictionCount` items.|0| +### Caffeine Cache + +A highly performant local cache implementation for Druid based on [Caffeine](https://github.com/ben-manes/caffeine). Requires a JRE8u60 or higher if using `COMMON_FJP`. + +##### Configuration + +Below are the configuration options known to this module: + +|`runtime.properties`|Description|Default| +|--------------------|-----------|-------| +|`druid.cache.type`| Set this to `caffeine`|`local`| +|`druid.cache.sizeInBytes`|The maximum size of the cache in bytes on heap.|None (unlimited)| +|`druid.cache.expireAfter`|The time (in ms) after an access for which a cache entry may be expired|None (no time limit)| +|`druid.cache.cacheExecutorFactory`|The executor factory to use for Caffeine maintenance. One of `COMMON_FJP`, `SINGLE_THREAD`, or `SAME_THREAD`|ForkJoinPool common pool (`COMMON_FJP`)| +|`druid.cache.evictOnClose`|If a close of a namespace (ex: removing a segment from a node) should cause an eager eviction of associated cache values|`false`| + +##### `druid.cache.cacheExecutorFactory` + +Here are the possible values for `druid.cache.cacheExecutorFactory`, which controls how maintenance tasks are run + +* `COMMON_FJP` (default) use the common ForkJoinPool. Should use with [JRE 8u60 or higher](https://github.com/druid-io/druid/pull/4810#issuecomment-329922810). Older versions of the JRE may have worse performance than newer JRE versions. +* `SINGLE_THREAD` Use a single-threaded executor. +* `SAME_THREAD` Cache maintenance is done eagerly. + +#### Metrics +In addition to the normal cache metrics, the caffeine cache implementation also reports the following in both `total` and `delta` + +|Metric|Description|Normal value| +|------|-----------|------------| +|`query/cache/caffeine/*/requests`|Count of hits or misses|hit + miss| +|`query/cache/caffeine/*/loadTime`|Length of time caffeine spends loading new values (unused feature)|0| +|`query/cache/caffeine/*/evictionBytes`|Size in bytes that have been evicted from the cache|Varies, should tune cache `sizeInBytes` so that `sizeInBytes`/`evictionBytes` is approximately the rate of cache churn you desire| + #### Memcached diff --git a/docs/content/development/extensions-core/caffeine-cache.md b/docs/content/development/extensions-core/caffeine-cache.md deleted file mode 100644 index b9d739904b2d..000000000000 --- a/docs/content/development/extensions-core/caffeine-cache.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -layout: doc_page ---- - -Druid Caffeine Cache --------------------- - -A highly performant local cache implementation for Druid based on [Caffeine](https://github.com/ben-manes/caffeine). Requires a JRE8u60 or higher - -# Configuration -Below are the configuration options known to this module: - -|`runtime.properties`|Description|Default| -|--------------------|-----------|-------| -|`druid.cache.sizeInBytes`|The maximum size of the cache in bytes on heap.|None (unlimited)| -|`druid.cache.expireAfter`|The time (in ms) after an access for which a cache entry may be expired|None (no time limit)| -|`druid.cache.cacheExecutorFactory`|The executor factory to use for Caffeine maintenance. One of `COMMON_FJP`, `SINGLE_THREAD`, or `SAME_THREAD`|ForkJoinPool common pool (`COMMON_FJP`)| -|`druid.cache.evictOnClose`|If a close of a namespace (ex: removing a segment from a node) should cause an eager eviction of associated cache values|`false`| - -## `druid.cache.cacheExecutorFactory` - -Here are the possible values for `druid.cache.cacheExecutorFactory`, which controls how maintenance tasks are run - -* `COMMON_FJP` (default) use the common ForkJoinPool. Do NOT use this option unless you are running 8u60 or higher -* `SINGLE_THREAD` Use a single-threaded executor -* `SAME_THREAD` Cache maintenance is done eagerly - -# Enabling - -To enable the caffeine cache, include this module on the loadList and set `druid.cache.type` to `caffeine` in your properties. - -# Metrics -In addition to the normal cache metrics, the caffeine cache implementation also reports the following in both `total` and `delta` - -|Metric|Description|Normal value| -|------|-----------|------------| -|`query/cache/caffeine/*/requests`|Count of hits or misses|hit + miss| -|`query/cache/caffeine/*/loadTime`|Length of time caffeine spends loading new values (unused feature)|0| -|`query/cache/caffeine/*/evictionBytes`|Size in bytes that have been evicted from the cache|Varies, should tune cache `sizeInBytes` so that `sizeInBytes`/`evictionBytes` is approximately the rate of cache churn you desire| diff --git a/extensions-core/caffeine-cache/pom.xml b/extensions-core/caffeine-cache/pom.xml deleted file mode 100644 index e4ff268a34e6..000000000000 --- a/extensions-core/caffeine-cache/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - 4.0.0 - - io.druid.extensions - druid-caffeine-cache - druid-caffeine-cache - Local cache implementation for Druid using Caffeine https://github.com/ben-manes/caffeine as the underlying implementation - - - io.druid - druid - 0.11.0-SNAPSHOT - ../../pom.xml - - - - - io.druid - druid-api - provided - ${project.parent.version} - - - io.druid - druid-server - ${project.parent.version} - provided - - - com.github.ben-manes.caffeine - caffeine - 2.3.1 - - - net.jpountz.lz4 - lz4 - provided - - - - - junit - junit - test - - - diff --git a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineDruidModule.java b/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineDruidModule.java deleted file mode 100644 index 20adb814bc44..000000000000 --- a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineDruidModule.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Metamarkets Group Inc. (Metamarkets) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Metamarkets licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.druid.client.cache; - -import com.fasterxml.jackson.databind.Module; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.common.collect.ImmutableList; -import com.google.inject.Binder; -import io.druid.initialization.DruidModule; - -import java.util.List; - -public class CaffeineDruidModule implements DruidModule -{ - - @Override - public void configure(Binder binder) - { - - } - - @Override - public List getJacksonModules() - { - return ImmutableList.of( - new SimpleModule("DruidCaffeineCache") - .registerSubtypes(CaffeineCacheProvider.class) - ); - } -} diff --git a/extensions-core/caffeine-cache/src/main/resources/META-INF/services/io.druid.initialization.DruidModule b/extensions-core/caffeine-cache/src/main/resources/META-INF/services/io.druid.initialization.DruidModule deleted file mode 100644 index 016f13528ad4..000000000000 --- a/extensions-core/caffeine-cache/src/main/resources/META-INF/services/io.druid.initialization.DruidModule +++ /dev/null @@ -1 +0,0 @@ -io.druid.client.cache.CaffeineDruidModule diff --git a/pom.xml b/pom.xml index 65b5d92443d2..1cb0bd3afb8d 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ 1.10.77 + 2.5.5 @@ -100,7 +101,6 @@ hll extensions-core/avro-extensions - extensions-core/caffeine-cache extensions-core/datasketches extensions-core/druid-kerberos extensions-core/hdfs-storage @@ -744,6 +744,11 @@ ${calcite.version} test-jar + + com.github.ben-manes.caffeine + caffeine + ${caffeine.version} + org.easymock easymock diff --git a/server/pom.xml b/server/pom.xml index 5a5dca163b1c..667ea2c02cd7 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -185,6 +185,10 @@ it.unimi.dsi fastutil + + com.github.ben-manes.caffeine + caffeine + diff --git a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CacheExecutorFactory.java b/server/src/main/java/io/druid/client/cache/CacheExecutorFactory.java similarity index 100% rename from extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CacheExecutorFactory.java rename to server/src/main/java/io/druid/client/cache/CacheExecutorFactory.java diff --git a/server/src/main/java/io/druid/client/cache/CacheProvider.java b/server/src/main/java/io/druid/client/cache/CacheProvider.java index e2bb3f35378e..68187f540e84 100644 --- a/server/src/main/java/io/druid/client/cache/CacheProvider.java +++ b/server/src/main/java/io/druid/client/cache/CacheProvider.java @@ -23,11 +23,12 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.inject.Provider; -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LocalCacheProvider.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = CaffeineCacheProvider.class) @JsonSubTypes(value = { @JsonSubTypes.Type(name = "local", value = LocalCacheProvider.class), @JsonSubTypes.Type(name = "memcached", value = MemcachedCacheProvider.class), - @JsonSubTypes.Type(name = "hybrid", value = HybridCacheProvider.class) + @JsonSubTypes.Type(name = "hybrid", value = HybridCacheProvider.class), + @JsonSubTypes.Type(name = "caffeine", value = CaffeineCacheProvider.class) }) public interface CacheProvider extends Provider { diff --git a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCache.java b/server/src/main/java/io/druid/client/cache/CaffeineCache.java similarity index 100% rename from extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCache.java rename to server/src/main/java/io/druid/client/cache/CaffeineCache.java diff --git a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCacheConfig.java b/server/src/main/java/io/druid/client/cache/CaffeineCacheConfig.java similarity index 100% rename from extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCacheConfig.java rename to server/src/main/java/io/druid/client/cache/CaffeineCacheConfig.java diff --git a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCacheProvider.java b/server/src/main/java/io/druid/client/cache/CaffeineCacheProvider.java similarity index 92% rename from extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCacheProvider.java rename to server/src/main/java/io/druid/client/cache/CaffeineCacheProvider.java index 23e18987fe7e..13dd6690ce17 100644 --- a/extensions-core/caffeine-cache/src/main/java/io/druid/client/cache/CaffeineCacheProvider.java +++ b/server/src/main/java/io/druid/client/cache/CaffeineCacheProvider.java @@ -19,9 +19,6 @@ package io.druid.client.cache; -import com.fasterxml.jackson.annotation.JsonTypeName; - -@JsonTypeName("caffeine") public class CaffeineCacheProvider extends CaffeineCacheConfig implements CacheProvider { @Override diff --git a/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java index e8e724fd7780..6e66fb0a6b86 100644 --- a/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java +++ b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java @@ -24,7 +24,9 @@ import javax.validation.constraints.Min; /** + * Deprecated, use {@CaffeineCacheProvider} */ +@Deprecated public class LocalCacheProvider implements CacheProvider { @JsonProperty diff --git a/extensions-core/caffeine-cache/src/test/java/io/druid/client/cache/CaffeineCacheTest.java b/server/src/test/java/io/druid/client/cache/CaffeineCacheTest.java similarity index 100% rename from extensions-core/caffeine-cache/src/test/java/io/druid/client/cache/CaffeineCacheTest.java rename to server/src/test/java/io/druid/client/cache/CaffeineCacheTest.java