From ebf8d39ceb6c4d63f1202868bb6529b39f9e4201 Mon Sep 17 00:00:00 2001 From: cecemei Date: Mon, 23 Sep 2024 10:54:42 -0700 Subject: [PATCH 01/99] Some debug configs --- .../conf/druid/single-server/micro-quickstart/broker/jvm.config | 1 + .../micro-quickstart/coordinator-overlord/jvm.config | 1 + .../druid/single-server/micro-quickstart/historical/jvm.config | 1 + .../single-server/micro-quickstart/middleManager/jvm.config | 1 + .../conf/druid/single-server/micro-quickstart/router/jvm.config | 1 + 5 files changed, 5 insertions(+) diff --git a/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config b/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config index 5d666f6db234..53f75be7a646 100644 --- a/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config +++ b/examples/conf/druid/single-server/micro-quickstart/broker/jvm.config @@ -7,3 +7,4 @@ -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5001 diff --git a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config index aae520d8325f..8cf32760f798 100644 --- a/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config +++ b/examples/conf/druid/single-server/micro-quickstart/coordinator-overlord/jvm.config @@ -7,3 +7,4 @@ -Djava.io.tmpdir=var/tmp -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dderby.stream.error.file=var/druid/derby.log +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5002 diff --git a/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config b/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config index 79edb06e2030..d13169a217b3 100644 --- a/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config +++ b/examples/conf/druid/single-server/micro-quickstart/historical/jvm.config @@ -7,3 +7,4 @@ -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5003 diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config b/examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config index 8a650fb358c9..a24e8037d8e7 100644 --- a/examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config +++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/jvm.config @@ -6,3 +6,4 @@ -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5004 diff --git a/examples/conf/druid/single-server/micro-quickstart/router/jvm.config b/examples/conf/druid/single-server/micro-quickstart/router/jvm.config index b822f4601790..909f7dfe705a 100644 --- a/examples/conf/druid/single-server/micro-quickstart/router/jvm.config +++ b/examples/conf/druid/single-server/micro-quickstart/router/jvm.config @@ -7,3 +7,4 @@ -Dfile.encoding=UTF-8 -Djava.io.tmpdir=var/tmp -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 From 9d01ab6f79cd51fc90c28132da1cc7e5905a5a31 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 17 Oct 2024 14:08:33 -0700 Subject: [PATCH 02/99] use postgresql as the default metadata store and set a few debug log --- .../_common/common.runtime.properties | 22 +++++++++---------- .../micro-quickstart/_common/log4j2.xml | 8 +++---- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties index d27c83ca569e..f1845df19b5f 100644 --- a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties @@ -24,7 +24,7 @@ # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html -druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "druid-parquet-extensions"] +druid.extensions.loadList=["postgresql-metadata-storage", "druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "druid-parquet-extensions"] # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory # and uncomment the line below to point to your directory. @@ -55,10 +55,10 @@ druid.zk.paths.base=/druid # # For Derby server on your Druid Coordinator (only viable in a cluster with a single Coordinator, no fail-over): -druid.metadata.storage.type=derby -druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true -druid.metadata.storage.connector.host=localhost -druid.metadata.storage.connector.port=1527 +# druid.metadata.storage.type=derby +# druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true +# druid.metadata.storage.connector.host=localhost +# druid.metadata.storage.connector.port=1527 # For MySQL (make sure to include the MySQL JDBC driver on the classpath): #druid.metadata.storage.type=mysql @@ -67,10 +67,10 @@ druid.metadata.storage.connector.port=1527 #druid.metadata.storage.connector.password=... # For PostgreSQL: -#druid.metadata.storage.type=postgresql -#druid.metadata.storage.connector.connectURI=jdbc:postgresql://db.example.com:5432/druid -#druid.metadata.storage.connector.user=... -#druid.metadata.storage.connector.password=... +druid.metadata.storage.type=postgresql +druid.metadata.storage.connector.connectURI=jdbc:postgresql://localhost:5432/druid +druid.metadata.storage.connector.user=druid +druid.metadata.storage.connector.password=druid # # Deep storage @@ -120,8 +120,8 @@ druid.selectors.coordinator.serviceName=druid/coordinator # druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor", "org.apache.druid.server.metrics.ServiceStatusMonitor"] -druid.emitter=noop -druid.emitter.logging.logLevel=info +druid.emitter=logging +druid.emitter.logging.logLevel=debug # Storage type of double columns # ommiting this will lead to index double as float at the storage layer diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml b/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml index eb24a2a89406..aa4816e5e515 100644 --- a/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml +++ b/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml @@ -53,20 +53,20 @@ - + - + - + - + From 1d325dca4d8f37a811889cb864fdbb9dfc7ac384 Mon Sep 17 00:00:00 2001 From: cecemei Date: Tue, 29 Oct 2024 12:38:44 -0700 Subject: [PATCH 03/99] Add s3 extension, update local storage directory, use emoji in website title --- .../micro-quickstart/_common/common.runtime.properties | 4 ++-- web-console/unified-console.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties index f1845df19b5f..ad69643b83d9 100644 --- a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties +++ b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties @@ -24,7 +24,7 @@ # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: https://druid.apache.org/docs/latest/operations/including-extensions.html -druid.extensions.loadList=["postgresql-metadata-storage", "druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "druid-parquet-extensions"] +druid.extensions.loadList=["druid-s3-extensions","postgresql-metadata-storage", "druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-multi-stage-query", "druid-parquet-extensions"] # If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory # and uncomment the line below to point to your directory. @@ -78,7 +78,7 @@ druid.metadata.storage.connector.password=druid # For local disk (only viable in a cluster if this is a network mount): druid.storage.type=local -druid.storage.storageDirectory=var/druid/segments +druid.storage.storageDirectory=/Users/cecemei/codespace/var/druid/segments # For HDFS: #druid.storage.type=hdfs diff --git a/web-console/unified-console.html b/web-console/unified-console.html index 330c5ece9d41..bc8471f285db 100644 --- a/web-console/unified-console.html +++ b/web-console/unified-console.html @@ -24,7 +24,7 @@ name="viewport" content="width=device-width, user-scalable=no, initial-scale=1, maximum-scale=1, minimum-scale=1" /> - Apache Druid + ⚡Apache Druid