From 585eb2fb4c858b7f568697868d88b601e4e47e88 Mon Sep 17 00:00:00 2001 From: 317brian <53799971+317brian@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:03:24 -0800 Subject: [PATCH 1/7] docs: add mermaid diagram support --- website/docusaurus.config.js | 6 +++++- website/package.json | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 2cd516d43e26..7ebaa7cc73e0 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -41,6 +41,9 @@ module.exports={ }, "onBrokenLinks": "log", "onBrokenMarkdownLinks": "throw", + markdown: { + mermaid: true, + }, "presets": [ [ "@docusaurus/preset-classic", @@ -73,7 +76,8 @@ module.exports={ "redirects": Redirects } ], - "docusaurus-lunr-search" + "docusaurus-lunr-search", + "@docusaurus/theme-mermaid" ], "themeConfig": { colorMode: { diff --git a/website/package.json b/website/package.json index 92e202f0d4a3..94f48351c7b5 100644 --- a/website/package.json +++ b/website/package.json @@ -31,6 +31,7 @@ "@docusaurus/core": "^2.4.1", "@docusaurus/plugin-client-redirects": "^2.4.1", "@docusaurus/preset-classic": "^2.4.1", + "@docusaurus/theme-mermaid": "^2.4.1", "clsx": "^1.1.1", "docusaurus-lunr-search": "^3.3.1", "fast-glob": "^3.2.2", From 0d49860102043452b9f2d22f9376a9ffe5588bf1 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 28 Feb 2024 14:37:24 -0800 Subject: [PATCH 2/7] fix crash when parsing data in data loader that can not be parsed (#15983) --- web-console/src/utils/object-change.spec.ts | 21 ++++++++++++++++++++- web-console/src/utils/object-change.ts | 5 +++-- web-console/src/utils/sampler.ts | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/web-console/src/utils/object-change.spec.ts b/web-console/src/utils/object-change.spec.ts index bdfecca7ec1e..547d88159268 100644 --- a/web-console/src/utils/object-change.spec.ts +++ b/web-console/src/utils/object-change.spec.ts @@ -18,7 +18,15 @@ import * as JSONBig from 'json-bigint-native'; -import { deepDelete, deepExtend, deepGet, deepSet, makePath, parsePath } from './object-change'; +import { + allowKeys, + deepDelete, + deepExtend, + deepGet, + deepSet, + makePath, + parsePath, +} from './object-change'; describe('object-change', () => { describe('parsePath', () => { @@ -36,6 +44,17 @@ describe('object-change', () => { }); }); + describe('allowKeys', () => { + it('works with bad objects', () => { + expect(allowKeys(null, ['a', 'b', 'c'] as any)).toEqual(null); + expect(allowKeys(undefined as any, ['a', 'b', 'c'] as any)).toEqual(undefined); + }); + + it('works in a normal case', () => { + expect(allowKeys({ a: 1, z: 4 }, ['a', 'b', 'c'] as any)).toEqual({ a: 1 }); + }); + }); + describe('deepGet', () => { const thing = { hello: { diff --git a/web-console/src/utils/object-change.ts b/web-console/src/utils/object-change.ts index 9b29cda5dbf4..9eeb8a4b6e99 100644 --- a/web-console/src/utils/object-change.ts +++ b/web-console/src/utils/object-change.ts @@ -160,9 +160,10 @@ export function deepExtend>(target: T, diff: Recor } export function allowKeys(obj: T, keys: (keyof T)[]): T { - const newObj: T = {} as any; + if (!obj || typeof obj !== 'object') return obj; + const newObj = {} as T; for (const key of keys) { - if (Object.prototype.hasOwnProperty.call(obj, key)) { + if (Object.hasOwn(obj, key)) { newObj[key] = obj[key]; } } diff --git a/web-console/src/utils/sampler.ts b/web-console/src/utils/sampler.ts index 6c31f2bb2b66..8b1d25320b7d 100644 --- a/web-console/src/utils/sampler.ts +++ b/web-console/src/utils/sampler.ts @@ -134,7 +134,7 @@ export interface SampleEntry { export function getCacheRowsFromSampleResponse(sampleResponse: SampleResponse): CacheRows { return filterMap(sampleResponse.data, d => ({ ...d.input, - ...allowKeys(d.parsed, ALL_POSSIBLE_SYSTEM_FIELDS), + ...allowKeys(d.parsed || {}, ALL_POSSIBLE_SYSTEM_FIELDS), })).slice(0, 20); } From eeb1e459df914d7f435b2e3c793d14ef18b9e683 Mon Sep 17 00:00:00 2001 From: Jan Werner <105367074+janjwerner-confluent@users.noreply.github.com> Date: Wed, 28 Feb 2024 20:27:31 -0500 Subject: [PATCH 3/7] update jetty to address CVE (#16000) --- licenses.yaml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index d01d89af6442..329862d95425 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -1999,7 +1999,7 @@ name: Jetty license_category: binary module: java-core license_name: Apache License version 2.0 -version: 9.4.53.v20231009 +version: 9.4.54.v20240208 libraries: - org.eclipse.jetty: jetty-client - org.eclipse.jetty: jetty-continuation diff --git a/pom.xml b/pom.xml index 4c324f201ac3..acccb23703c3 100644 --- a/pom.xml +++ b/pom.xml @@ -97,7 +97,7 @@ 32.0.1-jre 4.1.0 1.3 - 9.4.53.v20231009 + 9.4.54.v20240208 1.19.4 2.12.7.20221012 1.9.13 From aa64e3b6508f60713110e24870294082c4096b8a Mon Sep 17 00:00:00 2001 From: AmatyaAvadhanula Date: Thu, 29 Feb 2024 12:06:47 +0530 Subject: [PATCH 4/7] Concurrent replace should work with supervisors using concurrent locks (#15995) * Concurrent replace should work with supervisors using concurrent locks * Ignore supervisors with useConcurrentLocks set to false * Apply feedback --- .../supervisor/SupervisorManager.java | 27 +++++++++--- .../supervisor/SupervisorManagerTest.java | 42 +++++++++++++++++++ 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java index 207ff56f28f8..810a991c2f22 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -87,23 +87,38 @@ public Optional getActiveSupervisorIdForDatasourceWithAppendLock(String final Supervisor supervisor = entry.getValue().lhs; final SupervisorSpec supervisorSpec = entry.getValue().rhs; - TaskLockType taskLockType = null; + boolean hasAppendLock = Tasks.DEFAULT_USE_CONCURRENT_LOCKS; if (supervisorSpec instanceof SeekableStreamSupervisorSpec) { SeekableStreamSupervisorSpec seekableStreamSupervisorSpec = (SeekableStreamSupervisorSpec) supervisorSpec; Map context = seekableStreamSupervisorSpec.getContext(); if (context != null) { - taskLockType = QueryContexts.getAsEnum( - Tasks.TASK_LOCK_TYPE, - context.get(Tasks.TASK_LOCK_TYPE), - TaskLockType.class + Boolean useConcurrentLocks = QueryContexts.getAsBoolean( + Tasks.USE_CONCURRENT_LOCKS, + context.get(Tasks.USE_CONCURRENT_LOCKS) ); + if (useConcurrentLocks == null) { + TaskLockType taskLockType = QueryContexts.getAsEnum( + Tasks.TASK_LOCK_TYPE, + context.get(Tasks.TASK_LOCK_TYPE), + TaskLockType.class + ); + if (taskLockType == null) { + hasAppendLock = Tasks.DEFAULT_USE_CONCURRENT_LOCKS; + } else if (taskLockType == TaskLockType.APPEND) { + hasAppendLock = true; + } else { + hasAppendLock = false; + } + } else { + hasAppendLock = useConcurrentLocks; + } } } if (supervisor instanceof SeekableStreamSupervisor && !supervisorSpec.isSuspended() && supervisorSpec.getDataSources().contains(datasource) - && TaskLockType.APPEND.equals(taskLockType)) { + && (hasAppendLock)) { return Optional.of(supervisorId); } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java index e8c5d839cf19..5ffbd4b94608 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java @@ -468,6 +468,21 @@ public void testGetActiveSupervisorIdForDatasourceWithAppendLock() EasyMock.replay(activeSpec); metadataSupervisorManager.insert(EasyMock.anyString(), EasyMock.anyObject()); + SeekableStreamSupervisorSpec activeSpecWithConcurrentLocks = EasyMock.mock(SeekableStreamSupervisorSpec.class); + Supervisor activeSupervisorWithConcurrentLocks = EasyMock.mock(SeekableStreamSupervisor.class); + EasyMock.expect(activeSpecWithConcurrentLocks.getId()).andReturn("activeSpecWithConcurrentLocks").anyTimes(); + EasyMock.expect(activeSpecWithConcurrentLocks.isSuspended()).andReturn(false).anyTimes(); + EasyMock.expect(activeSpecWithConcurrentLocks.getDataSources()) + .andReturn(ImmutableList.of("activeConcurrentLocksDS")).anyTimes(); + EasyMock.expect(activeSpecWithConcurrentLocks.createSupervisor()) + .andReturn(activeSupervisorWithConcurrentLocks).anyTimes(); + EasyMock.expect(activeSpecWithConcurrentLocks.createAutoscaler(activeSupervisorWithConcurrentLocks)) + .andReturn(null).anyTimes(); + EasyMock.expect(activeSpecWithConcurrentLocks.getContext()) + .andReturn(ImmutableMap.of(Tasks.USE_CONCURRENT_LOCKS, true)).anyTimes(); + EasyMock.replay(activeSpecWithConcurrentLocks); + metadataSupervisorManager.insert(EasyMock.anyString(), EasyMock.anyObject()); + SeekableStreamSupervisorSpec activeAppendSpec = EasyMock.mock(SeekableStreamSupervisorSpec.class); Supervisor activeAppendSupervisor = EasyMock.mock(SeekableStreamSupervisor.class); EasyMock.expect(activeAppendSpec.getId()).andReturn("activeAppendSpec").anyTimes(); @@ -482,6 +497,25 @@ public void testGetActiveSupervisorIdForDatasourceWithAppendLock() EasyMock.replay(activeAppendSpec); metadataSupervisorManager.insert(EasyMock.anyString(), EasyMock.anyObject()); + // A supervisor with useConcurrentLocks set to false explicitly must not use an append lock + SeekableStreamSupervisorSpec specWithUseConcurrentLocksFalse = EasyMock.mock(SeekableStreamSupervisorSpec.class); + Supervisor supervisorWithUseConcurrentLocksFalse = EasyMock.mock(SeekableStreamSupervisor.class); + EasyMock.expect(specWithUseConcurrentLocksFalse.getId()).andReturn("useConcurrentLocksFalse").anyTimes(); + EasyMock.expect(specWithUseConcurrentLocksFalse.isSuspended()).andReturn(false).anyTimes(); + EasyMock.expect(specWithUseConcurrentLocksFalse.getDataSources()) + .andReturn(ImmutableList.of("dsWithuseConcurrentLocksFalse")).anyTimes(); + EasyMock.expect(specWithUseConcurrentLocksFalse.createSupervisor()).andReturn(supervisorWithUseConcurrentLocksFalse).anyTimes(); + EasyMock.expect(specWithUseConcurrentLocksFalse.createAutoscaler(supervisorWithUseConcurrentLocksFalse)) + .andReturn(null).anyTimes(); + EasyMock.expect(specWithUseConcurrentLocksFalse.getContext()).andReturn(ImmutableMap.of( + Tasks.USE_CONCURRENT_LOCKS, + false, + Tasks.TASK_LOCK_TYPE, + TaskLockType.APPEND.name() + )).anyTimes(); + EasyMock.replay(specWithUseConcurrentLocksFalse); + metadataSupervisorManager.insert(EasyMock.anyString(), EasyMock.anyObject()); + replayAll(); manager.start(); @@ -499,6 +533,14 @@ public void testGetActiveSupervisorIdForDatasourceWithAppendLock() manager.createOrUpdateAndStartSupervisor(activeAppendSpec); Assert.assertTrue(manager.getActiveSupervisorIdForDatasourceWithAppendLock("activeAppendDS").isPresent()); + manager.createOrUpdateAndStartSupervisor(activeSpecWithConcurrentLocks); + Assert.assertTrue(manager.getActiveSupervisorIdForDatasourceWithAppendLock("activeConcurrentLocksDS").isPresent()); + + manager.createOrUpdateAndStartSupervisor(specWithUseConcurrentLocksFalse); + Assert.assertFalse( + manager.getActiveSupervisorIdForDatasourceWithAppendLock("dsWithUseConcurrentLocksFalse").isPresent() + ); + verifyAll(); } From 00e1b01416ffa74f1fafa8b34707d62a5b247027 Mon Sep 17 00:00:00 2001 From: Sensor Date: Thu, 29 Feb 2024 15:28:14 +0800 Subject: [PATCH 5/7] Add pre-check for heavy debug logs (#15706) Co-authored-by: Kashif Faraz Co-authored-by: Benedict Jin --- .../parallel/ParallelIndexSupervisorTask.java | 2 +- .../supervisor/SeekableStreamSupervisor.java | 15 ++++++----- .../autoscaler/LagBasedAutoScaler.java | 2 +- .../clients/OverlordResourceTestClient.java | 8 ++++-- .../processor/FrameProcessorExecutor.java | 1 - .../guava/ParallelMergeCombiningSequence.java | 25 ++++++++----------- .../util/common/io/smoosh/FileSmoosher.java | 8 +++--- .../druid/discovery/DataServerClient.java | 4 ++- .../server/coordinator/DruidCoordinator.java | 7 +++--- .../loading/CuratorLoadQueuePeon.java | 12 ++++----- 10 files changed, 41 insertions(+), 43 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java index d0afc0756176..894fe1e5b06a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java @@ -752,7 +752,7 @@ TaskStatus runHashPartitionMultiPhaseParallel(TaskToolbox toolbox) throws Except ); // This is for potential debugging in case we suspect bad estimation of cardinalities etc, - LOG.debug("intervalToNumShards: %s", intervalToNumShards.toString()); + LOG.debug("intervalToNumShards: %s", intervalToNumShards); } else { intervalToNumShards = CollectionUtils.mapValues( diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 62d3caa83d68..5ea7a4168cb1 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -1155,14 +1155,13 @@ public void tryInit() Instant handleNoticeEndTime = Instant.now(); Duration timeElapsed = Duration.between(handleNoticeStartTime, handleNoticeEndTime); String noticeType = notice.getType(); - log.debug( - "Handled notice [%s] from notices queue in [%d] ms, " - + "current notices queue size [%d] for datasource [%s]", - noticeType, - timeElapsed.toMillis(), - getNoticesQueueSize(), - dataSource - ); + if (log.isDebugEnabled()) { + log.debug( + "Handled notice [%s] from notices queue in [%d] ms, " + + "current notices queue size [%d] for datasource [%s]", + noticeType, timeElapsed.toMillis(), getNoticesQueueSize(), dataSource + ); + } emitNoticeProcessTime(noticeType, timeElapsed.toMillis()); } catch (Throwable e) { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java index 8c645278d55b..a06f358435f8 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java @@ -149,7 +149,7 @@ private Runnable computeAndCollectLag() long totalLags = lagStats.getTotalLag(); lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0L); } - log.debug("Current lags [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); + log.debug("Current lags for dataSource[%s] are [%s].", dataSource, lagMetricsQueue); } else { log.warn("[%s] supervisor is suspended, skipping lag collection", dataSource); } diff --git a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java index bf0f549b190f..95f56ad4b759 100644 --- a/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java +++ b/integration-tests/src/main/java/org/apache/druid/testing/clients/OverlordResourceTestClient.java @@ -185,7 +185,9 @@ private List getTasks(String identifier) HttpMethod.GET, StringUtils.format("%s%s", getIndexerURL(), identifier) ); - LOG.debug("Tasks %s response %s", identifier, response.getContent()); + if (LOG.isDebugEnabled()) { + LOG.debug("Tasks %s response %s", identifier, response.getContent()); + } return jsonMapper.readValue( response.getContent(), new TypeReference>() { @@ -204,7 +206,9 @@ public TaskPayloadResponse getTaskPayload(String taskId) HttpMethod.GET, StringUtils.format("%stask/%s", getIndexerURL(), StringUtils.urlEncode(taskId)) ); - LOG.debug("Task %s response %s", taskId, response.getContent()); + if (LOG.isDebugEnabled()) { + LOG.debug("Task %s response %s", taskId, response.getContent()); + } return jsonMapper.readValue( response.getContent(), new TypeReference() { diff --git a/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorExecutor.java b/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorExecutor.java index 02161d5ad5ad..c0f79d30e581 100644 --- a/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorExecutor.java +++ b/processing/src/main/java/org/apache/druid/frame/processor/FrameProcessorExecutor.java @@ -595,7 +595,6 @@ private static void logProcessorStatusString( sb.append("; cancel=").append(finishedFuture.isCancelled() ? "y" : "n"); sb.append("; done=").append(finishedFuture.isDone() ? "y" : "n"); - log.debug(StringUtils.encodeForFormat(sb.toString())); } } diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java index 1df3c9a9dc16..517235a99f98 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java @@ -475,20 +475,17 @@ private int computeNumTasks() final int computedNumParallelTasks = Math.max(computedOptimalParallelism, 1); - LOG.debug( - "Computed parallel tasks: [%s]; ForkJoinPool details - sequence parallelism: [%s] " - + "active threads: [%s] running threads: [%s] queued submissions: [%s] queued tasks: [%s] " - + "pool parallelism: [%s] pool size: [%s] steal count: [%s]", - computedNumParallelTasks, - parallelism, - getPool().getActiveThreadCount(), - runningThreadCount, - submissionCount, - getPool().getQueuedTaskCount(), - getPool().getParallelism(), - getPool().getPoolSize(), - getPool().getStealCount() - ); + if (LOG.isDebugEnabled()) { + ForkJoinPool pool = getPool(); + LOG.debug( + "Computed parallel tasks: [%s]; ForkJoinPool details - sequence parallelism: [%s] " + + "active threads: [%s] running threads: [%s] queued submissions: [%s] queued tasks: [%s] " + + "pool parallelism: [%s] pool size: [%s] steal count: [%s]", + computedNumParallelTasks, parallelism, + pool.getActiveThreadCount(), runningThreadCount, submissionCount, pool.getQueuedTaskCount(), + pool.getParallelism(), pool.getPoolSize(), pool.getStealCount() + ); + } return computedNumParallelTasks; } diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/FileSmoosher.java b/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/FileSmoosher.java index 17eb469e9497..f4103ec18473 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/FileSmoosher.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/smoosh/FileSmoosher.java @@ -490,11 +490,9 @@ public boolean isOpen() public void close() throws IOException { closer.close(); - FileSmoosher.LOG.debug( - "Created smoosh file [%s] of size [%s] bytes.", - outFile.getAbsolutePath(), - outFile.length() - ); + if (LOG.isDebugEnabled()) { + LOG.debug("Created smoosh file [%s] of size [%s] bytes.", outFile.getAbsolutePath(), outFile.length()); + } } } } diff --git a/server/src/main/java/org/apache/druid/discovery/DataServerClient.java b/server/src/main/java/org/apache/druid/discovery/DataServerClient.java index 479ba9d4142d..ce7ac325b62b 100644 --- a/server/src/main/java/org/apache/druid/discovery/DataServerClient.java +++ b/server/src/main/java/org/apache/druid/discovery/DataServerClient.java @@ -91,7 +91,9 @@ public Sequence run(Query query, ResponseContext responseContext, Java requestBuilder = requestBuilder.jsonContent(objectMapper, query); } - log.debug("Sending request to servers for query[%s], request[%s]", query.getId(), requestBuilder.toString()); + if (log.isDebugEnabled()) { + log.debug("Sending request to servers for query[%s], request[%s]", query.getId(), requestBuilder); + } ListenableFuture resultStreamFuture = serviceClient.asyncRequest( requestBuilder, new DataServerResponseHandler(query, responseContext, objectMapper) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java index 5558d204e81a..caa1c7fb66f0 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java @@ -538,10 +538,9 @@ List makeIndexingServiceDuties() if (getCompactSegmentsDutyFromCustomGroups().isEmpty()) { duties.add(compactSegments); } - log.debug( - "Initialized indexing service duties [%s].", - duties.stream().map(duty -> duty.getClass().getName()).collect(Collectors.toList()) - ); + if (log.isDebugEnabled()) { + log.debug("Initialized indexing service duties [%s].", duties.stream().map(duty -> duty.getClass().getName()).collect(Collectors.toList())); + } return ImmutableList.copyOf(duties); } diff --git a/server/src/main/java/org/apache/druid/server/coordinator/loading/CuratorLoadQueuePeon.java b/server/src/main/java/org/apache/druid/server/coordinator/loading/CuratorLoadQueuePeon.java index 2303be26d8b5..7ea78a132fe9 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/loading/CuratorLoadQueuePeon.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/loading/CuratorLoadQueuePeon.java @@ -230,12 +230,12 @@ public void run() final String path = ZKPaths.makePath(basePath, segmentHolder.getSegmentIdentifier()); final byte[] payload = jsonMapper.writeValueAsBytes(segmentHolder.getChangeRequest()); curator.create().withMode(CreateMode.EPHEMERAL).forPath(path, payload); - log.debug( - "ZKNode created for server to [%s] %s [%s]", - basePath, - segmentHolder.getAction(), - segmentHolder.getSegmentIdentifier() - ); + if (log.isDebugEnabled()) { + log.debug( + "ZKNode created for server to [%s] %s [%s]", + basePath, segmentHolder.getAction(), segmentHolder.getSegmentIdentifier() + ); + } final ScheduledFuture nodeDeletedCheck = scheduleNodeDeletedCheck(path); final Stat stat = curator.checkExists().usingWatcher( (CuratorWatcher) watchedEvent -> { From 137ea04ca8637f30bab0c89085edca66402f4f9b Mon Sep 17 00:00:00 2001 From: Sensor Date: Thu, 29 Feb 2024 22:32:27 +0800 Subject: [PATCH 6/7] Remove helm paths from CodeQL config (#16006) --- .github/workflows/codeql.yml | 2 -- .github/workflows/unit-and-integration-tests-unified.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8d9c157a61bb..fb6bc9a3bb0a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -6,14 +6,12 @@ on: - '**/*.md' - 'dev/**' - 'docs/**' - - 'helm/**' branches: [ 'master', '0.6.x', '0.7.x', '0.7.1.x', '0.7.2.x', '0.8.0', '0.8.1', '0.8.2', '0.8.3', '0.9.0', '0.9.1' ] pull_request: paths-ignore: - '**/*.md' - 'dev/**' - 'docs/**' - - 'helm/**' # The branches below must be a subset of the branches above branches: [ 'master' ] schedule: diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 59ea3f95a585..f51985da90fa 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -21,7 +21,6 @@ on: - 'dev/**' - 'docs/**' - 'examples/**/jupyter-notebooks/**' - - 'helm/**' - 'web-console/**' - 'website/**' branches: @@ -34,7 +33,6 @@ on: - 'dev/**' - 'docs/**' - 'examples/**/jupyter-notebooks/**' - - 'helm/**' - 'web-console/**' - 'website/**' branches: From ff8b66bedf9525bcc66194266c6e10eda82aeb56 Mon Sep 17 00:00:00 2001 From: 317brian <53799971+317brian@users.noreply.github.com> Date: Thu, 29 Feb 2024 11:48:49 -0800 Subject: [PATCH 7/7] docs: mention acid-compliance for metadb --- docs/design/metadata-storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/metadata-storage.md b/docs/design/metadata-storage.md index 4d633937ac5a..4f66115916d2 100644 --- a/docs/design/metadata-storage.md +++ b/docs/design/metadata-storage.md @@ -44,7 +44,7 @@ See [Metadata storage configuration](../configuration/index.md#metadata-storage) ## Available metadata stores -Druid supports Derby, MySQL, and PostgreSQL for storing metadata. +Druid supports Derby, MySQL, and PostgreSQL for storing metadata. Note that your metadata store must be ACID-compliant. If it isn't ACID-compliant, you can encounter issues, such as tasks failing sporadically. To avoid issues with upgrades that require schema changes to a large metadata table, consider a metadata store version that supports instant ADD COLUMN semantics. See the database-specific docs for guidance on versions.