From 3fa1615989e824413c0159d35ff24de968679356 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 13 Apr 2021 12:24:11 +0800 Subject: [PATCH 1/2] [Metric] Add metrics for routine load Add following metrics for routine load: doris_fe_job{job="load", type="ROUTINE_LOAD", state="NEED_SCHEDULE"} 0 doris_fe_job{job="load", type="ROUTINE_LOAD", state="RUNNING"} 1 doris_fe_job{job="load", type="ROUTINE_LOAD", state="PAUSED"} 0 doris_fe_job{job="load", type="ROUTINE_LOAD", state="STOPPED"} 0 doris_fe_job{job="load", type="ROUTINE_LOAD", state="CANCELLED"} 0 --- .../apache/doris/common/util/TimeUtils.java | 3 +- .../org/apache/doris/metric/MetricRepo.java | 32 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index 0fbac6cee030c7..b8ba2a67a7775c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -49,13 +49,14 @@ public class TimeUtils { private static final Logger LOG = LogManager.getLogger(TimeUtils.class); + public static final String UTC_TIME_ZONE = "Africa/Abidjan"; public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai"; private static final TimeZone TIME_ZONE; // set CST to +08:00 instead of America/Chicago public static final ImmutableMap timeZoneAliasMap = ImmutableMap.of( - "CST", DEFAULT_TIME_ZONE, "PRC", DEFAULT_TIME_ZONE); + "CST", DEFAULT_TIME_ZONE, "PRC", DEFAULT_TIME_ZONE, "UTC", UTC_TIME_ZONE); // NOTICE: Date formats are not synchronized. // it must be used as synchronized externally. diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 0fddec811e75ba..8b1d3086b0219c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -26,6 +26,8 @@ import org.apache.doris.load.EtlJobType; import org.apache.doris.load.loadv2.JobState; import org.apache.doris.load.loadv2.LoadManager; +import org.apache.doris.load.routineload.RoutineLoadJob; +import org.apache.doris.load.routineload.RoutineLoadManager; import org.apache.doris.metric.Metric.MetricUnit; import org.apache.doris.monitor.jvm.JvmService; import org.apache.doris.monitor.jvm.JvmStats; @@ -36,12 +38,14 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -117,19 +121,41 @@ public Long getValue() { } }; gauge.addLabel(new MetricLabel("job", "load")) - .addLabel(new MetricLabel("type", jobType.name())) - .addLabel(new MetricLabel("state", state.name())); + .addLabel(new MetricLabel("type", jobType.name())) + .addLabel(new MetricLabel("state", state.name())); PALO_METRIC_REGISTER.addPaloMetrics(gauge); } } + // routine load jobs + RoutineLoadManager routineLoadManager = Catalog.getCurrentCatalog().getRoutineLoadManager(); + for (RoutineLoadJob.JobState jobState : RoutineLoadJob.JobState.values()) { + GaugeMetric gauge = (GaugeMetric) new GaugeMetric("job", + MetricUnit.NOUNIT, "routine load job statistics") { + @Override + public Long getValue() { + if (!Catalog.getCurrentCatalog().isMaster()) { + return 0L; + } + Set states = Sets.newHashSet(); + states.add(jobState); + List jobs = routineLoadManager.getRoutineLoadJobByState(states); + return Long.valueOf(jobs.size()); + } + }; + gauge.addLabel(new MetricLabel("job", "load")) + .addLabel(new MetricLabel("type", "ROUTINE_LOAD")) + .addLabel(new MetricLabel("state", jobState.name())); + PALO_METRIC_REGISTER.addPaloMetrics(gauge); + } + // running alter job Alter alter = Catalog.getCurrentCatalog().getAlterInstance(); for (JobType jobType : JobType.values()) { if (jobType != JobType.SCHEMA_CHANGE && jobType != JobType.ROLLUP) { continue; } - + GaugeMetric gauge = (GaugeMetric) new GaugeMetric("job", MetricUnit.NOUNIT, "job statistics") { @Override From f5d601c84fb1327e709d20a2e89e758dac3a540c Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 14 Apr 2021 09:40:04 +0800 Subject: [PATCH 2/2] change UTC --- .../src/main/java/org/apache/doris/common/util/TimeUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java index b8ba2a67a7775c..85901be163644a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/TimeUtils.java @@ -49,7 +49,7 @@ public class TimeUtils { private static final Logger LOG = LogManager.getLogger(TimeUtils.class); - public static final String UTC_TIME_ZONE = "Africa/Abidjan"; + public static final String UTC_TIME_ZONE = "Europe/London"; // This is just a Country to represent UTC offset +00:00 public static final String DEFAULT_TIME_ZONE = "Asia/Shanghai"; private static final TimeZone TIME_ZONE;