From 0811091359e621924e2b7133a2e07579ab472e55 Mon Sep 17 00:00:00 2001 From: daming Date: Tue, 14 Apr 2020 16:49:34 +0800 Subject: [PATCH 01/24] revert @Column --- .../storage/plugin/influxdb/InfluxClient.java | 17 ++ ...delConstants.java => InfluxConstants.java} | 24 +- .../influxdb/InfluxStorageProvider.java | 48 ++-- ...ssAlias.java => InfluxTableInstaller.java} | 28 +- .../plugin/influxdb/TableMetaInfo.java | 53 +++- .../plugin/influxdb/base/MetricsDAO.java | 36 +-- .../plugin/influxdb/base/NoneStreamDAO.java | 14 +- .../plugin/influxdb/base/RecordDAO.java | 11 +- .../influxdb/query/AggregationQuery.java | 17 +- .../query/InfluxMetadataQueryDAO.java | 146 ---------- .../plugin/influxdb/query/LogQuery.java | 4 +- .../plugin/influxdb/query/MetadataQuery.java | 255 ++++++++++++++++++ .../plugin/influxdb/query/MetricsQuery.java | 12 +- .../query/NetworkAddressAliasDAO.java | 86 ++++++ .../influxdb/query/ProfileTaskLogQuery.java | 5 +- .../influxdb/query/ProfileTaskQuery.java | 23 +- .../query/ProfileThreadSnapshotQuery.java | 25 +- .../influxdb/query/TopNRecordsQuery.java | 5 +- .../plugin/influxdb/query/TraceQuery.java | 5 +- 19 files changed, 559 insertions(+), 255 deletions(-) rename oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/{InfluxModelConstants.java => InfluxConstants.java} (70%) rename oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/{query/InfluxNetworkAddressAlias.java => InfluxTableInstaller.java} (52%) delete mode 100644 oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxMetadataQueryDAO.java create mode 100644 oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java create mode 100644 oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java index cc4c06ebc5ba..2282f5d809af 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import okhttp3.OkHttpClient; @@ -136,6 +137,22 @@ public QueryResult.Series queryForSingleSeries(Query query) throws IOException { return series.get(0); } + /** + * Execute a query against InfluxDB with a `select count(*)` statement and return the count only. + * + * @throws IOException if there is an error on the InfluxDB server or communication error + */ + public int getCounter(Query query) throws IOException { + QueryResult.Series series = queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + if (Objects.isNull(series)) { + return 0; + } + return ((Number) series.getValues().get(0).get(1)).intValue(); + } + /** * Data management, to drop a time-series by measurement and time-series name specified. If an exception isn't * thrown, it means execution success. Notice, drop series don't support to drop series by range diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxModelConstants.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java similarity index 70% rename from oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxModelConstants.java rename to oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java index 18dd47dc2816..1f3cea86dcfd 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxModelConstants.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java @@ -18,9 +18,23 @@ package org.apache.skywalking.oap.server.storage.plugin.influxdb; -public interface InfluxModelConstants { - /** - * Override column because the 'duration' is the identifier of InfluxDB. - */ - String DURATION = "dur"; +public interface InfluxConstants { + String ID_COLUMN = "id"; + + String ALL_FIELDS = "*::field"; + + String SORT_DES = "top"; + + String SORT_ASC = "bottom"; + + interface TagName { + + String ENTITY_ID = "_entity_id"; + + String TIME_BUCKET = "_time_bucket"; + + String NODE_TYPE = "_node_type"; + + String SERVICE_ID = "_service_id"; + } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java index e501b4f41920..ae677488cac7 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java @@ -23,6 +23,7 @@ import org.apache.skywalking.oap.server.core.storage.IBatchDAO; import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageException; import org.apache.skywalking.oap.server.core.storage.StorageModule; import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressAliasDAO; import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO; @@ -46,10 +47,10 @@ import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.InfluxStorageDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.AggregationQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.AlarmQuery; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.InfluxMetadataQueryDAO; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.InfluxNetworkAddressAlias; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.LogQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.MetadataQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.MetricsQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.NetworkAddressAliasDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.ProfileTaskLogQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.ProfileTaskQuery; import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.ProfileThreadSnapshotQuery; @@ -60,7 +61,7 @@ @Slf4j public class InfluxStorageProvider extends ModuleProvider { private InfluxStorageConfig config; - private InfluxClient influxClient; + private InfluxClient client; public InfluxStorageProvider() { config = new InfluxStorageConfig(); @@ -83,35 +84,42 @@ public ModuleConfig createConfigBeanIfAbsent() { @Override public void prepare() throws ServiceNotProvidedException { - influxClient = new InfluxClient(config); + client = new InfluxClient(config); - this.registerServiceImplementation(IBatchDAO.class, new BatchDAO(influxClient)); - this.registerServiceImplementation(StorageDAO.class, new InfluxStorageDAO(influxClient)); + this.registerServiceImplementation(IBatchDAO.class, new BatchDAO(client)); + this.registerServiceImplementation(StorageDAO.class, new InfluxStorageDAO(client)); - this.registerServiceImplementation(INetworkAddressAliasDAO.class, new InfluxNetworkAddressAlias(influxClient)); - this.registerServiceImplementation(IMetadataQueryDAO.class, new InfluxMetadataQueryDAO(influxClient)); + this.registerServiceImplementation(INetworkAddressAliasDAO.class, new NetworkAddressAliasDAO(client)); + this.registerServiceImplementation(IMetadataQueryDAO.class, new MetadataQuery(client)); - this.registerServiceImplementation(ITopologyQueryDAO.class, new TopologyQuery(influxClient)); - this.registerServiceImplementation(IMetricsQueryDAO.class, new MetricsQuery(influxClient)); - this.registerServiceImplementation(ITraceQueryDAO.class, new TraceQuery(influxClient)); - this.registerServiceImplementation(IAggregationQueryDAO.class, new AggregationQuery(influxClient)); - this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQuery(influxClient)); - this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new TopNRecordsQuery(influxClient)); - this.registerServiceImplementation(ILogQueryDAO.class, new LogQuery(influxClient)); + this.registerServiceImplementation(ITopologyQueryDAO.class, new TopologyQuery(client)); + this.registerServiceImplementation(IMetricsQueryDAO.class, new MetricsQuery(client)); + this.registerServiceImplementation(ITraceQueryDAO.class, new TraceQuery(client)); + this.registerServiceImplementation(IAggregationQueryDAO.class, new AggregationQuery(client)); + this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQuery(client)); + this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new TopNRecordsQuery(client)); + this.registerServiceImplementation(ILogQueryDAO.class, new LogQuery(client)); - this.registerServiceImplementation(IProfileTaskQueryDAO.class, new ProfileTaskQuery(influxClient)); + this.registerServiceImplementation(IProfileTaskQueryDAO.class, new ProfileTaskQuery(client)); this.registerServiceImplementation( - IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQuery(influxClient)); + IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQuery(client)); this.registerServiceImplementation( - IProfileTaskLogQueryDAO.class, new ProfileTaskLogQuery(influxClient, config.getFetchTaskLogMaxSize())); + IProfileTaskLogQueryDAO.class, new ProfileTaskLogQuery(client, config.getFetchTaskLogMaxSize())); this.registerServiceImplementation( - IHistoryDeleteDAO.class, new HistoryDeleteDAO(influxClient)); + IHistoryDeleteDAO.class, new HistoryDeleteDAO(client)); } @Override public void start() throws ServiceNotProvidedException, ModuleStartException { - influxClient.connect(); + client.connect(); + + InfluxTableInstaller installer = new InfluxTableInstaller(getManager()); + try { + installer.install(client); + } catch (StorageException e) { + throw new ModuleStartException(e.getMessage(), e); + } } @Override diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxNetworkAddressAlias.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxTableInstaller.java similarity index 52% rename from oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxNetworkAddressAlias.java rename to oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxTableInstaller.java index 90b47e22d9ba..585df52d0f5c 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxNetworkAddressAlias.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxTableInstaller.java @@ -16,22 +16,28 @@ * */ -package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; +package org.apache.skywalking.oap.server.storage.plugin.influxdb; -import java.util.List; -import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; -import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressAliasDAO; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller; +import org.apache.skywalking.oap.server.library.client.Client; +import org.apache.skywalking.oap.server.library.module.ModuleManager; -public class InfluxNetworkAddressAlias implements INetworkAddressAliasDAO { - private InfluxClient client; +public class InfluxTableInstaller extends ModelInstaller { - public InfluxNetworkAddressAlias(final InfluxClient client) { - this.client = client; + public InfluxTableInstaller(ModuleManager moduleManager) { + super(moduleManager); } @Override - public List loadLastUpdate(final long timeBucket) { - return null; + protected boolean isExists(final Client client, final Model model) throws StorageException { + TableMetaInfo.addModel(model); + return true; + } + + @Override + protected void createTable(final Client client, final Model model) throws StorageException { + // Automatically create table } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java index 4cc9e806acf3..0944870795f3 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java @@ -18,18 +18,65 @@ package org.apache.skywalking.oap.server.storage.plugin.influxdb; +import com.google.common.collect.Maps; import java.util.HashMap; +import java.util.List; import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.storage.model.ColumnName; import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; +@Getter +@Builder +@AllArgsConstructor public class TableMetaInfo { - private static Map TABLES = new HashMap<>(); + private static final Map TABLES = new HashMap<>(); + private static final String PREFIX = "_"; + + private Map storageAndColumnMap; + private Map storageAndTagMap; + private Model model; public static void addModel(Model model) { - TABLES.put(model.getName(), model); + final List columns = model.getColumns(); + final Map storageAndTagMap = Maps.newHashMap(); + final Map storageAndColumnMap = Maps.newHashMap(); + columns.forEach(column -> { + ColumnName columnName = column.getColumnName(); + storageAndColumnMap.put(columnName.getStorageName(), columnName.getName()); + }); + + // + if (storageAndColumnMap.containsKey(Metrics.ENTITY_ID)) { + storageAndTagMap.put(Metrics.ENTITY_ID, InfluxConstants.TagName.ENTITY_ID); + } + if (storageAndColumnMap.containsKey(Record.TIME_BUCKET)) { + storageAndTagMap.put(Record.TIME_BUCKET, InfluxConstants.TagName.TIME_BUCKET); + } + if (storageAndColumnMap.containsKey(ServiceTraffic.NODE_TYPE)) { + storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE); + } + if (storageAndColumnMap.containsKey(SegmentRecord.SERVICE_ID)) { + storageAndTagMap.put(SegmentRecord.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); + } + + TableMetaInfo info = TableMetaInfo.builder() + .model(model) + .storageAndTagMap(storageAndTagMap) + .storageAndColumnMap(storageAndColumnMap) + .build(); + TABLES.put(model.getName(), info); } - public static Model get(String moduleName) { + public static TableMetaInfo get(String moduleName) { return TABLES.get(moduleName); } + } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java index eb3008f5ddfe..e471d7e1e989 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java @@ -27,18 +27,15 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; -import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; -import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; -import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; import org.apache.skywalking.oap.server.core.storage.StorageBuilder; import org.apache.skywalking.oap.server.core.storage.model.Model; -import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject; import org.apache.skywalking.oap.server.library.client.request.InsertRequest; import org.apache.skywalking.oap.server.library.client.request.UpdateRequest; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; @@ -47,9 +44,6 @@ import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; public class MetricsDAO implements IMetricsDAO { - public static final String TAG_ENTITY_ID = "_entity_id"; - public static final String TAG_ENDPOINT_OWNER_SERVICE = "_service_id"; - public static final String TAG_ENDPOINT_NAME = "_endpoint_name"; private final StorageBuilder storageBuilder; private final InfluxClient client; @@ -73,10 +67,9 @@ public List multiGet(Model model, List ids) throws IOException final List metrics = Lists.newArrayList(); List columns = series.getColumns(); - Map storageAndColumnNames = Maps.newHashMap(); - for (ModelColumn column : model.getColumns()) { - storageAndColumnNames.put(column.getColumnName().getStorageName(), column.getColumnName().getName()); - } + + TableMetaInfo metaInfo = TableMetaInfo.get(model.getName()); + Map storageAndColumnMap = metaInfo.getStorageAndColumnMap(); series.getValues().forEach(values -> { Map data = Maps.newHashMap(); @@ -87,7 +80,7 @@ public List multiGet(Model model, List ids) throws IOException value = ((StorageDataComplexObject) value).toStorageData(); } - data.put(storageAndColumnNames.get(columns.get(i)), value); + data.put(storageAndColumnMap.get(columns.get(i)), value); } metrics.add(storageBuilder.map2Data(data)); @@ -99,16 +92,15 @@ public List multiGet(Model model, List ids) throws IOException @Override public InsertRequest prepareBatchInsert(Model model, Metrics metrics) throws IOException { final long timestamp = TimeBucket.getTimestamp(metrics.getTimeBucket(), model.getDownsampling()); - if (metrics instanceof EndpointTraffic || metrics instanceof ServiceTraffic || metrics instanceof InstanceTraffic) { - return new InfluxInsertRequest(model, metrics, storageBuilder) - .time(timestamp, TimeUnit.MILLISECONDS) - .addFieldAsTag(EndpointTraffic.SERVICE_ID, TAG_ENDPOINT_OWNER_SERVICE) - .addFieldAsTag(EndpointTraffic.NAME, TAG_ENDPOINT_NAME); - } else { - return new InfluxInsertRequest(model, metrics, storageBuilder) - .time(timestamp, TimeUnit.MILLISECONDS) - .addFieldAsTag(Metrics.ENTITY_ID, TAG_ENTITY_ID); - } + TableMetaInfo tableMetaInfo = TableMetaInfo.get(model.getName()); + + InfluxInsertRequest request = new InfluxInsertRequest(model, metrics, storageBuilder) + .time(timestamp, TimeUnit.MILLISECONDS); + + tableMetaInfo.getStorageAndTagMap().forEach((field, tag) -> { + request.addFieldAsTag(field, tag); + }); + return request; } @Override diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java index ae00288aa677..a9873b0106ce 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java @@ -23,12 +23,11 @@ import org.apache.skywalking.apm.commons.datacarrier.common.AtomicRangeInteger; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; -import org.apache.skywalking.oap.server.core.profile.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.storage.INoneStreamDAO; import org.apache.skywalking.oap.server.core.storage.StorageBuilder; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; -import org.influxdb.dto.Point; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; public class NoneStreamDAO implements INoneStreamDAO { public static final String TAG_SERVICE_ID = "_service_id"; @@ -48,10 +47,11 @@ public void insert(final Model model, final NoneStream noneStream) throws IOExce final long timestamp = TimeBucket.getTimestamp( noneStream.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); - Point point = new InfluxInsertRequest(model, noneStream, storageBuilder) - .time(timestamp, TimeUnit.NANOSECONDS) - .addFieldAsTag(ProfileTaskRecord.SERVICE_ID, TAG_SERVICE_ID).getPoint(); - - client.write(point); + InfluxInsertRequest request = new InfluxInsertRequest(model, noneStream, storageBuilder) + .time(timestamp, TimeUnit.NANOSECONDS); + TableMetaInfo.get(model.getName()).getStorageAndTagMap().forEach((field, tag) -> { + request.addFieldAsTag(field, tag); + }); + client.write(request.getPoint()); } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java index fae2e4109cd8..5cdb2afa6e11 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java @@ -22,13 +22,13 @@ import java.util.concurrent.TimeUnit; import org.apache.skywalking.apm.commons.datacarrier.common.AtomicRangeInteger; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; -import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.storage.IRecordDAO; import org.apache.skywalking.oap.server.core.storage.StorageBuilder; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.library.client.request.InsertRequest; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; public class RecordDAO implements IRecordDAO { public static final String TAG_SERVICE_ID = "_service_id"; @@ -48,8 +48,11 @@ public InsertRequest prepareBatchInsert(Model model, Record record) throws IOExc final long timestamp = TimeBucket.getTimestamp( record.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); - return new InfluxInsertRequest(model, record, storageBuilder) - .time(timestamp, TimeUnit.NANOSECONDS) - .addFieldAsTag(SegmentRecord.SERVICE_ID, TAG_SERVICE_ID); + InfluxInsertRequest request = new InfluxInsertRequest(model, record, storageBuilder) + .time(timestamp, TimeUnit.NANOSECONDS); + TableMetaInfo.get(model.getName()).getStorageAndTagMap().forEach((field, tag) -> { + request.addFieldAsTag(field, tag); + }); + return request; } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java index 2ab6b213710f..9d3c76a92a96 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java @@ -27,11 +27,12 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.query.entity.Order; import org.apache.skywalking.oap.server.core.query.entity.TopNEntity; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.MetricsDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.SelectSubQueryImpl; @@ -68,7 +69,7 @@ public List getServiceInstanceTopN(String serviceId, String indName, long startTB, long endTB, Order order) throws IOException { return getTopNEntity( downsampling, indName, - subQuery(InstanceTraffic.SERVICE_ID, serviceId, indName, valueCName, startTB, endTB), order, topN + subQuery(InfluxConstants.TagName.SERVICE_ID, serviceId, indName, valueCName, startTB, endTB), order, topN ); } @@ -84,7 +85,7 @@ public List getEndpointTopN(String serviceId, String indName, String long startTB, long endTB, Order order) throws IOException { return getTopNEntity( downsampling, indName, - subQuery(EndpointTraffic.SERVICE_ID, serviceId, indName, valueCName, startTB, endTB), order, topN + subQuery(InfluxConstants.TagName.SERVICE_ID, serviceId, indName, valueCName, startTB, endTB), order, topN ); } @@ -95,14 +96,14 @@ private List getTopNEntity(DownSampling downsampling, int topN) throws IOException { // Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`. Comparator comparator = DESCENDING; - String functionName = "top"; + String functionName = InfluxConstants.SORT_DES; if (order == Order.ASC) { - functionName = "bottom"; + functionName = InfluxConstants.SORT_ASC; comparator = ASCENDING; } SelectQueryImpl query = select().function(functionName, "mean", topN).as("value") - .column(MetricsDAO.TAG_ENTITY_ID) + .column(InfluxConstants.TagName.ENTITY_ID) .from(client.getDatabase(), measurement); query.setSubQuery(subQuery); @@ -135,7 +136,7 @@ private SelectSubQueryImpl subQuery(String serviceColumnName, S .and(eq(serviceColumnName, serviceId)) .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))) - .groupBy(MetricsDAO.TAG_ENTITY_ID); + .groupBy(InfluxConstants.TagName.ENTITY_ID); } private SelectSubQueryImpl subQuery(String name, String columnName, long startTB, long endTB) { @@ -143,7 +144,7 @@ private SelectSubQueryImpl subQuery(String name, String columnN .where() .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))) - .groupBy(MetricsDAO.TAG_ENTITY_ID); + .groupBy(InfluxConstants.TagName.ENTITY_ID); } private static final Comparator ASCENDING = Comparator.comparingLong(TopNEntity::getValue); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxMetadataQueryDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxMetadataQueryDAO.java deleted file mode 100644 index b300f85f3292..000000000000 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/InfluxMetadataQueryDAO.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; - -import com.google.common.base.Strings; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; -import org.apache.skywalking.oap.server.core.query.entity.Database; -import org.apache.skywalking.oap.server.core.query.entity.Endpoint; -import org.apache.skywalking.oap.server.core.query.entity.Service; -import org.apache.skywalking.oap.server.core.query.entity.ServiceInstance; -import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.MetricsDAO; -import org.influxdb.dto.Query; -import org.influxdb.dto.QueryResult; -import org.influxdb.querybuilder.SelectQueryImpl; -import org.influxdb.querybuilder.WhereQueryImpl; - -import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; -import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; -import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; - -public class InfluxMetadataQueryDAO implements IMetadataQueryDAO { - private InfluxClient client; - // 'name' is InfluxDB keyword, so escapes it - private static final String ENDPOINT_NAME = '\"' + EndpointTraffic.NAME + '\"'; - - public InfluxMetadataQueryDAO(final InfluxClient client) { - this.client = client; - } - - @Override - public int numOfService(final long startTimestamp, final long endTimestamp) throws IOException { - return 0; - } - - @Override - public int numOfEndpoint() throws IOException { - final SelectQueryImpl selectQuery = select() - .count(EndpointTraffic.ENTITY_ID) - .from(client.getDatabase(), EndpointTraffic.INDEX_NAME); - - Query query = new Query(selectQuery.getCommand()); - - final QueryResult.Series series = client.queryForSingleSeries(query); - if (series == null) { - return 0; - } - - return ((Number) series.getValues().get(0).get(1)).intValue(); - } - - @Override - public int numOfConjectural(final int nodeTypeValue) throws IOException { - return 0; - } - - @Override - public List getAllServices(final long startTimestamp, final long endTimestamp) throws IOException { - return null; - } - - @Override - public List getAllBrowserServices(final long startTimestamp, final long endTimestamp) throws IOException { - return null; - } - - @Override - public List getAllDatabases() throws IOException { - return null; - } - - @Override - public List searchServices(final long startTimestamp, - final long endTimestamp, - final String keyword) throws IOException { - return null; - } - - @Override - public Service searchService(final String serviceCode) throws IOException { - return null; - } - - @Override - public List searchEndpoint(final String keyword, - final String serviceId, - final int limit) throws IOException { - WhereQueryImpl endpointQuery = select() - .column(EndpointTraffic.SERVICE_ID) - .column(ENDPOINT_NAME) - .from(client.getDatabase(), EndpointTraffic.INDEX_NAME) - .where(); - endpointQuery.where(eq(MetricsDAO.TAG_ENDPOINT_OWNER_SERVICE, String.valueOf(serviceId))); - if (!Strings.isNullOrEmpty(keyword)) { - endpointQuery.where(contains(MetricsDAO.TAG_ENDPOINT_NAME, keyword.replaceAll("/", "\\\\/"))); - } - endpointQuery.limit(limit); - - Query query = new Query(endpointQuery.getCommand()); - - final QueryResult.Series series = client.queryForSingleSeries(query); - - List list = new ArrayList<>(limit); - if (series != null) { - series.getValues().forEach(values -> { - EndpointTraffic endpointTraffic = new EndpointTraffic(); - endpointTraffic.setServiceId((String) values.get(1)); - endpointTraffic.setName((String) values.get(2)); - - Endpoint endpoint = new Endpoint(); - endpoint.setId(IDManager.EndpointID.buildId(endpointTraffic.getServiceId(), endpointTraffic.getName())); - endpoint.setName(endpointTraffic.getName()); - list.add(endpoint); - }); - } - return list; - } - - @Override - public List getServiceInstances(final long startTimestamp, - final long endTimestamp, - final String serviceId) throws IOException { - return null; - } -} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java index fe69ac04e9c7..dc2d2720d7fd 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java @@ -33,7 +33,7 @@ import org.apache.skywalking.oap.server.core.query.entity.Pagination; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.elasticsearch.common.Strings; import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; @@ -72,7 +72,7 @@ public Logs queryLogs(String metricName, int serviceId, int serviceInstanceId, S .from(client.getDatabase(), metricName) .where(); if (serviceId != Const.NONE) { - recallQuery.and(eq(RecordDAO.TAG_SERVICE_ID, String.valueOf(serviceId))); + recallQuery.and(eq(InfluxConstants.TagName.SERVICE_ID, String.valueOf(serviceId))); } if (serviceInstanceId != Const.NONE) { recallQuery.and(eq(SERVICE_INSTANCE_ID, serviceInstanceId)); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java new file mode 100644 index 000000000000..91da45264009 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.NodeType; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.query.entity.Attribute; +import org.apache.skywalking.oap.server.core.query.entity.Database; +import org.apache.skywalking.oap.server.core.query.entity.Endpoint; +import org.apache.skywalking.oap.server.core.query.entity.Language; +import org.apache.skywalking.oap.server.core.query.entity.LanguageTrans; +import org.apache.skywalking.oap.server.core.query.entity.Service; +import org.apache.skywalking.oap.server.core.query.entity.ServiceInstance; +import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ID_COLUMN; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class MetadataQuery implements IMetadataQueryDAO { + private static final Gson GSON = new Gson(); + + private static final String NAME = "\"" + ServiceTraffic.NAME + "\""; // 'name' is InfluxDB keyword, so escapes it + private final InfluxClient client; + + public MetadataQuery(final InfluxClient client) { + this.client = client; + } + + @Override + public int numOfService(final long startTimestamp, final long endTimestamp) throws IOException { + WhereQueryImpl query = select().countAll() + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where() + .and(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + return client.getCounter(query); + } + + @Override + public int numOfEndpoint() throws IOException { + SelectQueryImpl query = select() + .count(EndpointTraffic.ENTITY_ID) + .from(client.getDatabase(), EndpointTraffic.INDEX_NAME); + return client.getCounter(query); + } + + @Override + public int numOfConjectural(final int nodeTypeValue) throws IOException { + WhereQueryImpl query = select().countAll() + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, nodeTypeValue)); + return client.getCounter(query); + } + + @Override + public List getAllServices(final long startTimestamp, final long endTimestamp) throws IOException { + WhereQueryImpl query = select(ID_COLUMN, NAME) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + return buildServices(query); + } + + @Override + public List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException { + WhereQueryImpl query = select(ID_COLUMN, NAME) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + return buildServices(query); + } + + @Override + public List getAllDatabases() throws IOException { + WhereQueryImpl query = select(ID_COLUMN, NAME) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Database.value())); + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + + List databases = Lists.newArrayList(); + if (Objects.nonNull(series)) { + for (List values : series.getValues()) { + Database database = new Database(); + database.setId((String) values.get(1)); + database.setName((String) values.get(2)); + databases.add(database); + } + } + return databases; + } + + @Override + public List searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException { + WhereQueryImpl query = select(ID_COLUMN, NAME) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + if (!Strings.isNullOrEmpty(keyword)) { + query.and(contains(ServiceTraffic.NAME, keyword)); + } + return buildServices(query); + } + + @Override + public Service searchService(String serviceCode) throws IOException { + WhereQueryImpl query = select(ID_COLUMN, NAME) + .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + query.and(eq(ServiceTraffic.NAME, serviceCode)); + return buildServices(query).get(0); + } + + @Override + public List searchEndpoint(final String keyword, + final String serviceId, + final int limit) throws IOException { + WhereQueryImpl query = select(EndpointTraffic.SERVICE_ID, NAME) + .from(client.getDatabase(), EndpointTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.SERVICE_ID, String.valueOf(serviceId))); + if (!Strings.isNullOrEmpty(keyword)) { + query.where(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/"))); + } + query.limit(limit); + + final QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + + List list = new ArrayList<>(limit); + if (series != null) { + series.getValues().forEach(values -> { + EndpointTraffic endpointTraffic = new EndpointTraffic(); + endpointTraffic.setServiceId((String) values.get(1)); + endpointTraffic.setName((String) values.get(2)); + + Endpoint endpoint = new Endpoint(); + endpoint.setId(IDManager.EndpointID.buildId(endpointTraffic.getServiceId(), endpointTraffic.getName())); + endpoint.setName(endpointTraffic.getName()); + list.add(endpoint); + }); + } + return list; + } + + @Override + public List getServiceInstances(final long startTimestamp, + final long endTimestamp, + final String serviceId) throws IOException { + final long minuteTimeBucket = TimeBucket.getMinuteTimeBucket(startTimestamp); + + WhereQueryImpl query = select(InstanceTraffic.SERVICE_ID, NAME, InstanceTraffic.PROPERTIES) + .from(client.getDatabase(), InstanceTraffic.INDEX_NAME) + .where() + .and(gte(InstanceTraffic.LAST_PING_TIME_BUCKET, minuteTimeBucket)) + .and(eq(InfluxConstants.TagName.SERVICE_ID, serviceId)); + + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + + if (Objects.isNull(series)) { + return Collections.EMPTY_LIST; + } + + List> result = series.getValues(); + List instances = Lists.newArrayList(); + for (List values : result) { + ServiceInstance serviceInstance = new ServiceInstance(); + + serviceInstance.setId((String) values.get(1)); + serviceInstance.setName((String) values.get(2)); + serviceInstance.setInstanceUUID(serviceInstance.getId()); + + String propertiesString = (String) values.get(3); + if (!Strings.isNullOrEmpty(propertiesString)) { + JsonObject properties = GSON.fromJson(propertiesString, JsonObject.class); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + String value = property.getValue().getAsString(); + if (key.equals(InstanceTraffic.PropertyUtil.LANGUAGE)) { + serviceInstance.setLanguage(LanguageTrans.INSTANCE.value(value)); + } else { + serviceInstance.getAttributes().add(new Attribute(key, value)); + } + + } + } else { + serviceInstance.setLanguage(Language.UNKNOWN); + } + instances.add(serviceInstance); + } + return instances; + } + + private List buildServices(WhereQueryImpl query) throws IOException { + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + + ArrayList services = Lists.newArrayList(); + if (Objects.nonNull(series)) { + for (List values : series.getValues()) { + Service service = new Service(); + service.setId((String) values.get(1)); + service.setName((String) values.get(2)); + services.add(service); + } + } + return services; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java index 09b2107e6377..dd0da3c2e997 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java @@ -39,8 +39,8 @@ import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.MetricsDAO; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.SelectionQueryImpl; @@ -74,7 +74,7 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s WhereQueryImpl queryWhereQuery = query.from(client.getDatabase(), measurement).where(); Map> columnTypes = Maps.newHashMap(); - for (ModelColumn column : TableMetaInfo.get(measurement).getColumns()) { + for (ModelColumn column : TableMetaInfo.get(measurement).getModel().getColumns()) { columnTypes.put(column.getColumnName().getStorageName(), column.getType()); } @@ -86,9 +86,9 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s final List values = kv.getValues(); Class type = columnTypes.get(kv.getKey()); - if (values.size() == 1) { + if (values.size() == 1) { // FIXME String value = kv.getValues().get(0); - if (type == String.class) { + if (type != String.class) { value = "'" + value + "'"; } clauseBuilder.append(kv.getKey()).append("=").append(value).append(" OR "); @@ -111,7 +111,7 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s queryWhereQuery .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB, downsampling))) .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB, downsampling))) - .groupBy(MetricsDAO.TAG_ENTITY_ID); + .groupBy(InfluxConstants.TagName.ENTITY_ID); IntValues intValues = new IntValues(); List seriesList = client.queryForSeries(queryWhereQuery); @@ -121,7 +121,7 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s if (!(seriesList == null || seriesList.isEmpty())) { for (QueryResult.Series series : seriesList) { KVInt kv = new KVInt(); - kv.setId(series.getTags().get(MetricsDAO.TAG_ENTITY_ID)); + kv.setId(series.getTags().get(InfluxConstants.TagName.ENTITY_ID)); Number value = (Number) series.getValues().get(0).get(1); kv.setValue(value.longValue()); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java new file mode 100644 index 000000000000..94c18cf2810d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; +import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressAliasDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class NetworkAddressAliasDAO implements INetworkAddressAliasDAO { + private final NetworkAddressAlias.Builder builder = new NetworkAddressAlias.Builder(); + private InfluxClient client; + + public NetworkAddressAliasDAO(final InfluxClient client) { + this.client = client; + } + + @Override + public List loadLastUpdate(final long timeBucket) { + List networkAddressAliases = new ArrayList<>(); + + WhereQueryImpl query = select().raw(InfluxConstants.ALL_FIELDS) + .from(client.getDatabase(), NetworkAddressAlias.INDEX_NAME) + .where(gte( + NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET, + timeBucket + )); + try { + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + if (Objects.isNull(series)) { + return networkAddressAliases; + } + + List> result = series.getValues(); + List columns = series.getColumns(); + + Map columnAndFieldMap = TableMetaInfo.get(NetworkAddressAlias.INDEX_NAME) + .getStorageAndColumnMap(); + for (List values : result) { + Map map = Maps.newHashMap(); + for (int i = 0; i < columns.size(); i++) { + map.put(columnAndFieldMap.get(columns.get(i)), values.get(1 + i)); + } + networkAddressAliases.add(builder.map2Data(map)); + } + } catch (IOException e) { + log.error(e.getMessage(), e); + } + + return networkAddressAliases; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java index cfac9db7a5d7..f96a4c9e0e47 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java @@ -30,6 +30,7 @@ import org.apache.skywalking.oap.server.core.query.entity.ProfileTaskLogOperationType; import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; @@ -50,7 +51,7 @@ public ProfileTaskLogQuery(InfluxClient client, int fetchTaskLogMaxSize) { public List getTaskLogList() throws IOException { WhereQueryImpl query = select() .function("top", ProfileTaskLogRecord.OPERATION_TIME, fetchTaskLogMaxSize) - .column("id") + .column(InfluxConstants.ID_COLUMN) .column(ProfileTaskLogRecord.TASK_ID) .column(ProfileTaskLogRecord.INSTANCE_ID) .column(ProfileTaskLogRecord.OPERATION_TIME) @@ -77,7 +78,7 @@ public List getTaskLogList() throws IOException { .sorted((a, b) -> Long.compare(((Number) b.get(1)).longValue(), ((Number) a.get(1)).longValue())) .forEach(values -> { taskLogs.add(ProfileTaskLog.builder() - .id((String) values.get(columnsMap.get("id"))) + .id((String) values.get(columnsMap.get(InfluxConstants.ID_COLUMN))) .taskId((String) values.get(columnsMap.get(ProfileTaskLogRecord.TASK_ID))) .instanceId( (String) values.get(columnsMap.get(ProfileTaskLogRecord.INSTANCE_ID))) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java index db2b2f2da858..72583b069342 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java @@ -22,12 +22,12 @@ import java.io.IOException; import java.util.List; import java.util.Objects; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.profile.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.entity.ProfileTask; import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxModelConstants; import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.NoneStreamDAO; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; @@ -38,8 +38,11 @@ import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; +@Slf4j public class ProfileTaskQuery implements IProfileTaskQueryDAO { - private InfluxClient client; + private static final String DURATION = "\"" + "duration" + "\""; + private static final String ID_COLUMN = "id"; + private final InfluxClient client; public ProfileTaskQuery(InfluxClient client) { this.client = client; @@ -52,10 +55,10 @@ public List getTaskList(final String serviceId, final Long endTimeBucket, final Integer limit) throws IOException { WhereQueryImpl query = - select("id", ProfileTaskRecord.SERVICE_ID, + select(ID_COLUMN, ProfileTaskRecord.SERVICE_ID, ProfileTaskRecord.ENDPOINT_NAME, ProfileTaskRecord.START_TIME, ProfileTaskRecord.CREATE_TIME, - InfluxModelConstants.DURATION, + DURATION, ProfileTaskRecord.MIN_DURATION_THRESHOLD, ProfileTaskRecord.DUMP_PERIOD, ProfileTaskRecord.MAX_SAMPLING_COUNT @@ -81,6 +84,9 @@ public List getTaskList(final String serviceId, List tasks = Lists.newArrayList(); QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } if (series != null) { series.getValues().forEach(values -> { tasks.add(profileTaskBuilder(values)); @@ -94,20 +100,23 @@ public ProfileTask getById(final String id) throws IOException { if (StringUtil.isEmpty(id)) { return null; } - SelectQueryImpl query = select("id", ProfileTaskRecord.SERVICE_ID, + SelectQueryImpl query = select(ID_COLUMN, ProfileTaskRecord.SERVICE_ID, ProfileTaskRecord.ENDPOINT_NAME, ProfileTaskRecord.START_TIME, ProfileTaskRecord.CREATE_TIME, - InfluxModelConstants.DURATION, + DURATION, ProfileTaskRecord.MIN_DURATION_THRESHOLD, ProfileTaskRecord.DUMP_PERIOD, ProfileTaskRecord.MAX_SAMPLING_COUNT ) .from(client.getDatabase(), ProfileTaskRecord.INDEX_NAME) .where() - .and(eq("id", id)) + .and(eq(ID_COLUMN, id)) .limit(1); QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } if (Objects.nonNull(series)) { return profileTaskBuilder(series.getValues().get(0)); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java index 36a99371ce70..f8f6a6b2dd73 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java @@ -26,6 +26,8 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.util.StringUtil; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; @@ -43,6 +45,7 @@ import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; +@Slf4j public class ProfileThreadSnapshotQuery implements IProfileThreadSnapshotQueryDAO { private final InfluxClient client; @@ -60,7 +63,7 @@ public List queryProfiledSegments(String taskId) throws IOException final LinkedList segments = new LinkedList<>(); QueryResult.Series series = client.queryForSingleSeries(query); - if (series == null) { + if (Objects.isNull(series)) { return Collections.emptyList(); } series.getValues().forEach(values -> { @@ -130,8 +133,15 @@ public List queryRecords(String segmentId, int minS .and(gte(ProfileThreadSnapshotRecord.SEQUENCE, minSequence)) .and(lte(ProfileThreadSnapshotRecord.SEQUENCE, maxSequence)); + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } + if (Objects.isNull(series)) { + return Collections.EMPTY_LIST; + } ArrayList result = new ArrayList<>(maxSequence - minSequence); - client.queryForSingleSeries(query).getValues().forEach(values -> { + series.getValues().forEach(values -> { ProfileThreadSnapshotRecord record = new ProfileThreadSnapshotRecord(); record.setTaskId((String) values.get(1)); @@ -165,7 +175,10 @@ public SegmentRecord getProfiledSegment(String segmentId) throws IOException { .where() .and(eq(SegmentRecord.SEGMENT_ID, segmentId)); List series = client.queryForSeries(query); - if (series == null || series.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (Objects.isNull(series) || series.isEmpty()) { return null; } @@ -198,10 +211,6 @@ private int querySequenceWithAgg(String function, String segmentId, long start, .and(eq(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId)) .and(gte(ProfileThreadSnapshotRecord.DUMP_TIME, start)) .and(lte(ProfileThreadSnapshotRecord.DUMP_TIME, end)); - QueryResult.Series series = client.queryForSingleSeries(query); - if (series == null) { - return -1; - } - return ((Number) series.getValues().get(0).get(1)).intValue(); + return client.getCounter(query); } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java index c247dd6c3d13..1603141a4038 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java @@ -30,6 +30,7 @@ import org.apache.skywalking.oap.server.core.query.entity.TopNRecord; import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.WhereQueryImpl; @@ -50,11 +51,11 @@ public TopNRecordsQuery(InfluxClient client) { @Override public List getTopNRecords(long startSecondTB, long endSecondTB, String metricName, String serviceId, int topN, Order order) throws IOException { - String function = "bottom"; + String function = InfluxConstants.SORT_ASC; // Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`. Comparator comparator = Comparator.comparingLong(TopNRecord::getLatency); if (order.equals(Order.DES)) { - function = "top"; + function = InfluxConstants.SORT_DES; comparator = (a, b) -> Long.compare(b.getLatency(), a.getLatency()); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java index eac9706d5207..277051deab3c 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java @@ -34,6 +34,7 @@ import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; import org.apache.skywalking.oap.server.library.util.BooleanUtils; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; import org.elasticsearch.common.Strings; import org.influxdb.dto.Query; @@ -78,7 +79,7 @@ public TraceBrief queryBasicTraces(long startSecondTB, } WhereQueryImpl recallQuery = select() - .function("top", orderBy, limit + from) + .function(InfluxConstants.SORT_DES, orderBy, limit + from) .column(SegmentRecord.SEGMENT_ID) .column(SegmentRecord.START_TIME) .column(SegmentRecord.ENDPOINT_NAME) @@ -102,7 +103,7 @@ public TraceBrief queryBasicTraces(long startSecondTB, recallQuery.and(contains(SegmentRecord.ENDPOINT_NAME, endpointName.replaceAll("/", "\\\\/"))); } if (StringUtil.isNotEmpty(serviceId)) { - recallQuery.and(eq(RecordDAO.TAG_SERVICE_ID, String.valueOf(serviceId))); + recallQuery.and(eq(RecordDAO.TAG_SERVICE_ID, serviceId)); } if (StringUtil.isNotEmpty(serviceInstanceId)) { recallQuery.and(eq(SegmentRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); From 160ab950fc420dd63a57efb11356b97b5bb341b4 Mon Sep 17 00:00:00 2001 From: daming Date: Tue, 14 Apr 2020 17:08:44 +0800 Subject: [PATCH 02/24] fix checkstyle --- .../server/storage/plugin/influxdb/query/AggregationQuery.java | 3 --- .../server/storage/plugin/influxdb/query/MetadataQuery.java | 1 - 2 files changed, 4 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java index 9d3c76a92a96..48cd1efc47ec 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java @@ -25,9 +25,6 @@ import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.analysis.DownSampling; -import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; -import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; -import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.query.entity.Order; import org.apache.skywalking.oap.server.core.query.entity.TopNEntity; import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java index 91da45264009..845149cc94b7 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -46,7 +46,6 @@ import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; -import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; From 2f8a2318000957b797186cdb0e5bde156e01b626 Mon Sep 17 00:00:00 2001 From: daming Date: Wed, 15 Apr 2020 15:41:51 +0800 Subject: [PATCH 03/24] add influxdb to e2e test --- .github/workflows/e2e.cluster.yaml | 2 +- .github/workflows/e2e.profiling.yaml | 2 +- .github/workflows/e2e.storages.yaml | 2 +- .github/workflows/e2e.ttl.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.cluster.yaml b/.github/workflows/e2e.cluster.yaml index 6b87a833c93e..1fbd3b3739bf 100644 --- a/.github/workflows/e2e.cluster.yaml +++ b/.github/workflows/e2e.cluster.yaml @@ -35,7 +35,7 @@ jobs: strategy: matrix: coordinator: ['zk'] - storage: ['mysql', 'es6', 'es7'] #TODO: 'influxdb' + storage: ['mysql', 'es6', 'es7', 'influxdb'] env: SW_COORDINATOR: ${{ matrix.coordinator }} SW_STORAGE: ${{ matrix.storage }} diff --git a/.github/workflows/e2e.profiling.yaml b/.github/workflows/e2e.profiling.yaml index 714fcc360c04..235495a14b34 100644 --- a/.github/workflows/e2e.profiling.yaml +++ b/.github/workflows/e2e.profiling.yaml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['h2', 'mysql', 'es6', 'es7'] #TODO: 'influxdb' + storage: ['h2', 'mysql', 'es6', 'es7', 'influxdb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/.github/workflows/e2e.storages.yaml b/.github/workflows/e2e.storages.yaml index 5d9abf40f9b9..b1d4ac0b18b6 100644 --- a/.github/workflows/e2e.storages.yaml +++ b/.github/workflows/e2e.storages.yaml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['mysql', 'es6', 'es7'] #TODO: 'influxdb' + storage: ['mysql', 'es6', 'es7', 'influxdb'] env: SW_STORAGE: ${{ matrix.storage }} steps: diff --git a/.github/workflows/e2e.ttl.yaml b/.github/workflows/e2e.ttl.yaml index 3f0912147616..cec82a10b1af 100644 --- a/.github/workflows/e2e.ttl.yaml +++ b/.github/workflows/e2e.ttl.yaml @@ -34,7 +34,7 @@ jobs: timeout-minutes: 90 strategy: matrix: - storage: ['es6', 'es7'] #TODO: 'influxdb' + storage: ['es6', 'es7', 'influxdb'] env: SW_STORAGE: ${{ matrix.storage }} steps: From dde4f2dc9be8f3ac9f8fcb2d67300cd5dcf22026 Mon Sep 17 00:00:00 2001 From: daming Date: Thu, 16 Apr 2020 01:34:33 +0800 Subject: [PATCH 04/24] fix traffic --- .../plugin/influxdb/InfluxConstants.java | 6 ++ .../plugin/influxdb/TableMetaInfo.java | 40 ++++++--- .../influxdb/base/InfluxInsertRequest.java | 7 +- .../plugin/influxdb/base/MetricsDAO.java | 10 ++- .../plugin/influxdb/base/NoneStreamDAO.java | 2 +- .../plugin/influxdb/base/RecordDAO.java | 2 +- .../plugin/influxdb/query/LogQuery.java | 12 ++- .../plugin/influxdb/query/MetadataQuery.java | 90 +++++++++++++------ .../plugin/influxdb/query/MetricsQuery.java | 32 +++---- .../query/NetworkAddressAliasDAO.java | 4 +- .../plugin/influxdb/query/TraceQuery.java | 4 +- 11 files changed, 140 insertions(+), 69 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java index 1f3cea86dcfd..ecf5abf649a6 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java @@ -21,6 +21,8 @@ public interface InfluxConstants { String ID_COLUMN = "id"; + String NAME = "\"name\""; + String ALL_FIELDS = "*::field"; String SORT_DES = "top"; @@ -29,6 +31,10 @@ public interface InfluxConstants { interface TagName { + String ID_COLUMN = "_id"; + + String NAME = "_name"; + String ENTITY_ID = "_entity_id"; String TIME_BUCKET = "_time_bucket"; diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java index 0944870795f3..bdf3b55f5106 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java @@ -25,6 +25,8 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; +import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; @@ -53,18 +55,32 @@ public static void addModel(Model model) { storageAndColumnMap.put(columnName.getStorageName(), columnName.getName()); }); - // - if (storageAndColumnMap.containsKey(Metrics.ENTITY_ID)) { - storageAndTagMap.put(Metrics.ENTITY_ID, InfluxConstants.TagName.ENTITY_ID); - } - if (storageAndColumnMap.containsKey(Record.TIME_BUCKET)) { - storageAndTagMap.put(Record.TIME_BUCKET, InfluxConstants.TagName.TIME_BUCKET); - } - if (storageAndColumnMap.containsKey(ServiceTraffic.NODE_TYPE)) { - storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE); - } - if (storageAndColumnMap.containsKey(SegmentRecord.SERVICE_ID)) { - storageAndTagMap.put(SegmentRecord.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); + if (model.getName().endsWith("_traffic")) { // It is not a good way. It has performance issue. + // instance_traffic name, service_id + // endpoint_traffic name, service_id + // service_traffic name, node_type + storageAndTagMap.put(InstanceTraffic.NAME, InfluxConstants.TagName.NAME); + if ("instance_traffic".equals(model.getName()) + || "endpoint_traffic".equals(model.getName())) { + storageAndTagMap.put(EndpointTraffic.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); + } else { + storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE); + } + } else { + + // Specifies ENTITY_ID, TIME_BUCKET, NODE_TYPE, SERVICE_ID as tag + if (storageAndColumnMap.containsKey(Metrics.ENTITY_ID)) { + storageAndTagMap.put(Metrics.ENTITY_ID, InfluxConstants.TagName.ENTITY_ID); + } + if (storageAndColumnMap.containsKey(Record.TIME_BUCKET)) { + storageAndTagMap.put(Record.TIME_BUCKET, InfluxConstants.TagName.TIME_BUCKET); + } + if (storageAndColumnMap.containsKey(ServiceTraffic.NODE_TYPE)) { + storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE); + } + if (storageAndColumnMap.containsKey(SegmentRecord.SERVICE_ID)) { + storageAndTagMap.put(SegmentRecord.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); + } } TableMetaInfo info = TableMetaInfo.builder() diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java index eaa0f4dbe738..fc48fd5c74ff 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java @@ -58,8 +58,7 @@ public InfluxInsertRequest(Model model, StorageData storageData, StorageBuilder } builder = Point.measurement(model.getName()) .addField(ID, storageData.id()) - .fields(fields) - .tag(InfluxClient.TAG_TIME_BUCKET, String.valueOf(fields.get(Metrics.TIME_BUCKET))); + .fields(fields); } public InfluxInsertRequest time(long time, TimeUnit unit) { @@ -68,9 +67,7 @@ public InfluxInsertRequest time(long time, TimeUnit unit) { } public InfluxInsertRequest addFieldAsTag(String fieldName, String tagName) { - if (fields.containsKey(fieldName)) { - builder.tag(tagName, String.valueOf(fields.get(fieldName))); - } + builder.tag(tagName, String.valueOf(fields.get(fieldName))); return this; } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java index e471d7e1e989..86f5c8c0d543 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; @@ -40,9 +41,11 @@ import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; +import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ALL_FIELDS; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; +@Slf4j public class MetricsDAO implements IMetricsDAO { private final StorageBuilder storageBuilder; @@ -56,10 +59,13 @@ public MetricsDAO(InfluxClient client, StorageBuilder storageBuilder) { @Override public List multiGet(Model model, List ids) throws IOException { WhereQueryImpl query = select() - .regex("*::field") + .raw(ALL_FIELDS) .from(client.getDatabase(), model.getName()) .where(contains("id", Joiner.on("|").join(ids))); QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result: {}", query.getCommand(), series); + } if (series == null) { return Collections.emptyList(); @@ -94,7 +100,7 @@ public InsertRequest prepareBatchInsert(Model model, Metrics metrics) throws IOE final long timestamp = TimeBucket.getTimestamp(metrics.getTimeBucket(), model.getDownsampling()); TableMetaInfo tableMetaInfo = TableMetaInfo.get(model.getName()); - InfluxInsertRequest request = new InfluxInsertRequest(model, metrics, storageBuilder) + final InfluxInsertRequest request = new InfluxInsertRequest(model, metrics, storageBuilder) .time(timestamp, TimeUnit.MILLISECONDS); tableMetaInfo.getStorageAndTagMap().forEach((field, tag) -> { diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java index a9873b0106ce..750219823933 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java @@ -47,7 +47,7 @@ public void insert(final Model model, final NoneStream noneStream) throws IOExce final long timestamp = TimeBucket.getTimestamp( noneStream.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); - InfluxInsertRequest request = new InfluxInsertRequest(model, noneStream, storageBuilder) + final InfluxInsertRequest request = new InfluxInsertRequest(model, noneStream, storageBuilder) .time(timestamp, TimeUnit.NANOSECONDS); TableMetaInfo.get(model.getName()).getStorageAndTagMap().forEach((field, tag) -> { request.addFieldAsTag(field, tag); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java index 5cdb2afa6e11..671c0278a50c 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java @@ -48,7 +48,7 @@ public InsertRequest prepareBatchInsert(Model model, Record record) throws IOExc final long timestamp = TimeBucket.getTimestamp( record.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); - InfluxInsertRequest request = new InfluxInsertRequest(model, record, storageBuilder) + final InfluxInsertRequest request = new InfluxInsertRequest(model, record, storageBuilder) .time(timestamp, TimeUnit.NANOSECONDS); TableMetaInfo.get(model.getName()).getStorageAndTagMap().forEach((field, tag) -> { request.addFieldAsTag(field, tag); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java index dc2d2720d7fd..6b6019d394c7 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java @@ -32,6 +32,7 @@ import org.apache.skywalking.oap.server.core.query.entity.Logs; import org.apache.skywalking.oap.server.core.query.entity.Pagination; import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.elasticsearch.common.Strings; @@ -51,6 +52,7 @@ import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.STATUS_CODE; import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.TIMESTAMP; import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.TRACE_ID; +import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ALL_FIELDS; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; @@ -68,7 +70,7 @@ public LogQuery(InfluxClient client) { public Logs queryLogs(String metricName, int serviceId, int serviceInstanceId, String endpointId, String traceId, LogState state, String stateCode, Pagination paging, int from, int limit, long startTB, long endTB) throws IOException { - WhereQueryImpl recallQuery = select().regex("*::field") + WhereQueryImpl recallQuery = select().raw(ALL_FIELDS) .from(client.getDatabase(), metricName) .where(); if (serviceId != Const.NONE) { @@ -131,8 +133,12 @@ public Logs queryLogs(String metricName, int serviceId, int serviceInstanceId, S Map data = Maps.newHashMap(); Log log = new Log(); - for (int i = 0; i < columns.size(); i++) { - data.put(columns.get(i), values.get(i)); + for (int i = 1; i < columns.size(); i++) { + Object value = values.get(i); + if (value instanceof StorageDataComplexObject) { + value = ((StorageDataComplexObject) value).toStorageData(); + } + data.put(columns.get(i), value); } log.setContent((String) data.get(CONTENT)); log.setContentType(ContentType.instanceOf((int) data.get(CONTENT_TYPE))); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java index 845149cc94b7..1b550d59fdee 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -46,11 +46,16 @@ import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; +import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.SelectSubQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; +import org.influxdb.querybuilder.WhereSubQueryImpl; import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ID_COLUMN; +import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.NAME; +import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.TagName; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; @@ -59,8 +64,6 @@ @Slf4j public class MetadataQuery implements IMetadataQueryDAO { private static final Gson GSON = new Gson(); - - private static final String NAME = "\"" + ServiceTraffic.NAME + "\""; // 'name' is InfluxDB keyword, so escapes it private final InfluxClient client; public MetadataQuery(final InfluxClient client) { @@ -69,24 +72,26 @@ public MetadataQuery(final InfluxClient client) { @Override public int numOfService(final long startTimestamp, final long endTimestamp) throws IOException { - WhereQueryImpl query = select().countAll() + WhereQueryImpl query = select().raw("count(distinct " + ID_COLUMN + ")") .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) .where() - .and(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + .and( + eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()) + )); return client.getCounter(query); } @Override public int numOfEndpoint() throws IOException { SelectQueryImpl query = select() - .count(EndpointTraffic.ENTITY_ID) + .raw("count(distinct " + ID_COLUMN + ")") .from(client.getDatabase(), EndpointTraffic.INDEX_NAME); return client.getCounter(query); } @Override public int numOfConjectural(final int nodeTypeValue) throws IOException { - WhereQueryImpl query = select().countAll() + WhereQueryImpl query = select().raw("count(distinct " + ID_COLUMN + ")") .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) .where(eq(InfluxConstants.TagName.NODE_TYPE, nodeTypeValue)); return client.getCounter(query); @@ -94,9 +99,14 @@ public int numOfConjectural(final int nodeTypeValue) throws IOException { @Override public List getAllServices(final long startTimestamp, final long endTimestamp) throws IOException { - WhereQueryImpl query = select(ID_COLUMN, NAME) - .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + SelectSubQueryImpl subQuery = select() + .fromSubQuery(client.getDatabase()) + .column(ID_COLUMN).column(NAME) + .from(ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))) + .groupBy(TagName.NAME, TagName.NODE_TYPE); + SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase()); + query.setSubQuery(subQuery); return buildServices(query); } @@ -104,15 +114,20 @@ public List getAllServices(final long startTimestamp, final long endTim public List getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException { WhereQueryImpl query = select(ID_COLUMN, NAME) .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); return buildServices(query); } @Override public List getAllDatabases() throws IOException { - WhereQueryImpl query = select(ID_COLUMN, NAME) - .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Database.value())); + SelectSubQueryImpl subQuery = select() + .fromSubQuery(client.getDatabase()) + .column(ID_COLUMN).column(NAME) + .from(ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Database.value())) + .groupBy(TagName.NAME, TagName.NODE_TYPE); + SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase()); + query.setSubQuery(subQuery); QueryResult.Series series = client.queryForSingleSeries(query); if (log.isDebugEnabled()) { log.debug("SQL: {} result: {}", query.getCommand(), series); @@ -132,12 +147,19 @@ public List getAllDatabases() throws IOException { @Override public List searchServices(long startTimestamp, long endTimestamp, String keyword) throws IOException { - WhereQueryImpl query = select(ID_COLUMN, NAME) - .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); + WhereSubQueryImpl, SelectQueryImpl> subQuery = select() + .fromSubQuery(client.getDatabase()) + .column(ID_COLUMN) + .column(NAME) + .from(ServiceTraffic.INDEX_NAME) + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); if (!Strings.isNullOrEmpty(keyword)) { - query.and(contains(ServiceTraffic.NAME, keyword)); + subQuery.and(contains(ServiceTraffic.NAME, keyword)); } + subQuery.groupBy(TagName.NAME, TagName.NODE_TYPE); + + SelectQueryImpl query = select(ID_COLUMN, NAME).from(client.getDatabase()); + query.setSubQuery(subQuery); return buildServices(query); } @@ -145,8 +167,8 @@ public List searchServices(long startTimestamp, long endTimestamp, Stri public Service searchService(String serviceCode) throws IOException { WhereQueryImpl query = select(ID_COLUMN, NAME) .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, NodeType.Normal.value())); - query.and(eq(ServiceTraffic.NAME, serviceCode)); + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(NodeType.Normal.value()))); + query.and(eq(ServiceTraffic.NODE_TYPE, serviceCode)); return buildServices(query).get(0); } @@ -154,12 +176,19 @@ public Service searchService(String serviceCode) throws IOException { public List searchEndpoint(final String keyword, final String serviceId, final int limit) throws IOException { - WhereQueryImpl query = select(EndpointTraffic.SERVICE_ID, NAME) - .from(client.getDatabase(), EndpointTraffic.INDEX_NAME) + WhereSubQueryImpl, SelectQueryImpl> subQuery = select() + .fromSubQuery(client.getDatabase()) + .column(ID_COLUMN) + .column(NAME) + .from(EndpointTraffic.INDEX_NAME) .where(eq(InfluxConstants.TagName.SERVICE_ID, String.valueOf(serviceId))); if (!Strings.isNullOrEmpty(keyword)) { - query.where(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/"))); + subQuery.where(contains(EndpointTraffic.NAME, keyword.replaceAll("/", "\\\\/"))); } + subQuery.groupBy(TagName.NAME, TagName.SERVICE_ID); + SelectQueryImpl query = select(ID_COLUMN, NAME) + .from(client.getDatabase()); + query.setSubQuery(subQuery); query.limit(limit); final QueryResult.Series series = client.queryForSingleSeries(query); @@ -189,11 +218,20 @@ public List getServiceInstances(final long startTimestamp, final String serviceId) throws IOException { final long minuteTimeBucket = TimeBucket.getMinuteTimeBucket(startTimestamp); - WhereQueryImpl query = select(InstanceTraffic.SERVICE_ID, NAME, InstanceTraffic.PROPERTIES) - .from(client.getDatabase(), InstanceTraffic.INDEX_NAME) + SelectSubQueryImpl subQuery = select() + .fromSubQuery(client.getDatabase()) + .column(ID_COLUMN).column(NAME).column(InstanceTraffic.PROPERTIES) + .from(InstanceTraffic.INDEX_NAME) .where() .and(gte(InstanceTraffic.LAST_PING_TIME_BUCKET, minuteTimeBucket)) - .and(eq(InfluxConstants.TagName.SERVICE_ID, serviceId)); + .and(eq(InfluxConstants.TagName.SERVICE_ID, serviceId)) + .groupBy(TagName.NAME, TagName.SERVICE_ID); + + SelectQueryImpl query = select().column(ID_COLUMN) + .column(NAME) + .column(InstanceTraffic.PROPERTIES) + .from(client.getDatabase(), InstanceTraffic.INDEX_NAME); + query.setSubQuery(subQuery); QueryResult.Series series = client.queryForSingleSeries(query); if (log.isDebugEnabled()) { @@ -234,7 +272,7 @@ public List getServiceInstances(final long startTimestamp, return instances; } - private List buildServices(WhereQueryImpl query) throws IOException { + private List buildServices(Query query) throws IOException { QueryResult.Series series = client.queryForSingleSeries(query); if (log.isDebugEnabled()) { log.debug("SQL: {} result: {}", query.getCommand(), series); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java index dd0da3c2e997..a2140015c472 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java @@ -38,6 +38,7 @@ import org.apache.skywalking.oap.server.core.query.sql.Where; import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; @@ -46,6 +47,7 @@ import org.influxdb.querybuilder.SelectionQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; +import static org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants.ID_COLUMN; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; @@ -86,7 +88,7 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s final List values = kv.getValues(); Class type = columnTypes.get(kv.getKey()); - if (values.size() == 1) { // FIXME + if (values.size() == 1) { String value = kv.getValues().get(0); if (type != String.class) { value = "'" + value + "'"; @@ -99,10 +101,10 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s .append(" =~ /") .append(Joiner.on("|").join(values)) .append("/ OR "); - continue; - } - for (String value : values) { - clauseBuilder.append(kv.getKey()).append(" = '").append(value).append("' OR "); + } else { + for (String value : values) { + clauseBuilder.append(kv.getKey()).append(" = ").append(value).append(" OR "); + } } } } @@ -118,7 +120,7 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s if (log.isDebugEnabled()) { log.debug("SQL: {} result set: {}", queryWhereQuery.getCommand(), seriesList); } - if (!(seriesList == null || seriesList.isEmpty())) { + if (CollectionUtils.isNotEmpty(seriesList)) { for (QueryResult.Series series : seriesList) { KVInt kv = new KVInt(); kv.setId(series.getTags().get(InfluxConstants.TagName.ENTITY_ID)); @@ -139,16 +141,16 @@ public IntValues getLinearIntValues(String measurement, String valueCName) throws IOException { WhereQueryImpl query = select() - .column("id") + .column(ID_COLUMN) .column(valueCName) .from(client.getDatabase(), measurement) .where(); - if (ids != null && !ids.isEmpty()) { + if (CollectionUtils.isNotEmpty(ids)) { if (ids.size() == 1) { - query.where(eq("id", ids.get(0))); + query.where(eq(ID_COLUMN, ids.get(0))); } else { - query.where(contains("id", Joiner.on("|").join(ids))); + query.where(contains(ID_COLUMN, Joiner.on("|").join(ids))); } } List seriesList = client.queryForSeries(query); @@ -157,7 +159,7 @@ public IntValues getLinearIntValues(String measurement, } IntValues intValues = new IntValues(); - if (!(seriesList == null || seriesList.isEmpty())) { + if (CollectionUtils.isNotEmpty(seriesList)) { seriesList.get(0).getValues().forEach(values -> { KVInt kv = new KVInt(); kv.setValue(((Number) values.get(2)).longValue()); @@ -197,7 +199,7 @@ public IntValues[] getMultipleLinearIntValues(String measurement, DownSampling d .from(client.getDatabase(), measurement) .where(); - if (ids != null && !ids.isEmpty()) { + if (CollectionUtils.isNotEmpty(ids)) { if (ids.size() == 1) { query.where(eq("id", ids.get(0))); } else { @@ -212,7 +214,7 @@ public IntValues[] getMultipleLinearIntValues(String measurement, DownSampling d for (int i = 0; i < intValues.length; i++) { intValues[i] = new IntValues(); } - if (series == null || series.isEmpty()) { + if (CollectionUtils.isEmpty(series)) { return intValues; } series.get(0).getValues().forEach(values -> { @@ -253,9 +255,9 @@ public Thermodynamic getThermodynamic(String measurement, DownSampling downsampl .column(ThermodynamicMetrics.STEP) .column(ThermodynamicMetrics.NUM_OF_STEPS) .column(ThermodynamicMetrics.DETAIL_GROUP) - .column("id") + .column(ID_COLUMN) .from(client.getDatabase(), measurement) - .where(contains("id", Joiner.on("|").join(ids))); + .where(contains(ID_COLUMN, Joiner.on("|").join(ids))); Map> thermodynamicValueMatrix = new HashMap<>(); QueryResult.Series series = client.queryForSingleSeries(query); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java index 94c18cf2810d..ef60af82a0fe 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/NetworkAddressAliasDAO.java @@ -72,8 +72,8 @@ public List loadLastUpdate(final long timeBucket) { .getStorageAndColumnMap(); for (List values : result) { Map map = Maps.newHashMap(); - for (int i = 0; i < columns.size(); i++) { - map.put(columnAndFieldMap.get(columns.get(i)), values.get(1 + i)); + for (int i = 1; i < columns.size(); i++) { + map.put(columnAndFieldMap.get(columns.get(i)), values.get(i)); } networkAddressAliases.add(builder.map2Data(map)); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java index 277051deab3c..aa14811a2f67 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java @@ -202,9 +202,9 @@ public List queryByTraceId(String traceId) throws IOException { segmentRecord.setEndTime((long) values.get(7)); segmentRecord.setLatency((int) values.get(8)); segmentRecord.setIsError((int) values.get(9)); - segmentRecord.setVersion((int) values.get(10)); + segmentRecord.setVersion((int) values.get(11)); - String base64 = (String) values.get(9); + String base64 = (String) values.get(10); if (!Strings.isNullOrEmpty(base64)) { segmentRecord.setDataBinary(Base64.getDecoder().decode(base64)); } From 50ba299e19cccc3682085aa038404869baf37595 Mon Sep 17 00:00:00 2001 From: daming Date: Thu, 16 Apr 2020 11:49:18 +0800 Subject: [PATCH 05/24] checkstyle --- .../storage/plugin/influxdb/base/InfluxInsertRequest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java index fc48fd5c74ff..6e56de04b691 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java @@ -21,7 +21,6 @@ import com.google.common.collect.Maps; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.storage.StorageBuilder; import org.apache.skywalking.oap.server.core.storage.StorageData; import org.apache.skywalking.oap.server.core.storage.model.Model; @@ -29,7 +28,6 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject; import org.apache.skywalking.oap.server.library.client.request.InsertRequest; import org.apache.skywalking.oap.server.library.client.request.UpdateRequest; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.influxdb.dto.Point; /** From 85edf0013942d2756be0bac34e532f401e7a3c95 Mon Sep 17 00:00:00 2001 From: daming Date: Thu, 16 Apr 2020 12:15:44 +0800 Subject: [PATCH 06/24] fix e2e configuration --- .../cluster/docker-compose.zk.influxdb.yml | 18 ------------------ .../docker/profile/docker-compose.influxdb.yml | 15 --------------- 2 files changed, 33 deletions(-) diff --git a/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml b/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml index 6d5d68dea379..82f6eb79b47f 100644 --- a/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml +++ b/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml @@ -16,22 +16,6 @@ version: '2.1' services: - mysql: - image: mysql/mysql-server:8.0.13 - networks: - - e2e - expose: - - 3306 - environment: - - MYSQL_ROOT_PASSWORD=root@1234 - - MYSQL_DATABASE=swtest - - MYSQL_ROOT_HOST=% - healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/3306"] - interval: 5s - timeout: 60s - retries: 120 - influxdb: image: influxdb:1.7.9 expose: @@ -56,8 +40,6 @@ services: - SW_CLUSTER_ZK_HOST_PORT=zk:2181 - SW_STORAGE=influxdb - SW_STORAGE_INFLUXDB_URL=http://influxdb:8086 - - SW_STORAGE_METABASE_TYPE=mysql - - SW_STORAGE_METABASE_URL=jdbc:mysql://mysql:3306/swtest volumes: - ./download-mysql.sh:/download-mysql.sh restart: on-failure diff --git a/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml b/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml index dd967b3902e9..cf4074eb3030 100644 --- a/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml +++ b/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml @@ -16,20 +16,6 @@ version: '2.1' services: - h2db: - build: - context: . - dockerfile: Dockerfile.h2 - networks: - - e2e - expose: - - 1521 - healthcheck: - test: ["CMD", "sh", "-c", "nc -z 127.0.0.1 1521"] - interval: 5s - timeout: 60s - retries: 120 - influxdb: image: influxdb:1.7.9 expose: @@ -54,7 +40,6 @@ services: environment: - SW_STORAGE=influxdb - SW_STORAGE_INFLUXDB_URL=http://influxdb:8086 - - SW_STORAGE_METABASE_URL=jdbc:h2:tcp://h2db:1521/skywalking-oap-db restart: on-failure depends_on: h2db: From b7f2f2d9f4a490c7536ad241326551dbd67da0d7 Mon Sep 17 00:00:00 2001 From: daming Date: Thu, 16 Apr 2020 15:46:14 +0800 Subject: [PATCH 07/24] fix e2e configuration --- test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml b/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml index a56263aabe96..86009a7d5d08 100644 --- a/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml +++ b/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml @@ -22,8 +22,6 @@ services: - 8086 networks: - e2e - depends_on: - - h2db healthcheck: test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/8086"] interval: 5s From 61f813f81b193641cf31e96c8022120067b17cfe Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 15:52:20 +0800 Subject: [PATCH 08/24] fix query issue --- .../storage/plugin/influxdb/InfluxClient.java | 3 +- .../plugin/influxdb/query/MetadataQuery.java | 10 +- .../plugin/influxdb/query/MetricsQuery.java | 6 +- .../plugin/influxdb/query/TopologyQuery.java | 133 +++++++++++------- 4 files changed, 89 insertions(+), 63 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java index 2282f5d809af..fe96dd4dbde6 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java @@ -74,6 +74,7 @@ public void connect() { InfluxDB.ResponseFormat.MSGPACK ); influx.query(new Query("CREATE DATABASE " + database)); + influx.enableGzip(); influx.enableBatch(config.getActions(), config.getDuration(), TimeUnit.MILLISECONDS); influx.setDatabase(database); @@ -100,7 +101,7 @@ public List query(Query query) throws IOException { } try { - QueryResult result = getInflux().query(query); + QueryResult result = getInflux().query(new Query(query.getCommand())); if (result.hasError()) { throw new IOException(result.getError()); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java index 1b550d59fdee..ae807f5e1136 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -93,7 +93,7 @@ public int numOfEndpoint() throws IOException { public int numOfConjectural(final int nodeTypeValue) throws IOException { WhereQueryImpl query = select().raw("count(distinct " + ID_COLUMN + ")") .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, nodeTypeValue)); + .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(nodeTypeValue))); return client.getCounter(query); } @@ -199,13 +199,9 @@ public List searchEndpoint(final String keyword, List list = new ArrayList<>(limit); if (series != null) { series.getValues().forEach(values -> { - EndpointTraffic endpointTraffic = new EndpointTraffic(); - endpointTraffic.setServiceId((String) values.get(1)); - endpointTraffic.setName((String) values.get(2)); - Endpoint endpoint = new Endpoint(); - endpoint.setId(IDManager.EndpointID.buildId(endpointTraffic.getServiceId(), endpointTraffic.getName())); - endpoint.setName(endpointTraffic.getName()); + endpoint.setId((String) values.get(1)); + endpoint.setName((String) values.get(2)); list.add(endpoint); }); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java index a2140015c472..a28f1c6d2040 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java @@ -86,16 +86,16 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s StringBuilder clauseBuilder = new StringBuilder(); for (KeyValues kv : whereKeyValues) { final List values = kv.getValues(); + ids.addAll(values); Class type = columnTypes.get(kv.getKey()); if (values.size() == 1) { String value = kv.getValues().get(0); - if (type != String.class) { + if (type == String.class) { value = "'" + value + "'"; } clauseBuilder.append(kv.getKey()).append("=").append(value).append(" OR "); } else { - ids.addAll(values); if (type == String.class) { clauseBuilder.append(kv.getKey()) .append(" =~ /") @@ -103,7 +103,7 @@ public IntValues getValues(String measurement, DownSampling downsampling, long s .append("/ OR "); } else { for (String value : values) { - clauseBuilder.append(kv.getKey()).append(" = ").append(value).append(" OR "); + clauseBuilder.append(kv.getKey()).append(" = '").append(value).append("' OR "); } } } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java index 804a78510481..914496866e5b 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java @@ -34,9 +34,13 @@ import org.apache.skywalking.oap.server.core.source.DetectPoint; import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; +import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.SelectSubQueryImpl; import org.influxdb.querybuilder.WhereNested; -import org.influxdb.querybuilder.WhereQueryImpl; +import org.influxdb.querybuilder.WhereSubQueryImpl; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; @@ -56,7 +60,7 @@ public List loadServiceRelationsDetectedAtServerSide(DownSampli long endTB, List serviceIds) throws IOException { String measurement = ServiceRelationServerSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceCallsQuery( + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceCallsQuery( measurement, startTB, endTB, @@ -64,7 +68,8 @@ public List loadServiceRelationsDetectedAtServerSide(DownSampli ServiceRelationServerSideMetrics.DEST_SERVICE_ID, serviceIds ); - return buildServiceCalls(query, DetectPoint.SERVER); + + return buildServiceCalls(buildQuery(subQuery), DetectPoint.SERVER); } @Override @@ -72,7 +77,7 @@ public List loadServiceRelationDetectedAtClientSide(DownSamplin long endTB, List serviceIds) throws IOException { String measurement = ServiceRelationClientSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceCallsQuery( + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceCallsQuery( measurement, startTB, endTB, @@ -80,14 +85,14 @@ public List loadServiceRelationDetectedAtClientSide(DownSamplin ServiceRelationServerSideMetrics.DEST_SERVICE_ID, serviceIds ); - return buildServiceCalls(query, DetectPoint.CLIENT); + return buildServiceCalls(buildQuery(subQuery), DetectPoint.CLIENT); } @Override public List loadServiceRelationsDetectedAtServerSide(DownSampling downsampling, long startTB, long endTB) throws IOException { String measurement = ServiceRelationServerSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceCallsQuery( + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceCallsQuery( measurement, startTB, endTB, @@ -95,14 +100,14 @@ public List loadServiceRelationsDetectedAtServerSide(DownSampli ServiceRelationServerSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0) ); - return buildServiceCalls(query, DetectPoint.SERVER); + return buildServiceCalls(buildQuery(subQuery), DetectPoint.SERVER); } @Override public List loadServiceRelationDetectedAtClientSide(DownSampling downsampling, long startTB, long endTB) throws IOException { String tableName = ServiceRelationClientSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceCallsQuery( + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceCallsQuery( tableName, startTB, endTB, @@ -110,7 +115,7 @@ public List loadServiceRelationDetectedAtClientSide(DownSamplin ServiceRelationServerSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0) ); - return buildServiceCalls(query, DetectPoint.CLIENT); + return buildServiceCalls(buildQuery(subQuery), DetectPoint.CLIENT); } @Override @@ -120,14 +125,15 @@ public List loadInstanceRelationDetectedAtServerSide(String cli long startTB, long endTB) throws IOException { String measurement = ServiceInstanceRelationServerSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceInstanceCallsQuery(measurement, - startTB, - endTB, - ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, - ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, - clientServiceId, serverServiceId + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceInstanceCallsQuery( + measurement, + startTB, + endTB, + ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, + ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, + clientServiceId, serverServiceId ); - return buildInstanceCalls(query, DetectPoint.SERVER); + return buildInstanceCalls(buildQuery(subQuery), DetectPoint.SERVER); } @Override @@ -137,14 +143,15 @@ public List loadInstanceRelationDetectedAtClientSide(String cli long startTB, long endTB) throws IOException { String measurement = ServiceInstanceRelationClientSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceInstanceCallsQuery(measurement, - startTB, - endTB, - ServiceInstanceRelationClientSideMetrics.SOURCE_SERVICE_ID, - ServiceInstanceRelationClientSideMetrics.DEST_SERVICE_ID, - clientServiceId, serverServiceId + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceInstanceCallsQuery( + measurement, + startTB, + endTB, + ServiceInstanceRelationClientSideMetrics.SOURCE_SERVICE_ID, + ServiceInstanceRelationClientSideMetrics.DEST_SERVICE_ID, + clientServiceId, serverServiceId ); - return buildInstanceCalls(query, DetectPoint.CLIENT); + return buildInstanceCalls(buildQuery(subQuery), DetectPoint.CLIENT); } @Override @@ -154,7 +161,7 @@ public List loadEndpointRelation(DownSampling downsampling, String destEndpointId) throws IOException { String measurement = EndpointRelationServerSideMetrics.INDEX_NAME; - WhereQueryImpl query = buildServiceCallsQuery( + WhereSubQueryImpl, SelectQueryImpl> subQuery = buildServiceCallsQuery( measurement, startTB, endTB, @@ -162,9 +169,9 @@ public List loadEndpointRelation(DownSampling downsampling, EndpointRelationServerSideMetrics.DEST_ENDPOINT, Collections.emptyList() ); - query.and(eq(EndpointRelationServerSideMetrics.DEST_ENDPOINT, destEndpointId)); + subQuery.and(eq(EndpointRelationServerSideMetrics.DEST_ENDPOINT, destEndpointId)); - WhereQueryImpl query2 = buildServiceCallsQuery( + WhereSubQueryImpl, SelectQueryImpl> subQuery2 = buildServiceCallsQuery( measurement, startTB, endTB, @@ -172,43 +179,54 @@ public List loadEndpointRelation(DownSampling downsampling, EndpointRelationServerSideMetrics.DEST_ENDPOINT, Collections.emptyList() ); - query2.and(eq(EndpointRelationServerSideMetrics.SOURCE_ENDPOINT, destEndpointId)); + subQuery2.and(eq(EndpointRelationServerSideMetrics.SOURCE_ENDPOINT, destEndpointId)); - List calls = buildEndpointCalls(query, DetectPoint.SERVER); - calls.addAll(buildEndpointCalls(query2, DetectPoint.CLIENT)); + List calls = buildEndpointCalls(buildQuery(subQuery), DetectPoint.SERVER); + calls.addAll(buildEndpointCalls(buildQuery(subQuery), DetectPoint.CLIENT)); return calls; } - private WhereQueryImpl buildServiceCallsQuery(String measurement, long startTB, long endTB, String sourceCName, - String destCName, List serviceIds) { - WhereQueryImpl query = select() - .function("distinct", Metrics.ENTITY_ID, ServiceRelationServerSideMetrics.COMPONENT_ID) - .from(client.getDatabase(), measurement) + private WhereSubQueryImpl, SelectQueryImpl> buildServiceCallsQuery( + String measurement, + long startTB, + long endTB, + String sourceCName, + String destCName, + List serviceIds) { + WhereSubQueryImpl, SelectQueryImpl> subQuery = select() + .fromSubQuery(client.getDatabase()) + .function("distinct", ServiceInstanceRelationServerSideMetrics.COMPONENT_ID) + .as(ServiceInstanceRelationClientSideMetrics.COMPONENT_ID) + .from(measurement) .where() .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))); if (!serviceIds.isEmpty()) { - WhereNested whereNested = query.andNested(); + WhereNested whereNested = subQuery.andNested(); for (String id : serviceIds) { whereNested.or(eq(sourceCName, id)) .or(eq(destCName, id)); } whereNested.close(); } - return query; + return subQuery; } - private WhereQueryImpl buildServiceInstanceCallsQuery(String measurement, - long startTB, - long endTB, - String sourceCName, - String destCName, - String sourceServiceId, - String destServiceId) { - WhereQueryImpl query = select() - .function("distinct", Metrics.ENTITY_ID, ServiceInstanceRelationServerSideMetrics.COMPONENT_ID) - .from(client.getDatabase(), measurement) + private WhereSubQueryImpl, SelectQueryImpl> buildServiceInstanceCallsQuery( + String measurement, + long startTB, + long endTB, + String sourceCName, + String destCName, + String sourceServiceId, + String destServiceId) { + + WhereSubQueryImpl, SelectQueryImpl> subQuery = select() + .fromSubQuery(client.getDatabase()) + .function("distinct", ServiceInstanceRelationServerSideMetrics.COMPONENT_ID) + .as(ServiceInstanceRelationClientSideMetrics.COMPONENT_ID) + .from(measurement) .where() .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))); @@ -222,11 +240,12 @@ private WhereQueryImpl buildServiceInstanceCallsQuery(String measurement, .append(") and (") .append(destCName).append("=").append(sourceServiceId) .append("))"); - query.where(builder.toString()); - return query; + subQuery.where(builder.toString()); + subQuery.groupBy(InfluxConstants.TagName.ENTITY_ID); + return subQuery; } - private List buildServiceCalls(WhereQueryImpl query, + private List buildServiceCalls(Query query, DetectPoint detectPoint) throws IOException { QueryResult.Series series = client.queryForSingleSeries(query); @@ -240,7 +259,9 @@ private List buildServiceCalls(WhereQueryImpl query, List calls = new ArrayList<>(); series.getValues().forEach(values -> { Call.CallDetail call = new Call.CallDetail(); - String entityId = (String) values.get(1); + System.out.println(values.get(1)); + System.out.println(values.get(2)); + String entityId = String.valueOf(values.get(1)); int componentId = (int) values.get(2); call.buildFromServiceRelation(entityId, componentId, detectPoint); calls.add(call); @@ -248,7 +269,15 @@ private List buildServiceCalls(WhereQueryImpl query, return calls; } - private List buildInstanceCalls(WhereQueryImpl query, + private Query buildQuery(WhereSubQueryImpl, SelectQueryImpl> subQuery) { + SelectQueryImpl query = select().column(InfluxConstants.TagName.ENTITY_ID) + .column(ServiceInstanceRelationClientSideMetrics.COMPONENT_ID) + .from(client.getDatabase()); + query.setSubQuery(subQuery.groupBy(InfluxConstants.TagName.ENTITY_ID)); + return query; + } + + private List buildInstanceCalls(Query query, DetectPoint detectPoint) throws IOException { QueryResult.Series series = client.queryForSingleSeries(query); @@ -270,7 +299,7 @@ private List buildInstanceCalls(WhereQueryImpl query, return calls; } - private List buildEndpointCalls(WhereQueryImpl query, + private List buildEndpointCalls(Query query, DetectPoint detectPoint) throws IOException { QueryResult.Series series = client.queryForSingleSeries(query); From f4b3b5185b20aabeeef9bacaa405a370aafcf3a9 Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 16:28:58 +0800 Subject: [PATCH 09/24] update --- .../server/storage/plugin/influxdb/query/MetadataQuery.java | 6 ++++-- .../server/storage/plugin/influxdb/query/TopologyQuery.java | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java index ae807f5e1136..bd041974aee5 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetadataQuery.java @@ -30,7 +30,6 @@ import java.util.Map; import java.util.Objects; import lombok.extern.slf4j.Slf4j; -import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.NodeType; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; @@ -93,7 +92,10 @@ public int numOfEndpoint() throws IOException { public int numOfConjectural(final int nodeTypeValue) throws IOException { WhereQueryImpl query = select().raw("count(distinct " + ID_COLUMN + ")") .from(client.getDatabase(), ServiceTraffic.INDEX_NAME) - .where(eq(InfluxConstants.TagName.NODE_TYPE, String.valueOf(nodeTypeValue))); + .where(eq( + InfluxConstants.TagName.NODE_TYPE, + String.valueOf(nodeTypeValue) + )); return client.getCounter(query); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java index 914496866e5b..637fb1f86658 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java @@ -29,7 +29,6 @@ import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics; import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics; import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; -import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.query.entity.Call; import org.apache.skywalking.oap.server.core.source.DetectPoint; import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; From ef1ce7d737a69dce5fce102202aa658add5dbd7a Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 16:47:12 +0800 Subject: [PATCH 10/24] checkstyleg --- .../oap/server/storage/plugin/influxdb/query/TopologyQuery.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java index 637fb1f86658..cef935c97440 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java @@ -258,8 +258,6 @@ private List buildServiceCalls(Query query, List calls = new ArrayList<>(); series.getValues().forEach(values -> { Call.CallDetail call = new Call.CallDetail(); - System.out.println(values.get(1)); - System.out.println(values.get(2)); String entityId = String.valueOf(values.get(1)); int componentId = (int) values.get(2); call.buildFromServiceRelation(entityId, componentId, detectPoint); From 580327243345785bc28c82c505f06a7b79f0d431 Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 18:33:23 +0800 Subject: [PATCH 11/24] polish --- .../plugin/influxdb/InfluxConstants.java | 2 + .../plugin/influxdb/TableMetaInfo.java | 5 +-- .../influxdb/base/InfluxInsertRequest.java | 5 +-- .../plugin/influxdb/base/NoneStreamDAO.java | 5 +-- .../plugin/influxdb/base/RecordDAO.java | 5 +-- .../influxdb/query/ProfileTaskLogQuery.java | 22 ++++------ .../influxdb/query/ProfileTaskQuery.java | 41 +++++++++++-------- .../query/ProfileThreadSnapshotQuery.java | 3 +- 8 files changed, 42 insertions(+), 46 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java index ecf5abf649a6..33bc7dbc09f5 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxConstants.java @@ -29,6 +29,8 @@ public interface InfluxConstants { String SORT_ASC = "bottom"; + String DURATION = "\"" + "duration" + "\""; + interface TagName { String ID_COLUMN = "_id"; diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java index bdf3b55f5106..d481c5d38831 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java @@ -40,7 +40,6 @@ @AllArgsConstructor public class TableMetaInfo { private static final Map TABLES = new HashMap<>(); - private static final String PREFIX = "_"; private Map storageAndColumnMap; private Map storageAndTagMap; @@ -55,15 +54,15 @@ public static void addModel(Model model) { storageAndColumnMap.put(columnName.getStorageName(), columnName.getName()); }); - if (model.getName().endsWith("_traffic")) { // It is not a good way. It has performance issue. + if (model.getName().endsWith("_traffic")) { // instance_traffic name, service_id // endpoint_traffic name, service_id - // service_traffic name, node_type storageAndTagMap.put(InstanceTraffic.NAME, InfluxConstants.TagName.NAME); if ("instance_traffic".equals(model.getName()) || "endpoint_traffic".equals(model.getName())) { storageAndTagMap.put(EndpointTraffic.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); } else { + // service_traffic name, node_type storageAndTagMap.put(ServiceTraffic.NODE_TYPE, InfluxConstants.TagName.NODE_TYPE); } } else { diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java index 6e56de04b691..f98437216543 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java @@ -28,14 +28,13 @@ import org.apache.skywalking.oap.server.core.storage.type.StorageDataComplexObject; import org.apache.skywalking.oap.server.library.client.request.InsertRequest; import org.apache.skywalking.oap.server.library.client.request.UpdateRequest; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.influxdb.dto.Point; /** * InfluxDB Point wrapper. */ public class InfluxInsertRequest implements InsertRequest, UpdateRequest { - public static final String ID = "id"; - private Point.Builder builder; private Map fields = Maps.newHashMap(); @@ -55,7 +54,7 @@ public InfluxInsertRequest(Model model, StorageData storageData, StorageBuilder } } builder = Point.measurement(model.getName()) - .addField(ID, storageData.id()) + .addField(InfluxConstants.ID_COLUMN, storageData.id()) .fields(fields); } diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java index 750219823933..3d89e419fa0a 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java @@ -30,7 +30,6 @@ import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; public class NoneStreamDAO implements INoneStreamDAO { - public static final String TAG_SERVICE_ID = "_service_id"; private static final int PADDING_SIZE = 1_000_000; private static final AtomicRangeInteger SUFFIX = new AtomicRangeInteger(0, PADDING_SIZE); @@ -44,8 +43,8 @@ public NoneStreamDAO(InfluxClient client, StorageBuilder storageBuil @Override public void insert(final Model model, final NoneStream noneStream) throws IOException { - final long timestamp = TimeBucket.getTimestamp( - noneStream.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); + final long timestamp = TimeBucket.getTimestamp(noneStream.getTimeBucket(), model.getDownsampling()) + * PADDING_SIZE + SUFFIX.getAndIncrement(); final InfluxInsertRequest request = new InfluxInsertRequest(model, noneStream, storageBuilder) .time(timestamp, TimeUnit.NANOSECONDS); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java index 671c0278a50c..63739c1708d3 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java @@ -31,7 +31,6 @@ import org.apache.skywalking.oap.server.storage.plugin.influxdb.TableMetaInfo; public class RecordDAO implements IRecordDAO { - public static final String TAG_SERVICE_ID = "_service_id"; private static final int PADDING_SIZE = 1_000_000; private static final AtomicRangeInteger SUFFIX = new AtomicRangeInteger(0, PADDING_SIZE); @@ -45,8 +44,8 @@ public RecordDAO(InfluxClient client, StorageBuilder storageBuilder) { @Override public InsertRequest prepareBatchInsert(Model model, Record record) throws IOException { - final long timestamp = TimeBucket.getTimestamp( - record.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); + final long timestamp = TimeBucket.getTimestamp(record.getTimeBucket(), model.getDownsampling()) + * PADDING_SIZE + SUFFIX.getAndIncrement(); final InfluxInsertRequest request = new InfluxInsertRequest(model, record, storageBuilder) .time(timestamp, TimeUnit.NANOSECONDS); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java index f96a4c9e0e47..d4066f07e8d3 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java @@ -50,7 +50,7 @@ public ProfileTaskLogQuery(InfluxClient client, int fetchTaskLogMaxSize) { @Override public List getTaskLogList() throws IOException { WhereQueryImpl query = select() - .function("top", ProfileTaskLogRecord.OPERATION_TIME, fetchTaskLogMaxSize) + .function(InfluxConstants.SORT_DES, ProfileTaskLogRecord.OPERATION_TIME, fetchTaskLogMaxSize) .column(InfluxConstants.ID_COLUMN) .column(ProfileTaskLogRecord.TASK_ID) .column(ProfileTaskLogRecord.INSTANCE_ID) @@ -66,26 +66,18 @@ public List getTaskLogList() throws IOException { if (series == null) { return Collections.emptyList(); } - List columns = series.getColumns(); - Map columnsMap = Maps.newHashMap(); - for (int i = 0; i < columns.size(); i++) { - columnsMap.put(columns.get(i), i); - } - - List taskLogs = Lists.newArrayList(); + final List taskLogs = Lists.newArrayList(); series.getValues().stream() // re-sort by self, because of the result order by time. .sorted((a, b) -> Long.compare(((Number) b.get(1)).longValue(), ((Number) a.get(1)).longValue())) .forEach(values -> { taskLogs.add(ProfileTaskLog.builder() - .id((String) values.get(columnsMap.get(InfluxConstants.ID_COLUMN))) - .taskId((String) values.get(columnsMap.get(ProfileTaskLogRecord.TASK_ID))) - .instanceId( - (String) values.get(columnsMap.get(ProfileTaskLogRecord.INSTANCE_ID))) - .operationTime( - (Long) values.get(columnsMap.get(ProfileTaskLogRecord.OPERATION_TIME))) + .id((String) values.get(2)) + .taskId((String) values.get(3)) + .instanceId((String) values.get(4)) + .operationTime(((Number) values.get(5)).longValue()) .operationType(ProfileTaskLogOperationType.parse( - (int) values.get(columnsMap.get(ProfileTaskLogRecord.OPERATION_TYPE)))) + ((Number) values.get(6)).intValue())) .build()); }); return taskLogs; diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java index 72583b069342..d6b595cb79dc 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java @@ -28,6 +28,7 @@ import org.apache.skywalking.oap.server.core.query.entity.ProfileTask; import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.NoneStreamDAO; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; @@ -40,8 +41,6 @@ @Slf4j public class ProfileTaskQuery implements IProfileTaskQueryDAO { - private static final String DURATION = "\"" + "duration" + "\""; - private static final String ID_COLUMN = "id"; private final InfluxClient client; public ProfileTaskQuery(InfluxClient client) { @@ -55,19 +54,22 @@ public List getTaskList(final String serviceId, final Long endTimeBucket, final Integer limit) throws IOException { WhereQueryImpl query = - select(ID_COLUMN, ProfileTaskRecord.SERVICE_ID, - ProfileTaskRecord.ENDPOINT_NAME, ProfileTaskRecord.START_TIME, - ProfileTaskRecord.CREATE_TIME, - DURATION, - ProfileTaskRecord.MIN_DURATION_THRESHOLD, - ProfileTaskRecord.DUMP_PERIOD, - ProfileTaskRecord.MAX_SAMPLING_COUNT + select( + InfluxConstants.ID_COLUMN, + ProfileTaskRecord.SERVICE_ID, + ProfileTaskRecord.ENDPOINT_NAME, + ProfileTaskRecord.START_TIME, + ProfileTaskRecord.CREATE_TIME, + InfluxConstants.DURATION, + ProfileTaskRecord.MIN_DURATION_THRESHOLD, + ProfileTaskRecord.DUMP_PERIOD, + ProfileTaskRecord.MAX_SAMPLING_COUNT ) .from(client.getDatabase(), ProfileTaskRecord.INDEX_NAME) .where(); if (StringUtil.isNotEmpty(serviceId)) { - query.and(eq(NoneStreamDAO.TAG_SERVICE_ID, serviceId)); + query.and(eq(InfluxConstants.TagName.SERVICE_ID, serviceId)); } if (StringUtil.isNotEmpty(endpointName)) { query.and(eq(ProfileTaskRecord.ENDPOINT_NAME, endpointName)); @@ -100,17 +102,20 @@ public ProfileTask getById(final String id) throws IOException { if (StringUtil.isEmpty(id)) { return null; } - SelectQueryImpl query = select(ID_COLUMN, ProfileTaskRecord.SERVICE_ID, - ProfileTaskRecord.ENDPOINT_NAME, ProfileTaskRecord.START_TIME, - ProfileTaskRecord.CREATE_TIME, - DURATION, - ProfileTaskRecord.MIN_DURATION_THRESHOLD, - ProfileTaskRecord.DUMP_PERIOD, - ProfileTaskRecord.MAX_SAMPLING_COUNT + SelectQueryImpl query = select( + InfluxConstants.ID_COLUMN, + ProfileTaskRecord.SERVICE_ID, + ProfileTaskRecord.ENDPOINT_NAME, + ProfileTaskRecord.START_TIME, + ProfileTaskRecord.CREATE_TIME, + InfluxConstants.DURATION, + ProfileTaskRecord.MIN_DURATION_THRESHOLD, + ProfileTaskRecord.DUMP_PERIOD, + ProfileTaskRecord.MAX_SAMPLING_COUNT ) .from(client.getDatabase(), ProfileTaskRecord.INDEX_NAME) .where() - .and(eq(ID_COLUMN, id)) + .and(eq(InfluxConstants.ID_COLUMN, id)) .limit(1); QueryResult.Series series = client.queryForSingleSeries(query); diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java index f8f6a6b2dd73..25dfc0e8d8fe 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java @@ -35,6 +35,7 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; import org.apache.skywalking.oap.server.library.util.BooleanUtils; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; import org.elasticsearch.common.Strings; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.WhereQueryImpl; @@ -75,7 +76,7 @@ public List queryProfiledSegments(String taskId) throws IOException } query = select() - .function("bottom", SegmentRecord.START_TIME, segments.size()) + .function(InfluxConstants.SORT_ASC, SegmentRecord.START_TIME, segments.size()) .column(SegmentRecord.SEGMENT_ID) .column(SegmentRecord.START_TIME) .column(SegmentRecord.ENDPOINT_NAME) From 8765d92bfe151ec3b45f60a55635ee5bc159b467 Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 19:16:11 +0800 Subject: [PATCH 12/24] checkstyle --- .../storage/plugin/influxdb/query/ProfileTaskLogQuery.java | 2 -- .../server/storage/plugin/influxdb/query/ProfileTaskQuery.java | 1 - 2 files changed, 3 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java index d4066f07e8d3..59982d834b3d 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java @@ -19,11 +19,9 @@ package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.profile.ProfileTaskLogRecord; import org.apache.skywalking.oap.server.core.query.entity.ProfileTaskLog; diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java index d6b595cb79dc..43099ef99b4e 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java @@ -29,7 +29,6 @@ import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.NoneStreamDAO; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.SelectQueryImpl; import org.influxdb.querybuilder.WhereQueryImpl; From 5b9976979f9b8e915b34eb7237de8a397bb16026 Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 23:29:34 +0800 Subject: [PATCH 13/24] remove TAG_SERVER_ID --- .../oap/server/storage/plugin/influxdb/query/TraceQuery.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java index aa14811a2f67..9e577fa2ef5d 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java @@ -35,7 +35,6 @@ import org.apache.skywalking.oap.server.library.util.BooleanUtils; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; import org.elasticsearch.common.Strings; import org.influxdb.dto.Query; import org.influxdb.dto.QueryResult; @@ -103,7 +102,7 @@ public TraceBrief queryBasicTraces(long startSecondTB, recallQuery.and(contains(SegmentRecord.ENDPOINT_NAME, endpointName.replaceAll("/", "\\\\/"))); } if (StringUtil.isNotEmpty(serviceId)) { - recallQuery.and(eq(RecordDAO.TAG_SERVICE_ID, serviceId)); + recallQuery.and(eq(InfluxConstants.TagName.SERVICE_ID, serviceId)); } if (StringUtil.isNotEmpty(serviceInstanceId)) { recallQuery.and(eq(SegmentRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); From f3f50b1699c86698343520262747011a754d4be4 Mon Sep 17 00:00:00 2001 From: daming Date: Fri, 17 Apr 2020 23:54:25 +0800 Subject: [PATCH 14/24] remove TAG_SERVER_ID --- .../server/storage/plugin/influxdb/query/TopNRecordsQuery.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java index 1603141a4038..1dda92c58c52 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java @@ -31,7 +31,6 @@ import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxConstants; -import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; import org.influxdb.dto.QueryResult; import org.influxdb.querybuilder.WhereQueryImpl; @@ -69,7 +68,7 @@ public List getTopNRecords(long startSecondTB, long endSecondTB, Str .and(lte(TopN.TIME_BUCKET, endSecondTB)); if (StringUtil.isNotEmpty(serviceId)) { - query.and(eq(RecordDAO.TAG_SERVICE_ID, serviceId)); + query.and(eq(InfluxConstants.TagName.SERVICE_ID, serviceId)); } QueryResult.Series series = client.queryForSingleSeries(query); From 17dff41463f2efb3e7625ab534331a9b16262dfc Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Sat, 18 Apr 2020 12:29:16 +0800 Subject: [PATCH 15/24] Revert unnecessary changes --- .../e2e-test/docker/cluster/docker-compose.zk.influxdb.yml | 2 -- test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml b/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml index 19c257e2e232..f45532133de3 100644 --- a/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml +++ b/test/e2e/e2e-test/docker/cluster/docker-compose.zk.influxdb.yml @@ -35,7 +35,6 @@ services: environment: SW_CLUSTER: zookeeper SW_STORAGE: influxdb - SW_STORAGE_INFLUXDB_URL: http://influxdb:8086 JAVA_OPTS: >- -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap1,destfile=/jacoco/oap1.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.* depends_on: @@ -51,7 +50,6 @@ services: environment: SW_CLUSTER: zookeeper SW_STORAGE: influxdb - SW_STORAGE_INFLUXDB_URL: http://influxdb:8086 JAVA_OPTS: >- -javaagent:/jacoco/jacocoagent.jar=classdumpdir=/jacoco/classes/oap2,destfile=/jacoco/oap2.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.oap.query.*:org.apache.skywalking.oap.server.core.query.* depends_on: diff --git a/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml b/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml index 86009a7d5d08..377ea791e024 100644 --- a/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml +++ b/test/e2e/e2e-test/docker/profile/docker-compose.influxdb.yml @@ -33,9 +33,7 @@ services: file: ../base-compose.yml service: oap environment: - - SW_STORAGE=influxdb - - SW_STORAGE_INFLUXDB_URL=http://influxdb:8086 - restart: on-failure + SW_STORAGE: influxdb depends_on: influxdb: condition: service_healthy From 060ab3de09bb5e732608c14b49f0ee4eb58422eb Mon Sep 17 00:00:00 2001 From: daming Date: Sat, 18 Apr 2020 18:34:58 +0800 Subject: [PATCH 16/24] remove image on Test finished --- test/plugin/scenarios/solrj-7.x-scenario/configuration.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/plugin/scenarios/solrj-7.x-scenario/configuration.yml b/test/plugin/scenarios/solrj-7.x-scenario/configuration.yml index aea4ef8281d2..be366df04c3b 100644 --- a/test/plugin/scenarios/solrj-7.x-scenario/configuration.yml +++ b/test/plugin/scenarios/solrj-7.x-scenario/configuration.yml @@ -25,6 +25,7 @@ dependencies: solr-server: image: solr:${CASE_SERVER_IMAGE_VERSION} hostname: solr-server + removeOnExit: true entrypoint: - docker-entrypoint.sh - solr-precreate From 316fcbcfb9faf94da99f8fbc50dcbdc1cf195e44 Mon Sep 17 00:00:00 2001 From: daming Date: Sat, 18 Apr 2020 18:36:20 +0800 Subject: [PATCH 17/24] polish --- .../oap/server/storage/plugin/influxdb/TableMetaInfo.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java index d481c5d38831..3d73663172ee 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/TableMetaInfo.java @@ -58,8 +58,8 @@ public static void addModel(Model model) { // instance_traffic name, service_id // endpoint_traffic name, service_id storageAndTagMap.put(InstanceTraffic.NAME, InfluxConstants.TagName.NAME); - if ("instance_traffic".equals(model.getName()) - || "endpoint_traffic".equals(model.getName())) { + if (InstanceTraffic.INDEX_NAME.equals(model.getName()) + || EndpointTraffic.INDEX_NAME.equals(model.getName())) { storageAndTagMap.put(EndpointTraffic.SERVICE_ID, InfluxConstants.TagName.SERVICE_ID); } else { // service_traffic name, node_type From c77f8e52e6e4e03dfa10f3a401935718ff04ee7c Mon Sep 17 00:00:00 2001 From: daming Date: Sat, 18 Apr 2020 21:46:19 +0800 Subject: [PATCH 18/24] remove workspace of testscase --- test/plugin/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/plugin/run.sh b/test/plugin/run.sh index 56b6c628478d..d81ecfffc95d 100755 --- a/test/plugin/run.sh +++ b/test/plugin/run.sh @@ -187,7 +187,6 @@ fi supported_versions=`grep -v -E "^$|^#" ${supported_version_file}` for version in ${supported_versions} do - waitForAvailable testcase_name="${scenario_name}-${version}" # testcase working directory, there are logs, data and packages. @@ -218,8 +217,10 @@ do [[ $? -ne 0 ]] && exitWithMessage "${testcase_name}, generate script failure!" echo "start container of testcase.name=${testcase_name}" - bash ${case_work_base}/scenario.sh ${task_state_house} 1>${case_work_logs_dir}/${testcase_name}.log & + bash ${case_work_base}/scenario.sh ${task_state_house} 1>${case_work_logs_dir}/${testcase_name}.log sleep 3 + waitForAvailable + rm -rf ${case_work_base} done echo -e "\033[33m${scenario_name} has already sumbitted\033[0m" From 813cfde137772235bc20ea78ed6a49948458ff38 Mon Sep 17 00:00:00 2001 From: daming Date: Sat, 18 Apr 2020 23:45:12 +0800 Subject: [PATCH 19/24] update logs when failed --- .github/workflows/plugins-test.0.yaml | 5 +++++ .github/workflows/plugins-test.1.yaml | 7 ++++++- .github/workflows/plugins-test.2.yaml | 5 +++++ .github/workflows/plugins-test.3.yaml | 10 ++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/plugins-test.0.yaml b/.github/workflows/plugins-test.0.yaml index e24f7ec44260..43957e368337 100644 --- a/.github/workflows/plugins-test.0.yaml +++ b/.github/workflows/plugins-test.0.yaml @@ -72,3 +72,8 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs diff --git a/.github/workflows/plugins-test.1.yaml b/.github/workflows/plugins-test.1.yaml index b9fc6c20a7d5..965a6c732531 100644 --- a/.github/workflows/plugins-test.1.yaml +++ b/.github/workflows/plugins-test.1.yaml @@ -46,7 +46,7 @@ jobs: - { name: 'kotlin-coroutine-scenario', title: 'Kotlin Coroutine 1.0.1-1.3.3 (4)' } - { name: 'lettuce-scenario', title: 'Lettuce 5.x (17)' } - { name: 'mongodb-3.x-scenario', title: 'Mongodb 3.4.0-3.11.1 (22)' } - - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (53)' } + - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' } - { name: 'netty-socketio-scenario', title: 'Netty-SocketIO 1.x (4)' } steps: - uses: actions/checkout@v2 @@ -66,3 +66,8 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs diff --git a/.github/workflows/plugins-test.2.yaml b/.github/workflows/plugins-test.2.yaml index 2f11f79c2979..749fb9e63b11 100644 --- a/.github/workflows/plugins-test.2.yaml +++ b/.github/workflows/plugins-test.2.yaml @@ -77,3 +77,8 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs diff --git a/.github/workflows/plugins-test.3.yaml b/.github/workflows/plugins-test.3.yaml index 4870180bc68a..b54794cf3609 100644 --- a/.github/workflows/plugins-test.3.yaml +++ b/.github/workflows/plugins-test.3.yaml @@ -54,6 +54,11 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs Oracle: name: Oracle @@ -81,3 +86,8 @@ jobs: bash test/plugin/run.sh oracle-scenario - name: Report Coverage run: bash -x tools/coverage/report.sh + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: logs + path: logs From 0e85466fafc6f03cceb410c38ded042fd7a7cef3 Mon Sep 17 00:00:00 2001 From: daming Date: Sun, 19 Apr 2020 01:02:19 +0800 Subject: [PATCH 20/24] move mysql to test.3.yaml --- .github/workflows/plugins-test.1.yaml | 1 - .github/workflows/plugins-test.3.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugins-test.1.yaml b/.github/workflows/plugins-test.1.yaml index 965a6c732531..e3bdfc314c59 100644 --- a/.github/workflows/plugins-test.1.yaml +++ b/.github/workflows/plugins-test.1.yaml @@ -46,7 +46,6 @@ jobs: - { name: 'kotlin-coroutine-scenario', title: 'Kotlin Coroutine 1.0.1-1.3.3 (4)' } - { name: 'lettuce-scenario', title: 'Lettuce 5.x (17)' } - { name: 'mongodb-3.x-scenario', title: 'Mongodb 3.4.0-3.11.1 (22)' } - - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' } - { name: 'netty-socketio-scenario', title: 'Netty-SocketIO 1.x (4)' } steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/plugins-test.3.yaml b/.github/workflows/plugins-test.3.yaml index b54794cf3609..1ec947d9bee6 100644 --- a/.github/workflows/plugins-test.3.yaml +++ b/.github/workflows/plugins-test.3.yaml @@ -36,6 +36,7 @@ jobs: - { name: 'undertow-scenario', title: 'Undertow 1.3.0-2.0.27 (23)' } - { name: 'webflux-scenario', title: 'Spring-WebFlux 2.x (7)' } - { name: 'zookeeper-scenario', title: 'Zookeeper 3.4.x (14)' } + - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' } steps: - uses: actions/checkout@v2 - name: checkout submodules From 6ee0f4c28ec05bb2a5d060549f108840229f80b1 Mon Sep 17 00:00:00 2001 From: daming Date: Sun, 19 Apr 2020 01:52:45 +0800 Subject: [PATCH 21/24] test mysql-scenario --- .../mysql-scenario/support-version.list | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/test/plugin/scenarios/mysql-scenario/support-version.list b/test/plugin/scenarios/mysql-scenario/support-version.list index 4768268ed6e1..e7f1040b3dbc 100644 --- a/test/plugin/scenarios/mysql-scenario/support-version.list +++ b/test/plugin/scenarios/mysql-scenario/support-version.list @@ -20,29 +20,4 @@ 8.0.14 8.0.13 8.0.12 -8.0.11 -6.0.6 -6.0.4 -6.0.2 -5.1.44 -5.1.42 -5.1.40 -5.1.38 -5.1.36 -5.1.34 -5.1.32 -5.1.30 -5.1.28 -5.1.26 -5.1.24 -5.1.22 -5.1.20 -5.1.18 -5.1.16 -5.1.14 -5.1.12 -5.1.10 -5.1.8 -5.1.6 -5.1.4 -5.1.2 \ No newline at end of file +8.0.11 \ No newline at end of file From 88e245bf35f917c3d3e2732eebae9c3b18b00978 Mon Sep 17 00:00:00 2001 From: daming Date: Sun, 19 Apr 2020 02:05:39 +0800 Subject: [PATCH 22/24] revert --- .github/workflows/plugins-test.0.yaml | 5 ---- .github/workflows/plugins-test.1.yaml | 5 ---- .github/workflows/plugins-test.2.yaml | 5 ---- .github/workflows/plugins-test.3.yaml | 12 +-------- .../mysql-scenario/support-version.list | 27 ++++++++++++++++++- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/plugins-test.0.yaml b/.github/workflows/plugins-test.0.yaml index 43957e368337..e24f7ec44260 100644 --- a/.github/workflows/plugins-test.0.yaml +++ b/.github/workflows/plugins-test.0.yaml @@ -72,8 +72,3 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: logs diff --git a/.github/workflows/plugins-test.1.yaml b/.github/workflows/plugins-test.1.yaml index e3bdfc314c59..a9a46b795b3e 100644 --- a/.github/workflows/plugins-test.1.yaml +++ b/.github/workflows/plugins-test.1.yaml @@ -65,8 +65,3 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: logs diff --git a/.github/workflows/plugins-test.2.yaml b/.github/workflows/plugins-test.2.yaml index 749fb9e63b11..2f11f79c2979 100644 --- a/.github/workflows/plugins-test.2.yaml +++ b/.github/workflows/plugins-test.2.yaml @@ -77,8 +77,3 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: logs diff --git a/.github/workflows/plugins-test.3.yaml b/.github/workflows/plugins-test.3.yaml index 1ec947d9bee6..80af4c7a1d89 100644 --- a/.github/workflows/plugins-test.3.yaml +++ b/.github/workflows/plugins-test.3.yaml @@ -33,10 +33,10 @@ jobs: fail-fast: true matrix: case: + - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' } - { name: 'undertow-scenario', title: 'Undertow 1.3.0-2.0.27 (23)' } - { name: 'webflux-scenario', title: 'Spring-WebFlux 2.x (7)' } - { name: 'zookeeper-scenario', title: 'Zookeeper 3.4.x (14)' } - - { name: 'mysql-scenario', title: 'MySQL 5.1.2-8.0.15 (30)' } steps: - uses: actions/checkout@v2 - name: checkout submodules @@ -55,11 +55,6 @@ jobs: run: bash test/plugin/run.sh ${{ matrix.case.name }} - name: Report Coverage run: bash -x tools/coverage/report.sh - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: logs Oracle: name: Oracle @@ -87,8 +82,3 @@ jobs: bash test/plugin/run.sh oracle-scenario - name: Report Coverage run: bash -x tools/coverage/report.sh - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: logs - path: logs diff --git a/test/plugin/scenarios/mysql-scenario/support-version.list b/test/plugin/scenarios/mysql-scenario/support-version.list index e7f1040b3dbc..4768268ed6e1 100644 --- a/test/plugin/scenarios/mysql-scenario/support-version.list +++ b/test/plugin/scenarios/mysql-scenario/support-version.list @@ -20,4 +20,29 @@ 8.0.14 8.0.13 8.0.12 -8.0.11 \ No newline at end of file +8.0.11 +6.0.6 +6.0.4 +6.0.2 +5.1.44 +5.1.42 +5.1.40 +5.1.38 +5.1.36 +5.1.34 +5.1.32 +5.1.30 +5.1.28 +5.1.26 +5.1.24 +5.1.22 +5.1.20 +5.1.18 +5.1.16 +5.1.14 +5.1.12 +5.1.10 +5.1.8 +5.1.6 +5.1.4 +5.1.2 \ No newline at end of file From a2ea5245ec924401e6b0f57ad43006c06265771e Mon Sep 17 00:00:00 2001 From: daming Date: Sun, 19 Apr 2020 02:44:54 +0800 Subject: [PATCH 23/24] delete some versions of mysql temporarily --- .../scenarios/mysql-scenario/support-version.list | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/plugin/scenarios/mysql-scenario/support-version.list b/test/plugin/scenarios/mysql-scenario/support-version.list index 4768268ed6e1..bf4ddf6ff3c7 100644 --- a/test/plugin/scenarios/mysql-scenario/support-version.list +++ b/test/plugin/scenarios/mysql-scenario/support-version.list @@ -36,13 +36,4 @@ 5.1.26 5.1.24 5.1.22 -5.1.20 -5.1.18 -5.1.16 -5.1.14 -5.1.12 -5.1.10 -5.1.8 -5.1.6 -5.1.4 -5.1.2 \ No newline at end of file +5.1.20 \ No newline at end of file From 365b7bb401afd21cb184f95fc371ed619e573e7b Mon Sep 17 00:00:00 2001 From: daming Date: Sun, 19 Apr 2020 19:15:02 +0800 Subject: [PATCH 24/24] fix encode --- .../plugin/influxdb/query/TopologyQuery.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java index cef935c97440..67b6f4a7b735 100644 --- a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java @@ -231,14 +231,14 @@ private WhereSubQueryImpl, SelectQueryImpl> .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))); StringBuilder builder = new StringBuilder("(("); - builder.append(sourceCName).append("=").append(sourceServiceId) - .append(" and ") - .append(destCName).append("=").append(destServiceId) - .append(") or (") - .append(sourceCName).append("=").append(destServiceId) - .append(") and (") - .append(destCName).append("=").append(sourceServiceId) - .append("))"); + builder.append(sourceCName).append("='").append(sourceServiceId) + .append("' and ") + .append(destCName).append("='").append(destServiceId) + .append("') or (") + .append(sourceCName).append("='").append(destServiceId) + .append("') and (") + .append(destCName).append("='").append(sourceServiceId) + .append("'))"); subQuery.where(builder.toString()); subQuery.groupBy(InfluxConstants.TagName.ENTITY_ID); return subQuery;