diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/ExecutionContext.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/ExecutionContext.java index e84612ed..26559593 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/ExecutionContext.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/ExecutionContext.java @@ -3,6 +3,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import java.time.Duration; +import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; @@ -48,7 +49,7 @@ public class ExecutionContext { private final LinkedHashSet allSelections; private final Optional timeSeriesPeriod; private final Filter queryRequestFilter; - private final Supplier> timeRangeDurationSupplier; + private final Supplier> queryTimeRangeSupplier; public ExecutionContext(String tenantId, QueryRequest request) { this.tenantId = tenantId; @@ -56,16 +57,16 @@ public ExecutionContext(String tenantId, QueryRequest request) { this.allSelections = new LinkedHashSet<>(); this.timeSeriesPeriod = calculateTimeSeriesPeriod(request); this.queryRequestFilter = request.getFilter(); - timeRangeDurationSupplier = + queryTimeRangeSupplier = Suppliers.memoize( - () -> findTimeRangeDuration(this.queryRequestFilter, this.timeFilterColumn)); + () -> buildQueryTimeRange(this.queryRequestFilter, this.timeFilterColumn)); analyze(request); } private Optional calculateTimeSeriesPeriod(QueryRequest request) { if (request.getGroupByCount() > 0) { for (Expression expression : request.getGroupByList()) { - if (isDateTimeFunction(expression)) { + if (QueryRequestUtil.isDateTimeFunction(expression)) { String timeSeriesPeriod = expression .getFunction() @@ -192,7 +193,7 @@ private void extractColumns(List columns, Expression expression) { } } - private Optional findTimeRangeDuration(Filter filter, String timeFilterColumn) { + private Optional buildQueryTimeRange(Filter filter, String timeFilterColumn) { // time filter will always be present with AND operator if (filter.getOperator() != Operator.AND) { @@ -218,11 +219,15 @@ private Optional findTimeRangeDuration(Filter filter, String timeFilte .findFirst(); if (timeRangeStart.isPresent() && timeRangeEnd.isPresent()) { - return Optional.of(Duration.ofMillis(timeRangeEnd.get() - timeRangeStart.get())); + return Optional.of( + new QueryTimeRange( + Instant.ofEpochMilli(timeRangeStart.get()), + Instant.ofEpochMilli(timeRangeEnd.get()), + Duration.ofMillis(timeRangeEnd.get() - timeRangeStart.get()))); } return filter.getChildFilterList().stream() - .map(childFilter -> this.findTimeRangeDuration(childFilter, timeFilterColumn)) + .map(childFilter -> this.buildQueryTimeRange(childFilter, timeFilterColumn)) .flatMap(Optional::stream) .findFirst(); } @@ -240,11 +245,6 @@ private Duration parseDuration(String timeSeriesPeriod) { return Duration.of(amount, unit); } - private boolean isDateTimeFunction(Expression expression) { - return expression.getValueCase() == ValueCase.FUNCTION - && expression.getFunction().getFunctionName().equals("dateTimeConvert"); - } - public void setTimeFilterColumn(String timeFilterColumn) { this.timeFilterColumn = timeFilterColumn; } @@ -274,6 +274,14 @@ public Optional getTimeSeriesPeriod() { } public Optional getTimeRangeDuration() { - return timeRangeDurationSupplier.get(); + return queryTimeRangeSupplier.get().map(QueryTimeRange::getDuration); + } + + public Optional getQueryTimeRange() { + return queryTimeRangeSupplier.get(); + } + + public String getTimeFilterColumn() { + return timeFilterColumn; } } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java index 402a8386..5cb3ba73 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryRequestUtil.java @@ -1,7 +1,10 @@ package org.hypertrace.core.query.service; +import static org.hypertrace.core.query.service.api.Expression.ValueCase.ATTRIBUTE_EXPRESSION; + import org.hypertrace.core.query.service.api.ColumnIdentifier; import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Expression.ValueCase; import org.hypertrace.core.query.service.api.LiteralConstant; import org.hypertrace.core.query.service.api.Value; import org.hypertrace.core.query.service.api.ValueType; @@ -68,4 +71,30 @@ public static Expression createNullNumberLiteralExpression() { .setValue(Value.newBuilder().setValueType(ValueType.NULL_NUMBER))) .build(); } + + public static boolean isDateTimeFunction(Expression expression) { + return expression.getValueCase() == ValueCase.FUNCTION + && expression.getFunction().getFunctionName().equals("dateTimeConvert"); + } + + public static boolean isComplexAttribute(Expression expression) { + return expression.getValueCase().equals(ATTRIBUTE_EXPRESSION) + && expression.getAttributeExpression().hasSubpath(); + } + + public static boolean isSimpleColumnExpression(Expression expression) { + return expression.getValueCase() == ValueCase.COLUMNIDENTIFIER + || (expression.getValueCase() == ATTRIBUTE_EXPRESSION && !isComplexAttribute(expression)); + } + + public static String getLogicalColumnNameForSimpleColumnExpression(Expression expression) { + if (!isSimpleColumnExpression(expression)) { + throw new RuntimeException("Expecting expression of type COLUMN or ATTRIBUTE"); + } + if (expression.getValueCase() == ValueCase.COLUMNIDENTIFIER) { + return expression.getColumnIdentifier().getColumnName(); + } else { + return expression.getAttributeExpression().getAttributeId(); + } + } } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java index b87ea932..09c5d84e 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryServiceModule.java @@ -8,6 +8,7 @@ import org.hypertrace.core.query.service.api.QueryServiceGrpc.QueryServiceImplBase; import org.hypertrace.core.query.service.pinot.PinotModule; import org.hypertrace.core.query.service.projection.ProjectionModule; +import org.hypertrace.core.query.service.prometheus.PrometheusModule; import org.hypertrace.core.serviceframework.spi.PlatformServiceLifecycle; class QueryServiceModule extends AbstractModule { @@ -31,5 +32,6 @@ protected void configure() { .in(Singleton.class); install(new PinotModule()); install(new ProjectionModule()); + install(new PrometheusModule()); } } diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTimeRange.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTimeRange.java new file mode 100644 index 00000000..77b11049 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/QueryTimeRange.java @@ -0,0 +1,14 @@ +package org.hypertrace.core.query.service; + +import java.time.Duration; +import java.time.Instant; +import lombok.AllArgsConstructor; +import lombok.Value; + +@Value +@AllArgsConstructor +public class QueryTimeRange { + Instant startTime; + Instant endTime; + Duration duration; +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/QueryRequestToPinotSQLConverter.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/QueryRequestToPinotSQLConverter.java index 146c8336..1180f6d0 100644 --- a/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/QueryRequestToPinotSQLConverter.java +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/pinot/QueryRequestToPinotSQLConverter.java @@ -1,5 +1,6 @@ package org.hypertrace.core.query.service.pinot; +import static org.hypertrace.core.query.service.QueryRequestUtil.isComplexAttribute; import static org.hypertrace.core.query.service.api.Expression.ValueCase.ATTRIBUTE_EXPRESSION; import static org.hypertrace.core.query.service.api.Expression.ValueCase.COLUMNIDENTIFIER; import static org.hypertrace.core.query.service.api.Expression.ValueCase.LITERAL; @@ -365,11 +366,6 @@ private LiteralConstant[] convertExpressionToMapLiterals(Expression expression) return literals; } - private boolean isComplexAttribute(Expression expression) { - return expression.getValueCase().equals(ATTRIBUTE_EXPRESSION) - && expression.getAttributeExpression().hasSubpath(); - } - /** TODO:Handle all types */ private String convertLiteralToString(LiteralConstant literal, Params.Builder paramsBuilder) { Value value = literal.getValue(); diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/FilterToPromqlConverter.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/FilterToPromqlConverter.java new file mode 100644 index 00000000..84e6060a --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/FilterToPromqlConverter.java @@ -0,0 +1,112 @@ +package org.hypertrace.core.query.service.prometheus; + +import static org.hypertrace.core.query.service.QueryRequestUtil.getLogicalColumnNameForSimpleColumnExpression; + +import com.google.protobuf.ByteString; +import java.util.List; +import java.util.function.Function; +import org.apache.commons.codec.binary.Hex; +import org.hypertrace.core.query.service.QueryRequestUtil; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.LiteralConstant; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.Value; + +class FilterToPromqlConverter { + + /** only `AND` operator in filter is allowed rhs of leaf filter should be literal */ + void convertFilterToString( + Filter filter, + String timeFilterColumn, + Function expressionToColumnConverter, + List filterList) { + if (filter.getChildFilterCount() > 0) { + for (Filter childFilter : filter.getChildFilterList()) { + convertFilterToString( + childFilter, timeFilterColumn, expressionToColumnConverter, filterList); + } + } else { + if (QueryRequestUtil.isSimpleColumnExpression(filter.getLhs()) + && timeFilterColumn.equals( + getLogicalColumnNameForSimpleColumnExpression(filter.getLhs()))) { + return; + } + StringBuilder builder = new StringBuilder(); + builder.append(expressionToColumnConverter.apply(filter.getLhs())); + builder.append(convertOperatorToString(filter.getOperator())); + builder.append(convertLiteralToString(filter.getRhs().getLiteral())); + filterList.add(builder.toString()); + } + } + + private String convertOperatorToString(Operator operator) { + switch (operator) { + case IN: + case EQ: + return "="; + case NEQ: + return "!="; + case LIKE: + return "=~"; + default: + throw new RuntimeException( + String.format("Equivalent %s operator not supported in promql", operator)); + } + } + + private String convertLiteralToString(LiteralConstant literal) { + Value value = literal.getValue(); + switch (value.getValueType()) { + case STRING_ARRAY: + StringBuilder builder = new StringBuilder("\""); + for (String item : value.getStringArrayList()) { + if (builder.length() > 1) { + builder.append("|"); + } + builder.append(item); + } + builder.append("\""); + return builder.toString(); + case BYTES_ARRAY: + builder = new StringBuilder("\""); + for (ByteString item : value.getBytesArrayList()) { + if (builder.length() > 1) { + builder.append("|"); + } + builder.append(Hex.encodeHexString(item.toByteArray())); + } + builder.append("\""); + return builder.toString(); + case STRING: + return "\"" + value.getString() + "\""; + case LONG: + return "\"" + value.getLong() + "\""; + case INT: + return "\"" + value.getInt() + "\""; + case FLOAT: + return "\"" + value.getFloat() + "\""; + case DOUBLE: + return "\"" + value.getDouble() + "\""; + case BYTES: + return "\"" + Hex.encodeHexString(value.getBytes().toByteArray()) + "\""; + case BOOL: + return "\"" + value.getBoolean() + "\""; + case TIMESTAMP: + return "\"" + value.getTimestamp() + "\""; + case NULL_NUMBER: + return "0"; + case NULL_STRING: + return "null"; + case LONG_ARRAY: + case INT_ARRAY: + case FLOAT_ARRAY: + case DOUBLE_ARRAY: + case BOOLEAN_ARRAY: + case UNRECOGNIZED: + default: + throw new RuntimeException( + String.format("Literal type %s not supported", value.getValueType())); + } + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusBasedRequestHandler.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusBasedRequestHandler.java new file mode 100644 index 00000000..d4c11191 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusBasedRequestHandler.java @@ -0,0 +1,86 @@ +package org.hypertrace.core.query.service.prometheus; + +import com.google.common.base.Preconditions; +import com.typesafe.config.Config; +import io.reactivex.rxjava3.core.Observable; +import java.util.Optional; +import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.QueryCost; +import org.hypertrace.core.query.service.RequestHandler; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.Row; + +public class PrometheusBasedRequestHandler implements RequestHandler { + + private static final String VIEW_DEFINITION_CONFIG_KEY = "prometheusViewDefinition"; + private static final String TENANT_ATTRIBUTE_NAME_CONFIG_KEY = "tenantAttributeName"; + private static final String START_TIME_ATTRIBUTE_NAME_CONFIG_KEY = "startTimeAttributeName"; + + private final QueryRequestEligibilityValidator queryRequestEligibilityValidator; + private final String name; + private PrometheusViewDefinition prometheusViewDefinition; + private Optional startTimeAttributeName; + private QueryRequestToPromqlConverter requestToPromqlConverter; + + PrometheusBasedRequestHandler(String name, Config config) { + this.name = name; + this.processConfig(config); + this.queryRequestEligibilityValidator = + new QueryRequestEligibilityValidator(prometheusViewDefinition); + } + + @Override + public String getName() { + return name; + } + + @Override + public Optional getTimeFilterColumn() { + return this.startTimeAttributeName; + } + + private void processConfig(Config config) { + + if (!config.hasPath(TENANT_ATTRIBUTE_NAME_CONFIG_KEY)) { + throw new RuntimeException( + TENANT_ATTRIBUTE_NAME_CONFIG_KEY + + " is not defined in the " + + name + + " request handler."); + } + + String tenantAttributeName = config.getString(TENANT_ATTRIBUTE_NAME_CONFIG_KEY); + this.prometheusViewDefinition = + PrometheusViewDefinition.parse( + config.getConfig(VIEW_DEFINITION_CONFIG_KEY), tenantAttributeName); + + this.startTimeAttributeName = + config.hasPath(START_TIME_ATTRIBUTE_NAME_CONFIG_KEY) + ? Optional.of(config.getString(START_TIME_ATTRIBUTE_NAME_CONFIG_KEY)) + : Optional.empty(); + + this.requestToPromqlConverter = new QueryRequestToPromqlConverter(prometheusViewDefinition); + } + + /** + * Returns a QueryCost that is an indication of whether the given query can be handled by this + * handler and if so, how costly is it to handle that query. + */ + @Override + public QueryCost canHandle(QueryRequest request, ExecutionContext executionContext) { + return queryRequestEligibilityValidator.calculateCost(request, executionContext); + } + + @Override + public Observable handleRequest( + QueryRequest originalRequest, ExecutionContext executionContext) { + + // Validate QueryContext and tenant id presence + Preconditions.checkNotNull(executionContext); + Preconditions.checkNotNull(executionContext.getTenantId()); + + // todo call convert and execute request using client here + + return null; + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusFunctionConverter.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusFunctionConverter.java new file mode 100644 index 00000000..7100c057 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusFunctionConverter.java @@ -0,0 +1,41 @@ +package org.hypertrace.core.query.service.prometheus; + +import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_AVG; +import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_COUNT; +import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_MAX; +import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_MIN; +import static org.hypertrace.core.query.service.QueryFunctionConstants.QUERY_FUNCTION_SUM; + +import java.util.List; +import org.hypertrace.core.query.service.api.Expression; + +class PrometheusFunctionConverter { + + static final List supportedFunctions = + List.of( + QUERY_FUNCTION_SUM, + QUERY_FUNCTION_MAX, + QUERY_FUNCTION_MIN, + QUERY_FUNCTION_AVG, + QUERY_FUNCTION_COUNT); + + String mapToPrometheusFunctionName(Expression functionSelection) { + String queryFunctionName = functionSelection.getFunction().getFunctionName().toUpperCase(); + switch (queryFunctionName) { + case QUERY_FUNCTION_SUM: + return "sum"; + case QUERY_FUNCTION_MAX: + return "max"; + case QUERY_FUNCTION_MIN: + return "min"; + case QUERY_FUNCTION_AVG: + return "avg"; + case QUERY_FUNCTION_COUNT: + return "count"; + default: + throw new RuntimeException( + String.format( + "Couldn't map query function [%s] to prometheus function", queryFunctionName)); + } + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusModule.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusModule.java new file mode 100644 index 00000000..b19c609f --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusModule.java @@ -0,0 +1,17 @@ +package org.hypertrace.core.query.service.prometheus; + +import com.google.inject.AbstractModule; +import com.google.inject.multibindings.Multibinder; +import org.hypertrace.core.query.service.RequestHandlerBuilder; +import org.hypertrace.core.query.service.RequestHandlerClientConfigRegistry; + +public class PrometheusModule extends AbstractModule { + + @Override + protected void configure() { + Multibinder.newSetBinder(binder(), RequestHandlerBuilder.class) + .addBinding() + .to(PrometheusRequestHandlerBuilder.class); + requireBinding(RequestHandlerClientConfigRegistry.class); + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusRequestHandlerBuilder.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusRequestHandlerBuilder.java new file mode 100644 index 00000000..c8df6d03 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusRequestHandlerBuilder.java @@ -0,0 +1,39 @@ +package org.hypertrace.core.query.service.prometheus; + +import javax.inject.Inject; +import org.hypertrace.core.query.service.QueryServiceConfig.RequestHandlerClientConfig; +import org.hypertrace.core.query.service.QueryServiceConfig.RequestHandlerConfig; +import org.hypertrace.core.query.service.RequestHandler; +import org.hypertrace.core.query.service.RequestHandlerBuilder; +import org.hypertrace.core.query.service.RequestHandlerClientConfigRegistry; + +public class PrometheusRequestHandlerBuilder implements RequestHandlerBuilder { + + private final RequestHandlerClientConfigRegistry clientConfigRegistry; + + @Inject + PrometheusRequestHandlerBuilder(RequestHandlerClientConfigRegistry clientConfigRegistry) { + this.clientConfigRegistry = clientConfigRegistry; + } + + @Override + public boolean canBuild(RequestHandlerConfig config) { + return "prometheus".equals(config.getType()); + } + + @Override + public RequestHandler build(RequestHandlerConfig config) { + + RequestHandlerClientConfig clientConfig = + this.clientConfigRegistry + .get(config.getClientConfig()) + .orElseThrow( + () -> + new UnsupportedOperationException( + "Client config requested but not registered: " + config.getClientConfig())); + + // todo build prom client + + return new PrometheusBasedRequestHandler(config.getName(), config.getRequestHandlerInfo()); + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusViewDefinition.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusViewDefinition.java new file mode 100644 index 00000000..8f202374 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/PrometheusViewDefinition.java @@ -0,0 +1,102 @@ +package org.hypertrace.core.query.service.prometheus; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigUtil; +import com.typesafe.config.ConfigValue; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import lombok.AllArgsConstructor; +import lombok.Value; + +/** Prometheus metric & attribute mapping for a pinot view */ +class PrometheusViewDefinition { + + private static final String VIEW_NAME_CONFIG_KEY = "viewName"; + private static final String ATTRIBUTE_MAP_CONFIG_KEY = "attributeMap"; + private static final String METRIC_MAP_CONFIG_KEY = "metricMap"; + private static final String METRIC_NAME_CONFIG_KEY = "metricName"; + private static final String METRIC_TYPE_CONFIG_KEY = "metricType"; + private static final String METRIC_SCOPE_CONFIG_KEY = "metricScope"; + + private final String viewName; + private final String tenantAttributeName; + private final Map metricMap; + private final Map attributeMap; + + public PrometheusViewDefinition( + String viewName, + String tenantAttributeName, + Map metricMap, + Map attributeMap) { + this.viewName = viewName; + this.tenantAttributeName = tenantAttributeName; + this.metricMap = metricMap; + this.attributeMap = attributeMap; + } + + public static PrometheusViewDefinition parse(Config config, String tenantAttributeName) { + String viewName = config.getString(VIEW_NAME_CONFIG_KEY); + + final Map attributeMap = Maps.newHashMap(); + Config fieldMapConfig = config.getConfig(ATTRIBUTE_MAP_CONFIG_KEY); + for (Entry element : fieldMapConfig.entrySet()) { + List keys = ConfigUtil.splitPath(element.getKey()); + attributeMap.put(keys.get(0), fieldMapConfig.getString(element.getKey())); + } + + Config metricMapConfig = config.getConfig(METRIC_MAP_CONFIG_KEY); + + Set metricNames = Sets.newHashSet(); + for (Entry element : metricMapConfig.entrySet()) { + List keys = ConfigUtil.splitPath(element.getKey()); + metricNames.add(keys.get(0)); + } + + final Map metricMap = Maps.newHashMap(); + String metricScope = config.getString(METRIC_SCOPE_CONFIG_KEY); + for (String metricName : metricNames) { + Config metricDef = metricMapConfig.getConfig(metricName); + metricMap.put( + metricScope + "." + metricName, + new MetricConfig( + metricDef.getString(METRIC_NAME_CONFIG_KEY), + MetricType.valueOf(metricDef.getString(METRIC_TYPE_CONFIG_KEY)))); + } + + return new PrometheusViewDefinition( + viewName, tenantAttributeName, + metricMap, attributeMap); + } + + public String getPhysicalColumnNameForLogicalColumnName(String logicalColumnName) { + return attributeMap.get(logicalColumnName); + } + + public MetricConfig getMetricConfigForLogicalMetricName(String logicalMetricName) { + return metricMap.get(logicalMetricName); + } + + public String getViewName() { + return viewName; + } + + public String getTenantAttributeName() { + return tenantAttributeName; + } + + @Value + @AllArgsConstructor + public static class MetricConfig { + String metricName; + MetricType metricType; + } + + public enum MetricType { + GAUGE, + COUNTER + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/QueryRequestEligibilityValidator.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/QueryRequestEligibilityValidator.java new file mode 100644 index 00000000..4b271810 --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/QueryRequestEligibilityValidator.java @@ -0,0 +1,153 @@ +package org.hypertrace.core.query.service.prometheus; + +import static org.hypertrace.core.query.service.QueryRequestUtil.getLogicalColumnNameForSimpleColumnExpression; + +import com.google.common.base.Preconditions; +import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.QueryCost; +import org.hypertrace.core.query.service.QueryRequestUtil; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Expression.ValueCase; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Function; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.prometheus.PrometheusViewDefinition.MetricConfig; +import org.hypertrace.core.query.service.prometheus.PrometheusViewDefinition.MetricType; + +/** Set of rules to check if the given request can be served by prometheus */ +class QueryRequestEligibilityValidator { + + private final PrometheusViewDefinition prometheusViewDefinition; + + public QueryRequestEligibilityValidator(PrometheusViewDefinition prometheusViewDefinition) { + this.prometheusViewDefinition = prometheusViewDefinition; + } + + QueryCost calculateCost(QueryRequest queryRequest, ExecutionContext executionContext) { + try { + // orderBy to be supported later + if (queryRequest.getOrderByCount() > 0) { + return QueryCost.UNSUPPORTED; + } + + // only aggregation queries are supported + if (queryRequest.getAggregationCount() == 0 || queryRequest.getGroupByCount() == 0) { + return QueryCost.UNSUPPORTED; + } + + if (queryRequest.getDistinctSelections()) { + return QueryCost.UNSUPPORTED; + } + + // all selection including group by and aggregations should be either on column or attribute + if (executionContext.getAllSelections().stream() + .filter(Predicate.not(QueryRequestUtil::isDateTimeFunction)) + .anyMatch(Predicate.not(QueryRequestUtil::isSimpleColumnExpression))) { + return QueryCost.UNSUPPORTED; + } + + Set referencedColumns = executionContext.getReferencedColumns(); + Preconditions.checkArgument(!referencedColumns.isEmpty()); + // all the columns in the request should have a mapping in the config + for (String referencedColumn : referencedColumns) { + if (prometheusViewDefinition.getPhysicalColumnNameForLogicalColumnName(referencedColumn) + == null + && prometheusViewDefinition.getMetricConfigForLogicalMetricName(referencedColumn) + == null) { + return QueryCost.UNSUPPORTED; + } + } + + if (areAggregationsNotSupported(queryRequest.getAggregationList())) { + return QueryCost.UNSUPPORTED; + } + + // if selection and groupBy should be on same column or simple attribute + if (selectionAndGroupByOnDifferentColumn( + queryRequest.getSelectionList(), queryRequest.getGroupByList())) { + return QueryCost.UNSUPPORTED; + } + + if (isFilterNotSupported(queryRequest.getFilter())) { + return QueryCost.UNSUPPORTED; + } + } catch (Exception e) { + return QueryCost.UNSUPPORTED; + } + // value 1.0 so that prometheus is preferred over others + return new QueryCost(1.0); + } + + private boolean selectionAndGroupByOnDifferentColumn( + List selectionList, List groupByList) { + + Set selections = + selectionList.stream() + .map(QueryRequestUtil::getLogicalColumnNameForSimpleColumnExpression) + .collect(Collectors.toSet()); + + Set groupBys = + groupByList.stream() + .filter(Predicate.not(QueryRequestUtil::isDateTimeFunction)) + .map(QueryRequestUtil::getLogicalColumnNameForSimpleColumnExpression) + .collect(Collectors.toSet()); + return !selections.equals(groupBys); + } + + private boolean areAggregationsNotSupported(List aggregationList) { + // supported aggregation must have single argument (except for dateTimeConvert) + // prometheusViewDef must have mapping for the metric + // function type must be supported + // right now only GAUGE type of metric is supported + return aggregationList.stream() + .filter(Predicate.not(QueryRequestUtil::isDateTimeFunction)) + .anyMatch( + expression -> { + Function function = expression.getFunction(); + if (function.getArgumentsCount() > 1) { + return true; + } + Expression functionArgument = function.getArgumentsList().get(0); + String attributeId = getLogicalColumnNameForSimpleColumnExpression(functionArgument); + if (!PrometheusFunctionConverter.supportedFunctions.contains( + function.getFunctionName())) { + return true; + } + MetricConfig metricConfig = + prometheusViewDefinition.getMetricConfigForLogicalMetricName(attributeId); + return null == metricConfig || metricConfig.getMetricType() != MetricType.GAUGE; + }); + } + + private boolean isFilterNotSupported(Filter filter) { + if (filter.getChildFilterCount() > 0) { + // Currently, `AND` high level composite operator is supported + // later OR operator can be supported for same column + if (filter.getOperator() != Operator.AND) { + return true; + } + for (Filter childFilter : filter.getChildFilterList()) { + if (!isFilterNotSupported(childFilter)) { + return true; + } + } + } else { + // rhs condition of filter should be literal only + if (filter.getRhs().getValueCase() != ValueCase.LITERAL) { + return true; + } + + // lhs condition of filter should be column or simple attribute + if (!QueryRequestUtil.isSimpleColumnExpression(filter.getLhs())) { + return true; + } + } + + return false; + } +} diff --git a/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/QueryRequestToPromqlConverter.java b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/QueryRequestToPromqlConverter.java new file mode 100644 index 00000000..5f54311b --- /dev/null +++ b/query-service-impl/src/main/java/org/hypertrace/core/query/service/prometheus/QueryRequestToPromqlConverter.java @@ -0,0 +1,151 @@ +package org.hypertrace.core.query.service.prometheus; + +import static org.hypertrace.core.query.service.QueryRequestUtil.getLogicalColumnNameForSimpleColumnExpression; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.QueryRequestUtil; +import org.hypertrace.core.query.service.QueryTimeRange; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Expression.ValueCase; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.prometheus.PrometheusViewDefinition.MetricConfig; + +class QueryRequestToPromqlConverter { + + private final PrometheusViewDefinition prometheusViewDefinition; + private final PrometheusFunctionConverter prometheusFunctionConverter; + private final FilterToPromqlConverter filterToPromqlConverter; + + QueryRequestToPromqlConverter(PrometheusViewDefinition prometheusViewDefinition) { + this.prometheusViewDefinition = prometheusViewDefinition; + this.prometheusFunctionConverter = new PrometheusFunctionConverter(); + this.filterToPromqlConverter = new FilterToPromqlConverter(); + } + + PromQLInstantQueries convertToPromqlInstantQuery( + ExecutionContext executionContext, + QueryRequest request, + LinkedHashSet allSelections) { + QueryTimeRange queryTimeRange = getQueryTimeRange(executionContext); + return new PromQLInstantQueries( + buildPromqlQueries( + executionContext.getTenantId(), + request, + allSelections, + queryTimeRange.getDuration(), + executionContext.getTimeFilterColumn()), + queryTimeRange.getEndTime()); + } + + PromQLRangeQueries convertToPromqlRangeQuery( + ExecutionContext executionContext, + QueryRequest request, + LinkedHashSet allSelections) { + QueryTimeRange queryTimeRange = getQueryTimeRange(executionContext); + return new PromQLRangeQueries( + buildPromqlQueries( + executionContext.getTenantId(), + request, + allSelections, + executionContext.getTimeSeriesPeriod().get(), + executionContext.getTimeFilterColumn()), + queryTimeRange.getStartTime(), + queryTimeRange.getEndTime(), + getTimeSeriesPeriod(executionContext)); + } + + private QueryTimeRange getQueryTimeRange(ExecutionContext executionContext) { + return executionContext + .getQueryTimeRange() + .orElseThrow(() -> new RuntimeException("Time Range missing in query")); + } + + private List buildPromqlQueries( + String tenantId, + QueryRequest request, + LinkedHashSet allSelections, + Duration duration, + String timeFilterColumn) { + List groupByList = getGroupByList(request); + + List filterList = new ArrayList<>(); + filterList.add( + String.format("%s=\"%s\"", prometheusViewDefinition.getTenantAttributeName(), tenantId)); + filterToPromqlConverter.convertFilterToString( + request.getFilter(), timeFilterColumn, this::convertColumnAttributeToString, filterList); + + // iterate over all the functions in the query except for date time function (which is handled + // separately and not a part of the query string) + return allSelections.stream() + .filter( + expression -> + expression.getValueCase().equals(ValueCase.FUNCTION) + && !QueryRequestUtil.isDateTimeFunction(expression)) + .map( + functionExpression -> + mapToPromqlQuery(functionExpression, groupByList, filterList, duration)) + .collect(Collectors.toUnmodifiableList()); + } + + private String mapToPromqlQuery( + Expression functionExpression, + List groupByList, + List filterList, + Duration duration) { + String functionName = + prometheusFunctionConverter.mapToPrometheusFunctionName(functionExpression); + MetricConfig metricConfig = getMetricConfigForFunction(functionExpression); + return buildQuery( + metricConfig.getMetricName(), + functionName, + String.join(", ", groupByList), + String.join(", ", filterList), + duration.toMillis()); + } + + private List getGroupByList(QueryRequest queryRequest) { + return queryRequest.getGroupByList().stream() + .filter(Predicate.not(QueryRequestUtil::isDateTimeFunction)) + .map(this::convertColumnAttributeToString) + .collect(Collectors.toUnmodifiableList()); + } + + private Duration getTimeSeriesPeriod(ExecutionContext executionContext) { + return executionContext.getTimeSeriesPeriod().get(); + } + + /** + * Builds a promql query. example query `sum by (a1, a2) (sum_over_time(num_calls{a4="..", + * a5=".."}[xms]))` + */ + private String buildQuery( + String metricName, String function, String groupByList, String filter, long durationMillis) { + String template = "%s by (%s) (%s(%s{%s}[%sms]))"; + + return String.format( + template, + function, + groupByList, + function + "_over_time", // assuming gauge type of metric + metricName, + filter, + durationMillis); + } + + private MetricConfig getMetricConfigForFunction(Expression functionSelection) { + return prometheusViewDefinition.getMetricConfigForLogicalMetricName( + getLogicalColumnNameForSimpleColumnExpression( + functionSelection.getFunction().getArgumentsList().get(0))); + } + + private String convertColumnAttributeToString(Expression expression) { + return prometheusViewDefinition.getPhysicalColumnNameForLogicalColumnName( + getLogicalColumnNameForSimpleColumnExpression(expression)); + } +} diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java index 721abaa2..a067c2ca 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/QueryServiceConfigTest.java @@ -33,7 +33,7 @@ public void testQueryServiceImplConfigParser() { assertEquals("query-service", appConfig.getString("service.name")); assertEquals(8091, appConfig.getInt("service.admin.port")); assertEquals(8090, appConfig.getInt("service.port")); - assertEquals(4, queryServiceConfig.getQueryRequestHandlersConfigs().size()); + assertEquals(6, queryServiceConfig.getQueryRequestHandlersConfigs().size()); assertEquals(2, queryServiceConfig.getRequestHandlerClientConfigs().size()); RequestHandlerConfig handler0 = queryServiceConfig.getQueryRequestHandlersConfigs().get(0); diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/ExecutionContextTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/ExecutionContextTest.java index 1aea0fbc..6a03fe3f 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/ExecutionContextTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/ExecutionContextTest.java @@ -15,6 +15,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.QueryTimeRange; import org.hypertrace.core.query.service.api.ColumnIdentifier; import org.hypertrace.core.query.service.api.Expression; import org.hypertrace.core.query.service.api.Filter; @@ -389,7 +390,7 @@ public void testEmptyGetTimeRangeDuration() { .build(); ExecutionContext context = new ExecutionContext("test", queryRequest); context.setTimeFilterColumn("SERVICE.startTime"); - assertEquals(Optional.empty(), context.getTimeRangeDuration()); + assertEquals(Optional.empty(), context.getQueryTimeRange()); } @ParameterizedTest @@ -397,7 +398,9 @@ public void testEmptyGetTimeRangeDuration() { public void testGetTimeRangeDuration(QueryRequest queryRequest) { ExecutionContext context = new ExecutionContext("test", queryRequest); context.setTimeFilterColumn("SERVICE.startTime"); - assertEquals(Optional.of(Duration.ofMinutes(60)), context.getTimeRangeDuration()); + assertEquals( + Optional.of(Duration.ofMinutes(60)), + context.getQueryTimeRange().map(QueryTimeRange::getDuration)); } private static Stream provideQueryRequest() { diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandlerTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandlerTest.java index e42a1f90..6ddbfc04 100644 --- a/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandlerTest.java +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/pinot/PinotBasedRequestHandlerTest.java @@ -75,6 +75,10 @@ public void setUp() { @Test public void testInit() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); } @@ -95,6 +99,10 @@ public void testInitFailure() { @Test public void testCanHandle() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -123,6 +131,10 @@ public void testCanHandle() { @Test public void testCanHandleNegativeCase() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -147,6 +159,10 @@ public void testCanHandleNegativeCase() { @Test public void testCanHandleWithInViewFilter() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -244,6 +260,10 @@ public void testCanHandleWithInViewFilter() { @Test public void testCanHandleWithEqViewFilter() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -447,6 +467,10 @@ public void testCanHandleWithEqViewFilter() { @Test public void testCanHandleWithMultipleViewFilters() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -702,6 +726,10 @@ public void testCanHandleWithMultipleViewFilters() { @Test public void testCanHandleWithMultipleViewFiltersAndRepeatedQueryFilters() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -896,6 +924,10 @@ public void testCanHandleWithMultipleViewFiltersAndRepeatedQueryFilters() { @Test void testCanHandle_costShouldBeLessThanHalf() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } + PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -928,6 +960,9 @@ void testCanHandle_costShouldBeLessThanHalf() { @Test void testCanHandle_costShouldBeMoreThanHalf() { for (Config config : serviceConfig.getConfigList("queryRequestHandlersConfig")) { + if (!isPinotConfig(config)) { + continue; + } PinotBasedRequestHandler handler = new PinotBasedRequestHandler( config.getString("name"), config.getConfig("requestHandlerInfo")); @@ -1211,7 +1246,8 @@ public boolean isResultTableResultSetType(ResultSet resultSet) { QueryRequestBuilderUtils.createColumnExpression("Trace.duration_millis")) .build(); ExecutionContext context = new ExecutionContext("__default", request); - verifyResponseRows(handler.handleRequest(request, context), resultTable); + Observable rows = handler.handleRequest(request, context); + verifyResponseRows(rows, resultTable); } } @@ -1458,4 +1494,8 @@ private void verifyResponseRows(Observable rowObservable, String[][] expect private String stringify(Object obj) throws JsonProcessingException { return objectMapper.writeValueAsString(obj); } + + private boolean isPinotConfig(Config config) { + return config.getString("type").equals("pinot"); + } } diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/prometheus/QueryRequestEligibilityValidatorTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/prometheus/QueryRequestEligibilityValidatorTest.java new file mode 100644 index 00000000..686bf390 --- /dev/null +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/prometheus/QueryRequestEligibilityValidatorTest.java @@ -0,0 +1,113 @@ +package org.hypertrace.core.query.service.prometheus; + +import static java.util.Objects.requireNonNull; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createColumnExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFunctionExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createOrderByExpression; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.QueryCost; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.QueryRequest.Builder; +import org.hypertrace.core.query.service.api.SortOrder; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class QueryRequestEligibilityValidatorTest { + + private static final String TENANT_COLUMN_NAME = "tenant_id"; + private static final String TEST_REQUEST_HANDLER_CONFIG_FILE = "prometheus_request_handler.conf"; + + private QueryRequestEligibilityValidator queryRequestEligibilityValidator; + + @BeforeEach + public void setup() { + queryRequestEligibilityValidator = + new QueryRequestEligibilityValidator(getDefaultPrometheusViewDefinition()); + } + + @Test + void testCalculateCost_orderBy() { + QueryRequest queryRequest = buildOrderByQuery(); + + ExecutionContext executionContext = new ExecutionContext("__default", queryRequest); + executionContext.setTimeFilterColumn("SERVICE.startTime"); + + Assertions.assertEquals( + QueryCost.UNSUPPORTED, + queryRequestEligibilityValidator.calculateCost(queryRequest, executionContext)); + } + + @Test + void testCalculateCost_groupByAndSelectionOnDifferentColumn() { + Builder builder = QueryRequest.newBuilder(); + Expression startTimeColumn = createColumnExpression("SERVICE.startTime").build(); + + builder.addSelection(createColumnExpression("SERVICE.id")); + builder.addSelection(startTimeColumn); + builder.addGroupBy(createColumnExpression("SERVICE.name")); + + QueryRequest queryRequest = builder.build(); + + ExecutionContext executionContext = new ExecutionContext("__default", queryRequest); + executionContext.setTimeFilterColumn("SERVICE.startTime"); + + Assertions.assertEquals( + QueryCost.UNSUPPORTED, + queryRequestEligibilityValidator.calculateCost(queryRequest, executionContext)); + } + + @Test + void testCalculateCost_aggregationNotSupported() { + Builder builder = QueryRequest.newBuilder(); + builder.addAggregation( + createFunctionExpression("Count", createColumnExpression("SERVICE.name").build())); + + Expression startTimeColumn = createColumnExpression("SERVICE.startTime").build(); + + builder.addSelection(createColumnExpression("SERVICE.id")); + builder.addSelection(startTimeColumn); + builder.addGroupBy(createColumnExpression("SERVICE.name")); + + QueryRequest queryRequest = builder.build(); + + ExecutionContext executionContext = new ExecutionContext("__default", queryRequest); + executionContext.setTimeFilterColumn("SERVICE.startTime"); + + Assertions.assertEquals( + QueryCost.UNSUPPORTED, + queryRequestEligibilityValidator.calculateCost(queryRequest, executionContext)); + } + + private PrometheusViewDefinition getDefaultPrometheusViewDefinition() { + Config fileConfig = + ConfigFactory.parseURL( + requireNonNull( + QueryRequestToPromqlConverterTest.class + .getClassLoader() + .getResource(TEST_REQUEST_HANDLER_CONFIG_FILE))); + + return PrometheusViewDefinition.parse( + fileConfig.getConfig("requestHandlerInfo.prometheusViewDefinition"), TENANT_COLUMN_NAME); + } + + private QueryRequest buildOrderByQuery() { + Builder builder = QueryRequest.newBuilder(); + Expression startTimeColumn = createColumnExpression("SERVICE.startTime").build(); + Expression endTimeColumn = createColumnExpression("SERVICE.endTime").build(); + + builder.addSelection(createColumnExpression("SERVICE.id")); + builder.addSelection(startTimeColumn); + builder.addSelection(endTimeColumn); + + builder.addOrderBy(createOrderByExpression(startTimeColumn.toBuilder(), SortOrder.DESC)); + builder.addOrderBy(createOrderByExpression(endTimeColumn.toBuilder(), SortOrder.ASC)); + + builder.setLimit(100); + return builder.build(); + } +} diff --git a/query-service-impl/src/test/java/org/hypertrace/core/query/service/prometheus/QueryRequestToPromqlConverterTest.java b/query-service-impl/src/test/java/org/hypertrace/core/query/service/prometheus/QueryRequestToPromqlConverterTest.java new file mode 100644 index 00000000..c5e1ce1f --- /dev/null +++ b/query-service-impl/src/test/java/org/hypertrace/core/query/service/prometheus/QueryRequestToPromqlConverterTest.java @@ -0,0 +1,219 @@ +package org.hypertrace.core.query.service.prometheus; + +import static java.util.Objects.requireNonNull; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createColumnExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createFunctionExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createInFilter; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createStringLiteralValueExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createTimeColumnGroupByExpression; +import static org.hypertrace.core.query.service.QueryRequestBuilderUtils.createTimeFilter; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import java.util.LinkedHashSet; +import java.util.List; +import org.hypertrace.core.query.service.ExecutionContext; +import org.hypertrace.core.query.service.api.Expression; +import org.hypertrace.core.query.service.api.Filter; +import org.hypertrace.core.query.service.api.Operator; +import org.hypertrace.core.query.service.api.QueryRequest; +import org.hypertrace.core.query.service.api.QueryRequest.Builder; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class QueryRequestToPromqlConverterTest { + + private static final String TENANT_COLUMN_NAME = "tenant_id"; + + private static final String TEST_REQUEST_HANDLER_CONFIG_FILE = "prometheus_request_handler.conf"; + + @Test + void testInstantQueryWithGroupByWithMultipleAggregates() { + QueryRequest query = buildMultipleGroupByMultipleAggQuery(); + Builder builder = QueryRequest.newBuilder(query); + builder.setLimit(20); + PrometheusViewDefinition prometheusViewDefinition = getDefaultPrometheusViewDefinition(); + + QueryRequest queryRequest = builder.build(); + + ExecutionContext executionContext = new ExecutionContext("__default", queryRequest); + executionContext.setTimeFilterColumn("SERVICE.startTime"); + PromQLInstantQueries promqlQuery = + new QueryRequestToPromqlConverter(prometheusViewDefinition) + .convertToPromqlInstantQuery( + executionContext, builder.build(), createSelectionsFromQueryRequest(queryRequest)); + + // time filter is removed from the query + String query1 = + "count by (service_name, api_name) (count_over_time(error_count{tenant_id=\"__default\"}[100ms]))"; + String query2 = + "avg by (service_name, api_name) (avg_over_time(num_calls{tenant_id=\"__default\"}[100ms]))"; + + Assertions.assertTrue(promqlQuery.getQueries().contains(query1)); + Assertions.assertTrue(promqlQuery.getQueries().contains(query2)); + } + + @Test + void testInstantQueryWithGroupByWithMultipleAggregatesWithMultipleFilters() { + QueryRequest query = buildMultipleGroupByMultipleAggQueryWithMultipleFilters(); + Builder builder = QueryRequest.newBuilder(query); + builder.setLimit(20); + PrometheusViewDefinition prometheusViewDefinition = getDefaultPrometheusViewDefinition(); + + QueryRequest queryRequest = builder.build(); + + ExecutionContext executionContext = new ExecutionContext("__default", queryRequest); + executionContext.setTimeFilterColumn("SERVICE.startTime"); + PromQLInstantQueries promqlQuery = + new QueryRequestToPromqlConverter(prometheusViewDefinition) + .convertToPromqlInstantQuery( + executionContext, builder.build(), createSelectionsFromQueryRequest(queryRequest)); + + // time filter is removed from the query + String query1 = + "count by (service_name, api_name) (count_over_time(error_count{tenant_id=\"__default\", service_id=\"1|2|3\", service_name=~\"someregex\"}[100ms]))"; + String query2 = + "avg by (service_name, api_name) (avg_over_time(num_calls{tenant_id=\"__default\", service_id=\"1|2|3\", service_name=~\"someregex\"}[100ms]))"; + + Assertions.assertTrue(promqlQuery.getQueries().contains(query1)); + Assertions.assertTrue(promqlQuery.getQueries().contains(query2)); + } + + @Test + void testTimeSeriesQueryWithGroupByWithMultipleAggregatesWithMultipleFilters() { + QueryRequest query = buildMultipleGroupByMultipleAggQueryWithMultipleFiltersAndDateTime(); + Builder builder = QueryRequest.newBuilder(query); + builder.setLimit(20); + PrometheusViewDefinition prometheusViewDefinition = getDefaultPrometheusViewDefinition(); + + QueryRequest queryRequest = builder.build(); + + ExecutionContext executionContext = new ExecutionContext("__default", queryRequest); + executionContext.setTimeFilterColumn("SERVICE.startTime"); + PromQLRangeQueries promqlQuery = + new QueryRequestToPromqlConverter(prometheusViewDefinition) + .convertToPromqlRangeQuery( + executionContext, builder.build(), createSelectionsFromQueryRequest(queryRequest)); + + // time filter is removed from the query + String query1 = + "count by (service_name, api_name) (count_over_time(error_count{tenant_id=\"__default\", service_id=\"1|2|3\", service_name=~\"someregex\"}[10ms]))"; + String query2 = + "avg by (service_name, api_name) (avg_over_time(num_calls{tenant_id=\"__default\", service_id=\"1|2|3\", service_name=~\"someregex\"}[10ms]))"; + + Assertions.assertTrue(promqlQuery.getQueries().contains(query1)); + Assertions.assertTrue(promqlQuery.getQueries().contains(query2)); + Assertions.assertEquals(10, promqlQuery.getPeriod().toMillis()); + } + + private QueryRequest buildMultipleGroupByMultipleAggQuery() { + Builder builder = QueryRequest.newBuilder(); + builder.addAggregation( + createFunctionExpression("Count", createColumnExpression("SERVICE.errorCount").build())); + Expression avg = + createFunctionExpression("AVG", createColumnExpression("SERVICE.numCalls").build()); + builder.addAggregation(avg); + + Filter startTimeFilter = createTimeFilter("SERVICE.startTime", Operator.GT, 100L); + Filter endTimeFilter = createTimeFilter("SERVICE.startTime", Operator.LT, 200L); + + Filter andFilter = + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(startTimeFilter) + .addChildFilter(endTimeFilter) + .build(); + builder.setFilter(andFilter); + + builder.addGroupBy(createColumnExpression("SERVICE.name")); + builder.addGroupBy(createColumnExpression("API.name")); + return builder.build(); + } + + private QueryRequest buildMultipleGroupByMultipleAggQueryWithMultipleFilters() { + Builder builder = QueryRequest.newBuilder(); + builder.addAggregation( + createFunctionExpression("Count", createColumnExpression("SERVICE.errorCount").build())); + Expression avg = + createFunctionExpression("AVG", createColumnExpression("SERVICE.numCalls").build()); + builder.addAggregation(avg); + + Filter startTimeFilter = createTimeFilter("SERVICE.startTime", Operator.GT, 100L); + Filter endTimeFilter = createTimeFilter("SERVICE.startTime", Operator.LT, 200L); + Filter inFilter = createInFilter("SERVICE.id", List.of("1", "2", "3")); + Filter likeFilter = + Filter.newBuilder() + .setOperator(Operator.LIKE) + .setLhs(createColumnExpression("SERVICE.name").build()) + .setRhs(createStringLiteralValueExpression("someregex")) + .build(); + Filter andFilter = + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(startTimeFilter) + .addChildFilter(endTimeFilter) + .addChildFilter(inFilter) + .addChildFilter(likeFilter) + .build(); + builder.setFilter(andFilter); + + builder.addGroupBy(createColumnExpression("SERVICE.name")); + builder.addGroupBy(createColumnExpression("API.name")); + return builder.build(); + } + + private QueryRequest buildMultipleGroupByMultipleAggQueryWithMultipleFiltersAndDateTime() { + Builder builder = QueryRequest.newBuilder(); + builder.addAggregation( + createFunctionExpression("Count", createColumnExpression("SERVICE.errorCount").build())); + Expression avg = + createFunctionExpression("AVG", createColumnExpression("SERVICE.numCalls").build()); + builder.addAggregation(avg); + + Filter startTimeFilter = createTimeFilter("SERVICE.startTime", Operator.GT, 100L); + Filter endTimeFilter = createTimeFilter("SERVICE.startTime", Operator.LT, 200L); + Filter inFilter = createInFilter("SERVICE.id", List.of("1", "2", "3")); + Filter likeFilter = + Filter.newBuilder() + .setOperator(Operator.LIKE) + .setLhs(createColumnExpression("SERVICE.name").build()) + .setRhs(createStringLiteralValueExpression("someregex")) + .build(); + Filter andFilter = + Filter.newBuilder() + .setOperator(Operator.AND) + .addChildFilter(startTimeFilter) + .addChildFilter(endTimeFilter) + .addChildFilter(inFilter) + .addChildFilter(likeFilter) + .build(); + builder.setFilter(andFilter); + + builder.addGroupBy(createColumnExpression("SERVICE.name")); + builder.addGroupBy(createColumnExpression("API.name")); + builder.addGroupBy(createTimeColumnGroupByExpression("SERVICE.startTime", "10:MILLISECONDS")); + return builder.build(); + } + + private PrometheusViewDefinition getDefaultPrometheusViewDefinition() { + Config fileConfig = + ConfigFactory.parseURL( + requireNonNull( + QueryRequestToPromqlConverterTest.class + .getClassLoader() + .getResource(TEST_REQUEST_HANDLER_CONFIG_FILE))); + + return PrometheusViewDefinition.parse( + fileConfig.getConfig("requestHandlerInfo.prometheusViewDefinition"), TENANT_COLUMN_NAME); + } + + private LinkedHashSet createSelectionsFromQueryRequest(QueryRequest queryRequest) { + LinkedHashSet selections = new LinkedHashSet<>(); + + selections.addAll(queryRequest.getGroupByList()); + selections.addAll(queryRequest.getSelectionList()); + selections.addAll(queryRequest.getAggregationList()); + + return selections; + } +} diff --git a/query-service-impl/src/test/resources/application.conf b/query-service-impl/src/test/resources/application.conf index 0be1f02a..6e306083 100644 --- a/query-service-impl/src/test/resources/application.conf +++ b/query-service-impl/src/test/resources/application.conf @@ -160,5 +160,67 @@ service.config = { } } } + { + name = raw-service-view-service-scope-prometheus-handler + type = prometheus + clientConfig = "" + requestHandlerInfo { + tenantAttributeName = tenant_id + startTimeAttributeName = "SERVICE.startTime" + prometheusViewDefinition { + viewName = rawServiceView + metricScope = SERVICE + metricMap { + numCalls { + metricName: "num_calls", + metricType: "GAUGE" + }, + errorCount { + metricName: "error_count", + metricType: "GAUGE" + } + } + attributeMap { + "SERVICE.id": "service_id", + "SERVICE.name": "service_name", + "API.id": "api_id", + "API.name": "api_name", + "SERVICE.startTime": "start_time_millis", + "SERVICE.endTime": "end_time_millis" + } + } + } + } + { + name = raw-service-view-api-scope-prometheus-handler + type = prometheus + clientConfig = "" + requestHandlerInfo { + tenantAttributeName = tenant_id + startTimeAttributeName = "API.startTime" + prometheusViewDefinition { + viewName = rawServiceView + metricScope = API + metricMap { + numCalls { + metricName: "num_calls", + metricType: "GAUGE" + }, + errorCount { + metricName: "error_count", + metricType: "GAUGE" + } + } + attributeMap { + "SERVICE.id": "service_id", + "SERVICE.name": "service_name", + "API.id": "api_id", + "API.name": "api_name", + "API.startTime": "start_time_millis", + "API.endTime": "end_time_millis" + } + } + } + } ] -} +} \ No newline at end of file diff --git a/query-service-impl/src/test/resources/prometheus_request_handler.conf b/query-service-impl/src/test/resources/prometheus_request_handler.conf new file mode 100644 index 00000000..76a31dac --- /dev/null +++ b/query-service-impl/src/test/resources/prometheus_request_handler.conf @@ -0,0 +1,31 @@ +{ + name = raw-service-view-service-prometheus-handler + type = prometheus + clientConfig = "" + requestHandlerInfo { + tenantAttributeName = tenant_id + startTimeAttributeName = "SERVICE.startTime" + prometheusViewDefinition { + viewName = rawServiceView + metricScope = SERVICE + metricMap { + numCalls { + metricName: "num_calls", + metricType: "GAUGE" + }, + errorCount { + metricName: "error_count", + metricType: "GAUGE" + } + } + attributeMap { + "SERVICE.id": "service_id", + "SERVICE.name": "service_name", + "API.id": "api_id", + "API.name": "api_name", + "SERVICE.startTime": "start_time_millis", + "SERVICE.endTime": "end_time_millis", + } + } + } +} \ No newline at end of file