From 0209c779a13deae1cba177b236ad5c9f01c32858 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Mon, 10 Aug 2020 11:47:32 -0400 Subject: [PATCH 01/20] initial commit of logging support extension --- sdk_extensions/logging_support/build.gradle | 19 ++ .../io/opentelemetry/common/AnyValue.java | 291 ++++++++++++++++++ .../logging/DefaultLogRecord.java | 171 ++++++++++ .../logging/LoggingBatchExporter.java | 33 ++ .../logging/LoggingBatchStrategy.java | 44 +++ .../logging/SdkLogSinkProvider.java | 82 +++++ .../logging/SizeOrLatencyBatchStrategy.java | 136 ++++++++ .../opentelemetry/logging/api/Exporter.java | 27 ++ .../opentelemetry/logging/api/LogRecord.java | 111 +++++++ .../io/opentelemetry/logging/api/LogSink.java | 34 ++ .../logging/SdkLogSinkProviderTest.java | 63 ++++ .../SizeOrLatencyBatchStrategyTest.java | 70 +++++ settings.gradle | 1 + 13 files changed, 1082 insertions(+) create mode 100644 sdk_extensions/logging_support/build.gradle create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/Exporter.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java create mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogSink.java create mode 100644 sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java create mode 100644 sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java diff --git a/sdk_extensions/logging_support/build.gradle b/sdk_extensions/logging_support/build.gradle new file mode 100644 index 00000000000..b225a87e312 --- /dev/null +++ b/sdk_extensions/logging_support/build.gradle @@ -0,0 +1,19 @@ +plugins { + id "java" + id "maven-publish" + + id "ru.vyarus.animalsniffer" +} + +description = 'OpenTelemetry Contrib Logging Support' +ext.moduleName = "io.opentelemetry.extensions.logging.support" + +dependencies { + api project(':opentelemetry-sdk') + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.4' + + annotationProcessor libraries.auto_value + + signature "org.codehaus.mojo.signature:java17:1.0@signature" + signature "net.sf.androidscents.signature:android-api-level-24:7.0_r2@signature" +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java new file mode 100644 index 00000000000..893fd4ec2ad --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.common; + +import com.google.auto.value.AutoValue; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * A class that represents all the possible values for a data body. An {@code AnyValue} can have 6 + * types of values: {@code String}, {@code boolean}, {@code int}, {@code double}, {@code array}, or + * {@code kvlist}. represented through {@code AnyValue.Type}. A {@code array} or a {@code kvlist} + * can in turn hold other {@code AnyValue} instances, allowing for mapping to JSON-like structures. + * + * @since 0.7.0 + */ +@Immutable +public abstract class AnyValue { + + /** + * An enum that represents all the possible value types for an {@code AnyValue}. + * + * @since 0.7.0 + */ + public enum Type { + STRING, + BOOL, + INT, + DOUBLE, + ARRAY, + KVLIST + } + + /** + * Returns an {@code AnyValue} with a string value. + * + * @param stringValue The new value. + * @return an {@code AnyValue} with a string value. + * @since 0.7.0 + */ + public static AnyValue stringAnyValue(String stringValue) { + return AnyValueString.create(stringValue); + } + + /** + * Returns the string value of this {@code AnyValue}. An UnsupportedOperationException will be + * thrown if getType() is not {@link AnyValue.Type#STRING}. + * + * @return the string value of this {@code AttributeValue}. + * @since 0.7.0 + */ + public String getStringValue() { + throw new UnsupportedOperationException( + String.format("This type can only return %s data", getType().name())); + } + + /** + * Returns an {@code AnyValue} with an int value. + * + * @param intValue The new value. + * @return an {@code AnyValue} with a int value. + * @since 0.7.0 + */ + public static AnyValue intAnyValue(int intValue) { + return AnyValueInt.create(intValue); + } + + public int getIntValue() { + throw new UnsupportedOperationException( + String.format("This type can only return %s data", getType().name())); + } + + /** + * Returns an {@code AnyValue} with a bool value. + * + * @param boolValue The new value. + * @return an {@code AnyValue} with a bool value. + * @since 0.7.0 + */ + public static AnyValue boolAnyValue(boolean boolValue) { + return AnyValueBool.create(boolValue); + } + + /** + * Returns the boolean value of this {@code AnyValue}. An UnsupportedOperationException will be + * thrown if getType() is not {@link AnyValue.Type#BOOL}. + * + * @return the boolean value of this {@code AttributeValue}. + * @since 0.7.0 + */ + public boolean getBoolValue() { + throw new UnsupportedOperationException( + String.format("This type can only return %s data", getType().name())); + } + + /** + * Returns an {@code AnyValue} with a double value. + * + * @param doubleValue The new value. + * @return an {@code AnyValue} with a double value. + * @since 0.7.0 + */ + public static AnyValue doubleAnyValue(double doubleValue) { + return AnyValueDouble.create(doubleValue); + } + + /** + * Returns the double value of this {@code AnyValue}. An UnsupportedOperationException will be + * thrown if getType() is not {@link AnyValue.Type#DOUBLE}. + * + * @return the double value of this {@code AttributeValue}. + * @since 0.7.0 + */ + public double getDoubleValue() { + throw new UnsupportedOperationException( + String.format("This type can only return %s data", getType().name())); + } + + /** + * Returns an {@code AnyValue} with a array value. + * + * @param values The new value. + * @return an {@code AnyValue} with a array value. + * @since 0.7.0 + */ + public static AnyValue arrayAnyValue(List values) { + return AnyValueArray.create(values); + } + + /** + * Returns the array value of this {@code AnyValue}. An UnsupportedOperationException will be + * thrown if getType() is not {@link AnyValue.Type#ARRAY}. + * + * @return the array value of this {@code AttributeValue}. + * @since 0.7.0 + */ + public List getArrayValue() { + throw new UnsupportedOperationException( + String.format("This type can only return %s data", getType().name())); + } + + /** + * Returns an {@code AnyValue} with a kvlist value. + * + * @param values The new value. + * @return an {@code AnyValue} with a kvlist value. + * @since 0.7.0 + */ + public static AnyValue kvlistAnyValue(Map values) { + return AnyValueKvlist.create(values); + } + + /** + * Returns the string value of this {@code AnyValue}. An UnsupportedOperationException will be + * thrown if getType() is not {@link AnyValue.Type#STRING}. + * + * @return the string value of this {@code AttributeValue}. + * @since 0.7.0 + */ + public Map getKvlistValue() { + throw new UnsupportedOperationException( + String.format("This type can only return %s data", getType().name())); + } + + public abstract Type getType(); + + @Immutable + @AutoValue + abstract static class AnyValueString extends AnyValue { + AnyValueString() {} + + static AnyValue create(String stringValue) { + return new AutoValue_AnyValue_AnyValueString(stringValue); + } + + @Override + public final Type getType() { + return Type.STRING; + } + + @Override + @Nullable + public abstract String getStringValue(); + } + + @Immutable + @AutoValue + abstract static class AnyValueInt extends AnyValue { + AnyValueInt() {} + + static AnyValue create(int intValue) { + return new AutoValue_AnyValue_AnyValueInt(intValue); + } + + @Override + public final Type getType() { + return Type.INT; + } + + @Override + public abstract int getIntValue(); + } + + @Immutable + @AutoValue + abstract static class AnyValueBool extends AnyValue { + AnyValueBool() {} + + static AnyValue create(boolean boolValue) { + return new AutoValue_AnyValue_AnyValueBool(boolValue); + } + + @Override + public final Type getType() { + return Type.BOOL; + } + + @Override + public abstract boolean getBoolValue(); + } + + @Immutable + @AutoValue + abstract static class AnyValueDouble extends AnyValue { + AnyValueDouble() {} + + static AnyValue create(double doubleValue) { + return new AutoValue_AnyValue_AnyValueDouble(doubleValue); + } + + @Override + public final Type getType() { + return Type.DOUBLE; + } + + @Override + public abstract double getDoubleValue(); + } + + @Immutable + @AutoValue + abstract static class AnyValueArray extends AnyValue { + AnyValueArray() {} + + static AnyValue create(List arrayValue) { + return new AutoValue_AnyValue_AnyValueArray(arrayValue); + } + + @Override + public final Type getType() { + return Type.ARRAY; + } + + @Override + public abstract List getArrayValue(); + } + + @Immutable + @AutoValue + abstract static class AnyValueKvlist extends AnyValue { + AnyValueKvlist() {} + + static AnyValue create(Map kvlistValue) { + return new AutoValue_AnyValue_AnyValueKvlist(kvlistValue); + } + + @Override + public final Type getType() { + return Type.KVLIST; + } + + @Override + public abstract Map getKvlistValue(); + } +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java new file mode 100644 index 00000000000..b0289ffa18d --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java @@ -0,0 +1,171 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import io.opentelemetry.common.AnyValue; +import io.opentelemetry.common.AttributeValue; +import io.opentelemetry.logging.api.LogRecord; +import java.util.Map; + +public class DefaultLogRecord implements LogRecord { + private long timestamp; + private byte[] traceId; + private byte[] spanId; + private int flags; + private Severity severity; + private String severityText; + private String name; + private AnyValue body; + private Map attributes; + + private DefaultLogRecord() {} + + /** + * Clone method, used so that changes through the builder do not affect the record built. + * + * @param template LogRecord from which to copy + */ + private DefaultLogRecord(DefaultLogRecord template) { + timestamp = template.timestamp; + traceId = template.traceId; + spanId = template.spanId; + flags = template.flags; + severity = template.severity; + severityText = template.severityText; + name = template.name; + body = template.body; + attributes = template.attributes; + } + + @Override + public long getTimeUnixNano() { + return timestamp; + } + + @Override + public byte[] getTraceId() { + return traceId; + } + + @Override + public byte[] getSpanId() { + return spanId; + } + + @Override + public int getFlags() { + return flags; + } + + @Override + public Severity getSeverity() { + return severity; + } + + @Override + public String getSeverityText() { + return severityText; + } + + @Override + public String getName() { + return name; + } + + @Override + public AnyValue getBody() { + return body; + } + + @Override + public Map getAttributes() { + return attributes; + } + + public static class Builder implements LogRecord.Builder { + DefaultLogRecord template = new DefaultLogRecord(); + + @Override + public LogRecord.Builder withUnixTimeNano(long timestamp) { + template.timestamp = timestamp; + return this; + } + + @Override + public LogRecord.Builder withUnixTimeMillis(long timestamp) { + return withUnixTimeNano(timestamp * 1_000_000); + } + + @Override + public LogRecord.Builder withTraceId(byte[] traceId) { + template.traceId = traceId; + return this; + } + + @Override + public LogRecord.Builder withSpanId(byte[] spanId) { + template.spanId = spanId; + return this; + } + + @Override + public LogRecord.Builder withFlags(int flags) { + template.flags = flags; + return this; + } + + @Override + public LogRecord.Builder withSeverity(Severity severity) { + template.severity = severity; + return this; + } + + @Override + public LogRecord.Builder withSeverityText(String severityText) { + template.severityText = severityText; + return this; + } + + @Override + public LogRecord.Builder withName(String name) { + template.name = name; + return this; + } + + @Override + public LogRecord.Builder withBody(AnyValue body) { + template.body = body; + return this; + } + + @Override + public LogRecord.Builder withBody(String body) { + return withBody(AnyValue.stringAnyValue(body)); + } + + @Override + public LogRecord.Builder withAttributes(Map attributes) { + template.attributes = attributes; + return this; + } + + @Override + public LogRecord build() { + return new DefaultLogRecord(template); + } + } +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java new file mode 100644 index 00000000000..2c14fb1a56d --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import io.opentelemetry.logging.api.LogRecord; +import java.util.List; + +/** + * A LoggingBatchExporter accepts a batch of records and handles the transmission of those records + * to a remote system. + */ +public interface LoggingBatchExporter { + /** + * Accept a batch of records for transmission or export + * + * @param batch list of records ready for transport + */ + void handleLogRecordBatch(List batch); +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java new file mode 100644 index 00000000000..28f97dde037 --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import io.opentelemetry.logging.api.LogRecord; +import java.util.List; + +/** + * A LoggingBatchStrategy encodes the logic for how to batch and aggregate records for transmission. + * {@see SizeOrLatencyBatchStrategy} + */ +public interface LoggingBatchStrategy { + /** + * Add a LogRecord to the existing batch. + * + * @param record record to enqueue + */ + void add(LogRecord record); + + /** Explicitly flush the batch. */ + void flush(); + + /** + * This sets the handler for the batch. {@link LoggingBatchExporter#handleLogRecordBatch(List)} + * will be called whenever this strategy's constraints are fulfilled. + * + * @param handler Exporter to invoke when the batch is full + */ + void setBatchHandler(LoggingBatchExporter handler); +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java new file mode 100644 index 00000000000..52902259788 --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import io.opentelemetry.logging.api.Exporter; +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.LogSink; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + +public class SdkLogSinkProvider implements LoggingBatchExporter { + private final LoggingBatchStrategy batchManager; + private final List exporters; + + private SdkLogSinkProvider(LoggingBatchStrategy batchManager, List exporters) { + this.batchManager = batchManager != null ? batchManager : getDefaultBatchManager(); + this.batchManager.setBatchHandler(this); + this.exporters = exporters; + } + + private static LoggingBatchStrategy getDefaultBatchManager() { + return new SizeOrLatencyBatchStrategy.Builder().build(); + } + + public LogSink get(String instrumentationName, String instrumentationVersion) { + // FIXME: caching + return new SdkLogSink(); + } + + @Override + public void handleLogRecordBatch(List batch) { + for (Exporter e : exporters) { + e.accept(batch); + } + } + + private class SdkLogSink implements LogSink { + @Override + public void offer(LogRecord record) { + batchManager.add(record); + } + + @Override + public LogRecord.Builder buildRecord() { + return new DefaultLogRecord.Builder(); + } + } + + public static class Builder { + @Nullable private LoggingBatchStrategy batchManager = null; + private final List exporters = new ArrayList<>(); + + public Builder withBatchManager(LoggingBatchStrategy manager) { + batchManager = manager; + return this; + } + + public Builder withExporter(Exporter exporter) { + exporters.add(exporter); + return this; + } + + public SdkLogSinkProvider build() { + return new SdkLogSinkProvider(batchManager, exporters); + } + } +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java new file mode 100644 index 00000000000..070bbf46034 --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java @@ -0,0 +1,136 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import io.opentelemetry.logging.api.LogRecord; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + +/** + * Batching strategy that allows specifying both a maximum size and a maximum amount of time before + * sending a batch of records. Batch will be flushed whenever the list of records queued exceeds the + * maximum threshold (default 50), or a set timeout expires (default 5 seconds). + */ +public class SizeOrLatencyBatchStrategy implements LoggingBatchStrategy { + private static final int DEFAULT_BATCH_SIZE = 50; + private static final int DEFAULT_MAX_DELAY = 5; + private static final TimeUnit DEFAULT_MAX_DELAY_UNITS = TimeUnit.SECONDS; + private static final Timer timer = new Timer(); + + private final int maxBatch; + private final int maxDelay; + private final TimeUnit maxDelayUnits; + + private WeakReference batchHandler; + private List batch = new ArrayList<>(); + private boolean timerPending = false; + + private SizeOrLatencyBatchStrategy(int maxBatch, int maxDelay, TimeUnit units) { + this.maxBatch = maxBatch > 0 ? maxBatch : DEFAULT_BATCH_SIZE; + this.maxDelay = maxDelay > 0 ? maxDelay : DEFAULT_MAX_DELAY; + this.maxDelayUnits = units != null ? units : DEFAULT_MAX_DELAY_UNITS; + } + + @Override + public void add(LogRecord record) { + synchronized (this) { + if (!timerPending) { + setTimer(); + } + batch.add(record); + if (batch.size() >= maxBatch) { + flush(); + } + } + } + + private void setTimer() { + timerPending = true; + timer.schedule( + new TimerTask() { + @Override + public void run() { + flush(); + } + }, + TimeUnit.MILLISECONDS.convert(maxDelay, maxDelayUnits)); + } + + @Override + public void flush() { + LoggingBatchExporter handler = batchHandler.get(); + if (handler != null && batch.size() > 0) { + synchronized (this) { + timer.purge(); + timerPending = false; + List completedBatch = batch; + batch = new ArrayList<>(); + handler.handleLogRecordBatch(completedBatch); + } + } + } + + @Override + public void setBatchHandler(LoggingBatchExporter handler) { + batchHandler = new WeakReference<>(handler); + } + + public static class Builder { + private int maxBatchSize = -1; + private int maxDelay = -1; + @Nullable private TimeUnit maxDelayTimeUnit = null; + + /** + * The maximum number of LogRecord objects to accumulate before dispatching a batch. + * + * @param maxBatchSize max number of records + * @return builder + */ + public Builder withMaxBatchSize(int maxBatchSize) { + this.maxBatchSize = maxBatchSize; + return this; + } + + /** + * The length of time to wait before dispatching a batch, even fi the number of records + * accumulated is less than the max number allowed. + * + * @param delay the maximum amount of time to wait before dispatching a batch + * @param units the units the amount of time should be measured in + * @return builder + */ + public Builder withMaxDelay(int delay, TimeUnit units) { + this.maxDelay = delay; + this.maxDelayTimeUnit = units; + return this; + } + + /** + * Instantiate the strategy + * + * @return built strategy + */ + public SizeOrLatencyBatchStrategy build() { + return new SizeOrLatencyBatchStrategy(maxBatchSize, maxDelay, maxDelayTimeUnit); + } + } +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/Exporter.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/Exporter.java new file mode 100644 index 00000000000..ca66f193cb1 --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/Exporter.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging.api; + +import java.util.List; + +/** + * An exporter is responsible for taking a list of {@link LogRecord}s and transmitting them to their + * ultimate destination. + */ +public interface Exporter { + void accept(List records); +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java new file mode 100644 index 00000000000..4ddb8aad39d --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java @@ -0,0 +1,111 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging.api; + +import io.opentelemetry.common.AnyValue; +import io.opentelemetry.common.AttributeValue; +import java.util.Map; + +/** + * A LogRecord is an implementation of the + * OpenTelemetry logging model + */ +public interface LogRecord { + long getTimeUnixNano(); + + byte[] getTraceId(); + + byte[] getSpanId(); + + int getFlags(); + + Severity getSeverity(); + + String getSeverityText(); + + String getName(); + + AnyValue getBody(); + + Map getAttributes(); + + enum Severity { + UNDEFINED_SEVERITY_NUMBER(0), + TRACE(1), + TRACE2(2), + TRACE3(3), + TRACE4(4), + DEBUG(5), + DEBUG2(6), + DEBUG3(7), + DEBUG4(8), + INFO(9), + INFO2(10), + INFO3(11), + INFO4(12), + WARN(13), + WARN2(14), + WARN3(15), + WARN4(16), + ERROR(17), + ERROR2(18), + ERROR3(19), + ERROR4(20), + FATAL(21), + FATAL2(22), + FATAL3(23), + FATAL4(24), + ; + + private final int severityNumber; + + Severity(int severityNumber) { + this.severityNumber = severityNumber; + } + + public int getSeverityNumber() { + return severityNumber; + } + } + + interface Builder { + Builder withUnixTimeNano(long timestamp); + + Builder withUnixTimeMillis(long currentTimeMillis); + + Builder withTraceId(byte[] traceId); + + Builder withSpanId(byte[] spanId); + + Builder withFlags(int flags); + + Builder withSeverity(Severity severity); + + Builder withSeverityText(String severityText); + + Builder withName(String name); + + Builder withBody(AnyValue body); + + Builder withBody(String body); + + Builder withAttributes(Map attributes); + + LogRecord build(); + } +} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogSink.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogSink.java new file mode 100644 index 00000000000..6592baa7d02 --- /dev/null +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogSink.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging.api; + +/** A LogSink accepts logging records for transmission to an aggregator or log processing system. */ +public interface LogSink { + /** + * Pass a record to the SDK for transmission to a logging exporter. + * + * @param record record to transmit + */ + void offer(LogRecord record); + + /** + * Convenience method for creating a record to be transmitted. + * + * @return builder instance + */ + LogRecord.Builder buildRecord(); +} diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java new file mode 100644 index 00000000000..de200d0a869 --- /dev/null +++ b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java @@ -0,0 +1,63 @@ +package io.opentelemetry.logging; + +import static com.google.common.truth.Truth.assertThat; + +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.LogRecord.Severity; +import io.opentelemetry.logging.api.LogSink; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@RunWith(JUnit4.class) +public class SdkLogSinkProviderTest { + @Test + public void testProviderAggregation() throws InterruptedException { + List records1 = new ArrayList<>(); + List records2 = new ArrayList<>(); + SdkLogSinkProvider provider = + new SdkLogSinkProvider.Builder() + .withBatchManager( + new SizeOrLatencyBatchStrategy.Builder() + .withMaxBatchSize(5) + .withMaxDelay(50, TimeUnit.MILLISECONDS) + .build()) + .withExporter(records1::addAll) + .withExporter(records2::addAll) + .build(); + LogSink sink = provider.get("test", "0.8.0"); + for (int i = 0; i < 11; i++) { + sink.offer( + sink.buildRecord() + .withUnixTimeMillis(System.currentTimeMillis()) + .withSeverity(Severity.DEBUG) + .withBody("test") + .build()); + } + assertThat(records1.size()).isEqualTo(10); + assertThat(records2.size()).isEqualTo(10); + Thread.sleep(55); + assertThat(records1.size()).isEqualTo(11); + assertThat(records2.size()).isEqualTo(11); + assertThat(records1).isEqualTo(records2); + } +} diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java new file mode 100644 index 00000000000..c564aa0ae7a --- /dev/null +++ b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -0,0 +1,70 @@ +package io.opentelemetry.logging; + +import static com.google.common.truth.Truth.assertThat; + +import io.opentelemetry.logging.api.LogRecord; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@RunWith(JUnit4.class) +public class SizeOrLatencyBatchStrategyTest { + + @Test + public void testSizeStrategy() { + SizeOrLatencyBatchStrategy strategy = + new SizeOrLatencyBatchStrategy.Builder().withMaxBatchSize(5).build(); + final List transmittedBatch = new ArrayList<>(); + + strategy.setBatchHandler(transmittedBatch::addAll); + + for (int i = 0; i < 7; i++) { + strategy.add(null); + } + + assertThat(transmittedBatch.size()).isEqualTo(5); + + for (int i = 0; i < 3; i++) { + strategy.add(null); + } + + assertThat(transmittedBatch.size()).isEqualTo(10); + } + + @Test + public void testLatencyStrategy() throws InterruptedException { + SizeOrLatencyBatchStrategy strategy = + new SizeOrLatencyBatchStrategy.Builder().withMaxDelay(50, TimeUnit.MILLISECONDS).build(); + + final List transmittedBatch = new ArrayList<>(); + + strategy.setBatchHandler(transmittedBatch::addAll); + + for (int i = 0; i < 7; i++) { + strategy.add(null); + Thread.sleep(10); + } + + assertThat(transmittedBatch.size()).isEqualTo(5); + Thread.sleep(55); + assertThat(transmittedBatch.size()).isEqualTo(7); + } +} diff --git a/settings.gradle b/settings.gradle index 74aaab52fb2..b0fa50fb6e2 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,6 +25,7 @@ include ":opentelemetry-all", ":opentelemetry-extension-runtime-metrics", ":opentelemetry-extension-trace-propagators", ":opentelemetry-extension-trace-utils", + ":opentelemetry-extension-logging-support", ":opentelemetry-exporters-inmemory", ":opentelemetry-exporters-jaeger", ":opentelemetry-exporters-logging", From 5d12042a2d829ef52f6afa6f0a75b5893c9b46e4 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Mon, 10 Aug 2020 16:36:37 -0400 Subject: [PATCH 02/20] complete move from extensions to sdk_extensions --- .../logging/LoggingBatchExporter.java | 2 +- .../logging/LoggingBatchStrategy.java | 2 +- .../logging/SizeOrLatencyBatchStrategy.java | 2 +- .../opentelemetry/logging/api/LogRecord.java | 2 +- .../logging/SdkLogSinkProviderTest.java | 28 +++++++++---------- .../SizeOrLatencyBatchStrategyTest.java | 25 +++++++++-------- settings.gradle | 3 +- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java index 2c14fb1a56d..9f45cfec517 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java @@ -25,7 +25,7 @@ */ public interface LoggingBatchExporter { /** - * Accept a batch of records for transmission or export + * Accept a batch of records for transmission or export. * * @param batch list of records ready for transport */ diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java index 28f97dde037..f57d565f390 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java @@ -21,7 +21,7 @@ /** * A LoggingBatchStrategy encodes the logic for how to batch and aggregate records for transmission. - * {@see SizeOrLatencyBatchStrategy} + * {@link SizeOrLatencyBatchStrategy} */ public interface LoggingBatchStrategy { /** diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java index 070bbf46034..159f2d19ac2 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java @@ -125,7 +125,7 @@ public Builder withMaxDelay(int delay, TimeUnit units) { } /** - * Instantiate the strategy + * Instantiate the strategy. * * @return built strategy */ diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java index 4ddb8aad39d..4eb718f7e1c 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java @@ -23,7 +23,7 @@ /** * A LogRecord is an implementation of the - * OpenTelemetry logging model + * OpenTelemetry logging model. */ public interface LogRecord { long getTimeUnixNano(); diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java index de200d0a869..9bd16d32d07 100644 --- a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java +++ b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java @@ -1,17 +1,3 @@ -package io.opentelemetry.logging; - -import static com.google.common.truth.Truth.assertThat; - -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.LogRecord.Severity; -import io.opentelemetry.logging.api.LogSink; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - /* * Copyright 2020, OpenTelemetry Authors * @@ -28,6 +14,20 @@ * limitations under the License. */ +package io.opentelemetry.logging; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.LogRecord.Severity; +import io.opentelemetry.logging.api.LogSink; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + @RunWith(JUnit4.class) public class SdkLogSinkProviderTest { @Test diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java index c564aa0ae7a..45061a7ad32 100644 --- a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -1,15 +1,3 @@ -package io.opentelemetry.logging; - -import static com.google.common.truth.Truth.assertThat; - -import io.opentelemetry.logging.api.LogRecord; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - /* * Copyright 2020, OpenTelemetry Authors * @@ -25,6 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +package io.opentelemetry.logging; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.opentelemetry.logging.api.LogRecord; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + @RunWith(JUnit4.class) public class SizeOrLatencyBatchStrategyTest { diff --git a/settings.gradle b/settings.gradle index b0fa50fb6e2..8b8bd9c3f8d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -25,7 +25,6 @@ include ":opentelemetry-all", ":opentelemetry-extension-runtime-metrics", ":opentelemetry-extension-trace-propagators", ":opentelemetry-extension-trace-utils", - ":opentelemetry-extension-logging-support", ":opentelemetry-exporters-inmemory", ":opentelemetry-exporters-jaeger", ":opentelemetry-exporters-logging", @@ -39,6 +38,7 @@ include ":opentelemetry-all", ":opentelemetry-sdk-extension-async-processor", ":opentelemetry-sdk-extension-auto-config", ":opentelemetry-sdk-extension-aws-v1-support", + ":opentelemetry-sdk-extension-logging-support", ":opentelemetry-sdk-extension-otproto", ":opentelemetry-sdk-extension-testbed", ":opentelemetry-sdk-extension-jaeger-remote-sampler", @@ -51,7 +51,6 @@ rootProject.children.each { .replace("opentelemetry-exporters-", "exporters/") .replace("opentelemetry-extension-", "extensions/") .replace("opentelemetry-sdk-extension-", "sdk_extensions/") - .replace("logging-", "logging/") .replace("opentelemetry-", "") .replace("-", "_") as File } From 3200d1735c67c759a6ef68cfac45ebae3271086e Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Mon, 24 Aug 2020 14:35:36 -0700 Subject: [PATCH 03/20] address code review comments --- sdk_extensions/logging_support/build.gradle | 1 + .../io/opentelemetry/common/AnyValue.java | 19 +- .../logging/DefaultLogRecord.java | 171 ------------------ .../logging/LoggingBatchStrategy.java | 2 + .../logging/SdkLogSinkProvider.java | 7 +- .../logging/SizeOrLatencyBatchStrategy.java | 66 +++---- .../opentelemetry/logging/api/LogRecord.java | 110 ++++++++--- .../logging/SdkLogSinkProviderTest.java | 36 ++-- .../SizeOrLatencyBatchStrategyTest.java | 10 +- 9 files changed, 156 insertions(+), 266 deletions(-) delete mode 100644 sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java diff --git a/sdk_extensions/logging_support/build.gradle b/sdk_extensions/logging_support/build.gradle index b225a87e312..89050f91f0c 100644 --- a/sdk_extensions/logging_support/build.gradle +++ b/sdk_extensions/logging_support/build.gradle @@ -11,6 +11,7 @@ ext.moduleName = "io.opentelemetry.extensions.logging.support" dependencies { api project(':opentelemetry-sdk') implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.4' + testImplementation libraries.awaitility annotationProcessor libraries.auto_value diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java index 893fd4ec2ad..4e0faf76e0f 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java @@ -28,16 +28,12 @@ * {@code kvlist}. represented through {@code AnyValue.Type}. A {@code array} or a {@code kvlist} * can in turn hold other {@code AnyValue} instances, allowing for mapping to JSON-like structures. * - * @since 0.7.0 + * @since 0.8.0 */ @Immutable public abstract class AnyValue { - /** - * An enum that represents all the possible value types for an {@code AnyValue}. - * - * @since 0.7.0 - */ + /** An enum that represents all the possible value types for an {@code AnyValue}. */ public enum Type { STRING, BOOL, @@ -52,7 +48,6 @@ public enum Type { * * @param stringValue The new value. * @return an {@code AnyValue} with a string value. - * @since 0.7.0 */ public static AnyValue stringAnyValue(String stringValue) { return AnyValueString.create(stringValue); @@ -63,7 +58,6 @@ public static AnyValue stringAnyValue(String stringValue) { * thrown if getType() is not {@link AnyValue.Type#STRING}. * * @return the string value of this {@code AttributeValue}. - * @since 0.7.0 */ public String getStringValue() { throw new UnsupportedOperationException( @@ -75,7 +69,6 @@ public String getStringValue() { * * @param intValue The new value. * @return an {@code AnyValue} with a int value. - * @since 0.7.0 */ public static AnyValue intAnyValue(int intValue) { return AnyValueInt.create(intValue); @@ -91,7 +84,6 @@ public int getIntValue() { * * @param boolValue The new value. * @return an {@code AnyValue} with a bool value. - * @since 0.7.0 */ public static AnyValue boolAnyValue(boolean boolValue) { return AnyValueBool.create(boolValue); @@ -102,7 +94,6 @@ public static AnyValue boolAnyValue(boolean boolValue) { * thrown if getType() is not {@link AnyValue.Type#BOOL}. * * @return the boolean value of this {@code AttributeValue}. - * @since 0.7.0 */ public boolean getBoolValue() { throw new UnsupportedOperationException( @@ -114,7 +105,6 @@ public boolean getBoolValue() { * * @param doubleValue The new value. * @return an {@code AnyValue} with a double value. - * @since 0.7.0 */ public static AnyValue doubleAnyValue(double doubleValue) { return AnyValueDouble.create(doubleValue); @@ -125,7 +115,6 @@ public static AnyValue doubleAnyValue(double doubleValue) { * thrown if getType() is not {@link AnyValue.Type#DOUBLE}. * * @return the double value of this {@code AttributeValue}. - * @since 0.7.0 */ public double getDoubleValue() { throw new UnsupportedOperationException( @@ -137,7 +126,6 @@ public double getDoubleValue() { * * @param values The new value. * @return an {@code AnyValue} with a array value. - * @since 0.7.0 */ public static AnyValue arrayAnyValue(List values) { return AnyValueArray.create(values); @@ -148,7 +136,6 @@ public static AnyValue arrayAnyValue(List values) { * thrown if getType() is not {@link AnyValue.Type#ARRAY}. * * @return the array value of this {@code AttributeValue}. - * @since 0.7.0 */ public List getArrayValue() { throw new UnsupportedOperationException( @@ -160,7 +147,6 @@ public List getArrayValue() { * * @param values The new value. * @return an {@code AnyValue} with a kvlist value. - * @since 0.7.0 */ public static AnyValue kvlistAnyValue(Map values) { return AnyValueKvlist.create(values); @@ -171,7 +157,6 @@ public static AnyValue kvlistAnyValue(Map values) { * thrown if getType() is not {@link AnyValue.Type#STRING}. * * @return the string value of this {@code AttributeValue}. - * @since 0.7.0 */ public Map getKvlistValue() { throw new UnsupportedOperationException( diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java deleted file mode 100644 index b0289ffa18d..00000000000 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/DefaultLogRecord.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright 2020, OpenTelemetry Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.opentelemetry.logging; - -import io.opentelemetry.common.AnyValue; -import io.opentelemetry.common.AttributeValue; -import io.opentelemetry.logging.api.LogRecord; -import java.util.Map; - -public class DefaultLogRecord implements LogRecord { - private long timestamp; - private byte[] traceId; - private byte[] spanId; - private int flags; - private Severity severity; - private String severityText; - private String name; - private AnyValue body; - private Map attributes; - - private DefaultLogRecord() {} - - /** - * Clone method, used so that changes through the builder do not affect the record built. - * - * @param template LogRecord from which to copy - */ - private DefaultLogRecord(DefaultLogRecord template) { - timestamp = template.timestamp; - traceId = template.traceId; - spanId = template.spanId; - flags = template.flags; - severity = template.severity; - severityText = template.severityText; - name = template.name; - body = template.body; - attributes = template.attributes; - } - - @Override - public long getTimeUnixNano() { - return timestamp; - } - - @Override - public byte[] getTraceId() { - return traceId; - } - - @Override - public byte[] getSpanId() { - return spanId; - } - - @Override - public int getFlags() { - return flags; - } - - @Override - public Severity getSeverity() { - return severity; - } - - @Override - public String getSeverityText() { - return severityText; - } - - @Override - public String getName() { - return name; - } - - @Override - public AnyValue getBody() { - return body; - } - - @Override - public Map getAttributes() { - return attributes; - } - - public static class Builder implements LogRecord.Builder { - DefaultLogRecord template = new DefaultLogRecord(); - - @Override - public LogRecord.Builder withUnixTimeNano(long timestamp) { - template.timestamp = timestamp; - return this; - } - - @Override - public LogRecord.Builder withUnixTimeMillis(long timestamp) { - return withUnixTimeNano(timestamp * 1_000_000); - } - - @Override - public LogRecord.Builder withTraceId(byte[] traceId) { - template.traceId = traceId; - return this; - } - - @Override - public LogRecord.Builder withSpanId(byte[] spanId) { - template.spanId = spanId; - return this; - } - - @Override - public LogRecord.Builder withFlags(int flags) { - template.flags = flags; - return this; - } - - @Override - public LogRecord.Builder withSeverity(Severity severity) { - template.severity = severity; - return this; - } - - @Override - public LogRecord.Builder withSeverityText(String severityText) { - template.severityText = severityText; - return this; - } - - @Override - public LogRecord.Builder withName(String name) { - template.name = name; - return this; - } - - @Override - public LogRecord.Builder withBody(AnyValue body) { - template.body = body; - return this; - } - - @Override - public LogRecord.Builder withBody(String body) { - return withBody(AnyValue.stringAnyValue(body)); - } - - @Override - public LogRecord.Builder withAttributes(Map attributes) { - template.attributes = attributes; - return this; - } - - @Override - public LogRecord build() { - return new DefaultLogRecord(template); - } - } -} diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java index f57d565f390..784794d27a3 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java @@ -24,6 +24,8 @@ * {@link SizeOrLatencyBatchStrategy} */ public interface LoggingBatchStrategy { + void stop(); + /** * Add a LogRecord to the existing batch. * diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java index 52902259788..b07b8779914 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java @@ -26,6 +26,7 @@ public class SdkLogSinkProvider implements LoggingBatchExporter { private final LoggingBatchStrategy batchManager; private final List exporters; + private final LogSink logSink = new SdkLogSink(); private SdkLogSinkProvider(LoggingBatchStrategy batchManager, List exporters) { this.batchManager = batchManager != null ? batchManager : getDefaultBatchManager(); @@ -38,8 +39,8 @@ private static LoggingBatchStrategy getDefaultBatchManager() { } public LogSink get(String instrumentationName, String instrumentationVersion) { - // FIXME: caching - return new SdkLogSink(); + // Currently there is no differentiation by instrumentation library + return logSink; } @Override @@ -57,7 +58,7 @@ public void offer(LogRecord record) { @Override public LogRecord.Builder buildRecord() { - return new DefaultLogRecord.Builder(); + return new LogRecord.Builder(); } } diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java index 159f2d19ac2..cd9c823e5c8 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java @@ -17,11 +17,11 @@ package io.opentelemetry.logging; import io.opentelemetry.logging.api.LogRecord; -import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.List; -import java.util.Timer; -import java.util.TimerTask; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -34,64 +34,58 @@ public class SizeOrLatencyBatchStrategy implements LoggingBatchStrategy { private static final int DEFAULT_BATCH_SIZE = 50; private static final int DEFAULT_MAX_DELAY = 5; private static final TimeUnit DEFAULT_MAX_DELAY_UNITS = TimeUnit.SECONDS; - private static final Timer timer = new Timer(); + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); private final int maxBatch; private final int maxDelay; private final TimeUnit maxDelayUnits; + private final ScheduledFuture schedule; - private WeakReference batchHandler; + private LoggingBatchExporter batchHandler; private List batch = new ArrayList<>(); - private boolean timerPending = false; private SizeOrLatencyBatchStrategy(int maxBatch, int maxDelay, TimeUnit units) { this.maxBatch = maxBatch > 0 ? maxBatch : DEFAULT_BATCH_SIZE; this.maxDelay = maxDelay > 0 ? maxDelay : DEFAULT_MAX_DELAY; this.maxDelayUnits = units != null ? units : DEFAULT_MAX_DELAY_UNITS; + this.schedule = + executor.scheduleWithFixedDelay( + new Runnable() { + @Override + public void run() { + flush(); + } + }, + this.maxDelay, + this.maxDelay, + this.maxDelayUnits); } @Override - public void add(LogRecord record) { - synchronized (this) { - if (!timerPending) { - setTimer(); - } - batch.add(record); - if (batch.size() >= maxBatch) { - flush(); - } - } + public void stop() { + schedule.cancel(false); } - private void setTimer() { - timerPending = true; - timer.schedule( - new TimerTask() { - @Override - public void run() { - flush(); - } - }, - TimeUnit.MILLISECONDS.convert(maxDelay, maxDelayUnits)); + @Override + public void add(LogRecord record) { + batch.add(record); + if (batch.size() >= maxBatch) { + flush(); + } } @Override public void flush() { - LoggingBatchExporter handler = batchHandler.get(); - if (handler != null && batch.size() > 0) { - synchronized (this) { - timer.purge(); - timerPending = false; - List completedBatch = batch; - batch = new ArrayList<>(); - handler.handleLogRecordBatch(completedBatch); - } + if (batchHandler != null && batch.size() > 0) { + List completedBatch = batch; + batch = new ArrayList<>(); + batchHandler.handleLogRecordBatch(completedBatch); } } @Override public void setBatchHandler(LoggingBatchExporter handler) { - batchHandler = new WeakReference<>(handler); + batchHandler = handler; } public static class Builder { diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java index 4eb718f7e1c..78da93a8639 100644 --- a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java +++ b/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java @@ -16,35 +16,44 @@ package io.opentelemetry.logging.api; +import com.google.auto.value.AutoValue; import io.opentelemetry.common.AnyValue; import io.opentelemetry.common.AttributeValue; +import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; /** * A LogRecord is an implementation of the * OpenTelemetry logging model. */ -public interface LogRecord { - long getTimeUnixNano(); +@AutoValue +public abstract class LogRecord { + abstract long getTimeUnixNano(); - byte[] getTraceId(); + @SuppressWarnings("mutable") + abstract byte[] getTraceId(); - byte[] getSpanId(); + @SuppressWarnings("mutable") + abstract byte[] getSpanId(); - int getFlags(); + abstract int getFlags(); - Severity getSeverity(); + abstract Severity getSeverity(); - String getSeverityText(); + @Nullable + abstract String getSeverityText(); - String getName(); + @Nullable + abstract String getName(); - AnyValue getBody(); + abstract AnyValue getBody(); - Map getAttributes(); + abstract Map getAttributes(); - enum Severity { + public enum Severity { UNDEFINED_SEVERITY_NUMBER(0), TRACE(1), TRACE2(2), @@ -83,29 +92,82 @@ public int getSeverityNumber() { } } - interface Builder { - Builder withUnixTimeNano(long timestamp); + public static class Builder { + private long timeUnixNano; + private byte[] traceId = new byte[0]; + private byte[] spanId = new byte[0]; + private int flags; + private Severity severity = Severity.UNDEFINED_SEVERITY_NUMBER; + private String severityText; + private String name; + private AnyValue body = AnyValue.stringAnyValue(""); + private final Map attributes = new HashMap<>(); + + public Builder withUnixTimeNano(long timestamp) { + this.timeUnixNano = timestamp; + return this; + } - Builder withUnixTimeMillis(long currentTimeMillis); + public Builder withUnixTimeMillis(long timestamp) { + return withUnixTimeNano(TimeUnit.MILLISECONDS.toNanos(timestamp)); + } - Builder withTraceId(byte[] traceId); + public Builder withTraceId(byte[] traceId) { + this.traceId = traceId; + return this; + } - Builder withSpanId(byte[] spanId); + public Builder withSpanId(byte[] spanId) { + this.spanId = spanId; + return this; + } - Builder withFlags(int flags); + public Builder withFlags(int flags) { + this.flags = flags; + return this; + } - Builder withSeverity(Severity severity); + public Builder withSeverity(Severity severity) { + this.severity = severity; + return this; + } - Builder withSeverityText(String severityText); + public Builder withSeverityText(String severityText) { + this.severityText = severityText; + return this; + } - Builder withName(String name); + public Builder withName(String name) { + this.name = name; + return this; + } - Builder withBody(AnyValue body); + public Builder withBody(AnyValue body) { + this.body = body; + return this; + } - Builder withBody(String body); + public Builder withBody(String body) { + this.body = AnyValue.stringAnyValue(body); + return this; + } - Builder withAttributes(Map attributes); + public Builder withAttributes(Map attributes) { + this.attributes.putAll(attributes); + return this; + } - LogRecord build(); + /** + * Build a LogRecord instance. + * + * @return value object being built + */ + public LogRecord build() { + if (timeUnixNano == 0) { + timeUnixNano = TimeUnit.MILLISECONDS.toNanos(System.currentTimeMillis()); + } + return new AutoValue_LogRecord( + timeUnixNano, traceId, spanId, flags, severity, severityText, name, body, attributes); + } } } diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java index 9bd16d32d07..8744cdcc9e8 100644 --- a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java +++ b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java @@ -17,7 +17,9 @@ package io.opentelemetry.logging; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import io.opentelemetry.logging.api.Exporter; import io.opentelemetry.logging.api.LogRecord; import io.opentelemetry.logging.api.LogRecord.Severity; import io.opentelemetry.logging.api.LogSink; @@ -30,19 +32,30 @@ @RunWith(JUnit4.class) public class SdkLogSinkProviderTest { + private static class TestExporter implements Exporter { + private final List records = new ArrayList<>(); + private int batches = 0; + + @Override + public void accept(List records) { + this.records.addAll(records); + batches++; + } + } + @Test - public void testProviderAggregation() throws InterruptedException { - List records1 = new ArrayList<>(); - List records2 = new ArrayList<>(); + public void testProviderAggregation() { + TestExporter exporter1 = new TestExporter(); + TestExporter exporter2 = new TestExporter(); SdkLogSinkProvider provider = new SdkLogSinkProvider.Builder() .withBatchManager( new SizeOrLatencyBatchStrategy.Builder() .withMaxBatchSize(5) - .withMaxDelay(50, TimeUnit.MILLISECONDS) + .withMaxDelay(250, TimeUnit.MILLISECONDS) .build()) - .withExporter(records1::addAll) - .withExporter(records2::addAll) + .withExporter(exporter1) + .withExporter(exporter2) .build(); LogSink sink = provider.get("test", "0.8.0"); for (int i = 0; i < 11; i++) { @@ -53,11 +66,10 @@ public void testProviderAggregation() throws InterruptedException { .withBody("test") .build()); } - assertThat(records1.size()).isEqualTo(10); - assertThat(records2.size()).isEqualTo(10); - Thread.sleep(55); - assertThat(records1.size()).isEqualTo(11); - assertThat(records2.size()).isEqualTo(11); - assertThat(records1).isEqualTo(records2); + await() + .atLeast(100, TimeUnit.MILLISECONDS) + .atMost(500, TimeUnit.MILLISECONDS) + .until(() -> exporter1.records.size() == 11); + assertThat(exporter1.batches).isEqualTo(3); } } diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java index 45061a7ad32..7c8193fab7d 100644 --- a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ b/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -17,6 +17,7 @@ package io.opentelemetry.logging; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import io.opentelemetry.logging.api.LogRecord; import java.util.ArrayList; @@ -52,8 +53,11 @@ public void testSizeStrategy() { @Test public void testLatencyStrategy() throws InterruptedException { + int maxDelay = 50; SizeOrLatencyBatchStrategy strategy = - new SizeOrLatencyBatchStrategy.Builder().withMaxDelay(50, TimeUnit.MILLISECONDS).build(); + new SizeOrLatencyBatchStrategy.Builder() + .withMaxDelay(maxDelay, TimeUnit.MILLISECONDS) + .build(); final List transmittedBatch = new ArrayList<>(); @@ -65,7 +69,7 @@ public void testLatencyStrategy() throws InterruptedException { } assertThat(transmittedBatch.size()).isEqualTo(5); - Thread.sleep(55); - assertThat(transmittedBatch.size()).isEqualTo(7); + + await().atMost(200, TimeUnit.MILLISECONDS).until(() -> transmittedBatch.size() == 7); } } From 52433b06eb1f6be0c33cf8b281f4107943a280b7 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Mon, 24 Aug 2020 15:01:06 -0700 Subject: [PATCH 04/20] move logging_support to logging/support --- sdk_extensions/{logging_support => logging/support}/build.gradle | 0 .../support}/src/main/java/io/opentelemetry/common/AnyValue.java | 0 .../main/java/io/opentelemetry/logging/LoggingBatchExporter.java | 0 .../main/java/io/opentelemetry/logging/LoggingBatchStrategy.java | 0 .../main/java/io/opentelemetry/logging/SdkLogSinkProvider.java | 0 .../java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java | 0 .../src/main/java/io/opentelemetry/logging/api/Exporter.java | 0 .../src/main/java/io/opentelemetry/logging/api/LogRecord.java | 0 .../src/main/java/io/opentelemetry/logging/api/LogSink.java | 0 .../java/io/opentelemetry/logging/SdkLogSinkProviderTest.java | 0 .../io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename sdk_extensions/{logging_support => logging/support}/build.gradle (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/common/AnyValue.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/api/Exporter.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/api/LogRecord.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/main/java/io/opentelemetry/logging/api/LogSink.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java (100%) rename sdk_extensions/{logging_support => logging/support}/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java (100%) diff --git a/sdk_extensions/logging_support/build.gradle b/sdk_extensions/logging/support/build.gradle similarity index 100% rename from sdk_extensions/logging_support/build.gradle rename to sdk_extensions/logging/support/build.gradle diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/common/AnyValue.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/Exporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/Exporter.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogRecord.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java diff --git a/sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogSink.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogSink.java similarity index 100% rename from sdk_extensions/logging_support/src/main/java/io/opentelemetry/logging/api/LogSink.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogSink.java diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java similarity index 100% rename from sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java rename to sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java diff --git a/sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java similarity index 100% rename from sdk_extensions/logging_support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java rename to sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java From 3e6be4d085d3b251f22b5d834de49b3b4d027c45 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Mon, 24 Aug 2020 15:57:48 -0700 Subject: [PATCH 05/20] missed test update in last commit --- .../SizeOrLatencyBatchStrategyTest.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java index 7c8193fab7d..dfba7a4f1fe 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -29,11 +29,24 @@ @RunWith(JUnit4.class) public class SizeOrLatencyBatchStrategyTest { + private static class TestBatchHandler implements LoggingBatchExporter { + private final List records = new ArrayList<>(); + private int callCount = 0; + + @Override + public void handleLogRecordBatch(List batch) { + records.addAll(batch); + callCount++; + } + } @Test public void testSizeStrategy() { SizeOrLatencyBatchStrategy strategy = - new SizeOrLatencyBatchStrategy.Builder().withMaxBatchSize(5).build(); + new SizeOrLatencyBatchStrategy.Builder() + .withMaxBatchSize(5) + .withMaxDelay(1, TimeUnit.DAYS) + .build(); final List transmittedBatch = new ArrayList<>(); strategy.setBatchHandler(transmittedBatch::addAll); @@ -54,22 +67,23 @@ public void testSizeStrategy() { @Test public void testLatencyStrategy() throws InterruptedException { int maxDelay = 50; + TestBatchHandler batchHandler = new TestBatchHandler(); SizeOrLatencyBatchStrategy strategy = new SizeOrLatencyBatchStrategy.Builder() .withMaxDelay(maxDelay, TimeUnit.MILLISECONDS) .build(); - final List transmittedBatch = new ArrayList<>(); - - strategy.setBatchHandler(transmittedBatch::addAll); + strategy.setBatchHandler(batchHandler); for (int i = 0; i < 7; i++) { strategy.add(null); Thread.sleep(10); } - assertThat(transmittedBatch.size()).isEqualTo(5); + await() + .atMost(200, TimeUnit.MILLISECONDS) + .until(() -> batchHandler.records.size() == 7); - await().atMost(200, TimeUnit.MILLISECONDS).until(() -> transmittedBatch.size() == 7); + assertThat(batchHandler.callCount).isEqualTo(2); } } From aa545852220f9476fe652febbfd765ff3ebd3f55 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Mon, 24 Aug 2020 16:02:11 -0700 Subject: [PATCH 06/20] code format --- .../opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java index dfba7a4f1fe..9e71b37cb18 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -80,9 +80,7 @@ public void testLatencyStrategy() throws InterruptedException { Thread.sleep(10); } - await() - .atMost(200, TimeUnit.MILLISECONDS) - .until(() -> batchHandler.records.size() == 7); + await().atMost(200, TimeUnit.MILLISECONDS).until(() -> batchHandler.records.size() == 7); assertThat(batchHandler.callCount).isEqualTo(2); } From 8573b4f795b4d729fbefbc6810219cd371dda8f2 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Tue, 25 Aug 2020 11:37:35 -0700 Subject: [PATCH 07/20] change list to queue in implementation, collection in api --- .../logging/LoggingBatchExporter.java | 4 ++-- .../logging/LoggingBatchStrategy.java | 6 +++--- .../logging/SdkLogSinkProvider.java | 15 ++++++++------- .../logging/SizeOrLatencyBatchStrategy.java | 17 +++++++++++------ .../io/opentelemetry/logging/api/Exporter.java | 4 ++-- .../logging/SdkLogSinkProviderTest.java | 3 ++- .../logging/SizeOrLatencyBatchStrategyTest.java | 5 +++-- 7 files changed, 31 insertions(+), 23 deletions(-) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java index 9f45cfec517..81dc9ca8c0c 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java @@ -17,7 +17,7 @@ package io.opentelemetry.logging; import io.opentelemetry.logging.api.LogRecord; -import java.util.List; +import java.util.Collection; /** * A LoggingBatchExporter accepts a batch of records and handles the transmission of those records @@ -29,5 +29,5 @@ public interface LoggingBatchExporter { * * @param batch list of records ready for transport */ - void handleLogRecordBatch(List batch); + void handleLogRecordBatch(Collection batch); } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java index 784794d27a3..a2130521b15 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java @@ -17,7 +17,6 @@ package io.opentelemetry.logging; import io.opentelemetry.logging.api.LogRecord; -import java.util.List; /** * A LoggingBatchStrategy encodes the logic for how to batch and aggregate records for transmission. @@ -37,8 +36,9 @@ public interface LoggingBatchStrategy { void flush(); /** - * This sets the handler for the batch. {@link LoggingBatchExporter#handleLogRecordBatch(List)} - * will be called whenever this strategy's constraints are fulfilled. + * This sets the handler for the batch. {@link + * LoggingBatchExporter#handleLogRecordBatch(Collection)} will be called whenever this strategy's + * constraints are fulfilled. * * @param handler Exporter to invoke when the batch is full */ diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java index b07b8779914..e4a233cfa7f 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java @@ -20,21 +20,22 @@ import io.opentelemetry.logging.api.LogRecord; import io.opentelemetry.logging.api.LogSink; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import javax.annotation.Nullable; public class SdkLogSinkProvider implements LoggingBatchExporter { - private final LoggingBatchStrategy batchManager; + private final LoggingBatchStrategy batchStrategy; private final List exporters; private final LogSink logSink = new SdkLogSink(); - private SdkLogSinkProvider(LoggingBatchStrategy batchManager, List exporters) { - this.batchManager = batchManager != null ? batchManager : getDefaultBatchManager(); - this.batchManager.setBatchHandler(this); + private SdkLogSinkProvider(LoggingBatchStrategy batchStrategy, List exporters) { + this.batchStrategy = batchStrategy != null ? batchStrategy : getDefaultBatchStrategy(); + this.batchStrategy.setBatchHandler(this); this.exporters = exporters; } - private static LoggingBatchStrategy getDefaultBatchManager() { + private static LoggingBatchStrategy getDefaultBatchStrategy() { return new SizeOrLatencyBatchStrategy.Builder().build(); } @@ -44,7 +45,7 @@ public LogSink get(String instrumentationName, String instrumentationVersion) { } @Override - public void handleLogRecordBatch(List batch) { + public void handleLogRecordBatch(Collection batch) { for (Exporter e : exporters) { e.accept(batch); } @@ -53,7 +54,7 @@ public void handleLogRecordBatch(List batch) { private class SdkLogSink implements LogSink { @Override public void offer(LogRecord record) { - batchManager.add(record); + batchStrategy.add(record); } @Override diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java index cd9c823e5c8..c33e787a28e 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java @@ -17,9 +17,10 @@ package io.opentelemetry.logging; import io.opentelemetry.logging.api.LogRecord; -import java.util.ArrayList; -import java.util.List; +import java.util.Collection; +import java.util.Queue; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -42,7 +43,11 @@ public class SizeOrLatencyBatchStrategy implements LoggingBatchStrategy { private final ScheduledFuture schedule; private LoggingBatchExporter batchHandler; - private List batch = new ArrayList<>(); + private Queue batch = buildNewQueue(); + + private Queue buildNewQueue() { + return new LinkedBlockingQueue<>(); + } private SizeOrLatencyBatchStrategy(int maxBatch, int maxDelay, TimeUnit units) { this.maxBatch = maxBatch > 0 ? maxBatch : DEFAULT_BATCH_SIZE; @@ -75,10 +80,10 @@ public void add(LogRecord record) { } @Override - public void flush() { + public synchronized void flush() { if (batchHandler != null && batch.size() > 0) { - List completedBatch = batch; - batch = new ArrayList<>(); + Collection completedBatch = batch; + batch = buildNewQueue(); batchHandler.handleLogRecordBatch(completedBatch); } } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java index ca66f193cb1..eac343b4c76 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java @@ -16,12 +16,12 @@ package io.opentelemetry.logging.api; -import java.util.List; +import java.util.Collection; /** * An exporter is responsible for taking a list of {@link LogRecord}s and transmitting them to their * ultimate destination. */ public interface Exporter { - void accept(List records); + void accept(Collection records); } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java index 8744cdcc9e8..1635e59515c 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java @@ -24,6 +24,7 @@ import io.opentelemetry.logging.api.LogRecord.Severity; import io.opentelemetry.logging.api.LogSink; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.TimeUnit; import org.junit.Test; @@ -37,7 +38,7 @@ private static class TestExporter implements Exporter { private int batches = 0; @Override - public void accept(List records) { + public void accept(Collection records) { this.records.addAll(records); batches++; } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java index 9e71b37cb18..48f0b24bdaf 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -21,6 +21,7 @@ import io.opentelemetry.logging.api.LogRecord; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.TimeUnit; import org.junit.Test; @@ -34,7 +35,7 @@ private static class TestBatchHandler implements LoggingBatchExporter { private int callCount = 0; @Override - public void handleLogRecordBatch(List batch) { + public void handleLogRecordBatch(Collection batch) { records.addAll(batch); callCount++; } @@ -49,7 +50,7 @@ public void testSizeStrategy() { .build(); final List transmittedBatch = new ArrayList<>(); - strategy.setBatchHandler(transmittedBatch::addAll); + strategy.setBatchHandler(c -> transmittedBatch.addAll(c)); for (int i = 0; i < 7; i++) { strategy.add(null); From 44a238d60443d96a000c77337fb37aabf96233df Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Wed, 26 Aug 2020 09:25:39 -0700 Subject: [PATCH 08/20] missed making a utility method static --- .../io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java index c33e787a28e..8239b4a343b 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java @@ -45,7 +45,7 @@ public class SizeOrLatencyBatchStrategy implements LoggingBatchStrategy { private LoggingBatchExporter batchHandler; private Queue batch = buildNewQueue(); - private Queue buildNewQueue() { + private static Queue buildNewQueue() { return new LinkedBlockingQueue<>(); } From 9f2ed0a99841bcab6e1b7db9ae0bf550de84e14a Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Wed, 26 Aug 2020 09:48:16 -0700 Subject: [PATCH 09/20] fix a broken test --- .../logging/SizeOrLatencyBatchStrategyTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java index 48f0b24bdaf..592dacdc8b4 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java @@ -53,13 +53,15 @@ public void testSizeStrategy() { strategy.setBatchHandler(c -> transmittedBatch.addAll(c)); for (int i = 0; i < 7; i++) { - strategy.add(null); + LogRecord record = new LogRecord.Builder().build(); + strategy.add(record); } assertThat(transmittedBatch.size()).isEqualTo(5); for (int i = 0; i < 3; i++) { - strategy.add(null); + LogRecord record = new LogRecord.Builder().build(); + strategy.add(record); } assertThat(transmittedBatch.size()).isEqualTo(10); @@ -77,7 +79,8 @@ public void testLatencyStrategy() throws InterruptedException { strategy.setBatchHandler(batchHandler); for (int i = 0; i < 7; i++) { - strategy.add(null); + LogRecord record = new LogRecord.Builder().build(); + strategy.add(record); Thread.sleep(10); } From fa90ba0ce29efdcaa3eeb00e086c79b58275481e Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 27 Aug 2020 15:10:07 -0700 Subject: [PATCH 10/20] Update sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java Co-authored-by: Tigran Najaryan <4194920+tigrannajaryan@users.noreply.github.com> --- .../src/main/java/io/opentelemetry/logging/api/LogRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java index 78da93a8639..2c791b1c7f1 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java @@ -26,7 +26,7 @@ /** * A LogRecord is an implementation of the + * href="https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/logs/data-model.md"> * OpenTelemetry logging model. */ @AutoValue From b9b14aed8c62d2360f14d91064c658d2f0ea4f2a Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 10 Sep 2020 09:11:44 -0700 Subject: [PATCH 11/20] reworked to align with how we batch process spans --- .../logging/BatchLogProcessor.java | 411 ++++++++++++++++++ ...kProvider.java => LogSinkSdkProvider.java} | 50 +-- .../logging/LoggingBatchStrategy.java | 46 -- .../logging/SizeOrLatencyBatchStrategy.java | 135 ------ .../opentelemetry/logging/api/Exporter.java | 5 +- .../LogProcessor.java} | 28 +- .../logging/LogSinkSdkProviderTest.java | 136 ++++++ .../logging/SdkLogSinkProviderTest.java | 76 ---- .../SizeOrLatencyBatchStrategyTest.java | 91 ---- 9 files changed, 585 insertions(+), 393 deletions(-) create mode 100644 sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/{SdkLogSinkProvider.java => LogSinkSdkProvider.java} (50%) delete mode 100644 sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java delete mode 100644 sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/{LoggingBatchExporter.java => api/LogProcessor.java} (53%) create mode 100644 sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java delete mode 100644 sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java delete mode 100644 sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java new file mode 100644 index 00000000000..ba06871010d --- /dev/null +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java @@ -0,0 +1,411 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import io.opentelemetry.OpenTelemetry; +import io.opentelemetry.common.Labels; +import io.opentelemetry.internal.Utils; +import io.opentelemetry.logging.api.Exporter; +import io.opentelemetry.logging.api.LogProcessor; +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.metrics.LongCounter; +import io.opentelemetry.metrics.LongCounter.BoundLongCounter; +import io.opentelemetry.metrics.Meter; +import io.opentelemetry.sdk.common.DaemonThreadFactory; +import io.opentelemetry.sdk.common.export.CompletableResultCode; +import io.opentelemetry.sdk.common.export.ConfigBuilder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; + +public class BatchLogProcessor implements LogProcessor { + private static final String WORKER_THREAD_NAME = + BatchLogProcessor.class.getSimpleName() + "_WorkerThread"; + private static final String TIMER_THREAD_NAME = + BatchLogProcessor.class.getSimpleName() + "_TimerThread"; + + private final Worker worker; + private final Thread workerThread; + + private BatchLogProcessor( + int maxQueueSize, + long scheduleDelayMillis, + int maxExportBatchSize, + long exporterTimeout, + Exporter exporter) { + this.worker = new Worker(maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeout, + exporter); + this.workerThread = new DaemonThreadFactory(WORKER_THREAD_NAME).newThread(worker); + this.workerThread.start(); + } + + public static Builder builder(Exporter exporter) { + return new Builder(exporter); + } + + @Override + public void addLogRecord(LogRecord record) { + worker.addLogRecord(record); + } + + @Override + public void shutdown() { + workerThread.interrupt(); + worker.shutdown(); + } + + @Override + public void forceFlush() { + worker.forceFlush(); + } + + private static class Worker implements Runnable { + static { + Meter meter = OpenTelemetry.getMeter("io.opentelemetry.sdk.logging"); + LongCounter logProcessorErrors = + meter + .longCounterBuilder("logProcessorErrors") + .setUnit("1") + .setDescription( + "Number of errors encountered while processing logs") + .build(); + Labels.Builder builder = Labels.of("logProcessorType", BatchLogProcessor.class.getName()) + .toBuilder(); + exporterFailureCounter = + logProcessorErrors.bind( + builder.setLabel("errorType", "exporter failure").build()); +// exporterExceptionCounter = +// logProcessorErrors.bind( +// builder.setLabel("errorType", "exporter exception").build()); + exporterBusyCounter = + logProcessorErrors.bind( + builder.setLabel("errorType", "exporter busy").build()); + droppedRecordCounter = + logProcessorErrors.bind( + builder.setLabel("errorType", "dropped record - queue full").build()); + } + + private static final BoundLongCounter exporterFailureCounter; + private static final BoundLongCounter exporterBusyCounter; + private static final BoundLongCounter droppedRecordCounter; + private static final Logger logger = Logger.getLogger(Worker.class.getName()); + + private final Object monitor = new Object(); + private final int maxQueueSize; + private final long scheduleDelayMillis; + private final ArrayList logRecords; + private final int maxExportBatchSize; + private final ExecutorService executor; + private final Exporter exporter; + private final Timer timer = new Timer(TIMER_THREAD_NAME, true); +// private final AtomicBoolean exportAvailable = new AtomicBoolean(true); + private final long exporterTimeout; + + private Worker( + int maxQueueSize, + long scheduleDelayMillis, + int maxExportBatchSize, + long exporterTimeout, + Exporter exporter) { + this.maxQueueSize = maxQueueSize; + this.maxExportBatchSize = maxExportBatchSize; + + this.exporterTimeout = exporterTimeout; + this.scheduleDelayMillis = scheduleDelayMillis; + this.exporter = exporter; + this.logRecords = new ArrayList<>(maxQueueSize); + // We should be able to drain a full queue without dropping a batch + int exportQueueSize = maxQueueSize / maxExportBatchSize; + RejectedExecutionHandler h = new RejectedExecutionHandler() { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + exporterBusyCounter.add(1); + } + }; + this.executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(exportQueueSize), h); + } + + @Override + public void run() { + ArrayList logsCopy; + while (!Thread.currentThread().isInterrupted()) { + synchronized (monitor) { + if (this.logRecords.size() < maxExportBatchSize) { + do { + try { + monitor.wait(scheduleDelayMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } while (this.logRecords.isEmpty()); + } + logsCopy = new ArrayList<>(this.logRecords); + logRecords.clear(); + } + exportBatches(logsCopy); + } + } + + private void exportBatches(ArrayList recordsToShip) { + for (int i = 0; i < recordsToShip.size(); ) { + int lastIndexToTake = Math.min(i + maxExportBatchSize, recordsToShip.size()); + onBatchExport(createLogDataForExport(recordsToShip, i, lastIndexToTake)); + i = lastIndexToTake; + } + } + + private void onBatchExport(final List logDataForExport) { + // The call to exporter.accept(logDataForExport) is presumed to be asynchronous + // but we ensure that the call takes less than exporterSubmissionTimeout time + // to enqueue the action. We then monitor completion of the CompletableResultCode + // and ensure that execution occurs within the exporterTimeout time frame. + final Future f = executor.submit( + new Callable() { + @Override + public CompletableResultCode call() { + return exporter.accept(logDataForExport); + } + }); +// final CompletableResultCode result; +// try { +// // This may block the worker thread for up to exporterSubmissionTimeout. This +// // means that there is at most a single call to exporter.accept() happening at +// // a time, and that they are in order. Ideally exporter.accept() should be +// // enqueuing the work and returning immediately to be completed asynchronously. +// result = f.get(exporterSubmissionTimeout, TimeUnit.MILLISECONDS); +// } catch (InterruptedException e) { +// logger.warning("log exporter interruption:" + e.getLocalizedMessage()); +// f.cancel(true); +// return; +// } catch (ExecutionException e) { +// logger.warning("log exporter exception:" + e.getLocalizedMessage()); +// exporterFailureCounter.add(1); +// return; +// } catch (TimeoutException e) { +// logger.warning("log exporter submission timeout"); +// exporterTimeoutCounter.add(1); +// f.cancel(true); +// return; +// } +// +// result.whenComplete( +// new Runnable() { +// @Override +// public void run() { +// if (!result.isSuccess()) { +// exporterFailureCounter.add(1); +// // TODO: would be nice to be able to capture an error in CompletableResultCode +// logger.warning("Error encountered while exporting log batch"); +// } +// exportAvailable.set(true); +// } +// }); + + timer.schedule( + new TimerTask() { + @Override + public void run() { + if (f.isDone()) { + try { + final CompletableResultCode result = f.get(0, TimeUnit.MILLISECONDS); + if (!result.isSuccess()) { + // This may mean that the export process has failed, or that it's still running + // but it's at the end of it's timeout. + logger.warning("log exporter has failed or timed out"); + exporterFailureCounter.add(1); + } + } catch (InterruptedException | ExecutionException e) { + logger.warning("log exporter failure:" + e.getLocalizedMessage()); + exporterFailureCounter.add(1); + } catch (TimeoutException e) { + logger.warning("log exporter has failed to return async result"); + exporterFailureCounter.add(1); + } + } + } + }, + exporterTimeout); + } + + private static List createLogDataForExport( + ArrayList recordsToShip, int startIndex, int endIndex) { + List logDataBuffer = new ArrayList<>(endIndex - startIndex); + for (int i = startIndex; i < endIndex; i++) { + logDataBuffer.add(recordsToShip.get(i)); + recordsToShip.set(i, null); + } + return Collections.unmodifiableList(logDataBuffer); + } + + public void addLogRecord(LogRecord record) { + synchronized (monitor) { + if (logRecords.size() >= maxQueueSize) { + droppedRecordCounter.add(1); + // FIXME: call callback + } + logRecords.add(record); + if (logRecords.size() >= maxExportBatchSize) { + monitor.notifyAll(); + } + } + } + + private void shutdown() { + forceFlush(); + timer.cancel(); + exporter.shutdown(); + } + + private void forceFlush() { + ArrayList logsCopy; + synchronized (monitor) { + logsCopy = new ArrayList<>(this.logRecords); + logRecords.clear(); + } + exportBatches(logsCopy); + } + } + + static class Builder extends ConfigBuilder { + /* @VisibleForTesting */ static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 5000; + static final int DEFAULT_MAX_QUEUE_SIZE = 2048; + static final int DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; + static final long DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; + + private final Exporter exporter; + private long scheduleDelayMillis = DEFAULT_SCHEDULE_DELAY_MILLIS; + private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; + private int maxExportBatchSize = DEFAULT_MAX_EXPORT_BATCH_SIZE; + private long exporterTimeoutMillis = DEFAULT_EXPORT_TIMEOUT_MILLIS; + + private Builder(Exporter exporter) { + this.exporter = Utils.checkNotNull(exporter, "Exporter argument can not be null"); + } + + public Builder newBuilder(Exporter exporter) { + return new Builder(exporter); + } + + public BatchLogProcessor build() { + return new BatchLogProcessor(maxQueueSize, scheduleDelayMillis, maxExportBatchSize, + exporterTimeoutMillis, exporter); + } + + + /** + * Sets the delay interval between two consecutive exports. The actual interval may be shorter + * if the batch size is getting larger than {@code maxQueuedSpans / 2}. + * + *

Default value is {@code 5000}ms. + * + * @param scheduleDelayMillis the delay interval between two consecutive exports. + * @return this. + * @see BatchLogProcessor.Builder#DEFAULT_SCHEDULE_DELAY_MILLIS + */ + public BatchLogProcessor.Builder setScheduleDelayMillis(long scheduleDelayMillis) { + this.scheduleDelayMillis = scheduleDelayMillis; + return this; + } + + long getScheduleDelayMillis() { + return scheduleDelayMillis; + } + + /** + * Sets the maximum time an exporter will be allowed to run before being cancelled. + * + *

Default value is {@code 30000}ms + * + * @param exporterTimeoutMillis the timeout for exports in milliseconds. + * @return this + * @see BatchLogProcessor.Builder#DEFAULT_EXPORT_TIMEOUT_MILLIS + */ + public Builder setExporterTimeoutMillis(int exporterTimeoutMillis) { + this.exporterTimeoutMillis = exporterTimeoutMillis; + return this; + } + + long getExporterTimeoutMillis() { + return exporterTimeoutMillis; + } + + /** + * Sets the maximum number of Spans that are kept in the queue before start dropping. + * + *

See the BatchSampledSpansProcessor class description for a high-level design description + * of this class. + * + *

Default value is {@code 2048}. + * + * @param maxQueueSize the maximum number of Spans that are kept in the queue before start + * dropping. + * @return this. + * @see BatchLogProcessor.Builder#DEFAULT_MAX_QUEUE_SIZE + */ + public Builder setMaxQueueSize(int maxQueueSize) { + this.maxQueueSize = maxQueueSize; + return this; + } + + int getMaxQueueSize() { + return maxQueueSize; + } + + /** + * Sets the maximum batch size for every export. This must be smaller or equal to {@code + * maxQueuedSpans}. + * + *

Default value is {@code 512}. + * + * @param maxExportBatchSize the maximum batch size for every export. + * @return this. + * @see BatchLogProcessor.Builder#DEFAULT_MAX_EXPORT_BATCH_SIZE + */ + public Builder setMaxExportBatchSize(int maxExportBatchSize) { + Utils.checkArgument(maxExportBatchSize > 0, "maxExportBatchSize must be positive."); + this.maxExportBatchSize = maxExportBatchSize; + return this; + } + + int getMaxExportBatchSize() { + return maxExportBatchSize; + } + + + @Override + protected Builder fromConfigMap(Map configMap, + NamingConvention namingConvention) { + return null; + } + } + +} diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java similarity index 50% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java index e4a233cfa7f..6cd52102d0e 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SdkLogSinkProvider.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java @@ -16,27 +16,18 @@ package io.opentelemetry.logging; -import io.opentelemetry.logging.api.Exporter; +import io.opentelemetry.internal.Utils; +import io.opentelemetry.logging.api.LogProcessor; import io.opentelemetry.logging.api.LogRecord; import io.opentelemetry.logging.api.LogSink; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import javax.annotation.Nullable; -public class SdkLogSinkProvider implements LoggingBatchExporter { - private final LoggingBatchStrategy batchStrategy; - private final List exporters; +public class LogSinkSdkProvider { private final LogSink logSink = new SdkLogSink(); + private final List processors = new ArrayList<>(); - private SdkLogSinkProvider(LoggingBatchStrategy batchStrategy, List exporters) { - this.batchStrategy = batchStrategy != null ? batchStrategy : getDefaultBatchStrategy(); - this.batchStrategy.setBatchHandler(this); - this.exporters = exporters; - } - - private static LoggingBatchStrategy getDefaultBatchStrategy() { - return new SizeOrLatencyBatchStrategy.Builder().build(); + private LogSinkSdkProvider() { } public LogSink get(String instrumentationName, String instrumentationVersion) { @@ -44,17 +35,22 @@ public LogSink get(String instrumentationName, String instrumentationVersion) { return logSink; } - @Override - public void handleLogRecordBatch(Collection batch) { - for (Exporter e : exporters) { - e.accept(batch); + public void addLogProcessor(LogProcessor processor) { + processors.add(Utils.checkNotNull(processor, "Processor can not be null")); + } + + public void forceFlush() { + for (LogProcessor processor : processors) { + processor.forceFlush(); } } private class SdkLogSink implements LogSink { @Override public void offer(LogRecord record) { - batchStrategy.add(record); + for (LogProcessor processor : processors) { + processor.addLogRecord(record); + } } @Override @@ -64,21 +60,9 @@ public LogRecord.Builder buildRecord() { } public static class Builder { - @Nullable private LoggingBatchStrategy batchManager = null; - private final List exporters = new ArrayList<>(); - - public Builder withBatchManager(LoggingBatchStrategy manager) { - batchManager = manager; - return this; - } - - public Builder withExporter(Exporter exporter) { - exporters.add(exporter); - return this; - } - public SdkLogSinkProvider build() { - return new SdkLogSinkProvider(batchManager, exporters); + public LogSinkSdkProvider build() { + return new LogSinkSdkProvider(); } } } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java deleted file mode 100644 index a2130521b15..00000000000 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchStrategy.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020, OpenTelemetry Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.opentelemetry.logging; - -import io.opentelemetry.logging.api.LogRecord; - -/** - * A LoggingBatchStrategy encodes the logic for how to batch and aggregate records for transmission. - * {@link SizeOrLatencyBatchStrategy} - */ -public interface LoggingBatchStrategy { - void stop(); - - /** - * Add a LogRecord to the existing batch. - * - * @param record record to enqueue - */ - void add(LogRecord record); - - /** Explicitly flush the batch. */ - void flush(); - - /** - * This sets the handler for the batch. {@link - * LoggingBatchExporter#handleLogRecordBatch(Collection)} will be called whenever this strategy's - * constraints are fulfilled. - * - * @param handler Exporter to invoke when the batch is full - */ - void setBatchHandler(LoggingBatchExporter handler); -} diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java deleted file mode 100644 index 8239b4a343b..00000000000 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategy.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2020, OpenTelemetry Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.opentelemetry.logging; - -import io.opentelemetry.logging.api.LogRecord; -import java.util.Collection; -import java.util.Queue; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; - -/** - * Batching strategy that allows specifying both a maximum size and a maximum amount of time before - * sending a batch of records. Batch will be flushed whenever the list of records queued exceeds the - * maximum threshold (default 50), or a set timeout expires (default 5 seconds). - */ -public class SizeOrLatencyBatchStrategy implements LoggingBatchStrategy { - private static final int DEFAULT_BATCH_SIZE = 50; - private static final int DEFAULT_MAX_DELAY = 5; - private static final TimeUnit DEFAULT_MAX_DELAY_UNITS = TimeUnit.SECONDS; - private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - - private final int maxBatch; - private final int maxDelay; - private final TimeUnit maxDelayUnits; - private final ScheduledFuture schedule; - - private LoggingBatchExporter batchHandler; - private Queue batch = buildNewQueue(); - - private static Queue buildNewQueue() { - return new LinkedBlockingQueue<>(); - } - - private SizeOrLatencyBatchStrategy(int maxBatch, int maxDelay, TimeUnit units) { - this.maxBatch = maxBatch > 0 ? maxBatch : DEFAULT_BATCH_SIZE; - this.maxDelay = maxDelay > 0 ? maxDelay : DEFAULT_MAX_DELAY; - this.maxDelayUnits = units != null ? units : DEFAULT_MAX_DELAY_UNITS; - this.schedule = - executor.scheduleWithFixedDelay( - new Runnable() { - @Override - public void run() { - flush(); - } - }, - this.maxDelay, - this.maxDelay, - this.maxDelayUnits); - } - - @Override - public void stop() { - schedule.cancel(false); - } - - @Override - public void add(LogRecord record) { - batch.add(record); - if (batch.size() >= maxBatch) { - flush(); - } - } - - @Override - public synchronized void flush() { - if (batchHandler != null && batch.size() > 0) { - Collection completedBatch = batch; - batch = buildNewQueue(); - batchHandler.handleLogRecordBatch(completedBatch); - } - } - - @Override - public void setBatchHandler(LoggingBatchExporter handler) { - batchHandler = handler; - } - - public static class Builder { - private int maxBatchSize = -1; - private int maxDelay = -1; - @Nullable private TimeUnit maxDelayTimeUnit = null; - - /** - * The maximum number of LogRecord objects to accumulate before dispatching a batch. - * - * @param maxBatchSize max number of records - * @return builder - */ - public Builder withMaxBatchSize(int maxBatchSize) { - this.maxBatchSize = maxBatchSize; - return this; - } - - /** - * The length of time to wait before dispatching a batch, even fi the number of records - * accumulated is less than the max number allowed. - * - * @param delay the maximum amount of time to wait before dispatching a batch - * @param units the units the amount of time should be measured in - * @return builder - */ - public Builder withMaxDelay(int delay, TimeUnit units) { - this.maxDelay = delay; - this.maxDelayTimeUnit = units; - return this; - } - - /** - * Instantiate the strategy. - * - * @return built strategy - */ - public SizeOrLatencyBatchStrategy build() { - return new SizeOrLatencyBatchStrategy(maxBatchSize, maxDelay, maxDelayTimeUnit); - } - } -} diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java index eac343b4c76..ee10e8154a6 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java @@ -16,6 +16,7 @@ package io.opentelemetry.logging.api; +import io.opentelemetry.sdk.common.export.CompletableResultCode; import java.util.Collection; /** @@ -23,5 +24,7 @@ * ultimate destination. */ public interface Exporter { - void accept(Collection records); + CompletableResultCode accept(Collection records); + + void shutdown(); } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogProcessor.java similarity index 53% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogProcessor.java index 81dc9ca8c0c..c4cfc0f0f3e 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LoggingBatchExporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogProcessor.java @@ -14,20 +14,26 @@ * limitations under the License. */ -package io.opentelemetry.logging; +package io.opentelemetry.logging.api; -import io.opentelemetry.logging.api.LogRecord; -import java.util.Collection; +import io.opentelemetry.sdk.trace.TracerSdkProvider; + +public interface LogProcessor { + + void addLogRecord(LogRecord record); + + /** + * Called when {@link TracerSdkProvider#shutdown()} is called. + * + *

Implementations must ensure that all span events are processed before returning. + */ + void shutdown(); -/** - * A LoggingBatchExporter accepts a batch of records and handles the transmission of those records - * to a remote system. - */ -public interface LoggingBatchExporter { /** - * Accept a batch of records for transmission or export. + * Processes all span events that have not yet been processed. * - * @param batch list of records ready for transport + *

This method is executed synchronously on the calling thread, and should not throw + * exceptions. */ - void handleLogRecordBatch(Collection batch); + void forceFlush(); } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java new file mode 100644 index 00000000000..6f5c00a7967 --- /dev/null +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java @@ -0,0 +1,136 @@ +package io.opentelemetry.logging; + +import io.opentelemetry.logging.api.Exporter; +import io.opentelemetry.logging.api.LogProcessor; +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.LogRecord.Severity; +import io.opentelemetry.logging.api.LogSink; +import io.opentelemetry.sdk.common.export.CompletableResultCode; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@RunWith(JUnit4.class) +public class LogSinkSdkProviderTest { + private static class TestExporter implements Exporter { + private final ArrayList records = new ArrayList<>(); + @Nullable private Runnable onCall = null; + private int callCount = 0; + + @Override + public CompletableResultCode accept(Collection records) { + this.records.addAll(records); + callCount++; + if (onCall != null) { + onCall.run(); + } + return null; + } + + @Override + public void shutdown() {} + } + + private static LogRecord createLog(LogRecord.Severity severity, String message) { + return new LogRecord.Builder() + .withUnixTimeMillis(System.currentTimeMillis()) + .withSeverity(severity) + .withBody(message) + .build(); + } + + @Test + public void testLogSinkSdkProvider() { + TestExporter exporter = new TestExporter(); + LogProcessor processor = BatchLogProcessor.builder(exporter).build(); + LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); + provider.addLogProcessor(processor); + LogSink sink = provider.get("test", "0.1a"); + sink.offer(createLog(Severity.ERROR, "test")); + provider.forceFlush(); + await() + .atMost(120, TimeUnit.MILLISECONDS) + .until(() -> exporter.records.size() > 0); + assertThat(exporter.records.size()).isEqualTo(1); + } + + @Test + public void testBatchSize() { + TestExporter exporter = new TestExporter(); + LogProcessor processor = BatchLogProcessor + .builder(exporter) + .setScheduleDelayMillis(3000) // Long enough to not be in play + .setMaxExportBatchSize(5) + .setMaxQueueSize(10) + .build(); + LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); + provider.addLogProcessor(processor); + LogSink sink = provider.get("test", "0.1a"); + + for (int i=0; i<7; i++) { + sink.offer(createLog(Severity.WARN, "test #" + i)); + } + // Ensure that more than batch size kicks off a flush + await() + .atMost(120, TimeUnit.MILLISECONDS) + .until(() -> exporter.records.size() > 0); + // Ensure that everything gets through + provider.forceFlush(); + await() + .atMost(120, TimeUnit.MILLISECONDS) + .until(() -> exporter.records.size() == 7); + assertThat(exporter.callCount).isEqualTo(2); + } + + @Test + public void testNoBlocking() { + TestExporter exporter = new TestExporter(); + exporter.onCall = () -> { + try { + Thread.sleep(250); + } catch(InterruptedException ignored) {} + }; + LogProcessor processor = BatchLogProcessor + .builder(exporter) + .setScheduleDelayMillis(3000) // Long enough to not be in play + .setMaxExportBatchSize(5) + .setMaxQueueSize(10) + .build(); + LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); + provider.addLogProcessor(processor); + LogSink sink = provider.get("test", "0.1a"); + + long start = System.currentTimeMillis(); + for (int i=0; i<700; i++) { + sink.offer(createLog(Severity.WARN, "test #" + i)); + } + long end = System.currentTimeMillis(); + assertThat(end - start).isLessThan(250L); + await() + .atMost(510, TimeUnit.MILLISECONDS) + .until(() -> exporter.callCount == 2); + assertThat(exporter.records.size()).isEqualTo(10); // Two exporter batches + } +} \ No newline at end of file diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java deleted file mode 100644 index 1635e59515c..00000000000 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SdkLogSinkProviderTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020, OpenTelemetry Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.opentelemetry.logging; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -import io.opentelemetry.logging.api.Exporter; -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.LogRecord.Severity; -import io.opentelemetry.logging.api.LogSink; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class SdkLogSinkProviderTest { - private static class TestExporter implements Exporter { - private final List records = new ArrayList<>(); - private int batches = 0; - - @Override - public void accept(Collection records) { - this.records.addAll(records); - batches++; - } - } - - @Test - public void testProviderAggregation() { - TestExporter exporter1 = new TestExporter(); - TestExporter exporter2 = new TestExporter(); - SdkLogSinkProvider provider = - new SdkLogSinkProvider.Builder() - .withBatchManager( - new SizeOrLatencyBatchStrategy.Builder() - .withMaxBatchSize(5) - .withMaxDelay(250, TimeUnit.MILLISECONDS) - .build()) - .withExporter(exporter1) - .withExporter(exporter2) - .build(); - LogSink sink = provider.get("test", "0.8.0"); - for (int i = 0; i < 11; i++) { - sink.offer( - sink.buildRecord() - .withUnixTimeMillis(System.currentTimeMillis()) - .withSeverity(Severity.DEBUG) - .withBody("test") - .build()); - } - await() - .atLeast(100, TimeUnit.MILLISECONDS) - .atMost(500, TimeUnit.MILLISECONDS) - .until(() -> exporter1.records.size() == 11); - assertThat(exporter1.batches).isEqualTo(3); - } -} diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java deleted file mode 100644 index 592dacdc8b4..00000000000 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/SizeOrLatencyBatchStrategyTest.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2020, OpenTelemetry Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.opentelemetry.logging; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -import io.opentelemetry.logging.api.LogRecord; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -@RunWith(JUnit4.class) -public class SizeOrLatencyBatchStrategyTest { - private static class TestBatchHandler implements LoggingBatchExporter { - private final List records = new ArrayList<>(); - private int callCount = 0; - - @Override - public void handleLogRecordBatch(Collection batch) { - records.addAll(batch); - callCount++; - } - } - - @Test - public void testSizeStrategy() { - SizeOrLatencyBatchStrategy strategy = - new SizeOrLatencyBatchStrategy.Builder() - .withMaxBatchSize(5) - .withMaxDelay(1, TimeUnit.DAYS) - .build(); - final List transmittedBatch = new ArrayList<>(); - - strategy.setBatchHandler(c -> transmittedBatch.addAll(c)); - - for (int i = 0; i < 7; i++) { - LogRecord record = new LogRecord.Builder().build(); - strategy.add(record); - } - - assertThat(transmittedBatch.size()).isEqualTo(5); - - for (int i = 0; i < 3; i++) { - LogRecord record = new LogRecord.Builder().build(); - strategy.add(record); - } - - assertThat(transmittedBatch.size()).isEqualTo(10); - } - - @Test - public void testLatencyStrategy() throws InterruptedException { - int maxDelay = 50; - TestBatchHandler batchHandler = new TestBatchHandler(); - SizeOrLatencyBatchStrategy strategy = - new SizeOrLatencyBatchStrategy.Builder() - .withMaxDelay(maxDelay, TimeUnit.MILLISECONDS) - .build(); - - strategy.setBatchHandler(batchHandler); - - for (int i = 0; i < 7; i++) { - LogRecord record = new LogRecord.Builder().build(); - strategy.add(record); - Thread.sleep(10); - } - - await().atMost(200, TimeUnit.MILLISECONDS).until(() -> batchHandler.records.size() == 7); - - assertThat(batchHandler.callCount).isEqualTo(2); - } -} From cb607c5ff7fd669987494588d56743588f2b1572 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 10 Sep 2020 09:58:54 -0700 Subject: [PATCH 12/20] move of CompletableResultCode and formatting --- .../logging/BatchLogProcessor.java | 113 ++++++------------ .../logging/LogSinkSdkProvider.java | 4 +- .../opentelemetry/logging/api/Exporter.java | 2 +- .../logging/LogSinkSdkProviderTest.java | 103 ++++++++-------- 4 files changed, 91 insertions(+), 131 deletions(-) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java index ba06871010d..26bce5301be 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java @@ -25,8 +25,8 @@ import io.opentelemetry.metrics.LongCounter; import io.opentelemetry.metrics.LongCounter.BoundLongCounter; import io.opentelemetry.metrics.Meter; +import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.common.DaemonThreadFactory; -import io.opentelemetry.sdk.common.export.CompletableResultCode; import io.opentelemetry.sdk.common.export.ConfigBuilder; import java.util.ArrayList; import java.util.Collections; @@ -60,8 +60,9 @@ private BatchLogProcessor( int maxExportBatchSize, long exporterTimeout, Exporter exporter) { - this.worker = new Worker(maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeout, - exporter); + this.worker = + new Worker( + maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeout, exporter); this.workerThread = new DaemonThreadFactory(WORKER_THREAD_NAME).newThread(worker); this.workerThread.start(); } @@ -93,20 +94,14 @@ private static class Worker implements Runnable { meter .longCounterBuilder("logProcessorErrors") .setUnit("1") - .setDescription( - "Number of errors encountered while processing logs") + .setDescription("Number of errors encountered while processing logs") .build(); - Labels.Builder builder = Labels.of("logProcessorType", BatchLogProcessor.class.getName()) - .toBuilder(); + Labels.Builder builder = + Labels.of("logProcessorType", BatchLogProcessor.class.getName()).toBuilder(); exporterFailureCounter = - logProcessorErrors.bind( - builder.setLabel("errorType", "exporter failure").build()); -// exporterExceptionCounter = -// logProcessorErrors.bind( -// builder.setLabel("errorType", "exporter exception").build()); + logProcessorErrors.bind(builder.setLabel("errorType", "exporter failure").build()); exporterBusyCounter = - logProcessorErrors.bind( - builder.setLabel("errorType", "exporter busy").build()); + logProcessorErrors.bind(builder.setLabel("errorType", "exporter busy").build()); droppedRecordCounter = logProcessorErrors.bind( builder.setLabel("errorType", "dropped record - queue full").build()); @@ -124,8 +119,7 @@ private static class Worker implements Runnable { private final int maxExportBatchSize; private final ExecutorService executor; private final Exporter exporter; - private final Timer timer = new Timer(TIMER_THREAD_NAME, true); -// private final AtomicBoolean exportAvailable = new AtomicBoolean(true); + private final Timer timer = new Timer(TIMER_THREAD_NAME, /* isDaemon= */ true); private final long exporterTimeout; private Worker( @@ -143,14 +137,21 @@ private Worker( this.logRecords = new ArrayList<>(maxQueueSize); // We should be able to drain a full queue without dropping a batch int exportQueueSize = maxQueueSize / maxExportBatchSize; - RejectedExecutionHandler h = new RejectedExecutionHandler() { - @Override - public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - exporterBusyCounter.add(1); - } - }; - this.executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue(exportQueueSize), h); + RejectedExecutionHandler h = + new RejectedExecutionHandler() { + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + exporterBusyCounter.add(1); + } + }; + this.executor = + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(exportQueueSize), + h); } @Override @@ -184,51 +185,14 @@ private void exportBatches(ArrayList recordsToShip) { } private void onBatchExport(final List logDataForExport) { - // The call to exporter.accept(logDataForExport) is presumed to be asynchronous - // but we ensure that the call takes less than exporterSubmissionTimeout time - // to enqueue the action. We then monitor completion of the CompletableResultCode - // and ensure that execution occurs within the exporterTimeout time frame. - final Future f = executor.submit( - new Callable() { - @Override - public CompletableResultCode call() { - return exporter.accept(logDataForExport); - } - }); -// final CompletableResultCode result; -// try { -// // This may block the worker thread for up to exporterSubmissionTimeout. This -// // means that there is at most a single call to exporter.accept() happening at -// // a time, and that they are in order. Ideally exporter.accept() should be -// // enqueuing the work and returning immediately to be completed asynchronously. -// result = f.get(exporterSubmissionTimeout, TimeUnit.MILLISECONDS); -// } catch (InterruptedException e) { -// logger.warning("log exporter interruption:" + e.getLocalizedMessage()); -// f.cancel(true); -// return; -// } catch (ExecutionException e) { -// logger.warning("log exporter exception:" + e.getLocalizedMessage()); -// exporterFailureCounter.add(1); -// return; -// } catch (TimeoutException e) { -// logger.warning("log exporter submission timeout"); -// exporterTimeoutCounter.add(1); -// f.cancel(true); -// return; -// } -// -// result.whenComplete( -// new Runnable() { -// @Override -// public void run() { -// if (!result.isSuccess()) { -// exporterFailureCounter.add(1); -// // TODO: would be nice to be able to capture an error in CompletableResultCode -// logger.warning("Error encountered while exporting log batch"); -// } -// exportAvailable.set(true); -// } -// }); + final Future f = + executor.submit( + new Callable() { + @Override + public CompletableResultCode call() { + return exporter.accept(logDataForExport); + } + }); timer.schedule( new TimerTask() { @@ -316,11 +280,10 @@ public Builder newBuilder(Exporter exporter) { } public BatchLogProcessor build() { - return new BatchLogProcessor(maxQueueSize, scheduleDelayMillis, maxExportBatchSize, - exporterTimeoutMillis, exporter); + return new BatchLogProcessor( + maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeoutMillis, exporter); } - /** * Sets the delay interval between two consecutive exports. The actual interval may be shorter * if the batch size is getting larger than {@code maxQueuedSpans / 2}. @@ -400,12 +363,10 @@ int getMaxExportBatchSize() { return maxExportBatchSize; } - @Override - protected Builder fromConfigMap(Map configMap, - NamingConvention namingConvention) { + protected Builder fromConfigMap( + Map configMap, NamingConvention namingConvention) { return null; } } - } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java index 6cd52102d0e..8af03b789ca 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java @@ -27,8 +27,7 @@ public class LogSinkSdkProvider { private final LogSink logSink = new SdkLogSink(); private final List processors = new ArrayList<>(); - private LogSinkSdkProvider() { - } + private LogSinkSdkProvider() {} public LogSink get(String instrumentationName, String instrumentationVersion) { // Currently there is no differentiation by instrumentation library @@ -39,6 +38,7 @@ public void addLogProcessor(LogProcessor processor) { processors.add(Utils.checkNotNull(processor, "Processor can not be null")); } + /** Flushes all attached processors. */ public void forceFlush() { for (LogProcessor processor : processors) { processor.forceFlush(); diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java index ee10e8154a6..d0e12794266 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java @@ -16,7 +16,7 @@ package io.opentelemetry.logging.api; -import io.opentelemetry.sdk.common.export.CompletableResultCode; +import io.opentelemetry.sdk.common.CompletableResultCode; import java.util.Collection; /** diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java index 6f5c00a7967..b2df762faae 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java @@ -1,22 +1,3 @@ -package io.opentelemetry.logging; - -import io.opentelemetry.logging.api.Exporter; -import io.opentelemetry.logging.api.LogProcessor; -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.LogRecord.Severity; -import io.opentelemetry.logging.api.LogSink; -import io.opentelemetry.sdk.common.export.CompletableResultCode; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - /* * Copyright 2020, OpenTelemetry Authors * @@ -32,6 +13,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +package io.opentelemetry.logging; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; + +import io.opentelemetry.logging.api.Exporter; +import io.opentelemetry.logging.api.LogProcessor; +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.LogRecord.Severity; +import io.opentelemetry.logging.api.LogSink; +import io.opentelemetry.sdk.common.CompletableResultCode; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + @RunWith(JUnit4.class) public class LogSinkSdkProviderTest { private static class TestExporter implements Exporter { @@ -70,67 +72,64 @@ public void testLogSinkSdkProvider() { LogSink sink = provider.get("test", "0.1a"); sink.offer(createLog(Severity.ERROR, "test")); provider.forceFlush(); - await() - .atMost(120, TimeUnit.MILLISECONDS) - .until(() -> exporter.records.size() > 0); + await().atMost(120, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); assertThat(exporter.records.size()).isEqualTo(1); } @Test public void testBatchSize() { TestExporter exporter = new TestExporter(); - LogProcessor processor = BatchLogProcessor - .builder(exporter) - .setScheduleDelayMillis(3000) // Long enough to not be in play - .setMaxExportBatchSize(5) - .setMaxQueueSize(10) - .build(); + LogProcessor processor = + BatchLogProcessor.builder(exporter) + .setScheduleDelayMillis(3000) // Long enough to not be in play + .setMaxExportBatchSize(5) + .setMaxQueueSize(10) + .build(); LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); provider.addLogProcessor(processor); LogSink sink = provider.get("test", "0.1a"); - for (int i=0; i<7; i++) { + for (int i = 0; i < 7; i++) { sink.offer(createLog(Severity.WARN, "test #" + i)); } // Ensure that more than batch size kicks off a flush - await() - .atMost(120, TimeUnit.MILLISECONDS) - .until(() -> exporter.records.size() > 0); + await().atMost(120, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); // Ensure that everything gets through provider.forceFlush(); - await() - .atMost(120, TimeUnit.MILLISECONDS) - .until(() -> exporter.records.size() == 7); + await().atMost(120, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() == 7); assertThat(exporter.callCount).isEqualTo(2); } @Test public void testNoBlocking() { TestExporter exporter = new TestExporter(); - exporter.onCall = () -> { - try { - Thread.sleep(250); - } catch(InterruptedException ignored) {} - }; - LogProcessor processor = BatchLogProcessor - .builder(exporter) - .setScheduleDelayMillis(3000) // Long enough to not be in play - .setMaxExportBatchSize(5) - .setMaxQueueSize(10) - .build(); + exporter.onCall = + () -> { + try { + Thread.sleep(250); + } catch (InterruptedException ex) { + fail("Exporter wait interrupted", ex); + } + }; + LogProcessor processor = + BatchLogProcessor.builder(exporter) + .setScheduleDelayMillis(3000) // Long enough to not be in play + .setMaxExportBatchSize(5) + .setMaxQueueSize(10) + .build(); LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); provider.addLogProcessor(processor); LogSink sink = provider.get("test", "0.1a"); long start = System.currentTimeMillis(); - for (int i=0; i<700; i++) { + for (int i = 0; i < 700; i++) { sink.offer(createLog(Severity.WARN, "test #" + i)); } long end = System.currentTimeMillis(); assertThat(end - start).isLessThan(250L); - await() - .atMost(510, TimeUnit.MILLISECONDS) - .until(() -> exporter.callCount == 2); - assertThat(exporter.records.size()).isEqualTo(10); // Two exporter batches + await().atMost(510, TimeUnit.MILLISECONDS).until(() -> exporter.callCount == 2); + assertThat(exporter.records.size()) // Two exporter batches + .isGreaterThan(5) + .isLessThanOrEqualTo(10); } -} \ No newline at end of file +} From 76a73bc497ba8c44a75d485080d9b056fbadc6a0 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 10 Sep 2020 11:16:09 -0700 Subject: [PATCH 13/20] update AnyValue to use long instead of int --- .../io/opentelemetry/common/AnyValue.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java index 4e0faf76e0f..5c48a2938f4 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java @@ -37,7 +37,7 @@ public abstract class AnyValue { public enum Type { STRING, BOOL, - INT, + INT64, DOUBLE, ARRAY, KVLIST @@ -67,14 +67,14 @@ public String getStringValue() { /** * Returns an {@code AnyValue} with an int value. * - * @param intValue The new value. + * @param longValue The new value. * @return an {@code AnyValue} with a int value. */ - public static AnyValue intAnyValue(int intValue) { - return AnyValueInt.create(intValue); + public static AnyValue longAnyValue(long longValue) { + return AnyValueLong.create(longValue); } - public int getIntValue() { + public long getLongValue() { throw new UnsupportedOperationException( String.format("This type can only return %s data", getType().name())); } @@ -186,20 +186,20 @@ public final Type getType() { @Immutable @AutoValue - abstract static class AnyValueInt extends AnyValue { - AnyValueInt() {} + abstract static class AnyValueLong extends AnyValue { + AnyValueLong() {} - static AnyValue create(int intValue) { - return new AutoValue_AnyValue_AnyValueInt(intValue); + static AnyValue create(long longValue) { + return new AutoValue_AnyValue_AnyValueLong(longValue); } @Override public final Type getType() { - return Type.INT; + return Type.INT64; } @Override - public abstract int getIntValue(); + public abstract long getLongValue(); } @Immutable From d8696e95c094a80f70b2833d87420c8b9cee1fa4 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 10 Sep 2020 14:59:07 -0700 Subject: [PATCH 14/20] address code review comments rename Exporter to LogExporter, move to 'export' package rename LogExporter.accept(data) to LogExporter.export(data) decrease flush interval from 5000 to 200ms return CompletableResultCode from LogExporter.shutdown() change traceId and spanId types to String from byte[] --- .../io/opentelemetry/common/AnyValue.java | 2 +- .../logging/BatchLogProcessor.java | 40 ++++++++++--------- .../opentelemetry/logging/api/LogRecord.java | 14 +++---- .../LogExporter.java} | 9 +++-- .../logging/LogSinkSdkProviderTest.java | 24 ++++++----- 5 files changed, 47 insertions(+), 42 deletions(-) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/{Exporter.java => export/LogExporter.java} (78%) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java index 5c48a2938f4..cba497f7302 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java @@ -28,7 +28,7 @@ * {@code kvlist}. represented through {@code AnyValue.Type}. A {@code array} or a {@code kvlist} * can in turn hold other {@code AnyValue} instances, allowing for mapping to JSON-like structures. * - * @since 0.8.0 + * @since 0.9.0 */ @Immutable public abstract class AnyValue { diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java index 26bce5301be..77e7fd8fd75 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java @@ -19,9 +19,9 @@ import io.opentelemetry.OpenTelemetry; import io.opentelemetry.common.Labels; import io.opentelemetry.internal.Utils; -import io.opentelemetry.logging.api.Exporter; import io.opentelemetry.logging.api.LogProcessor; import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.export.LogExporter; import io.opentelemetry.metrics.LongCounter; import io.opentelemetry.metrics.LongCounter.BoundLongCounter; import io.opentelemetry.metrics.Meter; @@ -59,16 +59,16 @@ private BatchLogProcessor( long scheduleDelayMillis, int maxExportBatchSize, long exporterTimeout, - Exporter exporter) { + LogExporter logExporter) { this.worker = new Worker( - maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeout, exporter); + maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeout, logExporter); this.workerThread = new DaemonThreadFactory(WORKER_THREAD_NAME).newThread(worker); this.workerThread.start(); } - public static Builder builder(Exporter exporter) { - return new Builder(exporter); + public static Builder builder(LogExporter logExporter) { + return new Builder(logExporter); } @Override @@ -118,7 +118,7 @@ private static class Worker implements Runnable { private final ArrayList logRecords; private final int maxExportBatchSize; private final ExecutorService executor; - private final Exporter exporter; + private final LogExporter logExporter; private final Timer timer = new Timer(TIMER_THREAD_NAME, /* isDaemon= */ true); private final long exporterTimeout; @@ -127,13 +127,13 @@ private Worker( long scheduleDelayMillis, int maxExportBatchSize, long exporterTimeout, - Exporter exporter) { + LogExporter logExporter) { this.maxQueueSize = maxQueueSize; this.maxExportBatchSize = maxExportBatchSize; this.exporterTimeout = exporterTimeout; this.scheduleDelayMillis = scheduleDelayMillis; - this.exporter = exporter; + this.logExporter = logExporter; this.logRecords = new ArrayList<>(maxQueueSize); // We should be able to drain a full queue without dropping a batch int exportQueueSize = maxQueueSize / maxExportBatchSize; @@ -190,7 +190,7 @@ private void onBatchExport(final List logDataForExport) { new Callable() { @Override public CompletableResultCode call() { - return exporter.accept(logDataForExport); + return logExporter.export(logDataForExport); } }); @@ -246,7 +246,7 @@ public void addLogRecord(LogRecord record) { private void shutdown() { forceFlush(); timer.cancel(); - exporter.shutdown(); + logExporter.shutdown(); } private void forceFlush() { @@ -260,35 +260,39 @@ private void forceFlush() { } static class Builder extends ConfigBuilder { - /* @VisibleForTesting */ static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 5000; + static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 200; static final int DEFAULT_MAX_QUEUE_SIZE = 2048; static final int DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; static final long DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; - private final Exporter exporter; + private final LogExporter logExporter; private long scheduleDelayMillis = DEFAULT_SCHEDULE_DELAY_MILLIS; private int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE; private int maxExportBatchSize = DEFAULT_MAX_EXPORT_BATCH_SIZE; private long exporterTimeoutMillis = DEFAULT_EXPORT_TIMEOUT_MILLIS; - private Builder(Exporter exporter) { - this.exporter = Utils.checkNotNull(exporter, "Exporter argument can not be null"); + private Builder(LogExporter logExporter) { + this.logExporter = Utils.checkNotNull(logExporter, "Exporter argument can not be null"); } - public Builder newBuilder(Exporter exporter) { - return new Builder(exporter); + public Builder newBuilder(LogExporter logExporter) { + return new Builder(logExporter); } public BatchLogProcessor build() { return new BatchLogProcessor( - maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeoutMillis, exporter); + maxQueueSize, + scheduleDelayMillis, + maxExportBatchSize, + exporterTimeoutMillis, + logExporter); } /** * Sets the delay interval between two consecutive exports. The actual interval may be shorter * if the batch size is getting larger than {@code maxQueuedSpans / 2}. * - *

Default value is {@code 5000}ms. + *

Default value is {@code 250}ms. * * @param scheduleDelayMillis the delay interval between two consecutive exports. * @return this. diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java index 2c791b1c7f1..0ca446122f9 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java @@ -33,11 +33,9 @@ public abstract class LogRecord { abstract long getTimeUnixNano(); - @SuppressWarnings("mutable") - abstract byte[] getTraceId(); + abstract String getTraceId(); - @SuppressWarnings("mutable") - abstract byte[] getSpanId(); + abstract String getSpanId(); abstract int getFlags(); @@ -94,8 +92,8 @@ public int getSeverityNumber() { public static class Builder { private long timeUnixNano; - private byte[] traceId = new byte[0]; - private byte[] spanId = new byte[0]; + private String traceId = ""; + private String spanId = ""; private int flags; private Severity severity = Severity.UNDEFINED_SEVERITY_NUMBER; private String severityText; @@ -112,12 +110,12 @@ public Builder withUnixTimeMillis(long timestamp) { return withUnixTimeNano(TimeUnit.MILLISECONDS.toNanos(timestamp)); } - public Builder withTraceId(byte[] traceId) { + public Builder withTraceId(String traceId) { this.traceId = traceId; return this; } - public Builder withSpanId(byte[] spanId) { + public Builder withSpanId(String spanId) { this.spanId = spanId; return this; } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/export/LogExporter.java similarity index 78% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/export/LogExporter.java index d0e12794266..add4e063eb3 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/Exporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/export/LogExporter.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package io.opentelemetry.logging.api; +package io.opentelemetry.logging.api.export; +import io.opentelemetry.logging.api.LogRecord; import io.opentelemetry.sdk.common.CompletableResultCode; import java.util.Collection; @@ -23,8 +24,8 @@ * An exporter is responsible for taking a list of {@link LogRecord}s and transmitting them to their * ultimate destination. */ -public interface Exporter { - CompletableResultCode accept(Collection records); +public interface LogExporter { + CompletableResultCode export(Collection records); - void shutdown(); + CompletableResultCode shutdown(); } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java index b2df762faae..4a492a74882 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java @@ -20,11 +20,11 @@ import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; -import io.opentelemetry.logging.api.Exporter; import io.opentelemetry.logging.api.LogProcessor; import io.opentelemetry.logging.api.LogRecord; import io.opentelemetry.logging.api.LogRecord.Severity; import io.opentelemetry.logging.api.LogSink; +import io.opentelemetry.logging.api.export.LogExporter; import io.opentelemetry.sdk.common.CompletableResultCode; import java.util.ArrayList; import java.util.Collection; @@ -36,13 +36,13 @@ @RunWith(JUnit4.class) public class LogSinkSdkProviderTest { - private static class TestExporter implements Exporter { + private static class TestLogExporter implements LogExporter { private final ArrayList records = new ArrayList<>(); @Nullable private Runnable onCall = null; private int callCount = 0; @Override - public CompletableResultCode accept(Collection records) { + public CompletableResultCode export(Collection records) { this.records.addAll(records); callCount++; if (onCall != null) { @@ -52,7 +52,9 @@ public CompletableResultCode accept(Collection records) { } @Override - public void shutdown() {} + public CompletableResultCode shutdown() { + return new CompletableResultCode().succeed(); + } } private static LogRecord createLog(LogRecord.Severity severity, String message) { @@ -65,7 +67,7 @@ private static LogRecord createLog(LogRecord.Severity severity, String message) @Test public void testLogSinkSdkProvider() { - TestExporter exporter = new TestExporter(); + TestLogExporter exporter = new TestLogExporter(); LogProcessor processor = BatchLogProcessor.builder(exporter).build(); LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); provider.addLogProcessor(processor); @@ -78,7 +80,7 @@ public void testLogSinkSdkProvider() { @Test public void testBatchSize() { - TestExporter exporter = new TestExporter(); + TestLogExporter exporter = new TestLogExporter(); LogProcessor processor = BatchLogProcessor.builder(exporter) .setScheduleDelayMillis(3000) // Long enough to not be in play @@ -93,16 +95,16 @@ public void testBatchSize() { sink.offer(createLog(Severity.WARN, "test #" + i)); } // Ensure that more than batch size kicks off a flush - await().atMost(120, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); + await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); // Ensure that everything gets through provider.forceFlush(); - await().atMost(120, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() == 7); - assertThat(exporter.callCount).isEqualTo(2); + await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() == 7); + assertThat(exporter.callCount).isGreaterThanOrEqualTo(2); } @Test public void testNoBlocking() { - TestExporter exporter = new TestExporter(); + TestLogExporter exporter = new TestLogExporter(); exporter.onCall = () -> { try { @@ -127,7 +129,7 @@ public void testNoBlocking() { } long end = System.currentTimeMillis(); assertThat(end - start).isLessThan(250L); - await().atMost(510, TimeUnit.MILLISECONDS).until(() -> exporter.callCount == 2); + await().atMost(1000, TimeUnit.MILLISECONDS).until(() -> exporter.callCount == 2); assertThat(exporter.records.size()) // Two exporter batches .isGreaterThan(5) .isLessThanOrEqualTo(10); From 64b30efe0a9b3ea11effb4f29f0d95e82401a380 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Fri, 11 Sep 2020 10:11:23 -0700 Subject: [PATCH 15/20] implement BatchLogProcessor.fromConfigMap --- .../logging/BatchLogProcessor.java | 33 ++++++++-- .../logging/BatchLogProcessorTest.java | 63 +++++++++++++++++++ 2 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java index 77e7fd8fd75..bf757cac920 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java @@ -234,7 +234,6 @@ public void addLogRecord(LogRecord record) { synchronized (monitor) { if (logRecords.size() >= maxQueueSize) { droppedRecordCounter.add(1); - // FIXME: call callback } logRecords.add(record); if (logRecords.size() >= maxExportBatchSize) { @@ -260,10 +259,15 @@ private void forceFlush() { } static class Builder extends ConfigBuilder { - static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 200; - static final int DEFAULT_MAX_QUEUE_SIZE = 2048; - static final int DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; - static final long DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; + private static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 200; + private static final int DEFAULT_MAX_QUEUE_SIZE = 2048; + private static final int DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; + private static final long DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; + + private static final String KEY_SCHEDULE_DELAY_MILLIS = "otel.log.schedule.delay"; + private static final String KEY_MAX_QUEUE_SIZE = "otel.log.max.queue"; + private static final String KEY_MAX_EXPORT_BATCH_SIZE = "otel.log.max.export.batch"; + private static final String KEY_EXPORT_TIMEOUT_MILLIS = "otel.log.export.timeout"; private final LogExporter logExporter; private long scheduleDelayMillis = DEFAULT_SCHEDULE_DELAY_MILLIS; @@ -370,7 +374,24 @@ int getMaxExportBatchSize() { @Override protected Builder fromConfigMap( Map configMap, NamingConvention namingConvention) { - return null; + configMap = namingConvention.normalize(configMap); + Long longValue = getLongProperty(KEY_SCHEDULE_DELAY_MILLIS, configMap); + if (longValue != null) { + this.setScheduleDelayMillis(longValue); + } + Integer intValue = getIntProperty(KEY_MAX_QUEUE_SIZE, configMap); + if (intValue != null) { + this.setMaxQueueSize(intValue); + } + intValue = getIntProperty(KEY_MAX_EXPORT_BATCH_SIZE, configMap); + if (intValue != null) { + this.setMaxExportBatchSize(intValue); + } + intValue = getIntProperty(KEY_EXPORT_TIMEOUT_MILLIS, configMap); + if (intValue != null) { + this.setExporterTimeoutMillis(intValue); + } + return this; } } } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java new file mode 100644 index 00000000000..1916b8399de --- /dev/null +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.logging; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; + +import io.opentelemetry.logging.api.LogRecord; +import io.opentelemetry.logging.api.export.LogExporter; +import io.opentelemetry.sdk.common.CompletableResultCode; +import java.util.Collection; +import java.util.Properties; +import org.junit.Test; + +public class BatchLogProcessorTest { + + @Test + public void testBuilder() { + Properties props = new Properties(); + long delay = 1234L; + int queue = 2345; + int batch = 521; + int timeout = 5432; + + props.put("otel.log.schedule.delay", Long.toString(delay)); + props.put("otel.log.max.queue", Integer.toString(queue)); + props.put("otel.log.max.export.batch", Integer.toString(batch)); + props.put("otel.log.export.timeout", Integer.toString(timeout)); + + BatchLogProcessor.Builder builder = + BatchLogProcessor.builder( + new LogExporter() { + @Override + public CompletableResultCode export(Collection records) { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + return CompletableResultCode.ofSuccess(); + } + }); + + builder.readProperties(props); + assertThat(builder.getScheduleDelayMillis()).isEqualTo(delay); + assertThat(builder.getMaxQueueSize()).isEqualTo(queue); + assertThat(builder.getMaxExportBatchSize()).isEqualTo(batch); + assertThat(builder.getExporterTimeoutMillis()).isEqualTo(timeout); + } +} From c44c10c872a83551ba6a401dad9e4a10170cca13 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Fri, 11 Sep 2020 13:06:40 -0700 Subject: [PATCH 16/20] rearrange packages --- sdk_extensions/logging/support/build.gradle | 2 +- .../api => sdk/logging}/LogProcessor.java | 3 ++- .../{logging/api => sdk/logging}/LogSink.java | 4 +++- .../{ => sdk}/logging/LogSinkSdkProvider.java | 6 ++--- .../logging/data}/AnyValue.java | 2 +- .../api => sdk/logging/data}/LogRecord.java | 3 +-- .../logging/export}/BatchLogProcessor.java | 24 +++++++++++-------- .../logging}/export/LogExporter.java | 4 ++-- .../logging/LogSinkSdkProviderTest.java | 13 +++++----- .../logging/sdk}/BatchLogProcessorTest.java | 7 +++--- 10 files changed, 36 insertions(+), 32 deletions(-) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{logging/api => sdk/logging}/LogProcessor.java (92%) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{logging/api => sdk/logging}/LogSink.java (91%) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{ => sdk}/logging/LogSinkSdkProvider.java (91%) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{common => sdk/logging/data}/AnyValue.java (99%) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{logging/api => sdk/logging/data}/LogRecord.java (98%) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{logging => sdk/logging/export}/BatchLogProcessor.java (96%) rename sdk_extensions/logging/support/src/main/java/io/opentelemetry/{logging/api => sdk/logging}/export/LogExporter.java (90%) rename sdk_extensions/logging/support/src/test/java/io/opentelemetry/{ => sdk}/logging/LogSinkSdkProviderTest.java (92%) rename sdk_extensions/logging/support/src/test/java/io/opentelemetry/{logging => sdk/logging/sdk}/BatchLogProcessorTest.java (90%) diff --git a/sdk_extensions/logging/support/build.gradle b/sdk_extensions/logging/support/build.gradle index 89050f91f0c..e00c0029e9e 100644 --- a/sdk_extensions/logging/support/build.gradle +++ b/sdk_extensions/logging/support/build.gradle @@ -6,7 +6,7 @@ plugins { } description = 'OpenTelemetry Contrib Logging Support' -ext.moduleName = "io.opentelemetry.extensions.logging.support" +ext.moduleName = "io.opentelemetry.sdk.extensions.logging" dependencies { api project(':opentelemetry-sdk') diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java similarity index 92% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogProcessor.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java index c4cfc0f0f3e..bd27950eb43 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package io.opentelemetry.logging.api; +package io.opentelemetry.sdk.logging; +import io.opentelemetry.sdk.logging.data.LogRecord; import io.opentelemetry.sdk.trace.TracerSdkProvider; public interface LogProcessor { diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogSink.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java similarity index 91% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogSink.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java index 6592baa7d02..b6391a54a8c 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogSink.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java @@ -14,7 +14,9 @@ * limitations under the License. */ -package io.opentelemetry.logging.api; +package io.opentelemetry.sdk.logging; + +import io.opentelemetry.sdk.logging.data.LogRecord; /** A LogSink accepts logging records for transmission to an aggregator or log processing system. */ public interface LogSink { diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java similarity index 91% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java index 8af03b789ca..659371cbe45 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/LogSinkSdkProvider.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java @@ -14,12 +14,10 @@ * limitations under the License. */ -package io.opentelemetry.logging; +package io.opentelemetry.sdk.logging; import io.opentelemetry.internal.Utils; -import io.opentelemetry.logging.api.LogProcessor; -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.LogSink; +import io.opentelemetry.sdk.logging.data.LogRecord; import java.util.ArrayList; import java.util.List; diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java similarity index 99% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java index cba497f7302..31a672b47c6 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/common/AnyValue.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.opentelemetry.common; +package io.opentelemetry.sdk.logging.data; import com.google.auto.value.AutoValue; import java.util.List; diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java similarity index 98% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java index 0ca446122f9..99602745761 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/LogRecord.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java @@ -14,10 +14,9 @@ * limitations under the License. */ -package io.opentelemetry.logging.api; +package io.opentelemetry.sdk.logging.data; import com.google.auto.value.AutoValue; -import io.opentelemetry.common.AnyValue; import io.opentelemetry.common.AttributeValue; import java.util.HashMap; import java.util.Map; diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java similarity index 96% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java index bf757cac920..c2e6c580d34 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/BatchLogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java @@ -14,20 +14,19 @@ * limitations under the License. */ -package io.opentelemetry.logging; +package io.opentelemetry.sdk.logging.export; import io.opentelemetry.OpenTelemetry; import io.opentelemetry.common.Labels; import io.opentelemetry.internal.Utils; -import io.opentelemetry.logging.api.LogProcessor; -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.export.LogExporter; import io.opentelemetry.metrics.LongCounter; import io.opentelemetry.metrics.LongCounter.BoundLongCounter; import io.opentelemetry.metrics.Meter; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.common.DaemonThreadFactory; import io.opentelemetry.sdk.common.export.ConfigBuilder; +import io.opentelemetry.sdk.logging.LogProcessor; +import io.opentelemetry.sdk.logging.data.LogRecord; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -258,7 +257,7 @@ private void forceFlush() { } } - static class Builder extends ConfigBuilder { + public static class Builder extends ConfigBuilder { private static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 200; private static final int DEFAULT_MAX_QUEUE_SIZE = 2048; private static final int DEFAULT_MAX_EXPORT_BATCH_SIZE = 512; @@ -275,7 +274,7 @@ static class Builder extends ConfigBuilder { private int maxExportBatchSize = DEFAULT_MAX_EXPORT_BATCH_SIZE; private long exporterTimeoutMillis = DEFAULT_EXPORT_TIMEOUT_MILLIS; - private Builder(LogExporter logExporter) { + public Builder(LogExporter logExporter) { this.logExporter = Utils.checkNotNull(logExporter, "Exporter argument can not be null"); } @@ -283,6 +282,11 @@ public Builder newBuilder(LogExporter logExporter) { return new Builder(logExporter); } + /** + * Build a BatchLogProcessor. + * + * @return configured processor + */ public BatchLogProcessor build() { return new BatchLogProcessor( maxQueueSize, @@ -307,7 +311,7 @@ public BatchLogProcessor.Builder setScheduleDelayMillis(long scheduleDelayMillis return this; } - long getScheduleDelayMillis() { + public long getScheduleDelayMillis() { return scheduleDelayMillis; } @@ -325,7 +329,7 @@ public Builder setExporterTimeoutMillis(int exporterTimeoutMillis) { return this; } - long getExporterTimeoutMillis() { + public long getExporterTimeoutMillis() { return exporterTimeoutMillis; } @@ -347,7 +351,7 @@ public Builder setMaxQueueSize(int maxQueueSize) { return this; } - int getMaxQueueSize() { + public int getMaxQueueSize() { return maxQueueSize; } @@ -367,7 +371,7 @@ public Builder setMaxExportBatchSize(int maxExportBatchSize) { return this; } - int getMaxExportBatchSize() { + public int getMaxExportBatchSize() { return maxExportBatchSize; } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/export/LogExporter.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java similarity index 90% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/export/LogExporter.java rename to sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java index add4e063eb3..a62fb4cf6ea 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/logging/api/export/LogExporter.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java @@ -14,10 +14,10 @@ * limitations under the License. */ -package io.opentelemetry.logging.api.export; +package io.opentelemetry.sdk.logging.export; -import io.opentelemetry.logging.api.LogRecord; import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logging.data.LogRecord; import java.util.Collection; /** diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java similarity index 92% rename from sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java rename to sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java index 4a492a74882..b5bee625ca7 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/LogSinkSdkProviderTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java @@ -14,18 +14,17 @@ * limitations under the License. */ -package io.opentelemetry.logging; +package io.opentelemetry.sdk.logging; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.awaitility.Awaitility.await; -import io.opentelemetry.logging.api.LogProcessor; -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.LogRecord.Severity; -import io.opentelemetry.logging.api.LogSink; -import io.opentelemetry.logging.api.export.LogExporter; import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logging.data.LogRecord; +import io.opentelemetry.sdk.logging.data.LogRecord.Severity; +import io.opentelemetry.sdk.logging.export.BatchLogProcessor; +import io.opentelemetry.sdk.logging.export.LogExporter; import java.util.ArrayList; import java.util.Collection; import java.util.concurrent.TimeUnit; @@ -74,7 +73,7 @@ public void testLogSinkSdkProvider() { LogSink sink = provider.get("test", "0.1a"); sink.offer(createLog(Severity.ERROR, "test")); provider.forceFlush(); - await().atMost(120, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); + await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); assertThat(exporter.records.size()).isEqualTo(1); } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java similarity index 90% rename from sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java rename to sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java index 1916b8399de..5069b1741fa 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/logging/BatchLogProcessorTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java @@ -14,13 +14,14 @@ * limitations under the License. */ -package io.opentelemetry.logging; +package io.opentelemetry.sdk.logging.sdk; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; -import io.opentelemetry.logging.api.LogRecord; -import io.opentelemetry.logging.api.export.LogExporter; import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logging.data.LogRecord; +import io.opentelemetry.sdk.logging.export.BatchLogProcessor; +import io.opentelemetry.sdk.logging.export.LogExporter; import java.util.Collection; import java.util.Properties; import org.junit.Test; From e77b91aac6b028820fc1a6f6c400dc41ed9ade0e Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Wed, 16 Sep 2020 06:34:29 -0700 Subject: [PATCH 17/20] add readme --- sdk_extensions/logging/support/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 sdk_extensions/logging/support/README.md diff --git a/sdk_extensions/logging/support/README.md b/sdk_extensions/logging/support/README.md new file mode 100644 index 00000000000..70affb87dfc --- /dev/null +++ b/sdk_extensions/logging/support/README.md @@ -0,0 +1,6 @@ +# OpenTelemetry Experimental Logging Support + +This project contains experimental support for transport of logs via OpenTelemetry. The API +presented is intended for the use of logging library adapters to enable resource and request +correlation with other observability signals and transport of logs through the OpenTelemetry +collector. From 7b238b4cc30402c8ce6677c9fd330c3ca5a83166 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 17 Sep 2020 14:25:17 -0700 Subject: [PATCH 18/20] respond to review feedback, refactored BatchLogProcessor --- .../sdk/logging/LogProcessor.java | 10 +- .../io/opentelemetry/sdk/logging/LogSink.java | 7 - .../sdk/logging/LogSinkSdkProvider.java | 33 ++- .../sdk/logging/data/LogRecord.java | 24 +- .../sdk/logging/export/BatchLogProcessor.java | 270 ++++++++---------- .../sdk/logging/LogSinkSdkProviderTest.java | 22 +- 6 files changed, 172 insertions(+), 194 deletions(-) diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java index bd27950eb43..7f9ac7bc7dc 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java @@ -16,6 +16,7 @@ package io.opentelemetry.sdk.logging; +import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.logging.data.LogRecord; import io.opentelemetry.sdk.trace.TracerSdkProvider; @@ -26,15 +27,14 @@ public interface LogProcessor { /** * Called when {@link TracerSdkProvider#shutdown()} is called. * - *

Implementations must ensure that all span events are processed before returning. + * @return result */ - void shutdown(); + CompletableResultCode shutdown(); /** * Processes all span events that have not yet been processed. * - *

This method is executed synchronously on the calling thread, and should not throw - * exceptions. + * @return result */ - void forceFlush(); + CompletableResultCode forceFlush(); } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java index b6391a54a8c..3b5ce9b1df0 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java @@ -26,11 +26,4 @@ public interface LogSink { * @param record record to transmit */ void offer(LogRecord record); - - /** - * Convenience method for creating a record to be transmitted. - * - * @return builder instance - */ - LogRecord.Builder buildRecord(); } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java index 659371cbe45..0d21be4de0b 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java @@ -17,8 +17,10 @@ package io.opentelemetry.sdk.logging; import io.opentelemetry.internal.Utils; +import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.logging.data.LogRecord; import java.util.ArrayList; +import java.util.Collection; import java.util.List; public class LogSinkSdkProvider { @@ -36,11 +38,30 @@ public void addLogProcessor(LogProcessor processor) { processors.add(Utils.checkNotNull(processor, "Processor can not be null")); } - /** Flushes all attached processors. */ - public void forceFlush() { + /** + * Flushes all attached processors. + * + * @return result + */ + public CompletableResultCode forceFlush() { + final List processorResults = new ArrayList<>(processors.size()); for (LogProcessor processor : processors) { - processor.forceFlush(); + processorResults.add(processor.forceFlush()); } + return CompletableResultCode.ofAll(processorResults); + } + + /** + * Shut down of provider and associated processors. + * + * @return result + */ + public CompletableResultCode shutdown() { + Collection processorResults = new ArrayList<>(processors.size()); + for (LogProcessor processor : processors) { + processorResults.add(processor.shutdown()); + } + return CompletableResultCode.ofAll(processorResults); } private class SdkLogSink implements LogSink { @@ -50,15 +71,9 @@ public void offer(LogRecord record) { processor.addLogRecord(record); } } - - @Override - public LogRecord.Builder buildRecord() { - return new LogRecord.Builder(); - } } public static class Builder { - public LogSinkSdkProvider build() { return new LogSinkSdkProvider(); } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java index 99602745761..fadf335688e 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java @@ -100,56 +100,56 @@ public static class Builder { private AnyValue body = AnyValue.stringAnyValue(""); private final Map attributes = new HashMap<>(); - public Builder withUnixTimeNano(long timestamp) { + public Builder setUnixTimeNano(long timestamp) { this.timeUnixNano = timestamp; return this; } - public Builder withUnixTimeMillis(long timestamp) { - return withUnixTimeNano(TimeUnit.MILLISECONDS.toNanos(timestamp)); + public Builder setUnixTimeMillis(long timestamp) { + return setUnixTimeNano(TimeUnit.MILLISECONDS.toNanos(timestamp)); } - public Builder withTraceId(String traceId) { + public Builder setTraceId(String traceId) { this.traceId = traceId; return this; } - public Builder withSpanId(String spanId) { + public Builder setSpanId(String spanId) { this.spanId = spanId; return this; } - public Builder withFlags(int flags) { + public Builder setFlags(int flags) { this.flags = flags; return this; } - public Builder withSeverity(Severity severity) { + public Builder setSeverity(Severity severity) { this.severity = severity; return this; } - public Builder withSeverityText(String severityText) { + public Builder setSeverityText(String severityText) { this.severityText = severityText; return this; } - public Builder withName(String name) { + public Builder setName(String name) { this.name = name; return this; } - public Builder withBody(AnyValue body) { + public Builder setBody(AnyValue body) { this.body = body; return this; } - public Builder withBody(String body) { + public Builder setBody(String body) { this.body = AnyValue.stringAnyValue(body); return this; } - public Builder withAttributes(Map attributes) { + public Builder setAttributes(Map attributes) { this.attributes.putAll(attributes); return this; } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java index c2e6c580d34..ec4e102c446 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java @@ -28,27 +28,15 @@ import io.opentelemetry.sdk.logging.LogProcessor; import io.opentelemetry.sdk.logging.data.LogRecord; import java.util.ArrayList; -import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; +import java.util.concurrent.atomic.AtomicReference; public class BatchLogProcessor implements LogProcessor { private static final String WORKER_THREAD_NAME = BatchLogProcessor.class.getSimpleName() + "_WorkerThread"; - private static final String TIMER_THREAD_NAME = - BatchLogProcessor.class.getSimpleName() + "_TimerThread"; private final Worker worker; private final Thread workerThread; @@ -57,11 +45,15 @@ private BatchLogProcessor( int maxQueueSize, long scheduleDelayMillis, int maxExportBatchSize, - long exporterTimeout, + long exporterTimeoutMillis, LogExporter logExporter) { this.worker = new Worker( - maxQueueSize, scheduleDelayMillis, maxExportBatchSize, exporterTimeout, logExporter); + logExporter, + scheduleDelayMillis, + maxExportBatchSize, + exporterTimeoutMillis, + new ArrayBlockingQueue(maxQueueSize)); this.workerThread = new DaemonThreadFactory(WORKER_THREAD_NAME).newThread(worker); this.workerThread.start(); } @@ -76,184 +68,164 @@ public void addLogRecord(LogRecord record) { } @Override - public void shutdown() { + public CompletableResultCode shutdown() { workerThread.interrupt(); - worker.shutdown(); + return worker.shutdown(); } @Override - public void forceFlush() { - worker.forceFlush(); + public CompletableResultCode forceFlush() { + return worker.forceFlush(); } private static class Worker implements Runnable { static { Meter meter = OpenTelemetry.getMeter("io.opentelemetry.sdk.logging"); - LongCounter logProcessorErrors = + LongCounter logRecordsProcessed = meter - .longCounterBuilder("logProcessorErrors") + .longCounterBuilder("logRecordsProcessed") .setUnit("1") - .setDescription("Number of errors encountered while processing logs") + .setDescription("Number of records processed") .build(); - Labels.Builder builder = - Labels.of("logProcessorType", BatchLogProcessor.class.getName()).toBuilder(); + successCounter = logRecordsProcessed.bind(Labels.of("result", "success")); exporterFailureCounter = - logProcessorErrors.bind(builder.setLabel("errorType", "exporter failure").build()); - exporterBusyCounter = - logProcessorErrors.bind(builder.setLabel("errorType", "exporter busy").build()); - droppedRecordCounter = - logProcessorErrors.bind( - builder.setLabel("errorType", "dropped record - queue full").build()); + logRecordsProcessed.bind( + Labels.of("result", "dropped record", "cause", "exporter failure")); + queueFullRecordCounter = + logRecordsProcessed.bind(Labels.of("result", "dropped record", "cause", "queue full")); } private static final BoundLongCounter exporterFailureCounter; - private static final BoundLongCounter exporterBusyCounter; - private static final BoundLongCounter droppedRecordCounter; - private static final Logger logger = Logger.getLogger(Worker.class.getName()); - - private final Object monitor = new Object(); - private final int maxQueueSize; - private final long scheduleDelayMillis; - private final ArrayList logRecords; + private static final BoundLongCounter queueFullRecordCounter; + private static final BoundLongCounter successCounter; + + private final long scheduleDelayNanos; private final int maxExportBatchSize; - private final ExecutorService executor; private final LogExporter logExporter; - private final Timer timer = new Timer(TIMER_THREAD_NAME, /* isDaemon= */ true); - private final long exporterTimeout; + private final long exporterTimeoutMillis; + private final ArrayList batch; + private final BlockingQueue queue; + + private final AtomicReference flushRequested = new AtomicReference<>(); + private volatile boolean continueWork = true; + private long nextExportTime; private Worker( - int maxQueueSize, + LogExporter logExporter, long scheduleDelayMillis, int maxExportBatchSize, - long exporterTimeout, - LogExporter logExporter) { - this.maxQueueSize = maxQueueSize; - this.maxExportBatchSize = maxExportBatchSize; - - this.exporterTimeout = exporterTimeout; - this.scheduleDelayMillis = scheduleDelayMillis; + long exporterTimeoutMillis, + BlockingQueue queue) { this.logExporter = logExporter; - this.logRecords = new ArrayList<>(maxQueueSize); - // We should be able to drain a full queue without dropping a batch - int exportQueueSize = maxQueueSize / maxExportBatchSize; - RejectedExecutionHandler h = - new RejectedExecutionHandler() { - @Override - public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - exporterBusyCounter.add(1); - } - }; - this.executor = - new ThreadPoolExecutor( - 1, - 1, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue(exportQueueSize), - h); + this.maxExportBatchSize = maxExportBatchSize; + this.exporterTimeoutMillis = exporterTimeoutMillis; + this.scheduleDelayNanos = TimeUnit.MILLISECONDS.toNanos(scheduleDelayMillis); + this.queue = queue; + this.batch = new ArrayList<>(this.maxExportBatchSize); } @Override public void run() { - ArrayList logsCopy; - while (!Thread.currentThread().isInterrupted()) { - synchronized (monitor) { - if (this.logRecords.size() < maxExportBatchSize) { - do { - try { - monitor.wait(scheduleDelayMillis); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } while (this.logRecords.isEmpty()); + updateNextExportTime(); + + while (continueWork) { + if (flushRequested.get() != null) { + flush(); + } + + try { + LogRecord lastElement = queue.poll(100, TimeUnit.MILLISECONDS); + if (lastElement != null) { + batch.add(lastElement); } - logsCopy = new ArrayList<>(this.logRecords); - logRecords.clear(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + + if (batch.size() >= maxExportBatchSize || System.nanoTime() >= nextExportTime) { + exportCurrentBatch(); + updateNextExportTime(); } - exportBatches(logsCopy); } } - private void exportBatches(ArrayList recordsToShip) { - for (int i = 0; i < recordsToShip.size(); ) { - int lastIndexToTake = Math.min(i + maxExportBatchSize, recordsToShip.size()); - onBatchExport(createLogDataForExport(recordsToShip, i, lastIndexToTake)); - i = lastIndexToTake; + private void flush() { + int recordsToFlush = queue.size(); + while (recordsToFlush > 0) { + LogRecord record = queue.poll(); + assert record != null; + batch.add(record); + recordsToFlush--; + if (batch.size() >= maxExportBatchSize) { + exportCurrentBatch(); + } } + exportCurrentBatch(); + CompletableResultCode result = flushRequested.get(); + assert result != null; + flushRequested.set(null); } - private void onBatchExport(final List logDataForExport) { - final Future f = - executor.submit( - new Callable() { - @Override - public CompletableResultCode call() { - return logExporter.export(logDataForExport); - } - }); - - timer.schedule( - new TimerTask() { - @Override - public void run() { - if (f.isDone()) { - try { - final CompletableResultCode result = f.get(0, TimeUnit.MILLISECONDS); - if (!result.isSuccess()) { - // This may mean that the export process has failed, or that it's still running - // but it's at the end of it's timeout. - logger.warning("log exporter has failed or timed out"); - exporterFailureCounter.add(1); - } - } catch (InterruptedException | ExecutionException e) { - logger.warning("log exporter failure:" + e.getLocalizedMessage()); - exporterFailureCounter.add(1); - } catch (TimeoutException e) { - logger.warning("log exporter has failed to return async result"); - exporterFailureCounter.add(1); - } - } - } - }, - exporterTimeout); + private void updateNextExportTime() { + nextExportTime = System.nanoTime() + scheduleDelayNanos; } - private static List createLogDataForExport( - ArrayList recordsToShip, int startIndex, int endIndex) { - List logDataBuffer = new ArrayList<>(endIndex - startIndex); - for (int i = startIndex; i < endIndex; i++) { - logDataBuffer.add(recordsToShip.get(i)); - recordsToShip.set(i, null); + private void exportCurrentBatch() { + if (batch.isEmpty()) { + return; } - return Collections.unmodifiableList(logDataBuffer); - } - public void addLogRecord(LogRecord record) { - synchronized (monitor) { - if (logRecords.size() >= maxQueueSize) { - droppedRecordCounter.add(1); - } - logRecords.add(record); - if (logRecords.size() >= maxExportBatchSize) { - monitor.notifyAll(); + try { + final CompletableResultCode result = logExporter.export(batch); + result.join(exporterTimeoutMillis, TimeUnit.MILLISECONDS); + if (result.isSuccess()) { + successCounter.add(batch.size()); + } else { + exporterFailureCounter.add(1); } + } catch (Exception t) { + exporterFailureCounter.add(batch.size()); + } finally { + batch.clear(); } } - private void shutdown() { - forceFlush(); - timer.cancel(); - logExporter.shutdown(); + private CompletableResultCode shutdown() { + final CompletableResultCode result = new CompletableResultCode(); + final CompletableResultCode flushResult = forceFlush(); + flushResult.whenComplete( + new Runnable() { + @Override + public void run() { + continueWork = false; + final CompletableResultCode shutdownResult = logExporter.shutdown(); + shutdownResult.whenComplete( + new Runnable() { + @Override + public void run() { + if (flushResult.isSuccess() && shutdownResult.isSuccess()) { + result.succeed(); + } else { + result.fail(); + } + } + }); + } + }); + return result; } - private void forceFlush() { - ArrayList logsCopy; - synchronized (monitor) { - logsCopy = new ArrayList<>(this.logRecords); - logRecords.clear(); + private CompletableResultCode forceFlush() { + CompletableResultCode flushResult = new CompletableResultCode(); + this.flushRequested.compareAndSet(null, flushResult); + return this.flushRequested.get(); + } + + public void addLogRecord(LogRecord record) { + if (!queue.offer(record)) { + queueFullRecordCounter.add(1); } - exportBatches(logsCopy); } } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java index b5bee625ca7..fea404b0b7c 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java @@ -58,9 +58,9 @@ public CompletableResultCode shutdown() { private static LogRecord createLog(LogRecord.Severity severity, String message) { return new LogRecord.Builder() - .withUnixTimeMillis(System.currentTimeMillis()) - .withSeverity(severity) - .withBody(message) + .setUnixTimeMillis(System.currentTimeMillis()) + .setSeverity(severity) + .setBody(message) .build(); } @@ -72,8 +72,7 @@ public void testLogSinkSdkProvider() { provider.addLogProcessor(processor); LogSink sink = provider.get("test", "0.1a"); sink.offer(createLog(Severity.ERROR, "test")); - provider.forceFlush(); - await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); + provider.forceFlush().join(500, TimeUnit.MILLISECONDS); assertThat(exporter.records.size()).isEqualTo(1); } @@ -96,8 +95,8 @@ public void testBatchSize() { // Ensure that more than batch size kicks off a flush await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); // Ensure that everything gets through - provider.forceFlush(); - await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() == 7); + CompletableResultCode result = provider.forceFlush(); + result.join(1, TimeUnit.SECONDS); assertThat(exporter.callCount).isGreaterThanOrEqualTo(2); } @@ -123,14 +122,13 @@ public void testNoBlocking() { LogSink sink = provider.get("test", "0.1a"); long start = System.currentTimeMillis(); - for (int i = 0; i < 700; i++) { + int testRecordCount = 700; + for (int i = 0; i < testRecordCount; i++) { sink.offer(createLog(Severity.WARN, "test #" + i)); } long end = System.currentTimeMillis(); assertThat(end - start).isLessThan(250L); - await().atMost(1000, TimeUnit.MILLISECONDS).until(() -> exporter.callCount == 2); - assertThat(exporter.records.size()) // Two exporter batches - .isGreaterThan(5) - .isLessThanOrEqualTo(10); + provider.forceFlush().join(1, TimeUnit.SECONDS); + assertThat(exporter.records.size()).isLessThan(testRecordCount); // We dropped records } } From d8a6d5ff08b4f380b86cf779472b4cab10fbbd9f Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Thu, 17 Sep 2020 15:42:33 -0700 Subject: [PATCH 19/20] added a couple tests --- .../sdk/logging/data/LogRecord.java | 5 ++ .../sdk/logging/LogSinkSdkProviderTest.java | 64 ++++++++++--------- .../logging/sdk/BatchLogProcessorTest.java | 25 +++++++- .../sdk/logging/util/TestLogExporter.java | 58 +++++++++++++++++ .../sdk/logging/util/TestLogProcessor.java | 58 +++++++++++++++++ 5 files changed, 179 insertions(+), 31 deletions(-) create mode 100644 sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java create mode 100644 sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java index fadf335688e..662fd37c123 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java +++ b/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java @@ -30,6 +30,7 @@ */ @AutoValue public abstract class LogRecord { + abstract long getTimeUnixNano(); abstract String getTraceId(); @@ -89,6 +90,10 @@ public int getSeverityNumber() { } } + public static Builder builder() { + return new Builder(); + } + public static class Builder { private long timeUnixNano; private String traceId = ""; diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java index fea404b0b7c..055f10ae9c6 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java @@ -24,37 +24,15 @@ import io.opentelemetry.sdk.logging.data.LogRecord; import io.opentelemetry.sdk.logging.data.LogRecord.Severity; import io.opentelemetry.sdk.logging.export.BatchLogProcessor; -import io.opentelemetry.sdk.logging.export.LogExporter; -import java.util.ArrayList; -import java.util.Collection; +import io.opentelemetry.sdk.logging.util.TestLogExporter; +import io.opentelemetry.sdk.logging.util.TestLogProcessor; import java.util.concurrent.TimeUnit; -import javax.annotation.Nullable; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class LogSinkSdkProviderTest { - private static class TestLogExporter implements LogExporter { - private final ArrayList records = new ArrayList<>(); - @Nullable private Runnable onCall = null; - private int callCount = 0; - - @Override - public CompletableResultCode export(Collection records) { - this.records.addAll(records); - callCount++; - if (onCall != null) { - onCall.run(); - } - return null; - } - - @Override - public CompletableResultCode shutdown() { - return new CompletableResultCode().succeed(); - } - } private static LogRecord createLog(LogRecord.Severity severity, String message) { return new LogRecord.Builder() @@ -73,7 +51,7 @@ public void testLogSinkSdkProvider() { LogSink sink = provider.get("test", "0.1a"); sink.offer(createLog(Severity.ERROR, "test")); provider.forceFlush().join(500, TimeUnit.MILLISECONDS); - assertThat(exporter.records.size()).isEqualTo(1); + assertThat(exporter.getRecords().size()).isEqualTo(1); } @Test @@ -93,24 +71,24 @@ public void testBatchSize() { sink.offer(createLog(Severity.WARN, "test #" + i)); } // Ensure that more than batch size kicks off a flush - await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.records.size() > 0); + await().atMost(500, TimeUnit.MILLISECONDS).until(() -> exporter.getRecords().size() > 0); // Ensure that everything gets through CompletableResultCode result = provider.forceFlush(); result.join(1, TimeUnit.SECONDS); - assertThat(exporter.callCount).isGreaterThanOrEqualTo(2); + assertThat(exporter.getCallCount()).isGreaterThanOrEqualTo(2); } @Test public void testNoBlocking() { TestLogExporter exporter = new TestLogExporter(); - exporter.onCall = + exporter.setOnCall( () -> { try { Thread.sleep(250); } catch (InterruptedException ex) { fail("Exporter wait interrupted", ex); } - }; + }); LogProcessor processor = BatchLogProcessor.builder(exporter) .setScheduleDelayMillis(3000) // Long enough to not be in play @@ -129,6 +107,32 @@ public void testNoBlocking() { long end = System.currentTimeMillis(); assertThat(end - start).isLessThan(250L); provider.forceFlush().join(1, TimeUnit.SECONDS); - assertThat(exporter.records.size()).isLessThan(testRecordCount); // We dropped records + assertThat(exporter.getRecords().size()).isLessThan(testRecordCount); // We dropped records + } + + @Test + public void testMultipleProcessors() { + TestLogProcessor processorOne = new TestLogProcessor(); + TestLogProcessor processorTwo = new TestLogProcessor(); + LogSinkSdkProvider provider = new LogSinkSdkProvider.Builder().build(); + provider.addLogProcessor(processorOne); + provider.addLogProcessor(processorTwo); + LogSink sink = provider.get("test", "0.1"); + LogRecord record = LogRecord.builder().setBody("test").build(); + sink.offer(record); + assertThat(processorOne.getRecords().size()).isEqualTo(1); + assertThat(processorTwo.getRecords().size()).isEqualTo(1); + assertThat(processorOne.getRecords().get(0)).isEqualTo(record); + assertThat(processorTwo.getRecords().get(0)).isEqualTo(record); + + CompletableResultCode flushResult = provider.forceFlush(); + flushResult.join(1, TimeUnit.SECONDS); + assertThat(processorOne.getFlushes()).isEqualTo(1); + assertThat(processorTwo.getFlushes()).isEqualTo(1); + + CompletableResultCode shutdownResult = provider.shutdown(); + shutdownResult.join(1, TimeUnit.SECONDS); + assertThat(processorOne.shutdownHasBeenCalled()).isEqualTo(true); + assertThat(processorTwo.shutdownHasBeenCalled()).isEqualTo(true); } } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java index 5069b1741fa..4d6558ff57d 100644 --- a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java @@ -17,17 +17,19 @@ package io.opentelemetry.sdk.logging.sdk; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; +import static org.awaitility.Awaitility.await; import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.logging.data.LogRecord; import io.opentelemetry.sdk.logging.export.BatchLogProcessor; import io.opentelemetry.sdk.logging.export.LogExporter; +import io.opentelemetry.sdk.logging.util.TestLogExporter; import java.util.Collection; import java.util.Properties; +import java.util.concurrent.TimeUnit; import org.junit.Test; public class BatchLogProcessorTest { - @Test public void testBuilder() { Properties props = new Properties(); @@ -61,4 +63,25 @@ public CompletableResultCode shutdown() { assertThat(builder.getMaxExportBatchSize()).isEqualTo(batch); assertThat(builder.getExporterTimeoutMillis()).isEqualTo(timeout); } + + @Test + public void testForceExport() { + int batchSize = 10; + int testRecordsToSend = 17; // greater than, but not a multiple of batch + TestLogExporter exporter = new TestLogExporter(); + BatchLogProcessor processor = + BatchLogProcessor.builder(exporter) + .setMaxExportBatchSize(batchSize) + .setMaxQueueSize(20) // more than we will send + .setScheduleDelayMillis(2000) // longer than test + .build(); + for (int i = 0; i < 17; i++) { + LogRecord record = LogRecord.builder().setBody(Integer.toString(i)).build(); + processor.addLogRecord(record); + } + await().until(() -> exporter.getCallCount() > 0); + assertThat(exporter.getRecords().size()).isEqualTo(batchSize); + processor.forceFlush().join(500, TimeUnit.MILLISECONDS); + assertThat(exporter.getRecords().size()).isEqualTo(testRecordsToSend); + } } diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java new file mode 100644 index 00000000000..52dd08f75a6 --- /dev/null +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java @@ -0,0 +1,58 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.sdk.logging.util; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logging.data.LogRecord; +import io.opentelemetry.sdk.logging.export.LogExporter; +import java.util.ArrayList; +import java.util.Collection; +import javax.annotation.Nullable; + +public class TestLogExporter implements LogExporter { + + private final ArrayList records = new ArrayList<>(); + @Nullable private Runnable onCall = null; + private int callCount = 0; + + @Override + public CompletableResultCode export(Collection records) { + this.records.addAll(records); + callCount++; + if (onCall != null) { + onCall.run(); + } + return null; + } + + @Override + public CompletableResultCode shutdown() { + return new CompletableResultCode().succeed(); + } + + public ArrayList getRecords() { + return records; + } + + public void setOnCall(@Nullable Runnable onCall) { + this.onCall = onCall; + } + + public int getCallCount() { + return callCount; + } +} diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java new file mode 100644 index 00000000000..c2aa404b14e --- /dev/null +++ b/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java @@ -0,0 +1,58 @@ +/* + * Copyright 2020, OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.opentelemetry.sdk.logging.util; + +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.logging.LogProcessor; +import io.opentelemetry.sdk.logging.data.LogRecord; +import java.util.ArrayList; +import java.util.List; + +public class TestLogProcessor implements LogProcessor { + private final List records = new ArrayList<>(); + private boolean shutdownCalled = false; + private int flushes = 0; + + @Override + public void addLogRecord(LogRecord record) { + records.add(record); + } + + @Override + public CompletableResultCode shutdown() { + shutdownCalled = true; + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode forceFlush() { + flushes++; + return CompletableResultCode.ofSuccess(); + } + + public List getRecords() { + return records; + } + + public int getFlushes() { + return flushes; + } + + public boolean shutdownHasBeenCalled() { + return shutdownCalled; + } +} From adf4a3005c7c7e22149628697acddc64e4d00715 Mon Sep 17 00:00:00 2001 From: David Poncelow Date: Wed, 23 Sep 2020 16:23:57 -0700 Subject: [PATCH 20/20] update to use new Attributes, remove support directory, public LogRecord getters --- .../logging/{support => }/README.md | 0 .../logging/{support => }/build.gradle | 0 .../sdk/logging/LogProcessor.java | 0 .../io/opentelemetry/sdk/logging/LogSink.java | 0 .../sdk/logging/LogSinkSdkProvider.java | 0 .../sdk/logging/data/AnyValue.java | 0 .../sdk/logging/data/LogRecord.java | 38 +++++++++++-------- .../sdk/logging/export/BatchLogProcessor.java | 0 .../sdk/logging/export/LogExporter.java | 0 .../sdk/logging/LogSinkSdkProviderTest.java | 0 .../logging/sdk/BatchLogProcessorTest.java | 0 .../sdk/logging/util/TestLogExporter.java | 0 .../sdk/logging/util/TestLogProcessor.java | 0 settings.gradle | 2 +- 14 files changed, 23 insertions(+), 17 deletions(-) rename sdk_extensions/logging/{support => }/README.md (100%) rename sdk_extensions/logging/{support => }/build.gradle (100%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java (100%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/LogSink.java (100%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java (100%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java (100%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java (81%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java (100%) rename sdk_extensions/logging/{support => }/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java (100%) rename sdk_extensions/logging/{support => }/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java (100%) rename sdk_extensions/logging/{support => }/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java (100%) rename sdk_extensions/logging/{support => }/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java (100%) rename sdk_extensions/logging/{support => }/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java (100%) diff --git a/sdk_extensions/logging/support/README.md b/sdk_extensions/logging/README.md similarity index 100% rename from sdk_extensions/logging/support/README.md rename to sdk_extensions/logging/README.md diff --git a/sdk_extensions/logging/support/build.gradle b/sdk_extensions/logging/build.gradle similarity index 100% rename from sdk_extensions/logging/support/build.gradle rename to sdk_extensions/logging/build.gradle diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java similarity index 100% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/LogProcessor.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/LogSink.java similarity index 100% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSink.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/LogSink.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java similarity index 100% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/LogSinkSdkProvider.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java similarity index 100% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/data/AnyValue.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java similarity index 81% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java index 662fd37c123..d86ea0abb9f 100644 --- a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java +++ b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/data/LogRecord.java @@ -17,9 +17,7 @@ package io.opentelemetry.sdk.logging.data; import com.google.auto.value.AutoValue; -import io.opentelemetry.common.AttributeValue; -import java.util.HashMap; -import java.util.Map; +import io.opentelemetry.common.Attributes; import java.util.concurrent.TimeUnit; import javax.annotation.Nullable; @@ -31,25 +29,25 @@ @AutoValue public abstract class LogRecord { - abstract long getTimeUnixNano(); + public abstract long getTimeUnixNano(); - abstract String getTraceId(); + public abstract String getTraceId(); - abstract String getSpanId(); + public abstract String getSpanId(); - abstract int getFlags(); + public abstract int getFlags(); - abstract Severity getSeverity(); + public abstract Severity getSeverity(); @Nullable - abstract String getSeverityText(); + public abstract String getSeverityText(); @Nullable - abstract String getName(); + public abstract String getName(); - abstract AnyValue getBody(); + public abstract AnyValue getBody(); - abstract Map getAttributes(); + public abstract Attributes getAttributes(); public enum Severity { UNDEFINED_SEVERITY_NUMBER(0), @@ -103,7 +101,7 @@ public static class Builder { private String severityText; private String name; private AnyValue body = AnyValue.stringAnyValue(""); - private final Map attributes = new HashMap<>(); + private final Attributes.Builder attributeBuilder = Attributes.newBuilder(); public Builder setUnixTimeNano(long timestamp) { this.timeUnixNano = timestamp; @@ -154,8 +152,8 @@ public Builder setBody(String body) { return this; } - public Builder setAttributes(Map attributes) { - this.attributes.putAll(attributes); + public Builder setAttributes(Attributes attributes) { + this.attributeBuilder.addAll(attributes); return this; } @@ -169,7 +167,15 @@ public LogRecord build() { timeUnixNano = TimeUnit.MILLISECONDS.toNanos(System.currentTimeMillis()); } return new AutoValue_LogRecord( - timeUnixNano, traceId, spanId, flags, severity, severityText, name, body, attributes); + timeUnixNano, + traceId, + spanId, + flags, + severity, + severityText, + name, + body, + attributeBuilder.build()); } } } diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java similarity index 100% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/export/BatchLogProcessor.java diff --git a/sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java b/sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java similarity index 100% rename from sdk_extensions/logging/support/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java rename to sdk_extensions/logging/src/main/java/io/opentelemetry/sdk/logging/export/LogExporter.java diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java b/sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java similarity index 100% rename from sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java rename to sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/LogSinkSdkProviderTest.java diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java b/sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java similarity index 100% rename from sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java rename to sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/sdk/BatchLogProcessorTest.java diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java b/sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java similarity index 100% rename from sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java rename to sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/util/TestLogExporter.java diff --git a/sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java b/sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java similarity index 100% rename from sdk_extensions/logging/support/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java rename to sdk_extensions/logging/src/test/java/io/opentelemetry/sdk/logging/util/TestLogProcessor.java diff --git a/settings.gradle b/settings.gradle index 1e58e4df780..5d4e3f7556d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,7 +42,7 @@ include ":opentelemetry-all", ":opentelemetry-sdk", ":opentelemetry-sdk-extension-async-processor", ":opentelemetry-sdk-extension-aws-v1-support", - ":opentelemetry-sdk-extension-logging-support", + ":opentelemetry-sdk-extension-logging", ":opentelemetry-sdk-extension-otproto", ":opentelemetry-sdk-extension-resources", ":opentelemetry-sdk-extension-testbed",