Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -216,14 +216,39 @@ public Dataflow create(PipelineOptions options) {

void setReaderCacheTimeoutSec(Integer value);

/** The max amount of time an UnboundedReader is consumed before checkpointing. */
/**
* The max amount of time an UnboundedReader is consumed before checkpointing.
*
* @deprecated use {@link DataflowPipelineDebugOptions#getUnboundedReaderMaxReadTimeMs()} instead
*/
@Description(
"The max amount of time before an UnboundedReader is consumed before checkpointing, in seconds.")
@Deprecated
@Default.Integer(10)
Integer getUnboundedReaderMaxReadTimeSec();

void setUnboundedReaderMaxReadTimeSec(Integer value);

/** The max amount of time an UnboundedReader is consumed before checkpointing. */
@Description(
"The max amount of time before an UnboundedReader is consumed before checkpointing, in millis.")
@Default.InstanceFactory(UnboundedReaderMaxReadTimeFactory.class)
Integer getUnboundedReaderMaxReadTimeMs();

void setUnboundedReaderMaxReadTimeMs(Integer value);

/**
* Sets Integer value based on old, deprecated field ({@link
* DataflowPipelineDebugOptions#getUnboundedReaderMaxReadTimeSec()}).
*/
final class UnboundedReaderMaxReadTimeFactory implements DefaultValueFactory<Integer> {
@Override
public Integer create(PipelineOptions options) {
DataflowPipelineDebugOptions debugOptions = options.as(DataflowPipelineDebugOptions.class);
return debugOptions.getUnboundedReaderMaxReadTimeSec() * 1000;
}
}

/** The max elements read from an UnboundedReader before checkpointing. */
@Description("The max elements read from an UnboundedReader before checkpointing. ")
@Default.Integer(10 * 1000)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -796,9 +796,8 @@ private UnboundedReaderIterator(
this.context = context;
this.started = started;
DataflowPipelineDebugOptions debugOptions = options.as(DataflowPipelineDebugOptions.class);
this.endTime =
Instant.now()
.plus(Duration.standardSeconds(debugOptions.getUnboundedReaderMaxReadTimeSec()));
long maxReadTimeMs = debugOptions.getUnboundedReaderMaxReadTimeMs();
this.endTime = Instant.now().plus(Duration.millis(maxReadTimeMs));
this.maxElems = debugOptions.getUnboundedReaderMaxElements();
this.backoffFactory =
FluentBackoff.DEFAULT
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -645,10 +645,10 @@ public void testReadUnboundedReader() throws Exception {
numReadOnThisIteration++;
}
Instant afterReading = Instant.now();
long maxReadSec = debugOptions.getUnboundedReaderMaxReadTimeSec();
long maxReadMs = debugOptions.getUnboundedReaderMaxReadTimeMs();
assertThat(
new Duration(beforeReading, afterReading).getStandardSeconds(),
lessThanOrEqualTo(maxReadSec + 1));
new Duration(beforeReading, afterReading).getMillis(),
lessThanOrEqualTo(maxReadMs + 1000L));
assertThat(
numReadOnThisIteration, lessThanOrEqualTo(debugOptions.getUnboundedReaderMaxElements()));

Expand Down