Skip to content

[core] remove nullable check for record-level.time-field #4837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 1 commit into from
Jan 6, 2025
Merged

[core] remove nullable check for record-level.time-field #4837

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,12 +203,6 @@ public static void validateTableSchema(TableSchema schema) {
"The record level time field type should be one of INT, BIGINT, or TIMESTAMP, but field type is %s.",
dataType));
}
if (dataType.isNullable()) {
throw new IllegalArgumentException(
String.format(
"Time field %s for record-level expire should be not null.",
recordLevelTimeField));
}
}

if (options.mergeEngine() == MergeEngine.FIRST_ROW) {
Expand Down
6 changes: 5 additions & 1 deletion paimon-core/src/test/java/org/apache/paimon/catalog/PrimaryKeyTableTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,11 @@ protected List<GenericRow> query(int[] projection) throws Exception {
r -> {
GenericRow newR = new GenericRow(projection.length);
for (int i = 0; i < projection.length; i++) {
newR.setField(i, r.getInt(i));
if (r.isNullAt(i)) {
newR.setField(i, null);
} else {
newR.setField(i, r.getInt(i));
}
}
rows.add(newR);
});
Expand Down
4 changes: 2 additions & 2 deletions paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import static org.apache.paimon.CoreOptions.BUCKET;
import static org.apache.paimon.CoreOptions.SCAN_SNAPSHOT_ID;
import static org.apache.paimon.schema.SchemaValidation.validateTableSchema;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatNoException;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

Expand Down Expand Up @@ -110,8 +111,7 @@ public void testRecordLevelTimeField() {
Map<String, String> options = new HashMap<>(2);
options.put(CoreOptions.RECORD_LEVEL_TIME_FIELD.key(), "f0");
options.put(CoreOptions.RECORD_LEVEL_EXPIRE_TIME.key(), "1 m");
assertThatThrownBy(() -> validateTableSchemaExec(options))
.hasMessageContaining("Time field f0 for record-level expire should be not null");
assertThatCode(() -> validateTableSchemaExec(options)).doesNotThrowAnyException();

options.put(CoreOptions.RECORD_LEVEL_TIME_FIELD.key(), "f10");
assertThatThrownBy(() -> validateTableSchemaExec(options))
Expand Down
12 changes: 11 additions & 1 deletion paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.util.UUID;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

class RecordLevelExpireTest extends PrimaryKeyTableTestBase {

Expand All @@ -54,7 +55,7 @@ public void beforeEachBase() throws Exception {
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
.column("col1", DataTypes.INT().notNull())
.column("col1", DataTypes.INT())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
Expand Down Expand Up @@ -99,5 +100,14 @@ public void test() throws Exception {
assertThat(query()).containsExactlyInAnyOrder(GenericRow.of(1, 4, currentSecs + 60 * 60));
assertThat(query(new int[] {2}))
.containsExactlyInAnyOrder(GenericRow.of(currentSecs + 60 * 60));

writeCommit(GenericRow.of(1, 5, null));
assertThat(query())
.containsExactlyInAnyOrder(
GenericRow.of(1, 4, currentSecs + 60 * 60), GenericRow.of(1, 5, null));

// null time field for record-level expire is not supported yet.
assertThatThrownBy(() -> compact(1))
.hasMessageContaining("Time field for record-level expire should not be null.");
}
}
11 changes: 10 additions & 1 deletion paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithMillisecondTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.util.UUID;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

class RecordLevelExpireWithMillisecondTest extends PrimaryKeyTableTestBase {
@Override
Expand All @@ -53,7 +54,7 @@ public void beforeEachBase() throws Exception {
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
.column("col1", DataTypes.BIGINT().notNull())
.column("col1", DataTypes.BIGINT())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
Expand Down Expand Up @@ -96,5 +97,13 @@ public void test() throws Exception {
// compact, expired
compact(1);
assertThat(query(new int[] {0, 1})).containsExactlyInAnyOrder(GenericRow.of(1, 4));

writeCommit(GenericRow.of(1, 5, null));
assertThat(query(new int[] {0, 1}))
.containsExactlyInAnyOrder(GenericRow.of(1, 4), GenericRow.of(1, 5));

// null time field for record-level expire is not supported yet.
assertThatThrownBy(() -> compact(1))
.hasMessageContaining("Time field for record-level expire should not be null.");
}
}
9 changes: 9 additions & 0 deletions ...on-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampBaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.time.Duration;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

abstract class RecordLevelExpireWithTimestampBaseTest extends PrimaryKeyTableTestBase {

Expand Down Expand Up @@ -61,5 +62,13 @@ public void testTimestampTypeExpire() throws Exception {
// compact, expired
compact(1);
assertThat(query(new int[] {0, 1})).containsExactlyInAnyOrder(GenericRow.of(1, 3));

writeCommit(GenericRow.of(1, 5, null));
assertThat(query(new int[] {0, 1}))
.containsExactlyInAnyOrder(GenericRow.of(1, 3), GenericRow.of(1, 5));

// null time field for record-level expire is not supported yet.
assertThatThrownBy(() -> compact(1))
.hasMessageContaining("Time field for record-level expire should not be null.");
}
}
2 changes: 1 addition & 1 deletion paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampLTZTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public void beforeEachBase() throws Exception {
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
.column("col1", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE().notNull())
.column("col1", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
Expand Down
2 changes: 1 addition & 1 deletion paimon-core/src/test/java/org/apache/paimon/table/RecordLevelExpireWithTimestampTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public void beforeEachBase() throws Exception {
Schema.newBuilder()
.column("pt", DataTypes.INT())
.column("pk", DataTypes.INT())
.column("col1", DataTypes.TIMESTAMP().notNull())
.column("col1", DataTypes.TIMESTAMP())
.partitionKeys("pt")
.primaryKey("pk", "pt")
.options(tableOptions().toMap())
Expand Down
Loading