Skip to content

[bugfix](iceberg)fix datetime conversion error for 2.0 #35831

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
Jun 6, 2024
Merged

[bugfix](iceberg)fix datetime conversion error for 2.0 #35831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -974,12 +974,25 @@ public static DateLiteral read(DataInput in) throws IOException {
}

public long unixTimestamp(TimeZone timeZone) {
ZonedDateTime zonedDateTime = ZonedDateTime.of((int) year, (int) month, (int) day, (int) hour,
(int) minute, (int) second, (int) microsecond, ZoneId.of(timeZone.getID()));
Timestamp timestamp = Timestamp.from(zonedDateTime.toInstant());
Timestamp timestamp = getTimestamp(timeZone);
return timestamp.getTime();
}

private Timestamp getTimestamp(TimeZone timeZone) {
ZonedDateTime zonedDateTime = ZonedDateTime.of((int) year, (int) month, (int) day, (int) hour,
(int) minute, (int) second, (int) microsecond * 1000, ZoneId.of(timeZone.getID()));
return Timestamp.from(zonedDateTime.toInstant());
}

public long getUnixTimestampWithMillisecond(TimeZone timeZone) {
return unixTimestamp(timeZone);
}

public long getUnixTimestampWithMicroseconds(TimeZone timeZone) {
Timestamp timestamp = getTimestamp(timeZone);
return timestamp.getTime() * 1000 + timestamp.getNanos() / 1000 % 1000;
}

public static boolean hasTimePart(String format) {
return format.chars().anyMatch(c -> TIME_PART_SET.contains((char) c));
}
Expand Down
3 changes: 1 addition & 2 deletions fe/fe-core/src/main/java/org/apache/doris/external/iceberg/util/IcebergUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ public Integer initialValue() {
return 0;
}
};
static long MILLIS_TO_NANO_TIME = 1000;
public static final String TOTAL_RECORDS = "total-records";
public static final String TOTAL_POSITION_DELETES = "total-position-deletes";
public static final String TOTAL_EQUALITY_DELETES = "total-equality-deletes";
Expand Down Expand Up @@ -454,7 +453,7 @@ private static Object extractDorisLiteral(org.apache.iceberg.types.Type icebergT
case DATE:
return dateLiteral.getStringValue();
case TIMESTAMP:
return dateLiteral.unixTimestamp(TimeUtils.getTimeZone()) * MILLIS_TO_NANO_TIME;
return dateLiteral.getUnixTimestampWithMicroseconds(TimeUtils.getTimeZone());
default:
return null;
}
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/test/java/org/apache/doris/analysis/DateLiteralTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@
import org.junit.Assert;
import org.junit.Test;

import java.time.ZoneOffset;
import java.util.TimeZone;

public class DateLiteralTest {

@Test
Expand Down Expand Up @@ -406,4 +409,16 @@ public void testCheckRangeForDateV2() {
}
Assert.assertFalse(hasException);
}

@Test
public void testUnixTimestampWithMilliMicroSecond() throws AnalysisException {
String s = "2020-12-13 12:13:14.123456";
Type type = Type.DATETIMEV2;
DateLiteral literal = new DateLiteral(s, type);
long l = literal.getUnixTimestampWithMillisecond(TimeZone.getTimeZone(ZoneOffset.UTC));
Assert.assertEquals(123, l % 1000);

long l2 = literal.getUnixTimestampWithMicroseconds(TimeZone.getTimeZone(ZoneOffset.UTC));
Assert.assertEquals(123456, l2 % 1000000);
}
}