From ff1efa10d00ebc12dc04118654b90d82c53f1b0f Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 24 Aug 2021 14:09:17 +0200 Subject: [PATCH 1/2] ARROW-13734: [Format] Clarify allowed values for time types --- format/Schema.fbs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/format/Schema.fbs b/format/Schema.fbs index 2d447d30791..10c69ec6807 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -194,8 +194,8 @@ enum DateUnit: short { MILLISECOND } -/// Date is either a 32-bit or 64-bit type representing elapsed time since UNIX -/// epoch (1970-01-01), stored in either of two units: +/// Date is either a 32-bit or 64-bit integer type representing an elapsed time +/// since UNIX epoch (1970-01-01), stored in either of two units: /// /// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no /// leap seconds), where the values are evenly divisible by 86400000 @@ -206,9 +206,20 @@ table Date { enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND } -/// Time type. The physical storage type depends on the unit -/// - SECOND and MILLISECOND: 32 bits -/// - MICROSECOND and NANOSECOND: 64 bits +/// Time is either a 32-bit or 64-bit integer type representing an elapsed time +/// since midnight, stored in either of four units: seconds, milliseconds, +/// microseconds or nanoseconds. +/// +/// The integer `bitWidth` depends on the `unit` and must be one of the following: +/// * SECOND and MILLISECOND: 32 bits +/// * MICROSECOND and NANOSECOND: 64 bits +/// +/// The allowed values are between 0 (inclusive) and 86400 (exclusive), adjusted +/// for the time unit (for example, up to 86400000 exclusive for the MILLISECOND +/// unit). +/// This definition doesn't allow for leap seconds. If converting data from a +/// system which may produce leap seconds, it will be necessary to adjust incoming +/// data (for example by converting 86400 to 86399). table Time { unit: TimeUnit = MILLISECOND; bitWidth: int = 32; From d3e756bb9f6c999f665904243185dd7c9573b9de Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 24 Aug 2021 16:35:13 +0200 Subject: [PATCH 2/2] Address review comments --- format/Schema.fbs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/format/Schema.fbs b/format/Schema.fbs index 10c69ec6807..225ccdd0057 100644 --- a/format/Schema.fbs +++ b/format/Schema.fbs @@ -194,8 +194,8 @@ enum DateUnit: short { MILLISECOND } -/// Date is either a 32-bit or 64-bit integer type representing an elapsed time -/// since UNIX epoch (1970-01-01), stored in either of two units: +/// Date is either a 32-bit or 64-bit signed integer type representing an +/// elapsed time since UNIX epoch (1970-01-01), stored in either of two units: /// /// * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no /// leap seconds), where the values are evenly divisible by 86400000 @@ -206,20 +206,20 @@ table Date { enum TimeUnit: short { SECOND, MILLISECOND, MICROSECOND, NANOSECOND } -/// Time is either a 32-bit or 64-bit integer type representing an elapsed time -/// since midnight, stored in either of four units: seconds, milliseconds, -/// microseconds or nanoseconds. +/// Time is either a 32-bit or 64-bit signed integer type representing an +/// elapsed time since midnight, stored in either of four units: seconds, +/// milliseconds, microseconds or nanoseconds. /// /// The integer `bitWidth` depends on the `unit` and must be one of the following: /// * SECOND and MILLISECOND: 32 bits /// * MICROSECOND and NANOSECOND: 64 bits /// -/// The allowed values are between 0 (inclusive) and 86400 (exclusive), adjusted -/// for the time unit (for example, up to 86400000 exclusive for the MILLISECOND -/// unit). -/// This definition doesn't allow for leap seconds. If converting data from a -/// system which may produce leap seconds, it will be necessary to adjust incoming -/// data (for example by converting 86400 to 86399). +/// The allowed values are between 0 (inclusive) and 86400 (=24*60*60) seconds +/// (exclusive), adjusted for the time unit (for example, up to 86400000 +/// exclusive for the MILLISECOND unit). +/// This definition doesn't allow for leap seconds. Time values from +/// measurements with leap seconds will need to be corrected when ingesting +/// into Arrow (for example by replacing the value 86400 with 86399). table Time { unit: TimeUnit = MILLISECOND; bitWidth: int = 32;