Skip to content

[fix](ParquetReader) Fix Parquet Reader to read int96 parquet type problem #32394

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 2 commits into from
Mar 19, 2024
Merged

[fix](ParquetReader) Fix Parquet Reader to read int96 parquet type problem #32394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
857e0b3
fix
BePPPower Mar 18, 2024
8760b82
fix2
BePPPower Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions be/src/vec/exec/format/parquet/parquet_column_convert.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,12 +318,13 @@ struct Int96toTimestamp : public ColumnConvert {
auto& data = static_cast<ColumnVector<UInt64>*>(dst_col.get())->get_data();

for (int i = 0; i < rows; i++) {
ParquetInt96 x = ParquetInt96_data[i];
auto& num = data[start_idx + i];
auto& value = reinterpret_cast<DateV2Value<DateTimeV2ValueType>&>(num);
int64_t micros = x.to_timestamp_micros();
value.from_unixtime(micros / 1000000, *_convert_params->ctz);
value.set_microsecond(micros % 1000000);
ParquetInt96 src_cell_data = ParquetInt96_data[i];
auto& dst_value =
reinterpret_cast<DateV2Value<DateTimeV2ValueType>&>(data[start_idx + i]);

int64_t timestamp_with_micros = src_cell_data.to_timestamp_micros();
dst_value.from_unixtime(timestamp_with_micros / 1000000, *_convert_params->ctz);
dst_value.set_microsecond(timestamp_with_micros % 1000000);
}
return Status::OK();
}
Expand Down
6 changes: 3 additions & 3 deletions be/src/vec/exec/format/parquet/parquet_common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@

namespace doris::vectorized {

const uint32_t ParquetInt96::JULIAN_EPOCH_OFFSET_DAYS = 2440588;
const uint64_t ParquetInt96::MICROS_IN_DAY = 86400000000;
const uint64_t ParquetInt96::NANOS_PER_MICROSECOND = 1000;
const int32_t ParquetInt96::JULIAN_EPOCH_OFFSET_DAYS = 2440588;
const int64_t ParquetInt96::MICROS_IN_DAY = 86400000000;
const int64_t ParquetInt96::NANOS_PER_MICROSECOND = 1000;

ColumnSelectVector::ColumnSelectVector(const uint8_t* filter_map, size_t filter_map_size,
bool filter_all) {
Expand Down
12 changes: 6 additions & 6 deletions be/src/vec/exec/format/parquet/parquet_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,10 @@ struct RowRange {

#pragma pack(1)
struct ParquetInt96 {
uint64_t lo; // time of nanoseconds in a day
uint32_t hi; // days from julian epoch
int64_t lo; // time of nanoseconds in a day
int32_t hi; // days from julian epoch

inline uint64_t to_timestamp_micros() const {
inline int64_t to_timestamp_micros() const {
return (hi - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + lo / NANOS_PER_MICROSECOND;
}
inline __int128 to_int128() const {
Expand All @@ -60,9 +60,9 @@ struct ParquetInt96 {
return ans;
}

static const uint32_t JULIAN_EPOCH_OFFSET_DAYS;
static const uint64_t MICROS_IN_DAY;
static const uint64_t NANOS_PER_MICROSECOND;
static const int32_t JULIAN_EPOCH_OFFSET_DAYS;
static const int64_t MICROS_IN_DAY;
static const int64_t NANOS_PER_MICROSECOND;
};
#pragma pack()
static_assert(sizeof(ParquetInt96) == 12, "The size of ParquetInt96 is not 12.");
Expand Down