Skip to content

[orc] Optimize configuration creating in orc file format #4716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 3 commits into from
Dec 16, 2024
Merged

[orc] Optimize configuration creating in orc file format #4716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ae9cd45
[orc][format] add cache for OrcFileFormat
zhangyazhe Dec 16, 2024
56115d4
[format] Switch to using the Configuration(boolean) constructor, swit…
zhangyazhe Dec 16, 2024
1cf3b74
[format] Switch to using the Configuration(boolean) constructor
zhangyazhe Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ public class OrcFileFormat extends FileFormat {
public OrcFileFormat(FormatContext formatContext) {
super(IDENTIFIER);
this.orcProperties = getOrcProperties(formatContext.options(), formatContext);
this.readerConf = new org.apache.hadoop.conf.Configuration();
Copy link
Contributor

@JingsongLi JingsongLi Dec 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just use new Configuration(false), and tests passed. Do you think it is ok to just use this way?

Copy link
Contributor Author

@zhangyazhe zhangyazhe Dec 16, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

this.readerConf = new org.apache.hadoop.conf.Configuration(false);
this.orcProperties.forEach((k, v) -> readerConf.set(k.toString(), v.toString()));
this.writerConf = new org.apache.hadoop.conf.Configuration();
this.writerConf = new org.apache.hadoop.conf.Configuration(false);
this.orcProperties.forEach((k, v) -> writerConf.set(k.toString(), v.toString()));
this.readBatchSize = formatContext.readBatchSize();
this.writeBatchSize = formatContext.writeBatchSize();
Expand Down
64 changes: 64 additions & 0 deletions paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,68 @@ public void testSupportedDataTypes() {
dataFields.add(new DataField(index++, "decimal_type", DataTypes.DECIMAL(10, 3)));
orc.validateDataFields(new RowType(dataFields));
}

@Test
public void testCreateCost() {
double createConfCost = createConfigCost();
for (int i = 0; i < 1000; i++) {
create();
}
int times = 10_000;
long start = System.nanoTime();
for (int i = 0; i < times; i++) {
create();
}
double cost = ((double) (System.nanoTime() - start)) / 1000_000 / times;
assertThat(cost * 500 < createConfCost).isTrue();
}

@Test
public void testCreateCostWithRandomConfig() {
double createConfCost = createConfigCost();
for (int i = 0; i < 1000; i++) {
createRandomConfig();
}
int times = 10_000;
long start = System.nanoTime();
for (int i = 0; i < times; i++) {
createRandomConfig();
}
double cost = ((double) (System.nanoTime() - start)) / 1000_000 / times;
assertThat(cost * 10 < createConfCost).isTrue();
}

private double createConfigCost() {
for (int i = 0; i < 1000; i++) {
createConfig();
}
int times = 10_000;
long start = System.nanoTime();
for (int i = 0; i < times; i++) {
createConfig();
}
return ((double) (System.nanoTime() - start)) / 1000_000 / times;
}

private void createConfig() {
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
conf.set("a", "a");
}

private void create() {
Options options = new Options();
options.setString("haha", "1");
options.setString("compress", "zlib");
OrcFileFormat orcFileFormat =
new OrcFileFormatFactory().create(new FormatContext(options, 1024, 1024));
}

private void createRandomConfig() {
Options options = new Options();
options.setString("haha", "1");
options.setString("compress", "zlib");
options.setString("a", Math.random() + "");
OrcFileFormat orcFileFormat =
new OrcFileFormatFactory().create(new FormatContext(options, 1024, 1024));
}
}
Loading