diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 3e90bb329be56..b0454c016640f 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -72,7 +72,7 @@ jobs: # Any TPC-DS related updates on this job need to be applied to tpcds-1g job of build_and_test.yml as well tpcds-1g-gen: name: "Generate an TPC-DS dataset with SF=1" - if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, 'ZStandardTPCDSDataBenchmark') || contains(inputs.class, '*') + if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, 'LZ4TPCDSDataBenchmark') || contains(inputs.class, 'ZStandardTPCDSDataBenchmark') || contains(inputs.class, '*') runs-on: ubuntu-latest env: SPARK_LOCAL_IP: localhost @@ -177,7 +177,7 @@ jobs: distribution: zulu java-version: ${{ inputs.jdk }} - name: Cache TPC-DS generated data - if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, 'ZStandardTPCDSDataBenchmark') || contains(inputs.class, '*') + if: contains(inputs.class, 'TPCDSQueryBenchmark') || contains(inputs.class, 'LZ4TPCDSDataBenchmark') || contains(inputs.class, 'ZStandardTPCDSDataBenchmark') || contains(inputs.class, '*') id: cache-tpcds-sf-1 uses: actions/cache@v4 with: diff --git a/core/benchmarks/LZ4TPCDSDataBenchmark-jdk21-results.txt b/core/benchmarks/LZ4TPCDSDataBenchmark-jdk21-results.txt new file mode 100644 index 0000000000000..64fb5dc7875ad --- /dev/null +++ b/core/benchmarks/LZ4TPCDSDataBenchmark-jdk21-results.txt @@ -0,0 +1,17 @@ +================================================================================================ +Benchmark LZ4CompressionCodec +================================================================================================ + +OpenJDK 64-Bit Server VM 21.0.9+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor +Compression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +Compression 4 times 2621 2633 16 0.0 655315176.3 1.0X + +OpenJDK 64-Bit Server VM 21.0.9+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor +Decompression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +Decompression 4 times 2270 2286 23 0.0 567457203.5 1.0X + + diff --git a/core/benchmarks/LZ4TPCDSDataBenchmark-results.txt b/core/benchmarks/LZ4TPCDSDataBenchmark-results.txt new file mode 100644 index 0000000000000..ff05d27454ece --- /dev/null +++ b/core/benchmarks/LZ4TPCDSDataBenchmark-results.txt @@ -0,0 +1,17 @@ +================================================================================================ +Benchmark LZ4CompressionCodec +================================================================================================ + +OpenJDK 64-Bit Server VM 17.0.17+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor +Compression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +Compression 4 times 2614 2621 10 0.0 653597000.8 1.0X + +OpenJDK 64-Bit Server VM 17.0.17+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor +Decompression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +------------------------------------------------------------------------------------------------------------------------ +Decompression 4 times 2346 2361 22 0.0 586455894.5 1.0X + + diff --git a/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt b/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt index 7630433a2f7a4..bdf21e4ad7641 100644 --- a/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt +++ b/core/benchmarks/ZStandardTPCDSDataBenchmark-jdk21-results.txt @@ -2,48 +2,48 @@ Benchmark ZStandardCompressionCodec ================================================================================================ -OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure +OpenJDK 64-Bit Server VM 21.0.9+10-LTS on Linux 6.11.0-1018-azure AMD EPYC 7763 64-Core Processor -Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +Compression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -Compression 4 times at level 1 without buffer pool 2567 2573 9 0.0 641834272.8 1.0X -Compression 4 times at level 2 without buffer pool 4171 4173 2 0.0 1042717610.7 0.6X -Compression 4 times at level 3 without buffer pool 6202 6205 4 0.0 1550416592.3 0.4X -Compression 4 times at level 1 with buffer pool 2562 2562 0 0.0 640541340.8 1.0X -Compression 4 times at level 2 with buffer pool 4172 4173 1 0.0 1043037897.5 0.6X -Compression 4 times at level 3 with buffer pool 6199 6203 4 0.0 1549874630.3 0.4X +Compression 4 times at level 1 without buffer pool 2585 2586 1 0.0 646239352.5 1.0X +Compression 4 times at level 2 without buffer pool 4211 4216 7 0.0 1052689958.8 0.6X +Compression 4 times at level 3 without buffer pool 6328 6328 1 0.0 1581904435.8 0.4X +Compression 4 times at level 1 with buffer pool 2585 2587 3 0.0 646306514.0 1.0X +Compression 4 times at level 2 with buffer pool 4193 4195 3 0.0 1048293261.0 0.6X +Compression 4 times at level 3 with buffer pool 6264 6284 28 0.0 1565962696.5 0.4X -OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure +OpenJDK 64-Bit Server VM 21.0.9+10-LTS on Linux 6.11.0-1018-azure AMD EPYC 7763 64-Core Processor -Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +Decompression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------------- -Decompression 4 times from level 1 without buffer pool 856 875 17 0.0 214044635.0 1.0X -Decompression 4 times from level 2 without buffer pool 1141 1147 10 0.0 285134371.0 0.8X -Decompression 4 times from level 3 without buffer pool 1345 1345 0 0.0 336239892.8 0.6X -Decompression 4 times from level 1 with buffer pool 844 851 12 0.0 210917530.8 1.0X -Decompression 4 times from level 2 with buffer pool 1111 1126 22 0.0 277630951.0 0.8X -Decompression 4 times from level 3 with buffer pool 1338 1348 13 0.0 334616619.0 0.6X +Decompression 4 times from level 1 without buffer pool 845 877 53 0.0 211244491.3 1.0X +Decompression 4 times from level 2 without buffer pool 1141 1145 6 0.0 285267829.3 0.7X +Decompression 4 times from level 3 without buffer pool 1358 1366 11 0.0 339611143.3 0.6X +Decompression 4 times from level 1 with buffer pool 856 862 10 0.0 213948489.5 1.0X +Decompression 4 times from level 2 with buffer pool 1153 1156 5 0.0 288195943.3 0.7X +Decompression 4 times from level 3 with buffer pool 1368 1375 9 0.0 342032342.0 0.6X -OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure +OpenJDK 64-Bit Server VM 21.0.9+10-LTS on Linux 6.11.0-1018-azure AMD EPYC 7763 64-Core Processor Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Parallel Compression with 0 workers 2099 2099 0 0.0 524768920.0 1.0X -Parallel Compression with 1 workers 2119 2119 0 0.0 529758581.0 1.0X -Parallel Compression with 2 workers 1087 1090 4 0.0 271687124.0 1.9X -Parallel Compression with 4 workers 761 764 2 0.0 190289606.5 2.8X -Parallel Compression with 8 workers 782 789 7 0.0 195588720.8 2.7X -Parallel Compression with 16 workers 897 908 10 0.0 224272497.0 2.3X +Parallel Compression with 0 workers 2128 2130 3 0.0 531987145.5 1.0X +Parallel Compression with 1 workers 2117 2131 20 0.0 529324483.0 1.0X +Parallel Compression with 2 workers 1106 1107 0 0.0 276552237.3 1.9X +Parallel Compression with 4 workers 771 774 3 0.0 192777338.5 2.8X +Parallel Compression with 8 workers 815 826 13 0.0 203648175.7 2.6X +Parallel Compression with 16 workers 950 963 11 0.0 237496152.5 2.2X -OpenJDK 64-Bit Server VM 21.0.8+9-LTS on Linux 6.11.0-1018-azure +OpenJDK 64-Bit Server VM 21.0.9+10-LTS on Linux 6.11.0-1018-azure AMD EPYC 7763 64-Core Processor Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Parallel Compression with 0 workers 8240 8242 2 0.0 2060062709.3 1.0X -Parallel Compression with 1 workers 6915 6924 13 0.0 1728757390.0 1.2X -Parallel Compression with 2 workers 3663 3664 1 0.0 915832528.8 2.2X -Parallel Compression with 4 workers 3239 3247 12 0.0 809829698.3 2.5X -Parallel Compression with 8 workers 3594 3610 23 0.0 898542204.8 2.3X -Parallel Compression with 16 workers 3888 3898 14 0.0 972017914.3 2.1X +Parallel Compression with 0 workers 8976 9140 231 0.0 2244064460.3 1.0X +Parallel Compression with 1 workers 7198 7214 24 0.0 1799376873.8 1.2X +Parallel Compression with 2 workers 3836 3850 20 0.0 959020774.8 2.3X +Parallel Compression with 4 workers 3271 3284 18 0.0 817803690.0 2.7X +Parallel Compression with 8 workers 3806 3807 2 0.0 951490679.3 2.4X +Parallel Compression with 16 workers 4004 4026 31 0.0 1000957120.3 2.2X diff --git a/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt b/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt index 01784a686f818..b69321de88614 100644 --- a/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt +++ b/core/benchmarks/ZStandardTPCDSDataBenchmark-results.txt @@ -2,48 +2,48 @@ Benchmark ZStandardCompressionCodec ================================================================================================ -OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +OpenJDK 64-Bit Server VM 17.0.17+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor +Compression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ---------------------------------------------------------------------------------------------------------------------------------- -Compression 4 times at level 1 without buffer pool 2513 2519 10 0.0 628127696.5 1.0X -Compression 4 times at level 2 without buffer pool 3914 3916 3 0.0 978555442.0 0.6X -Compression 4 times at level 3 without buffer pool 5417 5424 9 0.0 1354342306.3 0.5X -Compression 4 times at level 1 with buffer pool 2524 2526 4 0.0 630890054.5 1.0X -Compression 4 times at level 2 with buffer pool 3911 3911 0 0.0 977705511.5 0.6X -Compression 4 times at level 3 with buffer pool 5447 5451 5 0.0 1361830044.0 0.5X +Compression 4 times at level 1 without buffer pool 2572 2573 0 0.0 643092504.0 1.0X +Compression 4 times at level 2 without buffer pool 4223 4224 2 0.0 1055729472.5 0.6X +Compression 4 times at level 3 without buffer pool 6306 6307 1 0.0 1576606903.0 0.4X +Compression 4 times at level 1 with buffer pool 2566 2567 2 0.0 641403366.5 1.0X +Compression 4 times at level 2 with buffer pool 4229 4229 0 0.0 1057252850.2 0.6X +Compression 4 times at level 3 with buffer pool 6369 6370 1 0.0 1592238794.8 0.4X -OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz -Benchmark ZStandardCompressionCodec: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative +OpenJDK 64-Bit Server VM 17.0.17+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor +Decompression: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative -------------------------------------------------------------------------------------------------------------------------------------- -Decompression 4 times from level 1 without buffer pool 1053 1056 4 0.0 263342668.0 1.0X -Decompression 4 times from level 2 without buffer pool 1401 1403 3 0.0 350193821.5 0.8X -Decompression 4 times from level 3 without buffer pool 1683 1683 1 0.0 420625735.0 0.6X -Decompression 4 times from level 1 with buffer pool 1078 1079 2 0.0 269507650.5 1.0X -Decompression 4 times from level 2 with buffer pool 1397 1403 8 0.0 349213546.5 0.8X -Decompression 4 times from level 3 with buffer pool 1679 1680 2 0.0 419721856.3 0.6X +Decompression 4 times from level 1 without buffer pool 914 915 1 0.0 228498988.8 1.0X +Decompression 4 times from level 2 without buffer pool 1186 1188 2 0.0 296544159.0 0.8X +Decompression 4 times from level 3 without buffer pool 1424 1424 0 0.0 355996464.8 0.6X +Decompression 4 times from level 1 with buffer pool 915 916 2 0.0 228733774.3 1.0X +Decompression 4 times from level 2 with buffer pool 1189 1192 4 0.0 297227461.0 0.8X +Decompression 4 times from level 3 with buffer pool 1421 1425 5 0.0 355242896.0 0.6X -OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.17+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor Parallel Compression at level 3: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Parallel Compression with 0 workers 1815 1825 14 0.0 453734902.0 1.0X -Parallel Compression with 1 workers 1654 1663 12 0.0 413617141.5 1.1X -Parallel Compression with 2 workers 857 859 2 0.0 214195314.3 2.1X -Parallel Compression with 4 workers 777 781 3 0.0 194366718.5 2.3X -Parallel Compression with 8 workers 784 786 3 0.0 195965343.5 2.3X -Parallel Compression with 16 workers 836 858 19 0.0 208885356.8 2.2X +Parallel Compression with 0 workers 2142 2143 2 0.0 535512459.0 1.0X +Parallel Compression with 1 workers 2145 2146 2 0.0 536274942.0 1.0X +Parallel Compression with 2 workers 1094 1100 9 0.0 273416030.8 2.0X +Parallel Compression with 4 workers 768 769 2 0.0 192023783.8 2.8X +Parallel Compression with 8 workers 831 839 13 0.0 207743301.0 2.6X +Parallel Compression with 16 workers 960 972 16 0.0 240059822.0 2.2X -OpenJDK 64-Bit Server VM 17.0.16+8-LTS on Linux 6.11.0-1018-azure -Intel(R) Xeon(R) Platinum 8370C CPU @ 2.80GHz +OpenJDK 64-Bit Server VM 17.0.17+10-LTS on Linux 6.11.0-1018-azure +AMD EPYC 7763 64-Core Processor Parallel Compression at level 9: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative ------------------------------------------------------------------------------------------------------------------------ -Parallel Compression with 0 workers 8302 8310 12 0.0 2075384302.0 1.0X -Parallel Compression with 1 workers 6869 6871 3 0.0 1717196107.5 1.2X -Parallel Compression with 2 workers 3570 3587 25 0.0 892437927.8 2.3X -Parallel Compression with 4 workers 3152 3152 1 0.0 787940909.5 2.6X -Parallel Compression with 8 workers 3463 3470 10 0.0 865739615.8 2.4X -Parallel Compression with 16 workers 4061 4083 31 0.0 1015251097.3 2.0X +Parallel Compression with 0 workers 9016 9031 21 0.0 2254017491.8 1.0X +Parallel Compression with 1 workers 7423 7467 62 0.0 1855653850.8 1.2X +Parallel Compression with 2 workers 4203 4242 54 0.0 1050838310.7 2.1X +Parallel Compression with 4 workers 3472 3481 13 0.0 867935936.0 2.6X +Parallel Compression with 8 workers 3952 3996 62 0.0 988086500.5 2.3X +Parallel Compression with 16 workers 4081 4115 47 0.0 1020289087.7 2.2X diff --git a/core/src/test/scala/org/apache/spark/io/LZ4TPCDSDataBenchmark.scala b/core/src/test/scala/org/apache/spark/io/LZ4TPCDSDataBenchmark.scala new file mode 100644 index 0000000000000..f3f2bd1057b76 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/io/LZ4TPCDSDataBenchmark.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.io + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream, OutputStream} + +import org.apache.spark.SparkConf +import org.apache.spark.benchmark.Benchmark + +/** + * Benchmark for LZ4 codec performance. + * {{{ + * To run this benchmark: + * 1. without sbt: bin/spark-submit --class + * 2. build/sbt "core/Test/runMain " + * 3. generate result: SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "core/Test/runMain " + * Results will be written to "benchmarks/LZ4TPCDSDataBenchmark-results.txt". + * }}} + */ +object LZ4TPCDSDataBenchmark extends TPCDSDataBenchmark { + + override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { + prepareData() + runBenchmark("Benchmark LZ4CompressionCodec") { + compressionBenchmark() + decompressionBenchmark() + } + } + + private def compressionBenchmark(): Unit = { + val benchmark = new Benchmark("Compression", N, output = output) + val conf = new SparkConf(false) + benchmark.addCase(s"Compression $N times") { _ => + (1 until N).foreach { _ => + val os = new LZ4CompressionCodec(conf) + .compressedOutputStream(OutputStream.nullOutputStream()) + os.write(data) + os.close() + } + } + benchmark.run() + } + + private def decompressionBenchmark(): Unit = { + val benchmark = new Benchmark("Decompression", N, output = output) + val conf = new SparkConf(false) + val outputStream = new ByteArrayOutputStream() + val out = new LZ4CompressionCodec(conf).compressedOutputStream(outputStream) + out.write(data) + out.close() + val bytes = outputStream.toByteArray + + benchmark.addCase(s"Decompression $N times") { _ => + (1 until N).foreach { _ => + val bais = new ByteArrayInputStream(bytes) + val is = new LZ4CompressionCodec(conf).compressedInputStream(bais) + is.readAllBytes() + is.close() + } + } + benchmark.run() + } +} diff --git a/core/src/test/scala/org/apache/spark/io/TPCDSDataBenchmark.scala b/core/src/test/scala/org/apache/spark/io/TPCDSDataBenchmark.scala new file mode 100644 index 0000000000000..b6b419e0201ae --- /dev/null +++ b/core/src/test/scala/org/apache/spark/io/TPCDSDataBenchmark.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.io + +import java.nio.file.{Files, Paths} + +import org.apache.spark.benchmark.BenchmarkBase + +/** + * TPC-DS data preparation: + *

+ * 1. Follow https://github.com/gregrahn/tpcds-kit.git to set up tpcds-kit + *

+ * 2. Create a folder and export environment variable SPARK_TPCDS_DATA_TEXT + * {{{ + * mkdir -p /path/of/tpcds-sf1-text + * export SPARK_TPCDS_DATA_TEXT=/path/of/tpcds-sf1-text + * }}} + *

+ * 3. Generate TPC-DS (SF1) text data under SPARK_TPCDS_DATA_TEXT + * {{{ + * tpcds-kit/tools/dsdgen \ + * -DISTRIBUTIONS tpcds-kit/tools/tpcds.idx \ + * -SCALE 1 \ + * -DIR $SPARK_TPCDS_DATA_TEXT + * }}} + */ +abstract class TPCDSDataBenchmark extends BenchmarkBase { + + val N = 4 + + var data: Array[Byte] = _ + + protected def prepareData(): Unit = { + val tpcDsDataDir = sys.env.get("SPARK_TPCDS_DATA_TEXT") + assert(tpcDsDataDir.nonEmpty, "Can not find env var SPARK_TPCDS_DATA_TEXT") + + val catalogSalesDatPath = Paths.get(tpcDsDataDir.get, "catalog_sales.dat") + assert(Files.exists(catalogSalesDatPath), s"File $catalogSalesDatPath does not exists, " + + s"please follow instruction to generate the TPC-DS (SF1) text data first.") + + // the size of TPCDS catalog_sales.dat (SF1) is about 283M + data = Files.readAllBytes(catalogSalesDatPath) + } + + override def afterAll(): Unit = { + data = null + } +} diff --git a/core/src/test/scala/org/apache/spark/io/ZStandardTPCDSDataBenchmark.scala b/core/src/test/scala/org/apache/spark/io/ZStandardTPCDSDataBenchmark.scala index 69820c15be306..a546d03c8eb3f 100644 --- a/core/src/test/scala/org/apache/spark/io/ZStandardTPCDSDataBenchmark.scala +++ b/core/src/test/scala/org/apache/spark/io/ZStandardTPCDSDataBenchmark.scala @@ -18,10 +18,9 @@ package org.apache.spark.io import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectOutputStream, OutputStream} -import java.nio.file.{Files, Paths} import org.apache.spark.SparkConf -import org.apache.spark.benchmark.{Benchmark, BenchmarkBase} +import org.apache.spark.benchmark.Benchmark import org.apache.spark.internal.config.{IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED, IO_COMPRESSION_ZSTD_LEVEL, IO_COMPRESSION_ZSTD_WORKERS} /** @@ -34,28 +33,19 @@ import org.apache.spark.internal.config.{IO_COMPRESSION_ZSTD_BUFFERPOOL_ENABLED, * Results will be written to "benchmarks/ZStandardTPCDSDataBenchmark-results.txt". * }}} */ -object ZStandardTPCDSDataBenchmark extends BenchmarkBase { - - val N = 4 - - // the size of TPCDS catalog_sales.dat (SF1) is about 283M - val data = Files.readAllBytes(Paths.get(sys.env("SPARK_TPCDS_DATA_TEXT"), "catalog_sales.dat")) +object ZStandardTPCDSDataBenchmark extends TPCDSDataBenchmark { override def runBenchmarkSuite(mainArgs: Array[String]): Unit = { - val name = "Benchmark ZStandardCompressionCodec" - runBenchmark(name) { - val benchmark1 = new Benchmark(name, N, output = output) - compressionBenchmark(benchmark1, N) - benchmark1.run() - - val benchmark2 = new Benchmark(name, N, output = output) - decompressionBenchmark(benchmark2, N) - benchmark2.run() + prepareData() + runBenchmark("Benchmark ZStandardCompressionCodec") { + compressionBenchmark() + decompressionBenchmark() parallelCompressionBenchmark() } } - private def compressionBenchmark(benchmark: Benchmark, N: Int): Unit = { + private def compressionBenchmark(): Unit = { + val benchmark = new Benchmark("Compression", N, output = output) Seq(false, true).foreach { enablePool => Seq(1, 2, 3).foreach { level => val conf = new SparkConf(false) @@ -72,9 +62,11 @@ object ZStandardTPCDSDataBenchmark extends BenchmarkBase { } } } + benchmark.run() } - private def decompressionBenchmark(benchmark: Benchmark, N: Int): Unit = { + private def decompressionBenchmark(): Unit = { + val benchmark = new Benchmark("Decompression", N, output = output) Seq(false, true).foreach { enablePool => Seq(1, 2, 3).foreach { level => val conf = new SparkConf(false) @@ -97,6 +89,7 @@ object ZStandardTPCDSDataBenchmark extends BenchmarkBase { } } } + benchmark.run() } private def parallelCompressionBenchmark(): Unit = {