From b92bfd33c89277e2832d0b788861286f093e379a Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 3 Dec 2021 17:55:19 +0800 Subject: [PATCH 1/2] Update package.scala --- .../main/scala/org/apache/spark/internal/config/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 71a11f6d45018..ed7c640d742dc 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -1739,7 +1739,7 @@ package object config { "the codec") .version("0.8.0") .stringConf - .createWithDefaultString("lz4") + .createWithDefaultString("zstd") private[spark] val IO_COMPRESSION_ZSTD_BUFFERSIZE = ConfigBuilder("spark.io.compression.zstd.bufferSize") From ce477f454cd1efc1476e210c24f757eb515411eb Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 3 Dec 2021 17:57:18 +0800 Subject: [PATCH 2/2] Update configuration.md --- docs/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index 89871cbd3bf39..cad5a7a8b2386 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1452,7 +1452,7 @@ Apart from these, the following properties are also available, and may be useful spark.io.compression.codec - lz4 + zstd The codec used to compress internal data such as RDD partitions, event log, broadcast variables and shuffle outputs. By default, Spark provides four codecs: lz4, lzf,