From d5328c4fc0e2f6375ba9c5eee25e28b9973e68ef Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 9 Feb 2025 17:57:18 -0600 Subject: [PATCH] Validate PKzip header, throw for unknown compressions. Add tests for depot chunks. --- .../javasteam/steam/cdn/DepotChunk.kt | 219 +++++++++++------- .../in/dragonbra/javasteam/util/VZipUtil.kt | 8 +- .../in/dragonbra/javasteam/util/ZipUtil.kt | 1 + .../javasteam/steam/cdn/DepotChunkTest.java | 93 ++++++++ ...8567d9b3c09295cdbf4978c32b348d8e76c750.bin | 2 + ...c8e2657470b2eb70d6ddcd6c07004be8738697.bin | Bin 0 -> 320 bytes 6 files changed, 233 insertions(+), 90 deletions(-) create mode 100644 src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java create mode 100644 src/test/resources/depot/depot_232250_chunk_7b8567d9b3c09295cdbf4978c32b348d8e76c750.bin create mode 100644 src/test/resources/depot/depot_440_chunk_bac8e2657470b2eb70d6ddcd6c07004be8738697.bin diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt index cc01bae0..ea1e6160 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt @@ -1,87 +1,132 @@ -package `in`.dragonbra.javasteam.steam.cdn - -import `in`.dragonbra.javasteam.types.ChunkData -import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.Utils -import `in`.dragonbra.javasteam.util.VZipUtil -import `in`.dragonbra.javasteam.util.ZipUtil -import `in`.dragonbra.javasteam.util.crypto.CryptoHelper -import `in`.dragonbra.javasteam.util.stream.MemoryStream -import java.io.IOException -import javax.crypto.Cipher -import javax.crypto.spec.IvParameterSpec -import javax.crypto.spec.SecretKeySpec - -/** - * Provides a helper function to decrypt and decompress a single depot chunk. - */ -object DepotChunk { - - /** - * Processes the specified depot key by decrypting the data with the given depot encryption key, and then by decompressing the data. - * If the chunk has already been processed, this function does nothing. - * @param info The depot chunk data representing. - * @param data The encrypted chunk data. - * @param destination The buffer to receive the decrypted chunk data. - * @param depotKey The depot decryption key. - * @exception IOException Thrown if the processed data does not match the expected checksum given in its chunk information. - * @exception IllegalArgumentException Thrown if the destination size is too small or the depot key is not 32 bytes long - */ - fun process( - info: ChunkData, - data: ByteArray, - destination: ByteArray, - depotKey: ByteArray, - ): Int { - require(destination.size >= info.uncompressedLength) { - "The destination buffer must be longer than the chunk ${ChunkData::uncompressedLength.name}." - } - - require(depotKey.size == 32) { "Tried to decrypt depot chunk with non 32 byte key!" } - - // first 16 bytes of input is the ECB encrypted IV - val keySpec = SecretKeySpec(depotKey, "AES") - val ecbCipher = Cipher.getInstance("AES/ECB/NoPadding", CryptoHelper.SEC_PROV) - ecbCipher.init(Cipher.DECRYPT_MODE, keySpec) - - val iv = ByteArray(16) - val ivBytesRead = ecbCipher.doFinal(data, 0, iv.size, iv) - - require(iv.size == ivBytesRead) { "Failed to decrypt depot chunk iv (${iv.size} != $ivBytesRead)" } - - // With CBC and padding, the decrypted size will always be smaller - val buffer = ByteArray(data.size - iv.size) - val cbcCipher = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) - cbcCipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) - - val writtenDecompressed: Int - - try { - val bytesWrittenToBuffer = cbcCipher.doFinal(data, iv.size, data.size - iv.size, buffer) - - writtenDecompressed = if (buffer.size > 1 && buffer[0] == 'V'.code.toByte() && buffer[1] == 'Z'.code.toByte()) { - MemoryStream(buffer, 0, bytesWrittenToBuffer).use { ms -> - VZipUtil.decompress(ms, destination, verifyChecksum = false) - } - } else { - MemoryStream(buffer, 0, bytesWrittenToBuffer).use { ms -> - ZipUtil.decompress(ms, destination, verifyChecksum = false) - } - } - } catch (e: Exception) { - throw IOException("Failed to decompress chunk ${Strings.toHex(info.chunkID)}: $e\n${e.stackTraceToString()}") - } - - if (info.uncompressedLength != writtenDecompressed) { - throw IOException("Processed data checksum failed to decompress to the expected chunk uncompressed length. (was $writtenDecompressed, should be ${info.uncompressedLength})") - } - - val dataCrc = Utils.adlerHash(destination.sliceArray(0 until writtenDecompressed)) - - if (dataCrc != info.checksum) { - throw IOException("Processed data checksum is incorrect ($dataCrc != ${info.checksum})! Downloaded depot chunk is corrupt or invalid/wrong depot key?") - } - - return writtenDecompressed - } -} +package `in`.dragonbra.javasteam.steam.cdn + +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.util.Strings +import `in`.dragonbra.javasteam.util.Utils +import `in`.dragonbra.javasteam.util.VZipUtil +import `in`.dragonbra.javasteam.util.ZipUtil +import `in`.dragonbra.javasteam.util.crypto.CryptoHelper +import `in`.dragonbra.javasteam.util.stream.MemoryStream +import java.io.IOException +import javax.crypto.Cipher +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec + +/** + * Provides a helper function to decrypt and decompress a single depot chunk. + */ +object DepotChunk { + + /** + * Processes the specified depot key by decrypting the data with the given depot encryption key, and then by decompressing the data. + * If the chunk has already been processed, this function does nothing. + * @param info The depot chunk data representing. + * @param data The encrypted chunk data. + * @param destination The buffer to receive the decrypted chunk data. + * @param depotKey The depot decryption key. + * @exception IOException Thrown if the processed data does not match the expected checksum given in its chunk information. + * @exception IllegalArgumentException Thrown if the destination size is too small or the depot key is not 32 bytes long + */ + @JvmStatic + fun process( + info: ChunkData, + data: ByteArray, + destination: ByteArray, + depotKey: ByteArray, + ): Int { + require(destination.size >= info.uncompressedLength) { + "The destination buffer must be longer than the chunk ${ChunkData::uncompressedLength.name}." + } + + require(depotKey.size == 32) { "Tried to decrypt depot chunk with non 32 byte key!" } + + // first 16 bytes of input is the ECB encrypted IV + val keySpec = SecretKeySpec(depotKey, "AES") + val ecbCipher = Cipher.getInstance("AES/ECB/NoPadding", CryptoHelper.SEC_PROV) + ecbCipher.init(Cipher.DECRYPT_MODE, keySpec) + + val iv = ByteArray(16) + val ivBytesRead = ecbCipher.doFinal(data, 0, iv.size, iv) + + require(iv.size == ivBytesRead) { "Failed to decrypt depot chunk iv (${iv.size} != $ivBytesRead)" } + + // With CBC and padding, the decrypted size will always be smaller + val buffer = ByteArray(data.size - iv.size) + val cbcCipher = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) + cbcCipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) + + val writtenDecompressed: Int + + try { + val written = cbcCipher.doFinal(data, iv.size, data.size - iv.size, buffer) + + // Per SK: + // Steam client checks for like 20 bytes for pkzip, and 22 bytes for vzip, + // I'm just being safe and checking for a smaller value. + if (buffer.size < 16) { + throw IOException("Not enough data in the decrypted depot chunk (was ${buffer.size} bytes).") + } + + if (buffer[0] == 'V'.code.toByte() && + buffer[1] == 'S'.code.toByte() && + buffer[2] == 'Z'.code.toByte() && + buffer[3] == 'a'.code.toByte() + ) { + // Zstd + throw RuntimeException("Zstd compressed chunks are not yet implemented in JavaSteam.") + } else if (buffer[0] == 'V'.code.toByte() && + buffer[1] == 'Z'.code.toByte() && + buffer[2] == 'a'.code.toByte() + ) { + // LZMA + MemoryStream(buffer, 0, written).use { decryptedStream -> + writtenDecompressed = VZipUtil.decompress( + ms = decryptedStream, + destination = destination, + verifyChecksum = false + ) + } + } else if (buffer[0] == 'P'.code.toByte() && + buffer[1] == 'K'.code.toByte() && + buffer[2].toInt() == 0x03 && + buffer[3].toInt() == 0x04 + ) { + // Per SK: + // Steam client code performs the same check. + + // PKzip + MemoryStream(buffer, 0, written).use { decryptedStream -> + writtenDecompressed = ZipUtil.decompress( + ms = decryptedStream, + destination = destination, + verifyChecksum = false + ) + } + } else { + throw IOException( + "Unexpected depot chunk compression " + + "(first four bytes are ${Strings.toHex(buffer.copyOfRange(0, 4))})." + ) + } + } catch (e: Exception) { + throw IOException("Failed to decompress chunk ${Strings.toHex(info.chunkID)}: $e\n${e.stackTraceToString()}") + } finally { + buffer.fill(0) + } + + if (info.uncompressedLength != writtenDecompressed) { + throw IOException( + "Processed data checksum failed to decompress to the expected chunk uncompressed length. " + + "(was $writtenDecompressed, should be ${info.uncompressedLength})" + ) + } + + val dataCrc = Utils.adlerHash(destination.sliceArray(0 until writtenDecompressed)) + + if (dataCrc != info.checksum) { + throw IOException("Processed data checksum is incorrect ($dataCrc != ${info.checksum})! Downloaded depot chunk is corrupt or invalid/wrong depot key?") + } + + return writtenDecompressed + } +} diff --git a/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt b/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt index f626b8c3..021007f1 100644 --- a/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt +++ b/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt @@ -21,15 +21,16 @@ object VZipUtil { private const val HEADER_LENGTH = 7 // magic + version + timestamp/crc private const val FOOTER_LENGTH = 10 // crc + decompressed size + magic - private const val VERSION = 'a' + private const val VERSION: Byte = 'a'.code.toByte() + @JvmStatic fun decompress(ms: MemoryStream, destination: ByteArray, verifyChecksum: Boolean = true): Int { BinaryReader(ms).use { reader -> if (reader.readShort() != VZIP_HEADER) { throw IllegalArgumentException("Expecting VZipHeader at start of stream") } - if (reader.readChar() != VERSION) { + if (reader.readByte() != VERSION) { throw IllegalArgumentException("Expecting VZip version 'a'") } @@ -84,12 +85,13 @@ object VZipUtil { /** * Ported from SteamKit2 and is untested, use at your own risk */ + @JvmStatic fun compress(buffer: ByteArray): ByteArray { ByteArrayOutputStream().use { ms -> BinaryWriter(ms).use { writer -> val crc = CryptoHelper.crcHash(buffer) writer.writeShort(VZIP_HEADER) - writer.writeChar(VERSION) + writer.writeByte(VERSION) writer.write(crc) // Configure LZMA options to match SteamKit2's settings diff --git a/src/main/java/in/dragonbra/javasteam/util/ZipUtil.kt b/src/main/java/in/dragonbra/javasteam/util/ZipUtil.kt index f52242af..c5a24214 100644 --- a/src/main/java/in/dragonbra/javasteam/util/ZipUtil.kt +++ b/src/main/java/in/dragonbra/javasteam/util/ZipUtil.kt @@ -6,6 +6,7 @@ import java.util.zip.ZipInputStream object ZipUtil { + @JvmStatic fun decompress(ms: MemoryStream, destination: ByteArray, verifyChecksum: Boolean = true): Int { ZipInputStream(ms, Charsets.UTF_8).use { zip -> val entry = zip.nextEntry diff --git a/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java b/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java new file mode 100644 index 00000000..8b575519 --- /dev/null +++ b/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java @@ -0,0 +1,93 @@ +package in.dragonbra.javasteam.steam.cdn; + +import in.dragonbra.javasteam.types.ChunkData; +import in.dragonbra.javasteam.util.stream.MemoryStream; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.apache.commons.codec.binary.Hex; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class DepotChunkTest { + + @Test + public void decryptsAndDecompressesDepotChunkPKZip() throws IOException, NoSuchAlgorithmException { + var stream = getClass().getClassLoader() + .getResourceAsStream("depot/depot_440_chunk_bac8e2657470b2eb70d6ddcd6c07004be8738697.bin"); + + var ms = new MemoryStream(); + IOUtils.copy(stream, ms.asOutputStream()); + + var chunkData = ms.toByteArray(); + + var chunk = new ChunkData( + new byte[0], // id is not needed here + 2130218374, + 0, + 320, + 544 + ); + + var destination = new byte[chunk.getUncompressedLength()]; + var writtenLength = DepotChunk.process( + chunk, + chunkData, + destination, + new byte[]{ + (byte) 0x44, (byte) 0xCE, (byte) 0x5C, (byte) 0x52, (byte) 0x97, (byte) 0xA4, (byte) 0x15, (byte) 0xA1, + (byte) 0xA6, (byte) 0xF6, (byte) 0x9C, (byte) 0x85, (byte) 0x60, (byte) 0x37, (byte) 0xA5, (byte) 0xA2, + (byte) 0xFD, (byte) 0xD8, (byte) 0x2C, (byte) 0xD4, (byte) 0x74, (byte) 0xFA, (byte) 0x65, (byte) 0x9E, + (byte) 0xDF, (byte) 0xB4, (byte) 0xD5, (byte) 0x9B, (byte) 0x2A, (byte) 0xBC, (byte) 0x55, (byte) 0xFC + } + ); + + Assertions.assertEquals(chunk.getCompressedLength(), chunkData.length); + Assertions.assertEquals(chunk.getUncompressedLength(), writtenLength); + + var hash = Hex.encodeHexString(MessageDigest.getInstance("SHA-1").digest(destination), false); + + Assertions.assertEquals("BAC8E2657470B2EB70D6DDCD6C07004BE8738697", hash); + } + + @Test + public void decryptsAndDecompressesDepotChunkVZip() throws IOException, NoSuchAlgorithmException { + var stream = getClass().getClassLoader() + .getResourceAsStream("depot/depot_232250_chunk_7b8567d9b3c09295cdbf4978c32b348d8e76c750.bin"); + + var ms = new MemoryStream(); + IOUtils.copy(stream, ms.asOutputStream()); + + var chunkData = ms.toByteArray(); + + var chunk = new ChunkData( + new byte[0], // id is not needed here + Integer.parseUnsignedInt("2894626744"), + 0, + 304, + 798 + ); + + var destination = new byte[chunk.getUncompressedLength()]; + var writtenLength = DepotChunk.process( + chunk, + chunkData, + destination, + new byte[]{ + (byte) 0xE5, (byte) 0xF6, (byte) 0xAE, (byte) 0xD5, (byte) 0x5E, (byte) 0x9E, (byte) 0xCE, (byte) 0x42, + (byte) 0x9E, (byte) 0x56, (byte) 0xB8, (byte) 0x13, (byte) 0xFB, (byte) 0xF6, (byte) 0xBF, (byte) 0xE9, + (byte) 0x24, (byte) 0xF3, (byte) 0xCF, (byte) 0x72, (byte) 0x97, (byte) 0x2F, (byte) 0xDB, (byte) 0xD0, + (byte) 0x57, (byte) 0x1F, (byte) 0xFC, (byte) 0xAD, (byte) 0x9F, (byte) 0x2F, (byte) 0x7D, (byte) 0xAA, + } + ); + + Assertions.assertEquals(chunk.getCompressedLength(), chunkData.length); + Assertions.assertEquals(chunk.getUncompressedLength(), writtenLength); + + var hash = Hex.encodeHexString(MessageDigest.getInstance("SHA-1").digest(destination), false); + + Assertions.assertEquals("7B8567D9B3C09295CDBF4978C32B348D8E76C750", hash); + } +} diff --git a/src/test/resources/depot/depot_232250_chunk_7b8567d9b3c09295cdbf4978c32b348d8e76c750.bin b/src/test/resources/depot/depot_232250_chunk_7b8567d9b3c09295cdbf4978c32b348d8e76c750.bin new file mode 100644 index 00000000..bfce45b0 --- /dev/null +++ b/src/test/resources/depot/depot_232250_chunk_7b8567d9b3c09295cdbf4978c32b348d8e76c750.bin @@ -0,0 +1,2 @@ +ȔY%OY.i֐ϭԐŎō +c*, N>nTsOآ˝ ުXlo`,4߱pE/[ˆ# T=<yӃR"# 7$^9t?Y/_YUu;sPbw8 wު) \Q|A ܋-" Cι1ZWP$ \ No newline at end of file diff --git a/src/test/resources/depot/depot_440_chunk_bac8e2657470b2eb70d6ddcd6c07004be8738697.bin b/src/test/resources/depot/depot_440_chunk_bac8e2657470b2eb70d6ddcd6c07004be8738697.bin new file mode 100644 index 0000000000000000000000000000000000000000..9885d969b77412370c5fa964a3805fea23452bac GIT binary patch literal 320 zcmV-G0l)rounux<=gqp^(ZbpI#orp>1r1%As`31#R=efq0?9$!<2DE+))IX8qy8Hr zd6x1DVnLfoGX*<(n+fxZ9YD_7l>^->Ew?{GSULX$waN%;Y3+_tc<033lRrekUsGC< z(#L^o#n=4-LH^u!k17R;AQpmbv6)84a#XHBB#5R9wyy+i4q7-Sq*B~F@ehW|X18ty!C1m{ S1JDA)eJNpSZ4>Zj5{gs3I-C~( literal 0 HcmV?d00001