From ad4277b59cfa00ad92909d228742261b41f66c1a Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 18 Feb 2025 01:22:45 -0600 Subject: [PATCH 1/4] Fix parsing v4 depot manifests with tests. Note: Tests failing at this commit. --- .../javasteam/types/DepotManifest.kt | 103 ++++---- .../javasteam/types/Steam3Manifest.kt | 7 + .../javasteam/util/stream/MemoryStream.java | 7 + .../javasteam/types/DepotManifestTest.java | 242 ++++++++++++++++++ .../depot_440_1118032470228587934.manifest | Bin 0 -> 1271 bytes ...440_1118032470228587934_decrypted.manifest | Bin 0 -> 845 bytes .../depot_440_1118032470228587934_v4.manifest | Bin 0 -> 1186 bytes 7 files changed, 316 insertions(+), 43 deletions(-) create mode 100644 src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java create mode 100644 src/test/resources/depot/depot_440_1118032470228587934.manifest create mode 100644 src/test/resources/depot/depot_440_1118032470228587934_decrypted.manifest create mode 100644 src/test/resources/depot/depot_440_1118032470228587934_v4.manifest diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index a34cb018..220c3f6a 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -47,6 +47,7 @@ class DepotManifest { * @param stream Raw depot manifest stream to deserialize. * @exception NoSuchElementException Thrown if the given data is not something recognizable. */ + @JvmStatic fun deserialize(stream: InputStream): DepotManifest = deserialize(stream.readBytes()) /** @@ -55,6 +56,7 @@ class DepotManifest { * @param data Raw depot manifest data to deserialize. * @exception NoSuchElementException Thrown if the given data is not something recognizable. */ + @JvmStatic fun deserialize(data: ByteArray): DepotManifest = MemoryStream(data).use { ms -> val manifest = DepotManifest() manifest.internalDeserialize(ms) @@ -69,6 +71,7 @@ class DepotManifest { * @exception NoSuchElementException Thrown if the given data is not something recognizable. */ @Suppress("unused") + @JvmStatic fun loadFromFile(filename: String): DepotManifest? { val file = File(filename) if (!file.exists()) { @@ -169,7 +172,7 @@ class DepotManifest { var iv: ByteArray try { - for (file in files) { + files.forEach { file -> val decoded = Base64.getUrlDecoder().decode( file.fileName .replace('+', '-') @@ -252,6 +255,10 @@ class DepotManifest { if (marker != magic) { throw NoSuchElementException("Unable to find end of message marker for depot manifest") } + + // This is an intentional return because v4 manifest does not have the separate sections, + // and it will be parsed by ParseBinaryManifest. If we get here, the entire buffer has been already processed. + return } PROTOBUF_PAYLOAD_MAGIC -> { @@ -275,8 +282,8 @@ class DepotManifest { } if (payload != null && metadata != null && signature != null) { - parseProtobufManifestMetadata(metadata!!) - parseProtobufManifestPayload(payload!!) + parseProtobufManifestMetadata(metadata) + parseProtobufManifestPayload(payload) } else { throw NoSuchElementException("Missing ContentManifest sections required for parsing depot manifest") } @@ -290,8 +297,9 @@ class DepotManifest { creationTime = manifest.creationTime totalUncompressedSize = manifest.totalUncompressedSize totalCompressedSize = manifest.totalCompressedSize + encryptedCRC = manifest.encryptedCRC - for (fileMapping in manifest.fileMapping) { + manifest.fileMapping.forEach { fileMapping -> val fileData = FileData( fileName = fileMapping.fileName, fileNameHash = fileMapping.hashFileName, @@ -302,7 +310,7 @@ class DepotManifest { encrypted = filenamesEncrypted ) - for (chunk in fileMapping.chunks) { + fileMapping.chunks.forEach { chunk -> fileData.chunks.add( ChunkData( chunkID = chunk.chunkGID, @@ -321,7 +329,7 @@ class DepotManifest { internal fun parseProtobufManifestPayload(payload: ContentManifestPayload) { files.clear() - for (fileMapping in payload.mappingsList) { + payload.mappingsList.forEach { fileMapping -> val fileData = FileData( fileName = fileMapping.filename, fileNameHash = fileMapping.shaFilename.toByteArray(), @@ -332,7 +340,7 @@ class DepotManifest { encrypted = filenamesEncrypted ) - for (chunk in fileMapping.chunksList) { + fileMapping.chunksList.forEach { chunk -> fileData.chunks.add( ChunkData( chunkID = chunk.sha.toByteArray(), @@ -376,36 +384,35 @@ class DepotManifest { val payload = ContentManifestPayload.newBuilder() val uniqueChunks = hashSetOf() - for (file in files) { - val protoFile = ContentManifestPayload.FileMapping.newBuilder() - protoFile.setSize(file.totalSize) - protoFile.setFlags(EDepotFileFlag.code(file.flags)) + files.forEach { file -> + val protoFile = ContentManifestPayload.FileMapping.newBuilder().apply { + size = file.totalSize + flags = EDepotFileFlag.code(file.flags) + } if (filenamesEncrypted) { // Assume the name is unmodified - protoFile.setFilename(file.fileName) - protoFile.setShaFilename(ByteString.copyFrom(file.fileNameHash)) + protoFile.filename = file.fileName + protoFile.shaFilename = ByteString.copyFrom(file.fileNameHash) } else { - protoFile.setFilename(file.fileName.replace('/', '\\')) - protoFile.setShaFilename( - ByteString.copyFrom( - CryptoHelper.shaHash( - file.fileName - .replace('/', '\\') - .lowercase() - .toByteArray(Charsets.UTF_8) - ) + protoFile.filename = file.fileName.replace('/', '\\') + protoFile.shaFilename = ByteString.copyFrom( + CryptoHelper.shaHash( + file.fileName + .replace('/', '\\') + .lowercase() + .toByteArray(Charsets.UTF_8) ) ) } - protoFile.setShaContent(ByteString.copyFrom(file.fileHash)) + protoFile.shaContent = ByteString.copyFrom(file.fileHash) if (file.linkTarget.isNotBlank()) { protoFile.linktarget = file.linkTarget } - for (chunk in file.chunks) { + file.chunks.forEach { chunk -> val protoChunk = ContentManifestPayload.FileMapping.ChunkData.newBuilder().apply { sha = ByteString.copyFrom(chunk.chunkID) crc = chunk.checksum @@ -432,36 +439,46 @@ class DepotManifest { } // Calculate payload CRC - val payloadData = payload.build().toByteArray() - val len = payloadData.size - val data = ByteArray(Int.SIZE_BYTES + len) + MemoryStream().use { msPayload -> + payload.build().writeTo(msPayload.asOutputStream()) - System.arraycopy(ByteBuffer.allocate(Int.SIZE_BYTES).putInt(len).array(), 0, data, 0, 4) - System.arraycopy(payloadData, 0, data, 4, len) + val len = msPayload.length.toInt() + val data = ByteArray(4 + len) + System.arraycopy(ByteBuffer.allocate(4).putInt(len).array(), 0, data, 0, 4) + System.arraycopy(msPayload.toByteArray(), 0, data, 4, len) + val crc32 = Utils.crc32(data).toInt() - val crc32 = Utils.crc32(payloadData).toInt() + if (filenamesEncrypted) { + metadata.crcEncrypted = crc32 + metadata.crcClear = 0 + } else { + metadata.crcEncrypted = encryptedCRC + metadata.crcClear = crc32 + } - if (filenamesEncrypted) { - metadata.setCrcEncrypted(crc32) - metadata.setCrcClear(0) - } else { - metadata.setCrcEncrypted(encryptedCRC) - metadata.setCrcClear(crc32) + msPayload.toByteArray() } // Write the manifest to the stream and return the checksum return ByteArrayOutputStream().use { bw -> BinaryWriter(bw).use { writer -> // Write Protobuf payload - writer.writeInt(PROTOBUF_PAYLOAD_MAGIC) - writer.writeInt(payloadData.size) - writer.write(payloadData, 0, payloadData.size) + MemoryStream().use { msPayload -> + payload.build().writeTo(msPayload.asOutputStream()) + + writer.writeInt(PROTOBUF_PAYLOAD_MAGIC) + writer.writeInt(msPayload.length.toInt()) + writer.write(msPayload.buffer, 0, msPayload.length.toInt()) + } // Write Protobuf metadata - val metadataData = metadata.build().toByteArray() - writer.writeInt(PROTOBUF_METADATA_MAGIC) - writer.writeInt(metadataData.size) - writer.write(metadataData, 0, metadataData.size) + MemoryStream().use { msMetaData -> + metadata.build().writeTo(msMetaData.asOutputStream()) + + writer.writeInt(PROTOBUF_METADATA_MAGIC) + writer.writeInt(msMetaData.length.toInt()) + writer.write(msMetaData.buffer, 0, msMetaData.length.toInt()) + } // Write empty signature section writer.writeInt(PROTOBUF_SIGNATURE_MAGIC) diff --git a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt index 745ea5b3..a3c736e5 100644 --- a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt @@ -5,6 +5,7 @@ package `in`.dragonbra.javasteam.types import `in`.dragonbra.javasteam.enums.EDepotFileFlag import `in`.dragonbra.javasteam.util.compat.readNBytesCompat import `in`.dragonbra.javasteam.util.stream.BinaryReader +import java.lang.Exception import java.time.Instant import java.util.Date import java.util.EnumSet @@ -87,6 +88,12 @@ class Steam3Manifest( internal fun deserialize(ds: BinaryReader): Steam3Manifest { // The magic is verified by DepotManifest.InternalDeserialize, not checked here to avoid seeking val version = ds.readInt() + + if (version != CURRENT_VERSION) { + // Not Implemented Exception + throw Exception("Only version $CURRENT_VERSION is supported") + } + val depotID = ds.readInt() val manifestGID = ds.readLong() val creationTime = Date.from(Instant.ofEpochSecond(ds.readInt().toLong())) diff --git a/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java b/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java index f3c687d5..e97e3ef4 100644 --- a/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java +++ b/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java @@ -1,8 +1,10 @@ package in.dragonbra.javasteam.util.stream; import com.google.protobuf.ByteString; +import org.jetbrains.annotations.NotNull; import java.io.Closeable; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; @@ -379,6 +381,11 @@ public byte[] toByteArray() { return ret; } + @Override + public byte[] readAllBytes() { + return toByteArray(); + } + /** * Get an OutputStream that will write to this MemoryStream, at the current position. * diff --git a/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java b/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java new file mode 100644 index 00000000..b6505f7c --- /dev/null +++ b/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java @@ -0,0 +1,242 @@ +package in.dragonbra.javasteam.types; + +import in.dragonbra.javasteam.enums.EDepotFileFlag; +import in.dragonbra.javasteam.util.Strings; +import in.dragonbra.javasteam.util.crypto.CryptoHelper; +import in.dragonbra.javasteam.util.stream.BinaryReader; +import in.dragonbra.javasteam.util.stream.MemoryStream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Arrays; + +@SuppressWarnings({"resource", "DataFlowIssue"}) +public class DepotManifestTest { + + private static final byte[] DEPOT_440_DECRYPTION_KEY = new byte[]{ + (byte) 0x44, (byte) 0xCE, (byte) 0x5C, (byte) 0x52, (byte) 0x97, (byte) 0xA4, (byte) 0x15, (byte) 0xA1, + (byte) 0xA6, (byte) 0xF6, (byte) 0x9C, (byte) 0x85, (byte) 0x60, (byte) 0x37, (byte) 0xA5, (byte) 0xA2, + (byte) 0xFD, (byte) 0xD8, (byte) 0x2C, (byte) 0xD4, (byte) 0x74, (byte) 0xFA, (byte) 0x65, (byte) 0x9E, + (byte) 0xDF, (byte) 0xB4, (byte) 0xD5, (byte) 0x9B, (byte) 0x2A, (byte) 0xBC, (byte) 0x55, (byte) 0xFC + }; + + + @Test + public void parsesAndDecryptsManifestVersion4() throws IOException, NoSuchAlgorithmException { + var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_v4.manifest"); + + var ms = new MemoryStream(); + stream.transferTo(ms.asOutputStream()); + + var manifestData = ms.toByteArray(); + + var depotManifest = DepotManifest.deserialize(manifestData); + + Assertions.assertTrue(depotManifest.getFilenamesEncrypted()); + Assertions.assertEquals(1195249848L, depotManifest.getEncryptedCRC()); + + depotManifest.decryptFilenames(DEPOT_440_DECRYPTION_KEY); + + testDecryptedManifest(depotManifest); + } + + @Test + public void parsesAndDecryptsManifest() throws IOException, NoSuchAlgorithmException { + var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934.manifest"); + + var ms = new MemoryStream(); + stream.transferTo(ms.asOutputStream()); + + var manifestData = ms.toByteArray(); + + var depotManifest = DepotManifest.deserialize(manifestData); + + Assertions.assertTrue(depotManifest.getFilenamesEncrypted()); + Assertions.assertEquals(1606273976L, depotManifest.getEncryptedCRC()); + + depotManifest.decryptFilenames(DEPOT_440_DECRYPTION_KEY); + + testDecryptedManifest(depotManifest); + } + + @Test + public void parsesDecryptedManifest() throws IOException, NoSuchAlgorithmException { + var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_decrypted.manifest"); + + var ms = new MemoryStream(); + stream.transferTo(ms.asOutputStream()); + + var manifestData = ms.toByteArray(); + + var depotManifest = DepotManifest.deserialize(manifestData); + + testDecryptedManifest(depotManifest); + } + + @Test + public void roundtripSerializesManifestEncryptedManifest() throws IOException { + var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934.manifest"); + + var ms = new MemoryStream(); + stream.transferTo(ms.asOutputStream()); + + var manifestData = ms.toByteArray(); + + var depotManifest = DepotManifest.deserialize(manifestData); + + var actualStream = new MemoryStream(); + depotManifest.serialize(actualStream.asOutputStream()); + + var actual = actualStream.toByteArray(); + + // We are unable to write signatures, so validate everything except for the signature + var signature = new byte[]{(byte) 0x17, (byte) 0xB8, (byte) 0x81, (byte) 0x1B}; + + int actualOffset = indexOf(actual, signature); // DepotManifest.PROTOBUF_SIGNATURE_MAGIC + int expectedOffset = indexOf(manifestData, signature); + + byte expectedByte = manifestData[1109]; + byte actualByte = actual[1109]; + System.out.println("At index 1109:"); + System.out.println("Expected byte: " + expectedByte); + System.out.println("Actual byte: " + actualByte); + +// Maybe also look at surrounding bytes for context + System.out.println("Expected bytes around 1109:"); + for (int i = 1105; i < 1115; i++) { + System.out.println("Index " + i + ": " + manifestData[i]); + } + System.out.println("Actual bytes around 1109:"); + for (int i = 1105; i < 1115; i++) { + System.out.println("Index " + i + ": " + actual[i]); + } + + Assertions.assertTrue(actualOffset > 0); + Assertions.assertTrue(expectedOffset > 0); + Assertions.assertArrayEquals( + Arrays.copyOfRange(manifestData, 0, expectedOffset), + Arrays.copyOfRange(actual, 0, actualOffset) + ); + + // We dont have `BitConverter.ToInt32` + int expectedSignatureLength; + try (var bais = new ByteArrayInputStream(manifestData); + var br = new BinaryReader(bais)) { + expectedSignatureLength = br.readBytes(expectedOffset + 4).length; + } + int actualSignatureLength; + try (var bais = new ByteArrayInputStream(manifestData); + var br = new BinaryReader(bais)) { + actualSignatureLength = br.readBytes(expectedOffset + 4).length; + } + + //int expectedSignatureLength = readInt32(manifestData, expectedOffset + 4); + // int actualSignatureLength = readInt32(actual, actualOffset + 4); + + Assertions.assertEquals(131, expectedSignatureLength); + Assertions.assertEquals(0, actualSignatureLength); + Assertions.assertArrayEquals( + Arrays.copyOfRange(manifestData, expectedOffset + expectedSignatureLength + 8, manifestData.length), + Arrays.copyOfRange(actual, actualOffset + 8, actual.length) + ); + } + + @Test + public void roundtripSerializesManifestByteIdentical() throws IOException { + var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_decrypted.manifest"); + + var ms = new MemoryStream(); + stream.transferTo(ms.asOutputStream()); + + var manifestData = ms.toByteArray(); + + var depotManifest = DepotManifest.deserialize(manifestData); + + var actualStream = new MemoryStream(); + depotManifest.serialize(actualStream.asOutputStream()); + + var actual = actualStream.toByteArray(); + + Assertions.assertArrayEquals(manifestData, actual); + } + + private void testDecryptedManifest(DepotManifest depotManifest) throws NoSuchAlgorithmException { + Assertions.assertFalse(depotManifest.getFilenamesEncrypted()); + Assertions.assertEquals(440L, depotManifest.getDepotID()); + Assertions.assertEquals(1118032470228587934L, depotManifest.getManifestGID()); + Assertions.assertEquals(825745L, depotManifest.getTotalUncompressedSize()); + Assertions.assertEquals(43168L, depotManifest.getTotalCompressedSize()); + Assertions.assertEquals(7, depotManifest.getFiles().size()); + Assertions.assertEquals( + ZonedDateTime.of(2013, 4, 17, 20, 39, 24, 0, ZoneOffset.UTC).toInstant(), + depotManifest.getCreationTime().toInstant() + ); + + Assertions.assertEquals(Path.of("bin", "dxsupport.cfg").toString(), depotManifest.getFiles().get(0).getFileName()); + Assertions.assertEquals(Path.of("bin", "dxsupport.csv").toString(), depotManifest.getFiles().get(1).getFileName()); + Assertions.assertEquals(Path.of("bin", "dxsupport_episodic.cfg").toString(), depotManifest.getFiles().get(2).getFileName()); + Assertions.assertEquals(Path.of("bin", "dxsupport_sp.cfg").toString(), depotManifest.getFiles().get(3).getFileName()); + Assertions.assertEquals(Path.of("bin", "vidcfg.bin").toString(), depotManifest.getFiles().get(4).getFileName()); + Assertions.assertEquals(Path.of("hl2", "media", "startupvids.txt").toString(), depotManifest.getFiles().get(5).getFileName()); + Assertions.assertEquals(Path.of("tf", "media", "startupvids.txt").toString(), depotManifest.getFiles().get(6).getFileName()); + + Assertions.assertEquals(EDepotFileFlag.from(0), depotManifest.getFiles().get(0).getFlags()); + Assertions.assertEquals(398709L, depotManifest.getFiles().get(0).getTotalSize()); + Assertions.assertArrayEquals( + Strings.decodeHex("BAC8E2657470B2EB70D6DDCD6C07004BE8738697"), + depotManifest.getFiles().get(2).getFileHash() + ); + + for (var file : depotManifest.getFiles()) { + Assertions.assertArrayEquals( + file.getFileNameHash(), + CryptoHelper.shaHash(file.getFileName().replace('/', '\\').getBytes()) + ); + Assertions.assertNotNull(file.getLinkTarget()); + Assertions.assertEquals(1, file.getChunks().size()); + } + + var chunk = depotManifest.getFiles().get(6).getChunks().get(0); + Assertions.assertEquals(963249608L, chunk.getChecksum()); + Assertions.assertEquals(144L, chunk.getCompressedLength()); + Assertions.assertEquals(17L, chunk.getUncompressedLength()); + Assertions.assertEquals(0L, chunk.getOffset()); + Assertions.assertArrayEquals( + Strings.decodeHex("94020BDE145A521EDEC9A9424E7A90FD042481E9"), + chunk.getChunkID() + ); + } + + // Java or Apache doesn't have a indexOf(byte[], byte[]) + // This is taken from guava since we don't have that lib as a dependency. + private static int indexOf(byte[] array, byte[] target) { + if (target.length == 0) { + return 0; + } + + outer: + for (int i = 0; i < array.length - target.length + 1; i++) { + for (int j = 0; j < target.length; j++) { + if (array[i + j] != target[j]) { + continue outer; + } + } + return i; + } + return -1; + } + + private static int readInt32(byte[] data, int offset) { + return (data[offset] & 0xFF) | + ((data[offset + 1] & 0xFF) << 8) | + ((data[offset + 2] & 0xFF) << 16) | + ((data[offset + 3] & 0xFF) << 24); + } +} diff --git a/src/test/resources/depot/depot_440_1118032470228587934.manifest b/src/test/resources/depot/depot_440_1118032470228587934.manifest new file mode 100644 index 0000000000000000000000000000000000000000..da6770ec15e6159a03b05da9024f281fb2828b2f GIT binary patch literal 1271 zcmZwFZA=?=90%~0$9Y?{kapl4kD{w>u*a2hEprb%w7qieT}#XL+7~E&qF35-P$-2g zID#Xvi32A?vbiB+WE(Lf3<3i)VIpChC>k;ynG79sESrghftU^TYK*V?_M3mc`Q-ci z&&6$2WJIu7Kqni>DYJt*LoQZ^yQS4GlMyaAX=+tsCs=4Q(3ntBqAvsU{W`HFw~nZ= zcu<>-qNxJNWq=?BWG_${0ZvRjE0r6&Cud2heZYQqX#cJGt)YD{&n%P*xWI#3#QLKD zZe`w3Yww}!30Wt<>4j2(kU8)B$P==7R&or#jSV!ifplDk$<_L5U!7CybXF@fu~MH& zhdSgnO2~=%igcLHSS2MiBo!##2XMGSMeXti*-&?{LeV#P{($R)mEvFd8v<@?8$8i9 zJf)n*-c!LX`c7r4U_7V@mGjOe2igBzhrN~4y5swzW%E}}Dwv;h@n(u);{ zh9qrBWuD%VX|JwCjS2))=9N;2RH<{@v!D!uazkLjfBX2c^Wr&+Q5=q-mrvPHH$u5BwKt>IBn7Aoh>M7sXp=M*E;s$n~d%PkH`nJl2COAJ1W zhQ*nFmq+1OYOSycE>OyzCsHT`jIE#e1&# z^|0S|ei3KmS#f%yPrzMRyX`vE^z+)y$%el>*;ixO_2tb_8W1w)g%2iWF{?3!{H1!J zi_uj;sJOzK;VqPc${GnNl)6mbEWZ<1%gruWB2fsnV2N4iCe?~R>OdUQTA>f8GPD4vYa=#@oBgV4Z|c@&>Tr5W|J$0S0&Cin4gr^l z>FJA6B|iMPIDi_Ar|aHdZJ&T%148D!W2E~N2FDcgyAQL0;9&{823CoXJg2-27Zw#L zwHX1zC90wn<#un8C12(W*gY<}->R0;2<#+qg{@ZSG9a)Cf^kKj#SCyhYmE-_p50i9 z{k<9evS)I~xl5O`pSI*{z7}wY!_&p;zNouHkB>~mzuDJOJ=UHRJZz|(cVyuCFvDXC z`2*@}u}IP$7K^oQG@Ns$|J#MJ6OGqjihthr`LW1kHoyG}1a^NHCXGHg+WE&;B*Kat zZFr@Tc@7+APp|zYzmxJ|(zdiC_^yPj4-=w(oNgTH77l!Rw0T){Aa+?f&!dm;K9kU~ zdNKQl^*RUmEOUgO58TsQtyKE)-6QtZ;y>Fso$zK)&6TKw$!EDO7hV*3K#Awt3vsCF p>8x{n?|6bRdqS1?NSRl(eL-;3wtr{u{kfBOcE#k|sD9oQ^bZ9f;vxV5 literal 0 HcmV?d00001 diff --git a/src/test/resources/depot/depot_440_1118032470228587934_decrypted.manifest b/src/test/resources/depot/depot_440_1118032470228587934_decrypted.manifest new file mode 100644 index 0000000000000000000000000000000000000000..0edcb8e95ecd3a595fda602d2f5fdc8d0021c21b GIT binary patch literal 845 zcmca0{;iOWnSp^Tn@cb$GcP8kqPVo6AitHVt`xo3=rX_Og^R=v+wuhfj9c=tRopGliV_klSkqQ@9aZ%vfn3P2z}`AKD_84&*D|%^nfl7daI3$EQgo5RlF{i3*Sx{)#$R&mj`-XC>9OBb+c9AM(g z=Mq7+v$z0k+sto5AnVQ^EwNVm`$K7~uHx!=-v?gFDsQH0iG&F*S|;cz_k2cNg?C8! zoT{FW6Au`va$yx0?T*Nq3$zYspvHm*My?btK9GfFnJK_{(*rUEM8GlkO zmI&9=(_vrlmE9?F-qO3I=8&w}v~x>97Ge_@y~H}%5@?|y$ih4>iIOyY_Lcc9HNKD( zZp`fAy?%B|SoT@%oX1)sQ<%8#i9`j--8;F`$**d{Ulx_dmmvGFiHn|K&4SstPsl@F zn}LCWV+XUqyw&G#?V8+th+pE*ggxD?3Je+(Hyar&xX5I|?yzI}_rI(j?N9#P<`Uo0 NC=JxXuv+wp5diT!Rfzxq literal 0 HcmV?d00001 diff --git a/src/test/resources/depot/depot_440_1118032470228587934_v4.manifest b/src/test/resources/depot/depot_440_1118032470228587934_v4.manifest new file mode 100644 index 0000000000000000000000000000000000000000..7fb01739447f2e99f358b350b4553fe1800fefe3 GIT binary patch literal 1186 zcmZpiZX(9Qz`(GBk%3{})!@~$d0O~uIPwF5T!x7=d4MDX!-5rHih&(Sfp8R1!wvyE zcaW$S5ND?v1co^IW&7t?lx9VSo99RRR(cv|7z9Oz6!>YoB?f03xReDNCplGx6(;3+ zr=(;SWO^B8g%}yR8Kru;g>x|of{cO!uBWHNzTPXlQ{=p*cS+45S+i;9mI$dkKF|($ z%2>a}?(~(vTWlpy+)4+z5S!XdtdlKaW`WF|0L1>5ey%>jrNvbl0T~&k?k0Ze#gT#D zX|Cn&Mj3v^p@DvZ;UyMfz7d%f>9$-9A~5Yhx_5&4ftg#6xF7Rt@igxYp6;%swY&1k zw}{57`-N`HHZ8T!>U$CQOye8KHQ3b7XR`{183!^K?3!?+qD)^OQv=VE5Emzpz$pEI z3QxbtU}HBA--w75k0R&bG?UcQBJXfF4?lP3^h^&6_rRP~Gb6*W%p4;FFVygw!o+<~ zBq~Vm-pQ3tepM6xvZyq^Ec07xd?6{^nAyX7{p^&m?6cZAk3lZPruGDD7Djjl_?V}9 z`@1HknPyv71QaBO6lWHg8=I76<+_!*2PB*8n|rximRjgXMimrA86{adl^dAnnuK~s zq(+!}8(aEuF_en3!Mx3I>+`j&nzo0ZPaSOhKb>)>AY)yAN2Pd)V&38oHKw;>`^8g4 zll=>BEd#j_n_A{g@@6o@K;|acGcXhyc^eld8|DRB7`T_4Mrd1PMdq26W%&EJMrE0s zn!0HR7$in%=S27<8fS!?JQ868av?UgZM{G0V5Wi0bpR?1j_}m> zG%koT$V>|^D+^392}|=2E=b9Y2=LI(Ow|v~4=WBxEB1Aa^exQGs|+$ou1pHHNb^Xu zsMId@&om6+VvyVo%9#)lCb(#spr_pP8F3ZfA>nhXdOl7(aQ0}4wbI`oN?UamSI7H4 z@Jd#BGZo}QY-+nBa^}L!0-3Ae095B+Ztkh?;hf=`?XMl`@tl)q1Nhoi4s-KsvCgZQbx^E6gmAxdv(s42{6j F2>|$igD(I8 literal 0 HcmV?d00001 From 338fb4de632fbed1023dd3505ba7ae658156c64e Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 7 Mar 2025 00:35:38 -0600 Subject: [PATCH 2/4] Rework some classes to make tests pass, still fails. --- .../contentdownloader/FileManifestProvider.kt | 7 +- .../in/dragonbra/javasteam/types/ChunkData.kt | 38 +- .../javasteam/types/ChunkIdComparer.kt | 37 ++ .../javasteam/types/DepotManifest.kt | 362 +++++++++--------- .../in/dragonbra/javasteam/types/FileData.kt | 164 ++++---- .../javasteam/types/Steam3Manifest.kt | 271 +++++++------ .../javasteam/types/DepotManifestTest.java | 17 - 7 files changed, 453 insertions(+), 443 deletions(-) create mode 100644 src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt index ea0f6370..ec7201b6 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt @@ -4,6 +4,7 @@ import `in`.dragonbra.javasteam.types.DepotManifest import `in`.dragonbra.javasteam.util.compat.readNBytesCompat import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger +import `in`.dragonbra.javasteam.util.stream.MemoryStream import java.io.ByteArrayOutputStream import java.io.File import java.io.IOException @@ -190,7 +191,11 @@ class FileManifestProvider(private val file: Path) : IManifestProvider { } } // add manifest as uncompressed data - zipUncompressed(zip, getEntryName(manifest.depotID, manifest.manifestGID), manifest.toByteArray()) + MemoryStream().use { ms -> + manifest.serialize(ms.asOutputStream()) + + zipUncompressed(zip, getEntryName(manifest.depotID, manifest.manifestGID), ms.toByteArray()) + } } // save all data to the file try { diff --git a/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt b/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt index 576ff903..dfbea886 100644 --- a/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt @@ -8,43 +8,47 @@ class ChunkData { /** * Gets or sets the SHA-1 hash chunk id. */ - val chunkID: ByteArray? + var chunkID: ByteArray? = null /** * Gets or sets the expected Adler32 checksum of this chunk. */ - val checksum: Int + var checksum: Int = 0 /** * Gets or sets the chunk offset. */ - val offset: Long + var offset: Long = 0 /** * Gets or sets the compressed length of this chunk. */ - val compressedLength: Int + var compressedLength: Int = 0 /** * Gets or sets the decompressed length of this chunk. */ - val uncompressedLength: Int - - @JvmOverloads - constructor( - chunkID: ByteArray? = null, - checksum: Int = 0, - offset: Long = 0L, - compressedLength: Int = 0, - uncompressedLength: Int = 0, - ) { - this.chunkID = chunkID + var uncompressedLength: Int = 0 + + /** + * Initializes a new instance of the ChunkData class. + */ + constructor() + + /** + * Initializes a new instance of the [ChunkData] class with specified values. + */ + constructor(id: ByteArray, checksum: Int, offset: Long, compLength: Int, uncompLength: Int) { + this.chunkID = id this.checksum = checksum this.offset = offset - this.compressedLength = compressedLength - this.uncompressedLength = uncompressedLength + this.compressedLength = compLength + this.uncompressedLength = uncompLength } + /** + * Internal constructor helper + */ constructor(chunkData: ChunkData) { chunkID = chunkData.chunkID checksum = chunkData.checksum diff --git a/src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt b/src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt new file mode 100644 index 00000000..fd7a6514 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt @@ -0,0 +1,37 @@ +package `in`.dragonbra.javasteam.types + +class ChunkIdComparer : AbstractSet() { + + private val innerSet = HashSet() + + override val size: Int + get() = innerSet.size + + override fun contains(element: ByteArray): Boolean = innerSet.contains(ByteArrayWrapper(element)) + + override fun iterator(): Iterator = innerSet.map { it.data }.iterator() + + fun add(element: ByteArray): Boolean = innerSet.add(ByteArrayWrapper(element)) + + override fun isEmpty(): Boolean = innerSet.isEmpty() + + // Wrapper class to handle proper equals and hashCode for byte arrays + private class ByteArrayWrapper(val data: ByteArray) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || other !is ByteArrayWrapper) return false + return data.contentEquals(other.data) + } + + override fun hashCode(): Int { + // Similar to C# implementation - use first 4 bytes of the SHA-1 hash + if (data.size >= 4) { + return data[0].toInt() and 0xFF or + (data[1].toInt() and 0xFF shl 8) or + (data[2].toInt() and 0xFF shl 16) or + (data[3].toInt() and 0xFF shl 24) + } + return data.contentHashCode() + } + } +} diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index 220c3f6a..5120caf2 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -13,10 +13,7 @@ import `in`.dragonbra.javasteam.util.log.Logger import `in`.dragonbra.javasteam.util.stream.BinaryReader import `in`.dragonbra.javasteam.util.stream.BinaryWriter import `in`.dragonbra.javasteam.util.stream.MemoryStream -import java.io.ByteArrayOutputStream import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream import java.io.InputStream import java.io.OutputStream import java.nio.ByteBuffer @@ -26,51 +23,43 @@ import java.util.Date import javax.crypto.Cipher import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec +import kotlin.math.log /** * Represents a Steam3 depot manifest. */ -@Suppress("MemberVisibilityCanBePrivate", "SpellCheckingInspection") +@Suppress("unused") class DepotManifest { companion object { - const val PROTOBUF_PAYLOAD_MAGIC = 0x71F617D0 - const val PROTOBUF_METADATA_MAGIC = 0x1F4812BE - const val PROTOBUF_SIGNATURE_MAGIC = 0x1B81B817 - const val PROTOBUF_ENDOFMANIFEST_MAGIC = 0x32C415AB - private val logger: Logger = LogManager.getLogger(DepotManifest::class.java) + private const val PROTOBUF_PAYLOAD_MAGIC: Int = 0x71F617D0 + private const val PROTOBUF_METADATA_MAGIC: Int = 0x1F4812BE + private const val PROTOBUF_SIGNATURE_MAGIC: Int = 0x1B81B817 + private const val PROTOBUF_ENDOFMANIFEST_MAGIC: Int = 0x32C415AB + /** * Initializes a new instance of the [DepotManifest] class. * Depot manifests may come from the Steam CDN or from Steam/depotcache/ manifest files. - * @param stream Raw depot manifest stream to deserialize. - * @exception NoSuchElementException Thrown if the given data is not something recognizable. + * @param stream Raw depot manifest stream to deserialize. */ @JvmStatic - fun deserialize(stream: InputStream): DepotManifest = deserialize(stream.readBytes()) + fun deserialize(stream: InputStream): DepotManifest = DepotManifest().apply { internalDeserialize(stream) } /** * Initializes a new instance of the [DepotManifest] class. * Depot manifests may come from the Steam CDN or from Steam/depotcache/ manifest files. * @param data Raw depot manifest data to deserialize. - * @exception NoSuchElementException Thrown if the given data is not something recognizable. */ @JvmStatic - fun deserialize(data: ByteArray): DepotManifest = MemoryStream(data).use { ms -> - val manifest = DepotManifest() - manifest.internalDeserialize(ms) - manifest - } + fun deserialize(data: ByteArray): DepotManifest = MemoryStream(data).use { return deserialize(it) } /** * Loads binary manifest from a file and deserializes it. * @param filename Input file name. - * @return [Pair]<[DepotManifest], [ByteArray]> object where the first value is the depot manifest - * and the second is the checksum if the deserialization was successful or else the values will be null. - * @exception NoSuchElementException Thrown if the given data is not something recognizable. + * @return [DepotManifest] object if deserialization was successful; otherwise, **null**. */ - @Suppress("unused") @JvmStatic fun loadFromFile(filename: String): DepotManifest? { val file = File(filename) @@ -78,8 +67,8 @@ class DepotManifest { return null } - return FileInputStream(file).use { fs -> - deserialize(fs) + return file.inputStream().use { fileStream -> + deserialize(fileStream) } } } @@ -87,63 +76,51 @@ class DepotManifest { /** * Gets the list of files within this manifest. */ - val files: MutableList + var files: ArrayList = arrayListOf() /** * Gets a value indicating whether filenames within this depot are encrypted. + * @return **true** if the filenames are encrypted; otherwise, false. */ var filenamesEncrypted: Boolean = false - private set /** * Gets the depot id. */ var depotID: Int = 0 - private set /** * Gets the manifest id. */ - var manifestGID: Long = 0 - private set + var manifestGID: Long = 0L /** * Gets the depot creation time. */ var creationTime: Date = Date() - private set /** * Gets the total uncompressed size of all files in this depot. */ - var totalUncompressedSize: Long = 0 - private set + var totalUncompressedSize: Long = 0L /** * Gets the total compressed size of all files in this depot. */ - var totalCompressedSize: Long = 0 - private set + var totalCompressedSize: Long = 0L /** * Gets CRC-32 checksum of encrypted manifest payload. */ var encryptedCRC: Int = 0 - private set - constructor() { - files = mutableListOf() - filenamesEncrypted = false - depotID = 0 - manifestGID = 0 - creationTime = Date() - totalUncompressedSize = 0 - totalCompressedSize = 0 - encryptedCRC = 0 - } + constructor() + /** + * Internal constructor helper + */ constructor(manifest: DepotManifest) { - files = manifest.files.map { FileData(it) }.toMutableList() + files = arrayListOf(*manifest.files.map { FileData(it) }.toTypedArray()) filenamesEncrypted = manifest.filenamesEncrypted depotID = manifest.depotID manifestGID = manifest.manifestGID @@ -156,66 +133,94 @@ class DepotManifest { /** * Attempts to decrypt file names with the given encryption key. * @param encryptionKey The encryption key. - * @return `true` if the file names were successfully decrypted; otherwise `false`. + * @return **true** if the file names were successfully decrypted; otherwise, **false**. */ fun decryptFilenames(encryptionKey: ByteArray): Boolean { if (!filenamesEncrypted) { return true } - assert(encryptionKey.size == 32) { "Decrypt filenames used with non 32 byte key!" } + requireNotNull(files) { "Files was null when attempting to decrypt filenames." } + require(encryptionKey.size == 32) { "Decrypt filnames used with non 32 byte key!" } // This was originally copy-pasted in the SteamKit2 source from CryptoHelper.SymmetricDecrypt to avoid allocating Aes instance for every filename val ecbCipher = Cipher.getInstance("AES/ECB/NoPadding", CryptoHelper.SEC_PROV) val aes = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) val secretKey = SecretKeySpec(encryptionKey, "AES") - var iv: ByteArray + + val iv = ByteArray(16) + var filenameLength = 0 + var bufferDecoded = ByteArray(256) + var bufferDecrypted = ByteArray(256) try { - files.forEach { file -> - val decoded = Base64.getUrlDecoder().decode( - file.fileName - .replace('+', '-') - .replace('/', '_') - .replace("\n", "") - .replace("\r", "") - .replace(" ", "") - ) + files!!.forEach { file -> + var decodedLength = file.fileName.length / 4 * 3 // This may be higher due to padding + + // Majority of filenames are short, even when they are encrypted and base64 encoded, + // so this resize will be hit *very* rarely + if (decodedLength > bufferDecoded.size) { + // Simply create new arrays of the required size + bufferDecoded = ByteArray(decodedLength) + bufferDecrypted = ByteArray(decodedLength) + } + + val decoder = Base64.getDecoder() + decodedLength = try { + val tempBytes = decoder.decode(file.fileName) + if (tempBytes.size <= bufferDecoded.size) { + tempBytes.copyInto(bufferDecoded) + tempBytes.size + } else { + // Buffer too small + throw IllegalArgumentException("Buffer too small") + } + } catch (e: Exception) { + logger.error("Failed to base64 decode the filename: ${e.message}", e) + return false + } - val bufferDecrypted: ByteArray try { - // Extract IV from the first 16 bytes + // Get a slice of the decoded buffer up to decodedLength + val encryptedFilename = bufferDecoded.copyOfRange(0, decodedLength) + + // Decrypt the IV portion (first 16 bytes) using ECB mode ecbCipher.init(Cipher.DECRYPT_MODE, secretKey) - iv = ecbCipher.doFinal(decoded, 0, 16) + ecbCipher.doFinal(encryptedFilename, 0, iv.size, iv, 0) + + // Decrypt the rest using CBC mode with the IV we just decrypted + val ivSpec = IvParameterSpec(iv) + aes.init(Cipher.DECRYPT_MODE, secretKey, ivSpec) - // Decrypt filename - aes.init(Cipher.DECRYPT_MODE, secretKey, IvParameterSpec(iv)) - bufferDecrypted = aes.doFinal(decoded, iv.size, decoded.size - iv.size) + // Decrypt the remaining data after the IV + val remainingData = encryptedFilename.copyOfRange(iv.size, encryptedFilename.size) + filenameLength = aes.doFinal(remainingData, 0, remainingData.size, bufferDecrypted, 0) } catch (e: Exception) { - logger.error("Failed to decrypt the filename: $e") + logger.error("Failed to decrypt the filename.", e) return false } // Trim the ending null byte, safe for UTF-8 - val filenameLength = bufferDecrypted.size - if ( - bufferDecrypted.isNotEmpty() && - bufferDecrypted[bufferDecrypted.size - 1] == 0.toByte() - ) { - 1 - } else { - 0 + if (filenameLength > 0 && bufferDecrypted[filenameLength - 1] == 0.toByte()) { + filenameLength-- + } + + for (i in 0 until filenameLength) { + if (bufferDecrypted[i] == '\\'.code.toByte()) { + bufferDecrypted[i] = File.separatorChar.code.toByte() + } } file.fileName = String(bufferDecrypted, 0, filenameLength, Charsets.UTF_8) - .replace('\\', File.separatorChar) } - } catch (e: Exception) { - logger.error("Failed to decrypt filenames: $e") + } finally { + bufferDecoded.fill(0) + bufferDecrypted.fill(0) } // Sort file entries alphabetically because that's what Steam does // TODO: (SK) Doesn't match Steam sorting if there are non-ASCII names present - files.sortWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.fileName }) + files!!.sortWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.fileName }) filenamesEncrypted = false return true @@ -225,15 +230,14 @@ class DepotManifest { * Serializes depot manifest and saves the output to a file. * @param filename Output file name. */ - @Suppress("unused") fun saveToFile(filename: String) { - FileOutputStream(File(filename)).use { fs -> - serialize(fs) + File(filename).outputStream().use { fileStream -> + serialize(fileStream) } } @OptIn(ExperimentalStdlibApi::class) - internal fun internalDeserialize(stream: MemoryStream) { + private fun internalDeserialize(stream: InputStream) { var payload: ContentManifestPayload? = null var metadata: ContentManifestMetadata? = null var signature: ContentManifestSignature? = null @@ -248,7 +252,7 @@ class DepotManifest { when (magic) { Steam3Manifest.MAGIC -> { - val binaryManifest = Steam3Manifest.deserialize(br) + val binaryManifest = Steam3Manifest().apply { deserialize(br) } parseBinaryManifest(binaryManifest) val marker = br.readInt() @@ -276,7 +280,7 @@ class DepotManifest { signature = ContentManifestSignature.parseFrom(stream.readNBytesCompat(signatureLength)) } - else -> throw NoSuchElementException("Unrecognized magic value ${magic.toHexString(HexFormat.Default)} in depot manifest.") + else -> throw NoSuchElementException("Unrecognized magic value ${magic.toHexString()} in depot manifest.") } } } @@ -289,8 +293,8 @@ class DepotManifest { } } - internal fun parseBinaryManifest(manifest: Steam3Manifest) { - files.clear() + private fun parseBinaryManifest(manifest: Steam3Manifest) { + files = ArrayList(manifest.mapping!!.size) filenamesEncrypted = manifest.areFileNamesEncrypted depotID = manifest.depotID manifestGID = manifest.manifestGID @@ -299,64 +303,64 @@ class DepotManifest { totalCompressedSize = manifest.totalCompressedSize encryptedCRC = manifest.encryptedCRC - manifest.fileMapping.forEach { fileMapping -> - val fileData = FileData( - fileName = fileMapping.fileName, - fileNameHash = fileMapping.hashFileName, - flags = fileMapping.flags, - totalSize = fileMapping.totalSize, - fileHash = fileMapping.hashContent, + manifest.mapping!!.forEach { fileMapping -> + val filedata = FileData( + filename = fileMapping.fileName!!, + filenameHash = fileMapping.hashFileName!!, + flag = fileMapping.flags, + size = fileMapping.totalSize, + hash = fileMapping.hashContent!!, linkTarget = "", - encrypted = filenamesEncrypted + encrypted = filenamesEncrypted, + numChunks = fileMapping.chunks!!.size ) - fileMapping.chunks.forEach { chunk -> - fileData.chunks.add( - ChunkData( - chunkID = chunk.chunkGID, - checksum = chunk.checksum, - offset = chunk.offset, - compressedLength = chunk.compressedSize, - uncompressedLength = chunk.decompressedSize - ) + fileMapping.chunks!!.forEach { chunk -> + val chunkData = ChunkData( + id = chunk.chunkGID!!, + checksum = chunk.checksum, + offset = chunk.offset, + compLength = chunk.compressedSize, + uncompLength = chunk.decompressedSize, ) + filedata.chunks.add(chunkData) } - files.add(fileData) + files!!.add(filedata) } } - internal fun parseProtobufManifestPayload(payload: ContentManifestPayload) { - files.clear() + private fun parseProtobufManifestPayload(payload: ContentManifestPayload) { + files = ArrayList(payload.mappingsCount) payload.mappingsList.forEach { fileMapping -> - val fileData = FileData( - fileName = fileMapping.filename, - fileNameHash = fileMapping.shaFilename.toByteArray(), - flags = EDepotFileFlag.from(fileMapping.flags), - totalSize = fileMapping.size, - fileHash = fileMapping.shaContent.toByteArray(), + val filedata = FileData( + filename = fileMapping.filename, + filenameHash = fileMapping.shaFilename.toByteArray(), + flag = EDepotFileFlag.from(fileMapping.flags), + size = fileMapping.size, + hash = fileMapping.shaContent.toByteArray(), linkTarget = fileMapping.linktarget, - encrypted = filenamesEncrypted + encrypted = filenamesEncrypted, + numChunks = fileMapping.chunksList.size, ) fileMapping.chunksList.forEach { chunk -> - fileData.chunks.add( - ChunkData( - chunkID = chunk.sha.toByteArray(), - checksum = chunk.crc, - offset = chunk.offset, - compressedLength = chunk.cbCompressed, - uncompressedLength = chunk.cbOriginal - ) + val chunkData = ChunkData( + id = chunk.sha.toByteArray(), + checksum = chunk.crc, + offset = chunk.offset, + compLength = chunk.cbCompressed, + uncompLength = chunk.cbOriginal ) + filedata.chunks.add(chunkData) } - files.add(fileData) + files!!.add(filedata) } } - internal fun parseProtobufManifestMetadata(metadata: ContentManifestMetadata) { + private fun parseProtobufManifestMetadata(metadata: ContentManifestMetadata) { filenamesEncrypted = metadata.filenamesEncrypted depotID = metadata.depotId manifestGID = metadata.gidManifest @@ -369,34 +373,25 @@ class DepotManifest { /** * Serializes the depot manifest into the provided output stream. * @param output The stream to which the serialized depot manifest will be written. - * @return A pair object containing the amount of bytes written and the checksum of the manifest */ - fun serialize(output: OutputStream): Int { - val manifestBytes = toByteArray() - output.write(manifestBytes) - return manifestBytes.size - } + fun serialize(output: OutputStream) { + requireNotNull(files) { "Files was null when attempting to serialize manifest." } - /** - * Serializes the depot manifest into a byte array. - */ - fun toByteArray(): ByteArray { - val payload = ContentManifestPayload.newBuilder() - val uniqueChunks = hashSetOf() + var payload = ContentManifestPayload.newBuilder() + var uniqueChunks = ChunkIdComparer() files.forEach { file -> - val protoFile = ContentManifestPayload.FileMapping.newBuilder().apply { - size = file.totalSize - flags = EDepotFileFlag.code(file.flags) + var protofile = ContentManifestPayload.FileMapping.newBuilder().apply { + this.size = file.totalSize + this.flags = EDepotFileFlag.code(file.flags) } - if (filenamesEncrypted) { // Assume the name is unmodified - protoFile.filename = file.fileName - protoFile.shaFilename = ByteString.copyFrom(file.fileNameHash) + protofile.filename = file.fileName + protofile.shaFilename = ByteString.copyFrom(file.fileNameHash) } else { - protoFile.filename = file.fileName.replace('/', '\\') - protoFile.shaFilename = ByteString.copyFrom( + protofile.filename = file.fileName.replace('/', '\\') + protofile.shaFilename = ByteString.copyFrom( CryptoHelper.shaHash( file.fileName .replace('/', '\\') @@ -405,30 +400,28 @@ class DepotManifest { ) ) } - - protoFile.shaContent = ByteString.copyFrom(file.fileHash) - - if (file.linkTarget.isNotBlank()) { - protoFile.linktarget = file.linkTarget + protofile.shaContent = ByteString.copyFrom(file.fileHash) + if (!file.linkTarget.isNullOrBlank()) { + protofile.linktarget = file.linkTarget } file.chunks.forEach { chunk -> - val protoChunk = ContentManifestPayload.FileMapping.ChunkData.newBuilder().apply { - sha = ByteString.copyFrom(chunk.chunkID) - crc = chunk.checksum - offset = chunk.offset - cbOriginal = chunk.uncompressedLength - cbCompressed = chunk.compressedLength + var protochunk = ContentManifestPayload.FileMapping.ChunkData.newBuilder().apply { + this.sha = ByteString.copyFrom(chunk.chunkID) + this.crc = chunk.checksum + this.offset = chunk.offset + this.cbOriginal = chunk.uncompressedLength + this.cbCompressed = chunk.compressedLength } - protoFile.addChunks(protoChunk) + protofile.addChunks(protochunk) uniqueChunks.add(chunk.chunkID!!) } - payload.addMappings(protoFile) + payload.addMappings(protofile) } - val metadata = ContentManifestMetadata.newBuilder().apply { + var metadata = ContentManifestMetadata.newBuilder().apply { this.depotId = depotID this.gidManifest = manifestGID this.creationTime = this@DepotManifest.creationTime.toInstant().epochSecond.toInt() @@ -440,11 +433,11 @@ class DepotManifest { // Calculate payload CRC MemoryStream().use { msPayload -> - payload.build().writeTo(msPayload.asOutputStream()) + msPayload.asOutputStream().write(payload.build().toByteArray()) val len = msPayload.length.toInt() val data = ByteArray(4 + len) - System.arraycopy(ByteBuffer.allocate(4).putInt(len).array(), 0, data, 0, 4) + System.arraycopy(ByteBuffer.allocate(Int.SIZE_BYTES).putInt(len).array(), 0, data, 0, 4) System.arraycopy(msPayload.toByteArray(), 0, data, 4, len) val crc32 = Utils.crc32(data).toInt() @@ -455,46 +448,33 @@ class DepotManifest { metadata.crcEncrypted = encryptedCRC metadata.crcClear = crc32 } - - msPayload.toByteArray() } - // Write the manifest to the stream and return the checksum - return ByteArrayOutputStream().use { bw -> - BinaryWriter(bw).use { writer -> - // Write Protobuf payload - MemoryStream().use { msPayload -> - payload.build().writeTo(msPayload.asOutputStream()) - - writer.writeInt(PROTOBUF_PAYLOAD_MAGIC) - writer.writeInt(msPayload.length.toInt()) - writer.write(msPayload.buffer, 0, msPayload.length.toInt()) - } + var bw = BinaryWriter(output) - // Write Protobuf metadata - MemoryStream().use { msMetaData -> - metadata.build().writeTo(msMetaData.asOutputStream()) + // Write Protobuf payload + MemoryStream().use { msPayload -> + msPayload.asOutputStream().write(payload.build().toByteArray()) + bw.write(PROTOBUF_PAYLOAD_MAGIC) + bw.write(msPayload.length.toInt()) + bw.write(msPayload.buffer.copyOfRange(0, msPayload.length.toInt())) + } - writer.writeInt(PROTOBUF_METADATA_MAGIC) - writer.writeInt(msMetaData.length.toInt()) - writer.write(msMetaData.buffer, 0, msMetaData.length.toInt()) - } + // Write Protobuf metadata + MemoryStream().use { msMetadata -> + msMetadata.asOutputStream().write(payload.build().toByteArray()) + bw.write(PROTOBUF_METADATA_MAGIC) + bw.write(msMetadata.length.toInt()) + bw.write(msMetadata.buffer.copyOfRange(0, msMetadata.length.toInt())) + } - // Write empty signature section - writer.writeInt(PROTOBUF_SIGNATURE_MAGIC) - writer.writeInt(0) + // Write empty signature section + bw.write(PROTOBUF_SIGNATURE_MAGIC) + bw.write(0) - // Write EOF marker - writer.writeInt(PROTOBUF_ENDOFMANIFEST_MAGIC) - } + // Write EOF marker + bw.write(PROTOBUF_ENDOFMANIFEST_MAGIC) - bw.toByteArray() - } + bw.close() } - - /** - * Calculates the checksum of the depot manifest. - */ - @Suppress("unused") - fun calculateChecksum(): ByteArray = CryptoHelper.shaHash(toByteArray()) } diff --git a/src/main/java/in/dragonbra/javasteam/types/FileData.kt b/src/main/java/in/dragonbra/javasteam/types/FileData.kt index c23488d4..4e35120e 100644 --- a/src/main/java/in/dragonbra/javasteam/types/FileData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/FileData.kt @@ -1,80 +1,84 @@ -package `in`.dragonbra.javasteam.types - -import `in`.dragonbra.javasteam.enums.EDepotFileFlag -import java.io.File -import java.util.EnumSet - -/** - * Represents a single file within a manifest. - */ -class FileData { - - /** - * Gets the name of the file. - */ - var fileName: String - internal set - - /** - * Gets SHA-1 hash of this file's name. - */ - val fileNameHash: ByteArray - - /** - * Gets the chunks that this file is composed of. - */ - val chunks: MutableList - - /** - * Gets the file flags - */ - val flags: EnumSet - - /** - * Gets the total size of this file. - */ - val totalSize: Long - - /** - * Gets SHA-1 hash of this file. - */ - val fileHash: ByteArray - - /** - * Gets symlink target of this file. - */ - val linkTarget: String - - constructor( - fileName: String, - fileNameHash: ByteArray, - chunks: MutableList = mutableListOf(), - flags: EnumSet, - totalSize: Long, - fileHash: ByteArray, - linkTarget: String, - encrypted: Boolean, - ) { - if (encrypted) { - this.fileName = fileName - } else { - this.fileName = fileName.replace('\\', File.separatorChar) - } - this.fileNameHash = fileNameHash - this.chunks = chunks - this.flags = flags - this.totalSize = totalSize - this.fileHash = fileHash - this.linkTarget = linkTarget - } - - constructor(fileData: FileData) { - fileName = fileData.fileName - fileNameHash = fileData.fileNameHash - chunks = fileData.chunks.map { ChunkData(it) }.toMutableList() - flags = fileData.flags - totalSize = fileData.totalSize - fileHash = fileData.fileHash - linkTarget = fileData.linkTarget - } -} +package `in`.dragonbra.javasteam.types + +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import java.io.File +import java.util.EnumSet + +/** + * Represents a single file within a manifest. + * + * @constructor Initializes a new instance of the [FileData] class. + */ +@Suppress("unused") +class FileData { + + /** + * Gets the name of the file. + */ + var fileName: String = "" + + /** + * Gets SHA-1 hash of this file's name. + */ + var fileNameHash: ByteArray = byteArrayOf() + + /** + * Gets the chunks that this file is composed of. + */ + var chunks: MutableList = mutableListOf() + + /** + * Gets the file flags + */ + var flags: EnumSet = EnumSet.noneOf(EDepotFileFlag::class.java) + + /** + * Gets the total size of this file. + */ + var totalSize: Long = 0 + + /** + * Gets SHA-1 hash of this file. + */ + var fileHash: ByteArray = byteArrayOf() + + /** + * Gets symlink target of this file. + */ + var linkTarget: String? = null + + /** + * Initializes a new instance of the [FileData] class with specified values. + */ + constructor( + filename: String, + filenameHash: ByteArray, + flag: EnumSet, + size: Long, + hash: ByteArray, + linkTarget: String, + encrypted: Boolean, + numChunks: Int, + ) { + this.fileName = if (encrypted) filename else filename.replace('\\', File.separatorChar) + this.fileNameHash = filenameHash + this.flags = flag + this.totalSize = size + this.fileHash = hash + this.chunks = ArrayList(numChunks) + this.linkTarget = linkTarget + } + + /** + * Internal constructor helper + */ + constructor(fileData: FileData) { + fileName = fileData.fileName + fileNameHash = fileData.fileNameHash + chunks = fileData.chunks.map { ChunkData(it) }.toMutableList() + flags = fileData.flags + totalSize = fileData.totalSize + fileHash = fileData.fileHash + linkTarget = fileData.linkTarget + } +} diff --git a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt index a3c736e5..8f6e1492 100644 --- a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt @@ -1,137 +1,134 @@ -@file:Suppress("unused") - -package `in`.dragonbra.javasteam.types - -import `in`.dragonbra.javasteam.enums.EDepotFileFlag -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.stream.BinaryReader -import java.lang.Exception -import java.time.Instant -import java.util.Date -import java.util.EnumSet - -/** - * Represents the binary Steam3 manifest format. - */ -class Steam3Manifest( -// val magic: Int, - val version: Int, - val depotID: Int, - val manifestGID: Long, - val creationTime: Date, - val areFileNamesEncrypted: Boolean, - val totalUncompressedSize: Long, - val totalCompressedSize: Long, - val chunkCount: Int, - val fileEntryCount: Int, - val fileMappingSize: Int, - val encryptedCRC: Int, - val decryptedCRC: Int, - val flags: Int, - val fileMapping: List, -) { - - class FileMapping( - val fileName: String, - val totalSize: Long, - val flags: EnumSet, - val hashFileName: ByteArray, - val hashContent: ByteArray, - val numChunks: Int, - val chunks: Array, - ) { - - class Chunk( - val chunkGID: ByteArray, // sha1 hash for this chunk - val checksum: Int, - val offset: Long, - val decompressedSize: Int, - val compressedSize: Int, - ) { - companion object { - internal fun deserialize(ds: BinaryReader): Chunk = Chunk( - chunkGID = ds.readNBytesCompat(20), - checksum = ds.readInt(), - offset = ds.readLong(), - decompressedSize = ds.readInt(), - compressedSize = ds.readInt() - ) - } - } - - companion object { - internal fun deserialize(ds: BinaryReader): FileMapping { - val fileName = ds.readNullTermString(Charsets.UTF_8) - val totalSize = ds.readLong() - val flags = EDepotFileFlag.from(ds.readInt()) - val hashContent = ds.readNBytesCompat(20) - val hashFileName = ds.readNBytesCompat(20) - val numChunks = ds.readInt() - - return FileMapping( - fileName = fileName, - totalSize = totalSize, - flags = flags, - hashContent = hashContent, - hashFileName = hashFileName, - numChunks = numChunks, - chunks = Array(numChunks) { Chunk.deserialize(ds) } - ) - } - } - } - - companion object { - const val MAGIC: Int = 0x16349781 - const val CURRENT_VERSION: Int = 4 - - internal fun deserialize(ds: BinaryReader): Steam3Manifest { - // The magic is verified by DepotManifest.InternalDeserialize, not checked here to avoid seeking - val version = ds.readInt() - - if (version != CURRENT_VERSION) { - // Not Implemented Exception - throw Exception("Only version $CURRENT_VERSION is supported") - } - - val depotID = ds.readInt() - val manifestGID = ds.readLong() - val creationTime = Date.from(Instant.ofEpochSecond(ds.readInt().toLong())) - val areFileNamesEncrypted = ds.readInt() != 0 - val totalUncompressedSize = ds.readLong() - val totalCompressedSize = ds.readLong() - val chunkCount = ds.readInt() - val fileEntryCount = ds.readInt() - val fileMappingSize = ds.readInt() - val encryptedCRC = ds.readInt() - val decryptedCRC = ds.readInt() - val flags = ds.readInt() - - val fileMapping = mutableListOf() - var size = fileMappingSize - - while (size > 0) { - val start = ds.position - fileMapping.add(FileMapping.deserialize(ds)) - size -= ds.position - start - } - - return Steam3Manifest( - version = version, - depotID = depotID, - manifestGID = manifestGID, - creationTime = creationTime, - areFileNamesEncrypted = areFileNamesEncrypted, - totalUncompressedSize = totalUncompressedSize, - totalCompressedSize = totalCompressedSize, - chunkCount = chunkCount, - fileEntryCount = fileEntryCount, - fileMappingSize = fileMappingSize, - fileMapping = fileMapping, - encryptedCRC = encryptedCRC, - decryptedCRC = decryptedCRC, - flags = flags, - ) - } - } -} +package `in`.dragonbra.javasteam.types + +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.util.stream.BinaryReader +import java.nio.charset.StandardCharsets +import java.time.Instant +import java.util.Date +import java.util.EnumSet + +/** + * Represents the binary Steam3 manifest format. + */ +@Suppress("unused") +class Steam3Manifest { + + companion object { + const val MAGIC: Int = 0x16349781 + const val CURRENT_VERSION: Int = 4 + } + + class FileMapping { + + @Suppress("ArrayInDataClass") + data class Chunk( + var chunkGID: ByteArray? = null, // sha1 hash for this chunk + var checksum: Int = 0, + var offset: Long = 0L, + var decompressedSize: Int = 0, + var compressedSize: Int = 0, + ) { + internal fun deserialize(ds: BinaryReader) { + chunkGID = ds.readBytes(20) + checksum = ds.readInt() + offset = ds.readLong() + decompressedSize = ds.readInt() + compressedSize = ds.readInt() + } + } + + var fileName: String? = null + + var totalSize: Long = 0L + var flags: EnumSet = EnumSet.noneOf(EDepotFileFlag::class.java) + + var hashFileName: ByteArray? = null + var hashContent: ByteArray? = null + + var numChunks: Int = 0 + var chunks: ArrayList? = null + + internal fun deserialize(ds: BinaryReader) { + fileName = ds.readNullTermString(StandardCharsets.UTF_8) + + totalSize = ds.readLong() + + flags = EDepotFileFlag.from(ds.readInt()) + + hashContent = ds.readBytes(20) + hashFileName = ds.readBytes(20) + + numChunks = ds.readInt() + + chunks = ArrayList(numChunks) + + for (i in 0 until chunks!!.size) { + chunks!![i] = Chunk().apply { + deserialize(ds) + } + } + } + } + + var magic: Int = 0 + var version: Int = 0 + var depotID: Int = 0 + var manifestGID: Long = 0L + var creationTime: Date = Date() + var areFileNamesEncrypted: Boolean = false + var totalUncompressedSize: Long = 0L + var totalCompressedSize: Long = 0L + var chunkCount: Int = 0 + var fileEntryCount: Int = 0 + var fileMappingSize: Int = 0 + var encryptedCRC: Int = 0 + var decryptedCRC: Int = 0 + var flags: Int = 0 + var mapping: ArrayList? = null + + internal fun deserialize(ds: BinaryReader) { + // The magic is verified by DepotManifest.InternalDeserialize, not checked here to avoid seeking + // Magic = ds.readInt(); + // if (Magic != MAGIC) { + // throw new InvalidDataException("data is not a valid steam3 manifest: incorrect magic."); + // } + + version = ds.readInt() + + if (version != CURRENT_VERSION) { + throw IllegalArgumentException("Only version $CURRENT_VERSION is supported.") + } + + depotID = ds.readInt() + + manifestGID = ds.readLong() + creationTime = Date.from(Instant.ofEpochSecond(ds.readInt().toLong())) + + areFileNamesEncrypted = ds.readInt() != 0 + + totalUncompressedSize = ds.readLong() + totalCompressedSize = ds.readLong() + + chunkCount = ds.readInt() + + fileEntryCount = ds.readInt() + fileMappingSize = ds.readInt() + + mapping = ArrayList(fileMappingSize) + + encryptedCRC = ds.readInt() + decryptedCRC = ds.readInt() + + flags = ds.readInt() + + var i = fileMappingSize + while (i > 0) { + val start = ds.position + + val fileMapping = FileMapping().apply { deserialize(ds) } + mapping!!.add(fileMapping) + + i -= (ds.position - start) + } + } +} diff --git a/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java b/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java index b6505f7c..40a11094 100644 --- a/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java +++ b/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java @@ -11,7 +11,6 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Path; -import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -102,22 +101,6 @@ public void roundtripSerializesManifestEncryptedManifest() throws IOException { int actualOffset = indexOf(actual, signature); // DepotManifest.PROTOBUF_SIGNATURE_MAGIC int expectedOffset = indexOf(manifestData, signature); - byte expectedByte = manifestData[1109]; - byte actualByte = actual[1109]; - System.out.println("At index 1109:"); - System.out.println("Expected byte: " + expectedByte); - System.out.println("Actual byte: " + actualByte); - -// Maybe also look at surrounding bytes for context - System.out.println("Expected bytes around 1109:"); - for (int i = 1105; i < 1115; i++) { - System.out.println("Index " + i + ": " + manifestData[i]); - } - System.out.println("Actual bytes around 1109:"); - for (int i = 1105; i < 1115; i++) { - System.out.println("Index " + i + ": " + actual[i]); - } - Assertions.assertTrue(actualOffset > 0); Assertions.assertTrue(expectedOffset > 0); Assertions.assertArrayEquals( From 8b24d23de48ae528d7566547fd360a8338e48427 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 7 Mar 2025 14:41:34 -0600 Subject: [PATCH 3/4] All new tests pass. --- .run/javasteam [formatKotlin].run.xml | 24 +++ .../javasteam/types/ChunkIdComparer.kt | 37 ---- .../javasteam/types/DepotManifest.kt | 165 ++++++++++------ .../javasteam/types/Steam3Manifest.kt | 78 +++----- .../javasteam/types/DepotManifestTest.java | 187 +++++++++--------- 5 files changed, 255 insertions(+), 236 deletions(-) create mode 100644 .run/javasteam [formatKotlin].run.xml delete mode 100644 src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt diff --git a/.run/javasteam [formatKotlin].run.xml b/.run/javasteam [formatKotlin].run.xml new file mode 100644 index 00000000..2ffae927 --- /dev/null +++ b/.run/javasteam [formatKotlin].run.xml @@ -0,0 +1,24 @@ + + + + + + + true + true + false + false + + + \ No newline at end of file diff --git a/src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt b/src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt deleted file mode 100644 index fd7a6514..00000000 --- a/src/main/java/in/dragonbra/javasteam/types/ChunkIdComparer.kt +++ /dev/null @@ -1,37 +0,0 @@ -package `in`.dragonbra.javasteam.types - -class ChunkIdComparer : AbstractSet() { - - private val innerSet = HashSet() - - override val size: Int - get() = innerSet.size - - override fun contains(element: ByteArray): Boolean = innerSet.contains(ByteArrayWrapper(element)) - - override fun iterator(): Iterator = innerSet.map { it.data }.iterator() - - fun add(element: ByteArray): Boolean = innerSet.add(ByteArrayWrapper(element)) - - override fun isEmpty(): Boolean = innerSet.isEmpty() - - // Wrapper class to handle proper equals and hashCode for byte arrays - private class ByteArrayWrapper(val data: ByteArray) { - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other == null || other !is ByteArrayWrapper) return false - return data.contentEquals(other.data) - } - - override fun hashCode(): Int { - // Similar to C# implementation - use first 4 bytes of the SHA-1 hash - if (data.size >= 4) { - return data[0].toInt() and 0xFF or - (data[1].toInt() and 0xFF shl 8) or - (data[2].toInt() and 0xFF shl 16) or - (data[3].toInt() and 0xFF shl 24) - } - return data.contentHashCode() - } - } -} diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index 5120caf2..1d0d0e7b 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -16,14 +16,14 @@ import `in`.dragonbra.javasteam.util.stream.MemoryStream import java.io.File import java.io.InputStream import java.io.OutputStream -import java.nio.ByteBuffer import java.time.Instant -import java.util.Base64 -import java.util.Date +import java.util.* import javax.crypto.Cipher import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec -import kotlin.math.log +import kotlin.NoSuchElementException +import kotlin.collections.ArrayList +import kotlin.collections.HashSet /** * Represents a Steam3 depot manifest. @@ -45,7 +45,11 @@ class DepotManifest { * @param stream Raw depot manifest stream to deserialize. */ @JvmStatic - fun deserialize(stream: InputStream): DepotManifest = DepotManifest().apply { internalDeserialize(stream) } + fun deserialize(stream: InputStream): DepotManifest { + val manifest = DepotManifest() + manifest.internalDeserialize(stream) + return manifest + } /** * Initializes a new instance of the [DepotManifest] class. @@ -53,7 +57,12 @@ class DepotManifest { * @param data Raw depot manifest data to deserialize. */ @JvmStatic - fun deserialize(data: ByteArray): DepotManifest = MemoryStream(data).use { return deserialize(it) } + fun deserialize(data: ByteArray): DepotManifest { + val ms = MemoryStream(data) + val manifest = deserialize(ms) + ms.close() + return manifest + } /** * Loads binary manifest from a file and deserializes it. @@ -140,7 +149,6 @@ class DepotManifest { return true } - requireNotNull(files) { "Files was null when attempting to decrypt filenames." } require(encryptionKey.size == 32) { "Decrypt filnames used with non 32 byte key!" } // This was originally copy-pasted in the SteamKit2 source from CryptoHelper.SymmetricDecrypt to avoid allocating Aes instance for every filename @@ -149,12 +157,12 @@ class DepotManifest { val secretKey = SecretKeySpec(encryptionKey, "AES") val iv = ByteArray(16) - var filenameLength = 0 + var filenameLength: Int var bufferDecoded = ByteArray(256) var bufferDecrypted = ByteArray(256) try { - files!!.forEach { file -> + files.forEach { file -> var decodedLength = file.fileName.length / 4 * 3 // This may be higher due to padding // Majority of filenames are short, even when they are encrypted and base64 encoded, @@ -165,9 +173,17 @@ class DepotManifest { bufferDecrypted = ByteArray(decodedLength) } - val decoder = Base64.getDecoder() + val decoder = Base64.getUrlDecoder() decodedLength = try { - val tempBytes = decoder.decode(file.fileName) + val tempBytes = decoder.decode( + // :^) + file.fileName + .replace('+', '-') + .replace('/', '_') + .replace("\n", "") + .replace("\r", "") + .replace(" ", "") + ) if (tempBytes.size <= bufferDecoded.size) { tempBytes.copyInto(bufferDecoded) tempBytes.size @@ -220,7 +236,7 @@ class DepotManifest { // Sort file entries alphabetically because that's what Steam does // TODO: (SK) Doesn't match Steam sorting if there are non-ASCII names present - files!!.sortWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.fileName }) + files.sortWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.fileName }) filenamesEncrypted = false return true @@ -252,7 +268,8 @@ class DepotManifest { when (magic) { Steam3Manifest.MAGIC -> { - val binaryManifest = Steam3Manifest().apply { deserialize(br) } + val binaryManifest = Steam3Manifest() + binaryManifest.deserialize(br) parseBinaryManifest(binaryManifest) val marker = br.readInt() @@ -280,21 +297,23 @@ class DepotManifest { signature = ContentManifestSignature.parseFrom(stream.readNBytesCompat(signatureLength)) } - else -> throw NoSuchElementException("Unrecognized magic value ${magic.toHexString()} in depot manifest.") + else -> { + throw NoSuchElementException("Unrecognized magic value ${magic.toHexString()} in depot manifest.") + } } } } if (payload != null && metadata != null && signature != null) { - parseProtobufManifestMetadata(metadata) - parseProtobufManifestPayload(payload) + parseProtobufManifestMetadata(metadata!!) + parseProtobufManifestPayload(payload!!) } else { throw NoSuchElementException("Missing ContentManifest sections required for parsing depot manifest") } } private fun parseBinaryManifest(manifest: Steam3Manifest) { - files = ArrayList(manifest.mapping!!.size) + files = ArrayList(manifest.mapping.size) filenamesEncrypted = manifest.areFileNamesEncrypted depotID = manifest.depotID manifestGID = manifest.manifestGID @@ -303,7 +322,7 @@ class DepotManifest { totalCompressedSize = manifest.totalCompressedSize encryptedCRC = manifest.encryptedCRC - manifest.mapping!!.forEach { fileMapping -> + manifest.mapping.forEach { fileMapping -> val filedata = FileData( filename = fileMapping.fileName!!, filenameHash = fileMapping.hashFileName!!, @@ -326,7 +345,7 @@ class DepotManifest { filedata.chunks.add(chunkData) } - files!!.add(filedata) + files.add(filedata) } } @@ -356,7 +375,7 @@ class DepotManifest { filedata.chunks.add(chunkData) } - files!!.add(filedata) + files.add(filedata) } } @@ -375,16 +394,31 @@ class DepotManifest { * @param output The stream to which the serialized depot manifest will be written. */ fun serialize(output: OutputStream) { - requireNotNull(files) { "Files was null when attempting to serialize manifest." } + val payload = ContentManifestPayload.newBuilder() + val uniqueChunks = object : HashSet() { + // This acts like "ChunkIdComparer" + private val items = mutableListOf() + + override fun add(element: ByteArray): Boolean { + if (contains(element)) return false + items.add(element) + return true + } - var payload = ContentManifestPayload.newBuilder() - var uniqueChunks = ChunkIdComparer() + override fun contains(element: ByteArray): Boolean = items.any { it.contentEquals(element) } + + override fun iterator(): MutableIterator = items.iterator() + + override val size: Int + get() = items.size + } files.forEach { file -> - var protofile = ContentManifestPayload.FileMapping.newBuilder().apply { + val protofile = ContentManifestPayload.FileMapping.newBuilder().apply { this.size = file.totalSize this.flags = EDepotFileFlag.code(file.flags) } + if (filenamesEncrypted) { // Assume the name is unmodified protofile.filename = file.fileName @@ -395,85 +429,90 @@ class DepotManifest { CryptoHelper.shaHash( file.fileName .replace('/', '\\') - .lowercase() + .lowercase(Locale.getDefault()) .toByteArray(Charsets.UTF_8) ) ) } + protofile.shaContent = ByteString.copyFrom(file.fileHash) + if (!file.linkTarget.isNullOrBlank()) { protofile.linktarget = file.linkTarget } file.chunks.forEach { chunk -> - var protochunk = ContentManifestPayload.FileMapping.ChunkData.newBuilder().apply { + val protochunk = ContentManifestPayload.FileMapping.ChunkData.newBuilder().apply { this.sha = ByteString.copyFrom(chunk.chunkID) this.crc = chunk.checksum this.offset = chunk.offset this.cbOriginal = chunk.uncompressedLength this.cbCompressed = chunk.compressedLength - } + }.build() protofile.addChunks(protochunk) uniqueChunks.add(chunk.chunkID!!) } - payload.addMappings(protofile) + payload.addMappings(protofile.build()) } - var metadata = ContentManifestMetadata.newBuilder().apply { + val metadata = ContentManifestMetadata.newBuilder().apply { this.depotId = depotID this.gidManifest = manifestGID - this.creationTime = this@DepotManifest.creationTime.toInstant().epochSecond.toInt() - this.filenamesEncrypted = filenamesEncrypted + this.creationTime = (this@DepotManifest.creationTime.time / 1000).toInt() + this.filenamesEncrypted = this@DepotManifest.filenamesEncrypted this.cbDiskOriginal = totalUncompressedSize this.cbDiskCompressed = totalCompressedSize this.uniqueChunks = uniqueChunks.size } // Calculate payload CRC - MemoryStream().use { msPayload -> - msPayload.asOutputStream().write(payload.build().toByteArray()) - - val len = msPayload.length.toInt() - val data = ByteArray(4 + len) - System.arraycopy(ByteBuffer.allocate(Int.SIZE_BYTES).putInt(len).array(), 0, data, 0, 4) - System.arraycopy(msPayload.toByteArray(), 0, data, 4, len) - val crc32 = Utils.crc32(data).toInt() - - if (filenamesEncrypted) { - metadata.crcEncrypted = crc32 - metadata.crcClear = 0 - } else { - metadata.crcEncrypted = encryptedCRC - metadata.crcClear = crc32 - } + val msPayload = MemoryStream() + payload.build().writeTo(msPayload.asOutputStream()) + + val len = msPayload.length.toInt() + val data = ByteArray(4 + len) + + // BitConverter.GetBytes(len) + val lenBytes = ByteArray(4) + lenBytes[0] = (len and 0xFF).toByte() + lenBytes[1] = ((len shr 8) and 0xFF).toByte() + lenBytes[2] = ((len shr 16) and 0xFF).toByte() + lenBytes[3] = ((len shr 24) and 0xFF).toByte() + + System.arraycopy(lenBytes, 0, data, 0, 4) + System.arraycopy(msPayload.toByteArray(), 0, data, 4, len) + val crc32 = Utils.crc32(data).toInt() + + if (filenamesEncrypted) { + metadata.crcEncrypted = crc32 + metadata.crcClear = 0 + } else { + metadata.crcEncrypted = encryptedCRC + metadata.crcClear = crc32 } - var bw = BinaryWriter(output) + val bw = BinaryWriter(output) // Write Protobuf payload - MemoryStream().use { msPayload -> - msPayload.asOutputStream().write(payload.build().toByteArray()) - bw.write(PROTOBUF_PAYLOAD_MAGIC) - bw.write(msPayload.length.toInt()) - bw.write(msPayload.buffer.copyOfRange(0, msPayload.length.toInt())) - } + val payloadBytes = payload.build().toByteArray() + bw.writeInt(PROTOBUF_PAYLOAD_MAGIC) + bw.writeInt(payloadBytes.size) + bw.write(payloadBytes) // Write Protobuf metadata - MemoryStream().use { msMetadata -> - msMetadata.asOutputStream().write(payload.build().toByteArray()) - bw.write(PROTOBUF_METADATA_MAGIC) - bw.write(msMetadata.length.toInt()) - bw.write(msMetadata.buffer.copyOfRange(0, msMetadata.length.toInt())) - } + val metadataBytes = metadata.build().toByteArray() + bw.writeInt(PROTOBUF_METADATA_MAGIC) + bw.writeInt(metadataBytes.size) + bw.write(metadataBytes) // Write empty signature section - bw.write(PROTOBUF_SIGNATURE_MAGIC) - bw.write(0) + bw.writeInt(PROTOBUF_SIGNATURE_MAGIC) + bw.writeInt(0) // Write EOF marker - bw.write(PROTOBUF_ENDOFMANIFEST_MAGIC) + bw.writeInt(PROTOBUF_ENDOFMANIFEST_MAGIC) bw.close() } diff --git a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt index 8f6e1492..bd5e2f22 100644 --- a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt @@ -3,7 +3,6 @@ package `in`.dragonbra.javasteam.types import `in`.dragonbra.javasteam.enums.EDepotFileFlag import `in`.dragonbra.javasteam.util.stream.BinaryReader import java.nio.charset.StandardCharsets -import java.time.Instant import java.util.Date import java.util.EnumSet @@ -15,19 +14,17 @@ class Steam3Manifest { companion object { const val MAGIC: Int = 0x16349781 - const val CURRENT_VERSION: Int = 4 + private const val CURRENT_VERSION = 4 } class FileMapping { + class Chunk { + var chunkGID: ByteArray? = null // sha1 hash for this chunk + var checksum: Int = 0 + var offset: Long = 0 + var decompressedSize: Int = 0 + var compressedSize: Int = 0 - @Suppress("ArrayInDataClass") - data class Chunk( - var chunkGID: ByteArray? = null, // sha1 hash for this chunk - var checksum: Int = 0, - var offset: Long = 0L, - var decompressedSize: Int = 0, - var compressedSize: Int = 0, - ) { internal fun deserialize(ds: BinaryReader) { chunkGID = ds.readBytes(20) checksum = ds.readInt() @@ -38,34 +35,25 @@ class Steam3Manifest { } var fileName: String? = null - - var totalSize: Long = 0L + var totalSize: Long = 0 var flags: EnumSet = EnumSet.noneOf(EDepotFileFlag::class.java) - var hashFileName: ByteArray? = null var hashContent: ByteArray? = null - var numChunks: Int = 0 - var chunks: ArrayList? = null + var chunks: Array? = null internal fun deserialize(ds: BinaryReader) { fileName = ds.readNullTermString(StandardCharsets.UTF_8) - totalSize = ds.readLong() - flags = EDepotFileFlag.from(ds.readInt()) - hashContent = ds.readBytes(20) hashFileName = ds.readBytes(20) - numChunks = ds.readInt() - chunks = ArrayList(numChunks) - - for (i in 0 until chunks!!.size) { - chunks!![i] = Chunk().apply { - deserialize(ds) - } + chunks = Array(numChunks) { + val chunk = Chunk() + chunk.deserialize(ds) + chunk } } } @@ -73,62 +61,60 @@ class Steam3Manifest { var magic: Int = 0 var version: Int = 0 var depotID: Int = 0 - var manifestGID: Long = 0L + var manifestGID: Long = 0 var creationTime: Date = Date() var areFileNamesEncrypted: Boolean = false - var totalUncompressedSize: Long = 0L - var totalCompressedSize: Long = 0L + var totalUncompressedSize: Long = 0 + var totalCompressedSize: Long = 0 var chunkCount: Int = 0 var fileEntryCount: Int = 0 var fileMappingSize: Int = 0 var encryptedCRC: Int = 0 var decryptedCRC: Int = 0 var flags: Int = 0 - var mapping: ArrayList? = null + var mapping: MutableList = mutableListOf() internal fun deserialize(ds: BinaryReader) { + /* // The magic is verified by DepotManifest.InternalDeserialize, not checked here to avoid seeking - // Magic = ds.readInt(); - // if (Magic != MAGIC) { - // throw new InvalidDataException("data is not a valid steam3 manifest: incorrect magic."); - // } + magic = ds.readInt() + + if (magic != MAGIC) { + throw IOException("data is not a valid steam3 manifest: incorrect magic.") + } + */ version = ds.readInt() if (version != CURRENT_VERSION) { - throw IllegalArgumentException("Only version $CURRENT_VERSION is supported.") + throw NotImplementedError("Only version $CURRENT_VERSION is supported.") } depotID = ds.readInt() - manifestGID = ds.readLong() - creationTime = Date.from(Instant.ofEpochSecond(ds.readInt().toLong())) - + creationTime = Date(ds.readInt() * 1000L) areFileNamesEncrypted = ds.readInt() != 0 - totalUncompressedSize = ds.readLong() totalCompressedSize = ds.readLong() - chunkCount = ds.readInt() - fileEntryCount = ds.readInt() fileMappingSize = ds.readInt() - mapping = ArrayList(fileMappingSize) + mapping = ArrayList(fileMappingSize) encryptedCRC = ds.readInt() decryptedCRC = ds.readInt() - flags = ds.readInt() var i = fileMappingSize while (i > 0) { - val start = ds.position + val start = ds.position.toLong() - val fileMapping = FileMapping().apply { deserialize(ds) } - mapping!!.add(fileMapping) + val fileMapping = FileMapping() + fileMapping.deserialize(ds) + mapping.add(fileMapping) - i -= (ds.position - start) + i -= (ds.position - start.toInt()) } } } diff --git a/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java b/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java index 40a11094..2384672e 100644 --- a/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java +++ b/src/test/java/in/dragonbra/javasteam/types/DepotManifestTest.java @@ -1,23 +1,25 @@ package in.dragonbra.javasteam.types; +import in.dragonbra.javasteam.TestBase; import in.dragonbra.javasteam.enums.EDepotFileFlag; import in.dragonbra.javasteam.util.Strings; import in.dragonbra.javasteam.util.crypto.CryptoHelper; -import in.dragonbra.javasteam.util.stream.BinaryReader; import in.dragonbra.javasteam.util.stream.MemoryStream; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Arrays; -@SuppressWarnings({"resource", "DataFlowIssue"}) -public class DepotManifestTest { +public class DepotManifestTest extends TestBase { private static final byte[] DEPOT_440_DECRYPTION_KEY = new byte[]{ (byte) 0x44, (byte) 0xCE, (byte) 0x5C, (byte) 0x52, (byte) 0x97, (byte) 0xA4, (byte) 0x15, (byte) 0xA1, @@ -28,126 +30,139 @@ public class DepotManifestTest { @Test - public void parsesAndDecryptsManifestVersion4() throws IOException, NoSuchAlgorithmException { - var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_v4.manifest"); + public void parsesAndDecryptsManifestVersion4() { + try (var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_v4.manifest"); + var ms = new MemoryStream() + ) { + Assertions.assertNotNull(stream); - var ms = new MemoryStream(); - stream.transferTo(ms.asOutputStream()); + stream.transferTo(ms.asOutputStream()); + var manifestData = ms.toByteArray(); - var manifestData = ms.toByteArray(); + var depotManifest = DepotManifest.deserialize(manifestData); - var depotManifest = DepotManifest.deserialize(manifestData); + Assertions.assertTrue(depotManifest.getFilenamesEncrypted()); + Assertions.assertEquals(1195249848L, depotManifest.getEncryptedCRC()); - Assertions.assertTrue(depotManifest.getFilenamesEncrypted()); - Assertions.assertEquals(1195249848L, depotManifest.getEncryptedCRC()); + var result = depotManifest.decryptFilenames(DEPOT_440_DECRYPTION_KEY); + Assertions.assertTrue(result); - depotManifest.decryptFilenames(DEPOT_440_DECRYPTION_KEY); - - testDecryptedManifest(depotManifest); + testDecryptedManifest(depotManifest); + } catch (Exception e) { + Assertions.fail(e); + } } @Test public void parsesAndDecryptsManifest() throws IOException, NoSuchAlgorithmException { - var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934.manifest"); + try (var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934.manifest"); + var ms = new MemoryStream() + ) { + Assertions.assertNotNull(stream); - var ms = new MemoryStream(); - stream.transferTo(ms.asOutputStream()); + stream.transferTo(ms.asOutputStream()); - var manifestData = ms.toByteArray(); + var manifestData = ms.toByteArray(); - var depotManifest = DepotManifest.deserialize(manifestData); + var depotManifest = DepotManifest.deserialize(manifestData); - Assertions.assertTrue(depotManifest.getFilenamesEncrypted()); - Assertions.assertEquals(1606273976L, depotManifest.getEncryptedCRC()); + Assertions.assertTrue(depotManifest.getFilenamesEncrypted()); + Assertions.assertEquals(1606273976L, depotManifest.getEncryptedCRC()); - depotManifest.decryptFilenames(DEPOT_440_DECRYPTION_KEY); + var result = depotManifest.decryptFilenames(DEPOT_440_DECRYPTION_KEY); + Assertions.assertTrue(result); - testDecryptedManifest(depotManifest); + testDecryptedManifest(depotManifest); + } } @Test public void parsesDecryptedManifest() throws IOException, NoSuchAlgorithmException { - var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_decrypted.manifest"); + try (var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_decrypted.manifest"); + var ms = new MemoryStream() + ) { + Assertions.assertNotNull(stream); - var ms = new MemoryStream(); - stream.transferTo(ms.asOutputStream()); + stream.transferTo(ms.asOutputStream()); - var manifestData = ms.toByteArray(); + var manifestData = ms.toByteArray(); - var depotManifest = DepotManifest.deserialize(manifestData); + var depotManifest = DepotManifest.deserialize(manifestData); - testDecryptedManifest(depotManifest); + testDecryptedManifest(depotManifest); + } } @Test - public void roundtripSerializesManifestEncryptedManifest() throws IOException { - var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934.manifest"); + public void roundtripSerializesManifestEncryptedManifest() { + try (var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934.manifest"); + var ms = new MemoryStream() + ) { + Assertions.assertNotNull(stream); - var ms = new MemoryStream(); - stream.transferTo(ms.asOutputStream()); + IOUtils.copy(stream, ms.asOutputStream()); + stream.close(); - var manifestData = ms.toByteArray(); + var manifestData = ms.toByteArray(); - var depotManifest = DepotManifest.deserialize(manifestData); + DepotManifest depotManifest = DepotManifest.deserialize(manifestData); - var actualStream = new MemoryStream(); - depotManifest.serialize(actualStream.asOutputStream()); + var actualStream = new ByteArrayOutputStream(); + depotManifest.serialize(actualStream); - var actual = actualStream.toByteArray(); + var actual = actualStream.toByteArray(); - // We are unable to write signatures, so validate everything except for the signature - var signature = new byte[]{(byte) 0x17, (byte) 0xB8, (byte) 0x81, (byte) 0x1B}; + // We are unable to write signatures, so validate everything except for the signature + var signature = new byte[]{0x17, (byte) 0xB8, (byte) 0x81, 0x1B}; - int actualOffset = indexOf(actual, signature); // DepotManifest.PROTOBUF_SIGNATURE_MAGIC - int expectedOffset = indexOf(manifestData, signature); + int actualOffset = indexOf(actual, signature); // DepotManifest.PROTOBUF_SIGNATURE_MAGIC + int expectedOffset = indexOf(manifestData, signature); - Assertions.assertTrue(actualOffset > 0); - Assertions.assertTrue(expectedOffset > 0); - Assertions.assertArrayEquals( - Arrays.copyOfRange(manifestData, 0, expectedOffset), - Arrays.copyOfRange(actual, 0, actualOffset) - ); + Assertions.assertTrue(actualOffset > 0); + Assertions.assertTrue(expectedOffset > 0); - // We dont have `BitConverter.ToInt32` - int expectedSignatureLength; - try (var bais = new ByteArrayInputStream(manifestData); - var br = new BinaryReader(bais)) { - expectedSignatureLength = br.readBytes(expectedOffset + 4).length; - } - int actualSignatureLength; - try (var bais = new ByteArrayInputStream(manifestData); - var br = new BinaryReader(bais)) { - actualSignatureLength = br.readBytes(expectedOffset + 4).length; - } + Assertions.assertArrayEquals( + Arrays.copyOfRange(manifestData, 0, expectedOffset), + Arrays.copyOfRange(actual, 0, actualOffset)); - //int expectedSignatureLength = readInt32(manifestData, expectedOffset + 4); - // int actualSignatureLength = readInt32(actual, actualOffset + 4); + int expectedSignatureLength = ByteBuffer.wrap(manifestData, expectedOffset + 4, 4).order(ByteOrder.LITTLE_ENDIAN).getInt(); + int actualSignatureLength = ByteBuffer.wrap(actual, actualOffset + 4, 4).order(ByteOrder.LITTLE_ENDIAN).getInt(); - Assertions.assertEquals(131, expectedSignatureLength); - Assertions.assertEquals(0, actualSignatureLength); - Assertions.assertArrayEquals( - Arrays.copyOfRange(manifestData, expectedOffset + expectedSignatureLength + 8, manifestData.length), - Arrays.copyOfRange(actual, actualOffset + 8, actual.length) - ); + Assertions.assertEquals(131, expectedSignatureLength); + Assertions.assertEquals(0, actualSignatureLength); + + Assertions.assertArrayEquals( + Arrays.copyOfRange(manifestData, expectedOffset + expectedSignatureLength + 8, manifestData.length), + Arrays.copyOfRange(actual, actualOffset + 8, actual.length) + ); + } catch (IOException e) { + Assertions.fail(e); + } } + // C# Test "RoundtripSerializesManifestByteIndentical" @Test public void roundtripSerializesManifestByteIdentical() throws IOException { - var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_decrypted.manifest"); + try (var stream = getClass().getResourceAsStream("/depot/depot_440_1118032470228587934_decrypted.manifest"); + var ms = new MemoryStream() + ) { + Assertions.assertNotNull(stream); - var ms = new MemoryStream(); - stream.transferTo(ms.asOutputStream()); + stream.transferTo(ms.asOutputStream()); - var manifestData = ms.toByteArray(); + var manifestData = ms.toByteArray(); + ms.close(); - var depotManifest = DepotManifest.deserialize(manifestData); + DepotManifest depotManifest = DepotManifest.deserialize(manifestData); - var actualStream = new MemoryStream(); - depotManifest.serialize(actualStream.asOutputStream()); + var actualStream = new MemoryStream(); + depotManifest.serialize(actualStream.asOutputStream()); - var actual = actualStream.toByteArray(); + var actual = actualStream.toByteArray(); + actualStream.close(); - Assertions.assertArrayEquals(manifestData, actual); + Assertions.assertArrayEquals(manifestData, actual); + } } private void testDecryptedManifest(DepotManifest depotManifest) throws NoSuchAlgorithmException { @@ -197,15 +212,13 @@ private void testDecryptedManifest(DepotManifest depotManifest) throws NoSuchAlg ); } - // Java or Apache doesn't have a indexOf(byte[], byte[]) - // This is taken from guava since we don't have that lib as a dependency. - private static int indexOf(byte[] array, byte[] target) { - if (target.length == 0) { - return 0; + private int indexOf(byte[] array, byte[] target) { + if (array == null || target == null || array.length < target.length) { + return -1; } outer: - for (int i = 0; i < array.length - target.length + 1; i++) { + for (int i = 0; i <= array.length - target.length; i++) { for (int j = 0; j < target.length; j++) { if (array[i + j] != target[j]) { continue outer; @@ -213,13 +226,7 @@ private static int indexOf(byte[] array, byte[] target) { } return i; } - return -1; - } - private static int readInt32(byte[] data, int offset) { - return (data[offset] & 0xFF) | - ((data[offset + 1] & 0xFF) << 8) | - ((data[offset + 2] & 0xFF) << 16) | - ((data[offset + 3] & 0xFF) << 24); + return -1; } } From a518e3b2444088fffc70796ae22d618fb5486197 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 7 Mar 2025 15:27:04 -0600 Subject: [PATCH 4/4] Clean up a bit, making sure streams are closed. --- .../contentdownloader/ContentDownloader.kt | 12 ++- .../in/dragonbra/javasteam/types/ChunkData.kt | 81 +++++------------ .../javasteam/types/DepotManifest.kt | 38 +++----- .../in/dragonbra/javasteam/types/FileData.kt | 87 ++++++------------- .../javasteam/types/Steam3Manifest.kt | 28 +++--- .../javasteam/util/stream/MemoryStream.java | 3 +- 6 files changed, 79 insertions(+), 170 deletions(-) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt index 47819f0c..86d51deb 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt @@ -533,9 +533,7 @@ class ContentDownloader(val steamClient: SteamClient) { val chunkID = Strings.toHex(chunk.chunkID) - val chunkInfo = ChunkData(chunk) - - var outputChunkData = ByteArray(chunkInfo.uncompressedLength) + var outputChunkData = ByteArray(chunk.uncompressedLength) var writtenBytes = 0 do { @@ -544,10 +542,10 @@ class ContentDownloader(val steamClient: SteamClient) { try { connection = cdnPool.getConnection().await() - outputChunkData = ByteArray(chunkInfo.uncompressedLength) + outputChunkData = ByteArray(chunk.uncompressedLength) writtenBytes = cdnPool.cdnClient.downloadDepotChunk( depotId = depot.depotId, - chunk = chunkInfo, + chunk = chunk, server = connection!!, destination = outputChunkData, depotKey = depot.depotKey, @@ -587,7 +585,7 @@ class ContentDownloader(val steamClient: SteamClient) { fileStreamData.fileStream = randomAccessFile.channel } - fileStreamData.fileStream?.position(chunkInfo.offset) + fileStreamData.fileStream?.position(chunk.offset) fileStreamData.fileStream?.write(ByteBuffer.wrap(outputChunkData, 0, writtenBytes)) } finally { fileStreamData.fileLock.release() @@ -715,7 +713,7 @@ class ContentDownloader(val steamClient: SteamClient) { throw CancellationException("Unable to download manifest $manifestId for depot $depotId") } - val newProtoManifest = DepotManifest(depotManifest) + val newProtoManifest = depotManifest steamClient.configuration.depotManifestProvider.updateManifest(newProtoManifest) return@async newProtoManifest diff --git a/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt b/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt index dfbea886..3be2db1a 100644 --- a/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/ChunkData.kt @@ -1,59 +1,22 @@ -package `in`.dragonbra.javasteam.types - -/** - * Represents a single chunk within a file. - */ -class ChunkData { - - /** - * Gets or sets the SHA-1 hash chunk id. - */ - var chunkID: ByteArray? = null - - /** - * Gets or sets the expected Adler32 checksum of this chunk. - */ - var checksum: Int = 0 - - /** - * Gets or sets the chunk offset. - */ - var offset: Long = 0 - - /** - * Gets or sets the compressed length of this chunk. - */ - var compressedLength: Int = 0 - - /** - * Gets or sets the decompressed length of this chunk. - */ - var uncompressedLength: Int = 0 - - /** - * Initializes a new instance of the ChunkData class. - */ - constructor() - - /** - * Initializes a new instance of the [ChunkData] class with specified values. - */ - constructor(id: ByteArray, checksum: Int, offset: Long, compLength: Int, uncompLength: Int) { - this.chunkID = id - this.checksum = checksum - this.offset = offset - this.compressedLength = compLength - this.uncompressedLength = uncompLength - } - - /** - * Internal constructor helper - */ - constructor(chunkData: ChunkData) { - chunkID = chunkData.chunkID - checksum = chunkData.checksum - offset = chunkData.offset - compressedLength = chunkData.compressedLength - uncompressedLength = chunkData.uncompressedLength - } -} +package `in`.dragonbra.javasteam.types + +/** + * Represents a single chunk within a file. + * + * @constructor Initializes a new instance of the [ChunkData] class. + * @constructor Initializes a new instance of the [ChunkData] class with specified values. + * + * @param chunkID Gets or sets the SHA-1 hash chunk id. + * @param checksum Gets or sets the expected Adler32 checksum of this chunk. + * @param offset Gets or sets the chunk offset. + * @param compressedLength Gets or sets the compressed length of this chunk. + * @param uncompressedLength Gets or sets the decompressed length of this chunk. + */ +@Suppress("ArrayInDataClass") +data class ChunkData( + var chunkID: ByteArray? = null, + var checksum: Int = 0, + var offset: Long = 0, + var compressedLength: Int = 0, + var uncompressedLength: Int = 0, +) diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index 1d0d0e7b..66da7f59 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -123,22 +123,6 @@ class DepotManifest { */ var encryptedCRC: Int = 0 - constructor() - - /** - * Internal constructor helper - */ - constructor(manifest: DepotManifest) { - files = arrayListOf(*manifest.files.map { FileData(it) }.toTypedArray()) - filenamesEncrypted = manifest.filenamesEncrypted - depotID = manifest.depotID - manifestGID = manifest.manifestGID - creationTime = manifest.creationTime - totalUncompressedSize = manifest.totalUncompressedSize - totalCompressedSize = manifest.totalCompressedSize - encryptedCRC = manifest.encryptedCRC - } - /** * Attempts to decrypt file names with the given encryption key. * @param encryptionKey The encryption key. @@ -168,7 +152,6 @@ class DepotManifest { // Majority of filenames are short, even when they are encrypted and base64 encoded, // so this resize will be hit *very* rarely if (decodedLength > bufferDecoded.size) { - // Simply create new arrays of the required size bufferDecoded = ByteArray(decodedLength) bufferDecrypted = ByteArray(decodedLength) } @@ -176,7 +159,6 @@ class DepotManifest { val decoder = Base64.getUrlDecoder() decodedLength = try { val tempBytes = decoder.decode( - // :^) file.fileName .replace('+', '-') .replace('/', '_') @@ -331,16 +313,16 @@ class DepotManifest { hash = fileMapping.hashContent!!, linkTarget = "", encrypted = filenamesEncrypted, - numChunks = fileMapping.chunks!!.size + numChunks = fileMapping.chunks.size ) - fileMapping.chunks!!.forEach { chunk -> + fileMapping.chunks.forEach { chunk -> val chunkData = ChunkData( - id = chunk.chunkGID!!, + chunkID = chunk.chunkGID!!, checksum = chunk.checksum, offset = chunk.offset, - compLength = chunk.compressedSize, - uncompLength = chunk.decompressedSize, + compressedLength = chunk.compressedSize, + uncompressedLength = chunk.decompressedSize, ) filedata.chunks.add(chunkData) } @@ -366,11 +348,11 @@ class DepotManifest { fileMapping.chunksList.forEach { chunk -> val chunkData = ChunkData( - id = chunk.sha.toByteArray(), + chunkID = chunk.sha.toByteArray(), checksum = chunk.crc, offset = chunk.offset, - compLength = chunk.cbCompressed, - uncompLength = chunk.cbOriginal + compressedLength = chunk.cbCompressed, + uncompressedLength = chunk.cbOriginal ) filedata.chunks.add(chunkData) } @@ -474,7 +456,7 @@ class DepotManifest { val len = msPayload.length.toInt() val data = ByteArray(4 + len) - // BitConverter.GetBytes(len) + // Alternative of BitConverter.GetBytes(len) val lenBytes = ByteArray(4) lenBytes[0] = (len and 0xFF).toByte() lenBytes[1] = ((len shr 8) and 0xFF).toByte() @@ -485,6 +467,8 @@ class DepotManifest { System.arraycopy(msPayload.toByteArray(), 0, data, 4, len) val crc32 = Utils.crc32(data).toInt() + msPayload.close() + if (filenamesEncrypted) { metadata.crcEncrypted = crc32 metadata.crcClear = 0 diff --git a/src/main/java/in/dragonbra/javasteam/types/FileData.kt b/src/main/java/in/dragonbra/javasteam/types/FileData.kt index 4e35120e..53a7c2f4 100644 --- a/src/main/java/in/dragonbra/javasteam/types/FileData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/FileData.kt @@ -8,45 +8,25 @@ import java.util.EnumSet * Represents a single file within a manifest. * * @constructor Initializes a new instance of the [FileData] class. + * + * @param fileName Gets the name of the file. + * @param fileNameHash Gets SHA-1 hash of this file's name. + * @param chunks Gets the chunks that this file is composed of. + * @param flags Gets the file flags + * @param totalSize Gets the total size of this file. + * @param fileHash Gets SHA-1 hash of this file. + * @param linkTarget Gets symlink target of this file. */ -@Suppress("unused") -class FileData { - - /** - * Gets the name of the file. - */ - var fileName: String = "" - - /** - * Gets SHA-1 hash of this file's name. - */ - var fileNameHash: ByteArray = byteArrayOf() - - /** - * Gets the chunks that this file is composed of. - */ - var chunks: MutableList = mutableListOf() - - /** - * Gets the file flags - */ - var flags: EnumSet = EnumSet.noneOf(EDepotFileFlag::class.java) - - /** - * Gets the total size of this file. - */ - var totalSize: Long = 0 - - /** - * Gets SHA-1 hash of this file. - */ - var fileHash: ByteArray = byteArrayOf() - - /** - * Gets symlink target of this file. - */ - var linkTarget: String? = null - +@Suppress("ArrayInDataClass") +data class FileData( + var fileName: String = "", + var fileNameHash: ByteArray = byteArrayOf(), + var chunks: MutableList = mutableListOf(), + var flags: EnumSet = EnumSet.noneOf(EDepotFileFlag::class.java), + var totalSize: Long = 0, + var fileHash: ByteArray = byteArrayOf(), + var linkTarget: String? = null, +) { /** * Initializes a new instance of the [FileData] class with specified values. */ @@ -59,26 +39,13 @@ class FileData { linkTarget: String, encrypted: Boolean, numChunks: Int, - ) { - this.fileName = if (encrypted) filename else filename.replace('\\', File.separatorChar) - this.fileNameHash = filenameHash - this.flags = flag - this.totalSize = size - this.fileHash = hash - this.chunks = ArrayList(numChunks) - this.linkTarget = linkTarget - } - - /** - * Internal constructor helper - */ - constructor(fileData: FileData) { - fileName = fileData.fileName - fileNameHash = fileData.fileNameHash - chunks = fileData.chunks.map { ChunkData(it) }.toMutableList() - flags = fileData.flags - totalSize = fileData.totalSize - fileHash = fileData.fileHash - linkTarget = fileData.linkTarget - } + ) : this( + fileName = if (encrypted) filename else filename.replace('\\', File.separatorChar), + fileNameHash = filenameHash, + chunks = ArrayList(numChunks), + flags = flag, + totalSize = size, + fileHash = hash, + linkTarget = linkTarget, + ) } diff --git a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt index bd5e2f22..63377a44 100644 --- a/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/Steam3Manifest.kt @@ -9,19 +9,20 @@ import java.util.EnumSet /** * Represents the binary Steam3 manifest format. */ -@Suppress("unused") +@Suppress("unused", "MemberVisibilityCanBePrivate") class Steam3Manifest { companion object { const val MAGIC: Int = 0x16349781 - private const val CURRENT_VERSION = 4 + private const val CURRENT_VERSION: Int = 4 } + @Suppress("MemberVisibilityCanBePrivate") class FileMapping { class Chunk { var chunkGID: ByteArray? = null // sha1 hash for this chunk var checksum: Int = 0 - var offset: Long = 0 + var offset: Long = 0L var decompressedSize: Int = 0 var compressedSize: Int = 0 @@ -40,7 +41,8 @@ class Steam3Manifest { var hashFileName: ByteArray? = null var hashContent: ByteArray? = null var numChunks: Int = 0 - var chunks: Array? = null + var chunks: Array = arrayOf() + private set internal fun deserialize(ds: BinaryReader) { fileName = ds.readNullTermString(StandardCharsets.UTF_8) @@ -49,11 +51,10 @@ class Steam3Manifest { hashContent = ds.readBytes(20) hashFileName = ds.readBytes(20) numChunks = ds.readInt() + chunks = Array(numChunks) { Chunk() } - chunks = Array(numChunks) { - val chunk = Chunk() - chunk.deserialize(ds) - chunk + for (x in chunks.indices) { + chunks[x].deserialize(ds) } } } @@ -75,14 +76,11 @@ class Steam3Manifest { var mapping: MutableList = mutableListOf() internal fun deserialize(ds: BinaryReader) { - /* // The magic is verified by DepotManifest.InternalDeserialize, not checked here to avoid seeking - magic = ds.readInt() - - if (magic != MAGIC) { - throw IOException("data is not a valid steam3 manifest: incorrect magic.") - } - */ + // magic = ds.readInt() + // if (magic != MAGIC) { + // throw IOException("data is not a valid steam3 manifest: incorrect magic.") + // } version = ds.readInt() diff --git a/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java b/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java index e97e3ef4..52cda7f8 100644 --- a/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java +++ b/src/main/java/in/dragonbra/javasteam/util/stream/MemoryStream.java @@ -4,7 +4,6 @@ import org.jetbrains.annotations.NotNull; import java.io.Closeable; -import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.Arrays; @@ -382,7 +381,7 @@ public byte[] toByteArray() { } @Override - public byte[] readAllBytes() { + public byte @NotNull [] readAllBytes() { return toByteArray(); }