diff --git a/README.md b/README.md index 30dfb4ac..b758f697 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # JavaSteam + [![Java CI/CD](https://github.com/Longi94/JavaSteam/actions/workflows/javasteam-build-push.yml/badge.svg)](https://github.com/Longi94/JavaSteam/actions/workflows/javasteam-build.yml) [![Maven Central](https://img.shields.io/maven-central/v/in.dragonbra/javasteam)](https://mvnrepository.com/artifact/in.dragonbra/javasteam) [![Discord](https://img.shields.io/discord/420907597906968586.svg)](https://discord.gg/8F2JuTu) @@ -9,11 +10,19 @@ Java port of [SteamKit2](https://github.com/SteamRE/SteamKit). JavaSteam targets Latest version is available through [Maven](https://mvnrepository.com/artifact/in.dragonbra/javasteam) -If you get a `java.security.InvalidKeyException: Illegal key size or default parameters` exception when trying to encrypt a message you need to download the [Unlimited Strength Jurisdiction Policy Files](http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) and place them under `${java.home}/jre/lib/security/`. See [this stackoverflow question](https://stackoverflow.com/questions/6481627/java-security-illegal-key-size-or-default-parameters). +Snapshots may be available through [Maven central repository](https://central.sonatype.com/service/rest/repository/browse/maven-snapshots/in/dragonbra/) + +If you get a `java.security.InvalidKeyException: Illegal key size or default parameters` exception when trying to +encrypt a message you need to download +the [Unlimited Strength Jurisdiction Policy Files](http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html) +and place them under `${java.home}/jre/lib/security/`. + +See [this stackoverflow question](https://stackoverflow.com/questions/6481627/java-security-illegal-key-size-or-default-parameters). **1. Add the repository to your build.** Gradle + ```groovy repositories { mavenCentral() @@ -21,22 +30,27 @@ repositories { ``` Maven + ```xml + - central - https://repo.maven.apache.org/maven2 + central + https://repo.maven.apache.org/maven2 ``` **2. Add the JavaSteam dependency to your project.** Gradle + ```groovy implementation 'in.dragonbra:javasteam:x.y.z' ``` Maven + ```xml + in.dragonbra javasteam @@ -50,17 +64,28 @@ Maven [Android and Non-Android | Bouncy Castle](https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk18on) -**4. (Optional) Working with protobufs.** - -If you plan on working with protobuf builders directly to perform actions a handler doesn't support, you will need to add the protobuf-java dependency. +**4. Optional dependencies.** -Note: To eliminate any errors or warnings, you should try and match the same version JavaSteam uses.
You can find the latest version being used [here](https://github.com/Longi94/JavaSteam/blob/master/gradle/libs.versions.toml). +* Protobufs: + * If you plan on working with protobuf builders directly to perform actions a handler doesn't support, you will need + to add the [protobuf-java](https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java) dependency. + * Note: To eliminate any errors or warnings, you should try and match the same version JavaSteam uses. +

+* Content Downloading: + * If you plan on working with Content Downloading, Depot files may be compressed with Zstd *(Zstandard)*. + * You will need to implement the correct type + of [ztd implementation](https://mvnrepository.com/artifact/com.github.luben/zstd-jni) if using JVM or Android. + * Android uses `aar` for the library type. -[Protobuf Java](https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java) +You can find the latest version of these dependencies JavaSteam +supports [here](https://github.com/Longi94/JavaSteam/blob/master/gradle/libs.versions.toml). ## Getting Started -You can head to the very short [Getting Started](https://github.com/Longi94/JavaSteam/wiki/Getting-started) page or take a look at the [samples](https://github.com/Longi94/JavaSteam/tree/master/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples) to get you started with using this library. +You can head to the very short [Getting Started](https://github.com/Longi94/JavaSteam/wiki/Getting-started) page or take +a look at +the [samples](https://github.com/Longi94/JavaSteam/tree/master/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples) +to get you started with using this library. There some [open-source projects](https://github.com/Longi94/JavaSteam/wiki/Samples) too you can check out. @@ -78,11 +103,13 @@ Generated classes:
## Contributing -Contributions to the repository are always welcome! Checkout the [contribution guidelines](CONTRIBUTING.md) to get started. +Contributions to the repository are always welcome! Checkout the [contribution guidelines](CONTRIBUTING.md) to get +started. ## Other -Join the [discord server](https://discord.gg/8F2JuTu) if you have any questions related or unrelated to this repo or just want to chat! +Join the [discord server](https://discord.gg/8F2JuTu) if you have any questions related or unrelated to this repo or +just want to chat! ## License diff --git a/build.gradle.kts b/build.gradle.kts index 959a6bae..3ff2843a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -118,6 +118,7 @@ tasks.withType { } dependencies { + implementation(libs.bundles.ktor) implementation(libs.commons.io) implementation(libs.commons.lang3) implementation(libs.commons.validator) @@ -125,9 +126,9 @@ dependencies { implementation(libs.kotlin.coroutines) implementation(libs.kotlin.stdib) implementation(libs.okHttp) - implementation(libs.xz) implementation(libs.protobuf.java) - implementation(libs.bundles.ktor) + implementation(libs.xz) + compileOnly(libs.zstd) testImplementation(libs.bundles.testing) } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2dea79a6..c9e72842 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,6 +24,7 @@ protobuf-gradle = "0.9.5" # https://mvnrepository.com/artifact/com.google.protob publishPlugin = "2.0.0" # https://mvnrepository.com/artifact/io.github.gradle-nexus/publish-plugin qrCode = "1.0.1" # https://mvnrepository.com/artifact/pro.leaco.qrcode/console-qrcode xz = "1.10" # https://mvnrepository.com/artifact/org.tukaani/xz +zstd = "1.5.7-3" # https://search.maven.org/artifact/com.github.luben/zstd-jni # Testing Lib versions commonsCodec = "1.18.0" # https://mvnrepository.com/artifact/commons-codec/commons-codec @@ -39,14 +40,15 @@ commons-validator = { module = "commons-validator:commons-validator", version.re gson = { module = "com.google.code.gson:gson", version.ref = "gson" } kotlin-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlin-coroutines" } kotlin-stdib = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" } +ktor-client-cio = { module = "io.ktor:ktor-client-cio", version = "3.0.3" } +ktor-client-core = { module = "io.ktor:ktor-client-core", version = "3.0.3" } +ktor-client-websocket = { module = "io.ktor:ktor-client-websockets", version = "3.0.3" } okHttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okHttp" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" } qrCode = { module = "pro.leaco.qrcode:console-qrcode", version.ref = "qrCode" } xz = { module = "org.tukaani:xz", version.ref = "xz" } -ktor-client-core = { module = "io.ktor:ktor-client-core", version = "3.0.3" } -ktor-client-cio = { module = "io.ktor:ktor-client-cio", version = "3.0.3" } -ktor-client-websocket = { module = "io.ktor:ktor-client-websockets", version = "3.0.3" } +zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } test-commons-codec = { module = "commons-codec:commons-codec", version.ref = "commonsCodec" } test-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit5" } @@ -73,6 +75,7 @@ testing = [ "test-mock-webserver3", "test-mockito-core", "test-mockito-jupiter", + "zstd" ] ktor = [ diff --git a/javasteam-samples/build.gradle.kts b/javasteam-samples/build.gradle.kts index 07fe3c38..976e3a79 100644 --- a/javasteam-samples/build.gradle.kts +++ b/javasteam-samples/build.gradle.kts @@ -14,10 +14,11 @@ java { dependencies { implementation(rootProject) - implementation(libs.okHttp) implementation(libs.bouncyCastle) implementation(libs.gson) implementation(libs.kotlin.coroutines) + implementation(libs.okHttp) implementation(libs.protobuf.java) // To access protobufs directly as shown in Sample #2 implementation(libs.qrCode) + implementation(libs.zstd) // Content Downloading. } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index 5615ede5..aebd8bcb 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -214,7 +214,7 @@ class Client(steamClient: SteamClient) : Closeable { proxyServer: Server? = null, cdnAuthToken: String? = null, ): CompletableFuture = defaultScope.future { - downloadManifest( + return@future downloadManifest( depotId = depotId, manifestId = manifestId, manifestRequestCode = manifestRequestCode.toULong(), @@ -372,7 +372,7 @@ class Client(steamClient: SteamClient) : Closeable { proxyServer: Server? = null, cdnAuthToken: String? = null, ): CompletableFuture = defaultScope.future { - downloadDepotChunk( + return@future downloadDepotChunk( depotId = depotId, chunk = chunk, server = server, diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt index ea1e6160..9022fa20 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt @@ -4,6 +4,7 @@ import `in`.dragonbra.javasteam.types.ChunkData import `in`.dragonbra.javasteam.util.Strings import `in`.dragonbra.javasteam.util.Utils import `in`.dragonbra.javasteam.util.VZipUtil +import `in`.dragonbra.javasteam.util.VZstdUtil import `in`.dragonbra.javasteam.util.ZipUtil import `in`.dragonbra.javasteam.util.crypto.CryptoHelper import `in`.dragonbra.javasteam.util.stream.MemoryStream @@ -73,7 +74,11 @@ object DepotChunk { buffer[3] == 'a'.code.toByte() ) { // Zstd - throw RuntimeException("Zstd compressed chunks are not yet implemented in JavaSteam.") + writtenDecompressed = VZstdUtil.decompress( + buffer = buffer.copyOfRange(0, written), + destination = destination, + verifyChecksum = false, + ) } else if (buffer[0] == 'V'.code.toByte() && buffer[1] == 'Z'.code.toByte() && buffer[2] == 'a'.code.toByte() diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt index 86d51deb..92e5d764 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt @@ -1,721 +1,724 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.enums.EDepotFileFlag -import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.steam.cdn.ClientPool -import `in`.dragonbra.javasteam.steam.cdn.Server -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest -import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps -import `in`.dragonbra.javasteam.steam.handlers.steamapps.callback.PICSProductInfoCallback -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.types.ChunkData -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.types.FileData -import `in`.dragonbra.javasteam.types.KeyValue -import `in`.dragonbra.javasteam.util.SteamKitWebRequestException -import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.Utils -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.future.future -import kotlinx.coroutines.isActive -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit -import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream -import java.io.RandomAccessFile -import java.nio.ByteBuffer -import java.nio.file.Paths -import java.time.Instant -import java.time.temporal.ChronoUnit -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentLinkedQueue - -@Suppress("unused", "SpellCheckingInspection") -class ContentDownloader(val steamClient: SteamClient) { - - companion object { - private const val HTTP_UNAUTHORIZED = 401 - private const val HTTP_FORBIDDEN = 403 - private const val HTTP_NOT_FOUND = 404 - private const val SERVICE_UNAVAILABLE = 503 - - internal const val INVALID_APP_ID = Int.MAX_VALUE - internal const val INVALID_MANIFEST_ID = Long.MAX_VALUE - - private val logger: Logger = LogManager.getLogger(ContentDownloader::class.java) - } - - private val defaultScope = CoroutineScope(Dispatchers.IO) - - private fun requestDepotKey( - appId: Int, - depotId: Int, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.getDepotDecryptionKey(depotId, appId)?.await() - - return@async Pair(callback?.result ?: EResult.Fail, callback?.depotKey) - } - - private fun getDepotManifestId( - app: PICSProductInfo, - depotId: Int, - branchId: String, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val depot = app.keyValues["depots"][depotId.toString()] - if (depot == KeyValue.INVALID) { - logger.error("Could not find depot $depotId of ${app.id}") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val manifest = depot["manifests"][branchId] - if (manifest != KeyValue.INVALID) { - return@async Pair(app.id, manifest["gid"].asLong()) - } - - val depotFromApp = depot["depotfromapp"].asInteger(INVALID_APP_ID) - if (depotFromApp == app.id || depotFromApp == INVALID_APP_ID) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val innerApp = getAppInfo(depotFromApp, parentScope).await() - if (innerApp == null) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - return@async getDepotManifestId(innerApp, depotId, branchId, parentScope).await() - } - - private fun getAppDirName(app: PICSProductInfo): String { - val installDirKeyValue = app.keyValues["config"]["installdir"] - - return if (installDirKeyValue != KeyValue.INVALID) installDirKeyValue.value else app.id.toString() - } - - private fun getAppInfo( - appId: Int, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.picsGetProductInfo(PICSRequest(appId))?.await() - val apps = callback?.results?.flatMap { (it as PICSProductInfoCallback).apps.values } - - if (apps.isNullOrEmpty()) { - logger.error("Received empty apps list in PICSProductInfo response for $appId") - return@async null - } - - if (apps.size > 1) { - logger.debug("Received ${apps.size} apps from PICSProductInfo for $appId, using first result") - } - - return@async apps.first() - } - - /** - * Kotlin coroutines version - */ - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope = defaultScope, - ): Deferred = parentScope.async { - downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = onDownloadProgress, - scope = parentScope - ) - } - - /** - * Java-friendly version that returns a CompletableFuture - */ - @JvmOverloads - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - progressCallback: ProgressCallback? = null, - ): CompletableFuture = defaultScope.future { - downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = progressCallback?.let { callback -> { progress -> callback.onProgress(progress) } }, - scope = defaultScope - ) - } - - private suspend fun downloadAppInternal( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - scope: CoroutineScope, - ): Boolean { - if (!scope.isActive) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - return false - } - - val cdnPool = ClientPool(steamClient, appId, scope) - - val shiftedAppId: Int - val manifestId: Long - val appInfo = getAppInfo(appId, scope).await() - - if (appInfo == null) { - logger.error("Could not retrieve PICSProductInfo of $appId") - return false - } - - getDepotManifestId(appInfo, depotId, branch, scope).await().apply { - shiftedAppId = first - manifestId = second - } - - val depotKeyResult = requestDepotKey(shiftedAppId, depotId, scope).await() - - if (depotKeyResult.first != EResult.OK || depotKeyResult.second == null) { - logger.error("Depot key request for $appId failed with result ${depotKeyResult.first}") - return false - } - - val depotKey = depotKeyResult.second!! - - var newProtoManifest = steamClient.configuration.depotManifestProvider.fetchManifest(depotId, manifestId) - var oldProtoManifest = steamClient.configuration.depotManifestProvider.fetchLatestManifest(depotId) - - if (oldProtoManifest?.manifestGID == manifestId) { - oldProtoManifest = null - } - - // In case we have an early exit, this will force equiv of verifyall next run. - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, INVALID_MANIFEST_ID) - - try { - if (newProtoManifest == null) { - newProtoManifest = - downloadFilesManifestOf(shiftedAppId, depotId, manifestId, branch, depotKey, cdnPool, scope).await() - } else { - logger.debug("Already have manifest $manifestId for depot $depotId.") - } - - if (newProtoManifest == null) { - logger.error("Failed to retrieve files manifest for app: $shiftedAppId depot: $depotId manifest: $manifestId branch: $branch") - return false - } - - if (!scope.isActive) { - return false - } - - val downloadCounter = GlobalDownloadCounter() - val installDir = Paths.get(installPath, getAppDirName(appInfo)).toString() - val stagingDir = Paths.get(stagingPath, getAppDirName(appInfo)).toString() - val depotFileData = DepotFilesData( - depotDownloadInfo = DepotDownloadInfo(depotId, shiftedAppId, manifestId, branch, installDir, depotKey), - depotCounter = DepotDownloadCounter( - completeDownloadSize = newProtoManifest.totalUncompressedSize - ), - stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest - ) - - downloadDepotFiles(cdnPool, downloadCounter, depotFileData, maxDownloads, onDownloadProgress, scope).await() - - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, manifestId) - - cdnPool.shutdown() - - // delete the staging directory of this app - File(stagingDir).deleteRecursively() - - logger.debug( - "Depot $depotId - Downloaded ${depotFileData.depotCounter.depotBytesCompressed} " + - "bytes (${depotFileData.depotCounter.depotBytesUncompressed} bytes uncompressed)" - ) - - return true - } catch (e: CancellationException) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - - return false - } catch (e: Exception) { - logger.error("Error occurred while downloading app $shiftedAppId", e) - - return false - } - } - - private fun downloadDepotFiles( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - maxDownloads: Int, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!parentScope.isActive) { - return@async - } - - depotFilesData.manifest.files.forEach { file -> - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - if (file.flags.contains(EDepotFileFlag.Directory)) { - File(fileFinalPath).mkdirs() - File(fileStagingPath).mkdirs() - } else { - // Some manifests don't explicitly include all necessary directories - File(fileFinalPath).parentFile.mkdirs() - File(fileStagingPath).parentFile.mkdirs() - } - } - - logger.debug("Downloading depot ${depotFilesData.depotDownloadInfo.depotId}") - - val files = depotFilesData.manifest.files.filter { !it.flags.contains(EDepotFileFlag.Directory) }.toTypedArray() - val networkChunkQueue = ConcurrentLinkedQueue>() - - val downloadSemaphore = Semaphore(maxDownloads) - files.map { file -> - async { - downloadSemaphore.withPermit { - downloadDepotFile(depotFilesData, file, networkChunkQueue, onDownloadProgress, parentScope).await() - } - } - }.awaitAll() - - networkChunkQueue.map { (fileStreamData, fileData, chunk) -> - async { - downloadSemaphore.withPermit { - downloadSteam3DepotFileChunk( - cdnPool = cdnPool, - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = fileData, - fileStreamData = fileStreamData, - chunk = chunk, - onDownloadProgress = onDownloadProgress, - parentScope = parentScope - ).await() - } - } - }.awaitAll() - - // Check for deleted files if updating the depot. - depotFilesData.previousManifest?.apply { - val previousFilteredFiles = files.asSequence().map { it.fileName }.toMutableSet() - - // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names - previousFilteredFiles.removeAll(depotFilesData.manifest.files.map { it.fileName }.toSet()) - - for (existingFileName in previousFilteredFiles) { - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, existingFileName).toString() - - if (!File(fileFinalPath).exists()) { - continue - } - - File(fileFinalPath).delete() - logger.debug("Deleted $fileFinalPath") - } - } - } - - private fun downloadDepotFile( - depotFilesData: DepotFilesData, - file: FileData, - networkChunkQueue: ConcurrentLinkedQueue>, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depotDownloadCounter = depotFilesData.depotCounter - val oldManifestFile = depotFilesData.previousManifest?.files?.find { it.fileName == file.fileName } - - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - // This may still exist if the previous run exited before cleanup - File(fileStagingPath).takeIf { it.exists() }?.delete() - - val neededChunks: MutableList - val fi = File(fileFinalPath) - val fileDidExist = fi.exists() - - if (!fileDidExist) { - // create new file. need all chunks - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - } - - neededChunks = file.chunks.toMutableList() - } else { - // open existing - if (oldManifestFile != null) { - neededChunks = mutableListOf() - - val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) - if (!hashMatches) { - logger.debug("Validating $fileFinalPath") - - val matchingChunks = mutableListOf() - - for (chunk in file.chunks) { - val oldChunk = oldManifestFile.chunks.find { it.chunkID.contentEquals(chunk.chunkID) } - if (oldChunk != null) { - matchingChunks.add(ChunkMatch(oldChunk, chunk)) - } else { - neededChunks.add(chunk) - } - } - - val orderedChunks = matchingChunks.sortedBy { it.oldChunk.offset } - - val copyChunks = mutableListOf() - - FileInputStream(fileFinalPath).use { fsOld -> - for (match in orderedChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - val adler = Utils.adlerHash(tmp) - if (adler != match.oldChunk.checksum) { - neededChunks.add(match.newChunk) - } else { - copyChunks.add(match) - } - } - } - - if (neededChunks.isNotEmpty()) { - File(fileFinalPath).renameTo(File(fileStagingPath)) - - FileInputStream(fileStagingPath).use { fsOld -> - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - - for (match in copyChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - fs.channel.position(match.newChunk.offset) - fs.write(tmp) - } - } - } - - File(fileStagingPath).delete() - } - } - } else { - // No old manifest or file not in old manifest. We must validate. - RandomAccessFile(fileFinalPath, "rw").use { fs -> - if (fi.length() != file.totalSize) { - fs.channel.truncate(file.totalSize) - } - - logger.debug("Validating $fileFinalPath") - neededChunks = Utils.validateSteam3FileChecksums( - fs, - file.chunks.sortedBy { it.offset }.toTypedArray() - ) - } - } - - if (neededChunks.isEmpty()) { - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += file.totalSize - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - - return@async - } - - val sizeOnDisk = file.totalSize - neededChunks.sumOf { it.uncompressedLength.toLong() } - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += sizeOnDisk - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - } - - val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) - if (fileIsExecutable && - (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) - ) { - File(fileFinalPath).setExecutable(true) - } else if (!fileIsExecutable && oldManifestFile != null && oldManifestFile.flags.contains(EDepotFileFlag.Executable)) { - File(fileFinalPath).setExecutable(false) - } - - val fileStreamData = FileStreamData( - fileStream = null, - fileLock = Semaphore(1), - chunksToDownload = neededChunks.size - ) - - for (chunk in neededChunks) { - networkChunkQueue.add(Triple(fileStreamData, file, chunk)) - } - } - - private fun downloadSteam3DepotFileChunk( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - file: FileData, - fileStreamData: FileStreamData, - chunk: ChunkData, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depot = depotFilesData.depotDownloadInfo - val depotDownloadCounter = depotFilesData.depotCounter - - val chunkID = Strings.toHex(chunk.chunkID) - - var outputChunkData = ByteArray(chunk.uncompressedLength) - var writtenBytes = 0 - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - outputChunkData = ByteArray(chunk.uncompressedLength) - writtenBytes = cdnPool.cdnClient.downloadDepotChunk( - depotId = depot.depotId, - chunk = chunk, - server = connection!!, - destination = outputChunkData, - depotKey = depot.depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - when (e.statusCode) { - HTTP_UNAUTHORIZED, HTTP_FORBIDDEN -> { - logger.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") - break - } - - else -> logger.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") - } - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered unexpected error downloading chunk $chunkID", e) - } - } while (isActive && writtenBytes <= 0) - - if (writtenBytes <= 0) { - logger.error("Failed to find any server with chunk $chunkID for depot ${depot.depotId}. Aborting.") - throw CancellationException("Failed to download chunk") - } - - try { - fileStreamData.fileLock.acquire() - - if (fileStreamData.fileStream == null) { - val fileFinalPath = Paths.get(depot.installDir, file.fileName).toString() - val randomAccessFile = RandomAccessFile(fileFinalPath, "rw") - fileStreamData.fileStream = randomAccessFile.channel - } - - fileStreamData.fileStream?.position(chunk.offset) - fileStreamData.fileStream?.write(ByteBuffer.wrap(outputChunkData, 0, writtenBytes)) - } finally { - fileStreamData.fileLock.release() - } - - val remainingChunks = synchronized(fileStreamData) { - --fileStreamData.chunksToDownload - } - if (remainingChunks <= 0) { - fileStreamData.fileStream?.close() - } - - var sizeDownloaded: Long - synchronized(depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.sizeDownloaded + outputChunkData.size - depotDownloadCounter.sizeDownloaded = sizeDownloaded - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - - onDownloadProgress?.invoke( - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - ) - } - - private fun downloadFilesManifestOf( - appId: Int, - depotId: Int, - manifestId: Long, - branch: String, - depotKey: ByteArray, - cdnPool: ClientPool, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - if (!isActive) { - return@async null - } - - var depotManifest: DepotManifest? = null - var manifestRequestCode = 0UL - var manifestRequestCodeExpiration = Instant.MIN - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - if (connection == null) continue - - val now = Instant.now() - - // In order to download this manifest, we need the current manifest request code - // The manifest request code is only valid for a specific period of time - if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { - val steamContent = steamClient.getHandler(SteamContent::class.java)!! - - manifestRequestCode = steamContent.getManifestRequestCode( - depotId = depotId, - appId = appId, - manifestId = manifestId, - branch = branch, - parentScope = parentScope - ).await() - - // This code will hopefully be valid for one period following the issuing period - manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) - - // If we could not get the manifest code, this is a fatal error - if (manifestRequestCode == 0UL) { - throw CancellationException("No manifest request code was returned for manifest $manifestId in depot $depotId") - } - } - - depotManifest = cdnPool.cdnClient.downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode, - server = connection, - depotKey = depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: CancellationException) { - logger.error("Connection timeout downloading depot manifest $depotId $manifestId") - - return@async null - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - val statusName = when (e.statusCode) { - HTTP_UNAUTHORIZED -> HTTP_UNAUTHORIZED::class.java.name - HTTP_FORBIDDEN -> HTTP_FORBIDDEN::class.java.name - HTTP_NOT_FOUND -> HTTP_NOT_FOUND::class.java.name - SERVICE_UNAVAILABLE -> SERVICE_UNAVAILABLE::class.java.name - else -> null - } - - logger.error( - "Downloading of manifest $manifestId failed for depot $depotId with " + - if (statusName != null) { - "response of $statusName(${e.statusCode})" - } else { - "status code of ${e.statusCode}" - } - ) - - return@async null - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered error downloading manifest for depot $depotId $manifestId", e) - - return@async null - } - } while (isActive && depotManifest == null) - - if (depotManifest == null) { - throw CancellationException("Unable to download manifest $manifestId for depot $depotId") - } - - val newProtoManifest = depotManifest - steamClient.configuration.depotManifestProvider.updateManifest(newProtoManifest) - - return@async newProtoManifest - } -} +package `in`.dragonbra.javasteam.steam.contentdownloader + +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.enums.EResult +import `in`.dragonbra.javasteam.steam.cdn.ClientPool +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest +import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps +import `in`.dragonbra.javasteam.steam.handlers.steamapps.callback.PICSProductInfoCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.util.SteamKitWebRequestException +import `in`.dragonbra.javasteam.util.Strings +import `in`.dragonbra.javasteam.util.Utils +import `in`.dragonbra.javasteam.util.compat.readNBytesCompat +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.future.future +import kotlinx.coroutines.isActive +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import java.io.File +import java.io.FileInputStream +import java.io.FileOutputStream +import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.nio.file.Paths +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentLinkedQueue + +@Suppress("unused", "SpellCheckingInspection") +class ContentDownloader(val steamClient: SteamClient) { + + companion object { + private const val HTTP_UNAUTHORIZED = 401 + private const val HTTP_FORBIDDEN = 403 + private const val HTTP_NOT_FOUND = 404 + private const val SERVICE_UNAVAILABLE = 503 + + internal const val INVALID_APP_ID = Int.MAX_VALUE + internal const val INVALID_MANIFEST_ID = Long.MAX_VALUE + + private val logger: Logger = LogManager.getLogger(ContentDownloader::class.java) + } + + private val defaultScope = CoroutineScope(Dispatchers.IO) + + private fun requestDepotKey( + appId: Int, + depotId: Int, + parentScope: CoroutineScope, + ): Deferred> = parentScope.async { + val steamApps = steamClient.getHandler(SteamApps::class.java) + val callback = steamApps?.getDepotDecryptionKey(depotId, appId)?.await() + + return@async Pair(callback?.result ?: EResult.Fail, callback?.depotKey) + } + + private fun getDepotManifestId( + app: PICSProductInfo, + depotId: Int, + branchId: String, + parentScope: CoroutineScope, + ): Deferred> = parentScope.async { + val depot = app.keyValues["depots"][depotId.toString()] + if (depot == KeyValue.INVALID) { + logger.error("Could not find depot $depotId of ${app.id}") + return@async Pair(app.id, INVALID_MANIFEST_ID) + } + + val manifest = depot["manifests"][branchId] + if (manifest != KeyValue.INVALID) { + return@async Pair(app.id, manifest["gid"].asLong()) + } + + val depotFromApp = depot["depotfromapp"].asInteger(INVALID_APP_ID) + if (depotFromApp == app.id || depotFromApp == INVALID_APP_ID) { + logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") + return@async Pair(app.id, INVALID_MANIFEST_ID) + } + + val innerApp = getAppInfo(depotFromApp, parentScope).await() + if (innerApp == null) { + logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") + return@async Pair(app.id, INVALID_MANIFEST_ID) + } + + return@async getDepotManifestId(innerApp, depotId, branchId, parentScope).await() + } + + private fun getAppDirName(app: PICSProductInfo): String { + val installDirKeyValue = app.keyValues["config"]["installdir"] + + return if (installDirKeyValue != KeyValue.INVALID) installDirKeyValue.value else app.id.toString() + } + + private fun getAppInfo( + appId: Int, + parentScope: CoroutineScope, + ): Deferred = parentScope.async { + val steamApps = steamClient.getHandler(SteamApps::class.java) + val callback = steamApps?.picsGetProductInfo(PICSRequest(appId))?.await() + val apps = callback?.results?.flatMap { (it as PICSProductInfoCallback).apps.values } + + if (apps.isNullOrEmpty()) { + logger.error("Received empty apps list in PICSProductInfo response for $appId") + return@async null + } + + if (apps.size > 1) { + logger.debug("Received ${apps.size} apps from PICSProductInfo for $appId, using first result") + } + + return@async apps.first() + } + + /** + * Kotlin coroutines version + */ + fun downloadApp( + appId: Int, + depotId: Int, + installPath: String, + stagingPath: String, + branch: String = "public", + maxDownloads: Int = 8, + onDownloadProgress: ((Float) -> Unit)? = null, + parentScope: CoroutineScope = defaultScope, + ): Deferred = parentScope.async { + downloadAppInternal( + appId = appId, + depotId = depotId, + installPath = installPath, + stagingPath = stagingPath, + branch = branch, + maxDownloads = maxDownloads, + onDownloadProgress = onDownloadProgress, + scope = parentScope + ) + } + + /** + * Java-friendly version that returns a CompletableFuture + */ + @JvmOverloads + fun downloadApp( + appId: Int, + depotId: Int, + installPath: String, + stagingPath: String, + branch: String = "public", + maxDownloads: Int = 8, + progressCallback: ProgressCallback? = null, + ): CompletableFuture = defaultScope.future { + return@future downloadAppInternal( + appId = appId, + depotId = depotId, + installPath = installPath, + stagingPath = stagingPath, + branch = branch, + maxDownloads = maxDownloads, + onDownloadProgress = progressCallback?.let { callback -> { progress -> callback.onProgress(progress) } }, + scope = defaultScope + ) + } + + private suspend fun downloadAppInternal( + appId: Int, + depotId: Int, + installPath: String, + stagingPath: String, + branch: String = "public", + maxDownloads: Int = 8, + onDownloadProgress: ((Float) -> Unit)? = null, + scope: CoroutineScope, + ): Boolean { + if (!scope.isActive) { + logger.error("App $appId was not completely downloaded. Operation was canceled.") + return false + } + + val cdnPool = ClientPool(steamClient, appId, scope) + + val shiftedAppId: Int + val manifestId: Long + val appInfo = getAppInfo(appId, scope).await() + + if (appInfo == null) { + logger.error("Could not retrieve PICSProductInfo of $appId") + return false + } + + getDepotManifestId(appInfo, depotId, branch, scope).await().apply { + shiftedAppId = first + manifestId = second + } + + val depotKeyResult = requestDepotKey(shiftedAppId, depotId, scope).await() + + if (depotKeyResult.first != EResult.OK || depotKeyResult.second == null) { + logger.error("Depot key request for $appId failed with result ${depotKeyResult.first}") + return false + } + + val depotKey = depotKeyResult.second!! + + var newProtoManifest = steamClient.configuration.depotManifestProvider.fetchManifest(depotId, manifestId) + var oldProtoManifest = steamClient.configuration.depotManifestProvider.fetchLatestManifest(depotId) + + if (oldProtoManifest?.manifestGID == manifestId) { + oldProtoManifest = null + } + + // In case we have an early exit, this will force equiv of verifyall next run. + steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, INVALID_MANIFEST_ID) + + try { + if (newProtoManifest == null) { + newProtoManifest = + downloadFilesManifestOf(shiftedAppId, depotId, manifestId, branch, depotKey, cdnPool, scope).await() + } else { + logger.debug("Already have manifest $manifestId for depot $depotId.") + } + + if (newProtoManifest == null) { + logger.error("Failed to retrieve files manifest for app: $shiftedAppId depot: $depotId manifest: $manifestId branch: $branch") + return false + } + + if (!scope.isActive) { + return false + } + + val downloadCounter = GlobalDownloadCounter() + val installDir = Paths.get(installPath, getAppDirName(appInfo)).toString() + val stagingDir = Paths.get(stagingPath, getAppDirName(appInfo)).toString() + val depotFileData = DepotFilesData( + depotDownloadInfo = DepotDownloadInfo(depotId, shiftedAppId, manifestId, branch, installDir, depotKey), + depotCounter = DepotDownloadCounter( + completeDownloadSize = newProtoManifest.totalUncompressedSize + ), + stagingDir = stagingDir, + manifest = newProtoManifest, + previousManifest = oldProtoManifest + ) + + downloadDepotFiles(cdnPool, downloadCounter, depotFileData, maxDownloads, onDownloadProgress, scope).await() + + steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, manifestId) + + cdnPool.shutdown() + + // delete the staging directory of this app + File(stagingDir).deleteRecursively() + + logger.debug( + "Depot $depotId - Downloaded ${depotFileData.depotCounter.depotBytesCompressed} " + + "bytes (${depotFileData.depotCounter.depotBytesUncompressed} bytes uncompressed)" + ) + + return true + } catch (e: CancellationException) { + logger.error("App $appId was not completely downloaded. Operation was canceled.") + + return false + } catch (e: Exception) { + logger.error("Error occurred while downloading app $shiftedAppId", e) + + return false + } + } + + private fun downloadDepotFiles( + cdnPool: ClientPool, + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + maxDownloads: Int, + onDownloadProgress: ((Float) -> Unit)? = null, + parentScope: CoroutineScope, + ) = parentScope.async { + if (!parentScope.isActive) { + return@async + } + + depotFilesData.manifest.files.forEach { file -> + val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() + val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() + + if (file.flags.contains(EDepotFileFlag.Directory)) { + File(fileFinalPath).mkdirs() + File(fileStagingPath).mkdirs() + } else { + // Some manifests don't explicitly include all necessary directories + File(fileFinalPath).parentFile.mkdirs() + File(fileStagingPath).parentFile.mkdirs() + } + } + + logger.debug("Downloading depot ${depotFilesData.depotDownloadInfo.depotId}") + + val files = depotFilesData.manifest.files.filter { !it.flags.contains(EDepotFileFlag.Directory) }.toTypedArray() + val networkChunkQueue = ConcurrentLinkedQueue>() + + val downloadSemaphore = Semaphore(maxDownloads) + files.map { file -> + async { + downloadSemaphore.withPermit { + downloadDepotFile(depotFilesData, file, networkChunkQueue, onDownloadProgress, parentScope).await() + } + } + }.awaitAll() + + networkChunkQueue.map { (fileStreamData, fileData, chunk) -> + async { + downloadSemaphore.withPermit { + downloadSteam3DepotFileChunk( + cdnPool = cdnPool, + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = fileData, + fileStreamData = fileStreamData, + chunk = chunk, + onDownloadProgress = onDownloadProgress, + parentScope = parentScope + ).await() + } + } + }.awaitAll() + + // Check for deleted files if updating the depot. + depotFilesData.previousManifest?.apply { + val previousFilteredFiles = files.asSequence().map { it.fileName }.toMutableSet() + + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names + previousFilteredFiles.removeAll(depotFilesData.manifest.files.map { it.fileName }.toSet()) + + for (existingFileName in previousFilteredFiles) { + val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, existingFileName).toString() + + if (!File(fileFinalPath).exists()) { + continue + } + + File(fileFinalPath).delete() + logger.debug("Deleted $fileFinalPath") + } + } + } + + private fun downloadDepotFile( + depotFilesData: DepotFilesData, + file: FileData, + networkChunkQueue: ConcurrentLinkedQueue>, + onDownloadProgress: ((Float) -> Unit)? = null, + parentScope: CoroutineScope, + ) = parentScope.async { + if (!isActive) { + return@async + } + + val depotDownloadCounter = depotFilesData.depotCounter + val oldManifestFile = depotFilesData.previousManifest?.files?.find { it.fileName == file.fileName } + + val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() + val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() + + // This may still exist if the previous run exited before cleanup + File(fileStagingPath).takeIf { it.exists() }?.delete() + + val neededChunks: MutableList + val fi = File(fileFinalPath) + val fileDidExist = fi.exists() + + if (!fileDidExist) { + // create new file. need all chunks + FileOutputStream(fileFinalPath).use { fs -> + fs.channel.truncate(file.totalSize) + } + + neededChunks = file.chunks.toMutableList() + } else { + // open existing + if (oldManifestFile != null) { + neededChunks = mutableListOf() + + val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) + if (!hashMatches) { + logger.debug("Validating $fileFinalPath") + + val matchingChunks = mutableListOf() + + for (chunk in file.chunks) { + val oldChunk = oldManifestFile.chunks.find { it.chunkID.contentEquals(chunk.chunkID) } + if (oldChunk != null) { + matchingChunks.add(ChunkMatch(oldChunk, chunk)) + } else { + neededChunks.add(chunk) + } + } + + val orderedChunks = matchingChunks.sortedBy { it.oldChunk.offset } + + val copyChunks = mutableListOf() + + FileInputStream(fileFinalPath).use { fsOld -> + for (match in orderedChunks) { + fsOld.channel.position(match.oldChunk.offset) + + val tmp = ByteArray(match.oldChunk.uncompressedLength) + fsOld.readNBytesCompat(tmp, 0, tmp.size) + + val adler = Utils.adlerHash(tmp) + if (adler != match.oldChunk.checksum) { + neededChunks.add(match.newChunk) + } else { + copyChunks.add(match) + } + } + } + + if (neededChunks.isNotEmpty()) { + File(fileFinalPath).renameTo(File(fileStagingPath)) + + FileInputStream(fileStagingPath).use { fsOld -> + FileOutputStream(fileFinalPath).use { fs -> + fs.channel.truncate(file.totalSize) + + for (match in copyChunks) { + fsOld.channel.position(match.oldChunk.offset) + + val tmp = ByteArray(match.oldChunk.uncompressedLength) + fsOld.readNBytesCompat(tmp, 0, tmp.size) + + fs.channel.position(match.newChunk.offset) + fs.write(tmp) + } + } + } + + File(fileStagingPath).delete() + } + } + } else { + // No old manifest or file not in old manifest. We must validate. + RandomAccessFile(fileFinalPath, "rw").use { fs -> + if (fi.length() != file.totalSize) { + fs.channel.truncate(file.totalSize) + } + + logger.debug("Validating $fileFinalPath") + neededChunks = Utils.validateSteam3FileChecksums( + fs, + file.chunks.sortedBy { it.offset }.toTypedArray() + ) + } + } + + if (neededChunks.isEmpty()) { + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += file.totalSize + } + + onDownloadProgress?.apply { + val totalPercent = + depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize + this(totalPercent) + } + + return@async + } + + val sizeOnDisk = file.totalSize - neededChunks.sumOf { it.uncompressedLength.toLong() } + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += sizeOnDisk + } + + onDownloadProgress?.apply { + val totalPercent = + depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize + this(totalPercent) + } + } + + val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) + if (fileIsExecutable && + (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) + ) { + File(fileFinalPath).setExecutable(true) + } else if (!fileIsExecutable && oldManifestFile != null && oldManifestFile.flags.contains(EDepotFileFlag.Executable)) { + File(fileFinalPath).setExecutable(false) + } + + val fileStreamData = FileStreamData( + fileStream = null, + fileLock = Semaphore(1), + chunksToDownload = neededChunks.size + ) + + for (chunk in neededChunks) { + networkChunkQueue.add(Triple(fileStreamData, file, chunk)) + } + } + + private fun downloadSteam3DepotFileChunk( + cdnPool: ClientPool, + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + fileStreamData: FileStreamData, + chunk: ChunkData, + onDownloadProgress: ((Float) -> Unit)? = null, + parentScope: CoroutineScope, + ) = parentScope.async { + if (!isActive) { + return@async + } + + val depot = depotFilesData.depotDownloadInfo + val depotDownloadCounter = depotFilesData.depotCounter + + val chunkID = Strings.toHex(chunk.chunkID) + + var outputChunkData = ByteArray(chunk.uncompressedLength) + var writtenBytes = 0 + + do { + var connection: Server? = null + + try { + connection = cdnPool.getConnection().await() + + outputChunkData = ByteArray(chunk.uncompressedLength) + writtenBytes = cdnPool.cdnClient.downloadDepotChunk( + depotId = depot.depotId, + chunk = chunk, + server = connection!!, + destination = outputChunkData, + depotKey = depot.depotKey, + proxyServer = cdnPool.proxyServer + ) + + cdnPool.returnConnection(connection) + } catch (e: SteamKitWebRequestException) { + cdnPool.returnBrokenConnection(connection) + + when (e.statusCode) { + HTTP_UNAUTHORIZED, HTTP_FORBIDDEN -> { + logger.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") + break + } + + else -> logger.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") + } + } catch (e: NoClassDefFoundError) { + // Zstd is a 'compileOnly' dependency. + throw CancellationException(e.message) + } catch (e: Exception) { + cdnPool.returnBrokenConnection(connection) + + logger.error("Encountered unexpected error downloading chunk $chunkID", e) + } + } while (isActive && writtenBytes <= 0) + + if (writtenBytes <= 0) { + logger.error("Failed to find any server with chunk $chunkID for depot ${depot.depotId}. Aborting.") + throw CancellationException("Failed to download chunk") + } + + try { + fileStreamData.fileLock.acquire() + + if (fileStreamData.fileStream == null) { + val fileFinalPath = Paths.get(depot.installDir, file.fileName).toString() + val randomAccessFile = RandomAccessFile(fileFinalPath, "rw") + fileStreamData.fileStream = randomAccessFile.channel + } + + fileStreamData.fileStream?.position(chunk.offset) + fileStreamData.fileStream?.write(ByteBuffer.wrap(outputChunkData, 0, writtenBytes)) + } finally { + fileStreamData.fileLock.release() + } + + val remainingChunks = synchronized(fileStreamData) { + --fileStreamData.chunksToDownload + } + if (remainingChunks <= 0) { + fileStreamData.fileStream?.close() + } + + var sizeDownloaded: Long + synchronized(depotDownloadCounter) { + sizeDownloaded = depotDownloadCounter.sizeDownloaded + outputChunkData.size + depotDownloadCounter.sizeDownloaded = sizeDownloaded + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + onDownloadProgress?.invoke( + depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize + ) + } + + private fun downloadFilesManifestOf( + appId: Int, + depotId: Int, + manifestId: Long, + branch: String, + depotKey: ByteArray, + cdnPool: ClientPool, + parentScope: CoroutineScope, + ): Deferred = parentScope.async { + if (!isActive) { + return@async null + } + + var depotManifest: DepotManifest? = null + var manifestRequestCode = 0UL + var manifestRequestCodeExpiration = Instant.MIN + + do { + var connection: Server? = null + + try { + connection = cdnPool.getConnection().await() + + if (connection == null) continue + + val now = Instant.now() + + // In order to download this manifest, we need the current manifest request code + // The manifest request code is only valid for a specific period of time + if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { + val steamContent = steamClient.getHandler(SteamContent::class.java)!! + + manifestRequestCode = steamContent.getManifestRequestCode( + depotId = depotId, + appId = appId, + manifestId = manifestId, + branch = branch, + parentScope = parentScope + ).await() + + // This code will hopefully be valid for one period following the issuing period + manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) + + // If we could not get the manifest code, this is a fatal error + if (manifestRequestCode == 0UL) { + throw CancellationException("No manifest request code was returned for manifest $manifestId in depot $depotId") + } + } + + depotManifest = cdnPool.cdnClient.downloadManifest( + depotId = depotId, + manifestId = manifestId, + manifestRequestCode = manifestRequestCode, + server = connection, + depotKey = depotKey, + proxyServer = cdnPool.proxyServer + ) + + cdnPool.returnConnection(connection) + } catch (e: CancellationException) { + logger.error("Connection timeout downloading depot manifest $depotId $manifestId") + + return@async null + } catch (e: SteamKitWebRequestException) { + cdnPool.returnBrokenConnection(connection) + + val statusName = when (e.statusCode) { + HTTP_UNAUTHORIZED -> HTTP_UNAUTHORIZED::class.java.name + HTTP_FORBIDDEN -> HTTP_FORBIDDEN::class.java.name + HTTP_NOT_FOUND -> HTTP_NOT_FOUND::class.java.name + SERVICE_UNAVAILABLE -> SERVICE_UNAVAILABLE::class.java.name + else -> null + } + + logger.error( + "Downloading of manifest $manifestId failed for depot $depotId with " + + if (statusName != null) { + "response of $statusName(${e.statusCode})" + } else { + "status code of ${e.statusCode}" + } + ) + + return@async null + } catch (e: Exception) { + cdnPool.returnBrokenConnection(connection) + + logger.error("Encountered error downloading manifest for depot $depotId $manifestId", e) + + return@async null + } + } while (isActive && depotManifest == null) + + if (depotManifest == null) { + throw CancellationException("Unable to download manifest $manifestId for depot $depotId") + } + + val newProtoManifest = depotManifest + steamClient.configuration.depotManifestProvider.updateManifest(newProtoManifest) + + return@async newProtoManifest + } +} diff --git a/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt b/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt new file mode 100644 index 00000000..ca6c09b6 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt @@ -0,0 +1,75 @@ +package `in`.dragonbra.javasteam.util + +import com.github.luben.zstd.Zstd +import `in`.dragonbra.javasteam.util.log.LogManager +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.zip.CRC32 + +object VZstdUtil { + + private const val VZSTD_HEADER: Int = 0x615A5356 + + private val logger = LogManager.getLogger(VZstdUtil::class.java) + + @Throws(IOException::class, IllegalArgumentException::class) + @JvmStatic + @JvmOverloads + fun decompress(buffer: ByteArray, destination: ByteArray, verifyChecksum: Boolean = false): Int { + val byteBuffer = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN) // Convert the buffer. + + val header = byteBuffer.getInt(0) + if (header != VZSTD_HEADER) { + throw IOException("Expecting VZstdHeader at start of stream") + } + + val crc32 = byteBuffer.getInt(4) + val crc32Footer = byteBuffer.getInt(buffer.size - 15) + val sizeDecompressed = byteBuffer.getInt(buffer.size - 11) + + if (crc32 == crc32Footer) { + // They write CRC32 twice? + logger.debug("CRC32 appears to be written twice in the data") + } + + if (buffer[buffer.size - 3] != 'z'.code.toByte() || + buffer[buffer.size - 2] != 's'.code.toByte() || + buffer[buffer.size - 1] != 'v'.code.toByte() + ) { + throw IOException("Expecting VZstdFooter at end of stream") + } + + if (destination.size < sizeDecompressed) { + throw IllegalArgumentException("The destination buffer is smaller than the decompressed data size.") + } + + val compressedData = buffer.copyOfRange(8, buffer.size - 15) + + try { + val bytesDecompressed = Zstd.decompress(destination, compressedData) + + if (bytesDecompressed != sizeDecompressed.toLong()) { + throw IOException("Failed to decompress Zstd (expected $sizeDecompressed bytes, got $bytesDecompressed).") + } + + if (verifyChecksum) { + val crc = CRC32() + crc.update(destination, 0, sizeDecompressed) + val calculatedCrc = crc.value.toInt() + if (calculatedCrc != crc32Footer) { + throw IOException("CRC does not match decompressed data. VZstd data may be corrupted.") + } + } + + return sizeDecompressed + } catch (e: NoClassDefFoundError) { + // Zstd is a 'compileOnly' dependency. If it's missing, throw the correct type of error. + logger.error("Missing implementation of com.github.luben:zstd-jni") + throw e + } catch (e: Exception) { + // Catch all for the Zstd library. + throw IOException("Failed to decompress Zstd data: ${e.message}", e) + } + } +} diff --git a/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java b/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java index 8b575519..d14eb1f5 100644 --- a/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java +++ b/src/test/java/in/dragonbra/javasteam/steam/cdn/DepotChunkTest.java @@ -90,4 +90,43 @@ public void decryptsAndDecompressesDepotChunkVZip() throws IOException, NoSuchAl Assertions.assertEquals("7B8567D9B3C09295CDBF4978C32B348D8E76C750", hash); } + + @Test + public void decryptsAndDecompressesDepotChunkZStd() throws IOException, NoSuchAlgorithmException { + var stream = getClass().getClassLoader() + .getResourceAsStream("depot/depot_3441461_chunk_9e72678e305540630a665b93e1463bc3983eb55a.bin"); + + var ms = new MemoryStream(); + IOUtils.copy(stream, ms.asOutputStream()); + + var chunkData = ms.toByteArray(); + + var chunk = new ChunkData( + new byte[0], // id is not needed here + Integer.parseUnsignedInt("3753325726"), + 0, + 176, + 156 + ); + + var destination = new byte[chunk.getUncompressedLength()]; + var writtenLength = DepotChunk.process( + chunk, + chunkData, + destination, + new byte[]{ + (byte) 0x01, (byte) 0x02, (byte) 0x03, (byte) 0x04, (byte) 0x05, (byte) 0x06, (byte) 0x07, (byte) 0x08, + (byte) 0x09, (byte) 0x0A, (byte) 0x0B, (byte) 0x0C, (byte) 0x0D, (byte) 0x0E, (byte) 0x0F, (byte) 0x10, + (byte) 0x11, (byte) 0x12, (byte) 0x13, (byte) 0x14, (byte) 0x15, (byte) 0x16, (byte) 0x17, (byte) 0x18, + (byte) 0x19, (byte) 0x1A, (byte) 0x1B, (byte) 0x1C, (byte) 0x1D, (byte) 0x1E, (byte) 0x1F, (byte) 0x20 + } + ); + + Assertions.assertEquals(chunk.getCompressedLength(), chunkData.length); + Assertions.assertEquals(chunk.getUncompressedLength(), writtenLength); + + var hash = Hex.encodeHexString(MessageDigest.getInstance("SHA-1").digest(destination), false); + + Assertions.assertEquals("9E72678E305540630A665B93E1463BC3983EB55A", hash); + } } diff --git a/src/test/resources/depot/depot_3441461_chunk_9e72678e305540630a665b93e1463bc3983eb55a.bin b/src/test/resources/depot/depot_3441461_chunk_9e72678e305540630a665b93e1463bc3983eb55a.bin new file mode 100644 index 00000000..dd121572 --- /dev/null +++ b/src/test/resources/depot/depot_3441461_chunk_9e72678e305540630a665b93e1463bc3983eb55a.bin @@ -0,0 +1 @@ +˚b|tF#_ < Jݷz(GOX%94|X)ZϫRnM`"&>ΫT[NEP!kЌ9K(ͮŎ楹ڟyի=?lI0i` i ٱMx \ No newline at end of file