From d13d5fedb794f9b6ab6d217f4e271dbe4b034d0d Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 26 Apr 2023 15:38:59 +0300 Subject: [PATCH 1/3] Coverage algo update --- .../kotlin/org/utbot/python/PythonEngine.kt | 1 + .../evaluation/PythonCodeSocketExecutor.kt | 3 ++ .../evaluation/PythonCoverageReceiver.kt | 29 +++++++++++++++++++ .../python/evaluation/PythonWorkerManager.kt | 2 +- .../ExecutionRequestSerializer.kt | 1 + .../evaluation/utils/CoverageIdGenerator.kt | 14 +++++++++ .../utbot/python/utils/RequirementsUtils.kt | 2 +- 7 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 0bb1086aad..2a2a169825 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -11,6 +11,7 @@ import org.utbot.fuzzing.utils.Trie import org.utbot.python.evaluation.* import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree +import org.utbot.python.evaluation.utils.CoverageIdGenerator import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.PythonTreeWrapper diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 4190d22c14..36d5034ffa 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -12,6 +12,7 @@ import org.utbot.python.evaluation.serialiation.FailExecution import org.utbot.python.evaluation.serialiation.PythonExecutionResult import org.utbot.python.evaluation.serialiation.SuccessExecution import org.utbot.python.evaluation.serialiation.serializeObjects +import org.utbot.python.evaluation.utils.CoverageIdGenerator import org.utbot.python.framework.api.python.util.pythonAnyClassId import org.utbot.python.newtyping.pythonTypeName import org.utbot.python.newtyping.pythonTypeRepresentation @@ -60,6 +61,7 @@ class PythonCodeSocketExecutor( fullname.drop(moduleToImport.length).removePrefix(".") } + val coverageId = CoverageIdGenerator.createId() val request = ExecutionRequest( functionTextName, moduleToImport, @@ -69,6 +71,7 @@ class PythonCodeSocketExecutor( emptyMap(), // here can be only-kwargs arguments memory, method.moduleFilename, + coverageId, ) val message = ExecutionRequestSerializer.serializeRequest(request) ?: error("Cannot serialize request to python executor") try { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt new file mode 100644 index 0000000000..8eef6cfb7f --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -0,0 +1,29 @@ +package org.utbot.python.evaluation + +import mu.KotlinLogging +import java.io.IOException +import java.net.DatagramPacket +import java.net.DatagramSocket +import java.net.SocketException + +class PythonCoverageReceiver( + private val port: Int, +) { + val coverageStorage = mutableMapOf>() + private val socket = DatagramSocket(port) + private val logger = KotlinLogging.logger {} + + private fun run() { + try { + while (true) { + val request = DatagramPacket(byteArrayOf(1), 1) + socket.receive(request) + println(request.data) + } + } catch (ex: SocketException) { + logger.error("Socket error: " + ex.message); + } catch (ex: IOException) { + logger.error("IO error: " + ex.message); + } + } +} \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 1b701b0e70..11734ef16b 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -38,7 +38,7 @@ class PythonWorkerManager( "localhost", serverSocket.localPort.toString(), "--logfile", logfile.absolutePath, - "--loglevel", "INFO", // "DEBUG", "INFO", "WARNING", "ERROR" + "--loglevel", "DEBUG", // "DEBUG", "INFO", "WARNING", "ERROR" )) timeout = max(until - processStartTime, 0) workerSocket = try { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt index aa81e60087..80efc3393d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/serialiation/ExecutionRequestSerializer.kt @@ -24,4 +24,5 @@ data class ExecutionRequest( val kwargumentsIds: Map, val serializedMemory: String, val filepath: String, + val coverageId: String, ) diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt new file mode 100644 index 0000000000..6fc045979d --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt @@ -0,0 +1,14 @@ +package org.utbot.python.evaluation.utils + +import java.util.concurrent.atomic.AtomicLong + +object CoverageIdGenerator { + private const val lower_bound: Long = 1500_000_000 + + private val lastId: AtomicLong = AtomicLong(lower_bound) + + fun createId(): String { + return lastId.incrementAndGet().toString(radix = 16) + } + +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt index 4689bb0d09..e8f51f5560 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt @@ -3,7 +3,7 @@ package org.utbot.python.utils object RequirementsUtils { val requirements: List = listOf( "mypy==1.0.0", - "utbot-executor==1.4.21", + "utbot-executor==1.4.23", "utbot-mypy-runner==0.2.8", ) From 4e6c7d9829354b43daa95fb8d54271c74f46afa1 Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Thu, 27 Apr 2023 16:29:20 +0300 Subject: [PATCH 2/3] Update coverage --- .../kotlin/org/utbot/python/PythonEngine.kt | 26 ++++++++++++++++--- .../python/evaluation/CodeEvaluationApi.kt | 6 +++++ .../evaluation/PythonCodeSocketExecutor.kt | 10 ++++++- .../evaluation/PythonCoverageReceiver.kt | 22 +++++++++++----- .../python/evaluation/PythonWorkerManager.kt | 24 +++++++++++++++++ .../evaluation/utils/CoverageIdGenerator.kt | 1 - .../utbot/python/evaluation/utils/Utils.kt | 17 ++++++++++++ .../utbot/python/utils/RequirementsUtils.kt | 2 +- 8 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt diff --git a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt index 2a2a169825..4b627e6d2d 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/PythonEngine.kt @@ -12,6 +12,7 @@ import org.utbot.python.evaluation.* import org.utbot.python.evaluation.serialiation.MemoryDump import org.utbot.python.evaluation.serialiation.toPythonTree import org.utbot.python.evaluation.utils.CoverageIdGenerator +import org.utbot.python.evaluation.utils.coveredLinesToInstructions import org.utbot.python.framework.api.python.PythonTree import org.utbot.python.framework.api.python.PythonTreeModel import org.utbot.python.framework.api.python.PythonTreeWrapper @@ -85,6 +86,7 @@ class PythonEngine( private fun handleTimeoutResult( arguments: List, methodUnderTestDescription: PythonMethodDescription, + coveredLines: Collection, ): FuzzingExecutionFeedback { val summary = arguments .zip(methodUnderTest.arguments) @@ -101,12 +103,16 @@ class PythonEngine( val beforeThisObject = beforeThisObjectTree?.let { PythonTreeModel(it.tree) } val beforeModelList = beforeModelListTree.map { PythonTreeModel(it.tree) } + val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest) + val coverage = Coverage(coveredInstructions) + val utFuzzedExecution = PythonUtExecution( stateInit = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()), stateBefore = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()), stateAfter = EnvironmentModels(beforeThisObject, beforeModelList, emptyMap()), diffIds = emptyList(), result = executionResult, + coverage = coverage, testMethodName = testMethodName.testName?.camelToSnakeCase(), displayName = testMethodName.displayName, summary = summary.map { DocRegularStmt(it) } @@ -219,7 +225,9 @@ class PythonEngine( methodUnderTest.argumentsNames ) try { - return when (val evaluationResult = manager.run(functionArguments, localAdditionalModules)) { + val coverageId = CoverageIdGenerator.createId() + return when (val evaluationResult = + manager.runWithCoverage(functionArguments, localAdditionalModules, coverageId)) { is PythonEvaluationError -> { val utError = UtError( "Error evaluation: ${evaluationResult.status}, ${evaluationResult.message}", @@ -230,8 +238,20 @@ class PythonEngine( } is PythonEvaluationTimeout -> { - val utTimeoutException = handleTimeoutResult(arguments, description) - PythonExecutionResult(utTimeoutException, PythonFeedback(control = Control.PASS)) + val coveredLines = + manager.coverageReceiver.coverageStorage.getOrDefault(coverageId, mutableSetOf()) + val utTimeoutException = handleTimeoutResult(arguments, description, coveredLines) + val coveredInstructions = coveredLinesToInstructions(coveredLines, methodUnderTest) + val trieNode: Trie.Node = + if (coveredInstructions.isEmpty()) + Trie.emptyNode() + else description.tracer.add( + coveredInstructions + ) + PythonExecutionResult( + utTimeoutException, + PythonFeedback(control = Control.PASS, result = trieNode) + ) } is PythonEvaluationSuccess -> { diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt index 43744e2c63..06f0538871 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/CodeEvaluationApi.kt @@ -17,6 +17,12 @@ interface PythonCodeExecutor { additionalModulesToImport: Set ): PythonEvaluationResult + fun runWithCoverage( + fuzzedValues: FunctionArguments, + additionalModulesToImport: Set, + coverageId: String, + ): PythonEvaluationResult + fun stop() } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 36d5034ffa..5940c3bbc9 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -49,6 +49,15 @@ class PythonCodeSocketExecutor( override fun run( fuzzedValues: FunctionArguments, additionalModulesToImport: Set + ): PythonEvaluationResult { + val coverageId = CoverageIdGenerator.createId() + return runWithCoverage(fuzzedValues, additionalModulesToImport, coverageId) + } + + override fun runWithCoverage( + fuzzedValues: FunctionArguments, + additionalModulesToImport: Set, + coverageId: String ): PythonEvaluationResult { val (arguments, memory) = serializeObjects(fuzzedValues.allArguments.map { it.tree }) @@ -61,7 +70,6 @@ class PythonCodeSocketExecutor( fullname.drop(moduleToImport.length).removePrefix(".") } - val coverageId = CoverageIdGenerator.createId() val request = ExecutionRequest( functionTextName, moduleToImport, diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 8eef6cfb7f..5756a66f3f 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -7,18 +7,26 @@ import java.net.DatagramSocket import java.net.SocketException class PythonCoverageReceiver( - private val port: Int, -) { + private val until: Long, +) : Thread() { val coverageStorage = mutableMapOf>() - private val socket = DatagramSocket(port) + private val socket = DatagramSocket() private val logger = KotlinLogging.logger {} - private fun run() { + fun address(): Pair { + return "localhost" to socket.localPort.toString() + } + + override fun run() { try { - while (true) { - val request = DatagramPacket(byteArrayOf(1), 1) + while (System.currentTimeMillis() < until) { + val buf = ByteArray(256) + val request = DatagramPacket(buf, buf.size) socket.receive(request) - println(request.data) + val (id, line) = request.data.decodeToString().take(request.length).split(":") + logger.info { "Got coverage: $id, $line" } + val lineNumber = line.toInt() + coverageStorage.getOrPut(id) { mutableSetOf() } .add(lineNumber) } } catch (ex: SocketException) { logger.error("Socket error: " + ex.message); diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 11734ef16b..2bc8483de3 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -26,7 +26,10 @@ class PythonWorkerManager( private lateinit var workerSocket: Socket private lateinit var codeExecutor: PythonCodeExecutor + val coverageReceiver = PythonCoverageReceiver(until) + init { + coverageReceiver.start() connect() } @@ -37,6 +40,8 @@ class PythonWorkerManager( "-m", "utbot_executor", "localhost", serverSocket.localPort.toString(), + coverageReceiver.address().first, + coverageReceiver.address().second, "--logfile", logfile.absolutePath, "--loglevel", "DEBUG", // "DEBUG", "INFO", "WARNING", "ERROR" )) @@ -60,6 +65,7 @@ class PythonWorkerManager( fun disconnect() { workerSocket.close() process.destroy() + coverageReceiver.interrupt() } fun reconnect() { @@ -67,6 +73,24 @@ class PythonWorkerManager( connect() } + fun runWithCoverage( + fuzzedValues: FunctionArguments, + additionalModulesToImport: Set, + coverageId: String + ): PythonEvaluationResult { + val evaluationResult = try { + codeExecutor.runWithCoverage(fuzzedValues, additionalModulesToImport, coverageId) + } catch (_: SocketTimeoutException) { + logger.debug { "Socket timeout" } + reconnect() + PythonEvaluationTimeout() + } + if (evaluationResult is PythonEvaluationError) { + reconnect() + } + return evaluationResult + } + fun run( fuzzedValues: FunctionArguments, additionalModulesToImport: Set diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt index 6fc045979d..0e13ce16b7 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/CoverageIdGenerator.kt @@ -10,5 +10,4 @@ object CoverageIdGenerator { fun createId(): String { return lastId.incrementAndGet().toString(radix = 16) } - } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt new file mode 100644 index 0000000000..51d8a11e31 --- /dev/null +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/utils/Utils.kt @@ -0,0 +1,17 @@ +package org.utbot.python.evaluation.utils + +import org.utbot.framework.plugin.api.Instruction +import org.utbot.python.PythonMethod +import org.utbot.python.framework.api.python.util.pythonAnyClassId +import org.utbot.python.newtyping.pythonTypeRepresentation + +fun coveredLinesToInstructions(coveredLines: Collection, method: PythonMethod): List { + return coveredLines.map { + Instruction( + method.containingPythonClass?.pythonTypeRepresentation() ?: pythonAnyClassId.name, + method.methodSignature(), + it, + it.toLong() + ) + } +} diff --git a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt index e8f51f5560..9f16011099 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/utils/RequirementsUtils.kt @@ -3,7 +3,7 @@ package org.utbot.python.utils object RequirementsUtils { val requirements: List = listOf( "mypy==1.0.0", - "utbot-executor==1.4.23", + "utbot-executor==1.4.26", "utbot-mypy-runner==0.2.8", ) From b27a2a25fb8bb23b033742612983111f2394238b Mon Sep 17 00:00:00 2001 From: Vyacheslav Tamarin Date: Wed, 3 May 2023 13:42:17 +0300 Subject: [PATCH 3/3] Update coverage socket algorithm --- .../samples/easy_samples/long_function_coverage.py | 11 +++++++++++ .../python/evaluation/PythonCodeSocketExecutor.kt | 1 + .../utbot/python/evaluation/PythonCoverageReceiver.kt | 6 +++--- .../utbot/python/evaluation/PythonWorkerManager.kt | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 utbot-python/samples/easy_samples/long_function_coverage.py diff --git a/utbot-python/samples/easy_samples/long_function_coverage.py b/utbot-python/samples/easy_samples/long_function_coverage.py new file mode 100644 index 0000000000..64cbccc0c0 --- /dev/null +++ b/utbot-python/samples/easy_samples/long_function_coverage.py @@ -0,0 +1,11 @@ +import time + + +def long_function(x: int): + x += 4 + x /= 2 + x += 100 + x *= 3 + x -= 15 + time.sleep(2000) + return x diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt index 5940c3bbc9..7d8bb6a454 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCodeSocketExecutor.kt @@ -92,6 +92,7 @@ class PythonCodeSocketExecutor( val (status, response) = UtExecutorThread.run(pythonWorker, executionTimeout) return when (status) { UtExecutorThread.Status.TIMEOUT -> { + PythonEvaluationTimeout() } diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt index 5756a66f3f..348f32eb35 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonCoverageReceiver.kt @@ -24,14 +24,14 @@ class PythonCoverageReceiver( val request = DatagramPacket(buf, buf.size) socket.receive(request) val (id, line) = request.data.decodeToString().take(request.length).split(":") - logger.info { "Got coverage: $id, $line" } + logger.debug { "Got coverage: $id, $line" } val lineNumber = line.toInt() coverageStorage.getOrPut(id) { mutableSetOf() } .add(lineNumber) } } catch (ex: SocketException) { - logger.error("Socket error: " + ex.message); + logger.error("Socket error: " + ex.message) } catch (ex: IOException) { - logger.error("IO error: " + ex.message); + logger.error("IO error: " + ex.message) } } } \ No newline at end of file diff --git a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt index 2bc8483de3..a7bd0e4c41 100644 --- a/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt +++ b/utbot-python/src/main/kotlin/org/utbot/python/evaluation/PythonWorkerManager.kt @@ -43,7 +43,7 @@ class PythonWorkerManager( coverageReceiver.address().first, coverageReceiver.address().second, "--logfile", logfile.absolutePath, - "--loglevel", "DEBUG", // "DEBUG", "INFO", "WARNING", "ERROR" + "--loglevel", "INFO", // "DEBUG", "INFO", "WARNING", "ERROR" )) timeout = max(until - processStartTime, 0) workerSocket = try { @@ -85,7 +85,7 @@ class PythonWorkerManager( reconnect() PythonEvaluationTimeout() } - if (evaluationResult is PythonEvaluationError) { + if (evaluationResult is PythonEvaluationError || evaluationResult is PythonEvaluationTimeout) { reconnect() } return evaluationResult