From e7283f24e22bbc40899afbce6757b470c4e44836 Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Mon, 20 Apr 2020 14:37:20 +0800
Subject: [PATCH 1/7] rebalance

---
 .github/workflows/plugins-test.0.yaml | 6 +++---
 .github/workflows/plugins-test.1.yaml | 1 +
 .github/workflows/plugins-test.2.yaml | 7 +------
 .github/workflows/plugins-test.3.yaml | 4 ++++
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/plugins-test.0.yaml b/.github/workflows/plugins-test.0.yaml
index e24f7ec44260..8b4b00e85847 100644
--- a/.github/workflows/plugins-test.0.yaml
+++ b/.github/workflows/plugins-test.0.yaml
@@ -48,9 +48,9 @@ jobs:
           - { name: 'elasticsearch-6.x-scenario', title: 'ElasticSearch 6.7.1-6.8.4 (7)' }
           - { name: 'elasticsearch-7.x-scenario', title: 'ElasticSearch 7.0.0-7.5.2 (14)' }
           - { name: 'feign-scenario', title: 'Feign 9.0.0-9.5.1 (8)' }
-          - { name: 'finagle-17.10.x-scenario', title: 'Finagle 17.10.0-20.1.0' }
-          - { name: 'finagle-6.25.x-scenario', title: 'Finagle 6.25.0-6.43.0' }
-          - { name: 'finagle-6.44.x-scenario', title: 'Finagle 6.44.0-7.1.0' }
+          - { name: 'finagle-17.10.x-scenario', title: 'Finagle 17.10.0-20.1.0(19)' }
+          - { name: 'finagle-6.25.x-scenario', title: 'Finagle 6.25.0-6.43.0 (18)' }
+          - { name: 'finagle-6.44.x-scenario', title: 'Finagle 6.44.0-7.1.0 (4)' }
           - { name: 'gateway-scenario', title: 'Spring-Cloud-Gateway 2.1.x (3)' }
           - { name: 'grpc-scenario', title: 'gRPC 1.6.0-1.25.0 (22)' }
           - { name: 'gson-scenario', title: 'Gson (7)' }
diff --git a/.github/workflows/plugins-test.1.yaml b/.github/workflows/plugins-test.1.yaml
index a9a46b795b3e..20e4eaa9ef82 100644
--- a/.github/workflows/plugins-test.1.yaml
+++ b/.github/workflows/plugins-test.1.yaml
@@ -47,6 +47,7 @@ jobs:
           - { name: 'lettuce-scenario', title: 'Lettuce 5.x (17)' }
           - { name: 'mongodb-3.x-scenario', title: 'Mongodb 3.4.0-3.11.1 (22)' }
           - { name: 'netty-socketio-scenario', title: 'Netty-SocketIO 1.x (4)' }
+          - { name: 'postgresql-above9.4.1207-scenario', title: 'PostgreSQL 9.4.1207+ (62)' }
     steps:
       - uses: actions/checkout@v2
       - name: checkout submodules
diff --git a/.github/workflows/plugins-test.2.yaml b/.github/workflows/plugins-test.2.yaml
index 2f11f79c2979..ce041df2c82f 100644
--- a/.github/workflows/plugins-test.2.yaml
+++ b/.github/workflows/plugins-test.2.yaml
@@ -34,8 +34,7 @@ jobs:
       matrix:
         case:
           - { name: 'okhttp-scenario', title: 'OKHttp 3.0.x-3.14.x (34)' }
-          - { name: 'play-scenario', title: 'Play! Framework' }
-          - { name: 'postgresql-above9.4.1207-scenario', title: 'PostgreSQL 9.4.1207+ (62)' }
+          - { name: 'play-scenario', title: 'Play! Framework (12)' }
           - { name: 'postgresql-scenario', title: 'PostgreSQL 9.2.x-9.4.x (36)' }
           - { name: 'pulsar-scenario', title: 'Pulsar 2.2.0-2.4.1 (7)' }
           - { name: 'rabbitmq-scenario', title: 'RabbitMQ (12)' }
@@ -49,10 +48,6 @@ jobs:
           - { name: 'sofarpc-scenario', title: 'SofaRPC 5.4.0-5.6.2 (23)' }
           - { name: 'solrj-7.x-scenario', title: 'SolrJ 7.x (12)' }
           - { name: 'spring-3.0.x-scenario', title: 'Spring 3.0.x (8)' }
-          - { name: 'spring-3.1.x-scenario', title: 'Spring 3.1.x-4.0.x (25)' }
-          - { name: 'spring-4.1.x-scenario', title: 'Spring 4.1.x-4.2.x (20)' }
-          - { name: 'spring-4.3.x-scenario', title: 'Spring 4.3.x-5.2.x (54)' }
-          - { name: 'spring-async-scenario', title: 'Spring Async 4.3.x-5.1.x (35)' }
           - { name: 'spring-cloud-feign-1.1.x-scenario', title: 'SpringCloud Feign 1.1.x (8)' }
           - { name: 'spring-cloud-feign-1.2.x-scenario', title: 'SpringCloud Feign 1.2.x-1.4.x (23)' }
           - { name: 'spring-cloud-feign-2.x-scenario', title: 'SpringCloud Feign 2.x (14)' }
diff --git a/.github/workflows/plugins-test.3.yaml b/.github/workflows/plugins-test.3.yaml
index 80af4c7a1d89..f330223de035 100644
--- a/.github/workflows/plugins-test.3.yaml
+++ b/.github/workflows/plugins-test.3.yaml
@@ -37,6 +37,10 @@ jobs:
           - { name: 'undertow-scenario', title: 'Undertow 1.3.0-2.0.27 (23)' }
           - { name: 'webflux-scenario', title: 'Spring-WebFlux 2.x (7)' }
           - { name: 'zookeeper-scenario', title: 'Zookeeper 3.4.x (14)' }
+          - { name: 'spring-3.1.x-scenario', title: 'Spring 3.1.x-4.0.x (25)' }
+          - { name: 'spring-4.1.x-scenario', title: 'Spring 4.1.x-4.2.x (20)' }
+          - { name: 'spring-4.3.x-scenario', title: 'Spring 4.3.x-5.2.x (54)' }
+          - { name: 'spring-async-scenario', title: 'Spring Async 4.3.x-5.1.x (35)' }
     steps:
       - uses: actions/checkout@v2
       - name: checkout submodules

From 8be04b4e469380f37dc3a06a6e2b8bb613264362 Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Mon, 20 Apr 2020 14:38:12 +0800
Subject: [PATCH 2/7] restore mysql testcases

---
 .../scenarios/mysql-scenario/support-version.list     | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/plugin/scenarios/mysql-scenario/support-version.list b/test/plugin/scenarios/mysql-scenario/support-version.list
index bf4ddf6ff3c7..4768268ed6e1 100644
--- a/test/plugin/scenarios/mysql-scenario/support-version.list
+++ b/test/plugin/scenarios/mysql-scenario/support-version.list
@@ -36,4 +36,13 @@
 5.1.26
 5.1.24
 5.1.22
-5.1.20
\ No newline at end of file
+5.1.20
+5.1.18
+5.1.16
+5.1.14
+5.1.12
+5.1.10
+5.1.8
+5.1.6
+5.1.4
+5.1.2
\ No newline at end of file

From 1d955ecb9f68b7175d2df0cdfd882b523da08e8b Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Mon, 20 Apr 2020 14:42:04 +0800
Subject: [PATCH 3/7] support debug mode

---
 .../containers/jvm-container/docker/run.sh    | 13 +++--
 .../tomcat-container/docker/catalina.sh       |  2 +-
 .../containers/tomcat-container/docker/run.sh | 13 +++--
 test/plugin/run.sh                            | 54 +++++--------------
 .../plugin/test/helper/ConfigurationImpl.java |  6 +++
 .../plugin/test/helper/IConfiguration.java    |  2 +
 .../resources/container-start-script.template |  3 ++
 .../src/main/resources/scenario.sh            | 10 +---
 8 files changed, 42 insertions(+), 61 deletions(-)

diff --git a/test/plugin/containers/jvm-container/docker/run.sh b/test/plugin/containers/jvm-container/docker/run.sh
index 121eb732a054..90efeb4a9e5f 100644
--- a/test/plugin/containers/jvm-container/docker/run.sh
+++ b/test/plugin/containers/jvm-container/docker/run.sh
@@ -16,13 +16,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 function exitOnError() {
     echo -e "\033[31m[ERROR] $1\033[0m">&2
     exitAndClean 1
 }
 
 function exitAndClean() {
+    [[ -z $DEBUG_MODE ]] && exit $1;
+
     [[ -f ${SCENARIO_HOME}/data/actualData.yaml ]] && rm -rf ${SCENARIO_HOME}/data/actualData.yaml
     [[ -d ${SCENARIO_HOME}/logs ]] && rm -rf ${SCENARIO_HOME}/logs
     exit $1
@@ -53,13 +54,15 @@ TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
 JACOCO_HOME=${JACOCO_HOME:-/jacoco}
 
+mkdi -p ${LOGS_HOME}
+
 unzip -q ${SCENARIO_HOME}/*.zip -d /var/run/
 if [[ ! -f /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} ]]; then
     exitOnError "The required startup script not exists!"
 fi
 
 echo "To start mock collector"
-${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>/dev/null &
+${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>${SCENARIO_HOME}/logs/collector.out &
 healthCheck http://localhost:12800/receiveData
 
 # start applications
@@ -71,10 +74,10 @@ export agent_opts="
     -Dskywalking.collector.discovery_check_interval=2
     -Dskywalking.collector.backend_service=localhost:19876
     -Dskywalking.agent.service_name=${SCENARIO_NAME}
-    -Dskywalking.logging.dir=/usr/local/skywalking/scenario/logs
+    -Dskywalking.logging.dir=${SCENARIO_HOME}/logs
     -Dskywalking.agent.authentication=test-token
     -Xms256m -Xmx256m ${agent_opts}"
-exec /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} 1>/dev/null &
+exec /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} 1>${SCENARIO_HOME}/logs/scenario.out &
 
 healthCheck ${SCENARIO_HEALTH_CHECK_URL}
 
@@ -91,7 +94,7 @@ java -jar \
     -Xmx256m -Xms256m \
     -DcaseName="${SCENARIO_NAME}-${SCENARIO_VERSION}" \
     -DtestCasePath=${SCENARIO_HOME}/data/ \
-    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>/dev/null
+    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>1>${SCENARIO_HOME}/logs/validator.out
 status=$?
 
 if [[ $status -eq 0 ]]; then
diff --git a/test/plugin/containers/tomcat-container/docker/catalina.sh b/test/plugin/containers/tomcat-container/docker/catalina.sh
index 5b44bdfa7e10..70c6ff5e9c2b 100644
--- a/test/plugin/containers/tomcat-container/docker/catalina.sh
+++ b/test/plugin/containers/tomcat-container/docker/catalina.sh
@@ -118,7 +118,7 @@ if [ -f "${AGENT_FILE_PATH}/skywalking-agent.jar" ]; then
     -Dskywalking.collector.backend_service=localhost:19876
     -Dskywalking.agent.service_name=${SCENARIO_NAME}
     -Dskywalking.agent.authentication=test-token
-    -Dskywalking.logging.dir=/usr/local/skywalking/scenario/logs
+    -Dskywalking.logging.dir=${SCENARIO_HOME}/logs
     -Xms256m -Xmx256m -XX:PermSize=64M -XX:MaxPermSize=64"
 fi
 
diff --git a/test/plugin/containers/tomcat-container/docker/run.sh b/test/plugin/containers/tomcat-container/docker/run.sh
index 02a8b972c4b2..5ef8db6cbc15 100644
--- a/test/plugin/containers/tomcat-container/docker/run.sh
+++ b/test/plugin/containers/tomcat-container/docker/run.sh
@@ -22,8 +22,10 @@ function exitOnError() {
 }
 
 function exitAndClean() {
+    [[ -z $DEBUG_MODE ]] && exit $1
+
     [[ -f ${SCENARIO_HOME}/data/actualData.yaml ]] && rm -rf ${SCENARIO_HOME}/data/actualData.yaml
-    [[ -d ${SCENARIO_HOME}/logs ]] && rm -rf ${SCENARIO_HOME}/logs
+    [[ -d ${LOGS_HOME} ]] && rm -rf 1>${LOGS_HOME}
     exit $1
 }
 
@@ -46,6 +48,9 @@ function healthCheck() {
 
 TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
+LOGS_HOME=${SCENARIO_HOME}/logs
+
+[[ ! -d ${LOGS_HOME} ]] && mkdir $LOGS_HOME
 
 # Speed up launch tomcat
 rm /usr/local/tomcat/webapps/* -rf # remove needn't app
@@ -56,11 +61,11 @@ cp ${SCENARIO_HOME}/*.war /usr/local/tomcat/webapps/
 
 # start mock collector
 echo "To start mock collector"
-${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>/dev/null &
+${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>${LOGS_HOME}/collector.out &
 healthCheck http://localhost:12800/receiveData
 
 echo "To start tomcat"
-/usr/local/tomcat/bin/catalina.sh start 1>/dev/null &
+/usr/local/tomcat/bin/catalina.sh start 1>${LOGS_HOME}/catalina.out &
 
 healthCheck ${SCENARIO_HEALTH_CHECK_URL}
 
@@ -77,7 +82,7 @@ java -jar \
     -Xmx256m -Xms256m \
     -DcaseName="${SCENARIO_NAME}-${SCENARIO_VERSION}" \
     -DtestCasePath=${SCENARIO_HOME}/data/ \
-    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>/dev/null
+    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>${LOGS_HOME}/validator.out
 status=$?
 
 if [[ $status -eq 0 ]]; then
diff --git a/test/plugin/run.sh b/test/plugin/run.sh
index d81ecfffc95d..9e0089b3861e 100755
--- a/test/plugin/run.sh
+++ b/test/plugin/run.sh
@@ -20,9 +20,9 @@ set -ex
 
 home="$(cd "$(dirname $0)"; pwd)"
 scenario_name=""
-parallel_run_size=1
 force_build="off"
 cleanup="off"
+debug_mode=
 
 mvnw=${home}/../../mvnw
 agent_home="${home}"/../../skywalking-agent
@@ -32,7 +32,6 @@ scenarios_home="${home}/scenarios"
 print_help() {
     echo  "Usage: run.sh [OPTION] SCENARIO_NAME"
     echo -e "\t-f, --force_build \t\t do force to build Plugin-Test tools and images"
-    echo -e "\t--parallel_run_size, \t\t parallel size of test cases. Default: 1"
     echo -e "\t--cleanup, \t\t\t remove the related images and directories"
 }
 
@@ -47,22 +46,13 @@ parse_commandline() {
             --cleanup)
                 cleanup="on"
                 ;;
-            --parallel_run_size)
-                test $# -lt 2 && exitWithMessage "Missing value for the optional argument '$_key'."
-                parallel_run_size="$2"
-                shift
-                ;;
-            --parallel_run_size=*)
-                parallel_run_size="${_key##--parallel_run_size=}"
+            --debug)
+                debug_mode="on";
                 ;;
             -h|--help)
                 print_help
                 exit 0
                 ;;
-            -h*)
-                print_help
-                exit 0
-                ;;
             *)
                 scenario_name=$1
                 ;;
@@ -78,10 +68,8 @@ exitWithMessage() {
 
 exitAndClean() {
     elapsed=$(( `date +%s` - $start_stamp ))
-    num_of_testcases="`ls -l ${task_state_house} |grep -c FINISH`"
     [[ $1 -eq 1 ]] && printSystemInfo
-    printf "Scenarios: %s, Testcases: %d, parallel_run_size: %d, Elapsed: %02d:%02d:%02d \n" \
-        ${scenario_name} "${num_of_testcases}" "${parallel_run_size}" \
+    printf "Scenarios: ${scenario_name}, Testcases: ${num_of_testcases}, Elapsed: %02d:%02d:%02d \n" \
         $(( ${elapsed}/3600 )) $(( ${elapsed}%3600/60 )) $(( ${elapsed}%60 ))
     exit $1
 }
@@ -90,17 +78,6 @@ printSystemInfo(){
   bash ${home}/script/systeminfo.sh
 }
 
-waitForAvailable() {
-    while [[ `ls -l ${task_state_house} |grep -c RUNNING` -ge ${parallel_run_size} ]]
-    do
-        sleep 2
-    done
-
-    if [[ `ls -l ${task_state_house} |grep -c FAILURE` -gt 0 ]]; then
-        exitAndClean 1
-    fi
-}
-
 do_cleanup() {
     images=$(docker images -q "skywalking/agent-test-*:${BUILD_NO:=local}")
     [[ -n "${images}" ]] && docker rmi -f ${images}
@@ -155,9 +132,7 @@ fi
 [[ "$force_build" == "on" ]] && ${mvnw} --batch-mode -f ${home}/pom.xml clean package -DskipTests -DBUILD_NO=${BUILD_NO:=local} docker:build
 
 workspace="${home}/workspace/${scenario_name}"
-task_state_house="${workspace}/.states"
 [[ -d ${workspace} ]] && rm -rf $workspace
-mkdir -p ${task_state_house}
 
 plugin_runner_helper="${home}/dist/plugin-runner-helper.jar"
 if [[ ! -f ${plugin_runner_helper} ]]; then
@@ -211,27 +186,22 @@ do
         -Doutput.dir=${case_work_base} \
         -Dagent.dir=${_agent_home} \
         -Djacoco.home=${jacoco_home} \
+        -Ddebug.mode=${debug_mode} \
         -Ddocker.image.version=${BUILD_NO:=local} \
         ${plugin_runner_helper} 1>${case_work_logs_dir}/helper.log
 
     [[ $? -ne 0 ]] && exitWithMessage "${testcase_name}, generate script failure!"
 
     echo "start container of testcase.name=${testcase_name}"
-    bash ${case_work_base}/scenario.sh ${task_state_house} 1>${case_work_logs_dir}/${testcase_name}.log
-    sleep 3
-    waitForAvailable
-    rm -rf ${case_work_base}
+    bash ${case_work_base}/scenario.sh $debug 1>${case_work_logs_dir}/${testcase_name}.log
+    status=$?
+    if [[ $status == 0 ]]; then
+        rm -rf ${case_work_base}
+    else
+        exitWithMessage "Testcase ${testcase_name} failed!"
+    fi
 done
 
 echo -e "\033[33m${scenario_name} has already sumbitted\033[0m"
 
-# wait to finish
-while [[ `ls -l ${task_state_house} |grep -c RUNNING` -gt 0 ]]; do
-    sleep 1
-done
-
-if [[ `ls -l ${task_state_house} |grep -c FAILURE` -gt 0 ]]; then
-    exitAndClean 1
-fi
-
 exitAndClean 0
diff --git a/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/ConfigurationImpl.java b/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/ConfigurationImpl.java
index 1ad38ad34104..82737a5b08b3 100644
--- a/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/ConfigurationImpl.java
+++ b/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/ConfigurationImpl.java
@@ -159,6 +159,11 @@ public String jacocoHome() {
         return System.getProperty("jacoco.home");
     }
 
+    @Override
+    public String debugMode() {
+        return System.getProperty("debug.mode");
+    }
+
     @Override
     public Map<String, Object> toMap() {
         final Map<String, Object> root = new HashMap<>();
@@ -175,6 +180,7 @@ public Map<String, Object> toMap() {
         root.put("docker_image_version", dockerImageVersion());
         root.put("docker_container_name", dockerContainerName());
         root.put("jacoco_home", jacocoHome());
+        root.put("debug_mode", debugMode());
 
         root.put("expose", caseConfiguration().getExpose());
         root.put("hostname", caseConfiguration().getHostname());
diff --git a/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java b/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java
index 70d278fbbf5e..a8ae29f2939e 100644
--- a/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java
+++ b/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java
@@ -56,5 +56,7 @@ public interface IConfiguration {
 
     String jacocoHome();
 
+    String debugFlag();
+
     Map<String, Object> toMap();
 }
diff --git a/test/plugin/runner-helper/src/main/resources/container-start-script.template b/test/plugin/runner-helper/src/main/resources/container-start-script.template
index 5b410526c156..1e75240f2681 100644
--- a/test/plugin/runner-helper/src/main/resources/container-start-script.template
+++ b/test/plugin/runner-helper/src/main/resources/container-start-script.template
@@ -29,6 +29,9 @@ docker run \
         <#if catalina_opts??>
         --env CATALINA_OPTS=${catalina_opts} \
         </#if>
+        <#if debug_mode??>
+        --eve DEBUG_MODE=${debug_mode} \
+        </#if>
         -v ${agent_home}:/usr/local/skywalking/scenario/agent \
         -v ${scenario_home}:/usr/local/skywalking/scenario \
         -v ${jacoco_home}:/jacoco \
diff --git a/test/plugin/runner-helper/src/main/resources/scenario.sh b/test/plugin/runner-helper/src/main/resources/scenario.sh
index 3c0c6fe4d117..52c17fe37fd1 100644
--- a/test/plugin/runner-helper/src/main/resources/scenario.sh
+++ b/test/plugin/runner-helper/src/main/resources/scenario.sh
@@ -20,18 +20,10 @@ PRG="$0"
 PRGDIR=`dirname "$PRG"`
 [ -z "$SCENARIO_HOME" ] && SCENARIO_HOME=`cd "$PRGDIR" >/dev/null; pwd`
 
-state_house=$1
 testcase_name=${scenario_name}-${scenario_version}
 
 status=1
-<#noparse>touch ${state_house}/${testcase_name}.RUNNING</#noparse>
 
 ${running_script}
 
-<#noparse>
-if [[ ${status} -eq 0 ]]; then
-  mv ${state_house}/${testcase_name}.RUNNING ${state_house}/${testcase_name}.FINISH
-else
-  mv ${state_house}/${testcase_name}.RUNNING ${state_house}/${testcase_name}.FAILURE
-fi
-</#noparse>
+exit $status
\ No newline at end of file

From 2207dfb5a8f0ff688efb9ba245244221900b2dd1 Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Mon, 20 Apr 2020 15:07:05 +0800
Subject: [PATCH 4/7] support debug mode

---
 .../apache/skywalking/plugin/test/helper/IConfiguration.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java b/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java
index a8ae29f2939e..12c931d56ce6 100644
--- a/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java
+++ b/test/plugin/runner-helper/src/main/java/org/apache/skywalking/plugin/test/helper/IConfiguration.java
@@ -56,7 +56,7 @@ public interface IConfiguration {
 
     String jacocoHome();
 
-    String debugFlag();
+    String debugMode();
 
     Map<String, Object> toMap();
 }

From eac7b8f04cc4aa79d675087a678add8cf12ef90c Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Mon, 20 Apr 2020 15:21:52 +0800
Subject: [PATCH 5/7] correct spell

---
 .../src/main/resources/container-start-script.template          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/plugin/runner-helper/src/main/resources/container-start-script.template b/test/plugin/runner-helper/src/main/resources/container-start-script.template
index 1e75240f2681..8af65a1c4e52 100644
--- a/test/plugin/runner-helper/src/main/resources/container-start-script.template
+++ b/test/plugin/runner-helper/src/main/resources/container-start-script.template
@@ -30,7 +30,7 @@ docker run \
         --env CATALINA_OPTS=${catalina_opts} \
         </#if>
         <#if debug_mode??>
-        --eve DEBUG_MODE=${debug_mode} \
+        --env DEBUG_MODE=${debug_mode} \
         </#if>
         -v ${agent_home}:/usr/local/skywalking/scenario/agent \
         -v ${scenario_home}:/usr/local/skywalking/scenario \

From 283a9e5d6e6f467df4f84d43b8f1b6636737315d Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Wed, 22 Apr 2020 15:59:48 +0800
Subject: [PATCH 6/7] fix

---
 .../plugin/containers/jvm-container/docker/run.sh | 15 +++++++++------
 .../tomcat-container/docker/catalina.sh           |  3 +--
 .../containers/tomcat-container/docker/run.sh     |  8 +++++---
 test/plugin/run.sh                                |  7 +++++--
 .../runner-helper/src/main/resources/scenario.sh  |  2 ++
 5 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/test/plugin/containers/jvm-container/docker/run.sh b/test/plugin/containers/jvm-container/docker/run.sh
index 90efeb4a9e5f..7249244c230b 100644
--- a/test/plugin/containers/jvm-container/docker/run.sh
+++ b/test/plugin/containers/jvm-container/docker/run.sh
@@ -16,16 +16,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+[[ -n $DEBUG_MODE ]] && set -ex
+
 function exitOnError() {
     echo -e "\033[31m[ERROR] $1\033[0m">&2
     exitAndClean 1
 }
 
 function exitAndClean() {
-    [[ -z $DEBUG_MODE ]] && exit $1;
+    [[ -n $DEBUG_MODE ]] && exit $1;
 
     [[ -f ${SCENARIO_HOME}/data/actualData.yaml ]] && rm -rf ${SCENARIO_HOME}/data/actualData.yaml
-    [[ -d ${SCENARIO_HOME}/logs ]] && rm -rf ${SCENARIO_HOME}/logs
+    [[ -d ${LOGS_HOME} ]] && rm -rf ${LOGS_HOME}
     exit $1
 }
 
@@ -53,6 +55,7 @@ fi
 TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
 JACOCO_HOME=${JACOCO_HOME:-/jacoco}
+LOGS_HOME=${SCENARIO_NAME}/logs
 
 mkdi -p ${LOGS_HOME}
 
@@ -62,7 +65,7 @@ if [[ ! -f /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} ]]; then
 fi
 
 echo "To start mock collector"
-${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>${SCENARIO_HOME}/logs/collector.out &
+${TOOLS_HOME}/skywalking-mock-collector/bin/collector-startup.sh 1>${LOGS_HOME}/collector.out &
 healthCheck http://localhost:12800/receiveData
 
 # start applications
@@ -74,10 +77,10 @@ export agent_opts="
     -Dskywalking.collector.discovery_check_interval=2
     -Dskywalking.collector.backend_service=localhost:19876
     -Dskywalking.agent.service_name=${SCENARIO_NAME}
-    -Dskywalking.logging.dir=${SCENARIO_HOME}/logs
+    -Dskywalking.logging.dir=${LOGS_HOME}
     -Dskywalking.agent.authentication=test-token
     -Xms256m -Xmx256m ${agent_opts}"
-exec /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} 1>${SCENARIO_HOME}/logs/scenario.out &
+exec /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} 1>${LOGS_HOME}/scenario.out &
 
 healthCheck ${SCENARIO_HEALTH_CHECK_URL}
 
@@ -94,7 +97,7 @@ java -jar \
     -Xmx256m -Xms256m \
     -DcaseName="${SCENARIO_NAME}-${SCENARIO_VERSION}" \
     -DtestCasePath=${SCENARIO_HOME}/data/ \
-    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>1>${SCENARIO_HOME}/logs/validator.out
+    ${TOOLS_HOME}/skywalking-validator-tools.jar 1>${LOGS_HOME}/validatolr.out
 status=$?
 
 if [[ $status -eq 0 ]]; then
diff --git a/test/plugin/containers/tomcat-container/docker/catalina.sh b/test/plugin/containers/tomcat-container/docker/catalina.sh
index 70c6ff5e9c2b..1d3e784685bd 100644
--- a/test/plugin/containers/tomcat-container/docker/catalina.sh
+++ b/test/plugin/containers/tomcat-container/docker/catalina.sh
@@ -107,7 +107,6 @@
 JACOCO_HOME=${JACOCO_HOME:-/jacoco}
 
 export AGENT_FILE_PATH=/usr/local/skywalking/scenario/agent
-echo "${AGENT_FILE_PATH}/skywalking-agent.jar"
 if [ -f "${AGENT_FILE_PATH}/skywalking-agent.jar" ]; then
     CATALINA_OPTS="$CATALINA_OPTS
     -javaagent:${JACOCO_HOME}/jacocoagent.jar=classdumpdir=${JACOCO_HOME}/classes/${SCENARIO_NAME}${SCENARIO_VERSION},destfile=${JACOCO_HOME}/${SCENARIO_NAME}${SCENARIO_VERSION}.exec,includes=org.apache.skywalking.*,excludes=org.apache.skywalking.apm.dependencies.*:org.apache.skywalking.apm.testcase.*
@@ -118,7 +117,7 @@ if [ -f "${AGENT_FILE_PATH}/skywalking-agent.jar" ]; then
     -Dskywalking.collector.backend_service=localhost:19876
     -Dskywalking.agent.service_name=${SCENARIO_NAME}
     -Dskywalking.agent.authentication=test-token
-    -Dskywalking.logging.dir=${SCENARIO_HOME}/logs
+    -Dskywalking.logging.dir=${LOGS_HOME}
     -Xms256m -Xmx256m -XX:PermSize=64M -XX:MaxPermSize=64"
 fi
 
diff --git a/test/plugin/containers/tomcat-container/docker/run.sh b/test/plugin/containers/tomcat-container/docker/run.sh
index 5ef8db6cbc15..d8056eb8916f 100644
--- a/test/plugin/containers/tomcat-container/docker/run.sh
+++ b/test/plugin/containers/tomcat-container/docker/run.sh
@@ -16,16 +16,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+[[ -n $DEBUG_MODE ]] && set -ex
+
 function exitOnError() {
     echo -e "\033[31m[ERROR] $1\033[0m">&2
     exitAndClean 1
 }
 
 function exitAndClean() {
-    [[ -z $DEBUG_MODE ]] && exit $1
+    [[ -n $DEBUG_MODE ]] && exit $1
 
     [[ -f ${SCENARIO_HOME}/data/actualData.yaml ]] && rm -rf ${SCENARIO_HOME}/data/actualData.yaml
-    [[ -d ${LOGS_HOME} ]] && rm -rf 1>${LOGS_HOME}
+    [[ -d ${LOGS_HOME} ]] && rm -rf ${LOGS_HOME}
     exit $1
 }
 
@@ -48,7 +50,7 @@ function healthCheck() {
 
 TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
-LOGS_HOME=${SCENARIO_HOME}/logs
+export LOGS_HOME=${SCENARIO_HOME}/logs # share to catalina.sh
 
 [[ ! -d ${LOGS_HOME} ]] && mkdir $LOGS_HOME
 
diff --git a/test/plugin/run.sh b/test/plugin/run.sh
index 9e0089b3861e..a76822e7eb34 100755
--- a/test/plugin/run.sh
+++ b/test/plugin/run.sh
@@ -28,11 +28,13 @@ mvnw=${home}/../../mvnw
 agent_home="${home}"/../../skywalking-agent
 jacoco_home="${home}"/../jacoco
 scenarios_home="${home}/scenarios"
+num_of_testcases=0
 
 print_help() {
     echo  "Usage: run.sh [OPTION] SCENARIO_NAME"
     echo -e "\t-f, --force_build \t\t do force to build Plugin-Test tools and images"
     echo -e "\t--cleanup, \t\t\t remove the related images and directories"
+    echo -e "\t--debug, \t\t\t to save the log files and actualData.yaml"
 }
 
 parse_commandline() {
@@ -193,13 +195,14 @@ do
     [[ $? -ne 0 ]] && exitWithMessage "${testcase_name}, generate script failure!"
 
     echo "start container of testcase.name=${testcase_name}"
-    bash ${case_work_base}/scenario.sh $debug 1>${case_work_logs_dir}/${testcase_name}.log
+    bash ${case_work_base}/scenario.sh $debug_mode 1>${case_work_logs_dir}/${testcase_name}.log
     status=$?
     if [[ $status == 0 ]]; then
-        rm -rf ${case_work_base}
+        [[ -z $debug_mode ]] && rm -rf ${case_work_base}
     else
         exitWithMessage "Testcase ${testcase_name} failed!"
     fi
+    num_of_testcases=$(($num_of_testcases+1))
 done
 
 echo -e "\033[33m${scenario_name} has already sumbitted\033[0m"
diff --git a/test/plugin/runner-helper/src/main/resources/scenario.sh b/test/plugin/runner-helper/src/main/resources/scenario.sh
index 52c17fe37fd1..347980a6a731 100644
--- a/test/plugin/runner-helper/src/main/resources/scenario.sh
+++ b/test/plugin/runner-helper/src/main/resources/scenario.sh
@@ -16,6 +16,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+[[ -n $1 ]] && set -ex
+
 PRG="$0"
 PRGDIR=`dirname "$PRG"`
 [ -z "$SCENARIO_HOME" ] && SCENARIO_HOME=`cd "$PRGDIR" >/dev/null; pwd`

From 2ee41c3e28ce06519022e34da2a7779c7fca2370 Mon Sep 17 00:00:00 2001
From: daming <zteny@foxmail.com>
Date: Wed, 22 Apr 2020 16:18:26 +0800
Subject: [PATCH 7/7] fix type

---
 test/plugin/containers/jvm-container/docker/run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/plugin/containers/jvm-container/docker/run.sh b/test/plugin/containers/jvm-container/docker/run.sh
index 7249244c230b..7d34818f7ea4 100644
--- a/test/plugin/containers/jvm-container/docker/run.sh
+++ b/test/plugin/containers/jvm-container/docker/run.sh
@@ -55,9 +55,9 @@ fi
 TOOLS_HOME=/usr/local/skywalking/tools
 SCENARIO_HOME=/usr/local/skywalking/scenario
 JACOCO_HOME=${JACOCO_HOME:-/jacoco}
-LOGS_HOME=${SCENARIO_NAME}/logs
+LOGS_HOME=${SCENARIO_HOME}/logs
 
-mkdi -p ${LOGS_HOME}
+[[ ! -d $LOGS_HOME ]] && mkdir -p ${LOGS_HOME}
 
 unzip -q ${SCENARIO_HOME}/*.zip -d /var/run/
 if [[ ! -f /var/run/${SCENARIO_NAME}/${SCENARIO_START_SCRIPT} ]]; then