From 6ce23e251392feec6e8c9d57f3d70a29cc8e0c50 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 10 Nov 2022 07:26:48 +0000 Subject: [PATCH 1/7] Fix release_task cpp_run.sh Author: felixhjh <852142024@qq.com> Date: Thu Nov 10 07:26:48 2022 +0000 --- tests/release_task/cpp_run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 21f1df8bfb5..a50f17262eb 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -54,8 +54,8 @@ do backend=${RUN_CASE[i]} echo "Cpp Backend:" $backend if [ "$backend" != "trt" ];then - ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_cpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD + ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_$backend\_cpu_result.txt + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD fi if [ "$DEVICE" = "gpu" ];then @@ -63,8 +63,8 @@ do ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt --conf_threshold $CONF_THRESHOLD else - ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_gpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD + ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_$backend\_gpu_result.txt + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_$backend\_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD fi fi done From 85db044424ef6182495ee0d30060fa28ed5549bd Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 10 Nov 2022 08:02:41 +0000 Subject: [PATCH 2/7] Add option for infer_ppyoloe.cc author: felixhjh <852142024@qq.com> --- tests/release_task/infer_ppyoloe.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/release_task/infer_ppyoloe.cc b/tests/release_task/infer_ppyoloe.cc index 70ab5e71f63..e70453367a6 100644 --- a/tests/release_task/infer_ppyoloe.cc +++ b/tests/release_task/infer_ppyoloe.cc @@ -54,7 +54,7 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { return; } auto model = fastdeploy::vision::detection::PPYOLOE(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; From bd5e567715920f87f3fb63de0f684913cc5a96ce Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 10 Nov 2022 10:20:41 +0000 Subject: [PATCH 3/7] Fix bug in release task scripts --- tests/release_task/compare_with_gt.py | 32 +++++++++++++++++---------- tests/release_task/cpp_run.sh | 7 +++--- tests/release_task/py_run.sh | 5 ++++- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py index 564e74b579e..ba030999c8e 100644 --- a/tests/release_task/compare_with_gt.py +++ b/tests/release_task/compare_with_gt.py @@ -40,7 +40,7 @@ def all_sort(x): y = np.split(x1, len(x1)) z = list(reversed(y)) index = np.lexsort(z) - return x[index] + return np.squeeze(x[index]) def parse_arguments(): @@ -91,8 +91,8 @@ def write2file(error_file): with open(error_file, "a+") as f: from platform import python_version py_version = python_version() - f.write(args.platform + " " + py_version + " " + args.result_path.split( - ".")[0] + "\n") + f.write(args.platform + " " + py_version + " " + + args.result_path.split(".")[0] + "\n") def save_numpy_result(file_path, error_msg): @@ -108,29 +108,37 @@ def check_result(gt_result, infer_result, args): score_diff = diff[:, -2] boxes_diff = diff[:, :-2] boxes_diff_ratio = boxes_diff / (infer_result[:, :-2] + 1e-6) + label_diff_threshold = diff_score_threshold[platform]["label_diff"] + score_diff_threshold = diff_score_threshold[platform]["score_diff"] + boxes_diff_threshold = diff_score_threshold[platform]["boxes_diff"] + boxes_diff_ratio_threshold = diff_score_threshold[platform][ + "boxes_diff_ratio"] is_diff = False backend = args.result_path.split(".")[0] - if (label_diff > diff_score_threshold[platform]["label_diff"]).any(): + if (label_diff > label_diff_threshold).any(): print(args.platform, args.device, "label diff ", label_diff) is_diff = True label_diff_bool_file = args.platform + "_" + backend + "_" + "label_diff_bool.txt" - save_numpy_result(label_diff_bool_file, label_diff > 0) - if (score_diff > diff_score_threshold[platform]["score_diff"]).any(): + save_numpy_result(label_diff_bool_file, + label_diff > label_diff_threshold) + if (score_diff > score_diff_threshold).any(): print(args.platform, args.device, "score diff ", score_diff) is_diff = True score_diff_bool_file = args.platform + "_" + backend + "_" + "score_diff_bool.txt" - save_numpy_result(score_diff_bool_file, score_diff > 1e-4) - if (boxes_diff_ratio > diff_score_threshold[platform]["boxes_diff_ratio"] - ).any() and ( - boxes_diff > diff_score_threshold[platform]["boxes_diff"]).any(): + save_numpy_result(score_diff_bool_file, + score_diff > score_diff_threshold) + if (boxes_diff_ratio > boxes_diff_ratio_threshold).any() and ( + boxes_diff > boxes_diff_threshold).any(): print(args.platform, args.device, "boxes diff ", boxes_diff_ratio) is_diff = True boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt" boxes_diff_ratio_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio.txt" boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool" - save_numpy_result(boxes_diff_bool_file, boxes_diff > 1e-3) + save_numpy_result(boxes_diff_bool_file, + boxes_diff > boxes_diff_threshold) save_numpy_result(boxes_diff_ratio_file, boxes_diff_ratio) - save_numpy_result(boxes_diff_ratio_bool_file, boxes_diff_ratio > 1e-4) + save_numpy_result(boxes_diff_ratio_bool_file, + boxes_diff_ratio > boxes_diff_ratio_threshold) if is_diff: write2file("result.txt") else: diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index a50f17262eb..e2615194f49 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -48,7 +48,7 @@ tar xvf $CPP_FASTDEPLOY_PACKAGE.tgz mkdir build && cd build cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../$CPP_FASTDEPLOY_PACKAGE -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER make -j - +ret=0 for((i=0;i Date: Mon, 14 Nov 2022 07:52:42 +0000 Subject: [PATCH 4/7] Add error check function --- tests/release_task/py_run.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 2a4f73f16cf..7c7d9ba5659 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -38,10 +38,20 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) fi +ret=0 +check_ret(){ + if [ $? -ne 0 ];then + echo "ret before" + echo $ret + ret=-1 + echo "ret after" + echo $ret + fi +} + case_number=${#RUN_CASE[@]} py_version=$(python -V | awk '{print $2}') echo "py_version:" $py_version -ret=0 for((j=0;j> py_$backend\_cpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret fi if [ "$DEVICE" = "gpu" ];then if [ "$backend" = "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt + check_ret else python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_$backend\_gpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_gpu_result.txt --platform $PLATFORM --device gpu + check_ret fi fi - if [ $? -ne 0 ];then - ret=-1 - fi done - res_file="result.txt" if [ ! -f $res_file ];then if [ $ret -ne 0 ];then From 87634eb8d25d1d22ea7a285afc120b3d18c12ec3 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Mon, 14 Nov 2022 08:00:03 +0000 Subject: [PATCH 5/7] Format code --- tests/release_task/py_run.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 7c7d9ba5659..6001bfbbbcb 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -41,11 +41,7 @@ fi ret=0 check_ret(){ if [ $? -ne 0 ];then - echo "ret before" - echo $ret ret=-1 - echo "ret after" - echo $ret fi } From dc480ea917782e8832f312ca8ccc27529fa86763 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 17 Nov 2022 03:09:27 +0000 Subject: [PATCH 6/7] Add new download dir for release version --- tests/release_task/cpp_run.bat | 8 +++++++- tests/release_task/cpp_run.sh | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/release_task/cpp_run.bat b/tests/release_task/cpp_run.bat index 2c8a8aed628..4d92ea51b73 100644 --- a/tests/release_task/cpp_run.bat +++ b/tests/release_task/cpp_run.bat @@ -13,6 +13,12 @@ echo "PLATFORM: " %PLATFORM% echo "DEVICE: " %DEVICE% echo "VERSION: " %VERSION% +if "%VERSION%" == "0.0.0" ( + set DOWNLOAD_DIR=dev +) else ( + set DOWNLOAD_DIR=rel_tmp +) + if "%DEVICE%" == "gpu" ( set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%DEVICE%-%VERSION% set RUN_CASES=ort paddle trt @@ -33,7 +39,7 @@ set MODEL_PATH=%CURRENT_DIR%\ppyoloe_crn_l_300e_coco set GROUND_TRUTH_PATH=%CURRENT_DIR%\release_task_groud_truth_result.txt set COMPARE_SHELL=%CURRENT_DIR%\compare_with_gt.py -python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/dev/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" +python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/%DOWNLOAD_DIR%/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" mkdir build && cd build cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2" diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index e2615194f49..9a160b6e2bc 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -4,6 +4,11 @@ CURRENT_DIR=$(cd $(dirname $0); pwd) PLATFORM=$1 DEVICE=$2 VERSION=$3 +if [ "$VERSION" = "0.0.0" ];then + DOWNLOAD_DIR=dev +else + DOWNLOAD_DIR=rel_tmp +fi if [ "$DEVICE" = "gpu" ];then CPP_FASTDEPLOY_PACKAGE=fastdeploy-$PLATFORM-$DEVICE-$VERSION else @@ -42,7 +47,7 @@ fi case_number=${#RUN_CASE[@]} -wget -q https://fastdeploy.bj.bcebos.com/dev/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz +wget -q https://fastdeploy.bj.bcebos.com/$DOWNLOAD_DIR/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz tar xvf $CPP_FASTDEPLOY_PACKAGE.tgz mkdir build && cd build From 5c1e1f097e0f70105e1a96a8b2036af2bf54c737 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Wed, 23 Nov 2022 21:48:37 +0800 Subject: [PATCH 7/7] Fix precision diff for osx-arm64 --- tests/release_task/compare_with_gt.py | 16 ++++++++-------- tests/release_task/cpp_run.sh | 2 +- tests/release_task/py_run.sh | 11 ++++++++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py index ba030999c8e..57258f59b33 100644 --- a/tests/release_task/compare_with_gt.py +++ b/tests/release_task/compare_with_gt.py @@ -3,31 +3,31 @@ diff_score_threshold = { "linux-x64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 1e-4, "boxes_diff": 1e-3 }, "linux-aarch64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 1e-4, "boxes_diff": 1e-3 }, "osx-x86_64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 2e-4, "boxes_diff": 1e-3 }, "osx-arm64": { - "label_diff": 1e-02, - "score_diff": 1e-4, - "boxes_diff_ratio": 2e-4, + "label_diff": 0, + "score_diff": 1e-3, + "boxes_diff_ratio": 5e-4, "boxes_diff": 1e-3 }, "win-x64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 5e-4, "boxes_diff_ratio": 1e-3, "boxes_diff": 1e-3 @@ -133,7 +133,7 @@ def check_result(gt_result, infer_result, args): is_diff = True boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt" boxes_diff_ratio_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio.txt" - boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool" + boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool.txt" save_numpy_result(boxes_diff_bool_file, boxes_diff > boxes_diff_threshold) save_numpy_result(boxes_diff_ratio_file, boxes_diff_ratio) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 9a160b6e2bc..954a2a38e57 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -19,7 +19,7 @@ LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') #LINUX_AARCH_CPU_CASE=('ort' 'openvino') LINUX_AARCH_CPU_CASE=('ort') -MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') +MACOS_INTEL_CPU_CASE=('ort' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz wget -q https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 6001bfbbbcb..64e876747e9 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -14,7 +14,7 @@ LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') #LINUX_AARCH_CPU_CASE=('ort' 'openvino') LINUX_AARCH_CPU_CASE=('ort') -MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') +MACOS_INTEL_CPU_CASE=('ort' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz wget -q https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg @@ -54,8 +54,13 @@ do echo "Python Backend:" $backend if [ "$backend" != "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_$backend\_cpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu - check_ret + if [ "$PLATFORM" = "osx-arm64"] + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold 0.5 + check_ret + else + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret + fi fi if [ "$DEVICE" = "gpu" ];then