From 6ce23e251392feec6e8c9d57f3d70a29cc8e0c50 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 10 Nov 2022 07:26:48 +0000 Subject: [PATCH 001/136] Fix release_task cpp_run.sh Author: felixhjh <852142024@qq.com> Date: Thu Nov 10 07:26:48 2022 +0000 --- tests/release_task/cpp_run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 21f1df8bfb5..a50f17262eb 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -54,8 +54,8 @@ do backend=${RUN_CASE[i]} echo "Cpp Backend:" $backend if [ "$backend" != "trt" ];then - ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_cpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD + ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_$backend\_cpu_result.txt + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD fi if [ "$DEVICE" = "gpu" ];then @@ -63,8 +63,8 @@ do ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt --conf_threshold $CONF_THRESHOLD else - ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_gpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD + ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_$backend\_gpu_result.txt + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_$backend\_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD fi fi done From 85db044424ef6182495ee0d30060fa28ed5549bd Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 10 Nov 2022 08:02:41 +0000 Subject: [PATCH 002/136] Add option for infer_ppyoloe.cc author: felixhjh <852142024@qq.com> --- tests/release_task/infer_ppyoloe.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/release_task/infer_ppyoloe.cc b/tests/release_task/infer_ppyoloe.cc index 70ab5e71f63..e70453367a6 100644 --- a/tests/release_task/infer_ppyoloe.cc +++ b/tests/release_task/infer_ppyoloe.cc @@ -54,7 +54,7 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { return; } auto model = fastdeploy::vision::detection::PPYOLOE(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; From bd5e567715920f87f3fb63de0f684913cc5a96ce Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 10 Nov 2022 10:20:41 +0000 Subject: [PATCH 003/136] Fix bug in release task scripts --- tests/release_task/compare_with_gt.py | 32 +++++++++++++++++---------- tests/release_task/cpp_run.sh | 7 +++--- tests/release_task/py_run.sh | 5 ++++- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py index 564e74b579e..ba030999c8e 100644 --- a/tests/release_task/compare_with_gt.py +++ b/tests/release_task/compare_with_gt.py @@ -40,7 +40,7 @@ def all_sort(x): y = np.split(x1, len(x1)) z = list(reversed(y)) index = np.lexsort(z) - return x[index] + return np.squeeze(x[index]) def parse_arguments(): @@ -91,8 +91,8 @@ def write2file(error_file): with open(error_file, "a+") as f: from platform import python_version py_version = python_version() - f.write(args.platform + " " + py_version + " " + args.result_path.split( - ".")[0] + "\n") + f.write(args.platform + " " + py_version + " " + + args.result_path.split(".")[0] + "\n") def save_numpy_result(file_path, error_msg): @@ -108,29 +108,37 @@ def check_result(gt_result, infer_result, args): score_diff = diff[:, -2] boxes_diff = diff[:, :-2] boxes_diff_ratio = boxes_diff / (infer_result[:, :-2] + 1e-6) + label_diff_threshold = diff_score_threshold[platform]["label_diff"] + score_diff_threshold = diff_score_threshold[platform]["score_diff"] + boxes_diff_threshold = diff_score_threshold[platform]["boxes_diff"] + boxes_diff_ratio_threshold = diff_score_threshold[platform][ + "boxes_diff_ratio"] is_diff = False backend = args.result_path.split(".")[0] - if (label_diff > diff_score_threshold[platform]["label_diff"]).any(): + if (label_diff > label_diff_threshold).any(): print(args.platform, args.device, "label diff ", label_diff) is_diff = True label_diff_bool_file = args.platform + "_" + backend + "_" + "label_diff_bool.txt" - save_numpy_result(label_diff_bool_file, label_diff > 0) - if (score_diff > diff_score_threshold[platform]["score_diff"]).any(): + save_numpy_result(label_diff_bool_file, + label_diff > label_diff_threshold) + if (score_diff > score_diff_threshold).any(): print(args.platform, args.device, "score diff ", score_diff) is_diff = True score_diff_bool_file = args.platform + "_" + backend + "_" + "score_diff_bool.txt" - save_numpy_result(score_diff_bool_file, score_diff > 1e-4) - if (boxes_diff_ratio > diff_score_threshold[platform]["boxes_diff_ratio"] - ).any() and ( - boxes_diff > diff_score_threshold[platform]["boxes_diff"]).any(): + save_numpy_result(score_diff_bool_file, + score_diff > score_diff_threshold) + if (boxes_diff_ratio > boxes_diff_ratio_threshold).any() and ( + boxes_diff > boxes_diff_threshold).any(): print(args.platform, args.device, "boxes diff ", boxes_diff_ratio) is_diff = True boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt" boxes_diff_ratio_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio.txt" boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool" - save_numpy_result(boxes_diff_bool_file, boxes_diff > 1e-3) + save_numpy_result(boxes_diff_bool_file, + boxes_diff > boxes_diff_threshold) save_numpy_result(boxes_diff_ratio_file, boxes_diff_ratio) - save_numpy_result(boxes_diff_ratio_bool_file, boxes_diff_ratio > 1e-4) + save_numpy_result(boxes_diff_ratio_bool_file, + boxes_diff_ratio > boxes_diff_ratio_threshold) if is_diff: write2file("result.txt") else: diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index a50f17262eb..e2615194f49 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -48,7 +48,7 @@ tar xvf $CPP_FASTDEPLOY_PACKAGE.tgz mkdir build && cd build cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../$CPP_FASTDEPLOY_PACKAGE -DCMAKE_CXX_COMPILER=$CMAKE_CXX_COMPILER make -j - +ret=0 for((i=0;i Date: Mon, 14 Nov 2022 07:52:42 +0000 Subject: [PATCH 004/136] Add error check function --- tests/release_task/py_run.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 2a4f73f16cf..7c7d9ba5659 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -38,10 +38,20 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) fi +ret=0 +check_ret(){ + if [ $? -ne 0 ];then + echo "ret before" + echo $ret + ret=-1 + echo "ret after" + echo $ret + fi +} + case_number=${#RUN_CASE[@]} py_version=$(python -V | awk '{print $2}') echo "py_version:" $py_version -ret=0 for((j=0;j> py_$backend\_cpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret fi if [ "$DEVICE" = "gpu" ];then if [ "$backend" = "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt + check_ret else python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_$backend\_gpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_gpu_result.txt --platform $PLATFORM --device gpu + check_ret fi fi - if [ $? -ne 0 ];then - ret=-1 - fi done - res_file="result.txt" if [ ! -f $res_file ];then if [ $ret -ne 0 ];then From 87634eb8d25d1d22ea7a285afc120b3d18c12ec3 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Mon, 14 Nov 2022 08:00:03 +0000 Subject: [PATCH 005/136] Format code --- tests/release_task/py_run.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 7c7d9ba5659..6001bfbbbcb 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -41,11 +41,7 @@ fi ret=0 check_ret(){ if [ $? -ne 0 ];then - echo "ret before" - echo $ret ret=-1 - echo "ret after" - echo $ret fi } From dc480ea917782e8832f312ca8ccc27529fa86763 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 17 Nov 2022 03:09:27 +0000 Subject: [PATCH 006/136] Add new download dir for release version --- tests/release_task/cpp_run.bat | 8 +++++++- tests/release_task/cpp_run.sh | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/release_task/cpp_run.bat b/tests/release_task/cpp_run.bat index 2c8a8aed628..4d92ea51b73 100644 --- a/tests/release_task/cpp_run.bat +++ b/tests/release_task/cpp_run.bat @@ -13,6 +13,12 @@ echo "PLATFORM: " %PLATFORM% echo "DEVICE: " %DEVICE% echo "VERSION: " %VERSION% +if "%VERSION%" == "0.0.0" ( + set DOWNLOAD_DIR=dev +) else ( + set DOWNLOAD_DIR=rel_tmp +) + if "%DEVICE%" == "gpu" ( set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%DEVICE%-%VERSION% set RUN_CASES=ort paddle trt @@ -33,7 +39,7 @@ set MODEL_PATH=%CURRENT_DIR%\ppyoloe_crn_l_300e_coco set GROUND_TRUTH_PATH=%CURRENT_DIR%\release_task_groud_truth_result.txt set COMPARE_SHELL=%CURRENT_DIR%\compare_with_gt.py -python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/dev/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" +python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/%DOWNLOAD_DIR%/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" mkdir build && cd build cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2" diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index e2615194f49..9a160b6e2bc 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -4,6 +4,11 @@ CURRENT_DIR=$(cd $(dirname $0); pwd) PLATFORM=$1 DEVICE=$2 VERSION=$3 +if [ "$VERSION" = "0.0.0" ];then + DOWNLOAD_DIR=dev +else + DOWNLOAD_DIR=rel_tmp +fi if [ "$DEVICE" = "gpu" ];then CPP_FASTDEPLOY_PACKAGE=fastdeploy-$PLATFORM-$DEVICE-$VERSION else @@ -42,7 +47,7 @@ fi case_number=${#RUN_CASE[@]} -wget -q https://fastdeploy.bj.bcebos.com/dev/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz +wget -q https://fastdeploy.bj.bcebos.com/$DOWNLOAD_DIR/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz tar xvf $CPP_FASTDEPLOY_PACKAGE.tgz mkdir build && cd build From 5c1e1f097e0f70105e1a96a8b2036af2bf54c737 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Wed, 23 Nov 2022 21:48:37 +0800 Subject: [PATCH 007/136] Fix precision diff for osx-arm64 --- tests/release_task/compare_with_gt.py | 16 ++++++++-------- tests/release_task/cpp_run.sh | 2 +- tests/release_task/py_run.sh | 11 ++++++++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py index ba030999c8e..57258f59b33 100644 --- a/tests/release_task/compare_with_gt.py +++ b/tests/release_task/compare_with_gt.py @@ -3,31 +3,31 @@ diff_score_threshold = { "linux-x64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 1e-4, "boxes_diff": 1e-3 }, "linux-aarch64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 1e-4, "boxes_diff": 1e-3 }, "osx-x86_64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 2e-4, "boxes_diff": 1e-3 }, "osx-arm64": { - "label_diff": 1e-02, - "score_diff": 1e-4, - "boxes_diff_ratio": 2e-4, + "label_diff": 0, + "score_diff": 1e-3, + "boxes_diff_ratio": 5e-4, "boxes_diff": 1e-3 }, "win-x64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 5e-4, "boxes_diff_ratio": 1e-3, "boxes_diff": 1e-3 @@ -133,7 +133,7 @@ def check_result(gt_result, infer_result, args): is_diff = True boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt" boxes_diff_ratio_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio.txt" - boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool" + boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool.txt" save_numpy_result(boxes_diff_bool_file, boxes_diff > boxes_diff_threshold) save_numpy_result(boxes_diff_ratio_file, boxes_diff_ratio) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 9a160b6e2bc..954a2a38e57 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -19,7 +19,7 @@ LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') #LINUX_AARCH_CPU_CASE=('ort' 'openvino') LINUX_AARCH_CPU_CASE=('ort') -MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') +MACOS_INTEL_CPU_CASE=('ort' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz wget -q https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 6001bfbbbcb..64e876747e9 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -14,7 +14,7 @@ LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') #LINUX_AARCH_CPU_CASE=('ort' 'openvino') LINUX_AARCH_CPU_CASE=('ort') -MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') +MACOS_INTEL_CPU_CASE=('ort' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz wget -q https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg @@ -54,8 +54,13 @@ do echo "Python Backend:" $backend if [ "$backend" != "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_$backend\_cpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu - check_ret + if [ "$PLATFORM" = "osx-arm64"] + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold 0.5 + check_ret + else + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret + fi fi if [ "$DEVICE" = "gpu" ];then From 19684a25b924b38457f6d5dc3d8c2508c6a3c1b7 Mon Sep 17 00:00:00 2001 From: huangjianhui <852142024@qq.com> Date: Thu, 24 Nov 2022 21:21:53 +0800 Subject: [PATCH 008/136] Update py_run.sh --- tests/release_task/py_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 64e876747e9..07e5bce5094 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -54,7 +54,7 @@ do echo "Python Backend:" $backend if [ "$backend" != "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_$backend\_cpu_result.txt - if [ "$PLATFORM" = "osx-arm64"] + if [ "$PLATFORM" = "osx-arm64"];then python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold 0.5 check_ret else From d7084e3b771299651686755e678f68c60c202c4a Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Fri, 25 Nov 2022 03:36:53 +0000 Subject: [PATCH 009/136] Add threshold for osx-arm64 --- tests/release_task/cpp_run.sh | 1 + tests/release_task/py_run.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 954a2a38e57..4d821ed7b91 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -40,6 +40,7 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "linux-aarch64" ];then RUN_CASE=(${LINUX_AARCH_CPU_CASE[*]}) elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-x86_64" ];then RUN_CASE=(${MACOS_INTEL_CPU_CASE[*]}) + CONF_THRESHOLD=0.5 elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) CONF_THRESHOLD=0.5 diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 07e5bce5094..e0bd0154585 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -54,7 +54,7 @@ do echo "Python Backend:" $backend if [ "$backend" != "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_$backend\_cpu_result.txt - if [ "$PLATFORM" = "osx-arm64"];then + if [ "$PLATFORM" = "osx-arm64" ];then python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold 0.5 check_ret else From b05ec12b6ab1ff6bafbc4b061cfbea493c2e5a27 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Fri, 25 Nov 2022 05:19:08 +0000 Subject: [PATCH 010/136] Add osx-x86_64 threshold --- tests/release_task/py_run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index e0bd0154585..12d61199b7b 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -54,7 +54,7 @@ do echo "Python Backend:" $backend if [ "$backend" != "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device cpu --backend $backend >> py_$backend\_cpu_result.txt - if [ "$PLATFORM" = "osx-arm64" ];then + if [ "$PLATFORM" = "osx-arm64" ] || [ "$PLATFORM" = "osx-x86_64" ];then python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold 0.5 check_ret else From 39bb887f61bb4d340b319fbd72ff3e788b975cd3 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Mon, 14 Nov 2022 07:52:42 +0000 Subject: [PATCH 011/136] Add error check function --- tests/release_task/py_run.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 2a4f73f16cf..7c7d9ba5659 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -38,10 +38,20 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) fi +ret=0 +check_ret(){ + if [ $? -ne 0 ];then + echo "ret before" + echo $ret + ret=-1 + echo "ret after" + echo $ret + fi +} + case_number=${#RUN_CASE[@]} py_version=$(python -V | awk '{print $2}') echo "py_version:" $py_version -ret=0 for((j=0;j> py_$backend\_cpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret fi if [ "$DEVICE" = "gpu" ];then if [ "$backend" = "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt + check_ret else python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_$backend\_gpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_gpu_result.txt --platform $PLATFORM --device gpu + check_ret fi fi - if [ $? -ne 0 ];then - ret=-1 - fi done - res_file="result.txt" if [ ! -f $res_file ];then if [ $ret -ne 0 ];then From b5acec75dc5a145318686164b16f246fd728d9f3 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Fri, 25 Nov 2022 09:49:35 +0000 Subject: [PATCH 012/136] Remove root commit author --- tests/release_task/py_run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 98ba53b5f4e..270d2e77717 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -63,7 +63,6 @@ do fi fi if [ "$DEVICE" = "gpu" ];then - if [ "$backend" = "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt From 1261545cd98da4571cbc937ab590432a2e032ede Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Mon, 14 Nov 2022 07:52:42 +0000 Subject: [PATCH 013/136] Add error check function --- tests/release_task/py_run.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 2a4f73f16cf..7c7d9ba5659 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -38,10 +38,20 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) fi +ret=0 +check_ret(){ + if [ $? -ne 0 ];then + echo "ret before" + echo $ret + ret=-1 + echo "ret after" + echo $ret + fi +} + case_number=${#RUN_CASE[@]} py_version=$(python -V | awk '{print $2}') echo "py_version:" $py_version -ret=0 for((j=0;j> py_$backend\_cpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret fi if [ "$DEVICE" = "gpu" ];then if [ "$backend" = "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt + check_ret else python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_$backend\_gpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_gpu_result.txt --platform $PLATFORM --device gpu + check_ret fi fi - if [ $? -ne 0 ];then - ret=-1 - fi done - res_file="result.txt" if [ ! -f $res_file ];then if [ $ret -ne 0 ];then From f9971c2aba0a501224c0bb8e29bfbb9f5c529bed Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Mon, 14 Nov 2022 08:00:03 +0000 Subject: [PATCH 014/136] Format code --- tests/release_task/py_run.sh | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 2a4f73f16cf..1f20f6332fa 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -14,7 +14,7 @@ LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') #LINUX_AARCH_CPU_CASE=('ort' 'openvino') LINUX_AARCH_CPU_CASE=('ort') -MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') +MACOS_INTEL_CPU_CASE=('ort' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz wget -q https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg @@ -38,34 +38,44 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]}) fi +ret=0 +check_ret(){ + if [ $? -ne 0 ];then + ret=-1 + fi +} + case_number=${#RUN_CASE[@]} py_version=$(python -V | awk '{print $2}') echo "py_version:" $py_version -ret=0 for((j=0;j> py_$backend\_cpu_result.txt - python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + if [ "$PLATFORM" = "osx-arm64" ] || [ "$PLATFORM" = "osx-x86_64" ];then + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold 0.5 + check_ret + else + python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_cpu_result.txt --platform $PLATFORM --device cpu + check_ret + fi fi if [ "$DEVICE" = "gpu" ];then if [ "$backend" = "trt" ];then python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_trt_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_trt_result.txt --platform $PLATFORM --device trt + check_ret else python infer_ppyoloe.py --model_dir $MODEL_PATH --image $IMAGE_PATH --device gpu --backend $backend >> py_$backend\_gpu_result.txt python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path py_$backend\_gpu_result.txt --platform $PLATFORM --device gpu + check_ret fi fi - if [ $? -ne 0 ];then - ret=-1 - fi done - res_file="result.txt" if [ ! -f $res_file ];then if [ $ret -ne 0 ];then @@ -76,3 +86,5 @@ else cat $res_file exit -1 fi + + From 81e07cac73e5987e481c96a8d66dcb41912978d3 Mon Sep 17 00:00:00 2001 From: Jack Zhou Date: Mon, 14 Nov 2022 15:49:39 +0800 Subject: [PATCH 015/136] [Bug Fix]Remove profile (#579) remove profile --- python/fastdeploy/__init__.py | 1 - python/fastdeploy/utils/__init__.py | 2 -- python/fastdeploy/utils/profile.py | 28 ---------------------------- 3 files changed, 31 deletions(-) delete mode 100644 python/fastdeploy/utils/profile.py diff --git a/python/fastdeploy/__init__.py b/python/fastdeploy/__init__.py index bacad5bdf76..b767393f106 100644 --- a/python/fastdeploy/__init__.py +++ b/python/fastdeploy/__init__.py @@ -37,4 +37,3 @@ from . import pipeline from . import text from .download import download, download_and_decompress, download_model -from .utils import profile diff --git a/python/fastdeploy/utils/__init__.py b/python/fastdeploy/utils/__init__.py index 374afbad4ff..30fd1b3cb96 100644 --- a/python/fastdeploy/utils/__init__.py +++ b/python/fastdeploy/utils/__init__.py @@ -11,6 +11,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from .profile import profile from .example_resource import get_detection_test_image diff --git a/python/fastdeploy/utils/profile.py b/python/fastdeploy/utils/profile.py deleted file mode 100644 index 4c49cfe43e0..00000000000 --- a/python/fastdeploy/utils/profile.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import cProfile, pstats, io -from pstats import SortKey - - -def profile(func, *args, **kwargs): - pr = cProfile.Profile() - pr.enable() - func(*args, **kwargs) - pr.disable() - s = io.StringIO() - sortby = SortKey.CUMULATIVE - ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - ps.print_stats() - print(s.getvalue()) From 79343f91fc678cec0eea2ad77fb2e4562a4e5ef8 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Thu, 17 Nov 2022 03:09:27 +0000 Subject: [PATCH 016/136] Add new download dir for release version --- tests/release_task/cpp_run.bat | 8 +++++++- tests/release_task/cpp_run.sh | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/release_task/cpp_run.bat b/tests/release_task/cpp_run.bat index 2c8a8aed628..4d92ea51b73 100644 --- a/tests/release_task/cpp_run.bat +++ b/tests/release_task/cpp_run.bat @@ -13,6 +13,12 @@ echo "PLATFORM: " %PLATFORM% echo "DEVICE: " %DEVICE% echo "VERSION: " %VERSION% +if "%VERSION%" == "0.0.0" ( + set DOWNLOAD_DIR=dev +) else ( + set DOWNLOAD_DIR=rel_tmp +) + if "%DEVICE%" == "gpu" ( set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%DEVICE%-%VERSION% set RUN_CASES=ort paddle trt @@ -33,7 +39,7 @@ set MODEL_PATH=%CURRENT_DIR%\ppyoloe_crn_l_300e_coco set GROUND_TRUTH_PATH=%CURRENT_DIR%\release_task_groud_truth_result.txt set COMPARE_SHELL=%CURRENT_DIR%\compare_with_gt.py -python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/dev/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" +python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/%DOWNLOAD_DIR%/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')" mkdir build && cd build cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2" diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index e2615194f49..9a160b6e2bc 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -4,6 +4,11 @@ CURRENT_DIR=$(cd $(dirname $0); pwd) PLATFORM=$1 DEVICE=$2 VERSION=$3 +if [ "$VERSION" = "0.0.0" ];then + DOWNLOAD_DIR=dev +else + DOWNLOAD_DIR=rel_tmp +fi if [ "$DEVICE" = "gpu" ];then CPP_FASTDEPLOY_PACKAGE=fastdeploy-$PLATFORM-$DEVICE-$VERSION else @@ -42,7 +47,7 @@ fi case_number=${#RUN_CASE[@]} -wget -q https://fastdeploy.bj.bcebos.com/dev/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz +wget -q https://fastdeploy.bj.bcebos.com/$DOWNLOAD_DIR/cpp/$CPP_FASTDEPLOY_PACKAGE.tgz tar xvf $CPP_FASTDEPLOY_PACKAGE.tgz mkdir build && cd build From f54c477b8503f0656d6241c5deb5def45ee39b21 Mon Sep 17 00:00:00 2001 From: huangjianhui <852142024@qq.com> Date: Mon, 14 Nov 2022 16:02:12 +0800 Subject: [PATCH 017/136] [Bug Fix]Add check return code function (#581) * Fix release_task cpp_run.sh Author: felixhjh <852142024@qq.com> Date: Thu Nov 10 07:26:48 2022 +0000 * Add option for infer_ppyoloe.cc author: felixhjh <852142024@qq.com> * Fix bug in release task scripts * Add error check function * Format code Author: felixhjh <852142024@qq.com> --- tests/release_task/py_run.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh index 1f20f6332fa..98ba53b5f4e 100644 --- a/tests/release_task/py_run.sh +++ b/tests/release_task/py_run.sh @@ -87,4 +87,3 @@ else exit -1 fi - From 412deb7de38da736b585743f3b7da4879cb07544 Mon Sep 17 00:00:00 2001 From: yunyaoXYY <109218879+yunyaoXYY@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:39:31 +0800 Subject: [PATCH 018/136] [Quantization] Remove extra requirements for fd-auto-compress package (#582) * Add PaddleOCR Support * Add PaddleOCR Support * Add PaddleOCRv3 Support * Add PaddleOCRv3 Support * Update README.md * Update README.md * Update README.md * Update README.md * Add PaddleOCRv3 Support * Add PaddleOCRv3 Supports * Add PaddleOCRv3 Suport * Fix Rec diff * Remove useless functions * Remove useless comments * Add PaddleOCRv2 Support * Add PaddleOCRv3 & PaddleOCRv2 Support * remove useless parameters * Add utils of sorting det boxes * Fix code naming convention * Fix code naming convention * Fix code naming convention * Fix bug in the Classify process * Imporve OCR Readme * Fix diff in Cls model * Update Model Download Link in Readme * Fix diff in PPOCRv2 * Improve OCR readme * Imporve OCR readme * Improve OCR readme * Improve OCR readme * Imporve OCR readme * Improve OCR readme * Fix conflict * Add readme for OCRResult * Improve OCR readme * Add OCRResult readme * Improve OCR readme * Improve OCR readme * Add Model Quantization Demo * Fix Model Quantization Readme * Fix Model Quantization Readme * Add the function to do PTQ quantization * Improve quant tools readme * Improve quant tool readme * Improve quant tool readme * Add PaddleInference-GPU for OCR Rec model * Add QAT method to fastdeploy-quantization tool * Remove examples/slim for now * Move configs folder * Add Quantization Support for Classification Model * Imporve ways of importing preprocess * Upload YOLO Benchmark on readme * Upload YOLO Benchmark on readme * Upload YOLO Benchmark on readme * Improve Quantization configs and readme * Add support for multi-inputs model * Add backends and params file for YOLOv7 * Add quantized model deployment support for YOLO series * Fix YOLOv5 quantize readme * Fix YOLO quantize readme * Fix YOLO quantize readme * Improve quantize YOLO readme * Improve quantize YOLO readme * Improve quantize YOLO readme * Improve quantize YOLO readme * Improve quantize YOLO readme * Fix bug, change Fronted to ModelFormat * Change Fronted to ModelFormat * Add examples to deploy quantized paddleclas models * Fix readme * Add quantize Readme * Add quantize Readme * Add quantize Readme * Modify readme of quantization tools * Modify readme of quantization tools * Improve quantization tools readme * Improve quantization readme * Improve PaddleClas quantized model deployment readme * Add PPYOLOE-l quantized deployment examples * Improve quantization tools readme * Improve Quantize Readme * Fix conflicts * Fix conflicts * improve readme * Improve quantization tools and readme * Improve quantization tools and readme * Add quantized deployment examples for PaddleSeg model * Fix cpp readme * Fix memory leak of reader_wrapper function * Fix model file name in PaddleClas quantization examples * Update Runtime and E2E benchmark * Update Runtime and E2E benchmark * Rename quantization tools to auto compression tools * Remove PPYOLOE data when deployed on MKLDNN * Fix readme * Support PPYOLOE with OR without NMS and update readme * Update Readme * Update configs and readme * Update configs and readme * Add Paddle-TensorRT backend in quantized model deploy examples * Support PPYOLOE+ series * Add reused_input_tensors for PPYOLOE * Improve fastdeploy tools usage * improve fastdeploy tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * remove modify * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Improve fastdeploy auto compression tool * Remove extra requirements for fd-auto-compress package --- python/requirements.txt | 2 +- tools/README.md | 2 +- tools/README_EN.md | 2 +- tools/auto_compression/requirements.txt | 1 - tools/auto_compression/setup.py | 7 ++----- 5 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 tools/auto_compression/requirements.txt diff --git a/python/requirements.txt b/python/requirements.txt index 03755825d66..3d4a5600110 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,4 +3,4 @@ requests tqdm numpy opencv-python -fd-auto-compress>=0.0.0 +fd-auto-compress>=0.0.1 diff --git a/tools/README.md b/tools/README.md index ccc6cca690f..c166b8f9f09 100644 --- a/tools/README.md +++ b/tools/README.md @@ -20,7 +20,7 @@ python setup.py install ```bash # 通过pip安装fd-auto-compress. # FastDeploy的python包已包含此工具, 不需重复安装. -pip install fd-auto-compress +pip install fd-auto-compress==0.0.1 # 在当前目录执行以下命令 python setup.py install diff --git a/tools/README_EN.md b/tools/README_EN.md index 4e30972b162..3a8313fa9f5 100644 --- a/tools/README_EN.md +++ b/tools/README_EN.md @@ -21,7 +21,7 @@ python setup.py install ```bash # Installing fd-auto-compress via pip # This tool is included in the python installer of FastDeploy, so you don't need to install it again. -pip install fd-auto-compress +pip install fd-auto-compress==0.0.1 # Execute in the current directory python setup.py install diff --git a/tools/auto_compression/requirements.txt b/tools/auto_compression/requirements.txt deleted file mode 100644 index b9b109222d3..00000000000 --- a/tools/auto_compression/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -paddleslim diff --git a/tools/auto_compression/setup.py b/tools/auto_compression/setup.py index 5938874bc9b..8ee71b61862 100644 --- a/tools/auto_compression/setup.py +++ b/tools/auto_compression/setup.py @@ -4,11 +4,9 @@ long_description = "fd_auto_compress is a toolkit for model auto compression of FastDeploy.\n\n" long_description += "Usage: fastdeploy --auto_compress --config_path=./yolov7_tiny_qat_dis.yaml --method='QAT' --save_dir='../v7_qat_outmodel/' \n" -with open("requirements.txt") as fin: - REQUIRED_PACKAGES = fin.read() - setuptools.setup( - name="fd_auto_compress", # name of package + name="fd_auto_compress", + version="0.0.1", description="A toolkit for model auto compression of FastDeploy.", long_description=long_description, long_description_content_type="text/plain", @@ -16,7 +14,6 @@ author='fastdeploy', author_email='fastdeploy@baidu.com', url='https://github.com/PaddlePaddle/FastDeploy.git', - install_requires=REQUIRED_PACKAGES, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", From c84de29f94f9532c94054618369ff9c8396e7774 Mon Sep 17 00:00:00 2001 From: Wang Xinyu Date: Mon, 14 Nov 2022 18:44:00 +0800 Subject: [PATCH 019/136] [Backend] cuda normalize and permute, cuda concat, optimized ppcls, ppdet & ppseg (#546) * cuda normalize and permute, cuda concat * add use cuda option for preprocessor * ppyoloe use cuda normalize * ppseg use cuda normalize * add proclib cuda in processor base * ppcls add use cuda preprocess api * ppcls preprocessor set gpu id * fix pybind * refine ppcls preprocessing use gpu logic * fdtensor device id is -1 by default * refine assert message Co-authored-by: heliqi <1101791222@qq.com> --- fastdeploy/core/fd_tensor.cc | 7 +- fastdeploy/core/fd_tensor.h | 8 +- fastdeploy/function/concat.cc | 13 +-- fastdeploy/runtime.cc | 5 ++ .../vision/classification/ppcls/model.cc | 0 .../classification/ppcls/ppcls_pybind.cc | 3 + .../classification/ppcls/preprocessor.cc | 34 +++++++- .../classification/ppcls/preprocessor.h | 8 ++ fastdeploy/vision/common/processors/base.cc | 20 +++++ fastdeploy/vision/common/processors/base.h | 14 ++++ fastdeploy/vision/common/processors/mat.cc | 2 +- fastdeploy/vision/common/processors/mat.h | 1 + .../processors/normalize_and_permute.cc | 1 - .../processors/normalize_and_permute.cu | 82 +++++++++++++++++++ .../common/processors/normalize_and_permute.h | 3 + .../vision/common/processors/proc_lib.cc | 3 + .../vision/common/processors/proc_lib.h | 2 +- fastdeploy/vision/detection/ppdet/ppyoloe.cc | 3 - fastdeploy/vision/segmentation/ppseg/model.cc | 14 ++-- .../vision/classification/ppcls/__init__.py | 7 ++ 20 files changed, 204 insertions(+), 26 deletions(-) mode change 100755 => 100644 fastdeploy/vision/classification/ppcls/model.cc create mode 100644 fastdeploy/vision/common/processors/normalize_and_permute.cu diff --git a/fastdeploy/core/fd_tensor.cc b/fastdeploy/core/fd_tensor.cc index ff4ae61aaad..8a39421049a 100644 --- a/fastdeploy/core/fd_tensor.cc +++ b/fastdeploy/core/fd_tensor.cc @@ -252,7 +252,8 @@ void FDTensor::FreeFn() { } } -void FDTensor::CopyBuffer(void* dst, const void* src, size_t nbytes) { +void FDTensor::CopyBuffer(void* dst, const void* src, size_t nbytes, + const Device& device, bool is_pinned_memory) { if (device == Device::GPU) { #ifdef WITH_GPU FDASSERT(cudaMemcpy(dst, src, nbytes, cudaMemcpyDeviceToDevice) == 0, @@ -295,7 +296,7 @@ FDTensor::FDTensor(const FDTensor& other) size_t nbytes = Nbytes(); FDASSERT(ReallocFn(nbytes), "The FastDeploy FDTensor allocate memory error"); - CopyBuffer(buffer_, other.buffer_, nbytes); + CopyBuffer(buffer_, other.buffer_, nbytes, device, is_pinned_memory); } } @@ -325,7 +326,7 @@ FDTensor& FDTensor::operator=(const FDTensor& other) { } else { Resize(other.shape, other.dtype, other.name, other.device); size_t nbytes = Nbytes(); - CopyBuffer(buffer_, other.buffer_, nbytes); + CopyBuffer(buffer_, other.buffer_, nbytes, device, is_pinned_memory); } external_data_ptr = other.external_data_ptr; } diff --git a/fastdeploy/core/fd_tensor.h b/fastdeploy/core/fd_tensor.h index ef7fbff414c..b08e5f12ecc 100644 --- a/fastdeploy/core/fd_tensor.h +++ b/fastdeploy/core/fd_tensor.h @@ -39,6 +39,9 @@ struct FASTDEPLOY_DECL FDTensor { // GPU to inference the model // so we can skip data transfer, which may improve the efficience Device device = Device::CPU; + // By default the device id of FDTensor is -1, which means this value is + // invalid, and FDTensor is using the same device id as Runtime. + int device_id = -1; // Whether the data buffer is in pinned memory, which is allocated // with cudaMallocHost() @@ -130,8 +133,9 @@ struct FASTDEPLOY_DECL FDTensor { ~FDTensor() { FreeFn(); } - private: - void CopyBuffer(void* dst, const void* src, size_t nbytes); + static void CopyBuffer(void* dst, const void* src, size_t nbytes, + const Device& device = Device::CPU, + bool is_pinned_memory = false); }; } // namespace fastdeploy diff --git a/fastdeploy/function/concat.cc b/fastdeploy/function/concat.cc index cccd1768e4f..3a59e791086 100644 --- a/fastdeploy/function/concat.cc +++ b/fastdeploy/function/concat.cc @@ -85,8 +85,9 @@ struct ConcatFunctor { int64_t col_len = input_cols[j]; const T* input_data = reinterpret_cast(input[j].Data()); for (int64_t k = 0; k < out_rows; ++k) { - std::memcpy(output_data + k * out_cols + col_idx, - input_data + k * col_len, sizeof(T) * col_len); + FDTensor::CopyBuffer(output_data + k * out_cols + col_idx, + input_data + k * col_len, sizeof(T) * col_len, + input[j].device, input[j].is_pinned_memory); } col_idx += col_len; } @@ -97,7 +98,8 @@ template void ConcatKernel(const std::vector& input, FDTensor* output, int axis) { auto output_shape = ComputeAndCheckConcatOutputShape(input, axis); - output->Allocate(output_shape, TypeToDataType::dtype); + output->Resize(output_shape, TypeToDataType::dtype, output->name, + input[0].device); ConcatFunctor functor; functor(input, axis, output); @@ -115,10 +117,9 @@ void Concat(const std::vector& x, FDTensor* out, int axis) { if (axis < 0) { axis += rank; } - FDTensor out_temp; + FD_VISIT_ALL_TYPES(x[0].dtype, "Concat", - ([&] { ConcatKernel(x, &out_temp, axis); })); - *out = std::move(out_temp); + ([&] { ConcatKernel(x, out, axis); })); } } // namespace function diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index fc519fcfff1..3077afb51d6 100755 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -568,6 +568,11 @@ std::vector Runtime::GetOutputInfos() { bool Runtime::Infer(std::vector& input_tensors, std::vector* output_tensors) { + for (auto& tensor: input_tensors) { + FDASSERT(tensor.device_id < 0 || tensor.device_id == option.device_id, + "Device id of input tensor(%d) and runtime(%d) are not same.", + tensor.device_id, option.device_id); + } return backend_->Infer(input_tensors, output_tensors); } diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc old mode 100755 new mode 100644 diff --git a/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc b/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc index dc63744cc4b..74dfce845f0 100644 --- a/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc +++ b/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc @@ -28,6 +28,9 @@ void BindPaddleClas(pybind11::module& m) { pybind11::eval("raise Exception('Failed to preprocess the input data in PaddleClasPreprocessor.')"); } return outputs; + }) + .def("use_gpu", [](vision::classification::PaddleClasPreprocessor& self, int gpu_id = -1) { + self.UseGpu(gpu_id); }); pybind11::class_( diff --git a/fastdeploy/vision/classification/ppcls/preprocessor.cc b/fastdeploy/vision/classification/ppcls/preprocessor.cc index 8f7d9135819..bdc21ad1eee 100644 --- a/fastdeploy/vision/classification/ppcls/preprocessor.cc +++ b/fastdeploy/vision/classification/ppcls/preprocessor.cc @@ -15,17 +15,22 @@ #include "fastdeploy/vision/classification/ppcls/preprocessor.h" #include "fastdeploy/function/concat.h" #include "yaml-cpp/yaml.h" +#ifdef WITH_GPU +#include +#endif namespace fastdeploy { namespace vision { namespace classification { PaddleClasPreprocessor::PaddleClasPreprocessor(const std::string& config_file) { - FDASSERT(BuildPreprocessPipelineFromConfig(config_file), "Failed to create PaddleClasPreprocessor."); + FDASSERT(BuildPreprocessPipelineFromConfig(config_file), + "Failed to create PaddleClasPreprocessor."); initialized_ = true; } -bool PaddleClasPreprocessor::BuildPreprocessPipelineFromConfig(const std::string& config_file) { +bool PaddleClasPreprocessor::BuildPreprocessPipelineFromConfig( + const std::string& config_file) { processors_.clear(); YAML::Node cfg; try { @@ -73,6 +78,19 @@ bool PaddleClasPreprocessor::BuildPreprocessPipelineFromConfig(const std::string return true; } +void PaddleClasPreprocessor::UseGpu(int gpu_id) { +#ifdef WITH_GPU + use_cuda_ = true; + if (gpu_id < 0) return; + device_id_ = gpu_id; + cudaSetDevice(device_id_); +#else + FDWARNING << "FastDeploy didn't compile with WITH_GPU. " + << "Will force to use CPU to run preprocessing." << std::endl; + use_cuda_ = false; +#endif +} + bool PaddleClasPreprocessor::Run(std::vector* images, std::vector* outputs) { if (!initialized_) { FDERROR << "The preprocessor is not initialized." << std::endl; @@ -85,8 +103,15 @@ bool PaddleClasPreprocessor::Run(std::vector* images, std::vectorsize(); ++i) { for (size_t j = 0; j < processors_.size(); ++j) { - if (!(*(processors_[j].get()))(&((*images)[i]))) { - FDERROR << "Failed to processs image:" << i << " in " << processors_[i]->Name() << "." << std::endl; + bool ret = false; + if (processors_[j]->Name() == "NormalizeAndPermute" && use_cuda_) { + ret = (*(processors_[j].get()))(&((*images)[i]), ProcLib::CUDA); + } else { + ret = (*(processors_[j].get()))(&((*images)[i])); + } + if (!ret) { + FDERROR << "Failed to processs image:" << i << " in " + << processors_[i]->Name() << "." << std::endl; return false; } } @@ -104,6 +129,7 @@ bool PaddleClasPreprocessor::Run(std::vector* images, std::vector* images, std::vector* outputs); + /** \brief Use GPU to run preprocessing + * + * \param[in] gpu_id GPU device id + */ + void UseGpu(int gpu_id = -1); private: bool BuildPreprocessPipelineFromConfig(const std::string& config_file); std::vector> processors_; bool initialized_ = false; + bool use_cuda_ = false; + // GPU device id + int device_id_ = -1; }; } // namespace classification diff --git a/fastdeploy/vision/common/processors/base.cc b/fastdeploy/vision/common/processors/base.cc index 20c0584462d..f3d5b0a975b 100644 --- a/fastdeploy/vision/common/processors/base.cc +++ b/fastdeploy/vision/common/processors/base.cc @@ -30,12 +30,32 @@ bool Processor::operator()(Mat* mat, ProcLib lib) { return ImplByFlyCV(mat); #else FDASSERT(false, "FastDeploy didn't compile with FlyCV."); +#endif + } else if (target == ProcLib::CUDA) { +#ifdef WITH_GPU + return ImplByCuda(mat); +#else + FDASSERT(false, "FastDeploy didn't compile with WITH_GPU."); #endif } // DEFAULT & OPENCV return ImplByOpenCV(mat); } +FDTensor* Processor::UpdateAndGetReusedBuffer( + const std::vector& new_shape, const int& opencv_dtype, + const std::string& buffer_name, const Device& new_device, + const bool& use_pinned_memory) { + if (reused_buffers_.count(buffer_name) == 0) { + reused_buffers_[buffer_name] = FDTensor(); + } + reused_buffers_[buffer_name].is_pinned_memory = use_pinned_memory; + reused_buffers_[buffer_name].Resize(new_shape, + OpenCVDataTypeToFD(opencv_dtype), + buffer_name, new_device); + return &reused_buffers_[buffer_name]; +} + void EnableFlyCV() { #ifdef ENABLE_FLYCV DefaultProcLib::default_lib = ProcLib::FLYCV; diff --git a/fastdeploy/vision/common/processors/base.h b/fastdeploy/vision/common/processors/base.h index e9d7ede9045..c833ef8a379 100644 --- a/fastdeploy/vision/common/processors/base.h +++ b/fastdeploy/vision/common/processors/base.h @@ -18,6 +18,7 @@ #include "fastdeploy/vision/common/processors/mat.h" #include "opencv2/highgui/highgui.hpp" #include "opencv2/imgproc/imgproc.hpp" +#include namespace fastdeploy { namespace vision { @@ -55,7 +56,20 @@ class FASTDEPLOY_DECL Processor { return ImplByOpenCV(mat); } + virtual bool ImplByCuda(Mat* mat) { + return ImplByOpenCV(mat); + } + virtual bool operator()(Mat* mat, ProcLib lib = ProcLib::DEFAULT); + + protected: + FDTensor* UpdateAndGetReusedBuffer( + const std::vector& new_shape, const int& opencv_dtype, + const std::string& buffer_name, const Device& new_device = Device::CPU, + const bool& use_pinned_memory = false); + + private: + std::unordered_map reused_buffers_; }; } // namespace vision diff --git a/fastdeploy/vision/common/processors/mat.cc b/fastdeploy/vision/common/processors/mat.cc index 7ef4f9d7037..7e9a0efc19c 100644 --- a/fastdeploy/vision/common/processors/mat.cc +++ b/fastdeploy/vision/common/processors/mat.cc @@ -34,7 +34,7 @@ void* Mat::Data() { void Mat::ShareWithTensor(FDTensor* tensor) { tensor->SetExternalData({Channels(), Height(), Width()}, Type(), Data()); - tensor->device = Device::CPU; + tensor->device = device; if (layout == Layout::HWC) { tensor->shape = {Height(), Width(), Channels()}; } diff --git a/fastdeploy/vision/common/processors/mat.h b/fastdeploy/vision/common/processors/mat.h index 52537004329..8c19080301a 100644 --- a/fastdeploy/vision/common/processors/mat.h +++ b/fastdeploy/vision/common/processors/mat.h @@ -131,6 +131,7 @@ struct FASTDEPLOY_DECL Mat { ProcLib mat_type = ProcLib::OPENCV; Layout layout = Layout::HWC; + Device device = Device::CPU; // Create FD Mat from FD Tensor. This method only create a // new FD Mat with zero copy and it's data pointer is reference diff --git a/fastdeploy/vision/common/processors/normalize_and_permute.cc b/fastdeploy/vision/common/processors/normalize_and_permute.cc index ca1565ec870..93850b97fbf 100644 --- a/fastdeploy/vision/common/processors/normalize_and_permute.cc +++ b/fastdeploy/vision/common/processors/normalize_and_permute.cc @@ -73,7 +73,6 @@ bool NormalizeAndPermute::ImplByOpenCV(Mat* mat) { res.ptr() + i * origin_h * origin_w * 4), 0); } - mat->SetMat(res); mat->layout = Layout::CHW; return true; diff --git a/fastdeploy/vision/common/processors/normalize_and_permute.cu b/fastdeploy/vision/common/processors/normalize_and_permute.cu new file mode 100644 index 00000000000..8450417bf49 --- /dev/null +++ b/fastdeploy/vision/common/processors/normalize_and_permute.cu @@ -0,0 +1,82 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/normalize_and_permute.h" + +namespace fastdeploy { +namespace vision { + +__global__ void NormalizeAndPermuteKernel( + uint8_t* src, float* dst, const float* alpha, const float* beta, + int num_channel, bool swap_rb, int edge) { + int idx = blockDim.x * blockIdx.x + threadIdx.x; + if (idx >= edge) return; + + if (swap_rb) { + uint8_t tmp = src[num_channel * idx]; + src[num_channel * idx] = src[num_channel * idx + 2]; + src[num_channel * idx + 2] = tmp; + } + + for (int i = 0; i < num_channel; ++i) { + dst[idx + edge * i] = src[num_channel * idx + i] * alpha[i] + beta[i]; + } +} + +bool NormalizeAndPermute::ImplByCuda(Mat* mat) { + cv::Mat* im = mat->GetOpenCVMat(); + std::string buf_name = Name() + "_src"; + std::vector shape = {im->rows, im->cols, im->channels()}; + FDTensor* src = UpdateAndGetReusedBuffer(shape, im->type(), buf_name, + Device::GPU); + FDASSERT(cudaMemcpy(src->Data(), im->ptr(), src->Nbytes(), + cudaMemcpyHostToDevice) == 0, + "Error occurs while copy memory from CPU to GPU."); + + buf_name = Name() + "_dst"; + FDTensor* dst = UpdateAndGetReusedBuffer(shape, CV_32FC(im->channels()), + buf_name, Device::GPU); + cv::Mat res(im->rows, im->cols, CV_32FC(im->channels()), dst->Data()); + + buf_name = Name() + "_alpha"; + FDTensor* alpha = UpdateAndGetReusedBuffer({(int64_t)alpha_.size()}, CV_32FC1, + buf_name, Device::GPU); + FDASSERT(cudaMemcpy(alpha->Data(), alpha_.data(), alpha->Nbytes(), + cudaMemcpyHostToDevice) == 0, + "Error occurs while copy memory from CPU to GPU."); + + buf_name = Name() + "_beta"; + FDTensor* beta = UpdateAndGetReusedBuffer({(int64_t)beta_.size()}, CV_32FC1, + buf_name, Device::GPU); + FDASSERT(cudaMemcpy(beta->Data(), beta_.data(), beta->Nbytes(), + cudaMemcpyHostToDevice) == 0, + "Error occurs while copy memory from CPU to GPU."); + + int jobs = im->cols * im->rows; + int threads = 256; + int blocks = ceil(jobs / (float)threads); + NormalizeAndPermuteKernel<<>>( + reinterpret_cast(src->Data()), + reinterpret_cast(dst->Data()), + reinterpret_cast(alpha->Data()), + reinterpret_cast(beta->Data()), im->channels(), swap_rb_, jobs); + + mat->SetMat(res); + mat->device = Device::GPU; + mat->layout = Layout::CHW; + return true; +} + +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/normalize_and_permute.h b/fastdeploy/vision/common/processors/normalize_and_permute.h index 04715d9d7f3..ea7649d9281 100644 --- a/fastdeploy/vision/common/processors/normalize_and_permute.h +++ b/fastdeploy/vision/common/processors/normalize_and_permute.h @@ -28,6 +28,9 @@ class FASTDEPLOY_DECL NormalizeAndPermute : public Processor { bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV bool ImplByFlyCV(Mat* mat); +#endif +#ifdef WITH_GPU + bool ImplByCuda(Mat* mat); #endif std::string Name() { return "NormalizeAndPermute"; } diff --git a/fastdeploy/vision/common/processors/proc_lib.cc b/fastdeploy/vision/common/processors/proc_lib.cc index 07f8e83fd01..a09ed0ab5da 100644 --- a/fastdeploy/vision/common/processors/proc_lib.cc +++ b/fastdeploy/vision/common/processors/proc_lib.cc @@ -30,6 +30,9 @@ std::ostream& operator<<(std::ostream& out, const ProcLib& p) { case ProcLib::FLYCV: out << "ProcLib::FLYCV"; break; + case ProcLib::CUDA: + out << "ProcLib::CUDA"; + break; default: FDASSERT(false, "Unknow type of ProcLib."); } diff --git a/fastdeploy/vision/common/processors/proc_lib.h b/fastdeploy/vision/common/processors/proc_lib.h index deee4b4f23a..512ed9f837a 100644 --- a/fastdeploy/vision/common/processors/proc_lib.h +++ b/fastdeploy/vision/common/processors/proc_lib.h @@ -18,7 +18,7 @@ namespace fastdeploy { namespace vision { -enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV }; +enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV, CUDA }; FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const ProcLib& p); diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/fastdeploy/vision/detection/ppdet/ppyoloe.cc index 00a82ace528..1ae6294baf7 100755 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ b/fastdeploy/vision/detection/ppdet/ppyoloe.cc @@ -146,7 +146,6 @@ bool PPYOLOE::BuildPreprocessPipelineFromConfig() { // Fusion will improve performance FuseTransforms(&processors_); - return true; } @@ -161,8 +160,6 @@ bool PPYOLOE::Preprocess(Mat* mat, std::vector* outputs) { } } - Cast::Run(mat, "float"); - outputs->resize(2); (*outputs)[0].name = InputInfoOfRuntime(0).name; mat->ShareWithTensor(&((*outputs)[0])); diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index f2391199409..3d3b158a3a1 100644 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -39,6 +39,8 @@ PaddleSegModel::PaddleSegModel(const std::string& model_file, } bool PaddleSegModel::Initialize() { + reused_input_tensors_.resize(1); + reused_output_tensors_.resize(1); if (!BuildPreprocessPipelineFromConfig()) { FDERROR << "Failed to build preprocess pipeline from configuration file." << std::endl; @@ -133,6 +135,9 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() { if (!(this->disable_normalize_and_permute)) { processors_.push_back(std::make_shared()); } + + // Fusion will improve performance + FuseTransforms(&processors_); return true; } @@ -332,7 +337,6 @@ bool PaddleSegModel::Postprocess( bool PaddleSegModel::Predict(cv::Mat* im, SegmentationResult* result) { Mat mat(*im); - std::vector processed_data(1); std::map> im_info; @@ -340,18 +344,18 @@ bool PaddleSegModel::Predict(cv::Mat* im, SegmentationResult* result) { im_info["input_shape"] = {static_cast(mat.Height()), static_cast(mat.Width())}; - if (!Preprocess(&mat, &(processed_data[0]))) { + if (!Preprocess(&mat, &(reused_input_tensors_[0]))) { FDERROR << "Failed to preprocess input data while using model:" << ModelName() << "." << std::endl; return false; } - std::vector infer_result(1); - if (!Infer(processed_data, &infer_result)) { + + if (!Infer()) { FDERROR << "Failed to inference while using model:" << ModelName() << "." << std::endl; return false; } - if (!Postprocess(&infer_result[0], result, im_info)) { + if (!Postprocess(&reused_output_tensors_[0], result, im_info)) { FDERROR << "Failed to postprocess while using model:" << ModelName() << "." << std::endl; return false; diff --git a/python/fastdeploy/vision/classification/ppcls/__init__.py b/python/fastdeploy/vision/classification/ppcls/__init__.py index 879d5444142..75f20af38b2 100644 --- a/python/fastdeploy/vision/classification/ppcls/__init__.py +++ b/python/fastdeploy/vision/classification/ppcls/__init__.py @@ -35,6 +35,13 @@ def run(self, input_ims): """ return self._preprocessor.run(input_ims) + def use_gpu(self, gpu_id=-1): + """Use CUDA preprocessors + + :param: gpu_id: GPU device id + """ + return self._preprocessor.use_gpu(gpu_id) + class PaddleClasPostprocessor: def __init__(self, topk=1): From e26730328b7a46b702524c9e5c1e1448e817c0d3 Mon Sep 17 00:00:00 2001 From: ziqi-jin <67993288+ziqi-jin@users.noreply.github.com> Date: Mon, 14 Nov 2022 18:44:33 +0800 Subject: [PATCH 020/136] [Doc] Fix dead links (#584) * first commit for yolov7 * pybind for yolov7 * CPP README.md * CPP README.md * modified yolov7.cc * README.md * python file modify * delete license in fastdeploy/ * repush the conflict part * README.md modified * README.md modified * file path modified * file path modified * file path modified * file path modified * file path modified * README modified * README modified * move some helpers to private * add examples for yolov7 * api.md modified * api.md modified * api.md modified * YOLOv7 * yolov7 release link * yolov7 release link * yolov7 release link * copyright * change some helpers to private * change variables to const and fix documents. * gitignore * Transfer some funtions to private member of class * Transfer some funtions to private member of class * Merge from develop (#9) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * first commit for yolor * for merge * Develop (#11) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Yolor (#16) * Develop (#11) (#12) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Develop (#13) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * Develop (#14) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> * add is_dynamic for YOLO series (#22) * modify ppmatting backend and docs * modify ppmatting docs * fix the PPMatting size problem * fix LimitShort's log * retrigger ci * modify PPMatting docs * modify the way for dealing with LimitShort * add python comments for external models * modify resnet c++ comments * modify C++ comments for external models * modify python comments and add result class comments * fix comments compile error * modify result.h comments * deadlink check * deadlink check * deadlink check Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> --- README_CN.md | 24 ++++++++-------- README_EN.md | 28 +++++++++---------- docs/api_docs/cpp/main_page.md | 12 ++++---- examples/runtime/cpp/README.md | 8 +++--- examples/runtime/python/README.md | 6 ++-- examples/text/uie/cpp/README.md | 3 +- examples/text/uie/python/README.md | 2 +- .../rk1126/picodet_detection/README.md | 6 ++-- .../detection/yolov5/quantize/README_EN.md | 2 +- .../detection/yolov6/quantize/README_EN.md | 2 +- .../detection/yolov7/quantize/README_EN.md | 2 +- examples/vision/ocr/PP-OCRv3/web/README.md | 2 +- serving/docs/EN/model_repository-en.md | 4 +-- serving/docs/zh_CN/model_configuration.md | 2 +- serving/docs/zh_CN/model_repository.md | 4 +-- 15 files changed, 53 insertions(+), 54 deletions(-) diff --git a/README_CN.md b/README_CN.md index 30e50f4c5de..00a6fe1eda7 100644 --- a/README_CN.md +++ b/README_CN.md @@ -39,7 +39,7 @@ - 🔥 **2022.11.8：Release FastDeploy [release v0.6.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.6.0)** - - **🖥️ 服务端部署：支持推理速度更快的后端，支持更多的模型** + - **🖥️ 服务端部署：支持推理速度更快的后端，支持更多的模型** - 优化 YOLO系列、PaddleClas、PaddleDetection 前后处理内存创建逻辑； - 融合视觉预处理操作，优化PaddleClas、PaddleDetection预处理性能，提升端到端推理性能； - 服务化部署新增Clone接口支持，降低Paddle Inference/TensorRT/OpenVINO后端在多实例下内存/显存的使用； @@ -47,13 +47,13 @@ - **📲 移动端和端侧部署：移动端后端能力升级，支持更多的CV模型** - 集成 RKNPU2 后端，并提供与 Paddle Inference、Paddle Inference TensorRT、TensorRT、OpenVINO、ONNX Runtime、Paddle Lite 等推理后端一致的开发体验； - 支持 [PP-HumanSeg](./examples/vision/segmentation/paddleseg/rknpu2)、[Unet](./examples/vision/segmentation/paddleseg/rknpu2)、[PicoDet](examples/vision/detection/paddledetection/rknpu2)、[SCRFD](./examples/vision/facedet/scrfd/rknpu2) 等在NPU高需求的特色模型。 - + - [**more releases information**](./releases) ## 目录 *
📖 文档教程（点击可收缩）
- + - 安装文档 - [预编译库下载安装](docs/cn/build_and_install/download_prebuilt_libraries.md) - [GPU部署环境编译安装](docs/cn/build_and_install/gpu.md) @@ -112,7 +112,7 @@ ```bash pip install numpy opencv-python fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html ``` -##### [Conda安装(推荐)](docs/quick_start/Python_prebuilt_wheels.md) +##### [Conda安装(推荐)](docs/cn/build_and_install/download_prebuilt_libraries.md) ```bash conda config --add channels conda-forge && conda install cudatoolkit=11.2 cudnn=8.2 ``` @@ -155,7 +155,7 @@ cv2.imwrite("vis_image.jpg", vis_im)
C++ SDK快速开始（点开查看详情）
- + #### 安装 @@ -211,7 +211,7 @@ int main(int argc, char* argv[]) { |:----------------------:|:--------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------:|:-------:|:----------:|:-------:|:----------:|:-------:|:-------:|:-----------:|:-------------:|:-------------:|:-------:| | --- | --- | --- | X86 CPU | NVIDIA GPU | X86 CPU | NVIDIA GPU | X86 CPU | Arm CPU | AArch64 CPU | NVIDIA Jetson | Graphcore IPU | Serving | | Classification | [PaddleClas/ResNet50](./examples/vision/classification/paddleclas) | [Python](./examples/vision/classification/paddleclas/python)/[C++](./examples/vision/classification/paddleclas/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | -| Classification | [TorchVison/ResNet](examples/vision/classification/resnet) | [Python](./examples/vision/classification/resnet/python)/[C++](./examples/vision/classification/resnet/python/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | +| Classification | [TorchVison/ResNet](examples/vision/classification/resnet) | [Python](./examples/vision/classification/resnet/python)/[C++](./examples/vision/classification/resnet/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Classification | [ltralytics/YOLOv5Cls](examples/vision/classification/yolov5cls) | [Python](./examples/vision/classification/yolov5cls/python)/[C++](./examples/vision/classification/yolov5cls/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Classification | [PaddleClas/PP-LCNet](./examples/vision/classification/paddleclas) | [Python](./examples/vision/classification/paddleclas/python)/[C++](./examples/vision/classification/paddleclas/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | | Classification | [PaddleClas/PP-LCNetv2](./examples/vision/classification/paddleclas) | [Python](./examples/vision/classification/paddleclas/python)/[C++](./examples/vision/classification/paddleclas/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | @@ -243,9 +243,9 @@ int main(int argc, char* argv[]) { | Detection | [WongKinYiu/ScaledYOLOv4](./examples/vision/detection/scaledyolov4) | [Python](./examples/vision/detection/scaledyolov4/python)/[C++](./examples/vision/detection/scaledyolov4/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Detection | [ppogg/YOLOv5Lite](./examples/vision/detection/yolov5lite) | [Python](./examples/vision/detection/yolov5lite/python)/[C++](./examples/vision/detection/yolov5lite/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Detection | [RangiLyu/NanoDetPlus](./examples/vision/detection/nanodet_plus) | [Python](./examples/vision/detection/nanodet_plus/python)/[C++](./examples/vision/detection/nanodet_plus/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | -| KeyPoint | [PaddleDetection/TinyPose](./examples/vision/keypointdetection/tiny_pose) | [Python](./examples/vision/keypointdetection/tiny_pose/python)/[C++](./examples/vision/keypointdetection/tiny_pose/python/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | +| KeyPoint | [PaddleDetection/TinyPose](./examples/vision/keypointdetection/tiny_pose) | [Python](./examples/vision/keypointdetection/tiny_pose/python)/[C++](./examples/vision/keypointdetection/tiny_pose/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | KeyPoint | [PaddleDetection/PicoDet + TinyPose](./examples/vision/keypointdetection/det_keypoint_unite) | [Python](./examples/vision/keypointdetection/det_keypoint_unite/python)/[C++](./examples/vision/keypointdetection/det_keypoint_unite/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | -| HeadPose | [omasaht/headpose](examples/vision/headpose) | [Python](./xamples/vision/headpose/fsanet/python)/[C++](./xamples/vision/headpose/fsanet/cpp/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | +| HeadPose | [omasaht/headpose](examples/vision/headpose) | [Python](./examples/vision/headpose/fsanet/python)/[C++](./examples/vision/headpose/fsanet/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Tracking | [PaddleDetection/PP-Tracking](examples/vision/tracking/pptracking) | [Python](examples/vision/tracking/pptracking/python)/[C++](examples/vision/tracking/pptracking/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | OCR | [PaddleOCR/PP-OCRv2](./examples/vision/ocr) | [Python](./examples/vision/detection/nanodet_plus/python)/[C++](./examples/vision/ocr/PP-OCRv3/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | OCR | [PaddleOCR/PP-OCRv3](./examples/vision/ocr) | [Python](./examples/vision/ocr/PP-OCRv3/python)/[C++](./examples/vision/ocr/PP-OCRv3/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | @@ -331,12 +331,12 @@ int main(int argc, char* argv[]) { | OCR | [PaddleOCR/PP-OCRv2](examples/vision/ocr/PP-OCRv2) | 2.3+4.4 | ✅ | ❔ | ❔ | ❔ | -- | -- | -- | -- | | OCR | [PaddleOCR/PP-OCRv3](examples/vision/ocr/PP-OCRv3) | 2.4+10.6 | ✅ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | -- | | OCR | PaddleOCR/PP-OCRv3-tiny | 2.4+10.7 | ❔ | ❔ | ❔ | ❔ | -- | -- | -- | -- | - - + + ## 🌐 Web和小程序部署
- + | 任务场景 | 模型 | [web_demo](examples/application/js/web_demo) | |:------------------:|:-------------------------------------------------------------------------------------------:|:--------------------------------------------:| | --- | --- | [Paddle.js](examples/application/js) | @@ -346,7 +346,7 @@ int main(int argc, char* argv[]) { | Object Recognition | [GestureRecognition](examples/application/js/web_demo/src/pages/cv/recognition) | ✅ | | Object Recognition | [ItemIdentification](examples/application/js/web_demo/src/pages/cv/recognition) | ✅ | | OCR | [PaddleOCR/PP-OCRv3](./examples/application/js/web_demo/src/pages/cv/ocr) | ✅ | - +
## 社区交流 diff --git a/README_EN.md b/README_EN.md index b89e7f125e1..b439b2c1b82 100644 --- a/README_EN.md +++ b/README_EN.md @@ -38,7 +38,7 @@ Including image classification, object detection, image segmentation, face detec
- + - 🔥 **2022.11.8：Release FastDeploy [release v0.6.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.6.0)**
- **🖥️ Server-side and Cloud Deployment: Support more backend, Support more CV models** - Optimize preprocessing and postprocessing memory creation logic on YOLO series, PaddleClas, PaddleDetection; @@ -54,7 +54,7 @@ Including image classification, object detection, image segmentation, face detec ## Contents *
📖 Tutorials（click to fold）
- + - Install - [How to Install FastDeploy Prebuilt Libraries](docs/en/build_and_install/download_prebuilt_libraries.md) - [How to Build and Install FastDeploy Library on GPU Platform](docs/en/build_and_install/gpu.md) @@ -158,7 +158,7 @@ vis_im = vision.vis_detection(im, result, score_threshold=0.5) cv2.imwrite("vis_image.jpg", vis_im) ```
- +
@@ -213,13 +213,13 @@ Notes: ✅: already supported; ❔: to be supported in the future; N/A: Not Ava
- + | Task | Model | API | Linux | Linux | Win | Win | Mac | Mac | Linux | Linux | Linux | Linux | |:-----------------------------:|:---------------------------------------------------------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------:|:---------------------:|:------------------------:|:------------------------:|:------------------------:|:-----------------------:|:---------------------:|:--------------------------:|:---------------------------:|:--------------------------:|:---------------------------:| | --- | --- | --- | X86 CPU | NVIDIA GPU | Intel CPU | NVIDIA GPU | Intel CPU | Arm CPU | AArch64 CPU | NVIDIA Jetson | Graphcore IPU | Serving| | Classification | [PaddleClas/ResNet50](./examples/vision/classification/paddleclas) | [Python](./examples/vision/classification/paddleclas/python)/[C++](./examples/vision/classification/paddleclas/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | -| Classification | [TorchVison/ResNet](examples/vision/classification/resnet) | [Python](./examples/vision/classification/resnet/python)/[C++](./examples/vision/classification/resnet/python/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | +| Classification | [TorchVison/ResNet](examples/vision/classification/resnet) | [Python](./examples/vision/classification/resnet/python)/[C++](./examples/vision/classification/resnet/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Classification | [ltralytics/YOLOv5Cls](examples/vision/classification/yolov5cls) | [Python](./examples/vision/classification/yolov5cls/python)/[C++](./examples/vision/classification/yolov5cls/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Classification | [PaddleClas/PP-LCNet](./examples/vision/classification/paddleclas) | [Python](./examples/vision/classification/paddleclas/python)/[C++](./examples/vision/classification/paddleclas/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | | Classification | [PaddleClas/PP-LCNetv2](./examples/vision/classification/paddleclas) | [Python](./examples/vision/classification/paddleclas/python)/[C++](./examples/vision/classification/paddleclas/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | @@ -251,9 +251,9 @@ Notes: ✅: already supported; ❔: to be supported in the future; N/A: Not Ava | Detection | [WongKinYiu/ScaledYOLOv4](./examples/vision/detection/scaledyolov4) | [Python](./examples/vision/detection/scaledyolov4/python)/[C++](./examples/vision/detection/scaledyolov4/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Detection | [ppogg/YOLOv5Lite](./examples/vision/detection/yolov5lite) | [Python](./examples/vision/detection/yolov5lite/python)/[C++](./examples/vision/detection/yolov5lite/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Detection | [RangiLyu/NanoDetPlus](./examples/vision/detection/nanodet_plus) | [Python](./examples/vision/detection/nanodet_plus/python)/[C++](./examples/vision/detection/nanodet_plus/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | -| KeyPoint | [PaddleDetection/TinyPose](./examples/vision/keypointdetection/tiny_pose) | [Python](./examples/vision/keypointdetection/tiny_pose/python)/[C++](./examples/vision/keypointdetection/tiny_pose/python/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | +| KeyPoint | [PaddleDetection/TinyPose](./examples/vision/keypointdetection/tiny_pose) | [Python](./examples/vision/keypointdetection/tiny_pose/python)/[C++](./examples/vision/keypointdetection/tiny_pose/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | KeyPoint | [PaddleDetection/PicoDet + TinyPose](./examples/vision/keypointdetection/det_keypoint_unite) | [Python](./examples/vision/keypointdetection/det_keypoint_unite/python)/[C++](./examples/vision/keypointdetection/det_keypoint_unite/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | -| HeadPose | [omasaht/headpose](examples/vision/headpose) | [Python](./xamples/vision/headpose/fsanet/python)/[C++](./xamples/vision/headpose/fsanet/cpp/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | +| HeadPose | [omasaht/headpose](examples/vision/headpose) | [Python](./examples/vision/headpose/fsanet/python)/[C++](./examples/vision/headpose/fsanet/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | Tracking | [PaddleDetection/PP-Tracking](examples/vision/tracking/pptracking) | [Python](examples/vision/tracking/pptracking/python)/[C++](examples/vision/tracking/pptracking/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | OCR | [PaddleOCR/PP-OCRv2](./examples/vision/ocr) | [Python](./examples/vision/detection/nanodet_plus/python)/[C++](./examples/vision/ocr/PP-OCRv3/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | OCR | [PaddleOCR/PP-OCRv3](./examples/vision/ocr) | [Python](./examples/vision/ocr/PP-OCRv3/python)/[C++](./examples/vision/ocr/PP-OCRv3/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | @@ -280,21 +280,21 @@ Notes: ✅: already supported; ❔: to be supported in the future; N/A: Not Ava | Information Extraction | [PaddleNLP/UIE](./examples/text/uie) | [Python](./examples/text/uie/python)/[C++](./examples/text/uie/cpp) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ❔ | | NLP | [PaddleNLP/ERNIE-3.0](./examples/text/ernie-3.0) | Python/C++ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ✅ | | Speech | [PaddleSpeech/PP-TTS](./examples/text/uie) | [Python](examples/audio/pp-tts/python)/C++ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | -- | ✅ | - +
- + ## 📱 Mobile and Edge Device Deployment
- + ### Paddle Lite NPU Deployment - [Rockchip-NPU / Amlogic-NPU / NXP-NPU](./examples/vision/detection/paddledetection/rk1126)
- + ### Mobile and Edge Model List 🔥🔥🔥🔥
@@ -340,11 +340,11 @@ Notes: ✅: already supported; ❔: to be supported in the future; N/A: Not Ava | OCR | [PaddleOCR/PP-OCRv3](examples/vision/ocr/PP-OCRv3) | 2.4+10.6 | ✅ | ❔ | ❔ | ❔ | ❔ | ❔ | ❔ | -- | | OCR | PaddleOCR/PP-OCRv3-tiny | 2.4+10.7 | ❔ | ❔ | ❔ | ❔ | -- | -- | -- | -- | - + ## 🌐 Browser-based Model List
- + | Task | Model | [web_demo](examples/application/js/web_demo) | |:------------------:|:-------------------------------------------------------------------------------------------:|:--------------------------------------------:| | --- | --- | [Paddle.js](examples/application/js) | @@ -355,7 +355,7 @@ Notes: ✅: already supported; ❔: to be supported in the future; N/A: Not Ava | Object Recognition | [ItemIdentification](examples/application/js/web_demo/src/pages/cv/recognition) | ✅ | | OCR | [PaddleOCR/PP-OCRv3](./examples/application/js/web_demo/src/pages/cv/ocr) | ✅ | - + ## Community
diff --git a/docs/api_docs/cpp/main_page.md b/docs/api_docs/cpp/main_page.md index 017ad7b60d0..585e6d4286a 100644 --- a/docs/api_docs/cpp/main_page.md +++ b/docs/api_docs/cpp/main_page.md @@ -18,14 +18,14 @@ Currently, FastDeploy supported backends listed as below, - [C++ examples](./) ### Related APIs -- [RuntimeOption](./structfastdeploy_1_1RuntimeOption.html) -- [Runtime](./structfastdeploy_1_1Runtime.html) +- [RuntimeOption](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1RuntimeOption.html) +- [Runtime](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1Runtime.html) ## Vision Models | Task | Model | API | Example | | :---- | :---- | :---- | :----- | -| object detection | PaddleDetection/PPYOLOE | [fastdeploy::vision::detection::PPYOLOE](./classfastdeploy_1_1vision_1_1detection_1_1PPYOLOE.html) | [C++](./)/[Python](./) | -| keypoint detection | PaddleDetection/PPTinyPose | [fastdeploy::vision::keypointdetection::PPTinyPose](./classfastdeploy_1_1vision_1_1keypointdetection_1_1PPTinyPose.html) | [C++](./)/[Python](./) | -| image classification | PaddleClassification serials | [fastdeploy::vision::classification::PaddleClasModel](./classfastdeploy_1_1vision_1_1classification_1_1PaddleClasModel.html) | [C++](./)/[Python](./) | -| semantic segmentation | PaddleSegmentation serials | [fastdeploy::vision::classification::PaddleSegModel](./classfastdeploy_1_1vision_1_1segmentation_1_1PaddleSegModel.html) | [C++](./)/[Python](./) | +| object detection | PaddleDetection/PPYOLOE | [fastdeploy::vision::detection::PPYOLOE](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/classfastdeploy_1_1vision_1_1detection_1_1PPYOLOE.html) | [C++](./)/[Python](./) | +| keypoint detection | PaddleDetection/PPTinyPose | [fastdeploy::vision::keypointdetection::PPTinyPose](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/classfastdeploy_1_1pipeline_1_1PPTinyPose.html) | [C++](./)/[Python](./) | +| image classification | PaddleClassification serials | [fastdeploy::vision::classification::PaddleClasModel](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/classfastdeploy_1_1vision_1_1classification_1_1PaddleClasModel.html) | [C++](./)/[Python](./) | +| semantic segmentation | PaddleSegmentation serials | [fastdeploy::vision::classification::PaddleSegModel](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/classfastdeploy_1_1vision_1_1segmentation_1_1PaddleSegModel.html) | [C++](./)/[Python](./) | diff --git a/examples/runtime/cpp/README.md b/examples/runtime/cpp/README.md index 9de8b1d6271..38d25041dd3 100644 --- a/examples/runtime/cpp/README.md +++ b/examples/runtime/cpp/README.md @@ -2,8 +2,8 @@ 在运行demo前，需确认以下两个步骤 -- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) -- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../docs/cn/build_and_install/download_prebuilt_libraries.md) 本文档以 PaddleClas 分类模型 MobileNetV2 为例展示CPU上的推理示例 @@ -113,9 +113,9 @@ make -j source /Path/to/fastdeploy_cpp_sdk/fastdeploy_init.sh ``` -本示例代码在各平台(Windows/Linux/Mac)上通用，但编译过程仅支持(Linux/Mac)，Windows上使用msbuild进行编译，具体使用方式参考[Windows平台使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) +本示例代码在各平台(Windows/Linux/Mac)上通用，但编译过程仅支持(Linux/Mac)，Windows上使用msbuild进行编译，具体使用方式参考[Windows平台使用FastDeploy C++ SDK](../../../docs/cn/faq/use_sdk_on_windows.md) ## 其它文档 - [Runtime Python 示例](../python) -- [切换模型推理的硬件和后端](../../../../../docs/cn/faq/how_to_change_backend.md) +- [切换模型推理的硬件和后端](../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/runtime/python/README.md b/examples/runtime/python/README.md index c9692fca6b1..42f00705185 100644 --- a/examples/runtime/python/README.md +++ b/examples/runtime/python/README.md @@ -2,8 +2,8 @@ 在运行demo前，需确认以下两个步骤 -- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) -- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../docs/cn/build_and_install/download_prebuilt_libraries.md) 本文档以 PaddleClas 分类模型 MobileNetV2 为例展示 CPU 上的推理示例 @@ -50,4 +50,4 @@ print(results[0].shape) ## 其它文档 - [Runtime C++ 示例](../cpp) -- [切换模型推理的硬件和后端](../../../../../docs/cn/faq/how_to_change_backend.md) +- [切换模型推理的硬件和后端](../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/text/uie/cpp/README.md b/examples/text/uie/cpp/README.md index a3b8f801b14..c4fcc7f4f5f 100644 --- a/examples/text/uie/cpp/README.md +++ b/examples/text/uie/cpp/README.md @@ -466,8 +466,7 @@ void Predict( **参数** > * **texts**(list(str)): 文本列表 -> * **results**(list(dict())): UIE模型抽取结果。UIEResult结构详细可见[UIEResult说明](../../../../docs/api/text_results/uie_result.md)。 - +> * **results**(list(dict())): UIE模型抽取结果。 ## 相关文档 [UIE模型详细介绍](https://github.com/PaddlePaddle/PaddleNLP/blob/develop/model_zoo/uie/README.md) diff --git a/examples/text/uie/python/README.md b/examples/text/uie/python/README.md index 099820dbb2b..ca855c762bc 100644 --- a/examples/text/uie/python/README.md +++ b/examples/text/uie/python/README.md @@ -375,7 +375,7 @@ UIEModel模型加载和初始化，其中`model_file`, `params_file`为训练模 > > * **return_dict**(bool): 是否以字典形式输出UIE结果，默认为False。 > **返回** > -> > 返回`dict(str, list(fastdeploy.text.C.UIEResult))`, 详细可见[UIEResult说明](../../../../docs/api/text_results/uie_result.md)。 +> > 返回`dict(str, list(fastdeploy.text.C.UIEResult))`。 ## 相关文档 diff --git a/examples/vision/detection/paddledetection/rk1126/picodet_detection/README.md b/examples/vision/detection/paddledetection/rk1126/picodet_detection/README.md index 735ff1976e2..ddebdeb7add 100755 --- a/examples/vision/detection/paddledetection/rk1126/picodet_detection/README.md +++ b/examples/vision/detection/paddledetection/rk1126/picodet_detection/README.md @@ -118,7 +118,7 @@ Paddle-Lite-Demo/object_detection/linux/picodet_detection/run.sh ## 代码讲解 （使用 Paddle Lite `C++ API` 执行预测） -ARMLinux 示例基于 C++ API 开发，调用 Paddle Lite `C++s API` 包括以下五步。更详细的 `API` 描述参考：[Paddle Lite C++ API ](https://paddle-lite.readthedocs.io/zh/latest/api_reference/c++_api_doc.html)。 +ARMLinux 示例基于 C++ API 开发，调用 Paddle Lite `C++s API` 包括以下五步。更详细的 `API` 描述参考：[Paddle Lite C++ API ](https://paddle-lite.readthedocs.io/zh/latest/api_reference/cxx_api_doc.html)。 ```c++ #include @@ -198,7 +198,7 @@ export LD_LIBRARY_PATH=../Paddle-Lite/libs/$TARGET_ABI/ export GLOG_v=0 # Paddle-Lite 日志等级 export VSI_NN_LOG_LEVEL=0 # TIM-VX 日志等级 export VIV_VX_ENABLE_GRAPH_TRANSFORM=-pcq:1 # NPU 开启 perchannel 量化模型 -export VIV_VX_SET_PER_CHANNEL_ENTROPY=100 # 同上 +export VIV_VX_SET_PER_CHANNEL_ENTROPY=100 # 同上 build/object_detection_demo models/picodetv2_relu6_coco_no_fuse ../../assets/labels/coco_label_list.txt models/picodetv2_relu6_coco_no_fuse/subgraph.txt models/picodetv2_relu6_coco_no_fuse/picodet.yml # 执行 Demo 程序，4个 arg 分别为：模型、 label 文件、 自定义异构配置、 yaml ``` @@ -206,7 +206,7 @@ build/object_detection_demo models/picodetv2_relu6_coco_no_fuse ../../assets/lab ```shell # 代码文件 `object_detection_demo/rush.sh` -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${PADDLE_LITE_DIR}/libs/${TARGET_ARCH_ABI} +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${PADDLE_LITE_DIR}/libs/${TARGET_ARCH_ABI} build/object_detection_demo {模型} {label} {自定义异构配置文件} {yaml} ``` diff --git a/examples/vision/detection/yolov5/quantize/README_EN.md b/examples/vision/detection/yolov5/quantize/README_EN.md index c439470adc8..c704ccce2e7 100644 --- a/examples/vision/detection/yolov5/quantize/README_EN.md +++ b/examples/vision/detection/yolov5/quantize/README_EN.md @@ -6,7 +6,7 @@ Users can use the one-click model quantization tool to quantize and deploy the m ## FastDeploy One-Click Model Quantization Tool FastDeploy provides a one-click quantization tool that allows users to quantize a model simply with a configuration file. -For a detailed tutorial, please refer to: [One-Click Model Quantization Tool](... /... /... /... /... /... /... /tools/quantization/) +For a detailed tutorial, please refer to: [One-Click Model Quantization Tool](../../../../../tools/auto_compression/) ## Download Quantized YOLOv5s Model diff --git a/examples/vision/detection/yolov6/quantize/README_EN.md b/examples/vision/detection/yolov6/quantize/README_EN.md index 5fd3082bcc3..b7ae61d1150 100644 --- a/examples/vision/detection/yolov6/quantize/README_EN.md +++ b/examples/vision/detection/yolov6/quantize/README_EN.md @@ -6,7 +6,7 @@ Users can use the one-click model quantization tool to quantize and deploy the m ## FastDeploy One-Click Model Quantization Tool FastDeploy provides a one-click quantization tool that allows users to quantize a model simply with a configuration file. -For detailed tutorial, please refer to : [One-Click Model Quantization Tool](... /... /... /... /... /... /... /tools/quantization/) +For detailed tutorial, please refer to : [One-Click Model Quantization Tool](../../../../../tools/auto_compression/) ## Download Quantized YOLOv6s Model diff --git a/examples/vision/detection/yolov7/quantize/README_EN.md b/examples/vision/detection/yolov7/quantize/README_EN.md index 039000d9e96..4e6b2e3533f 100644 --- a/examples/vision/detection/yolov7/quantize/README_EN.md +++ b/examples/vision/detection/yolov7/quantize/README_EN.md @@ -6,7 +6,7 @@ Users can use the one-click model quantization tool to quantize and deploy the m ## FastDeploy One-Click Model Quantization Tool FastDeploy provides a one-click quantization tool that allows users to quantize a model simply with a configuration file. -For detailed tutorial, please refer to : [One-Click Model Quantization Tool](... /... /... /... /... /... /... /tools/quantization/) +For detailed tutorial, please refer to : [One-Click Model Quantization Tool](../../../../../tools/auto_compression/) ## Download Quantized YOLOv7 Model diff --git a/examples/vision/ocr/PP-OCRv3/web/README.md b/examples/vision/ocr/PP-OCRv3/web/README.md index 3afd2476127..b13e7547d50 100644 --- a/examples/vision/ocr/PP-OCRv3/web/README.md +++ b/examples/vision/ocr/PP-OCRv3/web/README.md @@ -37,4 +37,4 @@ ocr模型加载和初始化，其中模型为Paddle.js模型格式，js模型转 - [PP-OCRv3 C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) - [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) -- [PP-OCRv3 微信小程序部署文档](../../../../application/web_demo/examples/ocrXcx/) +- [PP-OCRv3 微信小程序部署文档](../mini_program/) diff --git a/serving/docs/EN/model_repository-en.md b/serving/docs/EN/model_repository-en.md index 6d8251549a0..099593be44a 100644 --- a/serving/docs/EN/model_repository-en.md +++ b/serving/docs/EN/model_repository-en.md @@ -1,6 +1,6 @@ # Model Repository -FastDeploy starts the serving by specifying one or more models in the model repository to deploy the service. When the serving is running, the models in the service can be modified following [Model Management](https://github.com/triton-inference-server/server/blob/main/docs/model_management.md), and obtain serving from one or more model repositories specified at the serving initiation. +FastDeploy starts the serving by specifying one or more models in the model repository to deploy the service. When the serving is running, the models in the service can be modified following [Model Management](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md), and obtain serving from one or more model repositories specified at the serving initiation. ## Repository Architecture @@ -39,7 +39,7 @@ Paddle models are saved in the version number subdirectory, which must be `model ## Model Version -Each model can have one or more versions available in the repository. The subdirectory named with a number in the model directory implies the version number. Subdirectories that are not named with a number, or that start with *0* will be ignored. A [version policy](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#version-policy) can be specified in the model configuration file to control which version of the model in model directory is launched by Triton. +Each model can have one or more versions available in the repository. The subdirectory named with a number in the model directory implies the version number. Subdirectories that are not named with a number, or that start with *0* will be ignored. A [version policy](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#version-policy) can be specified in the model configuration file to control which version of the model in model directory is launched by Triton. ## Repository Demo diff --git a/serving/docs/zh_CN/model_configuration.md b/serving/docs/zh_CN/model_configuration.md index ce3abc07591..6ae51af6403 100644 --- a/serving/docs/zh_CN/model_configuration.md +++ b/serving/docs/zh_CN/model_configuration.md @@ -2,7 +2,7 @@ 模型存储库中的每个模型都必须包含一个模型配置，该配置提供了关于模型的必要和可选信息。这些配置信息一般写在 *config.pbtxt* 文件中，[ModelConfig protobuf](https://github.com/triton-inference-server/common/blob/main/protobuf/model_config.proto)格式。 ## 模型通用最小配置 -详细的模型通用配置请看官网文档: [model_configuration](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md).Triton的最小模型配置必须包括: *platform* 或 *backend* 属性、*max_batch_size* 属性和模型的输入输出. +详细的模型通用配置请看官网文档: [model_configuration](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md).Triton的最小模型配置必须包括: *platform* 或 *backend* 属性、*max_batch_size* 属性和模型的输入输出. 例如一个Paddle模型，有两个输入*input0* 和 *input1*，一个输出*output0*，输入输出都是float32类型的tensor，最大batch为8.则最小的配置如下: diff --git a/serving/docs/zh_CN/model_repository.md b/serving/docs/zh_CN/model_repository.md index adff771ffa0..bc46cb35228 100644 --- a/serving/docs/zh_CN/model_repository.md +++ b/serving/docs/zh_CN/model_repository.md @@ -1,6 +1,6 @@ # 模型仓库(Model Repository) -FastDeploy启动服务时指定模型仓库中一个或多个模型部署服务。当服务运行时，可以用[Model Management](https://github.com/triton-inference-server/server/blob/main/docs/model_management.md)中描述的方式修改服务中的模型。 +FastDeploy启动服务时指定模型仓库中一个或多个模型部署服务。当服务运行时，可以用[Model Management](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_management.md)中描述的方式修改服务中的模型。 从服务器启动时指定的一个或多个模型存储库中为模型提供服务 ## 仓库结构 @@ -36,7 +36,7 @@ $ fastdeploy --model-repository= Paddle模型存在版本号子目录中，必须为`model.pdmodel`文件和`model.pdiparams`文件。 ## 模型版本 -每个模型在仓库中可以有一个或多个可用的版本，模型目录中以数字命名的子目录就是对应的版本，数字即版本号。没有以数字命名的子目录，或以*0*开头的子目录都会被忽略。模型配置文件中可以指定[版本策略](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#version-policy)，控制Triton启动模型目录中的哪个版本。 +每个模型在仓库中可以有一个或多个可用的版本，模型目录中以数字命名的子目录就是对应的版本，数字即版本号。没有以数字命名的子目录，或以*0*开头的子目录都会被忽略。模型配置文件中可以指定[版本策略](https://github.com/triton-inference-server/server/blob/main/docs/user_guide/model_configuration.md#version-policy)，控制Triton启动模型目录中的哪个版本。 ## 模型仓库示例 部署Paddle模型时需要的模型必须是2.0版本以上导出的推理模型，模型包含`model.pdmodel`和`model.pdiparams`两个文件放在版本目录中。 From 08cb45eb4b6877948d340c9bc4e6a5e888e47a27 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 14 Nov 2022 19:01:12 +0800 Subject: [PATCH 021/136] [Bug Fix] Fix preprocessor pybind api bug (#585) * Fix preprocessor pybind api bug * Update fd_tensor.cc --- fastdeploy/core/fd_tensor.cc | 23 ++++++++++--------- fastdeploy/core/fd_tensor.h | 14 ++++++++++- .../classification/ppcls/ppcls_pybind.cc | 3 +++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/fastdeploy/core/fd_tensor.cc b/fastdeploy/core/fd_tensor.cc index 8a39421049a..a3544756b37 100644 --- a/fastdeploy/core/fd_tensor.cc +++ b/fastdeploy/core/fd_tensor.cc @@ -120,11 +120,12 @@ void FDTensor::Resize(const std::vector& new_shape) { int numel = Numel(); int new_numel = std::accumulate(new_shape.begin(), new_shape.end(), 1, std::multiplies()); - if (new_numel > numel) { + if (new_numel > numel || external_data_ptr != nullptr) { size_t nbytes = new_numel * FDDataTypeSize(dtype); ReallocFn(nbytes); } shape.assign(new_shape.begin(), new_shape.end()); + external_data_ptr = nullptr; } void FDTensor::Resize(const std::vector& new_shape, @@ -143,9 +144,9 @@ void FDTensor::Resize(const std::vector& new_shape, } template -void CalculateStatisInfo(void* src_ptr, int size, double* mean, double* max, +void CalculateStatisInfo(const void* src_ptr, int size, double* mean, double* max, double* min) { - T* ptr = static_cast(src_ptr); + const T* ptr = static_cast(src_ptr); *mean = 0; *max = -99999999; *min = 99999999; @@ -161,24 +162,24 @@ void CalculateStatisInfo(void* src_ptr, int size, double* mean, double* max, *mean = *mean / size; } -void FDTensor::PrintInfo(const std::string& prefix) { +void FDTensor::PrintInfo(const std::string& prefix) const { double mean = 0; double max = -99999999; double min = 99999999; if (dtype == FDDataType::FP32) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else if (dtype == FDDataType::FP64) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else if (dtype == FDDataType::INT8) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else if (dtype == FDDataType::UINT8) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else if (dtype == FDDataType::INT32) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else if (dtype == FDDataType::INT64) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else if (dtype == FDDataType::FP16) { - CalculateStatisInfo(Data(), Numel(), &mean, &max, &min); + CalculateStatisInfo(CpuData(), Numel(), &mean, &max, &min); } else { FDASSERT(false, "PrintInfo function doesn't support current situation, maybe you " diff --git a/fastdeploy/core/fd_tensor.h b/fastdeploy/core/fd_tensor.h index b08e5f12ecc..32a0da86798 100644 --- a/fastdeploy/core/fd_tensor.h +++ b/fastdeploy/core/fd_tensor.h @@ -57,6 +57,18 @@ struct FASTDEPLOY_DECL FDTensor { void* Data(); + bool IsShared() { + return external_data_ptr != nullptr; + } + + void StopSharing() { + if (IsShared()) { + ReallocFn(Nbytes()); + CopyBuffer(buffer_, external_data_ptr, Nbytes()); + external_data_ptr = nullptr; + } + } + const void* Data() const; // Use this data to get the tensor data to process @@ -113,7 +125,7 @@ struct FASTDEPLOY_DECL FDTensor { // Debug function // Use this function to print shape, dtype, mean, max, min // prefix will also be printed as tag - void PrintInfo(const std::string& prefix = "TensorInfo: "); + void PrintInfo(const std::string& prefix = "TensorInfo: ") const; bool ReallocFn(size_t nbytes); diff --git a/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc b/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc index 74dfce845f0..267042ce054 100644 --- a/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc +++ b/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc @@ -27,6 +27,9 @@ void BindPaddleClas(pybind11::module& m) { if (!self.Run(&images, &outputs)) { pybind11::eval("raise Exception('Failed to preprocess the input data in PaddleClasPreprocessor.')"); } + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].StopSharing(); + } return outputs; }) .def("use_gpu", [](vision::classification::PaddleClasPreprocessor& self, int gpu_id = -1) { From 1f07ab5770c3212266028d87e713e52918b5af34 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Mon, 14 Nov 2022 19:31:20 +0800 Subject: [PATCH 022/136] [Android] Update Android SDK usage docs (#586) * [FlyCV] Add global SetProcLibCpuNumThreads method * [Android] Support segmentation and facedet in Android * [Android] add JNI instance check to j_runtime_option_obj * [Model] fixed ppseg flycv resize error * [FlyCV] fix FlyCV resize flags * [cmake] remove un-need lite compile option * [Android] add PaddleSegModel JNI and fix some bugs * [Android] bind PaddleSegModel via JNI * [Android] bind VisSegmentation via JNI * [Android] bind YOLOv5Face and SCRFD via JNI * [Android] fix NewJavaFaceDetectionResultFromCxx error * [Android] remove un-need option in Java RuntimeOption * [Android] fixed OcrMainActivity setting errors * [Android] Update Android SDK usage docs * [Android] Update Android SDK usage docs --- java/android/README.md | 332 +++++++++++++++++- java/android/app/src/main/AndroidManifest.xml | 4 +- .../detection/DetectionMainActivity.java | 15 +- 3 files changed, 344 insertions(+), 7 deletions(-) diff --git a/java/android/README.md b/java/android/README.md index 42b6f9a7146..faa88749de3 100644 --- a/java/android/README.md +++ b/java/android/README.md @@ -1,3 +1,333 @@ # FastDeploy Android AAR 包使用文档 +FastDeploy Android SDK 目前支持图像分类、目标检测、OCR文字识别、语义分割和人脸检测等任务，对更多的AI任务支持将会陆续添加进来。以下为各个任务对应的API文档，在Android下使用FastDeploy中集成的模型，只需以下几个步骤： +- 模型初始化 +- 调用`predict`接口 +- 可视化验证（可选） -- TODO +## 图像分类 +### PaddleClasModel Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleClasModel初始化参数说明如下： + - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径，如 model.pdiparams + - configFile: String, 模型推理的预处理配置文件，如 infer_cfg.yml + - labelFile: String, 可选参数，表示label标签文件所在路径，用于可视化，如 imagenet1k_label_list.txt，每一行包含一个label + - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + +```java +// 构造函数: constructor w/o label file +public PaddleClasModel(); // 空构造函数，之后可以调用init初始化 +public PaddleClasModel(String modelFile, String paramsFile, String configFile); +public PaddleClasModel(String modelFile, String paramsFile, String configFile, String labelFile); +public PaddleClasModel(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public PaddleClasModel(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public boolean init(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +``` +- 模型预测 API：模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指，不保存图片以及不渲染结果到Bitmap上，仅预测推理结果。预测并且可视化是指，预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测：不保存图片以及不渲染结果到Bitmap上 +public ClassifyResult predict(Bitmap ARGB8888Bitmap)； +// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 +public ClassifyResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold); +public ClassifyResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float scoreThreshold); // 只渲染 不保存图片 +``` +- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` + +## 目标检测 +### PicoDet Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PicoDet初始化参数说明如下： + - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径，如 model.pdiparams + - configFile: String, 模型推理的预处理配置文件，如 infer_cfg.yml + - labelFile: String, 可选参数，表示label标签文件所在路径，用于可视化，如 coco_label_list.txt，每一行包含一个label + - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + +```java +// 构造函数: constructor w/o label file +public PicoDet(); // 空构造函数，之后可以调用init初始化 +public PicoDet(String modelFile, String paramsFile, String configFile); +public PicoDet(String modelFile, String paramsFile, String configFile, String labelFile); +public PicoDet(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public PicoDet(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public boolean init(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +``` +- 模型预测 API：模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指，不保存图片以及不渲染结果到Bitmap上，仅预测推理结果。预测并且可视化是指，预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测：不保存图片以及不渲染结果到Bitmap上 +public DetectionResult predict(Bitmap ARGB8888Bitmap)； +// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 +public DetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold); +public DetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float scoreThreshold); // 只渲染 不保存图片 +``` +- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` + +## OCR文字识别 +### PP-OCRv2 & PP-OCRv3 Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。 PP-OCR初始化参数说明如下： + - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径，如 model.pdiparams + - labelFile: String, 可选参数，表示label标签文件所在路径，用于可视化，如 ppocr_keys_v1.txt，每一行包含一个label + - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 +与其他模型不同的是，PP-OCRv2 和 PP-OCRv3 包含 DBDetector、Classifier和Recognizer等基础模型，以及PPOCRv2和PPOCRv3等pipeline类型。 +```java +// 构造函数: constructor w/o label file +public DBDetector(String modelFile, String paramsFile); +public DBDetector(String modelFile, String paramsFile, RuntimeOption option); +public Classifier(String modelFile, String paramsFile); +public Classifier(String modelFile, String paramsFile, RuntimeOption option); +public Recognizer(String modelFile, String paramsFile, String labelPath); +public Recognizer(String modelFile, String paramsFile, String labelPath, RuntimeOption option); +public PPOCRv2(); // 空构造函数，之后可以调用init初始化 +// Constructor w/o classifier +public PPOCRv2(DBDetector detModel, Recognizer recModel); +public PPOCRv2(DBDetector detModel, Classifier clsModel, Recognizer recModel); +public PPOCRv3(); // 空构造函数，之后可以调用init初始化 +// Constructor w/o classifier +public PPOCRv3(DBDetector detModel, Recognizer recModel); +public PPOCRv3(DBDetector detModel, Classifier clsModel, Recognizer recModel); +``` +- 模型预测 API：模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指，不保存图片以及不渲染结果到Bitmap上，仅预测推理结果。预测并且可视化是指，预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测：不保存图片以及不渲染结果到Bitmap上 +public OCRResult predict(Bitmap ARGB8888Bitmap)； +// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 +public OCRResult predict(Bitmap ARGB8888Bitmap, String savedImagePath); +public OCRResult predict(Bitmap ARGB8888Bitmap, boolean rendering); // 只渲染 不保存图片 +``` +- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` + +## 语义分割 +### PaddleSegModel Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleSegModel初始化参数说明如下： + - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径，如 model.pdiparams + - configFile: String, 模型推理的预处理配置文件，如 infer_cfg.yml + - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + +```java +// 构造函数: constructor w/o label file +public PaddleSegModel(); // 空构造函数，之后可以调用init初始化 +public PaddleSegModel(String modelFile, String paramsFile, String configFile); +public PaddleSegModel(String modelFile, String paramsFile, String configFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, String configFile, RuntimeOption option); +``` +- 模型预测 API：模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指，不保存图片以及不渲染结果到Bitmap上，仅预测推理结果。预测并且可视化是指，预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测：不保存图片以及不渲染结果到Bitmap上 +public SegmentationResult predict(Bitmap ARGB8888Bitmap)； +// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 +public SegmentationResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float weight); +public SegmentationResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float weight); // 只渲染 不保存图片 +``` +- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` + +## 人脸检测 +### SCRFD Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleSegModel初始化参数说明如下： + - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径，如 model.pdiparams + - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + +```java +// 构造函数: constructor w/o label file +public SCRFD(); // 空构造函数，之后可以调用init初始化 +public SCRFD(String modelFile, String paramsFile); +public SCRFD(String modelFile, String paramsFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, RuntimeOption option); +``` +- 模型预测 API：模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指，不保存图片以及不渲染结果到Bitmap上，仅预测推理结果。预测并且可视化是指，预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测：不保存图片以及不渲染结果到Bitmap上 +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap)； +// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold); +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float scoreThreshold); // 只渲染 不保存图片 +``` +- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` + +### YOLOv5Face Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleSegModel初始化参数说明如下： + - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径，如 model.pdiparams + - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + +```java +// 构造函数: constructor w/o label file +public YOLOv5Face(); // 空构造函数，之后可以调用init初始化 +public YOLOv5Face(String modelFile, String paramsFile); +public YOLOv5Face(String modelFile, String paramsFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, RuntimeOption option); +``` +- 模型预测 API：模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指，不保存图片以及不渲染结果到Bitmap上，仅预测推理结果。预测并且可视化是指，预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测：不保存图片以及不渲染结果到Bitmap上 +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap)； +// 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold); +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float scoreThreshold); // 只渲染 不保存图片 +``` +- 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` + +## 结果说明 +- 图像分类ClassifyResult说明 +```java +public class ClassifyResult { + public float[] mScores; // [n] 每个类别的得分(概率) + public int[] mLabelIds; // [n] 分类ID 具体的类别类型 + public boolean initialized(); // 检测结果是否有效 +} +``` +其他参考：C++/Python对应的ClassifyResult说明 [api/vision_results/classification_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/classification_result.md) + +- 目标检测DetectionResult说明 +```java +public class DetectionResult { + public float[][] mBoxes; // [n,4] 检测框 (x1,y1,x2,y2) + public float[] mScores; // [n] 每个检测框得分(置信度，概率值) + public int[] mLabelIds; // [n] 分类ID + public boolean initialized(); // 检测结果是否有效 +} +``` +其他参考：C++/Python对应的DetectionResult说明 [api/vision_results/detection_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/detection_result.md) + +- OCR文字识别OCRResult说明 +```java +public class OCRResult { + public int[][] mBoxes; // [n,8] 表示单张图片检测出来的所有目标框坐标 每个框以8个int数值依次表示框的4个坐标点，顺序为左下，右下，右上，左上 + public String[] mText; // [n] 表示多个文本框内被识别出来的文本内容 + public float[] mRecScores; // [n] 表示文本框内识别出来的文本的置信度 + public float[] mClsScores; // [n] 表示文本框的分类结果的置信度 + public int[] mClsLabels; // [n] 表示文本框的方向分类类别 + public boolean initialized(); // 检测结果是否有效 +} +``` +其他参考：C++/Python对应的OCRResult说明 [api/vision_results/ocr_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/ocr_result.md) + +- 语义分割SegmentationResult结果说明 +```java +public class SegmentationResult { + public int[] mLabelMap; // 预测到的label map 每个像素位置对应一个label HxW + public float[] mScoreMap; // 预测到的得分 map 每个像素位置对应一个score HxW + public long[] mShape; // label map实际的shape (H,W) + public boolean mContainScoreMap = false; // 是否包含 score map + public boolean initialized(); // 检测结果是否有效 +} +``` +其他参考：C++/Python对应的SegmentationResult说明 [api/vision_results/segmentation_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/segmentation_result.md) + +- 人脸检测FaceDetectionResult结果说明 +```java +public class FaceDetectionResult { + public float[][] mBoxes; // [n,4] 检测框 (x1,y1,x2,y2) + public float[] mScores; // [n] 每个检测框得分(置信度，概率值) + public float[][] mLandmarks; // [nx?,2] 每个检测到的人脸对应关键点 + int mLandmarksPerFace = 0; // 每个人脸对应的关键点个数 + public boolean initialized(); // 检测结果是否有效 +} +``` +其他参考：C++/Python对应的FaceDetectionResult说明 [api/vision_results/face_detection_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/face_detection_result.md) + +## RuntimeOption说明 +- RuntimeOption设置说明 +```java +public class RuntimeOption { + public void enableLiteFp16(); // 开启fp16精度推理 + public void disableLiteFP16(); // 关闭fp16精度推理 + public void setCpuThreadNum(int threadNum); // 设置线程数 + public void setLitePowerMode(LitePowerMode mode); // 设置能耗模式 + public void setLitePowerMode(String modeStr); // 通过字符串形式设置能耗模式 +} +``` + +## 可视化接口 +FastDeploy Android SDK同时提供一些可视化接口，可用于快速验证推理结果。以下接口均把结果result渲染在输入的Bitmap上。具体的可视化API接口如下： + +```java +public class Visualize { + public static boolean visClassification(Bitmap ARGB8888Bitmap, ClassifyResult result); + public static boolean visDetection(Bitmap ARGB8888Bitmap, DetectionResult result); + public static boolean visFaceDetection(Bitmap ARGB8888Bitmap, FaceDetectionResult result); + public static boolean visOcr(Bitmap ARGB8888Bitmap, OCRResult result); + public static boolean visSegmentation(Bitmap ARGB8888Bitmap, SegmentationResult result); +} +``` +对应的可视化类型为： +```java +import com.baidu.paddle.fastdeploy.vision.Visualize; +``` + +## 使用示例 +- 模型调用示例1：使用构造函数以及默认的RuntimeOption +```java +import java.nio.ByteBuffer; +import android.graphics.Bitmap; +import android.opengl.GLES20; + +import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; + +// 初始化模型 +PicoDet model = new PicoDet("picodet_s_320_coco_lcnet/model.pdmodel", + "picodet_s_320_coco_lcnet/model.pdiparams", + "picodet_s_320_coco_lcnet/infer_cfg.yml"); + +// 模型推理 +DetectionResult result = model.predict(ARGB8888ImageBitmap); + +// 释放模型资源 +model.release(); +``` + +- 模型调用示例2: 在合适的程序节点，手动调用init，并自定义RuntimeOption +```java +// import 同上 ... +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.LitePowerMode; +import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; +// 新建空模型 +PicoDet model = new PicoDet(); +// 模型路径 +String modelFile = "picodet_s_320_coco_lcnet/model.pdmodel"; +String paramFile = "picodet_s_320_coco_lcnet/model.pdiparams"; +String configFile = "picodet_s_320_coco_lcnet/infer_cfg.yml"; +// 指定RuntimeOption +RuntimeOption option = new RuntimeOption(); +option.setCpuThreadNum(2); +option.setLitePowerMode(LitePowerMode.LITE_POWER_HIGH); +option.enableRecordTimeOfRuntime(); +option.enableLiteFp16(); +// 使用init函数初始化 +model.init(modelFile, paramFile, configFile, option); +// Bitmap读取、模型预测、资源释放 同上 ... +``` diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index d8856b08215..1cdc519e053 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -15,14 +15,14 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/AppTheme"> - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index bc588f37605..8eef10f5cc6 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -68,9 +68,10 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private Bitmap originShutterBitmap; private Bitmap picBitmap; private Bitmap originPicBitmap; + public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; public static final int ALBUM_SELECT = 1; - private static int TYPE = BTN_SHUTTER; + private static int TYPE = TYPE_UNKNOWN; private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; private static final int INTENT_CODE_PICK_IMAGE = 100; @@ -215,11 +216,17 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { synchronized (this) { savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; } - shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + if (TYPE == BTN_SHUTTER) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } else { + // Only reference in detecting loops. + shutterBitmap = ARGB8888ImageBitmap; + originShutterBitmap = ARGB8888ImageBitmap; + } boolean modified = false; DetectionResult result = predictor.predict( - ARGB8888ImageBitmap, savedImagePath, DetectionSettingsActivity.scoreThreshold); + ARGB8888ImageBitmap, true, DetectionSettingsActivity.scoreThreshold); modified = result.initialized(); if (!savedImagePath.isEmpty()) { synchronized (this) { From 3ac7c5d511866fef3b37125a21866c0d5c44615d Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 14 Nov 2022 20:28:31 +0800 Subject: [PATCH 023/136] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1a183e90deb..eb728c2f480 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,15 +1,10 @@ -### PR types(PR类型) - - -The title of this pull request should be `[PR type] Description of this pull request`, e.g `[Model] PP-Matting deployment support` - -PR的标题应该为 `[PR type] Description of this pull request`, 例如 `[Model] PP-Matting deployment support` +### PR types(PR类型) + ### Describe -- Support new model PP-Matting -- Add new data structure `MattingResult` + From 4283fc68687aed4f55f82023528825065da5ad44 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 14 Nov 2022 20:50:01 +0800 Subject: [PATCH 024/136] polish build docs --- docs/cn/build_and_install/cpu.md | 2 ++ docs/cn/build_and_install/gpu.md | 2 ++ docs/cn/build_and_install/ipu.md | 2 ++ docs/cn/build_and_install/jetson.md | 2 ++ docs/cn/build_and_install/rknpu2.md | 2 ++ docs/en/build_and_install/cpu.md | 2 ++ docs/en/build_and_install/gpu.md | 2 ++ docs/en/build_and_install/ipu.md | 2 ++ docs/en/build_and_install/jetson.md | 2 ++ 9 files changed, 18 insertions(+) diff --git a/docs/cn/build_and_install/cpu.md b/docs/cn/build_and_install/cpu.md index bd4d98e0071..7148b97de0e 100644 --- a/docs/cn/build_and_install/cpu.md +++ b/docs/cn/build_and_install/cpu.md @@ -66,6 +66,8 @@ msbuild INSTALL.vcxproj /m /p:Configuration=Release /p:Platform=x64 - cmake >= 3.18.0 - python >= 3.6 +Python打包依赖`wheel`，编译前请先执行`pip install wheel` + 所有编译选项通过环境变量导入 ### Linux & Mac diff --git a/docs/cn/build_and_install/gpu.md b/docs/cn/build_and_install/gpu.md index b87faf002bd..9569f50a070 100644 --- a/docs/cn/build_and_install/gpu.md +++ b/docs/cn/build_and_install/gpu.md @@ -88,6 +88,8 @@ msbuild INSTALL.vcxproj /m /p:Configuration=Release /p:Platform=x64 - cuda >= 11.2 - cudnn >= 8.2 +Python打包依赖`wheel`，编译前请先执行`pip install wheel` + 所有编译选项通过环境变量导入 ```bash diff --git a/docs/cn/build_and_install/ipu.md b/docs/cn/build_and_install/ipu.md index 54da4f952ee..9a0ca93c4d2 100644 --- a/docs/cn/build_and_install/ipu.md +++ b/docs/cn/build_and_install/ipu.md @@ -34,6 +34,8 @@ Linux编译过程同样需要满足 - popart >= 3.0.0 - python >= 3.6 +Python打包依赖`wheel`，编译前请先执行`pip install wheel` + 所有编译选项通过环境变量导入 ```bash diff --git a/docs/cn/build_and_install/jetson.md b/docs/cn/build_and_install/jetson.md index 4e20de529df..3c918a8b483 100644 --- a/docs/cn/build_and_install/jetson.md +++ b/docs/cn/build_and_install/jetson.md @@ -33,6 +33,8 @@ make install - jetpack >= 4.6.1 - python >= 3.6 +Python打包依赖`wheel`，编译前请先执行`pip install wheel` + 所有编译选项通过环境变量导入 ```bash diff --git a/docs/cn/build_and_install/rknpu2.md b/docs/cn/build_and_install/rknpu2.md index 61057252c4f..e395c92bfb7 100644 --- a/docs/cn/build_and_install/rknpu2.md +++ b/docs/cn/build_and_install/rknpu2.md @@ -80,6 +80,8 @@ make install ### 编译Python SDK +Python打包依赖`wheel`，编译前请先执行`pip install wheel` + ```bash git clone https://github.com/PaddlePaddle/FastDeploy.git cd FastDeploy diff --git a/docs/en/build_and_install/cpu.md b/docs/en/build_and_install/cpu.md index 6fc6469bbac..b8a14ef2f91 100644 --- a/docs/en/build_and_install/cpu.md +++ b/docs/en/build_and_install/cpu.md @@ -67,6 +67,8 @@ Prerequisite for Compiling: - cmake >= 3.18.0 - python >= 3.6 +Notice the `wheel` is required if you need to pack a wheel, execute `pip install wheel` first. + All compilation options are introduced via environment variables ### Linux & Mac diff --git a/docs/en/build_and_install/gpu.md b/docs/en/build_and_install/gpu.md index 7d3ad967443..4bbfaf35f36 100644 --- a/docs/en/build_and_install/gpu.md +++ b/docs/en/build_and_install/gpu.md @@ -79,6 +79,8 @@ If you use CMake GUI, please refer to [How to Compile with CMakeGUI + Visual Stu ## How to Build and Install Python SDK +Notice the `wheel` is required if you need to pack a wheel, execute `pip install wheel` first. + ### Linux Prerequisite for Compiling on Linux: diff --git a/docs/en/build_and_install/ipu.md b/docs/en/build_and_install/ipu.md index ca462006e8a..a998ab09f9f 100644 --- a/docs/en/build_and_install/ipu.md +++ b/docs/en/build_and_install/ipu.md @@ -34,6 +34,8 @@ Prerequisite for Compiling on Linux: - popart >= 3.0.0 - python >= 3.6 +Notice the `wheel` is required if you need to pack a wheel, execute `pip install wheel` first. + All compilation options are imported via environment variables ```bash diff --git a/docs/en/build_and_install/jetson.md b/docs/en/build_and_install/jetson.md index dceb5cba717..e606b6d2927 100644 --- a/docs/en/build_and_install/jetson.md +++ b/docs/en/build_and_install/jetson.md @@ -33,6 +33,8 @@ Prerequisite for Compiling on NVIDIA Jetson: - jetpack >= 4.6.1 - python >= 3.6 +Notice the `wheel` is required if you need to pack a wheel, execute `pip install wheel` first. + All compilation options are imported via environment variables ``` From ffb5042c6164c7418450a2e4dcc7d174a9a25460 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Mon, 14 Nov 2022 21:33:16 +0800 Subject: [PATCH 025/136] [Android] fix JNI OCRResult allocate errors (#588) * [FlyCV] Add global SetProcLibCpuNumThreads method * [Android] Support segmentation and facedet in Android * [Android] add JNI instance check to j_runtime_option_obj * [Model] fixed ppseg flycv resize error * [FlyCV] fix FlyCV resize flags * [cmake] remove un-need lite compile option * [Android] add PaddleSegModel JNI and fix some bugs * [Android] bind PaddleSegModel via JNI * [Android] bind VisSegmentation via JNI * [Android] bind YOLOv5Face and SCRFD via JNI * [Android] fix NewJavaFaceDetectionResultFromCxx error * [Android] remove un-need option in Java RuntimeOption * [Android] fixed OcrMainActivity setting errors * [Android] Update Android SDK usage docs * [Android] Update Android SDK usage docs * [Android] fix JNI OCRResult allocate errors --- .../detection/DetectionMainActivity.java | 5 +- .../cpp/fastdeploy_jni/vision/results_jni.cc | 91 ++++++++++++------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 8eef10f5cc6..ec5988f68fb 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -209,9 +209,6 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { - Log.d(TAG, "onTextureChanged: bitmap height: " - + ARGB8888ImageBitmap.getHeight() + " width: " - + ARGB8888ImageBitmap.getWidth()); String savedImagePath = ""; synchronized (this) { savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; @@ -224,10 +221,12 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { shutterBitmap = ARGB8888ImageBitmap; originShutterBitmap = ARGB8888ImageBitmap; } + boolean modified = false; DetectionResult result = predictor.predict( ARGB8888ImageBitmap, true, DetectionSettingsActivity.scoreThreshold); modified = result.initialized(); + if (!savedImagePath.isEmpty()) { synchronized (this) { DetectionMainActivity.this.savedImagePath = "result.jpg"; diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc index 9ba3dc49425..7118489f4d7 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc @@ -30,6 +30,7 @@ bool AllocateJavaClassifyResultFromCxx( // (1) mScores float[] shape (n): [F // (2) mLabelIds int[] shape (n): [I // (3) mInitialized boolean: Z + // Docs: docs/api/vision_results/classification_result.md if (cxx_result == nullptr) { return false; } @@ -85,6 +86,7 @@ bool AllocateJavaDetectionResultFromCxx( // (2) mScores float[] shape (n): [F // (3) mLabelIds int[] shape (n): [I // (4) mInitialized boolean: Z + // Docs: docs/api/vision_results/detection_result.md if (cxx_result == nullptr) { return false; } @@ -138,7 +140,7 @@ bool AllocateJavaDetectionResultFromCxx( env->SetObjectField(j_det_result_obj, j_det_label_ids_id, j_det_label_ids_int_arr); env->SetBooleanField(j_det_result_obj, j_det_initialized_id, JNI_TRUE); - // Release non static local Refs + // Release local Refs env->DeleteLocalRef(j_det_boxes_float_arr); env->DeleteLocalRef(j_det_scores_float_arr); env->DeleteLocalRef(j_det_label_ids_int_arr); @@ -158,6 +160,7 @@ bool AllocateJavaOCRResultFromCxx( // (4) mClsScores float[] shape (n): [F // (5) mClsLabels int[] shape (n): [I // (6) mInitialized boolean: Z + // Docs: docs/api/vision_results/ocr_result.md if (cxx_result == nullptr) { return false; } @@ -212,30 +215,35 @@ bool AllocateJavaOCRResultFromCxx( jfloatArray j_ocr_rec_scores_float_arr = env->NewFloatArray(len); env->SetFloatArrayRegion(j_ocr_rec_scores_float_arr, 0, len, rec_scores.data()); - // mClsScores float[] shape (n): [F - const auto &cls_scores = c_result_ptr->cls_scores; - jfloatArray j_ocr_cls_scores_float_arr = env->NewFloatArray(len); - env->SetFloatArrayRegion(j_ocr_cls_scores_float_arr, 0, len, cls_scores.data()); + const int cls_len = static_cast(c_result_ptr->cls_scores.size()); + if (cls_len > 0) { + // mClsScores float[] shape (n): [F + const auto &cls_scores = c_result_ptr->cls_scores; + jfloatArray j_ocr_cls_scores_float_arr = env->NewFloatArray(cls_len); + env->SetFloatArrayRegion(j_ocr_cls_scores_float_arr, 0, cls_len, cls_scores.data()); + + // mClsLabels int[] shape (n): [I + const auto &cls_labels = c_result_ptr->cls_labels; + jintArray j_ocr_cls_labels_int_arr = env->NewIntArray(cls_len); + env->SetIntArrayRegion(j_ocr_cls_labels_int_arr, 0, cls_len, cls_labels.data()); - // mClsLabels int[] shape (n): [I - const auto &cls_labels = c_result_ptr->cls_labels; - jintArray j_ocr_cls_labels_int_arr = env->NewIntArray(len); - env->SetIntArrayRegion(j_ocr_cls_labels_int_arr, 0, len, cls_labels.data()); + env->SetObjectField(j_ocr_result_obj, j_ocr_cls_scores_id, j_ocr_cls_scores_float_arr); + env->SetObjectField(j_ocr_result_obj, j_ocr_cls_labels_id, j_ocr_cls_labels_int_arr); + + env->DeleteLocalRef(j_ocr_cls_scores_float_arr); + env->DeleteLocalRef(j_ocr_cls_labels_int_arr); + } // Set object fields env->SetObjectField(j_ocr_result_obj, j_ocr_boxes_id, j_ocr_boxes_int_arr); env->SetObjectField(j_ocr_result_obj, j_ocr_text_id, j_ocr_text_arr); env->SetObjectField(j_ocr_result_obj, j_ocr_rec_scores_id, j_ocr_rec_scores_float_arr); - env->SetObjectField(j_ocr_result_obj, j_ocr_cls_scores_id, j_ocr_cls_scores_float_arr); - env->SetObjectField(j_ocr_result_obj, j_ocr_cls_labels_id, j_ocr_cls_labels_int_arr); env->SetBooleanField(j_ocr_result_obj, j_ocr_initialized_id, JNI_TRUE); - // Release non static local Refs + // Release local Refs env->DeleteLocalRef(j_ocr_boxes_int_arr); env->DeleteLocalRef(j_ocr_text_arr); env->DeleteLocalRef(j_ocr_rec_scores_float_arr); - env->DeleteLocalRef(j_ocr_cls_scores_float_arr); - env->DeleteLocalRef(j_ocr_cls_labels_int_arr); env->DeleteLocalRef(j_ocr_result_clazz); env->DeleteLocalRef(j_ocr_int_arr_clazz); env->DeleteLocalRef(j_ocr_str_clazz); @@ -253,6 +261,7 @@ bool AllocateJavaSegmentationResultFromCxx( // (3) mContainScoreMap boolean: Z // (4) mScoreMap float[] shape (n): [F // (5) mInitialized boolean: Z + // Docs: docs/api/vision_results/segmentation_result.md if (cxx_result == nullptr) { return false; } @@ -314,7 +323,7 @@ bool AllocateJavaSegmentationResultFromCxx( env->SetObjectField(j_seg_result_obj, j_seg_shape_id, j_seg_shape_long_arr); env->SetBooleanField(j_seg_result_obj, j_seg_initialized_id, JNI_TRUE); - // Release non static local Refs + // Release local Refs env->DeleteLocalRef(j_seg_label_map_int_arr); env->DeleteLocalRef(j_seg_shape_long_arr); env->DeleteLocalRef(j_seg_result_clazz); @@ -332,6 +341,7 @@ bool AllocateJavaFaceDetectionResultFromCxx( // (3) mLandmarks float[][] shape (n,2): [[F // (4) mLandmarksPerFace int: I // (5) mInitialized boolean: Z + // Docs: docs/api/vision_results/face_detection_result.md if (cxx_result == nullptr) { return false; } @@ -521,6 +531,7 @@ bool AllocateClassifyResultFromJava( // (1) mScores float[] shape (n): [F // (2) mLabelIds int[] shape (n): [I // (3) mInitialized boolean: Z + // Docs: docs/api/vision_results/classification_result.md if (cxx_result == nullptr || j_cls_result_obj == nullptr) { return false; } @@ -573,6 +584,7 @@ bool AllocateClassifyResultFromJava( std::memcpy(c_result_ptr->label_ids.data(), j_cls_label_ids_ptr, len * sizeof(int)); env->ReleaseIntArrayElements(j_cls_label_ids_int_arr, j_cls_label_ids_ptr, 0); + // Release local Refs env->DeleteLocalRef(j_cls_result_clazz_cc); return true; @@ -587,6 +599,7 @@ bool AllocateDetectionResultFromJava( // (2) mScores float[] shape (n): [F // (3) mLabelIds int[] shape (n): [I // (4) mInitialized boolean: Z + // Docs: docs/api/vision_results/detection_result.md if (cxx_result == nullptr || j_det_result_obj == nullptr) { return false; } @@ -662,6 +675,7 @@ bool AllocateDetectionResultFromJava( std::memcpy(c_result_ptr->label_ids.data(), j_det_label_ids_ptr, len * sizeof(int)); env->ReleaseIntArrayElements(j_det_label_ids_int_arr, j_det_label_ids_ptr, 0); + // Release local Refs env->DeleteLocalRef(j_det_result_clazz_cc); return true; @@ -677,6 +691,7 @@ bool AllocateOCRResultFromJava( // (4) mClsScores float[] shape (n): [F // (5) mClsLabels int[] shape (n): [I // (6) mInitialized boolean: Z + // Docs: docs/api/vision_results/ocr_result.md if (cxx_result == nullptr || j_ocr_result_obj == nullptr) { return false; } @@ -721,9 +736,12 @@ bool AllocateOCRResultFromJava( const int len = env->GetArrayLength(j_ocr_boxes_arr); if ((len == 0) || (len != env->GetArrayLength(j_ocr_text_arr)) || - (len != env->GetArrayLength(j_ocr_rec_scores_float_arr)) || - (len != env->GetArrayLength(j_ocr_cls_scores_float_arr)) || - (len != env->GetArrayLength(j_ocr_cls_labels_int_arr))) { + (len != env->GetArrayLength(j_ocr_rec_scores_float_arr))){ + return false; + } + + const int cls_len = env->GetArrayLength(j_ocr_cls_scores_float_arr); + if (cls_len != env->GetArrayLength(j_ocr_cls_labels_int_arr)) { return false; } @@ -731,8 +749,11 @@ bool AllocateOCRResultFromJava( c_result_ptr->Clear(); c_result_ptr->boxes.resize(len); c_result_ptr->rec_scores.resize(len); - c_result_ptr->cls_scores.resize(len); - c_result_ptr->cls_labels.resize(len); + + if (cls_len > 0) { + c_result_ptr->cls_scores.resize(cls_len); + c_result_ptr->cls_labels.resize(cls_len); + } // mBoxes int[][] shape (n,8): [[I bool c_check_validation = true; @@ -765,18 +786,21 @@ bool AllocateOCRResultFromJava( std::memcpy(c_result_ptr->rec_scores.data(), j_ocr_rec_scores_ptr, len * sizeof(float)); env->ReleaseFloatArrayElements(j_ocr_rec_scores_float_arr, j_ocr_rec_scores_ptr, 0); - // mClsScores float[] shape (n): [F - jfloat *j_ocr_cls_scores_ptr = - env->GetFloatArrayElements(j_ocr_cls_scores_float_arr, nullptr); - std::memcpy(c_result_ptr->rec_scores.data(), j_ocr_cls_scores_ptr, len * sizeof(float)); - env->ReleaseFloatArrayElements(j_ocr_cls_scores_float_arr, j_ocr_cls_scores_ptr, 0); - - // mClsLabels int[] shape (n): [I - jint *j_ocr_cls_labels_ptr = - env->GetIntArrayElements(j_ocr_cls_labels_int_arr, nullptr); - std::memcpy(c_result_ptr->cls_labels.data(), j_ocr_cls_labels_ptr, len * sizeof(int)); - env->ReleaseIntArrayElements(j_ocr_cls_labels_int_arr, j_ocr_cls_labels_ptr,0); + if (cls_len > 0) { + // mClsScores float[] shape (n): [F + jfloat *j_ocr_cls_scores_ptr = + env->GetFloatArrayElements(j_ocr_cls_scores_float_arr, nullptr); + std::memcpy(c_result_ptr->cls_scores.data(), j_ocr_cls_scores_ptr, cls_len * sizeof(float)); + env->ReleaseFloatArrayElements(j_ocr_cls_scores_float_arr, j_ocr_cls_scores_ptr, 0); + + // mClsLabels int[] shape (n): [I + jint *j_ocr_cls_labels_ptr = + env->GetIntArrayElements(j_ocr_cls_labels_int_arr, nullptr); + std::memcpy(c_result_ptr->cls_labels.data(), j_ocr_cls_labels_ptr, cls_len * sizeof(int)); + env->ReleaseIntArrayElements(j_ocr_cls_labels_int_arr, j_ocr_cls_labels_ptr,0); + } + // Release local Refs env->DeleteLocalRef(j_ocr_result_clazz_cc); return true; @@ -791,6 +815,7 @@ bool AllocateSegmentationResultFromJava( // (3) mContainScoreMap boolean: Z // (4) mScoreMap float[] shape (n): [F // (5) mInitialized boolean: Z + // Docs: docs/api/vision_results/segmentation_result.md if (cxx_result == nullptr || j_seg_result_obj == nullptr) { return false; } @@ -864,6 +889,7 @@ bool AllocateSegmentationResultFromJava( env->ReleaseFloatArrayElements(j_seg_score_map_float_arr, j_seg_score_map_float_ptr, 0); } + // Release local Refs env->DeleteLocalRef(j_seg_result_clazz_cc); return true; @@ -878,6 +904,7 @@ bool AllocateFaceDetectionResultFromJava( // (3) mLandmarks float[][] shape (n,2): [[F // (4) mLandmarksPerFace int: I // (5) mInitialized boolean: Z + // Docs: docs/api/vision_results/face_detection_result.md if (cxx_result == nullptr || j_face_det_result_obj == nullptr) { return false; } @@ -973,12 +1000,12 @@ bool AllocateFaceDetectionResultFromJava( return false; } + // Release local Refs env->DeleteLocalRef(j_face_det_result_clazz_cc); return true; } - bool AllocateCxxResultFromJava( JNIEnv *env, jobject j_result_obj, void *cxx_result, vision::ResultType type) { From 56fa16f80a8399703da8291d363bc5acff5325c6 Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Mon, 14 Nov 2022 21:33:33 +0800 Subject: [PATCH 026/136] [Other] Support batch eval for detection (#587) * add onnx_ort_runtime demo * rm in requirements * support batch eval --- .../fastdeploy/vision/evaluation/detection.py | 53 ++++++++++++++----- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/python/fastdeploy/vision/evaluation/detection.py b/python/fastdeploy/vision/evaluation/detection.py index 98c6794feab..a13e0429e02 100644 --- a/python/fastdeploy/vision/evaluation/detection.py +++ b/python/fastdeploy/vision/evaluation/detection.py @@ -23,7 +23,8 @@ def eval_detection(model, ann_file, conf_threshold=None, nms_iou_threshold=None, - plot=False): + plot=False, + batch_size=1): from .utils import CocoDetection from .utils import COCOMetric import cv2 @@ -54,6 +55,8 @@ def eval_detection(model, start_time = 0 end_time = 0 average_inference_time = 0 + im_list = list() + im_id_list = list() for image_info, i in zip(all_image_info, trange( image_num, desc="Inference Progress")): @@ -61,19 +64,43 @@ def eval_detection(model, start_time = time.time() im = cv2.imread(image_info["image"]) im_id = image_info["im_id"] - if conf_threshold is None and nms_iou_threshold is None: - result = model.predict(im.copy()) + if batch_size == 1: + if conf_threshold is None and nms_iou_threshold is None: + result = model.predict(im.copy()) + else: + result = model.predict(im, conf_threshold, nms_iou_threshold) + pred = { + 'bbox': [[c] + [s] + b + for b, s, c in zip(result.boxes, result.scores, + result.label_ids)], + 'bbox_num': len(result.boxes), + 'im_id': im_id + } + eval_metric.update(im_id, pred) else: - result = model.predict(im, conf_threshold, nms_iou_threshold) - pred = { - 'bbox': - [[c] + [s] + b - for b, s, c in zip(result.boxes, result.scores, result.label_ids) - ], - 'bbox_num': len(result.boxes), - 'im_id': im_id - } - eval_metric.update(im_id, pred) + im_list.append(im) + im_id_list.append(im_id) + # If the batch_size is not satisfied, the remaining pictures are formed into a batch + if (i + 1) % batch_size != 0 and i != image_num - 1: + continue + if conf_threshold is None and nms_iou_threshold is None: + results = model.batch_predict(im_list) + else: + model.postprocessor.conf_threshold = conf_threshold + model.postprocessor.nms_threshold = nms_iou_threshold + results = model.batch_predict(im_list) + for k in range(len(im_list)): + pred = { + 'bbox': [[c] + [s] + b + for b, s, c in zip(results[k].boxes, results[ + k].scores, results[k].label_ids)], + 'bbox_num': len(results[k].boxes), + 'im_id': im_id_list[k] + } + eval_metric.update(im_id_list[k], pred) + im_list.clear() + im_id_list.clear() + if i == image_num - 1: end_time = time.time() average_inference_time = round( From 3ab3f7dee2a9dee929857798e8c87a173405174b Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Tue, 15 Nov 2022 09:48:16 +0800 Subject: [PATCH 027/136] [Model] Refactor YOLOv5 module (#562) * add paddle_trt in benchmark * update benchmark in device * update benchmark * update result doc * fixed for CI * update python api_docs * update index.rst * add runtime cpp examples * deal with comments * Update infer_paddle_tensorrt.py * Add runtime quick start * deal with comments * fixed reused_input_tensors&&reused_output_tensors * fixed docs * fixed headpose typo * fixed typo * refactor yolov5 * update model infer * refactor pybind for yolov5 * rm origin yolov5 * fixed bugs * rm cuda preprocess * fixed bugs * fixed bugs * fixed bug * fixed bug * fix pybind * rm useless code * add convert_and_permute * fixed bugs * fixed im_info for bs_predict * fixed bug * add bs_predict for yolov5 * Add runtime test and batch eval * deal with comments * fixed bug * update testcase * fixed batch eval bug * fixed preprocess bug Co-authored-by: Jason <928090362@qq.com> Co-authored-by: Jason --- fastdeploy/core/fd_tensor.cc | 8 + fastdeploy/core/fd_tensor.h | 8 +- fastdeploy/vision.h | 2 +- .../common/processors/convert_and_permute.cc | 94 +++++ .../common/processors/convert_and_permute.h | 66 +++ .../vision/common/processors/transform.h | 1 + fastdeploy/vision/detection/contrib/yolov5.cc | 384 ------------------ fastdeploy/vision/detection/contrib/yolov5.h | 136 ------- .../detection/contrib/yolov5/postprocessor.cc | 130 ++++++ .../detection/contrib/yolov5/postprocessor.h | 75 ++++ .../detection/contrib/yolov5/preprocessor.cc | 127 ++++++ .../detection/contrib/yolov5/preprocessor.h | 87 ++++ .../vision/detection/contrib/yolov5/yolov5.cc | 89 ++++ .../vision/detection/contrib/yolov5/yolov5.h | 88 ++++ .../detection/contrib/yolov5/yolov5_pybind.cc | 87 ++++ .../vision/detection/contrib/yolov5_pybind.cc | 73 ---- .../fastdeploy/vision/detection/__init__.py | 2 +- .../vision/detection/contrib/yolov5.py | 241 +++++------ tests/models/test_yolov5.py | 165 ++++++++ 19 files changed, 1141 insertions(+), 722 deletions(-) mode change 100644 => 100755 fastdeploy/core/fd_tensor.cc mode change 100644 => 100755 fastdeploy/core/fd_tensor.h create mode 100644 fastdeploy/vision/common/processors/convert_and_permute.cc create mode 100644 fastdeploy/vision/common/processors/convert_and_permute.h mode change 100644 => 100755 fastdeploy/vision/common/processors/transform.h delete mode 100755 fastdeploy/vision/detection/contrib/yolov5.cc delete mode 100644 fastdeploy/vision/detection/contrib/yolov5.h create mode 100755 fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc create mode 100755 fastdeploy/vision/detection/contrib/yolov5/postprocessor.h create mode 100755 fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc create mode 100755 fastdeploy/vision/detection/contrib/yolov5/preprocessor.h create mode 100755 fastdeploy/vision/detection/contrib/yolov5/yolov5.cc create mode 100755 fastdeploy/vision/detection/contrib/yolov5/yolov5.h create mode 100755 fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc delete mode 100644 fastdeploy/vision/detection/contrib/yolov5_pybind.cc mode change 100644 => 100755 python/fastdeploy/vision/detection/__init__.py create mode 100755 tests/models/test_yolov5.py diff --git a/fastdeploy/core/fd_tensor.cc b/fastdeploy/core/fd_tensor.cc old mode 100644 new mode 100755 index a3544756b37..86ce866f4e5 --- a/fastdeploy/core/fd_tensor.cc +++ b/fastdeploy/core/fd_tensor.cc @@ -43,6 +43,14 @@ const void* FDTensor::Data() const { return buffer_; } +void FDTensor::StopSharing() { + if (IsShared()) { + ReallocFn(Nbytes()); + CopyBuffer(buffer_, external_data_ptr, Nbytes()); + external_data_ptr = nullptr; + } +} + const void* FDTensor::CpuData() const { if (device == Device::GPU) { #ifdef WITH_GPU diff --git a/fastdeploy/core/fd_tensor.h b/fastdeploy/core/fd_tensor.h old mode 100644 new mode 100755 index 32a0da86798..7deb4822988 --- a/fastdeploy/core/fd_tensor.h +++ b/fastdeploy/core/fd_tensor.h @@ -61,13 +61,7 @@ struct FASTDEPLOY_DECL FDTensor { return external_data_ptr != nullptr; } - void StopSharing() { - if (IsShared()) { - ReallocFn(Nbytes()); - CopyBuffer(buffer_, external_data_ptr, Nbytes()); - external_data_ptr = nullptr; - } - } + void StopSharing(); const void* Data() const; diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index 44054ee9372..15cc1d009db 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -21,7 +21,7 @@ #include "fastdeploy/vision/detection/contrib/nanodet_plus.h" #include "fastdeploy/vision/detection/contrib/scaledyolov4.h" #include "fastdeploy/vision/detection/contrib/yolor.h" -#include "fastdeploy/vision/detection/contrib/yolov5.h" +#include "fastdeploy/vision/detection/contrib/yolov5/yolov5.h" #include "fastdeploy/vision/detection/contrib/yolov5lite.h" #include "fastdeploy/vision/detection/contrib/yolov6.h" #include "fastdeploy/vision/detection/contrib/yolov7.h" diff --git a/fastdeploy/vision/common/processors/convert_and_permute.cc b/fastdeploy/vision/common/processors/convert_and_permute.cc new file mode 100644 index 00000000000..73cbb5b48f8 --- /dev/null +++ b/fastdeploy/vision/common/processors/convert_and_permute.cc @@ -0,0 +1,94 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/convert_and_permute.h" + +namespace fastdeploy { +namespace vision { + +ConvertAndPermute::ConvertAndPermute(const std::vector& alpha, + const std::vector& beta, + bool swap_rb) { + FDASSERT(alpha.size() == beta.size(), + "ConvertAndPermute: requires the size of alpha equal to the size of beta."); + FDASSERT(alpha.size() > 0 && beta.size() > 0, + "ConvertAndPermute: requires the size of alpha and beta > 0."); + alpha_.assign(alpha.begin(), alpha.end()); + beta_.assign(beta.begin(), beta.end()); + swap_rb_ = swap_rb; +} + +bool ConvertAndPermute::ImplByOpenCV(FDMat* mat) { + cv::Mat* im = mat->GetOpenCVMat(); + int origin_w = im->cols; + int origin_h = im->rows; + std::vector split_im; + cv::split(*im, split_im); + if (swap_rb_) std::swap(split_im[0], split_im[2]); + for (int c = 0; c < im->channels(); c++) { + split_im[c].convertTo(split_im[c], CV_32FC1, alpha_[c], beta_[c]); + } + cv::Mat res(origin_h, origin_w, CV_32FC(im->channels())); + for (int i = 0; i < im->channels(); ++i) { + cv::extractChannel(split_im[i], + cv::Mat(origin_h, origin_w, CV_32FC1, + res.ptr() + i * origin_h * origin_w * 4), + 0); + } + + mat->SetMat(res); + mat->layout = Layout::CHW; + return true; +} + +#ifdef ENABLE_FLYCV +bool ConvertAndPermute::ImplByFlyCV(FDMat* mat) { + if (mat->layout != Layout::HWC) { + FDERROR << "Only supports input with HWC layout." << std::endl; + return false; + } + fcv::Mat* im = mat->GetFlyCVMat(); + if (im->channels() != 3) { + FDERROR << "Only supports 3-channels image in FlyCV, but now it's " + << im->channels() << "." << std::endl; + return false; + } + std::vector mean(3, 0); + std::vector std(3, 0); + for (size_t i = 0; i < 3; ++i) { + std[i] = 1.0 / alpha_[i]; + mean[i] = -1 * beta_[i] * std[i]; + } + + std::vector channel_reorder_index = {0, 1, 2}; + if (swap_rb_) std::swap(channel_reorder_index[0], channel_reorder_index[2]); + + fcv::Mat new_im; + fcv::normalize_to_submean_to_reorder(*im, mean, std, channel_reorder_index, + new_im, false); + mat->SetMat(new_im); + mat->layout = Layout::CHW; + return true; +} +#endif + +bool ConvertAndPermute::Run(FDMat* mat, const std::vector& alpha, + const std::vector& beta, bool swap_rb, + ProcLib lib) { + auto n = ConvertAndPermute(alpha, beta, swap_rb); + return n(mat, lib); +} + +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/convert_and_permute.h b/fastdeploy/vision/common/processors/convert_and_permute.h new file mode 100644 index 00000000000..a36dfd56383 --- /dev/null +++ b/fastdeploy/vision/common/processors/convert_and_permute.h @@ -0,0 +1,66 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { +class FASTDEPLOY_DECL ConvertAndPermute : public Processor { + public: + ConvertAndPermute(const std::vector& alpha = std::vector(), + const std::vector& beta = std::vector(), + bool swap_rb = false); + bool ImplByOpenCV(FDMat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFlyCV(FDMat* mat); +#endif + std::string Name() { return "ConvertAndPermute"; } + + static bool Run(FDMat* mat, const std::vector& alpha, + const std::vector& beta, bool swap_rb = false, + ProcLib lib = ProcLib::DEFAULT); + + std::vector GetAlpha() const { return alpha_; } + + void SetAlpha(const std::vector& alpha) { + alpha_.clear(); + std::vector().swap(alpha_); + alpha_.assign(alpha.begin(), alpha.end()); + } + + std::vector GetBeta() const { return beta_; } + + void SetBeta(const std::vector& beta) { + beta_.clear(); + std::vector().swap(beta_); + beta_.assign(beta.begin(), beta.end()); + } + + bool GetSwapRB() { + return swap_rb_; + } + + void SetSwapRB(bool swap_rb) { + swap_rb_ = swap_rb; + } + + private: + std::vector alpha_; + std::vector beta_; + bool swap_rb_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/transform.h b/fastdeploy/vision/common/processors/transform.h old mode 100644 new mode 100755 index 7ba58167d46..e3f71463657 --- a/fastdeploy/vision/common/processors/transform.h +++ b/fastdeploy/vision/common/processors/transform.h @@ -18,6 +18,7 @@ #include "fastdeploy/vision/common/processors/center_crop.h" #include "fastdeploy/vision/common/processors/color_space_convert.h" #include "fastdeploy/vision/common/processors/convert.h" +#include "fastdeploy/vision/common/processors/convert_and_permute.h" #include "fastdeploy/vision/common/processors/crop.h" #include "fastdeploy/vision/common/processors/hwc2chw.h" #include "fastdeploy/vision/common/processors/limit_by_stride.h" diff --git a/fastdeploy/vision/detection/contrib/yolov5.cc b/fastdeploy/vision/detection/contrib/yolov5.cc deleted file mode 100755 index a94adaf6c08..00000000000 --- a/fastdeploy/vision/detection/contrib/yolov5.cc +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/contrib/yolov5.h" - -#include "fastdeploy/utils/perf.h" -#include "fastdeploy/vision/utils/utils.h" -#ifdef ENABLE_CUDA_PREPROCESS -#include "fastdeploy/vision/utils/cuda_utils.h" -#endif // ENABLE_CUDA_PREPROCESS - -namespace fastdeploy { -namespace vision { -namespace detection { - -void YOLOv5::LetterBox(Mat* mat, std::vector size, - std::vector color, bool _auto, bool scale_fill, - bool scale_up, int stride) { - float scale = - std::min(size[1] * 1.0 / mat->Height(), size[0] * 1.0 / mat->Width()); - if (!scale_up) { - scale = std::min(scale, 1.0f); - } - - int resize_h = int(round(mat->Height() * scale)); - int resize_w = int(round(mat->Width() * scale)); - - int pad_w = size[0] - resize_w; - int pad_h = size[1] - resize_h; - if (_auto) { - pad_h = pad_h % stride; - pad_w = pad_w % stride; - } else if (scale_fill) { - pad_h = 0; - pad_w = 0; - resize_h = size[1]; - resize_w = size[0]; - } - Resize::Run(mat, resize_w, resize_h); - if (pad_h > 0 || pad_w > 0) { - float half_h = pad_h * 1.0 / 2; - int top = int(round(half_h - 0.1)); - int bottom = int(round(half_h + 0.1)); - float half_w = pad_w * 1.0 / 2; - int left = int(round(half_w - 0.1)); - int right = int(round(half_w + 0.1)); - Pad::Run(mat, top, bottom, left, right, color); - } -} - -YOLOv5::YOLOv5(const std::string& model_file, const std::string& params_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - if (model_format == ModelFormat::ONNX) { - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; - valid_gpu_backends = {Backend::ORT, Backend::TRT}; - } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; - } - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; -#ifdef ENABLE_CUDA_PREPROCESS - cudaSetDevice(runtime_option.device_id); - cudaStream_t stream; - CUDA_CHECK(cudaStreamCreate(&stream)); - cuda_stream_ = reinterpret_cast(stream); - runtime_option.SetExternalStream(cuda_stream_); -#endif // ENABLE_CUDA_PREPROCESS - initialized = Initialize(); -} - -bool YOLOv5::Initialize() { - // parameters for preprocess - size_ = {640, 640}; - padding_value_ = {114.0, 114.0, 114.0}; - is_mini_pad_ = false; - is_no_pad_ = false; - is_scale_up_ = false; - stride_ = 32; - max_wh_ = 7680.0; - multi_label_ = true; - reused_input_tensors_.resize(1); - - if (!InitRuntime()) { - FDERROR << "Failed to initialize fastdeploy backend." << std::endl; - return false; - } - // Check if the input shape is dynamic after Runtime already initialized, - // Note that, We need to force is_mini_pad 'false' to keep static - // shape after padding (LetterBox) when the is_dynamic_shape is 'false'. - // TODO(qiuyanjun): remove - // is_dynamic_input_ = false; - // auto shape = InputInfoOfRuntime(0).shape; - // for (int i = 0; i < shape.size(); ++i) { - // // if height or width is dynamic - // if (i >= 2 && shape[i] <= 0) { - // is_dynamic_input_ = true; - // break; - // } - // } - // if (!is_dynamic_input_) { - // is_mini_pad_ = false; - // } - - return true; -} - -YOLOv5::~YOLOv5() { -#ifdef ENABLE_CUDA_PREPROCESS - if (use_cuda_preprocessing_) { - CUDA_CHECK(cudaFreeHost(input_img_cuda_buffer_host_)); - CUDA_CHECK(cudaFree(input_img_cuda_buffer_device_)); - CUDA_CHECK(cudaFree(input_tensor_cuda_buffer_device_)); - CUDA_CHECK(cudaStreamDestroy(reinterpret_cast(cuda_stream_))); - } -#endif // ENABLE_CUDA_PREPROCESS -} - -bool YOLOv5::Preprocess(Mat* mat, FDTensor* output, - std::map>* im_info, - const std::vector& size, - const std::vector padding_value, - bool is_mini_pad, bool is_no_pad, bool is_scale_up, - int stride, float max_wh, bool multi_label) { - // Record the shape of image and the shape of preprocessed image - (*im_info)["input_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - (*im_info)["output_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - - // process after image load - double ratio = (size[0] * 1.0) / std::max(static_cast(mat->Height()), - static_cast(mat->Width())); - if (std::fabs(ratio - 1.0f) > 1e-06) { - int interp = cv::INTER_AREA; - if (ratio > 1.0) { - interp = cv::INTER_LINEAR; - } - int resize_h = int(mat->Height() * ratio); - int resize_w = int(mat->Width() * ratio); - Resize::Run(mat, resize_w, resize_h, -1, -1, interp); - } - // yolov5's preprocess steps - // 1. letterbox - // 2. BGR->RGB - // 3. HWC->CHW - LetterBox(mat, size, padding_value, is_mini_pad, is_no_pad, is_scale_up, - stride); - BGR2RGB::Run(mat); - // Normalize::Run(mat, std::vector(mat->Channels(), 0.0), - // std::vector(mat->Channels(), 1.0)); - // Compute `result = mat * alpha + beta` directly by channel - std::vector alpha = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - std::vector beta = {0.0f, 0.0f, 0.0f}; - Convert::Run(mat, alpha, beta); - - // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); - mat->ShareWithTensor(output); - output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c - return true; -} - -void YOLOv5::UseCudaPreprocessing(int max_image_size) { -#ifdef ENABLE_CUDA_PREPROCESS - use_cuda_preprocessing_ = true; - is_scale_up_ = true; - if (input_img_cuda_buffer_host_ == nullptr) { - // prepare input data cache in GPU pinned memory - CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, - max_image_size * 3)); - // prepare input data cache in GPU device memory - CUDA_CHECK( - cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); - CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, - 3 * size_[0] * size_[1] * sizeof(float))); - } -#else - FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON." - << std::endl; - use_cuda_preprocessing_ = false; -#endif -} - -bool YOLOv5::CudaPreprocess( - Mat* mat, FDTensor* output, - std::map>* im_info, - const std::vector& size, const std::vector padding_value, - bool is_mini_pad, bool is_no_pad, bool is_scale_up, int stride, - float max_wh, bool multi_label) { -#ifdef ENABLE_CUDA_PREPROCESS - if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) { - FDERROR << "Preprocessing with CUDA is only available when the arguments " - "satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." - << std::endl; - return false; - } - - // Record the shape of image and the shape of preprocessed image - (*im_info)["input_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - (*im_info)["output_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - - cudaStream_t stream = reinterpret_cast(cuda_stream_); - int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels(); - memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size); - CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_, - input_img_cuda_buffer_host_, src_img_buf_size, - cudaMemcpyHostToDevice, stream)); - utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(), - mat->Height(), input_tensor_cuda_buffer_device_, - size[0], size[1], padding_value, stream); - - // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(size[0]), - static_cast(size[1])}; - - output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32, - input_tensor_cuda_buffer_device_); - output->device = Device::GPU; - output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c - return true; -#else - FDERROR << "CUDA src code was not enabled." << std::endl; - return false; -#endif // ENABLE_CUDA_PREPROCESS -} - -bool YOLOv5::Postprocess( - std::vector& infer_results, DetectionResult* result, - const std::map>& im_info, - float conf_threshold, float nms_iou_threshold, bool multi_label, - float max_wh) { - auto& infer_result = infer_results[0]; - FDASSERT(infer_result.shape[0] == 1, "Only support batch =1 now."); - result->Clear(); - if (multi_label) { - result->Reserve(infer_result.shape[1] * (infer_result.shape[2] - 5)); - } else { - result->Reserve(infer_result.shape[1]); - } - if (infer_result.dtype != FDDataType::FP32) { - FDERROR << "Only support post process with float32 data." << std::endl; - return false; - } - float* data = static_cast(infer_result.Data()); - for (size_t i = 0; i < infer_result.shape[1]; ++i) { - int s = i * infer_result.shape[2]; - float confidence = data[s + 4]; - if (multi_label) { - for (size_t j = 5; j < infer_result.shape[2]; ++j) { - confidence = data[s + 4]; - float* class_score = data + s + j; - confidence *= (*class_score); - // filter boxes by conf_threshold - if (confidence <= conf_threshold) { - continue; - } - int32_t label_id = std::distance(data + s + 5, class_score); - - // convert from [x, y, w, h] to [x1, y1, x2, y2] - result->boxes.emplace_back(std::array{ - data[s] - data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, - data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); - result->label_ids.push_back(label_id); - result->scores.push_back(confidence); - } - } else { - float* max_class_score = - std::max_element(data + s + 5, data + s + infer_result.shape[2]); - confidence *= (*max_class_score); - // filter boxes by conf_threshold - if (confidence <= conf_threshold) { - continue; - } - int32_t label_id = std::distance(data + s + 5, max_class_score); - // convert from [x, y, w, h] to [x1, y1, x2, y2] - result->boxes.emplace_back(std::array{ - data[s] - data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, - data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); - result->label_ids.push_back(label_id); - result->scores.push_back(confidence); - } - } - - if (result->boxes.size() == 0) { - return true; - } - - utils::NMS(result, nms_iou_threshold); - - // scale the boxes to the origin image shape - auto iter_out = im_info.find("output_shape"); - auto iter_ipt = im_info.find("input_shape"); - FDASSERT(iter_out != im_info.end() && iter_ipt != im_info.end(), - "Cannot find input_shape or output_shape from im_info."); - float out_h = iter_out->second[0]; - float out_w = iter_out->second[1]; - float ipt_h = iter_ipt->second[0]; - float ipt_w = iter_ipt->second[1]; - float scale = std::min(out_h / ipt_h, out_w / ipt_w); - for (size_t i = 0; i < result->boxes.size(); ++i) { - float pad_h = (out_h - ipt_h * scale) / 2; - float pad_w = (out_w - ipt_w * scale) / 2; - int32_t label_id = (result->label_ids)[i]; - // clip box - result->boxes[i][0] = result->boxes[i][0] - max_wh * label_id; - result->boxes[i][1] = result->boxes[i][1] - max_wh * label_id; - result->boxes[i][2] = result->boxes[i][2] - max_wh * label_id; - result->boxes[i][3] = result->boxes[i][3] - max_wh * label_id; - result->boxes[i][0] = std::max((result->boxes[i][0] - pad_w) / scale, 0.0f); - result->boxes[i][1] = std::max((result->boxes[i][1] - pad_h) / scale, 0.0f); - result->boxes[i][2] = std::max((result->boxes[i][2] - pad_w) / scale, 0.0f); - result->boxes[i][3] = std::max((result->boxes[i][3] - pad_h) / scale, 0.0f); - result->boxes[i][0] = std::min(result->boxes[i][0], ipt_w); - result->boxes[i][1] = std::min(result->boxes[i][1], ipt_h); - result->boxes[i][2] = std::min(result->boxes[i][2], ipt_w); - result->boxes[i][3] = std::min(result->boxes[i][3], ipt_h); - } - return true; -} - -bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, - float nms_iou_threshold) { - Mat mat(*im); - - std::map> im_info; - - if (use_cuda_preprocessing_) { - if (!CudaPreprocess(&mat, &reused_input_tensors_[0], &im_info, size_, - padding_value_, is_mini_pad_, is_no_pad_, is_scale_up_, - stride_, max_wh_, multi_label_)) { - FDERROR << "Failed to preprocess input image." << std::endl; - return false; - } - } else { - if (!Preprocess(&mat, &reused_input_tensors_[0], &im_info, size_, - padding_value_, is_mini_pad_, is_no_pad_, is_scale_up_, - stride_, max_wh_, multi_label_)) { - FDERROR << "Failed to preprocess input image." << std::endl; - return false; - } - } - - reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; - if (!Infer()) { - FDERROR << "Failed to inference." << std::endl; - return false; - } - - if (!Postprocess(reused_output_tensors_, result, im_info, conf_threshold, - nms_iou_threshold, multi_label_)) { - FDERROR << "Failed to post process." << std::endl; - return false; - } - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5.h b/fastdeploy/vision/detection/contrib/yolov5.h deleted file mode 100644 index 4c46acd0a21..00000000000 --- a/fastdeploy/vision/detection/contrib/yolov5.h +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. //NOLINT -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include "fastdeploy/fastdeploy_model.h" -#include "fastdeploy/vision/common/processors/transform.h" -#include "fastdeploy/vision/common/result.h" - -namespace fastdeploy { -namespace vision { -namespace detection { -/*! @brief YOLOv5 model object used when to load a YOLOv5 model exported by YOLOv5. - */ -class FASTDEPLOY_DECL YOLOv5 : public FastDeployModel { - public: - /** \brief Set path of model file and the configuration of runtime. - * - * \param[in] model_file Path of model file, e.g ./yolov5.onnx - * \param[in] params_file Path of parameter file, e.g ppyoloe/model.pdiparams, if the model format is ONNX, this parameter will be ignored - * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" - * \param[in] model_format Model format of the loaded model, default is ONNX format - */ - YOLOv5(const std::string& model_file, const std::string& params_file = "", - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::ONNX); - - ~YOLOv5(); - - std::string ModelName() const { return "yolov5"; } - /** \brief Predict the detection result for an input image - * - * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format - * \param[in] result The output detection result will be writen to this structure - * \param[in] conf_threshold confidence threashold for postprocessing, default is 0.25 - * \param[in] nms_iou_threshold iou threashold for NMS, default is 0.5 - * \return true if the prediction successed, otherwise false - */ - virtual bool Predict(cv::Mat* im, DetectionResult* result, - float conf_threshold = 0.25, - float nms_iou_threshold = 0.5); - - static bool Preprocess(Mat* mat, FDTensor* output, - std::map>* im_info, - const std::vector& size = {640, 640}, - const std::vector padding_value = {114.0, 114.0, - 114.0}, - bool is_mini_pad = false, bool is_no_pad = false, - bool is_scale_up = false, int stride = 32, - float max_wh = 7680.0, bool multi_label = true); - - void UseCudaPreprocessing(int max_img_size = 3840 * 2160); - - bool CudaPreprocess(Mat* mat, FDTensor* output, - std::map>* im_info, - const std::vector& size = {640, 640}, - const std::vector padding_value = {114.0, 114.0, - 114.0}, - bool is_mini_pad = false, bool is_no_pad = false, - bool is_scale_up = false, int stride = 32, - float max_wh = 7680.0, bool multi_label = true); - - static bool Postprocess( - std::vector& infer_results, DetectionResult* result, - const std::map>& im_info, - float conf_threshold, float nms_iou_threshold, bool multi_label, - float max_wh = 7680.0); - - /*! @brief - Argument for image preprocessing step, tuple of (width, height), decide the target size after resize, default size = {640, 640} - */ - std::vector size_; - // padding value, size should be the same as channels - - std::vector padding_value_; - // only pad to the minimum rectange which height and width is times of stride - bool is_mini_pad_; - // while is_mini_pad = false and is_no_pad = true, - // will resize the image to the set size - bool is_no_pad_; - // if is_scale_up is false, the input image only can be zoom out, - // the maximum resize scale cannot exceed 1.0 - bool is_scale_up_; - // padding stride, for is_mini_pad - int stride_; - // for offseting the boxes by classes when using NMS - float max_wh_; - /*! @brief - Argument for image preprocessing step, for different strategies to get boxes when postprocessing, default true - */ - bool multi_label_; - - private: - bool Initialize(); - - bool IsDynamicInput() const { return is_dynamic_input_; } - - static void LetterBox(Mat* mat, std::vector size, - std::vector color, bool _auto, - bool scale_fill = false, bool scale_up = true, - int stride = 32); - - // whether to inference with dynamic shape (e.g ONNX export with dynamic shape - // or not.) - // YOLOv5 official 'export_onnx.py' script will export dynamic ONNX by - // default. - // while is_dynamic_shape if 'false', is_mini_pad will force 'false'. This - // value will - // auto check by fastdeploy after the internal Runtime already initialized. - bool is_dynamic_input_; - // CUDA host buffer for input image - uint8_t* input_img_cuda_buffer_host_ = nullptr; - // CUDA device buffer for input image - uint8_t* input_img_cuda_buffer_device_ = nullptr; - // CUDA device buffer for TRT input tensor - float* input_tensor_cuda_buffer_device_ = nullptr; - // Whether to use CUDA preprocessing - bool use_cuda_preprocessing_ = false; - // CUDA stream - void* cuda_stream_ = nullptr; -}; - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc new file mode 100755 index 00000000000..dd61efb0023 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc @@ -0,0 +1,130 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/contrib/yolov5/postprocessor.h" +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +YOLOv5Postprocessor::YOLOv5Postprocessor() { + conf_threshold_ = 0.25; + nms_threshold_ = 0.5; + multi_label_ = true; + max_wh_ = 7680.0; +} + +bool YOLOv5Postprocessor::Run(const std::vector& tensors, std::vector* results, + const std::vector>>& ims_info) { + int batch = tensors[0].shape[0]; + + results->resize(batch); + + for (size_t bs = 0; bs < batch; ++bs) { + (*results)[bs].Clear(); + if (multi_label_) { + (*results)[bs].Reserve(tensors[0].shape[1] * (tensors[0].shape[2] - 5)); + } else { + (*results)[bs].Reserve(tensors[0].shape[1]); + } + if (tensors[0].dtype != FDDataType::FP32) { + FDERROR << "Only support post process with float32 data." << std::endl; + return false; + } + const float* data = reinterpret_cast(tensors[0].Data()) + bs * tensors[0].shape[1] * tensors[0].shape[2]; + for (size_t i = 0; i < tensors[0].shape[1]; ++i) { + int s = i * tensors[0].shape[2]; + float confidence = data[s + 4]; + if (multi_label_) { + for (size_t j = 5; j < tensors[0].shape[2]; ++j) { + confidence = data[s + 4]; + const float* class_score = data + s + j; + confidence *= (*class_score); + // filter boxes by conf_threshold + if (confidence <= conf_threshold_) { + continue; + } + int32_t label_id = std::distance(data + s + 5, class_score); + + // convert from [x, y, w, h] to [x1, y1, x2, y2] + (*results)[bs].boxes.emplace_back(std::array{ + data[s] - data[s + 2] / 2.0f + label_id * max_wh_, + data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh_, + data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh_, + data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh_}); + (*results)[bs].label_ids.push_back(label_id); + (*results)[bs].scores.push_back(confidence); + } + } else { + const float* max_class_score = + std::max_element(data + s + 5, data + s + tensors[0].shape[2]); + confidence *= (*max_class_score); + // filter boxes by conf_threshold + if (confidence <= conf_threshold_) { + continue; + } + int32_t label_id = std::distance(data + s + 5, max_class_score); + // convert from [x, y, w, h] to [x1, y1, x2, y2] + (*results)[bs].boxes.emplace_back(std::array{ + data[s] - data[s + 2] / 2.0f + label_id * max_wh_, + data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh_, + data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh_, + data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh_}); + (*results)[bs].label_ids.push_back(label_id); + (*results)[bs].scores.push_back(confidence); + } + } + + if ((*results)[bs].boxes.size() == 0) { + return true; + } + + utils::NMS(&((*results)[bs]), nms_threshold_); + + // scale the boxes to the origin image shape + auto iter_out = ims_info[bs].find("output_shape"); + auto iter_ipt = ims_info[bs].find("input_shape"); + FDASSERT(iter_out != ims_info[bs].end() && iter_ipt != ims_info[bs].end(), + "Cannot find input_shape or output_shape from im_info."); + float out_h = iter_out->second[0]; + float out_w = iter_out->second[1]; + float ipt_h = iter_ipt->second[0]; + float ipt_w = iter_ipt->second[1]; + float scale = std::min(out_h / ipt_h, out_w / ipt_w); + for (size_t i = 0; i < (*results)[bs].boxes.size(); ++i) { + float pad_h = (out_h - ipt_h * scale) / 2; + float pad_w = (out_w - ipt_w * scale) / 2; + int32_t label_id = ((*results)[bs].label_ids)[i]; + // clip box + (*results)[bs].boxes[i][0] = (*results)[bs].boxes[i][0] - max_wh_ * label_id; + (*results)[bs].boxes[i][1] = (*results)[bs].boxes[i][1] - max_wh_ * label_id; + (*results)[bs].boxes[i][2] = (*results)[bs].boxes[i][2] - max_wh_ * label_id; + (*results)[bs].boxes[i][3] = (*results)[bs].boxes[i][3] - max_wh_ * label_id; + (*results)[bs].boxes[i][0] = std::max(((*results)[bs].boxes[i][0] - pad_w) / scale, 0.0f); + (*results)[bs].boxes[i][1] = std::max(((*results)[bs].boxes[i][1] - pad_h) / scale, 0.0f); + (*results)[bs].boxes[i][2] = std::max(((*results)[bs].boxes[i][2] - pad_w) / scale, 0.0f); + (*results)[bs].boxes[i][3] = std::max(((*results)[bs].boxes[i][3] - pad_h) / scale, 0.0f); + (*results)[bs].boxes[i][0] = std::min((*results)[bs].boxes[i][0], ipt_w); + (*results)[bs].boxes[i][1] = std::min((*results)[bs].boxes[i][1], ipt_h); + (*results)[bs].boxes[i][2] = std::min((*results)[bs].boxes[i][2], ipt_w); + (*results)[bs].boxes[i][3] = std::min((*results)[bs].boxes[i][3], ipt_h); + } + } + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h new file mode 100755 index 00000000000..a1479dd9403 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h @@ -0,0 +1,75 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace detection { +/*! @brief Postprocessor object for YOLOv5 serials model. + */ +class FASTDEPLOY_DECL YOLOv5Postprocessor { + public: + /** \brief Create a postprocessor instance for YOLOv5 serials model + */ + YOLOv5Postprocessor(); + + /** \brief Process the result of runtime and fill to DetectionResult structure + * + * \param[in] tensors The inference result from runtime + * \param[in] result The output result of detection + * \param[in] ims_info The shape info list, record input_shape and output_shape + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector* results, + const std::vector>>& ims_info); + + /// Set conf_threshold, default 0.25 + void SetConfThreshold(const float& conf_threshold) { + conf_threshold_ = conf_threshold; + } + + /// Get conf_threshold, default 0.25 + float GetConfThreshold() const { return conf_threshold_; } + + /// Set nms_threshold, default 0.5 + void SetNMSThreshold(const float& nms_threshold) { + nms_threshold_ = nms_threshold; + } + + /// Get nms_threshold, default 0.5 + float GetNMSThreshold() const { return nms_threshold_; } + + /// Set multi_label, default true + void SetMultiLabel(bool multi_label) { + multi_label_ = multi_label; + } + + /// Get multi_label, default true + bool GetMultiLabel() const { return multi_label_; } + + protected: + float conf_threshold_; + float nms_threshold_; + bool multi_label_; + float max_wh_; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc new file mode 100755 index 00000000000..112a4d4d5da --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc @@ -0,0 +1,127 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/contrib/yolov5/preprocessor.h" +#include "fastdeploy/function/concat.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +YOLOv5Preprocessor::YOLOv5Preprocessor() { + size_ = {640, 640}; + padding_value_ = {114.0, 114.0, 114.0}; + is_mini_pad_ = false; + is_no_pad_ = false; + is_scale_up_ = false; + stride_ = 32; + max_wh_ = 7680.0; +} + +void YOLOv5Preprocessor::LetterBox(FDMat* mat) { + float scale = + std::min(size_[1] * 1.0 / mat->Height(), size_[0] * 1.0 / mat->Width()); + if (!is_scale_up_) { + scale = std::min(scale, 1.0f); + } + + int resize_h = int(round(mat->Height() * scale)); + int resize_w = int(round(mat->Width() * scale)); + + int pad_w = size_[0] - resize_w; + int pad_h = size_[1] - resize_h; + if (is_mini_pad_) { + pad_h = pad_h % stride_; + pad_w = pad_w % stride_; + } else if (is_no_pad_) { + pad_h = 0; + pad_w = 0; + resize_h = size_[1]; + resize_w = size_[0]; + } + Resize::Run(mat, resize_w, resize_h); + if (pad_h > 0 || pad_w > 0) { + float half_h = pad_h * 1.0 / 2; + int top = int(round(half_h - 0.1)); + int bottom = int(round(half_h + 0.1)); + float half_w = pad_w * 1.0 / 2; + int left = int(round(half_w - 0.1)); + int right = int(round(half_w + 0.1)); + Pad::Run(mat, top, bottom, left, right, padding_value_); + } +} + +bool YOLOv5Preprocessor::Preprocess(FDMat* mat, FDTensor* output, + std::map>* im_info) { + // Record the shape of image and the shape of preprocessed image + (*im_info)["input_shape"] = {static_cast(mat->Height()), + static_cast(mat->Width())}; + + // process after image load + double ratio = (size_[0] * 1.0) / std::max(static_cast(mat->Height()), + static_cast(mat->Width())); + if (std::fabs(ratio - 1.0f) > 1e-06) { + int interp = cv::INTER_AREA; + if (ratio > 1.0) { + interp = cv::INTER_LINEAR; + } + int resize_h = int(mat->Height() * ratio); + int resize_w = int(mat->Width() * ratio); + Resize::Run(mat, resize_w, resize_h, -1, -1, interp); + } + // yolov5's preprocess steps + // 1. letterbox + // 2. convert_and_permute(swap_rb=true) + LetterBox(mat); + std::vector alpha = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; + std::vector beta = {0.0f, 0.0f, 0.0f}; + ConvertAndPermute::Run(mat, alpha, beta, true); + + // Record output shape of preprocessed image + (*im_info)["output_shape"] = {static_cast(mat->Height()), + static_cast(mat->Width())}; + + mat->ShareWithTensor(output); + output->ExpandDim(0); // reshape to n, h, w, c + return true; +} + +bool YOLOv5Preprocessor::Run(std::vector* images, std::vector* outputs, + std::vector>>* ims_info) { + if (images->size() == 0) { + FDERROR << "The size of input images should be greater than 0." << std::endl; + return false; + } + ims_info->resize(images->size()); + outputs->resize(1); + // Concat all the preprocessed data to a batch tensor + std::vector tensors(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + if (!Preprocess(&(*images)[i], &tensors[i], &(*ims_info)[i])) { + FDERROR << "Failed to preprocess input image." << std::endl; + return false; + } + } + + if (tensors.size() == 1) { + (*outputs)[0] = std::move(tensors[0]); + } else { + function::Concat(tensors, &((*outputs)[0]), 0); + } + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h new file mode 100755 index 00000000000..b3559685db7 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h @@ -0,0 +1,87 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace detection { +/*! @brief Preprocessor object for YOLOv5 serials model. + */ +class FASTDEPLOY_DECL YOLOv5Preprocessor { + public: + /** \brief Create a preprocessor instance for YOLOv5 serials model + */ + YOLOv5Preprocessor(); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime + * \param[in] ims_info The shape info list, record input_shape and output_shape + * \return true if the preprocess successed, otherwise false + */ + bool Run(std::vector* images, std::vector* outputs, + std::vector>>* ims_info); + + /// Set target size, tuple of (width, height), default size = {640, 640} + void SetSize(const std::vector& size) { size_ = size; } + + /// Get target size, tuple of (width, height), default size = {640, 640} + std::vector GetSize() const { return size_; } + + /// Set padding value, size should be the same as channels + void SetPaddingValue(const std::vector& padding_value) { + padding_value_ = padding_value; + } + + /// Get padding value, size should be the same as channels + std::vector GetPaddingValue() const { return padding_value_; } + + protected: + bool Preprocess(FDMat* mat, FDTensor* output, + std::map>* im_info); + + void LetterBox(FDMat* mat); + + // target size, tuple of (width, height), default size = {640, 640} + std::vector size_; + + // padding value, size should be the same as channels + std::vector padding_value_; + + // only pad to the minimum rectange which height and width is times of stride + bool is_mini_pad_; + + // while is_mini_pad = false and is_no_pad = true, + // will resize the image to the set size + bool is_no_pad_; + + // if is_scale_up is false, the input image only can be zoom out, + // the maximum resize scale cannot exceed 1.0 + bool is_scale_up_; + + // padding stride, for is_mini_pad + int stride_; + + // for offseting the boxes by classes when using NMS + float max_wh_; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc b/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc new file mode 100755 index 00000000000..422cf501c42 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/yolov5.cc @@ -0,0 +1,89 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/contrib/yolov5/yolov5.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +YOLOv5::YOLOv5(const std::string& model_file, const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) { + if (model_format == ModelFormat::ONNX) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else { + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool YOLOv5::Initialize() { + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, float nms_threshold) { + postprocessor_.SetConfThreshold(conf_threshold); + postprocessor_.SetNMSThreshold(nms_threshold); + if (!Predict(*im, result)) { + return false; + } + return true; +} + +bool YOLOv5::Predict(const cv::Mat& im, DetectionResult* result) { + std::vector results; + if (!BatchPredict({im}, &results)) { + return false; + } + *result = std::move(results[0]); + return true; +} + +bool YOLOv5::BatchPredict(const std::vector& images, std::vector* results) { + std::vector>> ims_info; + std::vector fd_images = WrapMat(images); + + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_, &ims_info)) { + FDERROR << "Failed to preprocess the input image." << std::endl; + return false; + } + + reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; + return false; + } + + if (!postprocessor_.Run(reused_output_tensors_, results, ims_info)) { + FDERROR << "Failed to postprocess the inference results by runtime." << std::endl; + return false; + } + + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/yolov5.h b/fastdeploy/vision/detection/contrib/yolov5/yolov5.h new file mode 100755 index 00000000000..53bcfce755d --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/yolov5.h @@ -0,0 +1,88 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. //NOLINT +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/detection/contrib/yolov5/preprocessor.h" +#include "fastdeploy/vision/detection/contrib/yolov5/postprocessor.h" + +namespace fastdeploy { +namespace vision { +namespace detection { +/*! @brief YOLOv5 model object used when to load a YOLOv5 model exported by YOLOv5. + */ +class FASTDEPLOY_DECL YOLOv5 : public FastDeployModel { + public: + /** \brief Set path of model file and the configuration of runtime. + * + * \param[in] model_file Path of model file, e.g ./yolov5.onnx + * \param[in] params_file Path of parameter file, e.g ppyoloe/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" + * \param[in] model_format Model format of the loaded model, default is ONNX format + */ + YOLOv5(const std::string& model_file, const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::ONNX); + + std::string ModelName() const { return "yolov5"; } + + /** \brief DEPRECATED Predict the detection result for an input image, remove at 1.0 version + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output detection result will be writen to this structure + * \param[in] conf_threshold confidence threashold for postprocessing, default is 0.25 + * \param[in] nms_threshold iou threashold for NMS, default is 0.5 + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(cv::Mat* im, DetectionResult* result, + float conf_threshold = 0.25, + float nms_threshold = 0.5); + + /** \brief Predict the detection result for an input image + * + * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output detection result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(const cv::Mat& img, DetectionResult* result); + + /** \brief Predict the detection results for a batch of input images + * + * \param[in] imgs, The input image list, each element comes from cv::imread() + * \param[in] results The output detection result list + * \return true if the prediction successed, otherwise false + */ + virtual bool BatchPredict(const std::vector& imgs, + std::vector* results); + + /// Get preprocessor reference of YOLOv5 + virtual YOLOv5Preprocessor& GetPreprocessor() { + return preprocessor_; + } + + /// Get postprocessor reference of YOLOv5 + virtual YOLOv5Postprocessor& GetPostprocessor() { + return postprocessor_; + } + + protected: + bool Initialize(); + YOLOv5Preprocessor preprocessor_; + YOLOv5Postprocessor postprocessor_; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc b/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc new file mode 100755 index 00000000000..f44891d9846 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc @@ -0,0 +1,87 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindYOLOv5(pybind11::module& m) { + pybind11::class_( + m, "YOLOv5Preprocessor") + .def(pybind11::init<>()) + .def("run", [](vision::detection::YOLOv5Preprocessor& self, std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + std::vector>> ims_info; + if (!self.Run(&images, &outputs, &ims_info)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in PaddleClasPreprocessor.')"); + } + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].StopSharing(); + } + return make_pair(outputs, ims_info); + }) + .def_property("size", &vision::detection::YOLOv5Preprocessor::GetSize, &vision::detection::YOLOv5Preprocessor::SetSize) + .def_property("padding_value", &vision::detection::YOLOv5Preprocessor::GetPaddingValue, &vision::detection::YOLOv5Preprocessor::SetPaddingValue); + + pybind11::class_( + m, "YOLOv5Postprocessor") + .def(pybind11::init<>()) + .def("run", [](vision::detection::YOLOv5Postprocessor& self, std::vector& inputs, + const std::vector>>& ims_info) { + std::vector results; + if (!self.Run(inputs, &results, ims_info)) { + pybind11::eval("raise Exception('Failed to postprocess the runtime result in YOLOv5Postprocessor.')"); + } + return results; + }) + .def("run", [](vision::detection::YOLOv5Postprocessor& self, std::vector& input_array, + const std::vector>>& ims_info) { + std::vector results; + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + if (!self.Run(inputs, &results, ims_info)) { + pybind11::eval("raise Exception('Failed to postprocess the runtime result in YOLOv5Postprocessor.')"); + } + return results; + }) + .def_property("conf_threshold", &vision::detection::YOLOv5Postprocessor::GetConfThreshold, &vision::detection::YOLOv5Postprocessor::SetConfThreshold) + .def_property("nms_threshold", &vision::detection::YOLOv5Postprocessor::GetNMSThreshold, &vision::detection::YOLOv5Postprocessor::SetNMSThreshold) + .def_property("multi_label", &vision::detection::YOLOv5Postprocessor::GetMultiLabel, &vision::detection::YOLOv5Postprocessor::SetMultiLabel); + + pybind11::class_(m, "YOLOv5") + .def(pybind11::init()) + .def("predict", + [](vision::detection::YOLOv5& self, pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::DetectionResult res; + self.Predict(mat, &res); + return res; + }) + .def("batch_predict", [](vision::detection::YOLOv5& self, std::vector& data) { + std::vector images; + for (size_t i = 0; i < data.size(); ++i) { + images.push_back(PyArrayToCvMat(data[i])); + } + std::vector results; + self.BatchPredict(images, &results); + return results; + }) + .def_property_readonly("preprocessor", &vision::detection::YOLOv5::GetPreprocessor) + .def_property_readonly("postprocessor", &vision::detection::YOLOv5::GetPostprocessor); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov5_pybind.cc b/fastdeploy/vision/detection/contrib/yolov5_pybind.cc deleted file mode 100644 index 52d0d78c9b2..00000000000 --- a/fastdeploy/vision/detection/contrib/yolov5_pybind.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/pybind/main.h" - -namespace fastdeploy { -void BindYOLOv5(pybind11::module& m) { - pybind11::class_(m, "YOLOv5") - .def(pybind11::init()) - .def("predict", - [](vision::detection::YOLOv5& self, pybind11::array& data, - float conf_threshold, float nms_iou_threshold) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res, conf_threshold, nms_iou_threshold); - return res; - }) - .def("use_cuda_preprocessing", - [](vision::detection::YOLOv5& self, int max_image_size) { - self.UseCudaPreprocessing(max_image_size); - }) - .def_static("preprocess", - [](pybind11::array& data, const std::vector& size, - const std::vector padding_value, bool is_mini_pad, - bool is_no_pad, bool is_scale_up, int stride, float max_wh, - bool multi_label) { - auto mat = PyArrayToCvMat(data); - fastdeploy::vision::Mat fd_mat(mat); - FDTensor output; - std::map> im_info; - vision::detection::YOLOv5::Preprocess( - &fd_mat, &output, &im_info, size, padding_value, - is_mini_pad, is_no_pad, is_scale_up, stride, max_wh, - multi_label); - return make_pair(TensorToPyArray(output), im_info); - }) - .def_static( - "postprocess", - [](std::vector infer_results, - const std::map>& im_info, - float conf_threshold, float nms_iou_threshold, bool multi_label, - float max_wh) { - std::vector fd_infer_results(infer_results.size()); - PyArrayToTensorList(infer_results, &fd_infer_results, true); - vision::DetectionResult result; - vision::detection::YOLOv5::Postprocess( - fd_infer_results, &result, im_info, conf_threshold, - nms_iou_threshold, multi_label, max_wh); - return result; - }) - .def_readwrite("size", &vision::detection::YOLOv5::size_) - .def_readwrite("padding_value", - &vision::detection::YOLOv5::padding_value_) - .def_readwrite("is_mini_pad", &vision::detection::YOLOv5::is_mini_pad_) - .def_readwrite("is_no_pad", &vision::detection::YOLOv5::is_no_pad_) - .def_readwrite("is_scale_up", &vision::detection::YOLOv5::is_scale_up_) - .def_readwrite("stride", &vision::detection::YOLOv5::stride_) - .def_readwrite("max_wh", &vision::detection::YOLOv5::max_wh_) - .def_readwrite("multi_label", &vision::detection::YOLOv5::multi_label_); -} -} // namespace fastdeploy diff --git a/python/fastdeploy/vision/detection/__init__.py b/python/fastdeploy/vision/detection/__init__.py old mode 100644 new mode 100755 index a4fe4c035be..47d175af755 --- a/python/fastdeploy/vision/detection/__init__.py +++ b/python/fastdeploy/vision/detection/__init__.py @@ -18,7 +18,7 @@ from .contrib.scaled_yolov4 import ScaledYOLOv4 from .contrib.nanodet_plus import NanoDetPlus from .contrib.yolox import YOLOX -from .contrib.yolov5 import YOLOv5 +from .contrib.yolov5 import * from .contrib.yolov5lite import YOLOv5Lite from .contrib.yolov6 import YOLOv6 from .contrib.yolov7end2end_trt import YOLOv7End2EndTRT diff --git a/python/fastdeploy/vision/detection/contrib/yolov5.py b/python/fastdeploy/vision/detection/contrib/yolov5.py index 5ecef307bc8..42eccb88d4a 100644 --- a/python/fastdeploy/vision/detection/contrib/yolov5.py +++ b/python/fastdeploy/vision/detection/contrib/yolov5.py @@ -18,109 +18,31 @@ from .... import c_lib_wrap as C -class YOLOv5(FastDeployModel): - def __init__(self, - model_file, - params_file="", - runtime_option=None, - model_format=ModelFormat.ONNX): - """Load a YOLOv5 model exported by YOLOv5. - - :param model_file: (str)Path of model file, e.g ./yolov5.onnx - :param params_file: (str)Path of parameters file, e.g yolox/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string - :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU - :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model +class YOLOv5Preprocessor: + def __init__(self): + """Create a preprocessor for YOLOv5 """ - # 调用基函数进行backend_option的初始化 - # 初始化后的option保存在self._runtime_option - super(YOLOv5, self).__init__(runtime_option) + self._preprocessor = C.vision.detection.YOLOv5Preprocessor() - self._model = C.vision.detection.YOLOv5( - model_file, params_file, self._runtime_option, model_format) - # 通过self.initialized判断整个模型的初始化是否成功 - assert self.initialized, "YOLOv5 initialize failed." + def run(self, input_ims): + """Preprocess input images for YOLOv5 - def predict(self, input_image, conf_threshold=0.25, nms_iou_threshold=0.5): - """Detect an input image - - :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format - :param conf_threshold: confidence threashold for postprocessing, default is 0.25 - :param nms_iou_threshold: iou threashold for NMS, default is 0.5 - :return: DetectionResult + :param: input_ims: (list of numpy.ndarray)The input image + :return: list of FDTensor """ - return self._model.predict(input_image, conf_threshold, - nms_iou_threshold) - - @staticmethod - def preprocess(input_image, - size=[640, 640], - padding_value=[114.0, 114.0, 114.0], - is_mini_pad=False, - is_no_pad=False, - is_scale_up=False, - stride=32, - max_wh=7680.0, - multi_label=True): - return C.vision.detection.YOLOv5.preprocess( - input_image, size, padding_value, is_mini_pad, is_no_pad, - is_scale_up, stride, max_wh, multi_label) - - @staticmethod - def postprocess(infer_result, - im_info, - conf_threshold=0.25, - nms_iou_threshold=0.5, - multi_label=True, - max_wh=7680.0): - return C.vision.detection.YOLOv5.postprocess( - infer_result, im_info, conf_threshold, nms_iou_threshold, - multi_label, max_wh) - - # 一些跟YOLOv5模型有关的属性封装 - # 多数是预处理相关，可通过修改如model.size = [1280, 1280]改变预处理时resize的大小（前提是模型支持） + return self._preprocessor.run(input_ims) + @property def size(self): """ Argument for image preprocessing step, the preprocess image size, tuple of (width, height), default size = [640, 640] """ - return self._model.size + return self._preprocessor.size @property def padding_value(self): # padding value, size should be the same as channels - return self._model.padding_value - - @property - def is_no_pad(self): - # while is_mini_pad = false and is_no_pad = true, will resize the image to the set size - return self._model.is_no_pad - - @property - def is_mini_pad(self): - # only pad to the minimum rectange which height and width is times of stride - return self._model.is_mini_pad - - @property - def is_scale_up(self): - # if is_scale_up is false, the input image only can be zoom out, the maximum resize scale cannot exceed 1.0 - return self._model.is_scale_up - - @property - def stride(self): - # padding stride, for is_mini_pad - return self._model.stride - - @property - def max_wh(self): - # for offseting the boxes by classes when using NMS - return self._model.max_wh - - @property - def multi_label(self): - """ - Argument for image preprocessing step, for different strategies to get boxes when postprocessing, default True - """ - return self._model.multi_label + return self._preprocessor.padding_value @size.setter def size(self, wh): @@ -129,50 +51,129 @@ def size(self, wh): assert len(wh) == 2,\ "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( len(wh)) - self._model.size = wh + self._preprocessor.size = wh @padding_value.setter def padding_value(self, value): assert isinstance( value, list), "The value to set `padding_value` must be type of list." - self._model.padding_value = value + self._preprocessor.padding_value = value - @is_no_pad.setter - def is_no_pad(self, value): - assert isinstance( - value, bool), "The value to set `is_no_pad` must be type of bool." - self._model.is_no_pad = value - @is_mini_pad.setter - def is_mini_pad(self, value): - assert isinstance( - value, - bool), "The value to set `is_mini_pad` must be type of bool." - self._model.is_mini_pad = value +class YOLOv5Postprocessor: + def __init__(self): + """Create a postprocessor for YOLOv5 + """ + self._postprocessor = C.vision.detection.YOLOv5Postprocessor() - @is_scale_up.setter - def is_scale_up(self, value): - assert isinstance( - value, - bool), "The value to set `is_scale_up` must be type of bool." - self._model.is_scale_up = value + def run(self, runtime_results, ims_info): + """Postprocess the runtime results for YOLOv5 - @stride.setter - def stride(self, value): - assert isinstance( - value, int), "The value to set `stride` must be type of int." - self._model.stride = value + :param: runtime_results: (list of FDTensor)The output FDTensor results from runtime + :param: ims_info: (list of dict)Record input_shape and output_shape + :return: list of DetectionResult(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results, ims_info) - @max_wh.setter - def max_wh(self, value): - assert isinstance( - value, float), "The value to set `max_wh` must be type of float." - self._model.max_wh = value + @property + def conf_threshold(self): + """ + confidence threshold for postprocessing, default is 0.25 + """ + return self._postprocessor.conf_threshold + + @property + def nms_threshold(self): + """ + nms threshold for postprocessing, default is 0.5 + """ + return self._postprocessor.nms_threshold + + @property + def multi_label(self): + """ + multi_label for postprocessing, default is true + """ + return self._postprocessor.multi_label + + @conf_threshold.setter + def conf_threshold(self, conf_threshold): + assert isinstance(conf_threshold, float),\ + "The value to set `conf_threshold` must be type of float." + self._postprocessor.conf_threshold = conf_threshold + + @nms_threshold.setter + def nms_threshold(self, nms_threshold): + assert isinstance(nms_threshold, float),\ + "The value to set `nms_threshold` must be type of float." + self._postprocessor.nms_threshold = nms_threshold @multi_label.setter def multi_label(self, value): assert isinstance( value, bool), "The value to set `multi_label` must be type of bool." - self._model.multi_label = value + self._postprocessor.multi_label = value + + +class YOLOv5(FastDeployModel): + def __init__(self, + model_file, + params_file="", + runtime_option=None, + model_format=ModelFormat.ONNX): + """Load a YOLOv5 model exported by YOLOv5. + + :param model_file: (str)Path of model file, e.g ./yolov5.onnx + :param params_file: (str)Path of parameters file, e.g yolox/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + # 调用基函数进行backend_option的初始化 + # 初始化后的option保存在self._runtime_option + super(YOLOv5, self).__init__(runtime_option) + + assert model_format == ModelFormat.ONNX, "YOLOv5 only support model format of ModelFormat.ONNX now." + self._model = C.vision.detection.YOLOv5( + model_file, params_file, self._runtime_option, model_format) + # 通过self.initialized判断整个模型的初始化是否成功 + assert self.initialized, "YOLOv5 initialize failed." + + def predict(self, input_image, conf_threshold=0.25, nms_iou_threshold=0.5): + """Detect an input image + + :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :param conf_threshold: confidence threshold for postprocessing, default is 0.25 + :param nms_iou_threshold: iou threshold for NMS, default is 0.5 + :return: DetectionResult + """ + + self.postprocessor.conf_threshold = conf_threshold + self.postprocessor.nms_threshold = nms_iou_threshold + return self._model.predict(input_image) + + def batch_predict(self, images): + """Classify a batch of input image + + :param im: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return list of DetectionResult + """ + + return self._model.batch_predict(images) + + @property + def preprocessor(self): + """Get YOLOv5Preprocessor object of the loaded model + + :return YOLOv5Preprocessor + """ + return self._model.preprocessor + + @property + def postprocessor(self): + """Get YOLOv5Postprocessor object of the loaded model + + :return YOLOv5Postprocessor + """ + return self._model.postprocessor diff --git a/tests/models/test_yolov5.py b/tests/models/test_yolov5.py new file mode 100755 index 00000000000..3d79ed33d61 --- /dev/null +++ b/tests/models/test_yolov5.py @@ -0,0 +1,165 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fastdeploy import ModelFormat +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_yolov5(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s.onnx" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + input_url2 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000570688.jpg" + result_url1 = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov5_result1.pkl" + result_url2 = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov5_result2.pkl" + fd.download(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(input_url2, "resources") + fd.download(result_url1, "resources") + fd.download(result_url2, "resources") + + model_file = "resources/yolov5s.onnx" + model = fd.vision.detection.YOLOv5( + model_file, runtime_option=rc.test_option) + + with open("resources/yolov5_result1.pkl", "rb") as f: + expect1 = pickle.load(f) + + with open("resources/yolov5_result2.pkl", "rb") as f: + expect2 = pickle.load(f) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + im2 = cv2.imread("./resources/000000570688.jpg") + + for i in range(3): + # test single predict + result1 = model.predict(im1) + result2 = model.predict(im2) + + diff_boxes_1 = np.fabs( + np.array(result1.boxes) - np.array(expect1["boxes"])) + diff_boxes_2 = np.fabs( + np.array(result2.boxes) - np.array(expect2["boxes"])) + + diff_label_1 = np.fabs( + np.array(result1.label_ids) - np.array(expect1["label_ids"])) + diff_label_2 = np.fabs( + np.array(result2.label_ids) - np.array(expect2["label_ids"])) + + diff_scores_1 = np.fabs( + np.array(result1.scores) - np.array(expect1["scores"])) + diff_scores_2 = np.fabs( + np.array(result2.scores) - np.array(expect2["scores"])) + + assert diff_boxes_1.max( + ) < 1e-06, "There's difference in detection boxes 1." + assert diff_label_1.max( + ) < 1e-06, "There's difference in detection label 1." + assert diff_scores_1.max( + ) < 1e-05, "There's difference in detection score 1." + + assert diff_boxes_2.max( + ) < 1e-06, "There's difference in detection boxes 2." + assert diff_label_2.max( + ) < 1e-06, "There's difference in detection label 2." + assert diff_scores_2.max( + ) < 1e-05, "There's difference in detection score 2." + + # test batch predict + results = model.batch_predict([im1, im2]) + result1 = results[0] + result2 = results[1] + + diff_boxes_1 = np.fabs( + np.array(result1.boxes) - np.array(expect1["boxes"])) + diff_boxes_2 = np.fabs( + np.array(result2.boxes) - np.array(expect2["boxes"])) + + diff_label_1 = np.fabs( + np.array(result1.label_ids) - np.array(expect1["label_ids"])) + diff_label_2 = np.fabs( + np.array(result2.label_ids) - np.array(expect2["label_ids"])) + + diff_scores_1 = np.fabs( + np.array(result1.scores) - np.array(expect1["scores"])) + diff_scores_2 = np.fabs( + np.array(result2.scores) - np.array(expect2["scores"])) + assert diff_boxes_1.max( + ) < 1e-06, "There's difference in detection boxes 1." + assert diff_label_1.max( + ) < 1e-06, "There's difference in detection label 1." + assert diff_scores_1.max( + ) < 1e-05, "There's difference in detection score 1." + + assert diff_boxes_2.max( + ) < 1e-06, "There's difference in detection boxes 2." + assert diff_label_2.max( + ) < 1e-06, "There's difference in detection label 2." + assert diff_scores_2.max( + ) < 1e-05, "There's difference in detection score 2." + + +def test_detection_yolov5_runtime(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s.onnx" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url1 = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov5_result1.pkl" + fd.download(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url1, "resources") + + model_file = "resources/yolov5s.onnx" + + preprocessor = fd.vision.detection.YOLOv5Preprocessor() + postprocessor = fd.vision.detection.YOLOv5Postprocessor() + + rc.test_option.set_model_path(model_file, model_format=ModelFormat.ONNX) + rc.test_option.use_openvino_backend() + runtime = fd.Runtime(rc.test_option) + + with open("resources/yolov5_result1.pkl", "rb") as f: + expect1 = pickle.load(f) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + + for i in range(3): + # test runtime + input_tensors, ims_info = preprocessor.run([im1.copy()]) + output_tensors = runtime.infer({"images": input_tensors[0]}) + results = postprocessor.run(output_tensors, ims_info) + result1 = results[0] + + diff_boxes_1 = np.fabs( + np.array(result1.boxes) - np.array(expect1["boxes"])) + diff_label_1 = np.fabs( + np.array(result1.label_ids) - np.array(expect1["label_ids"])) + diff_scores_1 = np.fabs( + np.array(result1.scores) - np.array(expect1["scores"])) + + assert diff_boxes_1.max( + ) < 1e-04, "There's difference in detection boxes 1." + assert diff_label_1.max( + ) < 1e-06, "There's difference in detection label 1." + assert diff_scores_1.max( + ) < 1e-05, "There's difference in detection score 1." + + +if __name__ == "__main__": + test_detection_yolov5() + test_detection_yolov5_runtime() From 3c4f3a80290213a509fd1ee97fb123bf65670693 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 15 Nov 2022 10:43:23 +0800 Subject: [PATCH 028/136] [Model] Refactor PaddleDetection module (#575) * Add namespace for functions * Refactor PaddleDetection module * finish all the single image test * Update preprocessor.cc * fix some litte detail * add python api * Update postprocessor.cc --- .../paddledetection/python/infer_mask_rcnn.py | 1 - fastdeploy/pipeline/pptinypose/pipeline.cc | 2 +- fastdeploy/pipeline/pptinypose/pipeline.h | 4 +- .../pptinypose/pptinyposepipeline_pybind.cc | 23 -- fastdeploy/vision.h | 1 - .../vision/detection/contrib/rknpu2/model.h | 16 - .../detection/contrib/rknpu2/rkdet_pybind.cc | 29 -- .../detection/contrib/rknpu2/rkpicodet.cc | 201 ------------- .../detection/contrib/rknpu2/rkpicodet.h | 46 --- .../vision/detection/detection_pybind.cc | 3 - fastdeploy/vision/detection/ppdet/base.cc | 68 +++++ .../detection/ppdet/{ppyoloe.h => base.h} | 62 ++-- .../vision/detection/ppdet/mask_rcnn.cc | 120 -------- fastdeploy/vision/detection/ppdet/model.h | 156 +++++++++- fastdeploy/vision/detection/ppdet/picodet.cc | 66 ----- fastdeploy/vision/detection/ppdet/picodet.h | 36 --- .../vision/detection/ppdet/postprocessor.cc | 132 +++++++++ .../ppdet/{mask_rcnn.h => postprocessor.h} | 34 ++- .../vision/detection/ppdet/ppdet_pybind.cc | 136 ++++----- fastdeploy/vision/detection/ppdet/ppyolo.cc | 78 ----- fastdeploy/vision/detection/ppdet/ppyolo.h | 52 ---- fastdeploy/vision/detection/ppdet/ppyoloe.cc | 274 ------------------ .../vision/detection/ppdet/preprocessor.cc | 201 +++++++++++++ .../vision/detection/ppdet/preprocessor.h | 50 ++++ fastdeploy/vision/detection/ppdet/rcnn.cc | 84 ------ fastdeploy/vision/detection/ppdet/rcnn.h | 39 --- fastdeploy/vision/detection/ppdet/yolov3.cc | 64 ---- fastdeploy/vision/detection/ppdet/yolov3.h | 35 --- fastdeploy/vision/detection/ppdet/yolox.cc | 74 ----- fastdeploy/vision/detection/ppdet/yolox.h | 35 --- .../fastdeploy/vision/detection/__init__.py | 3 +- .../vision/detection/ppdet/__init__.py | 104 +++++-- tests/models/test_faster_rcnn.py | 70 +++++ tests/models/test_mask_rcnn.py | 68 +++++ tests/models/test_picodet.py | 69 +++++ tests/models/test_pp_yolox.py | 70 +++++ tests/models/test_ppyolo.py | 69 +++++ tests/models/test_ppyoloe.py | 68 +++++ tests/models/test_yolov3.py | 69 +++++ 39 files changed, 1278 insertions(+), 1434 deletions(-) delete mode 100644 fastdeploy/vision/detection/contrib/rknpu2/model.h delete mode 100644 fastdeploy/vision/detection/contrib/rknpu2/rkdet_pybind.cc delete mode 100644 fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.cc delete mode 100644 fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.h create mode 100755 fastdeploy/vision/detection/ppdet/base.cc rename fastdeploy/vision/detection/ppdet/{ppyoloe.h => base.h} (61%) delete mode 100644 fastdeploy/vision/detection/ppdet/mask_rcnn.cc delete mode 100644 fastdeploy/vision/detection/ppdet/picodet.cc delete mode 100644 fastdeploy/vision/detection/ppdet/picodet.h create mode 100644 fastdeploy/vision/detection/ppdet/postprocessor.cc rename fastdeploy/vision/detection/ppdet/{mask_rcnn.h => postprocessor.h} (50%) delete mode 100644 fastdeploy/vision/detection/ppdet/ppyolo.cc delete mode 100644 fastdeploy/vision/detection/ppdet/ppyolo.h delete mode 100755 fastdeploy/vision/detection/ppdet/ppyoloe.cc create mode 100644 fastdeploy/vision/detection/ppdet/preprocessor.cc create mode 100644 fastdeploy/vision/detection/ppdet/preprocessor.h delete mode 100644 fastdeploy/vision/detection/ppdet/rcnn.cc delete mode 100644 fastdeploy/vision/detection/ppdet/rcnn.h delete mode 100644 fastdeploy/vision/detection/ppdet/yolov3.cc delete mode 100644 fastdeploy/vision/detection/ppdet/yolov3.h delete mode 100644 fastdeploy/vision/detection/ppdet/yolox.cc delete mode 100644 fastdeploy/vision/detection/ppdet/yolox.h create mode 100755 tests/models/test_faster_rcnn.py create mode 100755 tests/models/test_mask_rcnn.py create mode 100755 tests/models/test_picodet.py create mode 100755 tests/models/test_pp_yolox.py create mode 100755 tests/models/test_ppyolo.py create mode 100755 tests/models/test_ppyoloe.py create mode 100755 tests/models/test_yolov3.py diff --git a/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py b/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py index 7d3c2e5e41b..8b6c49aa794 100644 --- a/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py +++ b/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py @@ -66,4 +66,3 @@ def build_option(args): vis_im = fd.vision.vis_detection(im, result, score_threshold=0.5) cv2.imwrite("visualized_result.jpg", vis_im) print("Visualized result save in ./visualized_result.jpg") -print(runtime_option) diff --git a/fastdeploy/pipeline/pptinypose/pipeline.cc b/fastdeploy/pipeline/pptinypose/pipeline.cc index 6a2f2d4ba52..8c9d40a84ff 100644 --- a/fastdeploy/pipeline/pptinypose/pipeline.cc +++ b/fastdeploy/pipeline/pptinypose/pipeline.cc @@ -17,7 +17,7 @@ namespace fastdeploy { namespace pipeline { PPTinyPose::PPTinyPose( - fastdeploy::vision::detection::PPYOLOE* det_model, + fastdeploy::vision::detection::PicoDet* det_model, fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model) : detector_(det_model), pptinypose_model_(pptinypose_model) {} diff --git a/fastdeploy/pipeline/pptinypose/pipeline.h b/fastdeploy/pipeline/pptinypose/pipeline.h index 90d6e21f0d9..0cdbee39958 100644 --- a/fastdeploy/pipeline/pptinypose/pipeline.h +++ b/fastdeploy/pipeline/pptinypose/pipeline.h @@ -35,7 +35,7 @@ class FASTDEPLOY_DECL PPTinyPose { * \param[in] pptinypose_model Initialized pptinypose model object */ PPTinyPose( - fastdeploy::vision::detection::PPYOLOE* det_model, + fastdeploy::vision::detection::PicoDet* det_model, fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model); /** \brief Predict the keypoint detection result for an input image @@ -52,7 +52,7 @@ class FASTDEPLOY_DECL PPTinyPose { float detection_model_score_threshold = 0; protected: - fastdeploy::vision::detection::PPYOLOE* detector_ = nullptr; + fastdeploy::vision::detection::PicoDet* detector_ = nullptr; fastdeploy::vision::keypointdetection::PPTinyPose* pptinypose_model_ = nullptr; diff --git a/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc b/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc index 8aee0474e4f..b020bb1b49d 100644 --- a/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc +++ b/fastdeploy/pipeline/pptinypose/pptinyposepipeline_pybind.cc @@ -18,31 +18,8 @@ namespace fastdeploy { void BindPPTinyPosePipeline(pybind11::module& m) { pybind11::class_(m, "PPTinyPose") - // explicitly pybind more kinds of detection models here - .def(pybind11::init()) - .def(pybind11::init()) - - .def(pybind11::init()) - - .def(pybind11::init()) - - .def(pybind11::init()) - - .def(pybind11::init()) - - .def(pybind11::init()) - - .def(pybind11::init()) - .def("predict", [](pipeline::PPTinyPose& self, pybind11::array& data) { auto mat = PyArrayToCvMat(data); diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index 15cc1d009db..2f8c7066131 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -29,7 +29,6 @@ #include "fastdeploy/vision/detection/contrib/yolov7end2end_trt.h" #include "fastdeploy/vision/detection/contrib/yolox.h" #include "fastdeploy/vision/detection/ppdet/model.h" -#include "fastdeploy/vision/detection/contrib/rknpu2/model.h" #include "fastdeploy/vision/facedet/contrib/retinaface.h" #include "fastdeploy/vision/facedet/contrib/scrfd.h" #include "fastdeploy/vision/facedet/contrib/ultraface.h" diff --git a/fastdeploy/vision/detection/contrib/rknpu2/model.h b/fastdeploy/vision/detection/contrib/rknpu2/model.h deleted file mode 100644 index f0f8616eed2..00000000000 --- a/fastdeploy/vision/detection/contrib/rknpu2/model.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.h" diff --git a/fastdeploy/vision/detection/contrib/rknpu2/rkdet_pybind.cc b/fastdeploy/vision/detection/contrib/rknpu2/rkdet_pybind.cc deleted file mode 100644 index 6482ea67553..00000000000 --- a/fastdeploy/vision/detection/contrib/rknpu2/rkdet_pybind.cc +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include "fastdeploy/pybind/main.h" - -namespace fastdeploy { -void BindRKDet(pybind11::module& m) { - pybind11::class_(m, "RKPicoDet") - .def(pybind11::init()) - .def("predict", - [](vision::detection::RKPicoDet& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); -} -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.cc b/fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.cc deleted file mode 100644 index 926214d86d6..00000000000 --- a/fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.cc +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.h" -#include "yaml-cpp/yaml.h" -namespace fastdeploy { -namespace vision { -namespace detection { - -RKPicoDet::RKPicoDet(const std::string& model_file, - const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::ORT}; - valid_rknpu_backends = {Backend::RKNPU2}; - if ((model_format == ModelFormat::RKNN) || - (model_format == ModelFormat::ONNX)) { - has_nms_ = false; - } - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - - // NMS parameters come from RKPicoDet_s_nms - background_label = -1; - keep_top_k = 100; - nms_eta = 1; - nms_threshold = 0.5; - nms_top_k = 1000; - normalized = true; - score_threshold = 0.3; - initialized = Initialize(); -} - -bool RKPicoDet::Initialize() { - if (!BuildPreprocessPipelineFromConfig()) { - FDERROR << "Failed to build preprocess pipeline from configuration file." - << std::endl; - return false; - } - if (!InitRuntime()) { - FDERROR << "Failed to initialize fastdeploy backend." << std::endl; - return false; - } - return true; -} - -bool RKPicoDet::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - for (size_t i = 0; i < processors_.size(); ++i) { - if (!(*(processors_[i].get()))(mat)) { - FDERROR << "Failed to process image data in " << processors_[i]->Name() - << "." << std::endl; - return false; - } - } - - Cast::Run(mat, "float"); - - scale_factor.resize(2); - scale_factor[0] = mat->Height() * 1.0 / origin_h; - scale_factor[1] = mat->Width() * 1.0 / origin_w; - - outputs->resize(1); - (*outputs)[0].name = InputInfoOfRuntime(0).name; - mat->ShareWithTensor(&((*outputs)[0])); - // reshape to [1, c, h, w] - (*outputs)[0].shape.insert((*outputs)[0].shape.begin(), 1); - return true; -} - -bool RKPicoDet::BuildPreprocessPipelineFromConfig() { - processors_.clear(); - YAML::Node cfg; - try { - cfg = YAML::LoadFile(config_file_); - } catch (YAML::BadFile& e) { - FDERROR << "Failed to load yaml file " << config_file_ - << ", maybe you should check this file." << std::endl; - return false; - } - - processors_.push_back(std::make_shared()); - - for (const auto& op : cfg["Preprocess"]) { - std::string op_name = op["type"].as(); - if (op_name == "NormalizeImage") { - continue; - } else if (op_name == "Resize") { - bool keep_ratio = op["keep_ratio"].as(); - auto target_size = op["target_size"].as>(); - int interp = op["interp"].as(); - FDASSERT(target_size.size() == 2, - "Require size of target_size be 2, but now it's %lu.", - target_size.size()); - if (!keep_ratio) { - int width = target_size[1]; - int height = target_size[0]; - processors_.push_back( - std::make_shared(width, height, -1.0, -1.0, interp, false)); - } else { - int min_target_size = std::min(target_size[0], target_size[1]); - int max_target_size = std::max(target_size[0], target_size[1]); - std::vector max_size; - if (max_target_size > 0) { - max_size.push_back(max_target_size); - max_size.push_back(max_target_size); - } - processors_.push_back(std::make_shared( - min_target_size, interp, true, max_size)); - } - } else if (op_name == "Permute") { - continue; - } else if (op_name == "Pad") { - auto size = op["size"].as>(); - auto value = op["fill_value"].as>(); - processors_.push_back(std::make_shared("float")); - processors_.push_back( - std::make_shared(size[1], size[0], value)); - } else if (op_name == "PadStride") { - auto stride = op["stride"].as(); - processors_.push_back( - std::make_shared(stride, std::vector(3, 0))); - } else { - FDERROR << "Unexcepted preprocess operator: " << op_name << "." - << std::endl; - return false; - } - } - return true; -} - -bool RKPicoDet::Postprocess(std::vector& infer_result, - DetectionResult* result) { - FDASSERT(infer_result[1].shape[0] == 1, - "Only support batch = 1 in FastDeploy now."); - - if (!has_nms_) { - int boxes_index = 0; - int scores_index = 1; - if (infer_result[0].shape[1] == infer_result[1].shape[2]) { - boxes_index = 0; - scores_index = 1; - } else if (infer_result[0].shape[2] == infer_result[1].shape[1]) { - boxes_index = 1; - scores_index = 0; - } else { - FDERROR << "The shape of boxes and scores should be [batch, boxes_num, " - "4], [batch, classes_num, boxes_num]" - << std::endl; - return false; - } - - backend::MultiClassNMS nms; - nms.background_label = background_label; - nms.keep_top_k = keep_top_k; - nms.nms_eta = nms_eta; - nms.nms_threshold = nms_threshold; - nms.score_threshold = score_threshold; - nms.nms_top_k = nms_top_k; - nms.normalized = normalized; - nms.Compute(static_cast(infer_result[boxes_index].Data()), - static_cast(infer_result[scores_index].Data()), - infer_result[boxes_index].shape, - infer_result[scores_index].shape); - if (nms.out_num_rois_data[0] > 0) { - result->Reserve(nms.out_num_rois_data[0]); - } - for (size_t i = 0; i < nms.out_num_rois_data[0]; ++i) { - result->label_ids.push_back(nms.out_box_data[i * 6]); - result->scores.push_back(nms.out_box_data[i * 6 + 1]); - result->boxes.emplace_back( - std::array{nms.out_box_data[i * 6 + 2] / scale_factor[1], - nms.out_box_data[i * 6 + 3] / scale_factor[0], - nms.out_box_data[i * 6 + 4] / scale_factor[1], - nms.out_box_data[i * 6 + 5] / scale_factor[0]}); - } - } else { - FDERROR << "Picodet in Backend::RKNPU2 don't support NMS" << std::endl; - } - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.h b/fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.h deleted file mode 100644 index dbb48c16d93..00000000000 --- a/fastdeploy/vision/detection/contrib/rknpu2/rkpicodet.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" - -namespace fastdeploy { -namespace vision { -namespace detection { -class FASTDEPLOY_DECL RKPicoDet : public PPYOLOE { - public: - RKPicoDet(const std::string& model_file, - const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::RKNN); - - virtual std::string ModelName() const { return "RKPicoDet"; } - - protected: - /// Build the preprocess pipeline from the loaded model - virtual bool BuildPreprocessPipelineFromConfig(); - /// Preprocess an input image, and set the preprocessed results to `outputs` - virtual bool Preprocess(Mat* mat, std::vector* outputs); - - /// Postprocess the inferenced results, and set the final result to `result` - virtual bool Postprocess(std::vector& infer_result, - DetectionResult* result); - virtual bool Initialize(); - private: - std::vector scale_factor{1.0, 1.0}; -}; -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/detection_pybind.cc b/fastdeploy/vision/detection/detection_pybind.cc index f55bf68bf1b..b3a7a6ad941 100644 --- a/fastdeploy/vision/detection/detection_pybind.cc +++ b/fastdeploy/vision/detection/detection_pybind.cc @@ -27,8 +27,6 @@ void BindNanoDetPlus(pybind11::module& m); void BindPPDet(pybind11::module& m); void BindYOLOv7End2EndTRT(pybind11::module& m); void BindYOLOv7End2EndORT(pybind11::module& m); -void BindRKDet(pybind11::module& m); - void BindDetection(pybind11::module& m) { auto detection_module = @@ -44,6 +42,5 @@ void BindDetection(pybind11::module& m) { BindNanoDetPlus(detection_module); BindYOLOv7End2EndTRT(detection_module); BindYOLOv7End2EndORT(detection_module); - BindRKDet(detection_module); } } // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/base.cc b/fastdeploy/vision/detection/ppdet/base.cc new file mode 100755 index 00000000000..1db42f1582d --- /dev/null +++ b/fastdeploy/vision/detection/ppdet/base.cc @@ -0,0 +1,68 @@ +#include "fastdeploy/vision/detection/ppdet/base.h" +#include "fastdeploy/vision/utils/utils.h" +#include "yaml-cpp/yaml.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +PPDetBase::PPDetBase(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) : preprocessor_(config_file) { + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; +} + +bool PPDetBase::Initialize() { + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool PPDetBase::Predict(cv::Mat* im, DetectionResult* result) { + return Predict(*im, result); +} + +bool PPDetBase::Predict(const cv::Mat& im, DetectionResult* result) { + std::vector results; + if (!BatchPredict({im}, &results)) { + return false; + } + *result = std::move(results[0]); + return true; +} + +bool PPDetBase::BatchPredict(const std::vector& imgs, std::vector* results) { + std::vector fd_images = WrapMat(imgs); + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_)) { + FDERROR << "Failed to preprocess the input image." << std::endl; + return false; + } + reused_input_tensors_[0].name = "image"; + reused_input_tensors_[1].name = "scale_factor"; + reused_input_tensors_[2].name = "im_shape"; + // Some models don't need im_shape as input + if (NumInputsOfRuntime() == 2) { + reused_input_tensors_.pop_back(); + } + + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; + return false; + } + + if (!postprocessor_.Run(reused_output_tensors_, results)) { + FDERROR << "Failed to postprocess the inference results by runtime." << std::endl; + return false; + } + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.h b/fastdeploy/vision/detection/ppdet/base.h similarity index 61% rename from fastdeploy/vision/detection/ppdet/ppyoloe.h rename to fastdeploy/vision/detection/ppdet/base.h index fd2a71cb171..bffc477a5b1 100644 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.h +++ b/fastdeploy/vision/detection/ppdet/base.h @@ -14,6 +14,8 @@ #pragma once #include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/detection/ppdet/preprocessor.h" +#include "fastdeploy/vision/detection/ppdet/postprocessor.h" #include "fastdeploy/vision/common/processors/transform.h" #include "fastdeploy/vision/common/result.h" @@ -26,9 +28,9 @@ namespace vision { */ namespace detection { -/*! @brief PPYOLOE model object used when to load a PPYOLOE model exported by PaddleDetection +/*! @brief Base model object used when to load a model exported by PaddleDetection */ -class FASTDEPLOY_DECL PPYOLOE : public FastDeployModel { +class FASTDEPLOY_DECL PPDetBase : public FastDeployModel { public: /** \brief Set path of model file and configuration file, and the configuration of runtime * @@ -38,49 +40,49 @@ class FASTDEPLOY_DECL PPYOLOE : public FastDeployModel { * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` * \param[in] model_format Model format of the loaded model, default is Paddle format */ - PPYOLOE(const std::string& model_file, const std::string& params_file, + PPDetBase(const std::string& model_file, const std::string& params_file, const std::string& config_file, const RuntimeOption& custom_option = RuntimeOption(), const ModelFormat& model_format = ModelFormat::PADDLE); /// Get model's name - virtual std::string ModelName() const { return "PaddleDetection/PPYOLOE"; } + virtual std::string ModelName() const { return "PaddleDetection/BaseModel"; } - /** \brief Predict the detection result for an input image + /** \brief DEPRECATED Predict the detection result for an input image * * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format - * \param[in] result The output detection result will be writen to this structure + * \param[in] result The output detection result * \return true if the prediction successed, otherwise false */ virtual bool Predict(cv::Mat* im, DetectionResult* result); - protected: - PPYOLOE() {} - virtual bool Initialize(); - /// Build the preprocess pipeline from the loaded model - virtual bool BuildPreprocessPipelineFromConfig(); - /// Preprocess an input image, and set the preprocessed results to `outputs` - virtual bool Preprocess(Mat* mat, std::vector* outputs); + /** \brief Predict the detection result for an input image + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output detection result + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(const cv::Mat& im, DetectionResult* result); + + /** \brief Predict the detection result for an input image list + * \param[in] im The input image list, all the elements come from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] results The output detection result list + * \return true if the prediction successed, otherwise false + */ + virtual bool BatchPredict(const std::vector& imgs, + std::vector* results); - /// Postprocess the inferenced results, and set the final result to `result` - virtual bool Postprocess(std::vector& infer_result, - DetectionResult* result); + PaddleDetPreprocessor& GetPreprocessor() { + return preprocessor_; + } - std::vector> processors_; - std::string config_file_; - // configuration for nms - int64_t background_label = -1; - int64_t keep_top_k = 300; - float nms_eta = 1.0; - float nms_threshold = 0.7; - float score_threshold = 0.01; - int64_t nms_top_k = 10000; - bool normalized = true; - bool has_nms_ = true; + PaddleDetPostprocessor& GetPostprocessor() { + return postprocessor_; + } - // This function will used to check if this model contains multiclass_nms - // and get parameters from the operator - void GetNmsInfo(); + protected: + virtual bool Initialize(); + PaddleDetPreprocessor preprocessor_; + PaddleDetPostprocessor postprocessor_; }; } // namespace detection diff --git a/fastdeploy/vision/detection/ppdet/mask_rcnn.cc b/fastdeploy/vision/detection/ppdet/mask_rcnn.cc deleted file mode 100644 index 7c656c66953..00000000000 --- a/fastdeploy/vision/detection/ppdet/mask_rcnn.cc +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/ppdet/mask_rcnn.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -MaskRCNN::MaskRCNN(const std::string& model_file, - const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER}; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - initialized = Initialize(); -} - -bool MaskRCNN::Postprocess(std::vector& infer_result, - DetectionResult* result) { - // index 0: bbox_data [N, 6] float32 - // index 1: bbox_num [B=1] int32 - // index 2: mask_data [N, h, w] int32 - FDASSERT(infer_result[1].shape[0] == 1, - "Only support batch = 1 in FastDeploy now."); - FDASSERT(infer_result.size() == 3, - "The infer_result must contains 3 otuput Tensors, but found %lu", - infer_result.size()); - - FDTensor& box_tensor = infer_result[0]; - FDTensor& box_num_tensor = infer_result[1]; - FDTensor& mask_tensor = infer_result[2]; - - int box_num = 0; - if (box_num_tensor.dtype == FDDataType::INT32) { - box_num = *(static_cast(box_num_tensor.Data())); - } else if (box_num_tensor.dtype == FDDataType::INT64) { - box_num = *(static_cast(box_num_tensor.Data())); - } else { - FDASSERT(false, - "The output box_num of PaddleDetection/MaskRCNN model should be " - "type of int32/int64."); - } - if (box_num <= 0) { - return true; // no object detected. - } - result->Resize(box_num); - float* box_data = static_cast(box_tensor.Data()); - for (size_t i = 0; i < box_num; ++i) { - result->label_ids[i] = static_cast(box_data[i * 6]); - result->scores[i] = box_data[i * 6 + 1]; - result->boxes[i] = - std::array{box_data[i * 6 + 2], box_data[i * 6 + 3], - box_data[i * 6 + 4], box_data[i * 6 + 5]}; - } - result->contain_masks = true; - // TODO(qiuyanjun): Cast int64/int8 to int32. - FDASSERT(mask_tensor.dtype == FDDataType::INT32, - "The dtype of mask Tensor must be int32 now!"); - // In PaddleDetection/MaskRCNN, the mask_h and mask_w - // are already aligned with original input image. So, - // we need to crop it from output mask according to - // the detected bounding box. - // +-----------------------+ - // | x1,y1 | - // | +---------------+ | - // | | | | - // | | Crop | | - // | | | | - // | | | | - // | +---------------+ | - // | x2,y2 | - // +-----------------------+ - int64_t out_mask_h = mask_tensor.shape[1]; - int64_t out_mask_w = mask_tensor.shape[2]; - int64_t out_mask_numel = out_mask_h * out_mask_w; - int32_t* out_mask_data = static_cast(mask_tensor.Data()); - for (size_t i = 0; i < box_num; ++i) { - // crop instance mask according to box - int64_t x1 = static_cast(result->boxes[i][0]); - int64_t y1 = static_cast(result->boxes[i][1]); - int64_t x2 = static_cast(result->boxes[i][2]); - int64_t y2 = static_cast(result->boxes[i][3]); - int64_t keep_mask_h = y2 - y1; - int64_t keep_mask_w = x2 - x1; - int64_t keep_mask_numel = keep_mask_h * keep_mask_w; - result->masks[i].Resize(keep_mask_numel); // int32 - result->masks[i].shape = {keep_mask_h, keep_mask_w}; - int32_t* mask_start_ptr = out_mask_data + i * out_mask_numel; - int32_t* keep_mask_ptr = static_cast(result->masks[i].Data()); - for (size_t row = y1; row < y2; ++row) { - size_t keep_nbytes_in_col = keep_mask_w * sizeof(int32_t); - int32_t* out_row_start_ptr = mask_start_ptr + row * out_mask_w + x1; - int32_t* keep_row_start_ptr = keep_mask_ptr + (row - y1) * keep_mask_w; - std::memcpy(keep_row_start_ptr, out_row_start_ptr, keep_nbytes_in_col); - } - } - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/model.h b/fastdeploy/vision/detection/ppdet/model.h index f8a40d64fc2..90a92f89300 100644 --- a/fastdeploy/vision/detection/ppdet/model.h +++ b/fastdeploy/vision/detection/ppdet/model.h @@ -13,10 +13,152 @@ // limitations under the License. #pragma once -#include "fastdeploy/vision/detection/ppdet/mask_rcnn.h" -#include "fastdeploy/vision/detection/ppdet/picodet.h" -#include "fastdeploy/vision/detection/ppdet/ppyolo.h" -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" -#include "fastdeploy/vision/detection/ppdet/rcnn.h" -#include "fastdeploy/vision/detection/ppdet/yolov3.h" -#include "fastdeploy/vision/detection/ppdet/yolox.h" +#include "fastdeploy/vision/detection/ppdet/base.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +class FASTDEPLOY_DECL PicoDet : public PPDetBase { + public: + /** \brief Set path of model file and configuration file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g picodet/model.pdmodel + * \param[in] params_file Path of parameter file, e.g picodet/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] config_file Path of configuration file for deployment, e.g picodet/infer_cfg.yml + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * \param[in] model_format Model format of the loaded model, default is Paddle format + */ + PicoDet(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, + Backend::PDINFER, Backend::LITE}; + valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PicoDet"; } +}; + +class FASTDEPLOY_DECL PPYOLOE : public PPDetBase { + public: + /** \brief Set path of model file and configuration file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g ppyoloe/model.pdmodel + * \param[in] params_file Path of parameter file, e.g picodet/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] config_file Path of configuration file for deployment, e.g picodet/infer_cfg.yml + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * \param[in] model_format Model format of the loaded model, default is Paddle format + */ + PPYOLOE(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, + Backend::PDINFER, Backend::LITE}; + valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PPYOLOE"; } +}; + +class FASTDEPLOY_DECL PPYOLO : public PPDetBase { + public: + /** \brief Set path of model file and configuration file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g ppyolo/model.pdmodel + * \param[in] params_file Path of parameter file, e.g ppyolo/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] config_file Path of configuration file for deployment, e.g picodet/infer_cfg.yml + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * \param[in] model_format Model format of the loaded model, default is Paddle format + */ + PPYOLO(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PaddleDetection/PP-YOLO"; } +}; + +class FASTDEPLOY_DECL YOLOv3 : public PPDetBase { + public: + YOLOv3(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER, + Backend::LITE}; + valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PaddleDetection/YOLOv3"; } +}; + +class FASTDEPLOY_DECL PaddleYOLOX : public PPDetBase { + public: + PaddleYOLOX(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER, + Backend::LITE}; + valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PaddleDetection/YOLOX"; } +}; + +class FASTDEPLOY_DECL FasterRCNN : public PPDetBase { + public: + FasterRCNN(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::PDINFER, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PaddleDetection/FasterRCNN"; } +}; + +class FASTDEPLOY_DECL MaskRCNN : public PPDetBase { + public: + MaskRCNN(const std::string& model_file, const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) + : PPDetBase(model_file, params_file, config_file, custom_option, + model_format) { + valid_cpu_backends = {Backend::PDINFER, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER}; + initialized = Initialize(); + } + + virtual std::string ModelName() const { return "PaddleDetection/MaskRCNN"; } +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/picodet.cc b/fastdeploy/vision/detection/ppdet/picodet.cc deleted file mode 100644 index 9b67db4a77f..00000000000 --- a/fastdeploy/vision/detection/ppdet/picodet.cc +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/ppdet/picodet.h" -#include "yaml-cpp/yaml.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -PicoDet::PicoDet(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; - valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - background_label = -1; - keep_top_k = 100; - nms_eta = 1; - nms_threshold = 0.6; - nms_top_k = 1000; - normalized = true; - score_threshold = 0.025; - CheckIfContainDecodeAndNMS(); - initialized = Initialize(); -} - -bool PicoDet::CheckIfContainDecodeAndNMS() { - YAML::Node cfg; - try { - cfg = YAML::LoadFile(config_file_); - } catch (YAML::BadFile& e) { - FDERROR << "Failed to load yaml file " << config_file_ - << ", maybe you should check this file." << std::endl; - return false; - } - - if (cfg["arch"].as() == "PicoDet") { - FDERROR << "The arch in config file is PicoDet, which means this model " - "doesn contain box decode and nms, please export model with " - "decode and nms." - << std::endl; - return false; - } - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/picodet.h b/fastdeploy/vision/detection/ppdet/picodet.h deleted file mode 100644 index 5f85d2dd9b1..00000000000 --- a/fastdeploy/vision/detection/ppdet/picodet.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -class FASTDEPLOY_DECL PicoDet : public PPYOLOE { - public: - PicoDet(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); - - // Only support picodet contains decode and nms - bool CheckIfContainDecodeAndNMS(); - - virtual std::string ModelName() const { return "PicoDet"; } -}; -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/postprocessor.cc b/fastdeploy/vision/detection/ppdet/postprocessor.cc new file mode 100644 index 00000000000..5e8312a7d80 --- /dev/null +++ b/fastdeploy/vision/detection/ppdet/postprocessor.cc @@ -0,0 +1,132 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/ppdet/postprocessor.h" +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +bool PaddleDetPostprocessor::ProcessMask(const FDTensor& tensor, std::vector* results) { + auto shape = tensor.Shape(); + if (tensor.Dtype() != FDDataType::INT32) { + FDERROR << "The data type of out mask tensor should be INT32, but now it's " << tensor.Dtype() << std::endl; + return false; + } + int64_t out_mask_h = shape[1]; + int64_t out_mask_w = shape[2]; + int64_t out_mask_numel = shape[1] * shape[2]; + const int32_t* data = reinterpret_cast(tensor.CpuData()); + int index = 0; + + for (int i = 0; i < results->size(); ++i) { + (*results)[i].contain_masks = true; + (*results)[i].masks.resize((*results)[i].boxes.size()); + for (int j = 0; j < (*results)[i].boxes.size(); ++j) { + int x1 = static_cast((*results)[i].boxes[j][0]); + int y1 = static_cast((*results)[i].boxes[j][1]); + int x2 = static_cast((*results)[i].boxes[j][2]); + int y2 = static_cast((*results)[i].boxes[j][3]); + int keep_mask_h = y2 - y1; + int keep_mask_w = x2 - x1; + int keep_mask_numel = keep_mask_h * keep_mask_w; + (*results)[i].masks[j].Resize(keep_mask_numel); + (*results)[i].masks[j].shape = {keep_mask_h, keep_mask_w}; + const int32_t* current_ptr = data + index * out_mask_numel; + + int32_t* keep_mask_ptr = reinterpret_cast((*results)[i].masks[j].Data()); + for (int row = y1; row < y2; ++row) { + size_t keep_nbytes_in_col = keep_mask_w * sizeof(int32_t); + const int32_t* out_row_start_ptr = current_ptr + row * out_mask_w + x1; + int32_t* keep_row_start_ptr = keep_mask_ptr + (row - y1) * keep_mask_w; + std::memcpy(keep_row_start_ptr, out_row_start_ptr, keep_nbytes_in_col); + } + index += 1; + } + } + return true; +} + +bool PaddleDetPostprocessor::Run(const std::vector& tensors, std::vector* results) { + if (tensors[0].shape[0] == 0) { + // No detected boxes + return true; + } + + // Get number of boxes for each input image + std::vector num_boxes(tensors[1].shape[0]); + int total_num_boxes = 0; + if (tensors[1].dtype == FDDataType::INT32) { + const int32_t* data = static_cast(tensors[1].CpuData()); + for (size_t i = 0; i < tensors[1].shape[0]; ++i) { + num_boxes[i] = static_cast(data[i]); + total_num_boxes += num_boxes[i]; + } + } else if (tensors[1].dtype == FDDataType::INT64) { + const int64_t* data = static_cast(tensors[1].CpuData()); + for (size_t i = 0; i < tensors[1].shape[0]; ++i) { + num_boxes[i] = static_cast(data[i]); + } + } + + // Special case for TensorRT, it has fixed output shape of NMS + // So there's invalid boxes in its' output boxes + int num_output_boxes = tensors[0].Shape()[0]; + bool contain_invalid_boxes = false; + if (total_num_boxes != num_output_boxes) { + if (num_output_boxes % num_boxes.size() == 0) { + contain_invalid_boxes = true; + } else { + FDERROR << "Cannot handle the output data for this model, unexpected situation." << std::endl; + return false; + } + } + + // Get boxes for each input image + results->resize(num_boxes.size()); + const float* box_data = static_cast(tensors[0].CpuData()); + int offset = 0; + for (size_t i = 0; i < num_boxes.size(); ++i) { + const float* ptr = box_data + offset; + (*results)[i].Reserve(num_boxes[i]); + for (size_t j = 0; j < num_boxes[i]; ++j) { + (*results)[i].label_ids.push_back(static_cast(round(ptr[j * 6]))); + (*results)[i].scores.push_back(ptr[j * 6 + 1]); + (*results)[i].boxes.emplace_back(std::array({ptr[j * 6 + 2], ptr[j * 6 + 3], ptr[j * 6 + 4], ptr[j * 6 + 5]})); + } + if (contain_invalid_boxes) { + offset += (num_output_boxes * 6 / num_boxes.size()); + } else { + offset += (num_boxes[i] * 6); + } + } + + // Only detection + if (tensors.size() <= 2) { + return true; + } + + if (tensors[2].Shape()[0] != num_output_boxes) { + FDERROR << "The first dimension of output mask tensor:" << tensors[2].Shape()[0] << " is not equal to the first dimension of output boxes tensor:" << num_output_boxes << "." << std::endl; + return false; + } + + // process for maskrcnn + return ProcessMask(tensors[2], results); +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/mask_rcnn.h b/fastdeploy/vision/detection/ppdet/postprocessor.h similarity index 50% rename from fastdeploy/vision/detection/ppdet/mask_rcnn.h rename to fastdeploy/vision/detection/ppdet/postprocessor.h index a24d1f42c59..54be1bfd95f 100644 --- a/fastdeploy/vision/detection/ppdet/mask_rcnn.h +++ b/fastdeploy/vision/detection/ppdet/postprocessor.h @@ -13,26 +13,30 @@ // limitations under the License. #pragma once -#include "fastdeploy/vision/detection/ppdet/rcnn.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" namespace fastdeploy { namespace vision { -namespace detection { -class FASTDEPLOY_DECL MaskRCNN : public FasterRCNN { +namespace detection { +/*! @brief Postprocessor object for PaddleDet serials model. + */ +class FASTDEPLOY_DECL PaddleDetPostprocessor { public: - MaskRCNN(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); - - virtual std::string ModelName() const { return "PaddleDetection/MaskRCNN"; } - - virtual bool Postprocess(std::vector& infer_result, - DetectionResult* result); - - protected: - MaskRCNN() {} + PaddleDetPostprocessor() = default; + /** \brief Process the result of runtime and fill to ClassifyResult structure + * + * \param[in] tensors The inference result from runtime + * \param[in] result The output result of detection + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector* result); + private: + // Process mask tensor for MaskRCNN + bool ProcessMask(const FDTensor& tensor, + std::vector* results); }; } // namespace detection diff --git a/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc b/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc index 01a6a8ce194..252097608e1 100644 --- a/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc +++ b/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc @@ -15,94 +15,94 @@ namespace fastdeploy { void BindPPDet(pybind11::module& m) { - pybind11::class_(m, "PPYOLOE") + pybind11::class_( + m, "PaddleDetPreprocessor") + .def(pybind11::init()) + .def("run", [](vision::detection::PaddleDetPreprocessor& self, std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + if (!self.Run(&images, &outputs)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in PaddleDetPreprocessor.')"); + } + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].StopSharing(); + } + return outputs; + }); + + pybind11::class_( + m, "PaddleDetPostprocessor") + .def(pybind11::init<>()) + .def("run", [](vision::detection::PaddleDetPostprocessor& self, std::vector& inputs) { + std::vector results; + if (!self.Run(inputs, &results)) { + pybind11::eval("raise Exception('Failed to postprocess the runtime result in PaddleDetPostprocessor.')"); + } + return results; + }) + .def("run", [](vision::detection::PaddleDetPostprocessor& self, std::vector& input_array) { + std::vector results; + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + if (!self.Run(inputs, &results)) { + pybind11::eval("raise Exception('Failed to postprocess the runtime result in PaddleDetPostprocessor.')"); + } + return results; + }); + + pybind11::class_(m, "PPDetBase") .def(pybind11::init()) .def("predict", - [](vision::detection::PPYOLOE& self, pybind11::array& data) { + [](vision::detection::PPDetBase& self, pybind11::array& data) { auto mat = PyArrayToCvMat(data); vision::DetectionResult res; self.Predict(&mat, &res); return res; - }); + }) + .def("batch_predict", + [](vision::detection::PPDetBase& self, std::vector& data) { + std::vector images; + for (size_t i = 0; i < data.size(); ++i) { + images.push_back(PyArrayToCvMat(data[i])); + } + std::vector results; + self.BatchPredict(images, &results); + return results; + }) + .def_property_readonly("preprocessor", &vision::detection::PPDetBase::GetPreprocessor) + .def_property_readonly("postprocessor", &vision::detection::PPDetBase::GetPostprocessor); + - pybind11::class_(m, "PPYOLO") + pybind11::class_(m, "PPYOLO") .def(pybind11::init()) - .def("predict", - [](vision::detection::PPYOLO& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); - pybind11::class_(m, "PPYOLOv2") + pybind11::class_(m, "PPYOLOE") .def(pybind11::init()) - .def("predict", - [](vision::detection::PPYOLOv2& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); - pybind11::class_(m, "PicoDet") + pybind11::class_(m, "PicoDet") .def(pybind11::init()) - .def("predict", - [](vision::detection::PicoDet& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); - pybind11::class_( - m, "PaddleYOLOX") + pybind11::class_(m, "PaddleYOLOX") .def(pybind11::init()) - .def("predict", - [](vision::detection::PaddleYOLOX& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); - pybind11::class_(m, - "FasterRCNN") + pybind11::class_(m, "FasterRCNN") .def(pybind11::init()) - .def("predict", - [](vision::detection::FasterRCNN& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); - pybind11::class_(m, "YOLOv3") + pybind11::class_(m, "YOLOv3") .def(pybind11::init()) - .def("predict", - [](vision::detection::YOLOv3& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); - pybind11::class_(m, "MaskRCNN") + pybind11::class_(m, "MaskRCNN") .def(pybind11::init()) - .def("predict", - [](vision::detection::MaskRCNN& self, pybind11::array& data) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res); - return res; - }); + ModelFormat>()); } } // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/ppyolo.cc b/fastdeploy/vision/detection/ppdet/ppyolo.cc deleted file mode 100644 index f0965e5f485..00000000000 --- a/fastdeploy/vision/detection/ppdet/ppyolo.cc +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/ppdet/ppyolo.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -PPYOLO::PPYOLO(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER}; - has_nms_ = true; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - initialized = Initialize(); -} - -bool PPYOLO::Initialize() { - if (!BuildPreprocessPipelineFromConfig()) { - FDERROR << "Failed to build preprocess pipeline from configuration file." - << std::endl; - return false; - } - if (!InitRuntime()) { - FDERROR << "Failed to initialize fastdeploy backend." << std::endl; - return false; - } - return true; -} - -bool PPYOLO::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - for (size_t i = 0; i < processors_.size(); ++i) { - if (!(*(processors_[i].get()))(mat)) { - FDERROR << "Failed to process image data in " << processors_[i]->Name() - << "." << std::endl; - return false; - } - } - - outputs->resize(3); - (*outputs)[0].Allocate({1, 2}, FDDataType::FP32, "im_shape"); - (*outputs)[2].Allocate({1, 2}, FDDataType::FP32, "scale_factor"); - float* ptr0 = static_cast((*outputs)[0].MutableData()); - ptr0[0] = mat->Height(); - ptr0[1] = mat->Width(); - float* ptr2 = static_cast((*outputs)[2].MutableData()); - ptr2[0] = mat->Height() * 1.0 / origin_h; - ptr2[1] = mat->Width() * 1.0 / origin_w; - (*outputs)[1].name = "image"; - mat->ShareWithTensor(&((*outputs)[1])); - // reshape to [1, c, h, w] - (*outputs)[1].shape.insert((*outputs)[1].shape.begin(), 1); - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/ppyolo.h b/fastdeploy/vision/detection/ppdet/ppyolo.h deleted file mode 100644 index 6f288a9dbe2..00000000000 --- a/fastdeploy/vision/detection/ppdet/ppyolo.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -class FASTDEPLOY_DECL PPYOLO : public PPYOLOE { - public: - PPYOLO(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); - - virtual std::string ModelName() const { return "PaddleDetection/PPYOLO"; } - - virtual bool Preprocess(Mat* mat, std::vector* outputs); - virtual bool Initialize(); - - protected: - PPYOLO() {} -}; - -class FASTDEPLOY_DECL PPYOLOv2 : public PPYOLO { - public: - PPYOLOv2(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE) - : PPYOLO(model_file, params_file, config_file, custom_option, - model_format) {} - - virtual std::string ModelName() const { return "PaddleDetection/PPYOLOv2"; } -}; - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/ppyoloe.cc b/fastdeploy/vision/detection/ppdet/ppyoloe.cc deleted file mode 100755 index 1ae6294baf7..00000000000 --- a/fastdeploy/vision/detection/ppdet/ppyoloe.cc +++ /dev/null @@ -1,274 +0,0 @@ -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" -#include "fastdeploy/vision/utils/utils.h" -#include "yaml-cpp/yaml.h" -#ifdef ENABLE_PADDLE_FRONTEND -#include "paddle2onnx/converter.h" -#endif - -namespace fastdeploy { -namespace vision { -namespace detection { - -PPYOLOE::PPYOLOE(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - initialized = Initialize(); -} - -void PPYOLOE::GetNmsInfo() { -#ifdef ENABLE_PADDLE_FRONTEND - if (runtime_option.model_format == ModelFormat::PADDLE) { - std::string contents; - if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { - return; - } - auto reader = paddle2onnx::PaddleReader(contents.c_str(), contents.size()); - if (reader.has_nms) { - has_nms_ = true; - background_label = reader.nms_params.background_label; - keep_top_k = reader.nms_params.keep_top_k; - nms_eta = reader.nms_params.nms_eta; - nms_threshold = reader.nms_params.nms_threshold; - score_threshold = reader.nms_params.score_threshold; - nms_top_k = reader.nms_params.nms_top_k; - normalized = reader.nms_params.normalized; - } - } -#endif -} - -bool PPYOLOE::Initialize() { - if (!BuildPreprocessPipelineFromConfig()) { - FDERROR << "Failed to build preprocess pipeline from configuration file." - << std::endl; - return false; - } - if (!InitRuntime()) { - FDERROR << "Failed to initialize fastdeploy backend." << std::endl; - return false; - } - reused_input_tensors_.resize(2); - - return true; -} - -bool PPYOLOE::BuildPreprocessPipelineFromConfig() { - processors_.clear(); - YAML::Node cfg; - try { - cfg = YAML::LoadFile(config_file_); - } catch (YAML::BadFile& e) { - FDERROR << "Failed to load yaml file " << config_file_ - << ", maybe you should check this file." << std::endl; - return false; - } - - processors_.push_back(std::make_shared()); - - bool has_permute = false; - for (const auto& op : cfg["Preprocess"]) { - std::string op_name = op["type"].as(); - if (op_name == "NormalizeImage") { - auto mean = op["mean"].as>(); - auto std = op["std"].as>(); - bool is_scale = true; - if (op["is_scale"]) { - is_scale = op["is_scale"].as(); - } - std::string norm_type = "mean_std"; - if (op["norm_type"]) { - norm_type = op["norm_type"].as(); - } - if (norm_type != "mean_std") { - std::fill(mean.begin(), mean.end(), 0.0); - std::fill(std.begin(), std.end(), 1.0); - } - processors_.push_back(std::make_shared(mean, std, is_scale)); - } else if (op_name == "Resize") { - bool keep_ratio = op["keep_ratio"].as(); - auto target_size = op["target_size"].as>(); - int interp = op["interp"].as(); - FDASSERT(target_size.size() == 2, - "Require size of target_size be 2, but now it's %lu.", - target_size.size()); - if (!keep_ratio) { - int width = target_size[1]; - int height = target_size[0]; - processors_.push_back( - std::make_shared(width, height, -1.0, -1.0, interp, false)); - } else { - int min_target_size = std::min(target_size[0], target_size[1]); - int max_target_size = std::max(target_size[0], target_size[1]); - std::vector max_size; - if (max_target_size > 0) { - max_size.push_back(max_target_size); - max_size.push_back(max_target_size); - } - processors_.push_back(std::make_shared( - min_target_size, interp, true, max_size)); - } - } else if (op_name == "Permute") { - // Do nothing, do permute as the last operation - has_permute = true; - continue; - // processors_.push_back(std::make_shared()); - } else if (op_name == "Pad") { - auto size = op["size"].as>(); - auto value = op["fill_value"].as>(); - processors_.push_back(std::make_shared("float")); - processors_.push_back( - std::make_shared(size[1], size[0], value)); - } else if (op_name == "PadStride") { - auto stride = op["stride"].as(); - processors_.push_back( - std::make_shared(stride, std::vector(3, 0))); - } else { - FDERROR << "Unexcepted preprocess operator: " << op_name << "." - << std::endl; - return false; - } - } - if (has_permute) { - // permute = cast + HWC2CHW - processors_.push_back(std::make_shared("float")); - processors_.push_back(std::make_shared()); - } else { - processors_.push_back(std::make_shared()); - } - - // Fusion will improve performance - FuseTransforms(&processors_); - return true; -} - -bool PPYOLOE::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - for (size_t i = 0; i < processors_.size(); ++i) { - if (!(*(processors_[i].get()))(mat)) { - FDERROR << "Failed to process image data in " << processors_[i]->Name() - << "." << std::endl; - return false; - } - } - - outputs->resize(2); - (*outputs)[0].name = InputInfoOfRuntime(0).name; - mat->ShareWithTensor(&((*outputs)[0])); - - // reshape to [1, c, h, w] - (*outputs)[0].shape.insert((*outputs)[0].shape.begin(), 1); - - (*outputs)[1].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(1).name); - float* ptr = static_cast((*outputs)[1].MutableData()); - ptr[0] = mat->Height() * 1.0 / origin_h; - ptr[1] = mat->Width() * 1.0 / origin_w; - return true; -} - -bool PPYOLOE::Postprocess(std::vector& infer_result, - DetectionResult* result) { - FDASSERT(infer_result[1].shape[0] == 1, - "Only support batch = 1 in FastDeploy now."); - - has_nms_ = true; - if (!has_nms_) { - int boxes_index = 0; - int scores_index = 1; - if (infer_result[0].shape[1] == infer_result[1].shape[2]) { - boxes_index = 0; - scores_index = 1; - } else if (infer_result[0].shape[2] == infer_result[1].shape[1]) { - boxes_index = 1; - scores_index = 0; - } else { - FDERROR << "The shape of boxes and scores should be [batch, boxes_num, " - "4], [batch, classes_num, boxes_num]" - << std::endl; - return false; - } - - backend::MultiClassNMS nms; - nms.background_label = background_label; - nms.keep_top_k = keep_top_k; - nms.nms_eta = nms_eta; - nms.nms_threshold = nms_threshold; - nms.score_threshold = score_threshold; - nms.nms_top_k = nms_top_k; - nms.normalized = normalized; - nms.Compute(static_cast(infer_result[boxes_index].Data()), - static_cast(infer_result[scores_index].Data()), - infer_result[boxes_index].shape, - infer_result[scores_index].shape); - if (nms.out_num_rois_data[0] > 0) { - result->Reserve(nms.out_num_rois_data[0]); - } - for (size_t i = 0; i < nms.out_num_rois_data[0]; ++i) { - result->label_ids.push_back(nms.out_box_data[i * 6]); - result->scores.push_back(nms.out_box_data[i * 6 + 1]); - result->boxes.emplace_back(std::array{ - nms.out_box_data[i * 6 + 2], nms.out_box_data[i * 6 + 3], - nms.out_box_data[i * 6 + 4], nms.out_box_data[i * 6 + 5]}); - } - } else { - std::vector num_boxes(infer_result[1].shape[0]); - if (infer_result[1].dtype == FDDataType::INT32) { - int32_t* data = static_cast(infer_result[1].Data()); - for (size_t i = 0; i < infer_result[1].shape[0]; ++i) { - num_boxes[i] = static_cast(data[i]); - } - } else if (infer_result[1].dtype == FDDataType::INT64) { - int64_t* data = static_cast(infer_result[1].Data()); - for (size_t i = 0; i < infer_result[1].shape[0]; ++i) { - num_boxes[i] = static_cast(data[i]); - } - } - - // Only support batch = 1 now - result->Reserve(num_boxes[0]); - float* box_data = static_cast(infer_result[0].Data()); - for (size_t i = 0; i < num_boxes[0]; ++i) { - result->label_ids.push_back(box_data[i * 6]); - result->scores.push_back(box_data[i * 6 + 1]); - result->boxes.emplace_back( - std::array{box_data[i * 6 + 2], box_data[i * 6 + 3], - box_data[i * 6 + 4], box_data[i * 6 + 5]}); - } - } - return true; -} - -bool PPYOLOE::Predict(cv::Mat* im, DetectionResult* result) { - Mat mat(*im); - - if (!Preprocess(&mat, &reused_input_tensors_)) { - FDERROR << "Failed to preprocess input data while using model:" - << ModelName() << "." << std::endl; - return false; - } - - if (!Infer()) { - FDERROR << "Failed to inference while using model:" << ModelName() << "." - << std::endl; - return false; - } - - if (!Postprocess(reused_output_tensors_, result)) { - FDERROR << "Failed to postprocess while using model:" << ModelName() << "." - << std::endl; - return false; - } - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/preprocessor.cc b/fastdeploy/vision/detection/ppdet/preprocessor.cc new file mode 100644 index 00000000000..b1179d0362c --- /dev/null +++ b/fastdeploy/vision/detection/ppdet/preprocessor.cc @@ -0,0 +1,201 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/ppdet/preprocessor.h" +#include "fastdeploy/function/concat.h" +#include "fastdeploy/function/pad.h" +#include "yaml-cpp/yaml.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +PaddleDetPreprocessor::PaddleDetPreprocessor(const std::string& config_file) { + FDASSERT(BuildPreprocessPipelineFromConfig(config_file), "Failed to create PaddleDetPreprocessor."); + initialized_ = true; +} + +bool PaddleDetPreprocessor::BuildPreprocessPipelineFromConfig(const std::string& config_file) { + processors_.clear(); + YAML::Node cfg; + try { + cfg = YAML::LoadFile(config_file); + } catch (YAML::BadFile& e) { + FDERROR << "Failed to load yaml file " << config_file + << ", maybe you should check this file." << std::endl; + return false; + } + + processors_.push_back(std::make_shared()); + + bool has_permute = false; + for (const auto& op : cfg["Preprocess"]) { + std::string op_name = op["type"].as(); + if (op_name == "NormalizeImage") { + auto mean = op["mean"].as>(); + auto std = op["std"].as>(); + bool is_scale = true; + if (op["is_scale"]) { + is_scale = op["is_scale"].as(); + } + std::string norm_type = "mean_std"; + if (op["norm_type"]) { + norm_type = op["norm_type"].as(); + } + if (norm_type != "mean_std") { + std::fill(mean.begin(), mean.end(), 0.0); + std::fill(std.begin(), std.end(), 1.0); + } + processors_.push_back(std::make_shared(mean, std, is_scale)); + } else if (op_name == "Resize") { + bool keep_ratio = op["keep_ratio"].as(); + auto target_size = op["target_size"].as>(); + int interp = op["interp"].as(); + FDASSERT(target_size.size() == 2, + "Require size of target_size be 2, but now it's %lu.", + target_size.size()); + if (!keep_ratio) { + int width = target_size[1]; + int height = target_size[0]; + processors_.push_back( + std::make_shared(width, height, -1.0, -1.0, interp, false)); + } else { + int min_target_size = std::min(target_size[0], target_size[1]); + int max_target_size = std::max(target_size[0], target_size[1]); + std::vector max_size; + if (max_target_size > 0) { + max_size.push_back(max_target_size); + max_size.push_back(max_target_size); + } + processors_.push_back(std::make_shared( + min_target_size, interp, true, max_size)); + } + } else if (op_name == "Permute") { + // Do nothing, do permute as the last operation + has_permute = true; + continue; + // processors_.push_back(std::make_shared()); + } else if (op_name == "Pad") { + auto size = op["size"].as>(); + auto value = op["fill_value"].as>(); + processors_.push_back(std::make_shared("float")); + processors_.push_back( + std::make_shared(size[1], size[0], value)); + } else if (op_name == "PadStride") { + auto stride = op["stride"].as(); + processors_.push_back( + std::make_shared(stride, std::vector(3, 0))); + } else { + FDERROR << "Unexcepted preprocess operator: " << op_name << "." + << std::endl; + return false; + } + } + if (has_permute) { + // permute = cast + HWC2CHW + processors_.push_back(std::make_shared("float")); + processors_.push_back(std::make_shared()); + } else { + processors_.push_back(std::make_shared()); + } + + // Fusion will improve performance + FuseTransforms(&processors_); + + return true; +} + +bool PaddleDetPreprocessor::Run(std::vector* images, std::vector* outputs) { + if (!initialized_) { + FDERROR << "The preprocessor is not initialized." << std::endl; + return false; + } + if (images->size() == 0) { + FDERROR << "The size of input images should be greater than 0." << std::endl; + return false; + } + + // There are 3 outputs, image, scale_factor, im_shape + // But im_shape is not used for all the PaddleDetection models + // So preprocessor will output the 3 FDTensors, and how to use `im_shape` + // is decided by the model itself + outputs->resize(3); + int batch = static_cast(images->size()); + // Allocate memory for scale_factor + (*outputs)[1].Resize({batch, 2}, FDDataType::FP32); + // Allocate memory for im_shape + (*outputs)[2].Resize({batch, 2}, FDDataType::FP32); + // Record the max size for a batch of input image + // All the tensor will pad to the max size to compose a batched tensor + std::vector max_hw({-1, -1}); + + float* scale_factor_ptr = reinterpret_cast((*outputs)[1].MutableData()); + float* im_shape_ptr = reinterpret_cast((*outputs)[2].MutableData()); + for (size_t i = 0; i < images->size(); ++i) { + int origin_w = (*images)[i].Width(); + int origin_h = (*images)[i].Height(); + scale_factor_ptr[2 * i] = 1.0; + scale_factor_ptr[2 * i + 1] = 1.0; + for (size_t j = 0; j < processors_.size(); ++j) { + if (!(*(processors_[j].get()))(&((*images)[i]))) { + FDERROR << "Failed to processs image:" << i << " in " << processors_[i]->Name() << "." << std::endl; + return false; + } + if (processors_[j]->Name().find("Resize") != std::string::npos) { + scale_factor_ptr[2 * i] = (*images)[i].Height() * 1.0 / origin_h; + scale_factor_ptr[2 * i + 1] = (*images)[i].Width() * 1.0 / origin_w; + } + } + if ((*images)[i].Height() > max_hw[0]) { + max_hw[0] = (*images)[i].Height(); + } + if ((*images)[i].Width() > max_hw[1]) { + max_hw[1] = (*images)[i].Width(); + } + im_shape_ptr[2 * i] = max_hw[0]; + im_shape_ptr[2 * i + 1] = max_hw[1]; + } + + // Concat all the preprocessed data to a batch tensor + std::vector im_tensors(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + if ((*images)[i].Height() < max_hw[0] || (*images)[i].Width() < max_hw[1]) { + // if the size of image less than max_hw, pad to max_hw + FDTensor tensor; + (*images)[i].ShareWithTensor(&tensor); + function::Pad(tensor, &(im_tensors[i]), {0, 0, max_hw[0] - (*images)[i].Height(), max_hw[1] - (*images)[i].Width()}, 0); + } else { + // No need pad + (*images)[i].ShareWithTensor(&(im_tensors[i])); + } + // Reshape to 1xCxHxW + im_tensors[i].ExpandDim(0); + } + + if (im_tensors.size() == 1) { + // If there's only 1 input, no need to concat + // skip memory copy + (*outputs)[0] = std::move(im_tensors[0]); + } else { + // Else concat the im tensor for each input image + // compose a batched input tensor + function::Concat(im_tensors, &((*outputs)[0]), 0); + } + + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/preprocessor.h b/fastdeploy/vision/detection/ppdet/preprocessor.h new file mode 100644 index 00000000000..2733c450e8a --- /dev/null +++ b/fastdeploy/vision/detection/ppdet/preprocessor.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace detection { +/*! @brief Preprocessor object for PaddleDet serials model. + */ +class FASTDEPLOY_DECL PaddleDetPreprocessor { + public: + PaddleDetPreprocessor() = default; + /** \brief Create a preprocessor instance for PaddleDet serials model + * + * \param[in] config_file Path of configuration file for deployment, e.g ppyoloe/infer_cfg.yml + */ + explicit PaddleDetPreprocessor(const std::string& config_file); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime, include image, scale_factor, im_shape + * \return true if the preprocess successed, otherwise false + */ + bool Run(std::vector* images, std::vector* outputs); + + private: + bool BuildPreprocessPipelineFromConfig(const std::string& config_file); + std::vector> processors_; + bool initialized_ = false; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/rcnn.cc b/fastdeploy/vision/detection/ppdet/rcnn.cc deleted file mode 100644 index 53cbffa5682..00000000000 --- a/fastdeploy/vision/detection/ppdet/rcnn.cc +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/ppdet/rcnn.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -FasterRCNN::FasterRCNN(const std::string& model_file, - const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER}; - has_nms_ = true; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - initialized = Initialize(); -} - -bool FasterRCNN::Initialize() { - if (!BuildPreprocessPipelineFromConfig()) { - FDERROR << "Failed to build preprocess pipeline from configuration file." - << std::endl; - return false; - } - if (!InitRuntime()) { - FDERROR << "Failed to initialize fastdeploy backend." << std::endl; - return false; - } - return true; -} - -bool FasterRCNN::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - float scale[2] = {1.0, 1.0}; - for (size_t i = 0; i < processors_.size(); ++i) { - if (!(*(processors_[i].get()))(mat)) { - FDERROR << "Failed to process image data in " << processors_[i]->Name() - << "." << std::endl; - return false; - } - if (processors_[i]->Name().find("Resize") != std::string::npos) { - scale[0] = mat->Height() * 1.0 / origin_h; - scale[1] = mat->Width() * 1.0 / origin_w; - } - } - - outputs->resize(3); - (*outputs)[0].Allocate({1, 2}, FDDataType::FP32, "im_shape"); - (*outputs)[2].Allocate({1, 2}, FDDataType::FP32, "scale_factor"); - float* ptr0 = static_cast((*outputs)[0].MutableData()); - ptr0[0] = mat->Height(); - ptr0[1] = mat->Width(); - float* ptr2 = static_cast((*outputs)[2].MutableData()); - ptr2[0] = scale[0]; - ptr2[1] = scale[1]; - (*outputs)[1].name = "image"; - mat->ShareWithTensor(&((*outputs)[1])); - // reshape to [1, c, h, w] - (*outputs)[1].shape.insert((*outputs)[1].shape.begin(), 1); - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/rcnn.h b/fastdeploy/vision/detection/ppdet/rcnn.h deleted file mode 100644 index df42b8efc4e..00000000000 --- a/fastdeploy/vision/detection/ppdet/rcnn.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -class FASTDEPLOY_DECL FasterRCNN : public PPYOLOE { - public: - FasterRCNN(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); - - virtual std::string ModelName() const { return "PaddleDetection/FasterRCNN"; } - - virtual bool Preprocess(Mat* mat, std::vector* outputs); - virtual bool Initialize(); - - protected: - FasterRCNN() {} -}; -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/yolov3.cc b/fastdeploy/vision/detection/ppdet/yolov3.cc deleted file mode 100644 index bcfb3aef9cb..00000000000 --- a/fastdeploy/vision/detection/ppdet/yolov3.cc +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/ppdet/yolov3.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -YOLOv3::YOLOv3(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT, Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - initialized = Initialize(); -} - -bool YOLOv3::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - for (size_t i = 0; i < processors_.size(); ++i) { - if (!(*(processors_[i].get()))(mat)) { - FDERROR << "Failed to process image data in " << processors_[i]->Name() - << "." << std::endl; - return false; - } - } - - outputs->resize(3); - (*outputs)[0].Allocate({1, 2}, FDDataType::FP32, "im_shape"); - (*outputs)[2].Allocate({1, 2}, FDDataType::FP32, "scale_factor"); - float* ptr0 = static_cast((*outputs)[0].MutableData()); - ptr0[0] = mat->Height(); - ptr0[1] = mat->Width(); - float* ptr2 = static_cast((*outputs)[2].MutableData()); - ptr2[0] = mat->Height() * 1.0 / origin_h; - ptr2[1] = mat->Width() * 1.0 / origin_w; - (*outputs)[1].name = "image"; - mat->ShareWithTensor(&((*outputs)[1])); - // reshape to [1, c, h, w] - (*outputs)[1].shape.insert((*outputs)[1].shape.begin(), 1); - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/yolov3.h b/fastdeploy/vision/detection/ppdet/yolov3.h deleted file mode 100644 index ebafa6eddb9..00000000000 --- a/fastdeploy/vision/detection/ppdet/yolov3.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -class FASTDEPLOY_DECL YOLOv3 : public PPYOLOE { - public: - YOLOv3(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); - - virtual std::string ModelName() const { return "PaddleDetection/YOLOv3"; } - - virtual bool Preprocess(Mat* mat, std::vector* outputs); -}; -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/yolox.cc b/fastdeploy/vision/detection/ppdet/yolox.cc deleted file mode 100644 index f7405d4de38..00000000000 --- a/fastdeploy/vision/detection/ppdet/yolox.cc +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/ppdet/yolox.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -PaddleYOLOX::PaddleYOLOX(const std::string& model_file, - const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - config_file_ = config_file; - valid_cpu_backends = {Backend::ORT, Backend::PDINFER, Backend::LITE}; - valid_gpu_backends = {Backend::ORT, Backend::PDINFER, Backend::TRT}; - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; - background_label = -1; - keep_top_k = 1000; - nms_eta = 1; - nms_threshold = 0.65; - nms_top_k = 10000; - normalized = true; - score_threshold = 0.001; - initialized = Initialize(); -} - -bool PaddleYOLOX::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - float scale[2] = {1.0, 1.0}; - for (size_t i = 0; i < processors_.size(); ++i) { - if (!(*(processors_[i].get()))(mat)) { - FDERROR << "Failed to process image data in " << processors_[i]->Name() - << "." << std::endl; - return false; - } - if (processors_[i]->Name().find("Resize") != std::string::npos) { - scale[0] = mat->Height() * 1.0 / origin_h; - scale[1] = mat->Width() * 1.0 / origin_w; - } - } - - outputs->resize(2); - (*outputs)[0].name = InputInfoOfRuntime(0).name; - mat->ShareWithTensor(&((*outputs)[0])); - - // reshape to [1, c, h, w] - (*outputs)[0].shape.insert((*outputs)[0].shape.begin(), 1); - - (*outputs)[1].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(1).name); - float* ptr = static_cast((*outputs)[1].MutableData()); - ptr[0] = scale[0]; - ptr[1] = scale[1]; - return true; -} -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/yolox.h b/fastdeploy/vision/detection/ppdet/yolox.h deleted file mode 100644 index dd0a11b57e6..00000000000 --- a/fastdeploy/vision/detection/ppdet/yolox.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/vision/detection/ppdet/ppyoloe.h" - -namespace fastdeploy { -namespace vision { -namespace detection { - -class FASTDEPLOY_DECL PaddleYOLOX : public PPYOLOE { - public: - PaddleYOLOX(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); - - virtual bool Preprocess(Mat* mat, std::vector* outputs); - - virtual std::string ModelName() const { return "PaddleDetection/YOLOX"; } -}; -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/python/fastdeploy/vision/detection/__init__.py b/python/fastdeploy/vision/detection/__init__.py index 47d175af755..6de4a3fa634 100755 --- a/python/fastdeploy/vision/detection/__init__.py +++ b/python/fastdeploy/vision/detection/__init__.py @@ -23,5 +23,4 @@ from .contrib.yolov6 import YOLOv6 from .contrib.yolov7end2end_trt import YOLOv7End2EndTRT from .contrib.yolov7end2end_ort import YOLOv7End2EndORT -from .ppdet import PPYOLOE, PPYOLO, PPYOLOv2, PaddleYOLOX, PicoDet, FasterRCNN, YOLOv3, MaskRCNN -from .rknpu2 import RKPicoDet +from .ppdet import * diff --git a/python/fastdeploy/vision/detection/ppdet/__init__.py b/python/fastdeploy/vision/detection/ppdet/__init__.py index 4497c75ee4b..4341a9a1ae5 100644 --- a/python/fastdeploy/vision/detection/ppdet/__init__.py +++ b/python/fastdeploy/vision/detection/ppdet/__init__.py @@ -19,6 +19,40 @@ from .... import c_lib_wrap as C +class PaddleDetPreprocessor: + def __init__(self, config_file): + """Create a preprocessor for PaddleDetection Model from configuration file + + :param config_file: (str)Path of configuration file, e.g ppyoloe/infer_cfg.yml + """ + self._preprocessor = C.vision.detection.PaddleDetPreprocessor( + config_file) + + def run(self, input_ims): + """Preprocess input images for PaddleDetection Model + + :param: input_ims: (list of numpy.ndarray)The input image + :return: list of FDTensor, include image, scale_factor, im_shape + """ + return self._preprocessor.run(input_ims) + + +class PaddleDetPostprocessor: + def __init__(self): + """Create a postprocessor for PaddleDetection Model + + """ + self._postprocessor = C.vision.detection.PaddleDetPostprocessor() + + def run(self, runtime_results): + """Postprocess the runtime results for PaddleDetection Model + + :param: runtime_results: (list of FDTensor)The output FDTensor results from runtime + :return: list of ClassifyResult(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results) + + class PPYOLOE(FastDeployModel): def __init__(self, model_file, @@ -52,6 +86,31 @@ def predict(self, im): assert im is not None, "The input image data is None." return self._model.predict(im) + def batch_predict(self, images): + """Detect a batch of input image list + + :param im: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return list of DetectionResult + """ + + return self._model.batch_predict(images) + + @property + def preprocessor(self): + """Get PaddleDetPreprocessor object of the loaded model + + :return PaddleDetPreprocessor + """ + return self._model.preprocessor + + @property + def postprocessor(self): + """Get PaddleDetPostprocessor object of the loaded model + + :return PaddleDetPostprocessor + """ + return self._model.postprocessor + class PPYOLO(PPYOLOE): def __init__(self, @@ -77,31 +136,6 @@ def __init__(self, assert self.initialized, "PPYOLO model initialize failed." -class PPYOLOv2(PPYOLOE): - def __init__(self, - model_file, - params_file, - config_file, - runtime_option=None, - model_format=ModelFormat.PADDLE): - """Load a PPYOLOv2 model exported by PaddleDetection. - - :param model_file: (str)Path of model file, e.g ppyolov2/model.pdmodel - :param params_file: (str)Path of parameters file, e.g ppyolov2/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string - :param config_file: (str)Path of configuration file for deployment, e.g ppyoloe/infer_cfg.yml - :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU - :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model - """ - - super(PPYOLOE, self).__init__(runtime_option) - - assert model_format == ModelFormat.PADDLE, "PPYOLOv2 model only support model format of ModelFormat.Paddle now." - self._model = C.vision.detection.PPYOLOv2( - model_file, params_file, config_file, self._runtime_option, - model_format) - assert self.initialized, "PPYOLOv2 model initialize failed." - - class PaddleYOLOX(PPYOLOE): def __init__(self, model_file, @@ -202,7 +236,7 @@ def __init__(self, assert self.initialized, "YOLOv3 model initialize failed." -class MaskRCNN(FastDeployModel): +class MaskRCNN(PPYOLOE): def __init__(self, model_file, params_file, @@ -211,14 +245,14 @@ def __init__(self, model_format=ModelFormat.PADDLE): """Load a MaskRCNN model exported by PaddleDetection. - :param model_file: (str)Path of model file, e.g maskrcnn/model.pdmodel - :param params_file: (str)Path of parameters file, e.g maskrcnn/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string + :param model_file: (str)Path of model file, e.g fasterrcnn/model.pdmodel + :param params_file: (str)Path of parameters file, e.g fasterrcnn/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string :param config_file: (str)Path of configuration file for deployment, e.g ppyoloe/infer_cfg.yml :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model """ - super(MaskRCNN, self).__init__(runtime_option) + super(PPYOLOE, self).__init__(runtime_option) assert model_format == ModelFormat.PADDLE, "MaskRCNN model only support model format of ModelFormat.Paddle now." self._model = C.vision.detection.MaskRCNN( @@ -226,6 +260,12 @@ def __init__(self, model_format) assert self.initialized, "MaskRCNN model initialize failed." - def predict(self, input_image): - assert input_image is not None, "The input image data is None." - return self._model.predict(input_image) + def batch_predict(self, images): + """Detect a batch of input image list, batch_predict is not supported for maskrcnn now. + + :param im: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return list of DetectionResult + """ + + raise Exception( + "batch_predict is not supported for MaskRCNN model now.") diff --git a/tests/models/test_faster_rcnn.py b/tests/models/test_faster_rcnn.py new file mode 100755 index 00000000000..b7ab217a2ae --- /dev/null +++ b/tests/models/test_faster_rcnn.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +print(fd.__path__) +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_faster_rcnn(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/faster_rcnn_r50_vd_fpn_2x_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/faster_rcnn_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/faster_rcnn_r50_vd_fpn_2x_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + rc.test_option.use_paddle_backend() + model = fd.vision.detection.FasterRCNN( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/faster_rcnn_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# result = model.predict(im1) +# with open("faster_rcnn_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_faster_rcnn() diff --git a/tests/models/test_mask_rcnn.py b/tests/models/test_mask_rcnn.py new file mode 100755 index 00000000000..df8641af1ac --- /dev/null +++ b/tests/models/test_mask_rcnn.py @@ -0,0 +1,68 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_mask_rcnn(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/mask_rcnn_r50_1x_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/mask_rcnn_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/mask_rcnn_r50_1x_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + model = fd.vision.detection.MaskRCNN( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/mask_rcnn_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# result = model.predict(im1) +# with open("mask_rcnn_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_mask_rcnn() diff --git a/tests/models/test_picodet.py b/tests/models/test_picodet.py new file mode 100755 index 00000000000..59e4bc784c8 --- /dev/null +++ b/tests/models/test_picodet.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_picodet(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/picodet_l_320_coco_lcnet.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/picodet_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/picodet_l_320_coco_lcnet" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + rc.test_option.use_paddle_backend() + model = fd.vision.detection.PicoDet( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/picodet_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-02, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# result = model.predict(im1) +# with open("picodet_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_picodet() diff --git a/tests/models/test_pp_yolox.py b/tests/models/test_pp_yolox.py new file mode 100755 index 00000000000..57aee0b991a --- /dev/null +++ b/tests/models/test_pp_yolox.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +print(fd.__path__) +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_yolox(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolox_s_300e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/ppyolox_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/yolox_s_300e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + rc.test_option.use_ort_backend() + model = fd.vision.detection.PaddleYOLOX( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/ppyolox_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.1 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# result = model.predict(im1) +# with open("ppyolox_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_yolox() diff --git a/tests/models/test_ppyolo.py b/tests/models/test_ppyolo.py new file mode 100755 index 00000000000..ede1c155095 --- /dev/null +++ b/tests/models/test_ppyolo.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_ppyolo(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppyolov2_r101vd_dcn_365e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/ppyolo_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/ppyolov2_r101vd_dcn_365e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + rc.test_option.use_paddle_backend() + model = fd.vision.detection.PPYOLO( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/ppyolo_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# result = model.predict(im1) +# with open("ppyolo_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_ppyolo() diff --git a/tests/models/test_ppyoloe.py b/tests/models/test_ppyoloe.py new file mode 100755 index 00000000000..b75f3467031 --- /dev/null +++ b/tests/models/test_ppyoloe.py @@ -0,0 +1,68 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_ppyoloe(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/ppyoloe_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/ppyoloe_crn_l_300e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + rc.test_option.use_ort_backend() + model = fd.vision.detection.PPYOLOE( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/ppyoloe_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# with open("ppyoloe_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_ppyoloe() diff --git a/tests/models/test_yolov3.py b/tests/models/test_yolov3.py new file mode 100755 index 00000000000..e9c2faa3efc --- /dev/null +++ b/tests/models/test_yolov3.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_yolov3(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov3_darknet53_270e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/yolov3_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/yolov3_darknet53_270e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + rc.test_option.use_ort_backend() + model = fd.vision.detection.YOLOv3( + model_file, params_file, config_file, runtime_option=rc.test_option) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + result = model.predict(im1) + with open("resources/yolov3_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.1 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + + +# result = model.predict(im1) +# with open("yolov3_baseline.pkl", "wb") as f: +# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) + +if __name__ == "__main__": + test_detection_yolov3() From bea3831373568dbc6a6032ac23a3057e26f76130 Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Tue, 15 Nov 2022 02:51:09 +0000 Subject: [PATCH 029/136] add more tests --- tests/models/test_faster_rcnn.py | 50 ++++++++++++++++++++++++++++ tests/models/test_mask_rcnn.py | 57 ++++++++++++++++++++++++++++++-- tests/models/test_picodet.py | 55 ++++++++++++++++++++++++++++++ tests/models/test_pp_yolox.py | 47 ++++++++++++++++++++++++++ tests/models/test_ppyolo.py | 51 ++++++++++++++++++++++++++++ tests/models/test_ppyoloe.py | 48 +++++++++++++++++++++++++-- 6 files changed, 303 insertions(+), 5 deletions(-) diff --git a/tests/models/test_faster_rcnn.py b/tests/models/test_faster_rcnn.py index b7ab217a2ae..de5b908c042 100755 --- a/tests/models/test_faster_rcnn.py +++ b/tests/models/test_faster_rcnn.py @@ -66,5 +66,55 @@ def test_detection_faster_rcnn(): # with open("faster_rcnn_baseline.pkl", "wb") as f: # pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) +def test_detection_faster_rcnn1(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/faster_rcnn_r50_vd_fpn_2x_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/faster_rcnn_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/faster_rcnn_r50_vd_fpn_2x_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + option = rc.test_option + option.set_model_path(model_file, params_file) + option.use_paddle_infer_backend() + runtime = fd.Runtime(option); + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + im1 = cv2.imread("./resources/000000014439.jpg") + input_tensors = preprocessor.run([im1]) + output_tensors = runtime.infer({"image": input_tensors[0], "scale_factor": input_tensors[1], "im_shape": input_tensors[2]}) + results = postprocessor.run(output_tensors) + result = results[0] + + with open("resources/faster_rcnn_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-04, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-04, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + if __name__ == "__main__": test_detection_faster_rcnn() + test_detection_faster_rcnn1() diff --git a/tests/models/test_mask_rcnn.py b/tests/models/test_mask_rcnn.py index df8641af1ac..8cd0a614eac 100755 --- a/tests/models/test_mask_rcnn.py +++ b/tests/models/test_mask_rcnn.py @@ -13,6 +13,7 @@ # limitations under the License. import fastdeploy as fd +import copy import cv2 import os import pickle @@ -38,9 +39,9 @@ def test_detection_mask_rcnn(): # compare diff im1 = cv2.imread("./resources/000000014439.jpg") for i in range(2): - result = model.predict(im1) with open("resources/mask_rcnn_baseline.pkl", "rb") as f: boxes, scores, label_ids = pickle.load(f) + result = model.predict(im1) pred_boxes = np.array(result.boxes) pred_scores = np.array(result.scores) pred_label_ids = np.array(result.label_ids) @@ -53,9 +54,9 @@ def test_detection_mask_rcnn(): score_threshold = 0.0 assert diff_boxes[scores > score_threshold].max( - ) < 1e-04, "There's diff in boxes." + ) < 1e-01, "There's diff in boxes." assert diff_scores[scores > score_threshold].max( - ) < 1e-04, "There's diff in scores." + ) < 1e-02, "There's diff in scores." assert diff_label_ids[scores > score_threshold].max( ) < 1e-04, "There's diff in label_ids." @@ -64,5 +65,55 @@ def test_detection_mask_rcnn(): # with open("mask_rcnn_baseline.pkl", "wb") as f: # pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) +def test_detection_mask_rcnn1(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/mask_rcnn_r50_1x_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/mask_rcnn_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/mask_rcnn_r50_1x_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + option = rc.test_option + option.set_model_path(model_file, params_file) + option.use_paddle_infer_backend() + runtime = fd.Runtime(option); + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + im1 = cv2.imread("./resources/000000014439.jpg") + input_tensors = preprocessor.run([im1]) + output_tensors = runtime.infer({"image": input_tensors[0], "scale_factor": input_tensors[1], "im_shape": input_tensors[2]}) + results = postprocessor.run(output_tensors) + result = results[0] + + with open("resources/mask_rcnn_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-01, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-02, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + if __name__ == "__main__": test_detection_mask_rcnn() + test_detection_mask_rcnn1() diff --git a/tests/models/test_picodet.py b/tests/models/test_picodet.py index 59e4bc784c8..eec76fd5002 100755 --- a/tests/models/test_picodet.py +++ b/tests/models/test_picodet.py @@ -36,6 +36,12 @@ def test_detection_picodet(): model = fd.vision.detection.PicoDet( model_file, params_file, config_file, runtime_option=rc.test_option) + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + rc.test_option.set_model_path(model_file, params_file) + runtime = fd.Runtime(rc.test_option); + # compare diff im1 = cv2.imread("./resources/000000014439.jpg") for i in range(2): @@ -65,5 +71,54 @@ def test_detection_picodet(): # with open("picodet_baseline.pkl", "wb") as f: # pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) +def test_detection_picodet1(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/picodet_l_320_coco_lcnet.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/picodet_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/picodet_l_320_coco_lcnet" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + rc.test_option.set_model_path(model_file, params_file) + runtime = fd.Runtime(rc.test_option); + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + input_tensors = preprocessor.run([im1]) + output_tensors = runtime.infer({"image": input_tensors[0], "scale_factor": input_tensors[1]}) + results = postprocessor.run(output_tensors) + result = results[0] + with open("resources/picodet_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + with open("resources/dump_result.pkl", "wb") as f: + pickle.dump([pred_boxes, pred_scores, pred_label_ids], f) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-01, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-03, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + if __name__ == "__main__": test_detection_picodet() + test_detection_picodet1() diff --git a/tests/models/test_pp_yolox.py b/tests/models/test_pp_yolox.py index 57aee0b991a..f9739993e32 100755 --- a/tests/models/test_pp_yolox.py +++ b/tests/models/test_pp_yolox.py @@ -66,5 +66,52 @@ def test_detection_yolox(): # with open("ppyolox_baseline.pkl", "wb") as f: # pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) +def test_detection_yolox_1(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolox_s_300e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/ppyolox_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/yolox_s_300e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + rc.test_option.set_model_path(model_file, params_file) + runtime = fd.Runtime(rc.test_option); + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(3): + input_tensors = preprocessor.run([im1]) + output_tensors = runtime.infer({"image": input_tensors[0], "scale_factor": input_tensors[1]}) + results = postprocessor.run(output_tensors) + result = results[0] + with open("resources/ppyolox_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-01, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-02, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + if __name__ == "__main__": test_detection_yolox() + test_detection_yolox_1() diff --git a/tests/models/test_ppyolo.py b/tests/models/test_ppyolo.py index ede1c155095..73a4c703762 100755 --- a/tests/models/test_ppyolo.py +++ b/tests/models/test_ppyolo.py @@ -65,5 +65,56 @@ def test_detection_ppyolo(): # with open("ppyolo_baseline.pkl", "wb") as f: # pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) +def test_detection_ppyolo1(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppyolov2_r101vd_dcn_365e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/ppyolo_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/ppyolov2_r101vd_dcn_365e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + option = rc.test_option + option.use_paddle_backend() + option.set_model_path(model_file, params_file) + runtime = fd.Runtime(option); + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + input_tensors = preprocessor.run([im1]) + output_tensors = runtime.infer({"image": input_tensors[0], "scale_factor": input_tensors[1], "im_shape": input_tensors[2]}) + results = postprocessor.run(output_tensors) + result = results[0] + with open("resources/ppyolo_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + with open("resources/dump_result.pkl", "wb") as f: + pickle.dump([pred_boxes, pred_scores, pred_label_ids], f) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-01, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-03, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." + if __name__ == "__main__": test_detection_ppyolo() + test_detection_ppyolo1() diff --git a/tests/models/test_ppyoloe.py b/tests/models/test_ppyoloe.py index b75f3467031..08b19bf91f2 100755 --- a/tests/models/test_ppyoloe.py +++ b/tests/models/test_ppyoloe.py @@ -60,9 +60,53 @@ def test_detection_ppyoloe(): assert diff_label_ids[scores > score_threshold].max( ) < 1e-04, "There's diff in label_ids." +def test_detection_ppyoloe1(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url = "https://bj.bcebos.com/fastdeploy/tests/data/ppyoloe_baseline.pkl" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url, "resources") + model_path = "resources/ppyoloe_crn_l_300e_coco" + + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + + preprocessor = fd.vision.detection.PaddleDetPreprocessor(config_file) + postprocessor = fd.vision.detection.PaddleDetPostprocessor() + + rc.test_option.set_model_path(model_file, params_file) + runtime = fd.Runtime(rc.test_option); + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + for i in range(2): + input_tensors = preprocessor.run([im1]) + output_tensors = runtime.infer({"image": input_tensors[0], "scale_factor": input_tensors[1]}) + results = postprocessor.run(output_tensors) + result = results[0] + with open("resources/ppyoloe_baseline.pkl", "rb") as f: + boxes, scores, label_ids = pickle.load(f) + pred_boxes = np.array(result.boxes) + pred_scores = np.array(result.scores) + pred_label_ids = np.array(result.label_ids) + + diff_boxes = np.fabs(boxes - pred_boxes) + diff_scores = np.fabs(scores - pred_scores) + diff_label_ids = np.fabs(label_ids - pred_label_ids) + + print(diff_boxes.max(), diff_scores.max(), diff_label_ids.max()) + + score_threshold = 0.0 + assert diff_boxes[scores > score_threshold].max( + ) < 1e-01, "There's diff in boxes." + assert diff_scores[scores > score_threshold].max( + ) < 1e-02, "There's diff in scores." + assert diff_label_ids[scores > score_threshold].max( + ) < 1e-04, "There's diff in label_ids." -# with open("ppyoloe_baseline.pkl", "wb") as f: -# pickle.dump([np.array(result.boxes), np.array(result.scores), np.array(result.label_ids)], f) if __name__ == "__main__": test_detection_ppyoloe() + test_detection_ppyoloe1() From ecddd7ef9746f7a86216a98a7b80292f472905f1 Mon Sep 17 00:00:00 2001 From: Jack Zhou Date: Tue, 15 Nov 2022 11:25:32 +0800 Subject: [PATCH 030/136] [Benchmark] Add benchmark for ernie sequence classification (#444) * Add benchmark for ernie sequence classification * Add pretty print * Update benchmark of ernie * get_table -> get_statistics_table * add comments * Update the output * Add cpu gpu memory statitis * Add gpu utilization sampling Co-authored-by: Jason --- benchmark/benchmark_ernie_seq_cls.py | 282 +++++++++++++++++++++++ benchmark/run_benchmark_ernie_seq_cls.sh | 25 ++ 2 files changed, 307 insertions(+) create mode 100644 benchmark/benchmark_ernie_seq_cls.py create mode 100644 benchmark/run_benchmark_ernie_seq_cls.sh diff --git a/benchmark/benchmark_ernie_seq_cls.py b/benchmark/benchmark_ernie_seq_cls.py new file mode 100644 index 00000000000..3d6466202c3 --- /dev/null +++ b/benchmark/benchmark_ernie_seq_cls.py @@ -0,0 +1,282 @@ +import paddlenlp +import numpy as np +from paddlenlp.transformers import AutoTokenizer +from paddlenlp.datasets import load_dataset +import fastdeploy as fd +import os +import time +import distutils.util +import sys +import pynvml +import psutil +import GPUtil +from prettytable import PrettyTable +import multiprocessing + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_dir", + required=True, + help="The directory of model and tokenizer.") + parser.add_argument( + "--device", + type=str, + default='gpu', + choices=['gpu', 'cpu'], + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--backend", + type=str, + default='pp', + choices=['ort', 'pp', 'trt', 'pp-trt'], + help="The inference runtime backend.") + parser.add_argument( + "--device_id", type=int, default=0, help="device(gpu) id") + parser.add_argument( + "--batch_size", type=int, default=32, help="The batch size of data.") + parser.add_argument( + "--max_length", + type=int, + default=128, + help="The max length of sequence.") + parser.add_argument( + "--log_interval", + type=int, + default=10, + help="The interval of logging.") + parser.add_argument( + "--cpu_num_threads", + type=int, + default=1, + help="The number of threads when inferring on cpu.") + parser.add_argument( + "--use_fp16", + type=distutils.util.strtobool, + default=False, + help="Use FP16 mode") + parser.add_argument( + "--use_fast", + type=distutils.util.strtobool, + default=True, + help="Whether to use fast_tokenizer to accelarate the tokenization.") + return parser.parse_args() + + +def create_fd_runtime(args): + option = fd.RuntimeOption() + model_path = os.path.join(args.model_dir, "infer.pdmodel") + params_path = os.path.join(args.model_dir, "infer.pdiparams") + option.set_model_path(model_path, params_path) + if args.device == 'cpu': + option.use_cpu() + option.set_cpu_thread_num(args.cpu_num_threads) + else: + option.use_gpu(args.device_id) + if args.backend == 'pp': + option.use_paddle_backend() + elif args.backend == 'ort': + option.use_ort_backend() + else: + option.use_trt_backend() + if args.backend == 'pp-trt': + option.enable_paddle_to_trt() + option.enable_paddle_trt_collect_shape() + trt_file = os.path.join(args.model_dir, "infer.trt") + option.set_trt_input_shape( + 'input_ids', + min_shape=[1, args.max_length], + opt_shape=[args.batch_size, args.max_length], + max_shape=[args.batch_size, args.max_length]) + option.set_trt_input_shape( + 'token_type_ids', + min_shape=[1, args.max_length], + opt_shape=[args.batch_size, args.max_length], + max_shape=[args.batch_size, args.max_length]) + if args.use_fp16: + option.enable_trt_fp16() + trt_file = trt_file + ".fp16" + option.set_trt_cache_file(trt_file) + return fd.Runtime(option) + + +def convert_examples_to_data(dataset, batch_size): + texts, text_pairs, labels = [], [], [] + batch_text, batch_text_pair, batch_label = [], [], [] + + for i, item in enumerate(dataset): + batch_text.append(item['sentence1']) + batch_text_pair.append(item['sentence2']) + batch_label.append(item['label']) + if (i + 1) % batch_size == 0: + texts.append(batch_text) + text_pairs.append(batch_text_pair) + labels.append(batch_label) + batch_text, batch_text_pair, batch_label = [], [], [] + return texts, text_pairs, labels + + +def postprocess(logits): + max_value = np.max(logits, axis=1, keepdims=True) + exp_data = np.exp(logits - max_value) + probs = exp_data / np.sum(exp_data, axis=1, keepdims=True) + out_dict = { + "label": probs.argmax(axis=-1), + "confidence": probs.max(axis=-1) + } + return out_dict + + +def get_statistics_table(tokenizer_time_costs, runtime_time_costs, + postprocess_time_costs): + x = PrettyTable() + x.field_names = [ + "Stage", "Mean latency", "P50 latency", "P90 latency", "P95 latency" + ] + x.add_row([ + "Tokenization", f"{np.mean(tokenizer_time_costs):.4f}", + f"{np.percentile(tokenizer_time_costs, 50):.4f}", + f"{np.percentile(tokenizer_time_costs, 90):.4f}", + f"{np.percentile(tokenizer_time_costs, 95):.4f}" + ]) + x.add_row([ + "Runtime", f"{np.mean(runtime_time_costs):.4f}", + f"{np.percentile(runtime_time_costs, 50):.4f}", + f"{np.percentile(runtime_time_costs, 90):.4f}", + f"{np.percentile(runtime_time_costs, 95):.4f}" + ]) + x.add_row([ + "Postprocessing", f"{np.mean(postprocess_time_costs):.4f}", + f"{np.percentile(postprocess_time_costs, 50):.4f}", + f"{np.percentile(postprocess_time_costs, 90):.4f}", + f"{np.percentile(postprocess_time_costs, 95):.4f}" + ]) + return x + + +def get_current_memory_mb(gpu_id=None): + pid = os.getpid() + p = psutil.Process(pid) + info = p.memory_full_info() + cpu_mem = info.uss / 1024. / 1024. + gpu_mem = 0 + if gpu_id is not None: + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_id) + meminfo = pynvml.nvmlDeviceGetMemoryInfo(handle) + gpu_mem = meminfo.used / 1024. / 1024. + return cpu_mem, gpu_mem + + +def get_current_gputil(gpu_id): + GPUs = GPUtil.getGPUs() + gpu_load = GPUs[gpu_id].load + return gpu_load + + +def sample_gpuutil(gpu_id, gpu_utilization=[]): + while True: + gpu_utilization.append(get_current_gputil(gpu_id)) + time.sleep(0.01) + + +def show_statistics(tokenizer_time_costs, + runtime_time_costs, + postprocess_time_costs, + correct_num, + total_num, + cpu_mem, + gpu_mem, + gpu_util, + prefix=""): + print( + f"{prefix}Acc = {correct_num/total_num*100:.2f} ({correct_num} /{total_num})." + f" CPU memory: {np.mean(cpu_mem):.2f} MB, GPU memory: {np.mean(gpu_mem):.2f} MB," + f" GPU utilization {np.max(gpu_util) * 100:.2f}%.") + print( + get_statistics_table(tokenizer_time_costs, runtime_time_costs, + postprocess_time_costs)) + + +if __name__ == "__main__": + args = parse_arguments() + + tokenizer = AutoTokenizer.from_pretrained( + "ernie-3.0-medium-zh", use_faster=args.use_fast) + runtime = create_fd_runtime(args) + input_ids_name = runtime.get_input_info(0).name + token_type_ids_name = runtime.get_input_info(1).name + + test_ds = load_dataset("clue", "afqmc", splits=['dev']) + texts, text_pairs, labels = convert_examples_to_data(test_ds, + args.batch_size) + gpu_id = args.device_id + + def run_inference(warmup_steps=None): + tokenizer_time_costs = [] + runtime_time_costs = [] + postprocess_time_costs = [] + cpu_mem = [] + gpu_mem = [] + + total_num = 0 + correct_num = 0 + + manager = multiprocessing.Manager() + gpu_util = manager.list() + p = multiprocessing.Process( + target=sample_gpuutil, args=(gpu_id, gpu_util)) + p.start() + for i, (text, text_pair, + label) in enumerate(zip(texts, text_pairs, labels)): + # Start the process to sample gpu utilization + start = time.time() + encoded_inputs = tokenizer( + text=text, + text_pair=text_pair, + max_length=args.max_length, + padding='max_length', + truncation=True, + return_tensors='np') + tokenizer_time_costs += [(time.time() - start) * 1000] + + start = time.time() + input_map = { + input_ids_name: encoded_inputs["input_ids"].astype('int64'), + token_type_ids_name: + encoded_inputs["token_type_ids"].astype('int64'), + } + results = runtime.infer(input_map) + runtime_time_costs += [(time.time() - start) * 1000] + + start = time.time() + output = postprocess(results[0]) + postprocess_time_costs += [(time.time() - start) * 1000] + + cm, gm = get_current_memory_mb(gpu_id) + cpu_mem.append(cm) + gpu_mem.append(gm) + + total_num += len(label) + correct_num += (label == output["label"]).sum() + if warmup_steps is not None and i >= warmup_steps: + break + if (i + 1) % args.log_interval == 0: + show_statistics(tokenizer_time_costs, runtime_time_costs, + postprocess_time_costs, correct_num, total_num, + cpu_mem, gpu_mem, gpu_util, + f"Step {i + 1: 6d}: ") + show_statistics(tokenizer_time_costs, runtime_time_costs, + postprocess_time_costs, correct_num, total_num, + cpu_mem, gpu_mem, gpu_util, f"Final statistics: ") + p.terminate() + + # Warm up + print("Warm up") + run_inference(10) + print("Start to test the benchmark") + run_inference() + print("Finish") diff --git a/benchmark/run_benchmark_ernie_seq_cls.sh b/benchmark/run_benchmark_ernie_seq_cls.sh new file mode 100644 index 00000000000..658cef1feb9 --- /dev/null +++ b/benchmark/run_benchmark_ernie_seq_cls.sh @@ -0,0 +1,25 @@ +# Download and decompress the ERNIE 3.0 Medium model finetuned on AFQMC +# wget https://bj.bcebos.com/fastdeploy/models/ernie-3.0/ernie-3.0-medium-zh-afqmc.tgz +# tar xvfz ernie-3.0-medium-zh-afqmc.tgz + +# Download and decompress the quantization model of ERNIE 3.0 Medium model +# wget https://bj.bcebos.com/fastdeploy/models/ernie-3.0/ernie-3.0-medium-zh-afqmc-new-quant.tgz +# tar xvfz ernie-3.0-medium-zh-afqmc-new-quant.tgz + +# PP-TRT +python benchmark_ernie_seq_cls.py --batch_size 40 --model_dir ernie-3.0-medium-zh-afqmc --backend pp-trt +python benchmark_ernie_seq_cls.py --batch_size 40 --model_dir ernie-3.0-medium-zh-afqmc --backend pp-trt --use_fp16 True +python benchmark_ernie_seq_cls.py --batch_size 40 --model_dir ernie-3.0-medium-zh-afqmc-new-quant --backend pp-trt + +# TRT +python benchmark_ernie_seq_cls.py --batch_size 40 --model_dir ernie-3.0-medium-zh-afqmc --backend trt +python benchmark_ernie_seq_cls.py --batch_size 40 --model_dir ernie-3.0-medium-zh-afqmc --backend trt --use_fp16 True +python benchmark_ernie_seq_cls.py --batch_size 40 --model_dir ernie-3.0-medium-zh-afqmc-new-quant --backend trt --use_fp16 True + +# CPU PP +python benchmark_ernie_seq_cls.py --batch_size 40 --cpu_num_threads 10 --model_dir ernie-3.0-medium-zh-afqmc --backend pp --device cpu +python benchmark_ernie_seq_cls.py --batch_size 40 --cpu_num_threads 10 --model_dir ernie-3.0-medium-zh-afqmc-new-quant --backend pp --device cpu + +# CPU ORT +python benchmark_ernie_seq_cls.py --batch_size 40 --cpu_num_threads 10 --model_dir ernie-3.0-medium-zh-afqmc --backend ort --device cpu +python benchmark_ernie_seq_cls.py --batch_size 40 --cpu_num_threads 10 --model_dir ernie-3.0-medium-zh-afqmc-new-quant --backend ort --device cpu From 686b71192f0f11acb6dffc2e111182785e936e88 Mon Sep 17 00:00:00 2001 From: Jason Date: Tue, 15 Nov 2022 11:51:57 +0800 Subject: [PATCH 031/136] Update jetson.md --- docs/cn/build_and_install/jetson.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/cn/build_and_install/jetson.md b/docs/cn/build_and_install/jetson.md index 3c918a8b483..f60aa280c00 100644 --- a/docs/cn/build_and_install/jetson.md +++ b/docs/cn/build_and_install/jetson.md @@ -16,7 +16,6 @@ cd FastDeploy mkdir build && cd build cmake .. -DBUILD_ON_JETSON=ON \ -DENABLE_VISION=ON \ - -DENABLE_TEXT=ON -DCMAKE_INSTALL_PREFIX=${PWD}/installed_fastdeploy make -j8 make install @@ -42,7 +41,6 @@ git clone https://github.com/PaddlePaddle/FastDeploy.git cd FastDeploy/python export BUILD_ON_JETSON=ON export ENABLE_VISION=ON -export ENABLE_TEXT=ON python setup.py build python setup.py bdist_wheel From 743f4c640bc71d8493a95534df66f6b1f76d5001 Mon Sep 17 00:00:00 2001 From: Winter <1558270516@qq.com> Date: Tue, 15 Nov 2022 12:53:26 +0800 Subject: [PATCH 032/136] [Android] Add classification app examples (#590) * 1.Classification Code submission; 2.Specification code format. * Specification code format. * Update strings.xml Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> --- .../assets/labels/imagenet1k_label_list.txt | 1000 +++++++++++++++++ .../ClassificationMainActivity.java | 387 +++++++ .../ClassificationSettingsActivity.java | 198 ++++ .../detection/DetectionMainActivity.java | 89 +- .../detection/DetectionSettingsActivity.java | 14 +- .../baidu/paddle/fastdeploy/app/ui/Utils.java | 45 + ...ultAdapter.java => BaseResultAdapter.java} | 6 +- .../layout/classification_activity_main.xml | 14 + .../res/layout/classification_camera_page.xml | 161 +++ .../res/layout/classification_result_page.xml | 160 +++ .../classification_result_page_item.xml | 26 + .../main/res/layout/detection_camera_page.xml | 5 +- .../app/src/main/res/values/strings.xml | 13 +- .../main/res/xml/classification_settings.xml | 43 +- .../app/src/main/res/xml/default_settings.xml | 45 - .../src/main/res/xml/detection_settings.xml | 6 +- 16 files changed, 2078 insertions(+), 134 deletions(-) create mode 100644 java/android/app/src/main/assets/labels/imagenet1k_label_list.txt create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationSettingsActivity.java rename java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/{DetectResultAdapter.java => BaseResultAdapter.java} (85%) create mode 100644 java/android/app/src/main/res/layout/classification_activity_main.xml create mode 100644 java/android/app/src/main/res/layout/classification_camera_page.xml create mode 100644 java/android/app/src/main/res/layout/classification_result_page.xml create mode 100644 java/android/app/src/main/res/layout/classification_result_page_item.xml delete mode 100644 java/android/app/src/main/res/xml/default_settings.xml diff --git a/java/android/app/src/main/assets/labels/imagenet1k_label_list.txt b/java/android/app/src/main/assets/labels/imagenet1k_label_list.txt new file mode 100644 index 00000000000..4b2ff820cde --- /dev/null +++ b/java/android/app/src/main/assets/labels/imagenet1k_label_list.txt @@ -0,0 +1,1000 @@ +0 tench, Tinca tinca +1 goldfish, Carassius auratus +2 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias +3 tiger shark, Galeocerdo cuvieri +4 hammerhead, hammerhead shark +5 electric ray, crampfish, numbfish, torpedo +6 stingray +7 cock +8 hen +9 ostrich, Struthio camelus +10 brambling, Fringilla montifringilla +11 goldfinch, Carduelis carduelis +12 house finch, linnet, Carpodacus mexicanus +13 junco, snowbird +14 indigo bunting, indigo finch, indigo bird, Passerina cyanea +15 robin, American robin, Turdus migratorius +16 bulbul +17 jay +18 magpie +19 chickadee +20 water ouzel, dipper +21 kite +22 bald eagle, American eagle, Haliaeetus leucocephalus +23 vulture +24 great grey owl, great gray owl, Strix nebulosa +25 European fire salamander, Salamandra salamandra +26 common newt, Triturus vulgaris +27 eft +28 spotted salamander, Ambystoma maculatum +29 axolotl, mud puppy, Ambystoma mexicanum +30 bullfrog, Rana catesbeiana +31 tree frog, tree-frog +32 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui +33 loggerhead, loggerhead turtle, Caretta caretta +34 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea +35 mud turtle +36 terrapin +37 box turtle, box tortoise +38 banded gecko +39 common iguana, iguana, Iguana iguana +40 American chameleon, anole, Anolis carolinensis +41 whiptail, whiptail lizard +42 agama +43 frilled lizard, Chlamydosaurus kingi +44 alligator lizard +45 Gila monster, Heloderma suspectum +46 green lizard, Lacerta viridis +47 African chameleon, Chamaeleo chamaeleon +48 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis +49 African crocodile, Nile crocodile, Crocodylus niloticus +50 American alligator, Alligator mississipiensis +51 triceratops +52 thunder snake, worm snake, Carphophis amoenus +53 ringneck snake, ring-necked snake, ring snake +54 hognose snake, puff adder, sand viper +55 green snake, grass snake +56 king snake, kingsnake +57 garter snake, grass snake +58 water snake +59 vine snake +60 night snake, Hypsiglena torquata +61 boa constrictor, Constrictor constrictor +62 rock python, rock snake, Python sebae +63 Indian cobra, Naja naja +64 green mamba +65 sea snake +66 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus +67 diamondback, diamondback rattlesnake, Crotalus adamanteus +68 sidewinder, horned rattlesnake, Crotalus cerastes +69 trilobite +70 harvestman, daddy longlegs, Phalangium opilio +71 scorpion +72 black and gold garden spider, Argiope aurantia +73 barn spider, Araneus cavaticus +74 garden spider, Aranea diademata +75 black widow, Latrodectus mactans +76 tarantula +77 wolf spider, hunting spider +78 tick +79 centipede +80 black grouse +81 ptarmigan +82 ruffed grouse, partridge, Bonasa umbellus +83 prairie chicken, prairie grouse, prairie fowl +84 peacock +85 quail +86 partridge +87 African grey, African gray, Psittacus erithacus +88 macaw +89 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita +90 lorikeet +91 coucal +92 bee eater +93 hornbill +94 hummingbird +95 jacamar +96 toucan +97 drake +98 red-breasted merganser, Mergus serrator +99 goose +100 black swan, Cygnus atratus +101 tusker +102 echidna, spiny anteater, anteater +103 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus +104 wallaby, brush kangaroo +105 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus +106 wombat +107 jellyfish +108 sea anemone, anemone +109 brain coral +110 flatworm, platyhelminth +111 nematode, nematode worm, roundworm +112 conch +113 snail +114 slug +115 sea slug, nudibranch +116 chiton, coat-of-mail shell, sea cradle, polyplacophore +117 chambered nautilus, pearly nautilus, nautilus +118 Dungeness crab, Cancer magister +119 rock crab, Cancer irroratus +120 fiddler crab +121 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica +122 American lobster, Northern lobster, Maine lobster, Homarus americanus +123 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish +124 crayfish, crawfish, crawdad, crawdaddy +125 hermit crab +126 isopod +127 white stork, Ciconia ciconia +128 black stork, Ciconia nigra +129 spoonbill +130 flamingo +131 little blue heron, Egretta caerulea +132 American egret, great white heron, Egretta albus +133 bittern +134 crane +135 limpkin, Aramus pictus +136 European gallinule, Porphyrio porphyrio +137 American coot, marsh hen, mud hen, water hen, Fulica americana +138 bustard +139 ruddy turnstone, Arenaria interpres +140 red-backed sandpiper, dunlin, Erolia alpina +141 redshank, Tringa totanus +142 dowitcher +143 oystercatcher, oyster catcher +144 pelican +145 king penguin, Aptenodytes patagonica +146 albatross, mollymawk +147 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus +148 killer whale, killer, orca, grampus, sea wolf, Orcinus orca +149 dugong, Dugong dugon +150 sea lion +151 Chihuahua +152 Japanese spaniel +153 Maltese dog, Maltese terrier, Maltese +154 Pekinese, Pekingese, Peke +155 Shih-Tzu +156 Blenheim spaniel +157 papillon +158 toy terrier +159 Rhodesian ridgeback +160 Afghan hound, Afghan +161 basset, basset hound +162 beagle +163 bloodhound, sleuthhound +164 bluetick +165 black-and-tan coonhound +166 Walker hound, Walker foxhound +167 English foxhound +168 redbone +169 borzoi, Russian wolfhound +170 Irish wolfhound +171 Italian greyhound +172 whippet +173 Ibizan hound, Ibizan Podenco +174 Norwegian elkhound, elkhound +175 otterhound, otter hound +176 Saluki, gazelle hound +177 Scottish deerhound, deerhound +178 Weimaraner +179 Staffordshire bullterrier, Staffordshire bull terrier +180 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier +181 Bedlington terrier +182 Border terrier +183 Kerry blue terrier +184 Irish terrier +185 Norfolk terrier +186 Norwich terrier +187 Yorkshire terrier +188 wire-haired fox terrier +189 Lakeland terrier +190 Sealyham terrier, Sealyham +191 Airedale, Airedale terrier +192 cairn, cairn terrier +193 Australian terrier +194 Dandie Dinmont, Dandie Dinmont terrier +195 Boston bull, Boston terrier +196 miniature schnauzer +197 giant schnauzer +198 standard schnauzer +199 Scotch terrier, Scottish terrier, Scottie +200 Tibetan terrier, chrysanthemum dog +201 silky terrier, Sydney silky +202 soft-coated wheaten terrier +203 West Highland white terrier +204 Lhasa, Lhasa apso +205 flat-coated retriever +206 curly-coated retriever +207 golden retriever +208 Labrador retriever +209 Chesapeake Bay retriever +210 German short-haired pointer +211 vizsla, Hungarian pointer +212 English setter +213 Irish setter, red setter +214 Gordon setter +215 Brittany spaniel +216 clumber, clumber spaniel +217 English springer, English springer spaniel +218 Welsh springer spaniel +219 cocker spaniel, English cocker spaniel, cocker +220 Sussex spaniel +221 Irish water spaniel +222 kuvasz +223 schipperke +224 groenendael +225 malinois +226 briard +227 kelpie +228 komondor +229 Old English sheepdog, bobtail +230 Shetland sheepdog, Shetland sheep dog, Shetland +231 collie +232 Border collie +233 Bouvier des Flandres, Bouviers des Flandres +234 Rottweiler +235 German shepherd, German shepherd dog, German police dog, alsatian +236 Doberman, Doberman pinscher +237 miniature pinscher +238 Greater Swiss Mountain dog +239 Bernese mountain dog +240 Appenzeller +241 EntleBucher +242 boxer +243 bull mastiff +244 Tibetan mastiff +245 French bulldog +246 Great Dane +247 Saint Bernard, St Bernard +248 Eskimo dog, husky +249 malamute, malemute, Alaskan malamute +250 Siberian husky +251 dalmatian, coach dog, carriage dog +252 affenpinscher, monkey pinscher, monkey dog +253 basenji +254 pug, pug-dog +255 Leonberg +256 Newfoundland, Newfoundland dog +257 Great Pyrenees +258 Samoyed, Samoyede +259 Pomeranian +260 chow, chow chow +261 keeshond +262 Brabancon griffon +263 Pembroke, Pembroke Welsh corgi +264 Cardigan, Cardigan Welsh corgi +265 toy poodle +266 miniature poodle +267 standard poodle +268 Mexican hairless +269 timber wolf, grey wolf, gray wolf, Canis lupus +270 white wolf, Arctic wolf, Canis lupus tundrarum +271 red wolf, maned wolf, Canis rufus, Canis niger +272 coyote, prairie wolf, brush wolf, Canis latrans +273 dingo, warrigal, warragal, Canis dingo +274 dhole, Cuon alpinus +275 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus +276 hyena, hyaena +277 red fox, Vulpes vulpes +278 kit fox, Vulpes macrotis +279 Arctic fox, white fox, Alopex lagopus +280 grey fox, gray fox, Urocyon cinereoargenteus +281 tabby, tabby cat +282 tiger cat +283 Persian cat +284 Siamese cat, Siamese +285 Egyptian cat +286 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor +287 lynx, catamount +288 leopard, Panthera pardus +289 snow leopard, ounce, Panthera uncia +290 jaguar, panther, Panthera onca, Felis onca +291 lion, king of beasts, Panthera leo +292 tiger, Panthera tigris +293 cheetah, chetah, Acinonyx jubatus +294 brown bear, bruin, Ursus arctos +295 American black bear, black bear, Ursus americanus, Euarctos americanus +296 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus +297 sloth bear, Melursus ursinus, Ursus ursinus +298 mongoose +299 meerkat, mierkat +300 tiger beetle +301 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle +302 ground beetle, carabid beetle +303 long-horned beetle, longicorn, longicorn beetle +304 leaf beetle, chrysomelid +305 dung beetle +306 rhinoceros beetle +307 weevil +308 fly +309 bee +310 ant, emmet, pismire +311 grasshopper, hopper +312 cricket +313 walking stick, walkingstick, stick insect +314 cockroach, roach +315 mantis, mantid +316 cicada, cicala +317 leafhopper +318 lacewing, lacewing fly +319 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk +320 damselfly +321 admiral +322 ringlet, ringlet butterfly +323 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus +324 cabbage butterfly +325 sulphur butterfly, sulfur butterfly +326 lycaenid, lycaenid butterfly +327 starfish, sea star +328 sea urchin +329 sea cucumber, holothurian +330 wood rabbit, cottontail, cottontail rabbit +331 hare +332 Angora, Angora rabbit +333 hamster +334 porcupine, hedgehog +335 fox squirrel, eastern fox squirrel, Sciurus niger +336 marmot +337 beaver +338 guinea pig, Cavia cobaya +339 sorrel +340 zebra +341 hog, pig, grunter, squealer, Sus scrofa +342 wild boar, boar, Sus scrofa +343 warthog +344 hippopotamus, hippo, river horse, Hippopotamus amphibius +345 ox +346 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis +347 bison +348 ram, tup +349 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis +350 ibex, Capra ibex +351 hartebeest +352 impala, Aepyceros melampus +353 gazelle +354 Arabian camel, dromedary, Camelus dromedarius +355 llama +356 weasel +357 mink +358 polecat, fitch, foulmart, foumart, Mustela putorius +359 black-footed ferret, ferret, Mustela nigripes +360 otter +361 skunk, polecat, wood pussy +362 badger +363 armadillo +364 three-toed sloth, ai, Bradypus tridactylus +365 orangutan, orang, orangutang, Pongo pygmaeus +366 gorilla, Gorilla gorilla +367 chimpanzee, chimp, Pan troglodytes +368 gibbon, Hylobates lar +369 siamang, Hylobates syndactylus, Symphalangus syndactylus +370 guenon, guenon monkey +371 patas, hussar monkey, Erythrocebus patas +372 baboon +373 macaque +374 langur +375 colobus, colobus monkey +376 proboscis monkey, Nasalis larvatus +377 marmoset +378 capuchin, ringtail, Cebus capucinus +379 howler monkey, howler +380 titi, titi monkey +381 spider monkey, Ateles geoffroyi +382 squirrel monkey, Saimiri sciureus +383 Madagascar cat, ring-tailed lemur, Lemur catta +384 indri, indris, Indri indri, Indri brevicaudatus +385 Indian elephant, Elephas maximus +386 African elephant, Loxodonta africana +387 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens +388 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca +389 barracouta, snoek +390 eel +391 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch +392 rock beauty, Holocanthus tricolor +393 anemone fish +394 sturgeon +395 gar, garfish, garpike, billfish, Lepisosteus osseus +396 lionfish +397 puffer, pufferfish, blowfish, globefish +398 abacus +399 abaya +400 academic gown, academic robe, judge's robe +401 accordion, piano accordion, squeeze box +402 acoustic guitar +403 aircraft carrier, carrier, flattop, attack aircraft carrier +404 airliner +405 airship, dirigible +406 altar +407 ambulance +408 amphibian, amphibious vehicle +409 analog clock +410 apiary, bee house +411 apron +412 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin +413 assault rifle, assault gun +414 backpack, back pack, knapsack, packsack, rucksack, haversack +415 bakery, bakeshop, bakehouse +416 balance beam, beam +417 balloon +418 ballpoint, ballpoint pen, ballpen, Biro +419 Band Aid +420 banjo +421 bannister, banister, balustrade, balusters, handrail +422 barbell +423 barber chair +424 barbershop +425 barn +426 barometer +427 barrel, cask +428 barrow, garden cart, lawn cart, wheelbarrow +429 baseball +430 basketball +431 bassinet +432 bassoon +433 bathing cap, swimming cap +434 bath towel +435 bathtub, bathing tub, bath, tub +436 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon +437 beacon, lighthouse, beacon light, pharos +438 beaker +439 bearskin, busby, shako +440 beer bottle +441 beer glass +442 bell cote, bell cot +443 bib +444 bicycle-built-for-two, tandem bicycle, tandem +445 bikini, two-piece +446 binder, ring-binder +447 binoculars, field glasses, opera glasses +448 birdhouse +449 boathouse +450 bobsled, bobsleigh, bob +451 bolo tie, bolo, bola tie, bola +452 bonnet, poke bonnet +453 bookcase +454 bookshop, bookstore, bookstall +455 bottlecap +456 bow +457 bow tie, bow-tie, bowtie +458 brass, memorial tablet, plaque +459 brassiere, bra, bandeau +460 breakwater, groin, groyne, mole, bulwark, seawall, jetty +461 breastplate, aegis, egis +462 broom +463 bucket, pail +464 buckle +465 bulletproof vest +466 bullet train, bullet +467 butcher shop, meat market +468 cab, hack, taxi, taxicab +469 caldron, cauldron +470 candle, taper, wax light +471 cannon +472 canoe +473 can opener, tin opener +474 cardigan +475 car mirror +476 carousel, carrousel, merry-go-round, roundabout, whirligig +477 carpenter's kit, tool kit +478 carton +479 car wheel +480 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM +481 cassette +482 cassette player +483 castle +484 catamaran +485 CD player +486 cello, violoncello +487 cellular telephone, cellular phone, cellphone, cell, mobile phone +488 chain +489 chainlink fence +490 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour +491 chain saw, chainsaw +492 chest +493 chiffonier, commode +494 chime, bell, gong +495 china cabinet, china closet +496 Christmas stocking +497 church, church building +498 cinema, movie theater, movie theatre, movie house, picture palace +499 cleaver, meat cleaver, chopper +500 cliff dwelling +501 cloak +502 clog, geta, patten, sabot +503 cocktail shaker +504 coffee mug +505 coffeepot +506 coil, spiral, volute, whorl, helix +507 combination lock +508 computer keyboard, keypad +509 confectionery, confectionary, candy store +510 container ship, containership, container vessel +511 convertible +512 corkscrew, bottle screw +513 cornet, horn, trumpet, trump +514 cowboy boot +515 cowboy hat, ten-gallon hat +516 cradle +517 crane +518 crash helmet +519 crate +520 crib, cot +521 Crock Pot +522 croquet ball +523 crutch +524 cuirass +525 dam, dike, dyke +526 desk +527 desktop computer +528 dial telephone, dial phone +529 diaper, nappy, napkin +530 digital clock +531 digital watch +532 dining table, board +533 dishrag, dishcloth +534 dishwasher, dish washer, dishwashing machine +535 disk brake, disc brake +536 dock, dockage, docking facility +537 dogsled, dog sled, dog sleigh +538 dome +539 doormat, welcome mat +540 drilling platform, offshore rig +541 drum, membranophone, tympan +542 drumstick +543 dumbbell +544 Dutch oven +545 electric fan, blower +546 electric guitar +547 electric locomotive +548 entertainment center +549 envelope +550 espresso maker +551 face powder +552 feather boa, boa +553 file, file cabinet, filing cabinet +554 fireboat +555 fire engine, fire truck +556 fire screen, fireguard +557 flagpole, flagstaff +558 flute, transverse flute +559 folding chair +560 football helmet +561 forklift +562 fountain +563 fountain pen +564 four-poster +565 freight car +566 French horn, horn +567 frying pan, frypan, skillet +568 fur coat +569 garbage truck, dustcart +570 gasmask, respirator, gas helmet +571 gas pump, gasoline pump, petrol pump, island dispenser +572 goblet +573 go-kart +574 golf ball +575 golfcart, golf cart +576 gondola +577 gong, tam-tam +578 gown +579 grand piano, grand +580 greenhouse, nursery, glasshouse +581 grille, radiator grille +582 grocery store, grocery, food market, market +583 guillotine +584 hair slide +585 hair spray +586 half track +587 hammer +588 hamper +589 hand blower, blow dryer, blow drier, hair dryer, hair drier +590 hand-held computer, hand-held microcomputer +591 handkerchief, hankie, hanky, hankey +592 hard disc, hard disk, fixed disk +593 harmonica, mouth organ, harp, mouth harp +594 harp +595 harvester, reaper +596 hatchet +597 holster +598 home theater, home theatre +599 honeycomb +600 hook, claw +601 hoopskirt, crinoline +602 horizontal bar, high bar +603 horse cart, horse-cart +604 hourglass +605 iPod +606 iron, smoothing iron +607 jack-o'-lantern +608 jean, blue jean, denim +609 jeep, landrover +610 jersey, T-shirt, tee shirt +611 jigsaw puzzle +612 jinrikisha, ricksha, rickshaw +613 joystick +614 kimono +615 knee pad +616 knot +617 lab coat, laboratory coat +618 ladle +619 lampshade, lamp shade +620 laptop, laptop computer +621 lawn mower, mower +622 lens cap, lens cover +623 letter opener, paper knife, paperknife +624 library +625 lifeboat +626 lighter, light, igniter, ignitor +627 limousine, limo +628 liner, ocean liner +629 lipstick, lip rouge +630 Loafer +631 lotion +632 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system +633 loupe, jeweler's loupe +634 lumbermill, sawmill +635 magnetic compass +636 mailbag, postbag +637 mailbox, letter box +638 maillot +639 maillot, tank suit +640 manhole cover +641 maraca +642 marimba, xylophone +643 mask +644 matchstick +645 maypole +646 maze, labyrinth +647 measuring cup +648 medicine chest, medicine cabinet +649 megalith, megalithic structure +650 microphone, mike +651 microwave, microwave oven +652 military uniform +653 milk can +654 minibus +655 miniskirt, mini +656 minivan +657 missile +658 mitten +659 mixing bowl +660 mobile home, manufactured home +661 Model T +662 modem +663 monastery +664 monitor +665 moped +666 mortar +667 mortarboard +668 mosque +669 mosquito net +670 motor scooter, scooter +671 mountain bike, all-terrain bike, off-roader +672 mountain tent +673 mouse, computer mouse +674 mousetrap +675 moving van +676 muzzle +677 nail +678 neck brace +679 necklace +680 nipple +681 notebook, notebook computer +682 obelisk +683 oboe, hautboy, hautbois +684 ocarina, sweet potato +685 odometer, hodometer, mileometer, milometer +686 oil filter +687 organ, pipe organ +688 oscilloscope, scope, cathode-ray oscilloscope, CRO +689 overskirt +690 oxcart +691 oxygen mask +692 packet +693 paddle, boat paddle +694 paddlewheel, paddle wheel +695 padlock +696 paintbrush +697 pajama, pyjama, pj's, jammies +698 palace +699 panpipe, pandean pipe, syrinx +700 paper towel +701 parachute, chute +702 parallel bars, bars +703 park bench +704 parking meter +705 passenger car, coach, carriage +706 patio, terrace +707 pay-phone, pay-station +708 pedestal, plinth, footstall +709 pencil box, pencil case +710 pencil sharpener +711 perfume, essence +712 Petri dish +713 photocopier +714 pick, plectrum, plectron +715 pickelhaube +716 picket fence, paling +717 pickup, pickup truck +718 pier +719 piggy bank, penny bank +720 pill bottle +721 pillow +722 ping-pong ball +723 pinwheel +724 pirate, pirate ship +725 pitcher, ewer +726 plane, carpenter's plane, woodworking plane +727 planetarium +728 plastic bag +729 plate rack +730 plow, plough +731 plunger, plumber's helper +732 Polaroid camera, Polaroid Land camera +733 pole +734 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria +735 poncho +736 pool table, billiard table, snooker table +737 pop bottle, soda bottle +738 pot, flowerpot +739 potter's wheel +740 power drill +741 prayer rug, prayer mat +742 printer +743 prison, prison house +744 projectile, missile +745 projector +746 puck, hockey puck +747 punching bag, punch bag, punching ball, punchball +748 purse +749 quill, quill pen +750 quilt, comforter, comfort, puff +751 racer, race car, racing car +752 racket, racquet +753 radiator +754 radio, wireless +755 radio telescope, radio reflector +756 rain barrel +757 recreational vehicle, RV, R.V. +758 reel +759 reflex camera +760 refrigerator, icebox +761 remote control, remote +762 restaurant, eating house, eating place, eatery +763 revolver, six-gun, six-shooter +764 rifle +765 rocking chair, rocker +766 rotisserie +767 rubber eraser, rubber, pencil eraser +768 rugby ball +769 rule, ruler +770 running shoe +771 safe +772 safety pin +773 saltshaker, salt shaker +774 sandal +775 sarong +776 sax, saxophone +777 scabbard +778 scale, weighing machine +779 school bus +780 schooner +781 scoreboard +782 screen, CRT screen +783 screw +784 screwdriver +785 seat belt, seatbelt +786 sewing machine +787 shield, buckler +788 shoe shop, shoe-shop, shoe store +789 shoji +790 shopping basket +791 shopping cart +792 shovel +793 shower cap +794 shower curtain +795 ski +796 ski mask +797 sleeping bag +798 slide rule, slipstick +799 sliding door +800 slot, one-armed bandit +801 snorkel +802 snowmobile +803 snowplow, snowplough +804 soap dispenser +805 soccer ball +806 sock +807 solar dish, solar collector, solar furnace +808 sombrero +809 soup bowl +810 space bar +811 space heater +812 space shuttle +813 spatula +814 speedboat +815 spider web, spider's web +816 spindle +817 sports car, sport car +818 spotlight, spot +819 stage +820 steam locomotive +821 steel arch bridge +822 steel drum +823 stethoscope +824 stole +825 stone wall +826 stopwatch, stop watch +827 stove +828 strainer +829 streetcar, tram, tramcar, trolley, trolley car +830 stretcher +831 studio couch, day bed +832 stupa, tope +833 submarine, pigboat, sub, U-boat +834 suit, suit of clothes +835 sundial +836 sunglass +837 sunglasses, dark glasses, shades +838 sunscreen, sunblock, sun blocker +839 suspension bridge +840 swab, swob, mop +841 sweatshirt +842 swimming trunks, bathing trunks +843 swing +844 switch, electric switch, electrical switch +845 syringe +846 table lamp +847 tank, army tank, armored combat vehicle, armoured combat vehicle +848 tape player +849 teapot +850 teddy, teddy bear +851 television, television system +852 tennis ball +853 thatch, thatched roof +854 theater curtain, theatre curtain +855 thimble +856 thresher, thrasher, threshing machine +857 throne +858 tile roof +859 toaster +860 tobacco shop, tobacconist shop, tobacconist +861 toilet seat +862 torch +863 totem pole +864 tow truck, tow car, wrecker +865 toyshop +866 tractor +867 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi +868 tray +869 trench coat +870 tricycle, trike, velocipede +871 trimaran +872 tripod +873 triumphal arch +874 trolleybus, trolley coach, trackless trolley +875 trombone +876 tub, vat +877 turnstile +878 typewriter keyboard +879 umbrella +880 unicycle, monocycle +881 upright, upright piano +882 vacuum, vacuum cleaner +883 vase +884 vault +885 velvet +886 vending machine +887 vestment +888 viaduct +889 violin, fiddle +890 volleyball +891 waffle iron +892 wall clock +893 wallet, billfold, notecase, pocketbook +894 wardrobe, closet, press +895 warplane, military plane +896 washbasin, handbasin, washbowl, lavabo, wash-hand basin +897 washer, automatic washer, washing machine +898 water bottle +899 water jug +900 water tower +901 whiskey jug +902 whistle +903 wig +904 window screen +905 window shade +906 Windsor tie +907 wine bottle +908 wing +909 wok +910 wooden spoon +911 wool, woolen, woollen +912 worm fence, snake fence, snake-rail fence, Virginia fence +913 wreck +914 yawl +915 yurt +916 web site, website, internet site, site +917 comic book +918 crossword puzzle, crossword +919 street sign +920 traffic light, traffic signal, stoplight +921 book jacket, dust cover, dust jacket, dust wrapper +922 menu +923 plate +924 guacamole +925 consomme +926 hot pot, hotpot +927 trifle +928 ice cream, icecream +929 ice lolly, lolly, lollipop, popsicle +930 French loaf +931 bagel, beigel +932 pretzel +933 cheeseburger +934 hotdog, hot dog, red hot +935 mashed potato +936 head cabbage +937 broccoli +938 cauliflower +939 zucchini, courgette +940 spaghetti squash +941 acorn squash +942 butternut squash +943 cucumber, cuke +944 artichoke, globe artichoke +945 bell pepper +946 cardoon +947 mushroom +948 Granny Smith +949 strawberry +950 orange +951 lemon +952 fig +953 pineapple, ananas +954 banana +955 jackfruit, jak, jack +956 custard apple +957 pomegranate +958 hay +959 carbonara +960 chocolate sauce, chocolate syrup +961 dough +962 meat loaf, meatloaf +963 pizza, pizza pie +964 potpie +965 burrito +966 red wine +967 espresso +968 cup +969 eggnog +970 alp +971 bubble +972 cliff, drop, drop-off +973 coral reef +974 geyser +975 lakeside, lakeshore +976 promontory, headland, head, foreland +977 sandbar, sand bar +978 seashore, coast, seacoast, sea-coast +979 valley, vale +980 volcano +981 ballplayer, baseball player +982 groom, bridegroom +983 scuba diver +984 rapeseed +985 daisy +986 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum +987 corn +988 acorn +989 hip, rose hip, rosehip +990 buckeye, horse chestnut, conker +991 coral fungus +992 agaric +993 gyromitra +994 stinkhorn, carrion fungus +995 earthstar +996 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa +997 bolete +998 ear, spike, capitulum +999 toilet tissue, toilet paper, bathroom tissue \ No newline at end of file diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java new file mode 100644 index 00000000000..c78f8f39783 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java @@ -0,0 +1,387 @@ +package com.baidu.paddle.fastdeploy.app.examples.classification; + +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.AlertDialog; +import android.content.DialogInterface; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.net.Uri; +import android.os.Bundle; +import android.os.SystemClock; +import android.preference.PreferenceManager; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; +import android.widget.ImageButton; +import android.widget.ImageView; +import android.widget.SeekBar; +import android.widget.TextView; + +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; +import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; +import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; +import com.baidu.paddle.fastdeploy.vision.ClassifyResult; +import com.baidu.paddle.fastdeploy.vision.classification.PaddleClasModel; + +import java.io.File; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +public class ClassificationMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { + private static final String TAG = ClassificationMainActivity.class.getSimpleName(); + + CameraSurfaceView svPreview; + TextView tvStatus; + ImageButton btnSwitch; + ImageButton btnShutter; + ImageButton btnSettings; + ImageView realtimeToggleButton; + boolean isRealtimeStatusRunning = false; + ImageView backInPreview; + private ImageView albumSelectButton; + private View cameraPageView; + private ViewGroup resultPageView; + private ImageView resultImage; + private ImageView backInResult; + private SeekBar confidenceSeekbar; + private TextView seekbarText; + private float resultNum = 1.0f; + private ResultListView detectResultView; + private Bitmap shutterBitmap; + private Bitmap originShutterBitmap; + private Bitmap picBitmap; + private Bitmap originPicBitmap; + + public static final int TYPE_UNKNOWN = -1; + public static final int BTN_SHUTTER = 0; + public static final int ALBUM_SELECT = 1; + private static int TYPE = TYPE_UNKNOWN; + + private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; + private static final int INTENT_CODE_PICK_IMAGE = 100; + + String savedImagePath = "result.jpg"; + int lastFrameIndex = 0; + long lastFrameTime; + + // Call 'init' and 'release' manually later + PaddleClasModel predictor = new PaddleClasModel(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + // Fullscreen + requestWindowFeature(Window.FEATURE_NO_TITLE); + getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); + + setContentView(R.layout.classification_activity_main); + + // Clear all setting items to avoid app crashing due to the incorrect settings + initSettings(); + + // Init the camera preview and UI components + initView(); + + // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions + if (!checkAllPermissions()) { + requestAllPermissions(); + } + } + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View v) { + switch (v.getId()) { + case R.id.btn_switch: + svPreview.switchCamera(); + break; + case R.id.btn_shutter: + TYPE = BTN_SHUTTER; + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + resultImage.setImageBitmap(shutterBitmap); + break; + case R.id.btn_settings: + startActivity(new Intent(ClassificationMainActivity.this, ClassificationSettingsActivity.class)); + break; + case R.id.realtime_toggle_btn: + toggleRealtimeStyle(); + break; + case R.id.back_in_preview: + finish(); + break; + case R.id.albumSelect: + TYPE = ALBUM_SELECT; + // Judge whether authority has been granted. + if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + // If this permission was requested before the application but the user refused the request, this method will return true. + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_PERMISSION_CODE_STORAGE); + } else { + Intent intent = new Intent(Intent.ACTION_PICK); + intent.setType("image/*"); + startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); + } + break; + case R.id.back_in_result: + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + svPreview.onResume(); + break; + + } + } + + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (requestCode == INTENT_CODE_PICK_IMAGE) { + if (resultCode == Activity.RESULT_OK) { + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + Uri uri = data.getData(); + String path = getRealPathFromURI(this, uri); + picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); + resultImage.setImageBitmap(picBitmap); + } + } + } + + private void toggleRealtimeStyle() { + if (isRealtimeStatusRunning) { + isRealtimeStatusRunning = false; + realtimeToggleButton.setImageResource(R.drawable.realtime_stop_btn); + svPreview.setOnTextureChangedListener(this); + tvStatus.setVisibility(View.VISIBLE); + } else { + isRealtimeStatusRunning = true; + realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); + tvStatus.setVisibility(View.GONE); + svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + return false; + } + }); + } + } + + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + String savedImagePath = ""; + synchronized (this) { + savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; + } + if (TYPE == BTN_SHUTTER) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } else { + // Only reference in predict loops. + shutterBitmap = ARGB8888ImageBitmap; + originShutterBitmap = ARGB8888ImageBitmap; + } + boolean modified = false; + ClassifyResult result = predictor.predict( + ARGB8888ImageBitmap, true, ClassificationSettingsActivity.scoreThreshold); + modified = result.initialized(); + if (!savedImagePath.isEmpty()) { + synchronized (this) { + ClassificationMainActivity.this.savedImagePath = "result.jpg"; + } + } + lastFrameIndex++; + if (lastFrameIndex >= 30) { + final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + tvStatus.setText(Integer.toString(fps) + "fps"); + } + }); + lastFrameIndex = 0; + lastFrameTime = System.nanoTime(); + } + return modified; + } + + @Override + protected void onResume() { + super.onResume(); + // Reload settings and re-initialize the predictor + checkAndUpdateSettings(); + // Open camera until the permissions have been granted + if (!checkAllPermissions()) { + svPreview.disableCamera(); + } + svPreview.onResume(); + } + + @Override + protected void onPause() { + super.onPause(); + svPreview.onPause(); + } + + @Override + protected void onDestroy() { + if (predictor != null) { + predictor.release(); + } + super.onDestroy(); + } + + public void initView() { + TYPE = BTN_SHUTTER; + svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); + svPreview.setOnTextureChangedListener(this); + tvStatus = (TextView) findViewById(R.id.tv_status); + btnSwitch = (ImageButton) findViewById(R.id.btn_switch); + btnSwitch.setOnClickListener(this); + btnShutter = (ImageButton) findViewById(R.id.btn_shutter); + btnShutter.setOnClickListener(this); + btnSettings = (ImageButton) findViewById(R.id.btn_settings); + btnSettings.setOnClickListener(this); + realtimeToggleButton = findViewById(R.id.realtime_toggle_btn); + realtimeToggleButton.setOnClickListener(this); + backInPreview = findViewById(R.id.back_in_preview); + backInPreview.setOnClickListener(this); + albumSelectButton = findViewById(R.id.albumSelect); + albumSelectButton.setOnClickListener(this); + cameraPageView = findViewById(R.id.camera_page); + resultPageView = findViewById(R.id.result_page); + resultImage = findViewById(R.id.result_image); + backInResult = findViewById(R.id.back_in_result); + backInResult.setOnClickListener(this); + confidenceSeekbar = findViewById(R.id.confidence_seekbar); + seekbarText = findViewById(R.id.seekbar_text); + detectResultView = findViewById(R.id.result_list_view); + + List results = new ArrayList<>(); + results.add(new BaseResultModel(1, "cup", 0.4f)); + results.add(new BaseResultModel(2, "pen", 0.6f)); + results.add(new BaseResultModel(3, "tang", 1.0f)); + final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.classification_result_page_item, results); + detectResultView.setAdapter(adapter); + detectResultView.invalidate(); + + confidenceSeekbar.setMax(100); + confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { + @Override + public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { + float resultConfidence = seekBar.getProgress() / 100f; + BigDecimal bd = new BigDecimal(resultConfidence); + resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + } + + @Override + public void onStartTrackingTouch(SeekBar seekBar) { + + } + + @Override + public void onStopTrackingTouch(SeekBar seekBar) { + runOnUiThread(new Runnable() { + @Override + public void run() { + if (TYPE == ALBUM_SELECT) { + SystemClock.sleep(500); + predictor.predict(picBitmap, savedImagePath, resultNum); + resultImage.setImageBitmap(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); + resultNum = 1.0f; + } else { + SystemClock.sleep(500); + predictor.predict(shutterBitmap, savedImagePath, resultNum); + resultImage.setImageBitmap(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); + resultNum = 1.0f; + } + } + }); + } + }); + } + + @SuppressLint("ApplySharedPref") + public void initSettings() { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.clear(); + editor.commit(); + ClassificationSettingsActivity.resetSettings(); + } + + public void checkAndUpdateSettings() { + if (ClassificationSettingsActivity.checkAndUpdateSettings(this)) { + String realModelDir = getCacheDir() + "/" + ClassificationSettingsActivity.modelDir; + Utils.copyDirectoryFromAssets(this, ClassificationSettingsActivity.modelDir, realModelDir); + String realLabelPath = getCacheDir() + "/" + ClassificationSettingsActivity.labelPath; + Utils.copyFileFromAssets(this, ClassificationSettingsActivity.labelPath, realLabelPath); + + String modelFile = realModelDir + "/" + "inference.pdmodel"; + String paramsFile = realModelDir + "/" + "inference.pdiparams"; + String configFile = realModelDir + "/" + "inference_cls.yaml"; + String labelFile = realLabelPath; + RuntimeOption option = new RuntimeOption(); + option.setCpuThreadNum(ClassificationSettingsActivity.cpuThreadNum); + option.setLitePowerMode(ClassificationSettingsActivity.cpuPowerMode); + if (Boolean.parseBoolean(ClassificationSettingsActivity.enableLiteFp16)) { + option.enableLiteFp16(); + } + predictor.init(modelFile, paramsFile, configFile, labelFile, option); + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, + @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (grantResults[0] != PackageManager.PERMISSION_GRANTED || grantResults[1] != PackageManager.PERMISSION_GRANTED) { + new AlertDialog.Builder(ClassificationMainActivity.this) + .setTitle("Permission denied") + .setMessage("Click to force quit the app, then open Settings->Apps & notifications->Target " + + "App->Permissions to grant all of the permissions.") + .setCancelable(false) + .setPositiveButton("Exit", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + ClassificationMainActivity.this.finish(); + } + }).show(); + } + } + + private void requestAllPermissions() { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.CAMERA}, 0); + } + + private boolean checkAllPermissions() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED + && ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationSettingsActivity.java new file mode 100644 index 00000000000..23ab16f9aaa --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationSettingsActivity.java @@ -0,0 +1,198 @@ +package com.baidu.paddle.fastdeploy.app.examples.classification; + +import android.annotation.SuppressLint; +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Bundle; +import android.preference.EditTextPreference; +import android.preference.ListPreference; +import android.preference.PreferenceManager; +import android.support.v7.app.ActionBar; + +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.AppCompatPreferenceActivity; + +import java.util.ArrayList; +import java.util.List; + +public class ClassificationSettingsActivity extends AppCompatPreferenceActivity implements + SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = ClassificationSettingsActivity.class.getSimpleName(); + + static public int selectedModelIdx = -1; + static public String modelDir = ""; + static public String labelPath = ""; + static public int cpuThreadNum = 2; + static public String cpuPowerMode = ""; + static public float scoreThreshold = 0.1f; + static public String enableLiteFp16 = "true"; + + ListPreference lpChoosePreInstalledModel = null; + EditTextPreference etModelDir = null; + EditTextPreference etLabelPath = null; + ListPreference lpCPUThreadNum = null; + ListPreference lpCPUPowerMode = null; + EditTextPreference etScoreThreshold = null; + ListPreference lpEnableLiteFp16 = null; + + List preInstalledModelDirs = null; + List preInstalledLabelPaths = null; + List preInstalledCPUThreadNums = null; + List preInstalledCPUPowerModes = null; + List preInstalledScoreThresholds = null; + List preInstalledEnableLiteFp16s = null; + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + addPreferencesFromResource(R.xml.classification_settings); + ActionBar supportActionBar = getSupportActionBar(); + if (supportActionBar != null) { + supportActionBar.setDisplayHomeAsUpEnabled(true); + } + + // Initialize pre-installed models + preInstalledModelDirs = new ArrayList(); + preInstalledLabelPaths = new ArrayList(); + preInstalledCPUThreadNums = new ArrayList(); + preInstalledCPUPowerModes = new ArrayList(); + preInstalledScoreThresholds = new ArrayList(); + preInstalledEnableLiteFp16s = new ArrayList(); + preInstalledModelDirs.add(getString(R.string.CLASSIFICATION_MODEL_DIR_DEFAULT)); + preInstalledLabelPaths.add(getString(R.string.CLASSIFICATION_LABEL_PATH_DEFAULT)); + preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); + preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); + preInstalledScoreThresholds.add(getString(R.string.SCORE_THRESHOLD_CLASSIFICATION)); + preInstalledEnableLiteFp16s.add(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + // Setup UI components + lpChoosePreInstalledModel = + (ListPreference) findPreference(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY)); + String[] preInstalledModelNames = new String[preInstalledModelDirs.size()]; + for (int i = 0; i < preInstalledModelDirs.size(); i++) { + preInstalledModelNames[i] = preInstalledModelDirs.get(i).substring(preInstalledModelDirs.get(i).lastIndexOf("/") + 1); + } + lpChoosePreInstalledModel.setEntries(preInstalledModelNames); + lpChoosePreInstalledModel.setEntryValues(preInstalledModelDirs.toArray(new String[preInstalledModelDirs.size()])); + lpCPUThreadNum = (ListPreference) findPreference(getString(R.string.CPU_THREAD_NUM_KEY)); + lpCPUPowerMode = (ListPreference) findPreference(getString(R.string.CPU_POWER_MODE_KEY)); + etModelDir = (EditTextPreference) findPreference(getString(R.string.MODEL_DIR_KEY)); + etModelDir.setTitle("Model dir (SDCard: " + Utils.getSDCardDirectory() + ")"); + etLabelPath = (EditTextPreference) findPreference(getString(R.string.LABEL_PATH_KEY)); + etLabelPath.setTitle("Label path (SDCard: " + Utils.getSDCardDirectory() + ")"); + etScoreThreshold = (EditTextPreference) findPreference(getString(R.string.SCORE_THRESHOLD_KEY)); + lpEnableLiteFp16 = (ListPreference) findPreference(getString(R.string.ENABLE_LITE_FP16_MODE_KEY)); + } + + @SuppressLint("ApplySharedPref") + private void reloadSettingsAndUpdateUI() { + SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); + + String selected_model_dir = sharedPreferences.getString(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY), + getString(R.string.CLASSIFICATION_MODEL_DIR_DEFAULT)); + int selected_model_idx = lpChoosePreInstalledModel.findIndexOfValue(selected_model_dir); + if (selected_model_idx >= 0 && selected_model_idx < preInstalledModelDirs.size() && selected_model_idx != selectedModelIdx) { + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.putString(getString(R.string.MODEL_DIR_KEY), preInstalledModelDirs.get(selected_model_idx)); + editor.putString(getString(R.string.LABEL_PATH_KEY), preInstalledLabelPaths.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_THREAD_NUM_KEY), preInstalledCPUThreadNums.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_POWER_MODE_KEY), preInstalledCPUPowerModes.get(selected_model_idx)); + editor.putString(getString(R.string.SCORE_THRESHOLD_KEY), preInstalledScoreThresholds.get(selected_model_idx)); + editor.putString(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT), preInstalledEnableLiteFp16s.get(selected_model_idx)); + editor.commit(); + lpChoosePreInstalledModel.setSummary(selected_model_dir); + selectedModelIdx = selected_model_idx; + } + + String model_dir = sharedPreferences.getString(getString(R.string.MODEL_DIR_KEY), + getString(R.string.CLASSIFICATION_MODEL_DIR_DEFAULT)); + String label_path = sharedPreferences.getString(getString(R.string.LABEL_PATH_KEY), + getString(R.string.CLASSIFICATION_LABEL_PATH_DEFAULT)); + String cpu_thread_num = sharedPreferences.getString(getString(R.string.CPU_THREAD_NUM_KEY), + getString(R.string.CPU_THREAD_NUM_DEFAULT)); + String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), + getString(R.string.CPU_POWER_MODE_DEFAULT)); + String score_threshold = sharedPreferences.getString(getString(R.string.SCORE_THRESHOLD_KEY), + getString(R.string.SCORE_THRESHOLD_CLASSIFICATION)); + String enable_lite_fp16 = sharedPreferences.getString(getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + etModelDir.setSummary(model_dir); + etLabelPath.setSummary(label_path); + lpCPUThreadNum.setValue(cpu_thread_num); + lpCPUThreadNum.setSummary(cpu_thread_num); + lpCPUPowerMode.setValue(cpu_power_mode); + lpCPUPowerMode.setSummary(cpu_power_mode); + etScoreThreshold.setSummary(score_threshold); + etScoreThreshold.setText(score_threshold); + lpEnableLiteFp16.setValue(enable_lite_fp16); + lpEnableLiteFp16.setSummary(enable_lite_fp16); + + } + + static boolean checkAndUpdateSettings(Context ctx) { + boolean settingsChanged = false; + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(ctx); + + String model_dir = sharedPreferences.getString(ctx.getString(R.string.MODEL_DIR_KEY), + ctx.getString(R.string.CLASSIFICATION_MODEL_DIR_DEFAULT)); + settingsChanged |= !modelDir.equalsIgnoreCase(model_dir); + modelDir = model_dir; + + String label_path = sharedPreferences.getString(ctx.getString(R.string.LABEL_PATH_KEY), + ctx.getString(R.string.CLASSIFICATION_LABEL_PATH_DEFAULT)); + settingsChanged |= !labelPath.equalsIgnoreCase(label_path); + labelPath = label_path; + + String cpu_thread_num = sharedPreferences.getString(ctx.getString(R.string.CPU_THREAD_NUM_KEY), + ctx.getString(R.string.CPU_THREAD_NUM_DEFAULT)); + settingsChanged |= cpuThreadNum != Integer.parseInt(cpu_thread_num); + cpuThreadNum = Integer.parseInt(cpu_thread_num); + + String cpu_power_mode = sharedPreferences.getString(ctx.getString(R.string.CPU_POWER_MODE_KEY), + ctx.getString(R.string.CPU_POWER_MODE_DEFAULT)); + settingsChanged |= !cpuPowerMode.equalsIgnoreCase(cpu_power_mode); + cpuPowerMode = cpu_power_mode; + + String score_threshold = sharedPreferences.getString(ctx.getString(R.string.SCORE_THRESHOLD_KEY), + ctx.getString(R.string.SCORE_THRESHOLD_CLASSIFICATION)); + settingsChanged |= scoreThreshold != Float.parseFloat(score_threshold); + scoreThreshold = Float.parseFloat(score_threshold); + + String enable_lite_fp16 = sharedPreferences.getString(ctx.getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + ctx.getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + settingsChanged |= !enableLiteFp16.equalsIgnoreCase(enable_lite_fp16); + enableLiteFp16 = enable_lite_fp16; + + return settingsChanged; + } + + static void resetSettings() { + selectedModelIdx = -1; + modelDir = ""; + labelPath = ""; + cpuThreadNum = 2; + cpuPowerMode = ""; + scoreThreshold = 0.1f; + enableLiteFp16 = "true"; + } + + @Override + protected void onResume() { + super.onResume(); + getPreferenceScreen().getSharedPreferences().registerOnSharedPreferenceChangeListener(this); + reloadSettingsAndUpdateUI(); + } + + @Override + protected void onPause() { + super.onPause(); + getPreferenceScreen().getSharedPreferences().unregisterOnSharedPreferenceChangeListener(this); + } + + @Override + public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key) { + reloadSettingsAndUpdateUI(); + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index ec5988f68fb..2501206a011 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -1,5 +1,8 @@ package com.baidu.paddle.fastdeploy.app.examples.detection; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; + import android.Manifest; import android.annotation.SuppressLint; import android.app.Activity; @@ -34,7 +37,7 @@ import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; import com.baidu.paddle.fastdeploy.app.ui.Utils; -import com.baidu.paddle.fastdeploy.app.ui.view.adapter.DetectResultAdapter; +import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.DetectionResult; import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; @@ -56,8 +59,8 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe boolean isRealtimeStatusRunning = false; ImageView backInPreview; private ImageView albumSelectButton; - private View mCameraPageView; - private ViewGroup mResultPageView; + private View cameraPageView; + private ViewGroup resultPageView; private ImageView resultImage; private ImageView backInResult; private SeekBar confidenceSeekbar; @@ -68,6 +71,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private Bitmap originShutterBitmap; private Bitmap picBitmap; private Bitmap originPicBitmap; + public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; public static final int ALBUM_SELECT = 1; @@ -115,8 +119,8 @@ public void onClick(View v) { case R.id.btn_shutter: TYPE = BTN_SHUTTER; svPreview.onPause(); - mCameraPageView.setVisibility(View.GONE); - mResultPageView.setVisibility(View.VISIBLE); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); resultImage.setImageBitmap(shutterBitmap); @@ -132,9 +136,9 @@ public void onClick(View v) { break; case R.id.albumSelect: TYPE = ALBUM_SELECT; - // 判断是否已经赋予权限 + // Judge whether authority has been granted. if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { - // 如果应用之前请求过此权限但用户拒绝了请求，此方法将返回 true。 + // If this permission was requested before the application but the user refused the request, this method will return true. ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_PERMISSION_CODE_STORAGE); } else { Intent intent = new Intent(Intent.ACTION_PICK); @@ -143,8 +147,8 @@ public void onClick(View v) { } break; case R.id.back_in_result: - mResultPageView.setVisibility(View.GONE); - mCameraPageView.setVisibility(View.VISIBLE); + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); svPreview.onResume(); break; @@ -156,12 +160,12 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { super.onActivityResult(requestCode, resultCode, data); if (requestCode == INTENT_CODE_PICK_IMAGE) { if (resultCode == Activity.RESULT_OK) { - mCameraPageView.setVisibility(View.GONE); - mResultPageView.setVisibility(View.VISIBLE); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); Uri uri = data.getData(); - String path = getRealPathFromURI(uri); + String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); @@ -169,25 +173,6 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { } } - private String getRealPathFromURI(Uri contentURI) { - String result; - Cursor cursor = null; - try { - cursor = getContentResolver().query(contentURI, null, null, null, null); - } catch (Throwable e) { - e.printStackTrace(); - } - if (cursor == null) { - result = contentURI.getPath(); - } else { - cursor.moveToFirst(); - int idx = cursor.getColumnIndex(MediaStore.Images.ImageColumns.DATA); - result = cursor.getString(idx); - cursor.close(); - } - return result; - } - private void toggleRealtimeStyle() { if (isRealtimeStatusRunning) { isRealtimeStatusRunning = false; @@ -217,7 +202,7 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } else { - // Only reference in detecting loops. + // Only reference in predict loops. shutterBitmap = ARGB8888ImageBitmap; originShutterBitmap = ARGB8888ImageBitmap; } @@ -226,7 +211,6 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { DetectionResult result = predictor.predict( ARGB8888ImageBitmap, true, DetectionSettingsActivity.scoreThreshold); modified = result.initialized(); - if (!savedImagePath.isEmpty()) { synchronized (this) { DetectionMainActivity.this.savedImagePath = "result.jpg"; @@ -247,38 +231,6 @@ public void run() { return modified; } - /** - * @param path 路径 - * @param displayWidth 需要显示的宽度 - * @param displayHeight 需要显示的高度 - * @return Bitmap - */ - public static Bitmap decodeBitmap(String path, int displayWidth, int displayHeight) { - BitmapFactory.Options op = new BitmapFactory.Options(); - op.inJustDecodeBounds = true; - // op.inJustDecodeBounds = true;表示我们只读取Bitmap的宽高等信息，不读取像素。 - Bitmap bmp = BitmapFactory.decodeFile(path, op); // 获取尺寸信息 - // op.outWidth表示的是图像真实的宽度 - // op.inSamplySize 表示的是缩小的比例 - // op.inSamplySize = 4,表示缩小1/4的宽和高，1/16的像素，android认为设置为2是最快的。 - // 获取比例大小 - int wRatio = (int) Math.ceil(op.outWidth / (float) displayWidth); - int hRatio = (int) Math.ceil(op.outHeight / (float) displayHeight); - // 如果超出指定大小，则缩小相应的比例 - if (wRatio > 1 && hRatio > 1) { - if (wRatio > hRatio) { - // 如果太宽，我们就缩小宽度到需要的大小，注意，高度就会变得更加的小。 - op.inSampleSize = wRatio; - } else { - op.inSampleSize = hRatio; - } - } - op.inJustDecodeBounds = false; - bmp = BitmapFactory.decodeFile(path, op); - // 从原Bitmap创建一个给定宽高的Bitmap - return Bitmap.createScaledBitmap(bmp, displayWidth, displayHeight, true); - } - @Override protected void onResume() { super.onResume(); @@ -306,6 +258,7 @@ protected void onDestroy() { } public void initView() { + TYPE = BTN_SHUTTER; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); tvStatus = (TextView) findViewById(R.id.tv_status); @@ -321,8 +274,8 @@ public void initView() { backInPreview.setOnClickListener(this); albumSelectButton = findViewById(R.id.albumSelect); albumSelectButton.setOnClickListener(this); - mCameraPageView = findViewById(R.id.camera_page); - mResultPageView = findViewById(R.id.result_page); + cameraPageView = findViewById(R.id.camera_page); + resultPageView = findViewById(R.id.result_page); resultImage = findViewById(R.id.result_image); backInResult = findViewById(R.id.back_in_result); backInResult.setOnClickListener(this); @@ -334,7 +287,7 @@ public void initView() { results.add(new BaseResultModel(1, "cup", 0.4f)); results.add(new BaseResultModel(2, "pen", 0.6f)); results.add(new BaseResultModel(3, "tang", 1.0f)); - final DetectResultAdapter adapter = new DetectResultAdapter(this, R.layout.detection_result_page_item, results); + final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.detection_result_page_item, results); detectResultView.setAdapter(adapter); detectResultView.invalidate(); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionSettingsActivity.java index e31c228cf08..21984fdcc85 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionSettingsActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionSettingsActivity.java @@ -59,8 +59,8 @@ public void onCreate(Bundle savedInstanceState) { preInstalledCPUPowerModes = new ArrayList(); preInstalledScoreThresholds = new ArrayList(); preInstalledEnableLiteFp16s = new ArrayList(); - preInstalledModelDirs.add(getString(R.string.MODEL_DIR_DEFAULT)); - preInstalledLabelPaths.add(getString(R.string.LABEL_PATH_DEFAULT)); + preInstalledModelDirs.add(getString(R.string.DETECTION_MODEL_DIR_DEFAULT)); + preInstalledLabelPaths.add(getString(R.string.DETECTION_LABEL_PATH_DEFAULT)); preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); preInstalledScoreThresholds.add(getString(R.string.SCORE_THRESHOLD_DEFAULT)); @@ -90,7 +90,7 @@ private void reloadSettingsAndUpdateUI() { SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); String selected_model_dir = sharedPreferences.getString(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY), - getString(R.string.MODEL_DIR_DEFAULT)); + getString(R.string.DETECTION_MODEL_DIR_DEFAULT)); int selected_model_idx = lpChoosePreInstalledModel.findIndexOfValue(selected_model_dir); if (selected_model_idx >= 0 && selected_model_idx < preInstalledModelDirs.size() && selected_model_idx != selectedModelIdx) { SharedPreferences.Editor editor = sharedPreferences.edit(); @@ -106,9 +106,9 @@ private void reloadSettingsAndUpdateUI() { } String model_dir = sharedPreferences.getString(getString(R.string.MODEL_DIR_KEY), - getString(R.string.MODEL_DIR_DEFAULT)); + getString(R.string.DETECTION_MODEL_DIR_DEFAULT)); String label_path = sharedPreferences.getString(getString(R.string.LABEL_PATH_KEY), - getString(R.string.LABEL_PATH_DEFAULT)); + getString(R.string.DETECTION_LABEL_PATH_DEFAULT)); String cpu_thread_num = sharedPreferences.getString(getString(R.string.CPU_THREAD_NUM_KEY), getString(R.string.CPU_THREAD_NUM_DEFAULT)); String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), @@ -136,12 +136,12 @@ static boolean checkAndUpdateSettings(Context ctx) { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(ctx); String model_dir = sharedPreferences.getString(ctx.getString(R.string.MODEL_DIR_KEY), - ctx.getString(R.string.MODEL_DIR_DEFAULT)); + ctx.getString(R.string.DETECTION_MODEL_DIR_DEFAULT)); settingsChanged |= !modelDir.equalsIgnoreCase(model_dir); modelDir = model_dir; String label_path = sharedPreferences.getString(ctx.getString(R.string.LABEL_PATH_KEY), - ctx.getString(R.string.LABEL_PATH_DEFAULT)); + ctx.getString(R.string.DETECTION_LABEL_PATH_DEFAULT)); settingsChanged |= !labelPath.equalsIgnoreCase(label_path); labelPath = label_path; diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java index ef990237fd3..427da7f2456 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java @@ -2,9 +2,14 @@ import android.content.Context; import android.content.res.Resources; +import android.database.Cursor; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; import android.hardware.Camera; +import android.net.Uri; import android.opengl.GLES20; import android.os.Environment; +import android.provider.MediaStore; import android.util.Log; import android.view.Surface; import android.view.WindowManager; @@ -241,4 +246,44 @@ public static boolean isSupportedNPU() { String hardware = android.os.Build.HARDWARE; return hardware.equalsIgnoreCase("kirin810") || hardware.equalsIgnoreCase("kirin990"); } + + public static Bitmap decodeBitmap(String path, int displayWidth, int displayHeight) { + BitmapFactory.Options op = new BitmapFactory.Options(); + op.inJustDecodeBounds = true;// Only the width and height information of Bitmap is read, not the pixels. + Bitmap bmp = BitmapFactory.decodeFile(path, op); // Get size information. + int wRatio = (int) Math.ceil(op.outWidth / (float) displayWidth);// Get Scale Size. + int hRatio = (int) Math.ceil(op.outHeight / (float) displayHeight); + // If the specified size is exceeded, reduce the corresponding scale. + if (wRatio > 1 && hRatio > 1) { + if (wRatio > hRatio) { + // If it is too wide, we will reduce the width to the required size. Note that the height will become smaller. + op.inSampleSize = wRatio; + } else { + op.inSampleSize = hRatio; + } + } + op.inJustDecodeBounds = false; + bmp = BitmapFactory.decodeFile(path, op); + // Create a Bitmap with a given width and height from the original Bitmap. + return Bitmap.createScaledBitmap(bmp, displayWidth, displayHeight, true); + } + + public static String getRealPathFromURI(Context context, Uri contentURI) { + String result; + Cursor cursor = null; + try { + cursor = context.getContentResolver().query(contentURI, null, null, null, null); + } catch (Throwable e) { + e.printStackTrace(); + } + if (cursor == null) { + result = contentURI.getPath(); + } else { + cursor.moveToFirst(); + int idx = cursor.getColumnIndex(MediaStore.Images.ImageColumns.DATA); + result = cursor.getString(idx); + cursor.close(); + } + return result; + } } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/DetectResultAdapter.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/BaseResultAdapter.java similarity index 85% rename from java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/DetectResultAdapter.java rename to java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/BaseResultAdapter.java index 404b9cbc5e6..62747965adc 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/DetectResultAdapter.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/adapter/BaseResultAdapter.java @@ -15,14 +15,14 @@ import java.text.DecimalFormat; import java.util.List; -public class DetectResultAdapter extends ArrayAdapter { +public class BaseResultAdapter extends ArrayAdapter { private int resourceId; - public DetectResultAdapter(@NonNull Context context, int resource) { + public BaseResultAdapter(@NonNull Context context, int resource) { super(context, resource); } - public DetectResultAdapter(@NonNull Context context, int resource, @NonNull List objects) { + public BaseResultAdapter(@NonNull Context context, int resource, @NonNull List objects) { super(context, resource, objects); resourceId = resource; } diff --git a/java/android/app/src/main/res/layout/classification_activity_main.xml b/java/android/app/src/main/res/layout/classification_activity_main.xml new file mode 100644 index 00000000000..8c6179530ac --- /dev/null +++ b/java/android/app/src/main/res/layout/classification_activity_main.xml @@ -0,0 +1,14 @@ + + + + + + + diff --git a/java/android/app/src/main/res/layout/classification_camera_page.xml b/java/android/app/src/main/res/layout/classification_camera_page.xml new file mode 100644 index 00000000000..15b009e2449 --- /dev/null +++ b/java/android/app/src/main/res/layout/classification_camera_page.xml @@ -0,0 +1,161 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/classification_result_page.xml b/java/android/app/src/main/res/layout/classification_result_page.xml new file mode 100644 index 00000000000..958a8594014 --- /dev/null +++ b/java/android/app/src/main/res/layout/classification_result_page.xml @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/classification_result_page_item.xml b/java/android/app/src/main/res/layout/classification_result_page_item.xml new file mode 100644 index 00000000000..6a2b09ebff1 --- /dev/null +++ b/java/android/app/src/main/res/layout/classification_result_page_item.xml @@ -0,0 +1,26 @@ + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/detection_camera_page.xml b/java/android/app/src/main/res/layout/detection_camera_page.xml index c4f79d2c91c..238a817bc0f 100644 --- a/java/android/app/src/main/res/layout/detection_camera_page.xml +++ b/java/android/app/src/main/res/layout/detection_camera_page.xml @@ -42,18 +42,17 @@ android:layout_height="wrap_content" android:text="@string/action_bar_take_photo" android:textAlignment="center" - android:visibility="gone"/> + android:visibility="gone" /> - EasyEdge EasyEdge + EasyEdge CHOOSE_INSTALLED_MODEL_KEY MODEL_DIR_KEY @@ -16,17 +17,21 @@ 2 LITE_POWER_HIGH 0.4 + 0.1 true - - models/picodet_s_320_coco_lcnet - labels/coco_label_list.txt + + models/picodet_s_320_coco_lcnet + labels/coco_label_list.txt models labels/ppocr_keys_v1.txt + + models/MobileNetV1_x0_25_infer + labels/imagenet1k_label_list.txt 拍照识别 - EasyEdge 实时识别 + 实时识别 < 模型名称 识别结果 diff --git a/java/android/app/src/main/res/xml/classification_settings.xml b/java/android/app/src/main/res/xml/classification_settings.xml index 624ed13aec2..c43c1cdd5c2 100644 --- a/java/android/app/src/main/res/xml/classification_settings.xml +++ b/java/android/app/src/main/res/xml/classification_settings.xml @@ -1,4 +1,45 @@ - + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/xml/default_settings.xml b/java/android/app/src/main/res/xml/default_settings.xml deleted file mode 100644 index 26329068b69..00000000000 --- a/java/android/app/src/main/res/xml/default_settings.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - diff --git a/java/android/app/src/main/res/xml/detection_settings.xml b/java/android/app/src/main/res/xml/detection_settings.xml index af63bb8a591..df9801aec26 100644 --- a/java/android/app/src/main/res/xml/detection_settings.xml +++ b/java/android/app/src/main/res/xml/detection_settings.xml @@ -1,17 +1,17 @@ Date: Tue, 15 Nov 2022 13:19:32 +0800 Subject: [PATCH 033/136] [Bug Fix] fix import download module error in python3.6 (#591) fix import download module error in python3.6 --- python/fastdeploy/utils/example_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/fastdeploy/utils/example_resource.py b/python/fastdeploy/utils/example_resource.py index 4bfc4f2d821..682a07f2b7b 100644 --- a/python/fastdeploy/utils/example_resource.py +++ b/python/fastdeploy/utils/example_resource.py @@ -11,14 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import fastdeploy.download as download +import fastdeploy as fd import fastdeploy.utils.hub_env as hubenv def get_detection_test_image(path=None): if path is None: path = hubenv.RESOURCE_HOME - fullpath = download.download( + fullpath = fd.download( url='https://bj.bcebos.com/paddlehub/fastdeploy/example/detection_test_image.jpg', path=path) return fullpath From 8601272c6be5b6f19f2d5b174155ce4b6f120f3e Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Tue, 15 Nov 2022 17:06:16 +0800 Subject: [PATCH 034/136] [Bug Fix] Fixed MattingResult bug (#593) * add onnx_ort_runtime demo * rm in requirements * support batch eval * fixed MattingResults bug --- fastdeploy/vision/common/result.cc | 4 ++-- tests/models/test_rvm.py | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index 3585713a67d..4d746797f48 100755 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -372,7 +372,7 @@ void MattingResult::Reserve(int size) { if (contain_foreground) { FDASSERT((shape.size() == 3), "Please initial shape (h,w,c) before call Reserve."); - int c = static_cast(shape[3]); + int c = static_cast(shape[2]); foreground.reserve(size * c); } } @@ -382,7 +382,7 @@ void MattingResult::Resize(int size) { if (contain_foreground) { FDASSERT((shape.size() == 3), "Please initial shape (h,w,c) before call Resize."); - int c = static_cast(shape[3]); + int c = static_cast(shape[2]); foreground.resize(size * c); } } diff --git a/tests/models/test_rvm.py b/tests/models/test_rvm.py index 10d680948f4..4fa3083e59a 100644 --- a/tests/models/test_rvm.py +++ b/tests/models/test_rvm.py @@ -19,6 +19,7 @@ import numpy as np import runtime_config as rc + def test_matting_rvm_cpu(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/rvm.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/video.mp4" @@ -38,7 +39,8 @@ def test_matting_rvm_cpu(): break result = model.predict(frame) # compare diff - expect_alpha = np.load("resources/rvm/result_alpha_" + str(frame_id) + ".npy") + expect_alpha = np.load("resources/rvm/result_alpha_" + str(frame_id) + + ".npy") result_alpha = np.array(result.alpha).reshape(1920, 1080) diff = np.fabs(expect_alpha - result_alpha) thres = 1e-05 @@ -51,3 +53,7 @@ def test_matting_rvm_cpu(): cap.release() cv2.destroyAllWindows() break + + +if __name__ == "__main__": + test_matting_rvm_cpu() From 62453f24fedfd88ba8a2798a6398ef3c9c1ecd22 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Tue, 15 Nov 2022 18:59:42 +0800 Subject: [PATCH 035/136] [Android] Run button shutter in sub Ui Thread (#598) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android * [Android] Run button shutter in sub Ui Thread * [Android] Update CameraSurfaceView --- .../detection/DetectionMainActivity.java | 90 ++++++++++++++----- .../app/ui/view/CameraSurfaceView.java | 22 ++++- .../app/src/main/res/xml/ocr_settings.xml | 2 +- 3 files changed, 87 insertions(+), 27 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 2501206a011..1c0ec7f1c7d 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -47,6 +47,8 @@ import java.util.ArrayList; import java.util.List; +import javax.microedition.khronos.opengles.GL10; + public class DetectionMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { private static final String TAG = DetectionMainActivity.class.getSimpleName(); @@ -75,10 +77,12 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; public static final int ALBUM_SELECT = 1; - private static int TYPE = TYPE_UNKNOWN; + public static final int REALTIME_DETECT = 2; + private static int TYPE = REALTIME_DETECT; private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; private static final int INTENT_CODE_PICK_IMAGE = 100; + private static final int TIME_SLEEP_INTERVAL = 50; // ms String savedImagePath = "result.jpg"; int lastFrameIndex = 0; @@ -118,12 +122,7 @@ public void onClick(View v) { break; case R.id.btn_shutter: TYPE = BTN_SHUTTER; - svPreview.onPause(); - cameraPageView.setVisibility(View.GONE); - resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultNum + ""); - confidenceSeekbar.setProgress((int) (resultNum * 100)); - resultImage.setImageBitmap(shutterBitmap); + runOnShutterUiThread(); break; case R.id.btn_settings: startActivity(new Intent(DetectionMainActivity.this, DetectionSettingsActivity.class)); @@ -149,9 +148,49 @@ public void onClick(View v) { case R.id.back_in_result: resultPageView.setVisibility(View.GONE); cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; svPreview.onResume(); break; + } + } + + private void runOnShutterUiThread() { + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + try { + Thread.sleep(TIME_SLEEP_INTERVAL); + + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + resultImage.setImageBitmap(shutterBitmap); + } else { + new AlertDialog.Builder(DetectionMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + } + private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { + if (ARGB8888ImageBitmap == null) { + return; + } + if (!ARGB8888ImageBitmap.isRecycled()) { + synchronized (this) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } } } @@ -194,18 +233,17 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + synchronized (this) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; + } + } + String savedImagePath = ""; synchronized (this) { savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; } - if (TYPE == BTN_SHUTTER) { - shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - } else { - // Only reference in predict loops. - shutterBitmap = ARGB8888ImageBitmap; - originShutterBitmap = ARGB8888ImageBitmap; - } boolean modified = false; DetectionResult result = predictor.predict( @@ -258,7 +296,7 @@ protected void onDestroy() { } public void initView() { - TYPE = BTN_SHUTTER; + TYPE = REALTIME_DETECT; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); tvStatus = (TextView) findViewById(R.id.tv_status); @@ -313,16 +351,20 @@ public void onStopTrackingTouch(SeekBar seekBar) { @Override public void run() { if (TYPE == ALBUM_SELECT) { - SystemClock.sleep(500); - predictor.predict(picBitmap, savedImagePath, resultNum); - resultImage.setImageBitmap(picBitmap); - picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); + if (!picBitmap.isRecycled()) { + predictor.predict(picBitmap, true, resultNum); + resultImage.setImageBitmap(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); + } resultNum = 1.0f; } else { - SystemClock.sleep(500); - predictor.predict(shutterBitmap, savedImagePath, resultNum); - resultImage.setImageBitmap(shutterBitmap); - shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); + if (!shutterBitmap.isRecycled()) { + predictor.predict(shutterBitmap, true, resultNum); + resultImage.setImageBitmap(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); + } resultNum = 1.0f; } } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java index 06248fede42..5a315d20fab 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java @@ -46,6 +46,9 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer, protected int textureWidth = 0; protected int textureHeight = 0; + protected Bitmap ARGB8888ImageBitmap; + protected boolean bitmapReleaseMode = true; + // In order to manipulate the camera preview data and render the modified one // to the screen, three textures are created and the data flow is shown as following: // previewdata->camTextureId->fboTexureId->drawTexureId->framebuffer @@ -198,9 +201,12 @@ public void onDrawFrame(GL10 gl) { // Read pixels of FBO to a bitmap ByteBuffer pixelBuffer = ByteBuffer.allocate(textureWidth * textureHeight * 4); GLES20.glReadPixels(0, 0, textureWidth, textureHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuffer); - Bitmap ARGB8888ImageBitmap = Bitmap.createBitmap(textureWidth, textureHeight, Bitmap.Config.ARGB_8888); + + ARGB8888ImageBitmap = Bitmap.createBitmap(textureWidth, textureHeight, Bitmap.Config.ARGB_8888); ARGB8888ImageBitmap.copyPixelsFromBuffer(pixelBuffer); + boolean modified = onTextureChangedListener.onTextureChanged(ARGB8888ImageBitmap); + if (modified) { targetTexureId = drawTexureId[0]; // Update a bitmap to the GL texture if modified @@ -209,7 +215,9 @@ public void onDrawFrame(GL10 gl) { GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, targetTexureId); GLUtils.texImage2D(GL10.GL_TEXTURE_2D, 0, ARGB8888ImageBitmap, 0); } - ARGB8888ImageBitmap.recycle(); + if (bitmapReleaseMode) { + ARGB8888ImageBitmap.recycle(); + } } // fboTexureId/drawTexureId->Screen @@ -229,6 +237,16 @@ public void onDrawFrame(GL10 gl) { GLES20.glFlush(); } + public void setBitmapReleaseMode(boolean mode) { + synchronized (this) { + bitmapReleaseMode = mode; + } + } + + public Bitmap getBitmap() { + return ARGB8888ImageBitmap; // may null or recycled. + } + private float[] transformTextureCoordinates(float[] coords, float[] matrix) { float[] result = new float[coords.length]; float[] vt = new float[4]; diff --git a/java/android/app/src/main/res/xml/ocr_settings.xml b/java/android/app/src/main/res/xml/ocr_settings.xml index 59c77e5a7f8..692b74b4cd2 100644 --- a/java/android/app/src/main/res/xml/ocr_settings.xml +++ b/java/android/app/src/main/res/xml/ocr_settings.xml @@ -1,7 +1,7 @@ Date: Wed, 16 Nov 2022 10:11:43 +0800 Subject: [PATCH 036/136] [Serving]Update images version to 0.6.0 (#594) * serving images update to 0.6 * serving images update to 0.6 * serving images update to 0.6 --- .../classification/paddleclas/serving/README.md | 6 +++--- examples/vision/detection/yolov5/serving/README.md | 6 +++--- .../vision/detection/yolov5/serving/README_EN.md | 4 ++-- serving/Dockerfile | 2 +- serving/Dockerfile_cpu | 2 +- serving/README_CN.md | 14 +++++++++++--- serving/README_EN.md | 13 +++++++++++-- serving/docs/EN/compile-en.md | 6 +++--- serving/docs/zh_CN/compile.md | 6 +++--- serving/scripts/build.sh | 7 ++++--- 10 files changed, 42 insertions(+), 24 deletions(-) diff --git a/examples/vision/classification/paddleclas/serving/README.md b/examples/vision/classification/paddleclas/serving/README.md index 606770e17ca..59e5d2c0538 100644 --- a/examples/vision/classification/paddleclas/serving/README.md +++ b/examples/vision/classification/paddleclas/serving/README.md @@ -21,12 +21,12 @@ mv ResNet50_vd_infer/inference.pdiparams models/runtime/1/model.pdiparams # 拉取fastdeploy镜像 # GPU镜像 -docker pull paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 # CPU镜像 -docker pull paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-cpu-only-21.10 # 运行容器.容器名字为 fd_serving, 并挂载当前目录为容器的 /serving 目录 -nvidia-docker run -it --net=host --name fd_serving -v `pwd`/:/serving paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 bash +nvidia-docker run -it --net=host --name fd_serving -v `pwd`/:/serving paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 bash # 启动服务(不设置CUDA_VISIBLE_DEVICES环境变量，会拥有所有GPU卡的调度权限) CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=/serving/models --backend-config=python,shm-default-byte-size=10485760 diff --git a/examples/vision/detection/yolov5/serving/README.md b/examples/vision/detection/yolov5/serving/README.md index 521f9c41969..4f95df44118 100644 --- a/examples/vision/detection/yolov5/serving/README.md +++ b/examples/vision/detection/yolov5/serving/README.md @@ -14,13 +14,13 @@ wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s.onnx mv yolov5s.onnx models/runtime/1/model.onnx # 拉取fastdeploy镜像 -docker pull paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 # 运行容器.容器名字为 fd_serving, 并挂载当前目录为容器的 /yolov5_serving 目录 -nvidia-docker run -it --net=host --name fd_serving -v `pwd`/:/yolov5_serving paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 bash +nvidia-docker run -it --net=host --name fd_serving -v `pwd`/:/yolov5_serving paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 bash # 启动服务(不设置CUDA_VISIBLE_DEVICES环境变量，会拥有所有GPU卡的调度权限) -CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=models --backend-config=python,shm-default-byte-size=10485760 +CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=/yolov5_serving/models --backend-config=python,shm-default-byte-size=10485760 ``` >> **注意**: 当出现"Address already in use", 请使用`--grpc-port`指定端口号来启动服务，同时更改yolov5_grpc_client.py中的请求端口号 diff --git a/examples/vision/detection/yolov5/serving/README_EN.md b/examples/vision/detection/yolov5/serving/README_EN.md index db110efabd8..cc85355d89f 100644 --- a/examples/vision/detection/yolov5/serving/README_EN.md +++ b/examples/vision/detection/yolov5/serving/README_EN.md @@ -10,10 +10,10 @@ wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov5s.onnx mv yolov5s.onnx models/infer/1/ # Pull fastdeploy image -docker pull paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 # Run the docker. The docker name is fd_serving, and the current directory is mounted as the docker's /yolov5_serving directory -nvidia-docker run -it --net=host --name fd_serving -v `pwd`/:/yolov5_serving paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 bash +nvidia-docker run -it --net=host --name fd_serving -v `pwd`/:/yolov5_serving paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 bash # Start the service (Without setting the CUDA_VISIBLE_DEVICES environment variable, it will have scheduling privileges for all GPU cards) CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=models --backend-config=python,shm-default-byte-size=10485760 diff --git a/serving/Dockerfile b/serving/Dockerfile index eb27ca4eb6b..cca94ebe931 100644 --- a/serving/Dockerfile +++ b/serving/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update \ && apt-get update && apt-get install -y --no-install-recommends datacenter-gpu-manager=1:2.2.9 RUN apt-get update \ - && apt-get install -y --no-install-recommends libre2-5 libb64-0d python3 python3-pip libarchive-dev \ + && apt-get install -y --no-install-recommends libre2-5 libb64-0d python3 python3-pip libarchive-dev ffmpeg libsm6 libxext6 \ && python3 -m pip install -U pip \ && python3 -m pip install paddlepaddle-gpu paddlenlp faster_tokenizer diff --git a/serving/Dockerfile_cpu b/serving/Dockerfile_cpu index e671f03a170..9e6d8b0ef47 100644 --- a/serving/Dockerfile_cpu +++ b/serving/Dockerfile_cpu @@ -17,7 +17,7 @@ FROM paddlepaddle/fastdeploy:22.09-cpu-only-min ENV TZ=Asia/Shanghai \ DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libgomp1 \ +RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libgomp1 ffmpeg libsm6 libxext6 \ && python3 -m pip install -U pip \ && python3 -m pip install paddlepaddle paddlenlp faster_tokenizer diff --git a/serving/README_CN.md b/serving/README_CN.md index b26b4599fbc..377698873a2 100644 --- a/serving/README_CN.md +++ b/serving/README_CN.md @@ -17,13 +17,13 @@ FastDeploy基于[Triton Inference Server](https://github.com/triton-inference-se #### CPU镜像 CPU镜像仅支持Paddle/ONNX模型在CPU上进行服务化部署，支持的推理后端包括OpenVINO、Paddle Inference和ONNX Runtime ``` shell -docker pull paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-cpu-only-21.10 ``` #### GPU镜像 GPU镜像支持Paddle/ONNX模型在GPU/CPU上进行服务化部署，支持的推理后端包括OpenVINO、TensorRT、Paddle Inference和ONNX Runtime ``` -docker pull paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 ``` 用户也可根据自身需求，参考如下文档自行编译镜像 @@ -33,4 +33,12 @@ docker pull paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 - [服务化模型目录说明](docs/zh_CN/model_repository.md) (说明如何准备模型目录) - [服务化部署配置说明](docs/zh_CN/model_configuration.md) (说明runtime的配置选项) - [服务化部署示例](docs/zh_CN/demo.md) - - [YOLOV5 检测任务](../examples/vision/detection/yolov5/serving/README.md) + +### 模型示例列表 + +| 任务场景 | 模型 | +|---|---| +| Classification | [PaddleClas](../examples/vision/classification/paddleclas/serving/README.md) | +| Detection | [ultralytics/YOLOv5](../examples/vision/detection/yolov5/serving/README.md) | +| NLP | [PaddleNLP/ERNIE-3.0](../examples/text/ernie-3.0/serving/README.md)| +| Speech | [PaddleSpeech/PP-TTS](../examples/audio/pp-tts/serving/README.md)| diff --git a/serving/README_EN.md b/serving/README_EN.md index b7bb028f5fa..30bc405c0f7 100644 --- a/serving/README_EN.md +++ b/serving/README_EN.md @@ -20,7 +20,7 @@ FastDeploy builds an end-to-end serving deployment based on [Triton Inference Se CPU images only support Paddle/ONNX models for serving deployment on CPUs, and supported inference backends include OpenVINO, Paddle Inference, and ONNX Runtime ```shell -docker pull paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-cpu-only-21.10 ``` #### GPU Image @@ -28,7 +28,7 @@ docker pull paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 GPU images support Paddle/ONNX models for serving deployment on GPU and CPU, and supported inference backends including OpenVINO, TensorRT, Paddle Inference, and ONNX Runtime ``` -docker pull paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 +docker pull paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 ``` Users can also compile the image by themselves according to their own needs, referring to the following documents: @@ -41,3 +41,12 @@ Users can also compile the image by themselves according to their own needs, ref - [Serving Deployment Configuration for Runtime](docs/zh_CN/model_configuration.md) - [Serving Deployment Demo](docs/zh_CN/demo.md) - [YOLOV5 - Detection Task](../examples/vision/detection/yolov5/serving/README.md) + +### Model List + +| Task | Model | +|---|---| +| Classification | [PaddleClas](../examples/vision/classification/paddleclas/serving/README.md) | +| Detection | [ultralytics/YOLOv5](../examples/vision/detection/yolov5/serving/README.md) | +| NLP | [PaddleNLP/ERNIE-3.0](../examples/text/ernie-3.0/serving/README.md)| +| Speech | [PaddleSpeech/PP-TTS](../examples/audio/pp-tts/serving/README.md)| diff --git a/serving/docs/EN/compile-en.md b/serving/docs/EN/compile-en.md index fd72d767346..32476c19a87 100644 --- a/serving/docs/EN/compile-en.md +++ b/serving/docs/EN/compile-en.md @@ -13,7 +13,7 @@ bash scripts/build.sh # Exit to the FastDeploy home directory and create the image cd ../ -docker build -t paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 -f serving/Dockerfile . +docker build -t paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 -f serving/Dockerfile . ``` ## CPU Image @@ -26,7 +26,7 @@ bash scripts/build.sh OFF # Exit to the FastDeploy home directory and create the image cd ../ -docker build -t paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 -f serving/Dockerfile_cpu . +docker build -t paddlepaddle/fastdeploy:0.6.0-cpu-only-21.10 -f serving/Dockerfile_cpu . ``` ## IPU Image @@ -38,5 +38,5 @@ bash scripts/build_fd_ipu.sh # Exit to the FastDeploy home directory and create the image cd ../ -docker build -t paddlepaddle/fastdeploy:0.3.0-ipu-only-21.10 -f serving/Dockerfile_ipu . +docker build -t paddlepaddle/fastdeploy:0.6.0-ipu-only-21.10 -f serving/Dockerfile_ipu . ``` diff --git a/serving/docs/zh_CN/compile.md b/serving/docs/zh_CN/compile.md index 21a9c2ab433..ebbb25da44a 100644 --- a/serving/docs/zh_CN/compile.md +++ b/serving/docs/zh_CN/compile.md @@ -13,7 +13,7 @@ bash scripts/build.sh # 退出到FastDeploy主目录，制作镜像 cd ../ -docker build -t paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 -f serving/Dockerfile . +docker build -t paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 -f serving/Dockerfile . ``` ## 制作CPU镜像 @@ -25,7 +25,7 @@ bash scripts/build.sh OFF # 退出到FastDeploy主目录，制作镜像 cd ../ -docker build -t paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 -f serving/Dockerfile_cpu . +docker build -t paddlepaddle/fastdeploy:0.6.0-cpu-only-21.10 -f serving/Dockerfile_cpu . ``` ## 制作IPU镜像 @@ -37,5 +37,5 @@ bash scripts/build_fd_ipu.sh # 退出到FastDeploy主目录，制作镜像 cd ../ -docker build -t paddlepaddle/fastdeploy:0.3.0-ipu-only-21.10 -f serving/Dockerfile_ipu . +docker build -t paddlepaddle/fastdeploy:0.6.0-ipu-only-21.10 -f serving/Dockerfile_ipu . ``` diff --git a/serving/scripts/build.sh b/serving/scripts/build.sh index e09af35d802..465e3f03af5 100644 --- a/serving/scripts/build.sh +++ b/serving/scripts/build.sh @@ -30,7 +30,7 @@ if [ ! -d "./TensorRT-8.4.1.5/" ]; then rm -rf TensorRT-8.4.1.5.Linux.x86_64-gnu.cuda-11.6.cudnn8.4.tar.gz fi -nvidia-docker run -it --rm --name build_fd_vison \ +nvidia-docker run -it --rm --name build_fd \ -v`pwd`/..:/workspace/fastdeploy \ nvcr.io/nvidia/tritonserver:21.10-py3-min \ bash -c \ @@ -50,7 +50,8 @@ nvidia-docker run -it --rm --name build_fd_vison \ export ENABLE_TEXT=ON; python setup.py build; python setup.py bdist_wheel; - cd ../;rm -rf build; mkdir -p build;cd build; + cd /workspace/fastdeploy; + rm -rf build; mkdir -p build;cd build; cmake .. -DENABLE_TRT_BACKEND=ON -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy_install -DWITH_GPU=ON -DTRT_DIRECTORY=/workspace/fastdeploy/serving/TensorRT-8.4.1.5/ -DENABLE_PADDLE_BACKEND=ON -DENABLE_ORT_BACKEND=ON -DENABLE_OPENVINO_BACKEND=ON -DENABLE_VISION=OFF -DBUILD_FASTDEPLOY_PYTHON=OFF -DENABLE_PADDLE_FRONTEND=ON -DENABLE_TEXT=OFF -DLIBRARY_NAME=fastdeploy_runtime; make -j`nproc`; make install; @@ -65,7 +66,7 @@ else echo "start build FD CPU library" -docker run -it --rm --name build_fd_vison \ +docker run -it --rm --name build_fd \ -v`pwd`/..:/workspace/fastdeploy \ paddlepaddle/fastdeploy:21.10-cpu-only-buildbase \ bash -c \ From 26ab27a4d6cdd2e688dafabe9bd18ee2538292b9 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Wed, 16 Nov 2022 11:11:02 +0800 Subject: [PATCH 037/136] [Doc] Update Android SDK usage docs (#605) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android * [Android] Run button shutter in sub Ui Thread * [Android] Update CameraSurfaceView * [Android] Update Android SDK usage docs --- java/android/README.md | 73 ++++++++++++++++--- .../detection/DetectionMainActivity.java | 2 +- .../app/ui/view/CameraSurfaceView.java | 1 - .../vision/detection/detection_utils_jni.cc | 4 +- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/java/android/README.md b/java/android/README.md index faa88749de3..aae25b68008 100644 --- a/java/android/README.md +++ b/java/android/README.md @@ -4,7 +4,22 @@ FastDeploy Android SDK 目前支持图像分类、目标检测、OCR文字识别 - 调用`predict`接口 - 可视化验证（可选） -## 图像分类 +## 目录 + +- [图像分类API](#Classification) +- [目标检测API](#Detection) +- [语义分割API](#Segmentation) +- [OCR文字识别API](#OCR) +- [人脸检测API](#FaceDetection) +- [识别结果说明](#VisionResults) +- [RuntimeOption说明](#RuntimeOption) +- [可视化接口API](#Visualize) +- [模型使用示例](#Demo) + +## 图像分类 + +
+ ### PaddleClasModel Java API 说明 - 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleClasModel初始化参数说明如下： - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel @@ -13,6 +28,7 @@ FastDeploy Android SDK 目前支持图像分类、目标检测、OCR文字识别 - labelFile: String, 可选参数，表示label标签文件所在路径，用于可视化，如 imagenet1k_label_list.txt，每一行包含一个label - option: RuntimeOption，可选参数，模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + ```java // 构造函数: constructor w/o label file public PaddleClasModel(); // 空构造函数，之后可以调用init初始化 @@ -39,6 +55,9 @@ public boolean initialized(); // 检查是否初始化成功 ``` ## 目标检测 + +
+ ### PicoDet Java API 说明 - 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PicoDet初始化参数说明如下： - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel @@ -73,6 +92,9 @@ public boolean initialized(); // 检查是否初始化成功 ``` ## OCR文字识别 + +
+ ### PP-OCRv2 & PP-OCRv3 Java API 说明 - 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。 PP-OCR初始化参数说明如下： - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel @@ -112,6 +134,9 @@ public boolean initialized(); // 检查是否初始化成功 ``` ## 语义分割 + +
+ ### PaddleSegModel Java API 说明 - 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleSegModel初始化参数说明如下： - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel @@ -142,6 +167,9 @@ public boolean initialized(); // 检查是否初始化成功 ``` ## 人脸检测 + +
+ ### SCRFD Java API 说明 - 模型初始化 API: 模型初始化API包含两种方式，方式一是通过构造函数直接初始化；方式二是，通过调用init函数，在合适的程序节点进行初始化。PaddleSegModel初始化参数说明如下： - modelFile: String, paddle格式的模型文件路径，如 model.pdmodel @@ -198,7 +226,10 @@ public boolean release(); // 释放native资源 public boolean initialized(); // 检查是否初始化成功 ``` -## 结果说明 +## 识别结果说明 + +
+ - 图像分类ClassifyResult说明 ```java public class ClassifyResult { @@ -207,7 +238,7 @@ public class ClassifyResult { public boolean initialized(); // 检测结果是否有效 } ``` -其他参考：C++/Python对应的ClassifyResult说明 [api/vision_results/classification_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/classification_result.md) +其他参考：C++/Python对应的ClassifyResult说明: [api/vision_results/classification_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/classification_result.md) - 目标检测DetectionResult说明 ```java @@ -218,7 +249,7 @@ public class DetectionResult { public boolean initialized(); // 检测结果是否有效 } ``` -其他参考：C++/Python对应的DetectionResult说明 [api/vision_results/detection_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/detection_result.md) +其他参考：C++/Python对应的DetectionResult说明: [api/vision_results/detection_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/detection_result.md) - OCR文字识别OCRResult说明 ```java @@ -231,7 +262,7 @@ public class OCRResult { public boolean initialized(); // 检测结果是否有效 } ``` -其他参考：C++/Python对应的OCRResult说明 [api/vision_results/ocr_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/ocr_result.md) +其他参考：C++/Python对应的OCRResult说明: [api/vision_results/ocr_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/ocr_result.md) - 语义分割SegmentationResult结果说明 ```java @@ -243,7 +274,7 @@ public class SegmentationResult { public boolean initialized(); // 检测结果是否有效 } ``` -其他参考：C++/Python对应的SegmentationResult说明 [api/vision_results/segmentation_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/segmentation_result.md) +其他参考：C++/Python对应的SegmentationResult说明: [api/vision_results/segmentation_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/segmentation_result.md) - 人脸检测FaceDetectionResult结果说明 ```java @@ -255,9 +286,12 @@ public class FaceDetectionResult { public boolean initialized(); // 检测结果是否有效 } ``` -其他参考：C++/Python对应的FaceDetectionResult说明 [api/vision_results/face_detection_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/face_detection_result.md) +其他参考：C++/Python对应的FaceDetectionResult说明: [api/vision_results/face_detection_result.md](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/api/vision_results/face_detection_result.md) + +## RuntimeOption说明 + +
-## RuntimeOption说明 - RuntimeOption设置说明 ```java public class RuntimeOption { @@ -270,15 +304,33 @@ public class RuntimeOption { ``` ## 可视化接口 + +
+ FastDeploy Android SDK同时提供一些可视化接口，可用于快速验证推理结果。以下接口均把结果result渲染在输入的Bitmap上。具体的可视化API接口如下： ```java public class Visualize { + // 默认参数接口 public static boolean visClassification(Bitmap ARGB8888Bitmap, ClassifyResult result); public static boolean visDetection(Bitmap ARGB8888Bitmap, DetectionResult result); public static boolean visFaceDetection(Bitmap ARGB8888Bitmap, FaceDetectionResult result); public static boolean visOcr(Bitmap ARGB8888Bitmap, OCRResult result); public static boolean visSegmentation(Bitmap ARGB8888Bitmap, SegmentationResult result); + // 有可设置参数的可视化接口 + // visDetection: 可设置阈值（大于该阈值的框进行绘制）、框线大小、字体大小、类别labels等 + public static boolean visDetection(Bitmap ARGB8888Bitmap, DetectionResult result, float scoreThreshold); + public static boolean visDetection(Bitmap ARGB8888Bitmap, DetectionResult result, float scoreThreshold, int lineSize, float fontSize); + public static boolean visDetection(Bitmap ARGB8888Bitmap, DetectionResult result, String[] labels); + public static boolean visDetection(Bitmap ARGB8888Bitmap, DetectionResult result, String[] labels, float scoreThreshold, int lineSize, float fontSize); + // visClassification: 可设置阈值（大于该阈值的框进行绘制）、字体大小、类别labels等 + public static boolean visClassification(Bitmap ARGB8888Bitmap, ClassifyResult result, float scoreThreshold,float fontSize); + public static boolean visClassification(Bitmap ARGB8888Bitmap, ClassifyResult result, String[] labels); + public static boolean visClassification(Bitmap ARGB8888Bitmap, ClassifyResult result, String[] labels, float scoreThreshold,float fontSize); + // visSegmentation: weight背景权重 + public static boolean visSegmentation(Bitmap ARGB8888Bitmap, SegmentationResult result, float weight); + // visFaceDetection: 线大小、字体大小等 + public static boolean visFaceDetection(Bitmap ARGB8888Bitmap, FaceDetectionResult result, int lineSize, float fontSize); } ``` 对应的可视化类型为： @@ -286,7 +338,10 @@ public class Visualize { import com.baidu.paddle.fastdeploy.vision.Visualize; ``` -## 使用示例 +## 模型使用示例 + +
+ - 模型调用示例1：使用构造函数以及默认的RuntimeOption ```java import java.nio.ByteBuffer; diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 1c0ec7f1c7d..0b2ca905c7a 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -159,7 +159,7 @@ private void runOnShutterUiThread() { @SuppressLint("SetTextI18n") public void run() { try { - Thread.sleep(TIME_SLEEP_INTERVAL); + Thread.sleep(TIME_SLEEP_INTERVAL * 2); svPreview.onPause(); cameraPageView.setVisibility(View.GONE); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java index 5a315d20fab..b4456e8a4db 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java @@ -33,7 +33,6 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer, public static final int EXPECTED_PREVIEW_WIDTH = 1280; public static final int EXPECTED_PREVIEW_HEIGHT = 720; - protected int numberOfCameras; protected int selectedCameraId; protected boolean disableCamera = false; diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/detection/detection_utils_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/detection/detection_utils_jni.cc index 25e24c3108a..e0248c5866c 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/detection/detection_utils_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/detection/detection_utils_jni.cc @@ -30,10 +30,10 @@ void RenderingDetection(JNIEnv *env, const cv::Mat &c_bgr, if (AssetsLoader::IsDetectionLabelsLoaded()) { c_vis_im = vision::VisDetection(c_bgr, c_result, AssetsLoader::GetDetectionLabels(), - score_threshold, 2, 1.0f); + score_threshold, 2, 0.5f); } else { c_vis_im = - vision::VisDetection(c_bgr, c_result, score_threshold, 2, 1.0f); + vision::VisDetection(c_bgr, c_result, score_threshold, 2, 0.5f); } LOGD("Visualize from native costs %f ms", GetElapsedTime(t)); From 48f32aa26c5afca4a12fe5fb35a25100531b329a Mon Sep 17 00:00:00 2001 From: chenjian Date: Wed, 16 Nov 2022 14:04:24 +0800 Subject: [PATCH 038/136] [Bug Fix] fix import error in python3.6 (#608) fix import error in python3.6 --- python/fastdeploy/utils/example_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/fastdeploy/utils/example_resource.py b/python/fastdeploy/utils/example_resource.py index 682a07f2b7b..d6dd68697b6 100644 --- a/python/fastdeploy/utils/example_resource.py +++ b/python/fastdeploy/utils/example_resource.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. import fastdeploy as fd -import fastdeploy.utils.hub_env as hubenv +from . import hub_env def get_detection_test_image(path=None): if path is None: - path = hubenv.RESOURCE_HOME + path = hub_env.RESOURCE_HOME fullpath = fd.download( url='https://bj.bcebos.com/paddlehub/fastdeploy/example/detection_test_image.jpg', path=path) From 752f1369fbe234be40517c406e8debed7dfc609e Mon Sep 17 00:00:00 2001 From: ziqi-jin <67993288+ziqi-jin@users.noreply.github.com> Date: Wed, 16 Nov 2022 14:15:10 +0800 Subject: [PATCH 039/136] [Model] Add PIPNet and FaceLandmark1000 Support (#548) * first commit for yolov7 * pybind for yolov7 * CPP README.md * CPP README.md * modified yolov7.cc * README.md * python file modify * delete license in fastdeploy/ * repush the conflict part * README.md modified * README.md modified * file path modified * file path modified * file path modified * file path modified * file path modified * README modified * README modified * move some helpers to private * add examples for yolov7 * api.md modified * api.md modified * api.md modified * YOLOv7 * yolov7 release link * yolov7 release link * yolov7 release link * copyright * change some helpers to private * change variables to const and fix documents. * gitignore * Transfer some funtions to private member of class * Transfer some funtions to private member of class * Merge from develop (#9) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * first commit for yolor * for merge * Develop (#11) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Yolor (#16) * Develop (#11) (#12) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Develop (#13) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * Develop (#14) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> * add is_dynamic for YOLO series (#22) * modify ppmatting backend and docs * modify ppmatting docs * fix the PPMatting size problem * fix LimitShort's log * retrigger ci * modify PPMatting docs * modify the way for dealing with LimitShort * add python comments for external models * modify resnet c++ comments * modify C++ comments for external models * modify python comments and add result class comments * fix comments compile error * modify result.h comments * c++ version for FaceLandmark1000 * add pipnet land1000 sigle test and python code * fix facelandmark1000 sigle test * fix python examples for PIPNet and FaceLandmark1000 * fix examples links for PIPNet and FaceLandmark1000 * modify test_vision_colorspace_convert.cc * modify facealign readme * retrigger ci * modify README * test ci * fix download_prebuilt_libraries.md * fix download_prebuilt_libraries.md * modify for comments * modify supported_num_landmarks * retrigger ci * check code style * check code style Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> --- .../download_prebuilt_libraries.md | 4 +- examples/vision/facealign/README.md | 2 + .../facealign/face_landmark_1000/README.md | 25 + .../face_landmark_1000/cpp/CMakeLists.txt | 18 + .../face_landmark_1000/cpp/README.md | 84 +++ .../facealign/face_landmark_1000/cpp/infer.cc | 110 +++ .../face_landmark_1000/python/README.md | 71 ++ .../face_landmark_1000/python/infer.py | 90 +++ examples/vision/facealign/pipnet/README.md | 33 + .../facealign/pipnet/cpp/CMakeLists.txt | 18 + .../vision/facealign/pipnet/cpp/README.md | 84 +++ examples/vision/facealign/pipnet/cpp/infer.cc | 111 +++ .../vision/facealign/pipnet/python/README.md | 71 ++ .../vision/facealign/pipnet/python/infer.py | 93 +++ fastdeploy/vision.h | 2 + .../common/processors/color_space_convert.cc | 69 +- .../common/processors/color_space_convert.h | 36 +- fastdeploy/vision/common/result.cc | 5 +- .../facealign/contrib/face_landmark_1000.cc | 134 ++++ .../facealign/contrib/face_landmark_1000.h | 75 ++ .../contrib/face_landmark_1000_pybind.cc | 34 + fastdeploy/vision/facealign/contrib/pipnet.cc | 687 ++++++++++++++++++ fastdeploy/vision/facealign/contrib/pipnet.h | 127 ++++ .../vision/facealign/contrib/pipnet_pybind.cc | 39 + .../vision/facealign/facealign_pybind.cc | 5 + .../fastdeploy/vision/facealign/__init__.py | 2 + .../facealign/contrib/face_landmark_1000.py | 68 ++ .../vision/facealign/contrib/pipnet.py | 107 +++ tests/eval_example/test_facelandmark1000.py | 43 ++ tests/eval_example/test_pipnet.py | 43 ++ .../test_vision_colorspace_convert.cc | 50 ++ 31 files changed, 2325 insertions(+), 15 deletions(-) create mode 100644 examples/vision/facealign/face_landmark_1000/README.md create mode 100644 examples/vision/facealign/face_landmark_1000/cpp/CMakeLists.txt create mode 100644 examples/vision/facealign/face_landmark_1000/cpp/README.md create mode 100644 examples/vision/facealign/face_landmark_1000/cpp/infer.cc create mode 100644 examples/vision/facealign/face_landmark_1000/python/README.md create mode 100644 examples/vision/facealign/face_landmark_1000/python/infer.py create mode 100644 examples/vision/facealign/pipnet/README.md create mode 100644 examples/vision/facealign/pipnet/cpp/CMakeLists.txt create mode 100644 examples/vision/facealign/pipnet/cpp/README.md create mode 100644 examples/vision/facealign/pipnet/cpp/infer.cc create mode 100644 examples/vision/facealign/pipnet/python/README.md create mode 100644 examples/vision/facealign/pipnet/python/infer.py create mode 100644 fastdeploy/vision/facealign/contrib/face_landmark_1000.cc create mode 100644 fastdeploy/vision/facealign/contrib/face_landmark_1000.h create mode 100644 fastdeploy/vision/facealign/contrib/face_landmark_1000_pybind.cc create mode 100644 fastdeploy/vision/facealign/contrib/pipnet.cc create mode 100644 fastdeploy/vision/facealign/contrib/pipnet.h create mode 100644 fastdeploy/vision/facealign/contrib/pipnet_pybind.cc create mode 100644 python/fastdeploy/vision/facealign/contrib/face_landmark_1000.py create mode 100644 python/fastdeploy/vision/facealign/contrib/pipnet.py create mode 100644 tests/eval_example/test_facelandmark1000.py create mode 100644 tests/eval_example/test_pipnet.py diff --git a/docs/en/build_and_install/download_prebuilt_libraries.md b/docs/en/build_and_install/download_prebuilt_libraries.md index 9bf7122ce2d..58edd1d56eb 100755 --- a/docs/en/build_and_install/download_prebuilt_libraries.md +++ b/docs/en/build_and_install/download_prebuilt_libraries.md @@ -3,8 +3,8 @@ FastDeploy provides pre-built libraries for developers to download and install directly. Meanwhile, FastDeploy also offers easy access to compile so that developers can compile FastDeploy according to their own needs. This article is divided into two parts: -- [1.GPU Deployment Environment](##GPU Deployment Environment) -- [2.CPU Deployment Environment](##CPU Deployment Environment) +- [1.GPU Deployment Environment](#gpu-deployment-environment) +- [2.CPU Deployment Environment](#cpu-deployment-environment) ## GPU Deployment Environment diff --git a/examples/vision/facealign/README.md b/examples/vision/facealign/README.md index d145c7c8f9c..df0b441112a 100644 --- a/examples/vision/facealign/README.md +++ b/examples/vision/facealign/README.md @@ -5,3 +5,5 @@ FastDeploy目前支持如下人脸对齐(关键点检测）模型部署 | 模型 | 说明 | 模型格式 | 版本 | | :--- | :--- | :------- | :--- | | [Hsintao/pfld_106_face_landmarks](./pfld) | PFLD 系列模型 | ONNX | [CommitID:e150195](https://github.com/Hsintao/pfld_106_face_landmarks/commit/e150195) | +| [Single430/FaceLandmark1000](./face_landmark_1000) | FaceLandmark1000 系列模型 | ONNX | [CommitID:1a951b6](https://github.com/Single430/FaceLandmark1000/tree/1a951b6) | +| [jhb86253817/PIPNet](./pipnet) | PIPNet 系列模型 | ONNX | [CommitID:b9eab58](https://github.com/jhb86253817/PIPNet/tree/b9eab58) | diff --git a/examples/vision/facealign/face_landmark_1000/README.md b/examples/vision/facealign/face_landmark_1000/README.md new file mode 100644 index 00000000000..ce45cc285b5 --- /dev/null +++ b/examples/vision/facealign/face_landmark_1000/README.md @@ -0,0 +1,25 @@ +# FaceLandmark 模型部署 + +## 模型版本说明 + +- [FaceLandmark1000](https://github.com/Single430/FaceLandmark1000/tree/1a951b6) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [FaceLandmark1000 模型](https://github.com/Single430/FaceLandmark1000) + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了FaceLandmark导出的各系列模型，开发者可直接下载使用。 + +| 模型 | 参数大小 | 精度 | 备注 | +|:---------------------------------------------------------------- |:----- |:----- | :------ | +| [FaceLandmark1000](https://bj.bcebos.com/paddlehub/fastdeploy/FaceLandmark1000.onnx) | 2.1M | - | + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/facealign/face_landmark_1000/cpp/CMakeLists.txt b/examples/vision/facealign/face_landmark_1000/cpp/CMakeLists.txt new file mode 100644 index 00000000000..c417fcb3880 --- /dev/null +++ b/examples/vision/facealign/face_landmark_1000/cpp/CMakeLists.txt @@ -0,0 +1,18 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") +include(${FASTDEPLOY_INSTALL_DIR}/utils/gflags.cmake) +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +if(UNIX AND (NOT APPLE) AND (NOT ANDROID)) + target_link_libraries(infer_demo ${FASTDEPLOY_LIBS} gflags pthread) +else() + target_link_libraries(infer_demo ${FASTDEPLOY_LIBS} gflags) +endif() diff --git a/examples/vision/facealign/face_landmark_1000/cpp/README.md b/examples/vision/facealign/face_landmark_1000/cpp/README.md new file mode 100644 index 00000000000..033f54164d1 --- /dev/null +++ b/examples/vision/facealign/face_landmark_1000/cpp/README.md @@ -0,0 +1,84 @@ +# FaceLandmark1000 C++部署示例 + +本目录下提供`infer.cc`快速完成FaceLandmark1000在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上CPU推理为例，在本目录执行如下命令即可完成编译测试，保证 FastDeploy 版本0.7.0以上(x.x.x >= 0.7.0)支持FaceLandmark1000模型 + +```bash +mkdir build +cd build +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-x.x.x.tgz +tar xvf fastdeploy-linux-x64-x.x.x.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-x.x.x +make -j + +#下载官方转换好的 FaceLandmark1000 模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/FaceLandmark1000.onnx +wget https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png + +# CPU推理 +./infer_demo --model FaceLandmark1000.onnx --image facealign_input.png --device cpu +# GPU推理 +./infer_demo --model FaceLandmark1000.onnx --image facealign_input.png --device gpu +# GPU上TensorRT推理 +./infer_demo --model FaceLandmark1000.onnx --image facealign_input.png --device gpu --backend trt +``` + +运行完成可视化结果如下图所示 + +
+ +
+ +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## FaceLandmark1000 C++接口 + +### FaceLandmark1000 类 + +```c++ +fastdeploy::vision::facealign::FaceLandmark1000( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::ONNX) +``` + +FaceLandmark1000模型加载和初始化，其中model_file为导出的ONNX模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径，当模型格式为ONNX时，此参数传入空字符串即可 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为ONNX格式 + +#### Predict函数 + +> ```c++ +> FaceLandmark1000::Predict(cv::Mat* im, FaceAlignmentResult* result) +> ``` +> +> 模型预测接口，输入图像直接输出landmarks结果。 +> +> **参数** +> +> > * **im**: 输入图像，注意需为HWC，BGR格式 +> > * **result**: landmarks结果, FaceAlignmentResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员变量 + +用户可按照自己的实际需求，修改下列预处理参数，从而影响最终的推理和部署效果 + +> > * **size**(vector<int>): 通过此参数修改预处理过程中resize的大小，包含两个整型元素，表示[width, height], 默认值为[128, 128] + +- [模型介绍](../../) +- [Python部署](../python) +- [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/facealign/face_landmark_1000/cpp/infer.cc b/examples/vision/facealign/face_landmark_1000/cpp/infer.cc new file mode 100644 index 00000000000..702d9609a80 --- /dev/null +++ b/examples/vision/facealign/face_landmark_1000/cpp/infer.cc @@ -0,0 +1,110 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" +#include "gflags/gflags.h" + +DEFINE_string(model, "", "Directory of the inference model."); +DEFINE_string(image, "", "Path of the image file."); +DEFINE_string(device, "cpu", + "Type of inference device, support 'cpu' or 'gpu'."); +DEFINE_string(backend, "default", + "The inference runtime backend, support: ['default', 'ort', " + "'paddle', 'ov', 'trt', 'paddle_trt']"); +DEFINE_bool(use_fp16, false, "Whether to use FP16 mode, only support 'trt' and 'paddle_trt' backend"); + +void PrintUsage() { + std::cout << "Usage: infer_demo --model model_path --image img_path --device [cpu|gpu] --backend " + "[default|ort|paddle|ov|trt|paddle_trt] " + "--use_fp16 false" + << std::endl; + std::cout << "Default value of device: cpu" << std::endl; + std::cout << "Default value of backend: default" << std::endl; + std::cout << "Default value of use_fp16: false" << std::endl; +} + +bool CreateRuntimeOption(fastdeploy::RuntimeOption* option) { + if (FLAGS_device == "gpu") { + option->UseGpu(); + if (FLAGS_backend == "ort") { + option->UseOrtBackend(); + } else if (FLAGS_backend == "paddle") { + option->UsePaddleBackend(); + } else if (FLAGS_backend == "trt" || + FLAGS_backend == "paddle_trt") { + option->UseTrtBackend(); + option->SetTrtInputShape("input", {1, 3, 128, 128}); + if (FLAGS_backend == "paddle_trt") { + option->EnablePaddleToTrt(); + } + if (FLAGS_use_fp16) { + option->EnableTrtFP16(); + } + } else if (FLAGS_backend == "default") { + return true; + } else { + std::cout << "While inference with GPU, only support default/ort/paddle/trt/paddle_trt now, " << FLAGS_backend << " is not supported." << std::endl; + return false; + } + } else if (FLAGS_device == "cpu") { + if (FLAGS_backend == "ort") { + option->UseOrtBackend(); + } else if (FLAGS_backend == "ov") { + option->UseOpenVINOBackend(); + } else if (FLAGS_backend == "paddle") { + option->UsePaddleBackend(); + } else if (FLAGS_backend == "default") { + return true; + } else { + std::cout << "While inference with CPU, only support default/ort/ov/paddle now, " << FLAGS_backend << " is not supported." << std::endl; + return false; + } + } else { + std::cerr << "Only support device CPU/GPU now, " << FLAGS_device << " is not supported." << std::endl; + return false; + } + + return true; +} + +int main(int argc, char* argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + auto option = fastdeploy::RuntimeOption(); + if (!CreateRuntimeOption(&option)) { + PrintUsage(); + return -1; + } + + auto model = fastdeploy::vision::facealign::FaceLandmark1000(FLAGS_model, "", option); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return -1; + } + + auto im = cv::imread(FLAGS_image); + auto im_bak = im.clone(); + + fastdeploy::vision::FaceAlignmentResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return -1; + } + std::cout << res.Str() << std::endl; + + auto vis_im = fastdeploy::vision::VisFaceAlignment(im_bak, res); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + + return 0; +} diff --git a/examples/vision/facealign/face_landmark_1000/python/README.md b/examples/vision/facealign/face_landmark_1000/python/README.md new file mode 100644 index 00000000000..380d1cb2705 --- /dev/null +++ b/examples/vision/facealign/face_landmark_1000/python/README.md @@ -0,0 +1,71 @@ +# FaceLandmark1000 Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成FaceLandmark1000在CPU/GPU，以及GPU上通过TensorRT加速部署的示例，保证 FastDeploy 版本 >= 0.7.0 支持FaceLandmark1000模型。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/facealign/facelandmark1000/python + +# 下载FaceLandmark1000模型文件和测试图片 +## 原版ONNX模型 +wget https://bj.bcebos.com/paddlehub/fastdeploy/FaceLandmark1000.onnx +wget https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png + +# CPU推理 +python infer.py --model FaceLandmark1000.onnx --image facealign_input.png --device cpu +# GPU推理 +python infer.py --model FaceLandmark1000.onnx --image facealign_input.png --device gpu +# TRT推理 +python infer.py --model FaceLandmark1000.onnx --image facealign_input.png --device gpu --backend trt +``` + +运行完成可视化结果如下图所示 + +
+ +
+ +## FaceLandmark1000 Python接口 + +```python +fd.vision.facealign.FaceLandmark1000(model_file, params_file=None, runtime_option=None, model_format=ModelFormat.ONNX) +``` + +FaceLandmark1000模型加载和初始化，其中model_file为导出的ONNX模型格式 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径，当模型格式为ONNX格式时，此参数无需设定 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为ONNX + +### predict函数 + +> ```python +> FaceLandmark1000.predict(input_image) +> ``` +> +> 模型预测结口，输入图像直接输出landmarks坐标结果。 +> +> **参数** +> +> > * **input_image**(np.ndarray): 输入数据，注意需为HWC，BGR格式 + +> **返回** +> +> > 返回`fastdeploy.vision.FaceAlignmentResult`结构体，结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/) + + +## 其它文档 + +- [FaceLandmark1000 模型介绍](..) +- [FaceLandmark1000 C++部署](../cpp) +- [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/facealign/face_landmark_1000/python/infer.py b/examples/vision/facealign/face_landmark_1000/python/infer.py new file mode 100644 index 00000000000..d6f52760c20 --- /dev/null +++ b/examples/vision/facealign/face_landmark_1000/python/infer.py @@ -0,0 +1,90 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", required=True, help="Path of FaceLandmark1000 model.") + parser.add_argument("--image", type=str, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--backend", + type=str, + default="ort", + help="inference backend, ort, ov, trt, paddle, paddle_trt.") + parser.add_argument( + "--enable_trt_fp16", + type=ast.literal_eval, + default=False, + help="whether enable fp16 in trt/paddle_trt backend") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + device = args.device + backend = args.backend + enable_trt_fp16 = args.enable_trt_fp16 + if device == "gpu": + option.use_gpu() + if backend == "ort": + option.use_ort_backend() + elif backend == "paddle": + option.use_paddle_backend() + elif backend in ["trt", "paddle_trt"]: + option.use_trt_backend() + option.set_trt_input_shape("input", [1, 3, 112, 112]) + if backend == "paddle_trt": + option.enable_paddle_to_trt() + if enable_trt_fp16: + option.enable_trt_fp16() + elif backend == "default": + return option + else: + raise Exception( + "While inference with GPU, only support default/ort/paddle/trt/paddle_trt now, {} is not supported.". + format(backend)) + elif device == "cpu": + if backend == "ort": + option.use_ort_backend() + elif backend == "ov": + option.use_openvino_backend() + elif backend == "paddle": + option.use_paddle_backend() + elif backend == "default": + return option + else: + raise Exception( + "While inference with CPU, only support default/ort/ov/paddle now, {} is not supported.". + format(backend)) + else: + raise Exception( + "Only support device CPU/GPU now, {} is not supported.".format( + device)) + + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.facealign.FaceLandmark1000( + args.model, runtime_option=runtime_option) + +# for image +im = cv2.imread(args.image) +result = model.predict(im.copy()) +print(result) +# 可视化结果 +vis_im = fd.vision.vis_face_alignment(im, result) +cv2.imwrite("visualized_result.jpg", vis_im) +print("Visualized result save in ./visualized_result.jpg") diff --git a/examples/vision/facealign/pipnet/README.md b/examples/vision/facealign/pipnet/README.md new file mode 100644 index 00000000000..504aac2241c --- /dev/null +++ b/examples/vision/facealign/pipnet/README.md @@ -0,0 +1,33 @@ +# PIPNet 模型部署 + +## 模型版本说明 + +- [PIPNet](https://github.com/jhb86253817/PIPNet/tree/b9eab58) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [PIPNet 模型](https://github.com/jhb86253817/PIPNet) + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了PIPNet导出的各系列模型，开发者可直接下载使用。 + +| 模型 | 参数大小 | 精度 | 备注 | +|:---------------------------------------------------------------- |:----- |:----- | :------ | +| [PIPNet19_ResNet18_AFLW](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x19x32x256_aflw.onnx) | 45.6M | - | +| [PIPNet29_ResNet18_COFW](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x29x32x256_cofw.onnx) | 46.1M | - | +| [PIPNet68_ResNet18_300W](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x68x32x256_300w.onnx) | 47.9M | - | +| [PIPNet98_ResNet18_WFLW](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x98x32x256_wflw.onnx) | 49.3M | - | +| [PIPNet19_ResNet101_AFLW](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet101_10x19x32x256_aflw.onnx) | 173.4M | - | +| [PIPNet29_ResNet101_COFW](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet101_10x29x32x256_cofw.onnx) | 175.3M | - | +| [PIPNet68_ResNet101_300W](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet101_10x68x32x256_300w.onnx) | 182.6M | - | +| [PIPNet98_ResNet101_WFLW](https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet101_10x98x32x256_wflw.onnx) | 188.3M | - | + + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/facealign/pipnet/cpp/CMakeLists.txt b/examples/vision/facealign/pipnet/cpp/CMakeLists.txt new file mode 100644 index 00000000000..c417fcb3880 --- /dev/null +++ b/examples/vision/facealign/pipnet/cpp/CMakeLists.txt @@ -0,0 +1,18 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") +include(${FASTDEPLOY_INSTALL_DIR}/utils/gflags.cmake) +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +if(UNIX AND (NOT APPLE) AND (NOT ANDROID)) + target_link_libraries(infer_demo ${FASTDEPLOY_LIBS} gflags pthread) +else() + target_link_libraries(infer_demo ${FASTDEPLOY_LIBS} gflags) +endif() diff --git a/examples/vision/facealign/pipnet/cpp/README.md b/examples/vision/facealign/pipnet/cpp/README.md new file mode 100644 index 00000000000..3f11bf62b4f --- /dev/null +++ b/examples/vision/facealign/pipnet/cpp/README.md @@ -0,0 +1,84 @@ +# PIPNet C++部署示例 + +本目录下提供`infer.cc`快速完成PIPNet在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上CPU推理为例，在本目录执行如下命令即可完成编译测试，保证 FastDeploy 版本0.7.0以上(x.x.x >= 0.7.0)支持PIPNet模型 + +```bash +mkdir build +cd build +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-x.x.x.tgz +tar xvf fastdeploy-linux-x64-x.x.x.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-x.x.x +make -j + +#下载官方转换好的 PIPNet 模型文件和测试图片 +wget https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x19x32x256_aflw.onnx +wget https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png + +# CPU推理 +./infer_demo --model pipnet_resnet18_10x19x32x256_aflw.onnx --image facealign_input.png --device cpu +# GPU推理 +./infer_demo --model pipnet_resnet18_10x19x32x256_aflw.onnx --image facealign_input.png --device gpu +# GPU上TensorRT推理 +./infer_demo --model pipnet_resnet18_10x19x32x256_aflw.onnx --image facealign_input.png --device gpu --backend trt +``` + +运行完成可视化结果如下图所示 + +
+ +
+ +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## PIPNet C++接口 + +### PIPNet 类 + +```c++ +fastdeploy::vision::facealign::PIPNet( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::ONNX) +``` + +PIPNet模型加载和初始化，其中model_file为导出的ONNX模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径，当模型格式为ONNX时，此参数传入空字符串即可 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为ONNX格式 + +#### Predict函数 + +> ```c++ +> PIPNet::Predict(cv::Mat* im, FaceAlignmentResult* result) +> ``` +> +> 模型预测接口，输入图像直接输出landmarks结果。 +> +> **参数** +> +> > * **im**: 输入图像，注意需为HWC，BGR格式 +> > * **result**: landmarks结果, FaceAlignmentResult说明参考[视觉模型预测结果](../../../../../docs/api/vision_results/) + +### 类成员变量 + +用户可按照自己的实际需求，修改下列预处理参数，从而影响最终的推理和部署效果 + +> > * **size**(vector<int>): 通过此参数修改预处理过程中resize的大小，包含两个整型元素，表示[width, height], 默认值为[256, 256] + +- [模型介绍](../../) +- [Python部署](../python) +- [视觉模型预测结果](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/facealign/pipnet/cpp/infer.cc b/examples/vision/facealign/pipnet/cpp/infer.cc new file mode 100644 index 00000000000..cba9667d92e --- /dev/null +++ b/examples/vision/facealign/pipnet/cpp/infer.cc @@ -0,0 +1,111 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" +#include "gflags/gflags.h" + +DEFINE_string(model, "", "Directory of the inference model."); +DEFINE_string(image, "", "Path of the image file."); +DEFINE_string(device, "cpu", + "Type of inference device, support 'cpu' or 'gpu'."); +DEFINE_string(backend, "default", + "The inference runtime backend, support: ['default', 'ort', " + "'paddle', 'ov', 'trt', 'paddle_trt']"); +DEFINE_bool(use_fp16, false, "Whether to use FP16 mode, only support 'trt' and 'paddle_trt' backend"); +DEFINE_int32(num_landmarks, 19, "number of landmarks for PIPNet, the num_lms should be a number in {19, 29, 68, 98}"); + +void PrintUsage() { + std::cout << "Usage: infer_demo --model model_path --image img_path --device [cpu|gpu] --backend " + "[default|ort|paddle|ov|trt|paddle_trt] " + "--use_fp16 false" + << std::endl; + std::cout << "Default value of device: cpu" << std::endl; + std::cout << "Default value of backend: default" << std::endl; + std::cout << "Default value of use_fp16: false" << std::endl; +} + +bool CreateRuntimeOption(fastdeploy::RuntimeOption* option) { + if (FLAGS_device == "gpu") { + option->UseGpu(); + if (FLAGS_backend == "ort") { + option->UseOrtBackend(); + } else if (FLAGS_backend == "paddle") { + option->UsePaddleBackend(); + } else if (FLAGS_backend == "trt" || + FLAGS_backend == "paddle_trt") { + option->UseTrtBackend(); + option->SetTrtInputShape("input", {1, 3, 128, 128}); + if (FLAGS_backend == "paddle_trt") { + option->EnablePaddleToTrt(); + } + if (FLAGS_use_fp16) { + option->EnableTrtFP16(); + } + } else if (FLAGS_backend == "default") { + return true; + } else { + std::cout << "While inference with GPU, only support default/ort/paddle/trt/paddle_trt now, " << FLAGS_backend << " is not supported." << std::endl; + return false; + } + } else if (FLAGS_device == "cpu") { + if (FLAGS_backend == "ort") { + option->UseOrtBackend(); + } else if (FLAGS_backend == "ov") { + option->UseOpenVINOBackend(); + } else if (FLAGS_backend == "paddle") { + option->UsePaddleBackend(); + } else if (FLAGS_backend == "default") { + return true; + } else { + std::cout << "While inference with CPU, only support default/ort/ov/paddle now, " << FLAGS_backend << " is not supported." << std::endl; + return false; + } + } else { + std::cerr << "Only support device CPU/GPU now, " << FLAGS_device << " is not supported." << std::endl; + return false; + } + + return true; +} + +int main(int argc, char* argv[]) { + google::ParseCommandLineFlags(&argc, &argv, true); + auto option = fastdeploy::RuntimeOption(); + if (!CreateRuntimeOption(&option)) { + PrintUsage(); + return -1; + } + + auto model = fastdeploy::vision::facealign::PIPNet(FLAGS_model, "", option); + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return -1; + } + model.SetNumLandmarks(FLAGS_num_landmarks); + auto im = cv::imread(FLAGS_image); + auto im_bak = im.clone(); + + fastdeploy::vision::FaceAlignmentResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return -1; + } + std::cout << res.Str() << std::endl; + + auto vis_im = fastdeploy::vision::VisFaceAlignment(im_bak, res); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + + return 0; +} diff --git a/examples/vision/facealign/pipnet/python/README.md b/examples/vision/facealign/pipnet/python/README.md new file mode 100644 index 00000000000..ae46849c38e --- /dev/null +++ b/examples/vision/facealign/pipnet/python/README.md @@ -0,0 +1,71 @@ +# PIPNet Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成PIPNet在CPU/GPU，以及GPU上通过TensorRT加速部署的示例，保证 FastDeploy 版本 >= 0.7.0 支持PIPNet模型。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/facealign/pipnet/python + +# 下载PIPNet模型文件和测试图片以及视频 +## 原版ONNX模型 +wget https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x19x32x256_aflw.onnx +wget https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png + +# CPU推理 +python infer.py --model pipnet_resnet18_10x19x32x256_aflw.onnx --image facealign_input.png --device cpu +# GPU推理 +python infer.py --model pipnet_resnet18_10x19x32x256_aflw.onnx --image facealign_input.png --device gpu +# TRT推理 +python infer.py --model pipnet_resnet18_10x19x32x256_aflw.onnx --image facealign_input.png --device gpu --backend trt +``` + +运行完成可视化结果如下图所示 + +
+ +
+ +## PIPNet Python接口 + +```python +fd.vision.facealign.PIPNet(model_file, params_file=None, runtime_option=None, model_format=ModelFormat.ONNX) +``` + +PIPNet模型加载和初始化，其中model_file为导出的ONNX模型格式 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径，当模型格式为ONNX格式时，此参数无需设定 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为ONNX + +### predict函数 + +> ```python +> PIPNet.predict(input_image) +> ``` +> +> 模型预测结口，输入图像直接输出landmarks坐标结果。 +> +> **参数** +> +> > * **input_image**(np.ndarray): 输入数据，注意需为HWC，BGR格式 + +> **返回** +> +> > 返回`fastdeploy.vision.FaceAlignmentResult`结构体，结构体说明参考文档[视觉模型预测结果](../../../../../docs/api/vision_results/) + + +## 其它文档 + +- [PIPNet 模型介绍](..) +- [PIPNet C++部署](../cpp) +- [模型预测结果说明](../../../../../docs/api/vision_results/) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/facealign/pipnet/python/infer.py b/examples/vision/facealign/pipnet/python/infer.py new file mode 100644 index 00000000000..fa34d38444f --- /dev/null +++ b/examples/vision/facealign/pipnet/python/infer.py @@ -0,0 +1,93 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of PIPNet model.") + parser.add_argument("--image", type=str, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--backend", + type=str, + default="ort", + help="inference backend, ort, ov, trt, paddle, paddle_trt.") + parser.add_argument( + "--enable_trt_fp16", + type=ast.literal_eval, + default=False, + help="whether enable fp16 in trt/paddle_trt backend") + parser.add_argument( + "--num_landmarks", + type=int, + default=19, + help="whether enable fp16 in trt/paddle_trt backend") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + device = args.device + backend = args.backend + enable_trt_fp16 = args.enable_trt_fp16 + if device == "gpu": + option.use_gpu() + if backend == "ort": + option.use_ort_backend() + elif backend == "paddle": + option.use_paddle_backend() + elif backend in ["trt", "paddle_trt"]: + option.use_trt_backend() + option.set_trt_input_shape("input", [1, 3, 112, 112]) + if backend == "paddle_trt": + option.enable_paddle_to_trt() + if enable_trt_fp16: + option.enable_trt_fp16() + elif backend == "default": + return option + else: + raise Exception( + "While inference with GPU, only support default/ort/paddle/trt/paddle_trt now, {} is not supported.". + format(backend)) + elif device == "cpu": + if backend == "ort": + option.use_ort_backend() + elif backend == "ov": + option.use_openvino_backend() + elif backend == "paddle": + option.use_paddle_backend() + elif backend == "default": + return option + else: + raise Exception( + "While inference with CPU, only support default/ort/ov/paddle now, {} is not supported.". + format(backend)) + else: + raise Exception( + "Only support device CPU/GPU now, {} is not supported.".format( + device)) + + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model = fd.vision.facealign.PIPNet(args.model, runtime_option=runtime_option) +model.num_landmarks = args.num_landmarks +# for image +im = cv2.imread(args.image) +result = model.predict(im.copy()) +print(result) +# 可视化结果 +vis_im = fd.vision.vis_face_alignment(im, result) +cv2.imwrite("visualized_result.jpg", vis_im) +print("Visualized result save in ./visualized_result.jpg") diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index 2f8c7066131..c9719cc9ea7 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -34,6 +34,8 @@ #include "fastdeploy/vision/facedet/contrib/ultraface.h" #include "fastdeploy/vision/facedet/contrib/yolov5face.h" #include "fastdeploy/vision/facealign/contrib/pfld.h" +#include "fastdeploy/vision/facealign/contrib/face_landmark_1000.h" +#include "fastdeploy/vision/facealign/contrib/pipnet.h" #include "fastdeploy/vision/faceid/contrib/adaface.h" #include "fastdeploy/vision/faceid/contrib/arcface.h" #include "fastdeploy/vision/faceid/contrib/cosface.h" diff --git a/fastdeploy/vision/common/processors/color_space_convert.cc b/fastdeploy/vision/common/processors/color_space_convert.cc index 4ccfb651008..c486c537940 100644 --- a/fastdeploy/vision/common/processors/color_space_convert.cc +++ b/fastdeploy/vision/common/processors/color_space_convert.cc @@ -16,7 +16,7 @@ namespace fastdeploy { namespace vision { -bool BGR2RGB::ImplByOpenCV(Mat* mat) { +bool BGR2RGB::ImplByOpenCV(FDMat* mat) { cv::Mat* im = mat->GetOpenCVMat(); cv::Mat new_im; cv::cvtColor(*im, new_im, cv::COLOR_BGR2RGB); @@ -25,7 +25,7 @@ bool BGR2RGB::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool BGR2RGB::ImplByFlyCV(Mat* mat) { +bool BGR2RGB::ImplByFlyCV(FDMat* mat) { fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { FDERROR << "[BGR2RGB] The channel of input image must be 3, but not it's " @@ -39,7 +39,7 @@ bool BGR2RGB::ImplByFlyCV(Mat* mat) { } #endif -bool RGB2BGR::ImplByOpenCV(Mat* mat) { +bool RGB2BGR::ImplByOpenCV(FDMat* mat) { cv::Mat* im = mat->GetOpenCVMat(); cv::Mat new_im; cv::cvtColor(*im, new_im, cv::COLOR_RGB2BGR); @@ -48,7 +48,7 @@ bool RGB2BGR::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool RGB2BGR::ImplByFlyCV(Mat* mat) { +bool RGB2BGR::ImplByFlyCV(FDMat* mat) { fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { FDERROR << "[RGB2BGR] The channel of input image must be 3, but not it's " @@ -62,15 +62,72 @@ bool RGB2BGR::ImplByFlyCV(Mat* mat) { } #endif -bool BGR2RGB::Run(Mat* mat, ProcLib lib) { +bool BGR2GRAY::ImplByOpenCV(FDMat* mat) { + cv::Mat* im = mat->GetOpenCVMat(); + cv::Mat new_im; + cv::cvtColor(*im, new_im, cv::COLOR_BGR2GRAY); + mat->SetMat(new_im); + mat->SetChannels(1); + return true; +} + +#ifdef ENABLE_FLYCV +bool BGR2GRAY::ImplByFalconCV(FDMat* mat) { + fcv::Mat* im = mat->GetFalconCVMat(); + if (im->channels() != 3) { + FDERROR << "[BGR2GRAY] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl; + return false; + } + fcv::Mat new_im; + fcv::cvt_color(*im, new_im, fcv::ColorConvertType::CVT_PA_BGR2GRAY); + mat->SetMat(new_im); + return true; +} +#endif + +bool RGB2GRAY::ImplByOpenCV(FDMat* mat) { + cv::Mat* im = mat->GetOpenCVMat(); + cv::Mat new_im; + cv::cvtColor(*im, new_im, cv::COLOR_RGB2GRAY); + mat->SetMat(new_im); + return true; +} + +#ifdef ENABLE_FLYCV +bool RGB2GRAY::ImplByFalconCV(FDMat* mat) { + fcv::Mat* im = mat->GetFalconCVMat(); + if (im->channels() != 3) { + FDERROR << "[RGB2GRAY] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl; + return false; + } + fcv::Mat new_im; + fcv::cvt_color(*im, new_im, fcv::ColorConvertType::CVT_PA_RGB2GRAY); + mat->SetMat(new_im); + return true; +} +#endif + + +bool BGR2RGB::Run(FDMat* mat, ProcLib lib) { auto b = BGR2RGB(); return b(mat, lib); } -bool RGB2BGR::Run(Mat* mat, ProcLib lib) { +bool RGB2BGR::Run(FDMat* mat, ProcLib lib) { auto r = RGB2BGR(); return r(mat, lib); } +bool BGR2GRAY::Run(FDMat* mat, ProcLib lib) { + auto b = BGR2GRAY(); + return b(mat, lib); +} + +bool RGB2GRAY::Run(FDMat* mat, ProcLib lib) { + auto r = RGB2GRAY(); + return r(mat, lib); +} + + } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/color_space_convert.h b/fastdeploy/vision/common/processors/color_space_convert.h index e090bc62d57..f9cb10a1905 100644 --- a/fastdeploy/vision/common/processors/color_space_convert.h +++ b/fastdeploy/vision/common/processors/color_space_convert.h @@ -21,24 +21,48 @@ namespace vision { class FASTDEPLOY_DECL BGR2RGB : public Processor { public: - bool ImplByOpenCV(Mat* mat); + bool ImplByOpenCV(FDMat* mat); #ifdef ENABLE_FLYCV - bool ImplByFlyCV(Mat* mat); + bool ImplByFlyCV(FDMat* mat); #endif virtual std::string Name() { return "BGR2RGB"; } - static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); + static bool Run(FDMat* mat, ProcLib lib = ProcLib::DEFAULT); }; class FASTDEPLOY_DECL RGB2BGR : public Processor { public: - bool ImplByOpenCV(Mat* mat); + bool ImplByOpenCV(FDMat* mat); #ifdef ENABLE_FLYCV - bool ImplByFlyCV(Mat* mat); + bool ImplByFlyCV(FDMat* mat); #endif std::string Name() { return "RGB2BGR"; } - static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); + static bool Run(FDMat* mat, ProcLib lib = ProcLib::DEFAULT); }; + +class FASTDEPLOY_DECL BGR2GRAY : public Processor { + public: + bool ImplByOpenCV(FDMat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFalconCV(FDMat* mat); +#endif + virtual std::string Name() { return "BGR2GRAY"; } + + static bool Run(FDMat* mat, ProcLib lib = ProcLib::OPENCV); +}; + +class FASTDEPLOY_DECL RGB2GRAY : public Processor { + public: + bool ImplByOpenCV(FDMat* mat); +#ifdef ENABLE_FLYCV + bool ImplByFalconCV(FDMat* mat); +#endif + std::string Name() { return "RGB2GRAY"; } + + static bool Run(FDMat* mat, ProcLib lib = ProcLib::OPENCV); +}; + + } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index 4d746797f48..ddb5f616a31 100755 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -259,7 +259,10 @@ std::string FaceAlignmentResult::Str() { std::string out; out = "FaceAlignmentResult: [x, y]\n"; - for (size_t i = 0; i < landmarks.size(); ++i) { + out = out + "There are " +std::to_string(landmarks.size()) + " landmarks, the top 10 are listed as below:\n"; + int landmarks_size = landmarks.size(); + size_t result_length = std::min(10, landmarks_size); + for (size_t i = 0; i < result_length; ++i) { out = out + std::to_string(landmarks[i][0]) + "," + std::to_string(landmarks[i][1]) + "\n"; } diff --git a/fastdeploy/vision/facealign/contrib/face_landmark_1000.cc b/fastdeploy/vision/facealign/contrib/face_landmark_1000.cc new file mode 100644 index 00000000000..f7b689575c0 --- /dev/null +++ b/fastdeploy/vision/facealign/contrib/face_landmark_1000.cc @@ -0,0 +1,134 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/facealign/contrib/face_landmark_1000.h" + +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { + +namespace vision { + +namespace facealign { + +FaceLandmark1000::FaceLandmark1000(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) { + if (model_format == ModelFormat::ONNX) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else { + valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool FaceLandmark1000::Initialize() { + // parameters for preprocess + size_ = {128, 128}; + + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool FaceLandmark1000::Preprocess( + Mat* mat, FDTensor* output, + std::map>* im_info) { + // Resize + int resize_w = size_[0]; + int resize_h = size_[1]; + if (resize_h != mat->Height() || resize_w != mat->Width()) { + Resize::Run(mat, resize_w, resize_h); + } + + // BRG2GRAY + BGR2GRAY::Run(mat); + + // Record output shape of preprocessed image + (*im_info)["output_shape"] = {mat->Height(), mat->Width()}; + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + mat->ShareWithTensor(output); + output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c + return true; +} + +bool FaceLandmark1000::Postprocess( + FDTensor& infer_result, FaceAlignmentResult* result, + const std::map>& im_info) { + FDASSERT(infer_result.shape[0] == 1, "Only support batch = 1 now."); + if (infer_result.dtype != FDDataType::FP32) { + FDERROR << "Only support post process with float32 data." << std::endl; + return false; + } + + auto iter_in = im_info.find("input_shape"); + FDASSERT(iter_in != im_info.end(), "Cannot find input_shape from im_info."); + int in_h = iter_in->second[0]; + int in_w = iter_in->second[1]; + + result->Clear(); + float* data = static_cast(infer_result.Data()); + for (size_t i = 0; i < infer_result.shape[1]; i += 2) { + float x = data[i]; + float y = data[i + 1]; + x = std::min(std::max(0.f, x), 1.0f); + y = std::min(std::max(0.f, y), 1.0f); + // decode landmarks (default 106 landmarks) + result->landmarks.emplace_back(std::array{x * in_w, y * in_h}); + } + + return true; +} + +bool FaceLandmark1000::Predict(cv::Mat* im, FaceAlignmentResult* result) { + Mat mat(*im); + std::vector input_tensors(1); + + std::map> im_info; + + // Record the shape of image and the shape of preprocessed image + im_info["input_shape"] = {mat.Height(), mat.Width()}; + im_info["output_shape"] = {mat.Height(), mat.Width()}; + + if (!Preprocess(&mat, &input_tensors[0], &im_info)) { + FDERROR << "Failed to preprocess input image." << std::endl; + return false; + } + input_tensors[0].name = InputInfoOfRuntime(0).name; + std::vector output_tensors; + if (!Infer(input_tensors, &output_tensors)) { + FDERROR << "Failed to inference." << std::endl; + return false; + } + if (!Postprocess(output_tensors[0], result, im_info)) { + FDERROR << "Failed to post process." << std::endl; + return false; + } + return true; +} + +} // namespace facealign +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/facealign/contrib/face_landmark_1000.h b/fastdeploy/vision/facealign/contrib/face_landmark_1000.h new file mode 100644 index 00000000000..909edfe155a --- /dev/null +++ b/fastdeploy/vision/facealign/contrib/face_landmark_1000.h @@ -0,0 +1,75 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { + +namespace vision { + +namespace facealign { +/*! @brief FaceLandmark1000 model object used when to load a FaceLandmark1000 model exported by FaceLandmark1000. + */ +class FASTDEPLOY_DECL FaceLandmark1000 : public FastDeployModel { + public: + /** \brief Set path of model file and the configuration of runtime. + * + * \param[in] model_file Path of model file, e.g ./face_landmarks_1000.onnx + * \param[in] params_file Path of parameter file, e.g ppyoloe/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" + * \param[in] model_format Model format of the loaded model, default is ONNX format + */ + FaceLandmark1000(const std::string& model_file, + const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::ONNX); + + std::string ModelName() const { return "FaceLandmark1000"; } + /** \brief Predict the face detection result for an input image + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output face detection result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(cv::Mat* im, FaceAlignmentResult* result); + + /** \brief Get the input size of image + * + * \return Vector of int values, default {128,128} + */ + std::vector GetSize() { return size_; } + /** \brief Set the input size of image + * + * \param[in] size Vector of int values which represents {width, height} of image + */ + void SetSize(const std::vector& size) { size_ = size; } + + private: + bool Initialize(); + + bool Preprocess(Mat* mat, FDTensor* outputs, + std::map>* im_info); + + bool Postprocess(FDTensor& infer_result, FaceAlignmentResult* result, + const std::map>& im_info); + // tuple of (width, height), default (128, 128) + std::vector size_; +}; + +} // namespace facealign +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/facealign/contrib/face_landmark_1000_pybind.cc b/fastdeploy/vision/facealign/contrib/face_landmark_1000_pybind.cc new file mode 100644 index 00000000000..6cb337e535d --- /dev/null +++ b/fastdeploy/vision/facealign/contrib/face_landmark_1000_pybind.cc @@ -0,0 +1,34 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindFaceLandmark1000(pybind11::module& m) { + pybind11::class_( + m, "FaceLandmark1000") + .def(pybind11::init()) + .def( + "predict", + [](vision::facealign::FaceLandmark1000& self, pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::FaceAlignmentResult res; + self.Predict(&mat, &res); + return res; + }) + .def_property("size", &vision::facealign::FaceLandmark1000::GetSize, + &vision::facealign::FaceLandmark1000::SetSize); +} +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/facealign/contrib/pipnet.cc b/fastdeploy/vision/facealign/contrib/pipnet.cc new file mode 100644 index 00000000000..27ec35c0dd6 --- /dev/null +++ b/fastdeploy/vision/facealign/contrib/pipnet.cc @@ -0,0 +1,687 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/facealign/contrib/pipnet.h" + +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { + +namespace vision { + +namespace facealign { + +void PIPNet::GenerateLandmarks(std::vector& infer_result, + FaceAlignmentResult* result, float img_height, + float img_width) { + FDTensor outputs_cls = infer_result.at(0); + FDTensor outputs_x = infer_result.at(1); + FDTensor outputs_y = infer_result.at(2); + FDTensor outputs_nb_x = infer_result.at(3); + FDTensor outputs_nb_y = infer_result.at(4); + int grid_h = outputs_cls.shape[2]; // 8 + int grid_w = outputs_cls.shape[3]; // 8 + int grid_length = grid_h * grid_w; // 8 * 8 = 64 + int input_h = size_[1]; + int input_w = size_[0]; + // fetch data from pointers + const float* outputs_cls_ptr = static_cast(outputs_cls.Data()); + const float* outputs_x_ptr = static_cast(outputs_x.Data()); + const float* outputs_y_ptr = static_cast(outputs_y.Data()); + const float* outputs_nb_x_ptr = static_cast(outputs_nb_x.Data()); + const float* outputs_nb_y_ptr = static_cast(outputs_nb_y.Data()); + + // find max_ids + std::vector max_ids(num_landmarks_); + for (unsigned int i = 0; i < num_landmarks_; ++i) { + const float* score_ptr = outputs_cls_ptr + i * grid_length; + unsigned int max_id = 0; + float max_score = score_ptr[0]; + for (unsigned int j = 0; j < grid_length; ++j) { + if (score_ptr[j] > max_score) { + max_score = score_ptr[j]; + max_id = j; + } + } + max_ids[i] = max_id; // range 0~64 + } + // find x & y offsets + std::vector output_x_select(num_landmarks_); + std::vector output_y_select(num_landmarks_); + for (unsigned int i = 0; i < num_landmarks_; ++i) { + const float* offset_x_ptr = outputs_x_ptr + i * grid_length; + const float* offset_y_ptr = outputs_y_ptr + i * grid_length; + const unsigned int max_id = max_ids.at(i); + output_x_select[i] = offset_x_ptr[max_id]; + output_y_select[i] = offset_y_ptr[max_id]; + } + + // find nb_x & nb_y offsets + std::map> output_nb_x_select; + std::map> output_nb_y_select; + // initialize offsets map + for (unsigned int i = 0; i < num_landmarks_; ++i) { + std::vector nb_x_offset(num_nb_); + std::vector nb_y_offset(num_nb_); + output_nb_x_select[i] = nb_x_offset; + output_nb_y_select[i] = nb_y_offset; + } + for (unsigned int i = 0; i < num_landmarks_; ++i) { + for (unsigned int j = 0; j < num_nb_; ++j) { + const unsigned int max_id = max_ids.at(i); + const float* offset_nb_x_ptr = + outputs_nb_x_ptr + (i * num_nb_ + j) * grid_length; + const float* offset_nb_y_ptr = + outputs_nb_y_ptr + (i * num_nb_ + j) * grid_length; + output_nb_x_select[i][j] = offset_nb_x_ptr[max_id]; + output_nb_y_select[i][j] = offset_nb_y_ptr[max_id]; + } + } + + // calculate coords + std::vector lms_pred_x(num_landmarks_); // 19 + std::vector lms_pred_y(num_landmarks_); // 19 + std::map> lms_pred_nb_x; // 19,10 + std::map> lms_pred_nb_y; // 19,10 + + // initialize pred maps + for (unsigned int i = 0; i < num_landmarks_; ++i) { + std::vector nb_x_offset(num_nb_); + std::vector nb_y_offset(num_nb_); + lms_pred_nb_x[i] = nb_x_offset; + lms_pred_nb_y[i] = nb_y_offset; + } + for (unsigned int i = 0; i < num_landmarks_; ++i) { + float cx = static_cast(max_ids.at(i) % grid_w); + float cy = static_cast(max_ids.at(i) / grid_w); + // calculate coords & normalize + lms_pred_x[i] = + ((cx + output_x_select[i]) * (float)net_stride_) / (float)input_w; + lms_pred_y[i] = + ((cy + output_y_select[i]) * (float)net_stride_) / (float)input_h; + for (unsigned int j = 0; j < num_nb_; ++j) { + lms_pred_nb_x[i][j] = + ((cx + output_nb_x_select[i][j]) * (float)net_stride_) / + (float)input_w; + lms_pred_nb_y[i][j] = + ((cy + output_nb_y_select[i][j]) * (float)net_stride_) / + (float)input_h; + } + } + + // reverse indexes + std::map> + tmp_nb_x; // 19,max_len_map_[num_landmarks_] + std::map> + tmp_nb_y; // 19,max_len_map_[num_landmarks_] + // initialize reverse maps + for (unsigned int i = 0; i < num_landmarks_; ++i) { + std::vector tmp_x(max_len_map_[num_landmarks_]); + std::vector tmp_y(max_len_map_[num_landmarks_]); + tmp_nb_x[i] = tmp_x; + tmp_nb_y[i] = tmp_y; + } + for (unsigned int i = 0; i < num_landmarks_; ++i) { + for (unsigned int j = 0; j < max_len_map_[num_landmarks_]; ++j) { + unsigned int ri = + reverse_index1_map_[num_landmarks_] + [i * max_len_map_[num_landmarks_] + j]; + unsigned int rj = + reverse_index2_map_[num_landmarks_] + [i * max_len_map_[num_landmarks_] + j]; + tmp_nb_x[i][j] = lms_pred_nb_x[ri][rj]; + tmp_nb_y[i][j] = lms_pred_nb_y[ri][rj]; + } + } + + // merge predictions + result->Clear(); + for (unsigned int i = 0; i < num_landmarks_; ++i) { + float total_x = lms_pred_x[i]; + float total_y = lms_pred_y[i]; + for (unsigned int j = 0; j < max_len_map_[num_landmarks_]; ++j) { + total_x += tmp_nb_x[i][j]; + total_y += tmp_nb_y[i][j]; + } + float x = total_x / ((float)max_len_map_[num_landmarks_] + 1.f); + float y = total_y / ((float)max_len_map_[num_landmarks_] + 1.f); + x = std::min(std::max(0.f, x), 1.0f); + y = std::min(std::max(0.f, y), 1.0f); + result->landmarks.emplace_back( + std::array{x * img_width, y * img_height}); + } +}; + +void PIPNet::SetNumLandmarks(const int& num_landmarks) { + if (std::find(supported_num_landmarks_.begin(), + supported_num_landmarks_.end(), + num_landmarks) == supported_num_landmarks_.end()) { + FDWARNING << "The number of landmarks should be in {19, 29, 68, 98}." + << std::endl; + } + num_landmarks_ = num_landmarks; +} +PIPNet::PIPNet(const std::string& model_file, const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) { + if (model_format == ModelFormat::ONNX) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else { + valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool PIPNet::Initialize() { + // parameters for preprocess + size_ = {256, 256}; + mean_vals_ = {0.485f, 0.456f, 0.406f}; + std_vals_ = {0.229f, 0.224f, 0.225f}; + num_nb_ = 10; + net_stride_ = 32; + num_landmarks_ = 19; + supported_num_landmarks_ = {19, 29, 68, 98}; + // parameters for num_landmarks_ == 19 + reverse_index1_map_[19] = { + 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, 7, 8, 1, 2, 6, + 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, 6, 7, 8, 0, 2, 3, 4, + 0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 3, 4, 5, 6, + 0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 14, 0, 1, 2, 4, 5, 6, + 1, 2, 3, 5, 9, 10, 11, 1, 2, 3, 5, 9, 10, 11, 1, 2, 3, 5, + 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, 10, 11, 3, 4, 9, + 0, 1, 2, 3, 7, 8, 12, 13, 15, 0, 1, 2, 3, 7, 8, 12, 13, 15, + 0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 15, 16, 18, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 13, 14, 16, 17, 18, 0, 1, + 3, 4, 5, 9, 10, 14, 17, 3, 4, 5, 9, 10, 14, 17, 3, 4, 5, 9, + 0, 1, 6, 7, 8, 13, 14, 15, 16, 17, 18, 0, 1, 6, 7, 8, 13, 14, + 0, 2, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17, 18, 0, 2, 5, + 4, 5, 9, 10, 11, 12, 13, 15, 16, 17, 18, 4, 5, 9, 10, 11, 12, 13, + 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, + 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, 12, 13, 14, 15, 17, 18, + 12, 13, 14, 15, 16, 18, 12, 13, 14, 15, 16, 18, 12, 13, 14, 15, 16, 18, + 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17, 15, 16, 17}; + reverse_index2_map_[19] = { + 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 4, 6, 0, 6, 1, 0, 1, 8, 7, 2, 2, 3, + 0, 1, 8, 7, 2, 2, 3, 0, 1, 8, 7, 3, 1, 3, 5, 5, 4, 3, 1, 5, 6, 6, 9, 3, 1, + 3, 5, 5, 4, 5, 5, 3, 1, 3, 7, 5, 5, 1, 3, 4, 9, 5, 5, 3, 1, 3, 7, 7, 8, 1, + 0, 3, 2, 2, 7, 8, 1, 0, 3, 2, 2, 7, 8, 1, 0, 6, 0, 6, 4, 1, 6, 0, 6, 4, 1, + 6, 0, 6, 4, 1, 6, 0, 6, 1, 3, 4, 9, 1, 2, 6, 9, 8, 1, 3, 4, 9, 1, 2, 6, 9, + 8, 2, 2, 2, 7, 8, 9, 0, 0, 9, 9, 9, 5, 7, 7, 8, 8, 2, 2, 4, 4, 0, 5, 6, 6, + 3, 0, 4, 5, 7, 4, 3, 8, 6, 6, 9, 6, 7, 6, 5, 0, 4, 4, 8, 6, 4, 0, 3, 8, 4, + 4, 9, 7, 6, 7, 9, 8, 7, 2, 2, 2, 9, 9, 9, 0, 0, 8, 5, 9, 7, 9, 9, 8, 4, 3, + 1, 2, 1, 6, 8, 4, 3, 1, 2, 1, 6, 8, 4, 3, 1, 2, 6, 9, 5, 7, 8, 0, 2, 1, 3, + 4, 4, 6, 9, 5, 7, 8, 0, 2, 8, 9, 8, 6, 8, 7, 7, 8, 8, 0, 0, 2, 2, 2, 5, 8, + 9, 8, 9, 7, 8, 7, 5, 2, 1, 4, 4, 1, 3, 9, 7, 8, 7, 5, 2, 1, 1, 5, 7, 0, 3, + 1, 1, 5, 7, 0, 3, 1, 1, 5, 7, 0, 3, 1, 3, 2, 3, 0, 0, 0, 3, 2, 3, 0, 0, 0, + 3, 2, 3, 0, 0, 0, 7, 6, 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 7, 6, 1, 3, 1, 2, 5, + 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5}; + max_len_map_[19] = 18; + // parameters for num_landmarks_ == 29 + reverse_index1_map_[29] = { + 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, 12, 13, 16, 2, 4, 5, 8, + 12, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, 9, 14, 15, 17, 3, 6, 7, + 9, 14, 0, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 16, 0, 3, 4, + 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 17, 0, + 1, 2, 4, 5, 0, 2, 5, 8, 10, 12, 13, 16, 0, 2, 5, 8, 10, 12, + 13, 16, 0, 2, 5, 0, 2, 4, 8, 10, 12, 13, 16, 0, 2, 4, 8, 10, + 12, 13, 16, 0, 2, 4, 1, 3, 7, 9, 11, 14, 15, 17, 1, 3, 7, 9, + 11, 14, 15, 17, 1, 3, 7, 1, 3, 6, 9, 11, 14, 15, 17, 1, 3, 6, + 9, 11, 14, 15, 17, 1, 3, 6, 0, 2, 4, 5, 10, 12, 13, 16, 0, 2, + 4, 5, 10, 12, 13, 16, 0, 2, 4, 1, 3, 6, 7, 11, 14, 15, 17, 1, + 3, 6, 7, 11, 14, 15, 17, 1, 3, 6, 0, 2, 3, 4, 5, 8, 12, 13, + 16, 18, 20, 0, 2, 3, 4, 5, 8, 12, 13, 1, 2, 3, 6, 7, 9, 14, + 15, 17, 19, 20, 21, 1, 2, 3, 6, 7, 9, 14, 0, 2, 4, 5, 8, 10, + 13, 16, 0, 2, 4, 5, 8, 10, 13, 16, 0, 2, 4, 0, 2, 4, 5, 8, + 10, 12, 16, 18, 22, 0, 2, 4, 5, 8, 10, 12, 16, 18, 1, 3, 6, 7, + 9, 11, 15, 17, 1, 3, 6, 7, 9, 11, 15, 17, 1, 3, 6, 1, 3, 6, + 7, 9, 11, 14, 17, 19, 23, 1, 3, 6, 7, 9, 11, 14, 17, 19, 0, 2, + 4, 5, 8, 10, 12, 13, 18, 0, 2, 4, 5, 8, 10, 12, 13, 18, 0, 1, + 3, 6, 7, 9, 11, 14, 15, 19, 1, 3, 6, 7, 9, 11, 14, 15, 19, 1, + 0, 4, 5, 8, 10, 12, 13, 16, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 0, 1, 6, 7, 9, 11, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 1, 1, 8, 9, 10, 11, 13, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, + 26, 27, 28, 18, 19, 20, 22, 23, 24, 25, 26, 27, 28, 18, 19, 20, 22, 23, + 24, 25, 26, 27, 18, 20, 21, 24, 25, 26, 27, 28, 18, 20, 21, 24, 25, 26, + 27, 28, 18, 20, 21, 19, 21, 24, 25, 26, 27, 28, 19, 21, 24, 25, 26, 27, + 28, 19, 21, 24, 25, 26, 18, 19, 20, 21, 22, 23, 25, 26, 27, 28, 18, 19, + 20, 21, 22, 23, 25, 26, 27, 18, 19, 20, 21, 22, 23, 24, 26, 27, 28, 18, + 19, 20, 21, 22, 23, 24, 26, 27, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, + 18, 19, 20, 21, 22, 23, 24, 25, 27, 20, 21, 22, 23, 24, 25, 26, 28, 20, + 21, 22, 23, 24, 25, 26, 28, 20, 21, 22, 22, 23, 24, 25, 26, 27, 22, 23, + 24, 25, 26, 27, 22, 23, 24, 25, 26, 27, 22}; + reverse_index2_map_[29] = { + 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 9, 3, 5, 3, 7, + 7, 7, 9, 3, 5, 3, 7, 7, 7, 9, 3, 5, 3, 7, 7, 6, 6, 6, 8, 9, 7, 0, 9, 6, + 5, 9, 6, 7, 6, 6, 6, 8, 9, 9, 7, 6, 8, 9, 6, 6, 7, 8, 0, 9, 6, 6, 6, 9, + 7, 6, 8, 9, 2, 5, 0, 5, 5, 3, 6, 5, 2, 5, 0, 5, 5, 3, 6, 5, 2, 5, 0, 1, + 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, 2, 4, 0, 5, 5, 3, + 5, 5, 2, 4, 0, 5, 5, 3, 5, 5, 2, 4, 0, 1, 3, 0, 4, 4, 2, 4, 2, 1, 3, 0, + 4, 4, 2, 4, 2, 1, 3, 0, 0, 7, 4, 3, 6, 5, 3, 4, 0, 7, 4, 3, 6, 5, 3, 4, + 0, 7, 4, 0, 7, 4, 3, 6, 5, 2, 4, 0, 7, 4, 3, 6, 5, 2, 4, 0, 7, 4, 6, 0, + 8, 7, 7, 6, 4, 2, 3, 5, 6, 6, 0, 8, 7, 7, 6, 4, 2, 6, 8, 0, 7, 7, 6, 4, + 3, 3, 5, 7, 9, 6, 8, 0, 7, 7, 6, 4, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, + 2, 3, 1, 0, 3, 1, 1, 5, 4, 5, 4, 0, 2, 1, 1, 6, 9, 5, 4, 5, 4, 0, 2, 1, + 1, 6, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 1, 2, 3, 1, 0, 3, 1, 1, 5, 5, 5, + 4, 0, 2, 1, 1, 7, 9, 5, 5, 5, 4, 0, 2, 1, 1, 7, 4, 2, 2, 2, 1, 1, 0, 0, + 9, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 4, 2, 2, 2, 1, 1, 0, 0, 9, 4, 2, 2, 2, + 1, 1, 0, 0, 9, 4, 8, 9, 8, 8, 7, 8, 8, 8, 8, 1, 3, 0, 8, 5, 8, 9, 9, 9, + 8, 8, 9, 8, 8, 7, 8, 8, 8, 8, 2, 4, 8, 0, 6, 7, 8, 8, 7, 8, 9, 9, 9, 9, + 8, 9, 9, 9, 9, 0, 0, 0, 6, 6, 4, 4, 6, 7, 8, 1, 1, 0, 5, 5, 2, 3, 3, 4, + 6, 1, 1, 0, 5, 5, 2, 3, 3, 4, 2, 8, 7, 7, 5, 4, 6, 5, 2, 8, 7, 7, 5, 4, + 6, 5, 2, 8, 7, 2, 8, 8, 6, 5, 5, 4, 2, 8, 8, 6, 5, 5, 4, 2, 8, 8, 6, 5, + 3, 3, 3, 1, 2, 3, 0, 2, 2, 3, 3, 3, 3, 1, 2, 3, 0, 2, 2, 4, 4, 4, 2, 1, + 1, 0, 0, 1, 2, 4, 4, 4, 2, 1, 1, 0, 0, 1, 7, 6, 5, 5, 3, 2, 1, 1, 0, 1, + 7, 6, 5, 5, 3, 2, 1, 1, 0, 9, 6, 4, 4, 3, 2, 1, 0, 9, 6, 4, 4, 3, 2, 1, + 0, 9, 6, 4, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7, 7, 9, 9, 7, 3, 7}; + max_len_map_[29] = 19; + // parameters for num_landmarks_ == 68 + reverse_index1_map_[68] = { + 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, 18, 36, 1, 2, 17, + 18, 36, 1, 2, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, 3, 17, 0, 2, + 3, 17, 0, 2, 3, 17, 0, 2, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, + 3, 4, 0, 1, 3, 4, 0, 1, 3, 4, 0, 1, 1, 2, 4, 5, 1, 2, + 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 4, 5, 1, 2, 2, 3, + 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, 2, 3, 5, 6, + 2, 3, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, 3, 4, 6, 7, + 3, 4, 6, 7, 3, 4, 3, 4, 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, + 5, 7, 8, 3, 4, 5, 7, 8, 3, 4, 5, 6, 8, 9, 5, 6, 8, 9, + 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 8, 9, 5, 6, 6, 7, 9, 10, + 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, 9, 10, 6, 7, + 7, 8, 10, 11, 7, 8, 10, 11, 7, 8, 10, 11, 7, 8, 10, 11, 7, 8, + 10, 11, 7, 8, 8, 9, 11, 12, 13, 8, 9, 11, 12, 13, 8, 9, 11, 12, + 13, 8, 9, 11, 12, 13, 8, 9, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, + 12, 13, 9, 10, 12, 13, 9, 10, 12, 13, 9, 10, 10, 11, 13, 14, 10, 11, + 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 13, 14, 10, 11, 11, 12, + 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, 11, 12, 14, 15, + 11, 12, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 15, 16, 12, 13, 15, 16, + 12, 13, 15, 16, 12, 13, 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 16, 26, + 13, 14, 16, 26, 13, 14, 16, 26, 13, 14, 14, 15, 25, 26, 45, 14, 15, 25, + 26, 45, 14, 15, 25, 26, 45, 14, 15, 25, 26, 45, 14, 15, 0, 1, 2, 18, + 19, 36, 37, 41, 0, 1, 2, 18, 19, 36, 37, 41, 0, 1, 2, 18, 19, 36, + 0, 1, 17, 19, 20, 36, 37, 38, 41, 0, 1, 17, 19, 20, 36, 37, 38, 41, + 0, 1, 17, 19, 0, 17, 18, 20, 21, 36, 37, 38, 40, 41, 0, 17, 18, 20, + 21, 36, 37, 38, 40, 41, 0, 17, 17, 18, 19, 21, 36, 37, 38, 39, 40, 41, + 17, 18, 19, 21, 36, 37, 38, 39, 40, 41, 17, 18, 18, 19, 20, 22, 27, 28, + 37, 38, 39, 40, 41, 18, 19, 20, 22, 27, 28, 37, 38, 39, 40, 41, 21, 23, + 24, 25, 27, 28, 42, 43, 44, 46, 47, 21, 23, 24, 25, 27, 28, 42, 43, 44, + 46, 47, 22, 24, 25, 26, 42, 43, 44, 45, 46, 47, 22, 24, 25, 26, 42, 43, + 44, 45, 46, 47, 22, 24, 16, 22, 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, + 23, 25, 26, 43, 44, 45, 46, 47, 16, 22, 15, 16, 23, 24, 26, 43, 44, 45, + 46, 15, 16, 23, 24, 26, 43, 44, 45, 46, 15, 16, 23, 24, 14, 15, 16, 24, + 25, 44, 45, 46, 14, 15, 16, 24, 25, 44, 45, 46, 14, 15, 16, 24, 25, 44, + 20, 21, 22, 23, 28, 29, 38, 39, 40, 42, 43, 47, 20, 21, 22, 23, 28, 29, + 38, 39, 40, 42, 21, 22, 27, 29, 30, 39, 40, 42, 47, 21, 22, 27, 29, 30, + 39, 40, 42, 47, 21, 22, 27, 29, 27, 28, 30, 31, 35, 39, 42, 27, 28, 30, + 31, 35, 39, 42, 27, 28, 30, 31, 35, 39, 42, 27, 28, 29, 31, 32, 33, 34, + 35, 28, 29, 31, 32, 33, 34, 35, 28, 29, 31, 32, 33, 34, 35, 28, 2, 3, + 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, 32, 33, 48, 49, 2, 3, 29, 30, + 32, 33, 29, 30, 31, 33, 34, 35, 49, 50, 29, 30, 31, 33, 34, 35, 49, 50, + 29, 30, 31, 33, 34, 35, 29, 30, 31, 32, 34, 35, 50, 51, 52, 29, 30, 31, + 32, 34, 35, 50, 51, 52, 29, 30, 31, 32, 29, 30, 31, 32, 33, 35, 52, 53, + 29, 30, 31, 32, 33, 35, 52, 53, 29, 30, 31, 32, 33, 35, 13, 14, 29, 30, + 32, 33, 34, 53, 54, 13, 14, 29, 30, 32, 33, 34, 53, 54, 13, 14, 29, 30, + 0, 1, 2, 17, 18, 19, 20, 37, 38, 39, 40, 41, 0, 1, 2, 17, 18, 19, + 20, 37, 38, 39, 0, 1, 17, 18, 19, 20, 21, 36, 38, 39, 40, 41, 0, 1, + 17, 18, 19, 20, 21, 36, 38, 39, 0, 1, 17, 18, 19, 20, 21, 27, 28, 36, + 37, 39, 40, 41, 0, 1, 17, 18, 19, 20, 21, 27, 19, 20, 21, 27, 28, 29, + 36, 37, 38, 40, 41, 19, 20, 21, 27, 28, 29, 36, 37, 38, 40, 41, 0, 1, + 17, 18, 19, 20, 21, 27, 28, 36, 37, 38, 39, 41, 0, 1, 17, 18, 19, 20, + 21, 27, 0, 1, 2, 17, 18, 19, 20, 21, 36, 37, 38, 39, 40, 0, 1, 2, + 17, 18, 19, 20, 21, 36, 22, 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 22, + 23, 24, 27, 28, 29, 43, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, 27, + 42, 44, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, 27, 42, 15, 16, 22, 23, + 24, 25, 26, 42, 43, 45, 46, 47, 15, 16, 22, 23, 24, 25, 26, 42, 43, 45, + 14, 15, 16, 23, 24, 25, 26, 42, 43, 44, 46, 47, 14, 15, 16, 23, 24, 25, + 26, 42, 43, 44, 14, 15, 16, 22, 23, 24, 25, 26, 42, 43, 44, 45, 47, 14, + 15, 16, 22, 23, 24, 25, 26, 42, 15, 16, 22, 23, 24, 25, 26, 27, 28, 42, + 43, 44, 45, 46, 15, 16, 22, 23, 24, 25, 26, 27, 2, 3, 4, 5, 6, 49, + 59, 60, 2, 3, 4, 5, 6, 49, 59, 60, 2, 3, 4, 5, 6, 49, 3, 4, + 5, 31, 32, 48, 50, 51, 59, 60, 61, 67, 3, 4, 5, 31, 32, 48, 50, 51, + 59, 60, 30, 31, 32, 33, 34, 48, 49, 51, 52, 58, 59, 60, 61, 62, 66, 67, + 30, 31, 32, 33, 34, 48, 30, 31, 32, 33, 34, 35, 48, 49, 50, 52, 53, 54, + 56, 58, 60, 61, 62, 63, 64, 65, 66, 67, 30, 32, 33, 34, 35, 50, 51, 53, + 54, 55, 56, 62, 63, 64, 65, 30, 32, 33, 34, 35, 50, 51, 11, 12, 13, 34, + 35, 52, 54, 55, 63, 64, 65, 11, 12, 13, 34, 35, 52, 54, 55, 63, 64, 65, + 10, 11, 12, 13, 14, 53, 55, 64, 10, 11, 12, 13, 14, 53, 55, 64, 10, 11, + 12, 13, 14, 53, 8, 9, 10, 11, 12, 13, 53, 54, 56, 57, 63, 64, 65, 8, + 9, 10, 11, 12, 13, 53, 54, 56, 7, 8, 9, 10, 11, 12, 54, 55, 57, 58, + 63, 64, 65, 66, 7, 8, 9, 10, 11, 12, 54, 55, 6, 7, 8, 9, 10, 55, + 56, 58, 59, 62, 65, 66, 67, 6, 7, 8, 9, 10, 55, 56, 58, 59, 4, 5, + 6, 7, 8, 9, 48, 56, 57, 59, 60, 61, 62, 66, 67, 4, 5, 6, 7, 8, + 9, 48, 3, 4, 5, 6, 7, 8, 48, 49, 57, 58, 60, 61, 67, 3, 4, 5, + 6, 7, 8, 48, 49, 57, 2, 3, 4, 5, 6, 31, 48, 49, 59, 2, 3, 4, + 5, 6, 31, 48, 49, 59, 2, 3, 4, 5, 31, 32, 33, 48, 49, 50, 51, 52, + 57, 58, 59, 60, 62, 63, 66, 67, 31, 32, 33, 48, 49, 50, 33, 34, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 63, 64, 65, 66, 67, 33, + 34, 35, 50, 51, 52, 53, 54, 55, 56, 57, 61, 62, 64, 65, 66, 34, 35, 50, + 51, 52, 53, 54, 10, 11, 12, 13, 14, 35, 53, 54, 55, 10, 11, 12, 13, 14, + 35, 53, 54, 55, 10, 11, 12, 13, 9, 10, 11, 12, 51, 52, 53, 54, 55, 56, + 57, 58, 61, 62, 63, 64, 66, 67, 9, 10, 11, 12, 7, 8, 9, 50, 51, 52, + 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 67, 7, 8, 9, 50, 4, 5, + 6, 7, 48, 49, 50, 51, 56, 57, 58, 59, 60, 61, 62, 63, 65, 66, 4, 5, + 6, 7}; + reverse_index2_map_[68] = { + 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 7, 8, 0, 3, 1, 1, 4, + 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 4, 9, 1, 1, 6, 1, 1, 5, 6, 1, + 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 1, 5, 6, 1, 5, 0, 0, 6, 5, 0, 0, 6, 5, + 0, 0, 6, 5, 0, 0, 6, 5, 0, 0, 6, 5, 0, 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 1, 7, + 2, 0, 1, 7, 2, 0, 1, 7, 2, 0, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, 6, 2, 1, 1, + 6, 2, 1, 1, 6, 2, 1, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, 1, 4, 9, 4, 0, + 1, 4, 9, 4, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, 0, 1, 3, 5, + 0, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 0, 4, 4, 0, 3, 0, + 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 0, 5, 3, 0, 3, 1, 0, 4, 9, + 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 0, 4, 9, 3, 1, 6, 1, 0, 2, 6, 1, 0, 2, + 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 0, 2, 6, 1, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 0, + 2, 7, 1, 0, 2, 7, 1, 0, 2, 7, 1, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, 1, 4, 6, 1, + 1, 4, 6, 1, 1, 4, 6, 1, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, 1, 0, 6, 5, + 1, 0, 6, 5, 1, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, 3, 0, 0, 9, + 3, 0, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 7, 2, 8, 3, 1, 0, + 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, 8, 6, 0, 3, 9, 0, 4, 4, 3, 8, 0, 0, + 6, 5, 7, 9, 7, 3, 8, 0, 0, 6, 5, 7, 9, 7, 3, 8, 0, 0, 7, 4, 1, 1, 6, 6, 5, + 7, 9, 5, 7, 4, 1, 1, 6, 6, 5, 7, 9, 5, 7, 4, 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, + 8, 4, 1, 0, 9, 6, 4, 7, 6, 8, 8, 4, 9, 6, 0, 4, 2, 7, 9, 6, 5, 5, 9, 9, 6, + 0, 4, 2, 7, 9, 6, 5, 5, 9, 4, 1, 6, 9, 3, 8, 5, 6, 9, 9, 6, 4, 1, 6, 9, 3, + 8, 5, 6, 9, 9, 6, 0, 1, 4, 8, 7, 5, 7, 9, 8, 5, 0, 1, 4, 8, 7, 5, 7, 9, 8, + 5, 0, 1, 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, 0, 1, 4, 7, 5, 6, 6, 9, 7, 6, + 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, 0, 9, 6, 5, 7, 8, 3, 5, 0, 8, 3, 1, + 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 4, 5, 8, 3, 1, 4, 0, 8, 9, 1, 1, 9, 1, 2, + 8, 4, 7, 2, 8, 7, 9, 1, 1, 9, 1, 2, 8, 4, 7, 2, 8, 8, 0, 0, 6, 6, 8, 6, 8, + 8, 8, 0, 0, 6, 6, 8, 6, 8, 8, 8, 0, 0, 5, 0, 0, 9, 9, 9, 9, 5, 0, 0, 9, 9, + 9, 9, 5, 0, 0, 9, 9, 9, 9, 5, 4, 1, 2, 2, 2, 2, 2, 4, 1, 2, 2, 2, 2, 2, 4, + 1, 2, 2, 2, 2, 2, 4, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, 6, 5, 0, 7, 7, 9, 8, 8, + 6, 5, 0, 7, 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, 5, 8, 7, 4, 3, 0, 0, 4, + 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, 1, 1, 5, 8, 5, 7, 2, 1, 1, 3, 1, + 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 6, 9, 3, 1, 5, 4, 1, 0, 8, 9, 5, 4, 9, + 6, 0, 8, 7, 8, 9, 5, 4, 9, 6, 0, 8, 7, 8, 9, 5, 4, 2, 2, 4, 2, 3, 5, 8, 1, + 5, 8, 4, 1, 2, 2, 4, 2, 3, 5, 8, 1, 5, 8, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, 3, + 0, 5, 6, 3, 2, 2, 3, 7, 1, 1, 3, 9, 9, 6, 6, 3, 2, 2, 7, 9, 3, 2, 1, 0, 3, + 9, 9, 6, 6, 3, 2, 2, 7, 9, 4, 3, 4, 3, 9, 7, 4, 2, 1, 4, 9, 4, 3, 4, 3, 9, + 7, 4, 2, 1, 4, 8, 7, 7, 8, 8, 5, 5, 8, 5, 2, 3, 0, 0, 2, 8, 7, 7, 8, 8, 5, + 5, 8, 4, 4, 5, 5, 5, 7, 7, 9, 0, 0, 3, 2, 2, 4, 4, 5, 5, 5, 7, 7, 9, 0, 3, + 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 3, 4, 9, 1, 2, 8, 2, 4, 7, 4, 2, 9, 9, 2, 2, + 3, 6, 6, 6, 1, 2, 3, 3, 0, 9, 9, 2, 2, 3, 6, 6, 6, 1, 6, 5, 7, 3, 2, 2, 3, + 4, 1, 1, 1, 3, 6, 5, 7, 3, 2, 2, 3, 4, 1, 1, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, + 0, 4, 4, 2, 2, 8, 5, 3, 1, 8, 4, 1, 5, 5, 4, 9, 7, 7, 5, 5, 3, 3, 0, 0, 1, + 5, 5, 4, 9, 7, 7, 5, 5, 3, 7, 8, 5, 6, 8, 8, 7, 9, 6, 0, 0, 3, 2, 2, 7, 8, + 5, 6, 8, 8, 7, 9, 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, 2, 5, 3, 3, 0, 6, 3, 2, + 2, 5, 3, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, 5, 8, 6, 7, 8, 4, 6, 1, 3, 9, 4, 1, + 7, 3, 3, 4, 8, 5, 1, 1, 7, 9, 8, 5, 1, 6, 9, 5, 7, 3, 3, 4, 8, 5, 9, 6, 5, + 3, 5, 6, 9, 6, 1, 1, 6, 9, 8, 8, 8, 3, 0, 3, 8, 6, 6, 6, 8, 8, 5, 3, 3, 8, + 2, 1, 5, 8, 9, 7, 1, 5, 4, 8, 8, 5, 3, 3, 8, 2, 8, 7, 6, 6, 4, 3, 1, 3, 5, + 1, 8, 8, 7, 6, 6, 4, 3, 1, 3, 5, 1, 8, 5, 2, 2, 4, 6, 2, 4, 0, 5, 2, 2, 4, + 6, 2, 4, 0, 5, 2, 2, 4, 6, 2, 7, 5, 2, 3, 6, 7, 5, 2, 2, 9, 8, 2, 5, 7, 5, + 2, 3, 6, 7, 5, 2, 2, 7, 5, 2, 3, 7, 8, 6, 0, 1, 5, 7, 6, 3, 8, 7, 5, 2, 3, + 7, 8, 6, 0, 8, 4, 2, 4, 8, 7, 0, 0, 7, 8, 7, 4, 7, 8, 4, 2, 4, 8, 7, 0, 0, + 7, 9, 7, 3, 2, 6, 7, 6, 5, 0, 0, 6, 7, 9, 7, 3, 9, 7, 3, 2, 6, 7, 6, 7, 6, + 3, 2, 5, 8, 2, 5, 8, 2, 2, 8, 4, 7, 6, 3, 2, 5, 8, 2, 5, 8, 7, 5, 3, 4, 6, + 8, 0, 0, 1, 7, 5, 3, 4, 6, 8, 0, 0, 1, 7, 5, 3, 4, 7, 7, 9, 3, 2, 0, 3, 9, + 6, 4, 5, 3, 2, 6, 3, 0, 7, 7, 9, 3, 2, 0, 8, 9, 8, 7, 2, 0, 2, 7, 8, 9, 6, + 5, 6, 9, 7, 2, 2, 7, 2, 0, 2, 8, 7, 7, 9, 4, 0, 3, 3, 5, 4, 7, 6, 3, 3, 0, + 5, 7, 7, 9, 4, 0, 3, 3, 6, 4, 3, 5, 7, 8, 0, 0, 1, 6, 4, 3, 5, 7, 8, 0, 0, + 1, 6, 4, 3, 5, 8, 9, 9, 9, 7, 4, 4, 4, 2, 1, 4, 7, 9, 5, 0, 4, 2, 9, 8, 9, + 9, 9, 9, 9, 9, 6, 5, 8, 6, 3, 2, 3, 6, 9, 4, 1, 4, 9, 1, 1, 9, 9, 9, 6, 8, + 9, 9, 8, 4, 4, 4, 6, 7, 3, 1, 2, 4, 0, 4, 9, 9, 1, 8, 9, 9, 8}; + max_len_map_[68] = 22; + // parameters for num_landmarks_ == 98 + reverse_index1_map_[98] = { + 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 33, 1, 2, 3, 4, 5, 0, + 2, 3, 4, 5, 6, 33, 0, 2, 3, 4, 5, 6, 33, 0, 2, 3, 0, 1, + 3, 4, 5, 6, 0, 1, 3, 4, 5, 6, 0, 1, 3, 4, 5, 0, 1, 2, + 4, 5, 6, 7, 0, 1, 2, 4, 5, 6, 7, 0, 1, 2, 0, 1, 2, 3, + 5, 6, 7, 8, 0, 1, 2, 3, 5, 6, 7, 8, 0, 1, 2, 3, 4, 6, + 7, 8, 9, 1, 2, 3, 4, 6, 7, 8, 9, 1, 2, 3, 4, 5, 7, 8, + 9, 10, 2, 3, 4, 5, 7, 8, 9, 10, 2, 3, 4, 5, 6, 8, 9, 10, + 3, 4, 5, 6, 8, 9, 10, 3, 4, 5, 4, 5, 6, 7, 9, 10, 11, 4, + 5, 6, 7, 9, 10, 11, 4, 5, 6, 4, 5, 6, 7, 8, 10, 11, 12, 4, + 5, 6, 7, 8, 10, 11, 12, 4, 5, 6, 7, 8, 9, 11, 12, 13, 76, 5, + 6, 7, 8, 9, 11, 12, 13, 7, 8, 9, 10, 12, 13, 14, 76, 88, 7, 8, + 9, 10, 12, 13, 14, 76, 8, 9, 10, 11, 13, 14, 15, 8, 9, 10, 11, 13, + 14, 15, 8, 9, 10, 10, 11, 12, 14, 15, 16, 10, 11, 12, 14, 15, 16, 10, + 11, 12, 14, 15, 11, 12, 13, 15, 16, 17, 11, 12, 13, 15, 16, 17, 11, 12, + 13, 15, 16, 12, 13, 14, 16, 17, 18, 12, 13, 14, 16, 17, 18, 12, 13, 14, + 16, 17, 13, 14, 15, 17, 18, 19, 13, 14, 15, 17, 18, 19, 13, 14, 15, 17, + 18, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, 20, 14, 15, 16, 18, 19, + 15, 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 21, 15, 16, 17, 19, 20, 16, + 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, 22, 16, 17, 18, 20, 21, 17, 18, + 19, 21, 22, 23, 24, 17, 18, 19, 21, 22, 23, 24, 17, 18, 19, 18, 19, 20, + 22, 23, 24, 25, 82, 18, 19, 20, 22, 23, 24, 25, 82, 18, 19, 20, 21, 23, + 24, 25, 26, 27, 19, 20, 21, 23, 24, 25, 26, 27, 19, 20, 21, 22, 24, 25, + 26, 27, 28, 20, 21, 22, 24, 25, 26, 27, 28, 20, 21, 22, 23, 25, 26, 27, + 28, 21, 22, 23, 25, 26, 27, 28, 21, 22, 23, 21, 22, 23, 24, 26, 27, 28, + 29, 21, 22, 23, 24, 26, 27, 28, 29, 21, 22, 23, 24, 25, 27, 28, 29, 30, + 22, 23, 24, 25, 27, 28, 29, 30, 22, 23, 24, 25, 26, 28, 29, 30, 31, 23, + 24, 25, 26, 28, 29, 30, 31, 23, 24, 25, 26, 27, 29, 30, 31, 32, 24, 25, + 26, 27, 29, 30, 31, 32, 24, 25, 26, 27, 28, 30, 31, 32, 25, 26, 27, 28, + 30, 31, 32, 25, 26, 27, 26, 27, 28, 29, 31, 32, 26, 27, 28, 29, 31, 32, + 26, 27, 28, 29, 31, 26, 27, 28, 29, 30, 32, 46, 26, 27, 28, 29, 30, 32, + 46, 26, 27, 28, 27, 28, 29, 30, 31, 46, 27, 28, 29, 30, 31, 46, 27, 28, + 29, 30, 31, 0, 1, 2, 3, 34, 41, 60, 0, 1, 2, 3, 34, 41, 60, 0, + 1, 2, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, 41, 60, 0, 33, 35, 40, + 41, 33, 34, 36, 37, 39, 40, 41, 60, 61, 62, 33, 34, 36, 37, 39, 40, 41, + 34, 35, 37, 38, 39, 40, 63, 64, 34, 35, 37, 38, 39, 40, 63, 64, 34, 36, + 38, 39, 51, 64, 36, 38, 39, 51, 64, 36, 38, 39, 51, 64, 36, 38, 36, 37, + 39, 51, 52, 63, 64, 65, 36, 37, 39, 51, 52, 63, 64, 65, 36, 35, 36, 37, + 38, 40, 62, 63, 64, 65, 66, 67, 96, 35, 36, 37, 38, 40, 33, 34, 35, 36, + 37, 38, 39, 41, 60, 61, 62, 63, 65, 66, 67, 96, 33, 0, 1, 2, 33, 34, + 35, 40, 60, 61, 67, 0, 1, 2, 33, 34, 35, 40, 43, 49, 50, 51, 68, 43, + 49, 50, 51, 68, 43, 49, 50, 51, 68, 43, 49, 42, 44, 45, 48, 49, 50, 68, + 69, 42, 44, 45, 48, 49, 50, 68, 69, 42, 42, 43, 45, 46, 47, 48, 49, 70, + 42, 43, 45, 46, 47, 48, 49, 70, 42, 32, 44, 46, 47, 48, 71, 72, 73, 32, + 44, 46, 47, 48, 71, 72, 73, 32, 29, 30, 31, 32, 45, 47, 72, 29, 30, 31, + 32, 45, 47, 72, 29, 30, 31, 30, 31, 32, 44, 45, 46, 48, 71, 72, 73, 30, + 31, 32, 44, 45, 46, 48, 42, 43, 44, 45, 46, 47, 49, 50, 69, 70, 71, 72, + 73, 74, 75, 97, 42, 42, 43, 44, 48, 50, 68, 69, 70, 74, 75, 97, 42, 43, + 44, 48, 50, 68, 42, 43, 49, 51, 52, 68, 69, 75, 42, 43, 49, 51, 52, 68, + 69, 75, 42, 37, 38, 42, 50, 52, 53, 64, 68, 37, 38, 42, 50, 52, 53, 64, + 68, 37, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, 53, 54, 51, + 53, 51, 52, 54, 55, 56, 57, 59, 51, 52, 54, 55, 56, 57, 59, 51, 52, 54, + 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 56, 57, 58, 59, 52, 53, 55, 53, + 54, 56, 57, 76, 77, 78, 88, 53, 54, 56, 57, 76, 77, 78, 88, 53, 53, 54, + 55, 57, 58, 77, 78, 79, 88, 53, 54, 55, 57, 58, 77, 78, 79, 53, 54, 55, + 56, 58, 59, 78, 79, 80, 90, 53, 54, 55, 56, 58, 59, 78, 53, 54, 56, 57, + 59, 79, 80, 81, 82, 92, 53, 54, 56, 57, 59, 79, 80, 53, 54, 57, 58, 80, + 81, 82, 92, 53, 54, 57, 58, 80, 81, 82, 92, 53, 0, 1, 2, 3, 4, 33, + 34, 41, 61, 62, 66, 67, 96, 0, 1, 2, 3, 0, 1, 33, 34, 35, 40, 41, + 60, 62, 63, 65, 66, 67, 96, 0, 1, 33, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 60, 61, 63, 64, 65, 66, 67, 96, 35, 36, 37, 38, 39, 40, 51, 52, 61, + 62, 64, 65, 66, 67, 96, 35, 36, 36, 37, 38, 39, 51, 52, 53, 63, 65, 66, + 96, 36, 37, 38, 39, 51, 52, 36, 37, 38, 39, 52, 61, 62, 63, 64, 66, 67, + 96, 36, 37, 38, 39, 52, 41, 60, 61, 62, 63, 64, 65, 67, 96, 41, 60, 61, + 62, 63, 64, 65, 67, 0, 1, 2, 3, 33, 34, 35, 40, 41, 60, 61, 62, 65, + 66, 96, 0, 1, 42, 43, 49, 50, 51, 52, 53, 69, 74, 75, 97, 42, 43, 49, + 50, 51, 52, 42, 43, 44, 48, 49, 50, 51, 68, 70, 71, 73, 74, 75, 97, 42, + 43, 44, 42, 43, 44, 45, 46, 47, 48, 49, 50, 68, 69, 71, 72, 73, 74, 75, + 97, 31, 32, 44, 45, 46, 47, 48, 69, 70, 72, 73, 74, 75, 97, 31, 32, 44, + 28, 29, 30, 31, 32, 45, 46, 47, 70, 71, 73, 74, 97, 28, 29, 30, 31, 29, + 30, 31, 32, 44, 45, 46, 47, 48, 70, 71, 72, 74, 75, 97, 29, 30, 47, 68, + 69, 70, 71, 72, 73, 75, 97, 47, 68, 69, 70, 71, 72, 73, 75, 42, 43, 49, + 50, 52, 68, 69, 70, 71, 72, 73, 74, 97, 42, 43, 49, 50, 6, 7, 8, 9, + 10, 11, 12, 55, 77, 87, 88, 89, 95, 6, 7, 8, 9, 55, 56, 76, 78, 86, + 87, 88, 89, 95, 55, 56, 76, 78, 86, 87, 88, 89, 54, 55, 56, 57, 58, 76, + 77, 79, 80, 85, 86, 87, 88, 89, 90, 94, 95, 54, 55, 56, 57, 58, 59, 77, + 78, 80, 81, 84, 85, 86, 89, 90, 91, 94, 54, 57, 58, 59, 78, 79, 81, 82, + 83, 84, 85, 90, 91, 92, 93, 94, 54, 58, 59, 80, 82, 83, 84, 91, 92, 93, + 58, 59, 80, 82, 83, 84, 91, 92, 20, 21, 22, 23, 24, 25, 26, 59, 81, 83, + 91, 92, 93, 20, 21, 22, 23, 17, 18, 19, 20, 21, 22, 23, 81, 82, 84, 91, + 92, 93, 17, 18, 19, 20, 16, 17, 18, 19, 20, 81, 82, 83, 85, 91, 92, 93, + 94, 16, 17, 18, 19, 14, 15, 16, 17, 18, 83, 84, 86, 87, 90, 93, 94, 95, + 14, 15, 16, 17, 11, 12, 13, 14, 15, 16, 76, 77, 85, 87, 88, 89, 94, 95, + 11, 12, 13, 9, 10, 11, 12, 13, 14, 76, 77, 86, 88, 89, 95, 9, 10, 11, + 12, 13, 7, 8, 9, 10, 11, 12, 13, 55, 76, 77, 86, 87, 89, 95, 7, 8, + 9, 55, 56, 76, 77, 78, 79, 86, 87, 88, 90, 95, 55, 56, 76, 77, 78, 79, + 56, 57, 58, 78, 79, 80, 83, 84, 85, 86, 87, 89, 91, 92, 93, 94, 95, 58, + 59, 79, 80, 81, 82, 83, 84, 85, 90, 92, 93, 94, 58, 59, 79, 80, 19, 20, + 21, 22, 23, 24, 25, 59, 81, 82, 83, 84, 91, 93, 19, 20, 21, 18, 19, 79, + 80, 81, 82, 83, 84, 85, 90, 91, 92, 94, 18, 19, 79, 80, 15, 16, 17, 78, + 79, 80, 83, 84, 85, 86, 87, 89, 90, 91, 93, 95, 15, 13, 14, 15, 76, 77, + 78, 85, 86, 87, 88, 89, 90, 94, 13, 14, 15, 76, 34, 35, 36, 38, 39, 40, + 41, 60, 61, 62, 63, 64, 65, 66, 67, 34, 35, 43, 44, 45, 47, 48, 49, 50, + 68, 69, 70, 71, 72, 73, 74, 75, 43, 44}; + reverse_index2_map_[98] = { + 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 4, 0, 2, 4, 6, 8, 0, 0, 2, 4, 6, 8, 8, 0, + 0, 2, 4, 6, 8, 8, 0, 0, 2, 1, 1, 0, 2, 4, 6, 1, 1, 0, 2, 4, 6, 1, 1, 0, 2, + 4, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 0, 2, 4, 6, 3, 2, 1, 6, 3, 3, 1, 0, 2, 4, + 7, 6, 3, 3, 1, 0, 2, 4, 7, 6, 6, 4, 3, 1, 0, 2, 4, 8, 6, 4, 3, 1, 0, 2, 4, + 8, 6, 7, 5, 3, 1, 0, 2, 4, 9, 7, 5, 3, 1, 0, 2, 4, 9, 7, 6, 5, 3, 1, 0, 2, + 4, 6, 5, 3, 1, 0, 2, 4, 6, 5, 3, 7, 5, 3, 1, 0, 2, 4, 7, 5, 3, 1, 0, 2, 4, + 7, 5, 3, 9, 7, 5, 3, 1, 0, 2, 5, 9, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, + 0, 2, 5, 8, 9, 7, 5, 3, 1, 0, 2, 5, 7, 5, 3, 1, 0, 2, 5, 9, 9, 7, 5, 3, 1, + 0, 2, 5, 9, 9, 5, 3, 1, 0, 2, 4, 9, 5, 3, 1, 0, 2, 4, 9, 5, 3, 6, 3, 1, 0, + 2, 6, 6, 3, 1, 0, 2, 6, 6, 3, 1, 0, 2, 7, 3, 1, 0, 3, 7, 7, 3, 1, 0, 3, 7, + 7, 3, 1, 0, 3, 6, 3, 1, 1, 3, 6, 6, 3, 1, 1, 3, 6, 6, 3, 1, 1, 3, 7, 3, 1, + 1, 3, 7, 7, 3, 1, 1, 3, 7, 7, 3, 1, 1, 3, 6, 3, 0, 1, 3, 6, 6, 3, 0, 1, 3, + 6, 6, 3, 0, 1, 3, 7, 2, 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 7, 2, 0, 1, 3, 5, 2, + 0, 1, 3, 5, 5, 2, 0, 1, 3, 5, 5, 2, 0, 1, 3, 4, 2, 0, 1, 3, 5, 8, 4, 2, 0, + 1, 3, 5, 8, 4, 2, 0, 5, 2, 0, 1, 3, 5, 7, 9, 5, 2, 0, 1, 3, 5, 7, 9, 5, 4, + 2, 0, 1, 3, 5, 7, 9, 4, 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, 5, 7, 9, 4, + 2, 0, 1, 3, 5, 7, 9, 4, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, 1, 3, 5, 7, 4, 2, 0, + 9, 4, 2, 0, 1, 3, 5, 6, 9, 4, 2, 0, 1, 3, 5, 6, 9, 9, 4, 2, 0, 1, 3, 5, 6, + 9, 4, 2, 0, 1, 3, 5, 6, 9, 8, 4, 2, 0, 1, 3, 4, 6, 8, 4, 2, 0, 1, 3, 4, 6, + 8, 6, 4, 2, 0, 1, 3, 3, 5, 6, 4, 2, 0, 1, 3, 3, 5, 6, 6, 4, 2, 0, 1, 2, 3, + 6, 4, 2, 0, 1, 2, 3, 6, 4, 2, 6, 4, 2, 0, 1, 1, 6, 4, 2, 0, 1, 1, 6, 4, 2, + 0, 1, 8, 6, 4, 2, 0, 0, 9, 8, 6, 4, 2, 0, 0, 9, 8, 6, 4, 8, 6, 4, 2, 0, 6, + 8, 6, 4, 2, 0, 6, 8, 6, 4, 2, 0, 2, 4, 5, 8, 3, 1, 6, 2, 4, 5, 8, 3, 1, 6, + 2, 4, 5, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 8, 7, 1, 1, 5, 0, 7, 1, 2, 8, 6, + 0, 5, 9, 8, 8, 7, 1, 2, 8, 6, 0, 5, 8, 2, 1, 4, 0, 6, 7, 9, 8, 2, 1, 4, 0, + 6, 7, 9, 8, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 5, 5, 7, 1, 0, 4, 0, 2, 2, + 6, 6, 2, 8, 4, 0, 2, 2, 6, 6, 2, 8, 4, 4, 0, 2, 1, 4, 7, 4, 4, 5, 9, 9, 7, + 4, 0, 2, 1, 4, 5, 2, 0, 3, 9, 9, 4, 2, 7, 5, 4, 8, 9, 8, 6, 6, 5, 5, 7, 9, + 0, 0, 3, 3, 2, 6, 7, 5, 7, 9, 0, 0, 3, 3, 2, 5, 0, 6, 7, 2, 5, 0, 6, 7, 2, + 5, 0, 6, 7, 2, 5, 1, 1, 8, 5, 0, 4, 9, 7, 1, 1, 8, 5, 0, 4, 9, 7, 1, 8, 1, + 1, 7, 4, 0, 6, 9, 8, 1, 1, 7, 4, 0, 6, 9, 8, 7, 2, 1, 0, 6, 9, 8, 9, 7, 2, + 1, 0, 6, 9, 8, 9, 7, 8, 5, 4, 2, 2, 1, 6, 8, 5, 4, 2, 2, 1, 6, 8, 5, 4, 9, + 7, 6, 3, 0, 0, 3, 6, 2, 7, 9, 7, 6, 3, 0, 0, 3, 7, 3, 0, 3, 5, 2, 2, 9, 8, + 4, 5, 7, 6, 7, 9, 6, 7, 2, 0, 4, 2, 1, 3, 2, 7, 9, 5, 8, 2, 0, 4, 2, 1, 3, + 0, 4, 3, 1, 5, 2, 6, 8, 0, 4, 3, 1, 5, 2, 6, 8, 0, 5, 6, 5, 5, 1, 5, 8, 8, + 5, 6, 5, 5, 1, 5, 8, 8, 5, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, 1, 9, 0, + 1, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 9, 9, 9, 9, 7, 0, 1, 4, 0, 5, 2, 0, 2, 4, + 4, 0, 5, 2, 0, 2, 4, 4, 0, 5, 6, 5, 0, 8, 6, 6, 9, 6, 6, 5, 0, 8, 6, 6, 9, + 6, 6, 3, 2, 0, 2, 7, 7, 5, 7, 8, 3, 2, 0, 2, 7, 7, 5, 7, 2, 0, 2, 1, 1, 2, + 4, 3, 5, 7, 2, 0, 2, 1, 1, 2, 4, 4, 3, 7, 1, 0, 5, 4, 8, 8, 8, 4, 3, 7, 1, + 0, 5, 4, 7, 4, 7, 0, 9, 6, 6, 6, 7, 4, 7, 0, 9, 6, 6, 6, 7, 4, 5, 6, 7, 8, + 2, 5, 4, 1, 9, 6, 1, 9, 4, 5, 6, 7, 8, 9, 3, 4, 6, 2, 3, 1, 2, 9, 7, 4, 0, + 5, 8, 9, 3, 9, 6, 5, 6, 7, 7, 3, 1, 7, 4, 2, 3, 6, 4, 1, 4, 0, 8, 5, 3, 3, + 1, 8, 8, 9, 7, 3, 1, 0, 5, 8, 3, 8, 5, 8, 4, 2, 8, 4, 3, 9, 1, 1, 7, 8, 8, + 4, 2, 8, 4, 3, 9, 6, 5, 9, 7, 9, 6, 0, 0, 3, 5, 2, 9, 6, 5, 9, 7, 9, 3, 4, + 1, 5, 5, 3, 2, 1, 9, 3, 4, 1, 5, 5, 3, 2, 9, 8, 8, 9, 6, 7, 9, 9, 6, 0, 0, + 5, 6, 2, 4, 9, 8, 4, 8, 8, 2, 3, 2, 8, 1, 8, 1, 9, 4, 8, 8, 2, 3, 2, 3, 5, + 8, 8, 1, 3, 9, 0, 3, 7, 8, 5, 0, 5, 3, 5, 8, 9, 6, 5, 6, 8, 6, 1, 4, 7, 6, + 4, 2, 5, 4, 2, 4, 0, 9, 8, 6, 4, 3, 3, 4, 9, 1, 1, 0, 4, 7, 2, 9, 8, 6, 8, + 7, 7, 5, 4, 5, 2, 5, 8, 1, 1, 6, 7, 8, 7, 7, 5, 9, 8, 8, 9, 9, 7, 4, 7, 9, + 5, 0, 0, 1, 6, 3, 9, 8, 9, 5, 5, 2, 4, 3, 2, 3, 1, 9, 5, 5, 2, 4, 3, 2, 3, + 6, 9, 9, 6, 8, 1, 0, 6, 8, 9, 5, 3, 4, 6, 9, 9, 6, 9, 8, 6, 6, 5, 6, 7, 8, + 4, 2, 0, 8, 7, 9, 8, 6, 6, 1, 5, 2, 7, 5, 3, 2, 0, 3, 1, 5, 2, 7, 5, 3, 2, + 0, 7, 4, 3, 4, 9, 7, 5, 1, 3, 7, 7, 6, 7, 2, 2, 3, 4, 6, 7, 4, 3, 4, 6, 9, + 0, 0, 9, 9, 6, 9, 7, 0, 7, 2, 8, 5, 3, 3, 3, 2, 5, 7, 6, 7, 8, 3, 2, 7, 4, + 4, 8, 5, 1, 6, 2, 3, 5, 0, 2, 3, 5, 1, 6, 2, 3, 5, 0, 2, 7, 6, 6, 6, 7, 8, + 9, 8, 4, 2, 8, 0, 8, 7, 6, 6, 6, 8, 7, 6, 5, 7, 8, 9, 3, 1, 1, 3, 1, 2, 8, + 7, 6, 5, 7, 5, 4, 5, 9, 7, 5, 5, 1, 4, 5, 1, 5, 7, 5, 4, 5, 8, 5, 4, 6, 8, + 8, 2, 2, 8, 4, 9, 0, 9, 8, 5, 4, 6, 9, 8, 4, 4, 6, 8, 5, 8, 2, 5, 5, 4, 6, + 1, 9, 8, 4, 9, 8, 5, 4, 6, 7, 1, 3, 1, 1, 3, 2, 9, 8, 5, 4, 6, 9, 8, 7, 7, + 8, 9, 9, 6, 0, 2, 8, 1, 5, 5, 9, 8, 7, 3, 6, 3, 0, 2, 8, 3, 4, 3, 6, 0, 3, + 6, 3, 0, 2, 8, 8, 6, 8, 1, 0, 1, 9, 6, 3, 6, 9, 6, 6, 9, 7, 1, 8, 6, 5, 6, + 2, 0, 3, 4, 3, 9, 5, 3, 0, 9, 6, 5, 6, 2, 9, 8, 8, 7, 7, 9, 9, 7, 2, 0, 1, + 8, 5, 5, 9, 8, 8, 9, 8, 9, 8, 1, 4, 0, 0, 4, 8, 1, 4, 7, 9, 8, 9, 8, 8, 9, + 9, 6, 4, 7, 7, 4, 0, 4, 7, 9, 1, 9, 6, 6, 8, 8, 9, 9, 4, 1, 8, 5, 0, 0, 4, + 1, 9, 8, 8, 9, 9, 4, 9, 7, 7, 8, 7, 7, 8, 5, 3, 0, 2, 3, 2, 0, 3, 9, 7, 7, + 7, 9, 8, 7, 7, 8, 4, 3, 0, 3, 4, 3, 0, 2, 7, 7}; + max_len_map_[98] = 17; + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool PIPNet::Preprocess(Mat* mat, FDTensor* output, + std::map>* im_info) { + // Resize + int resize_w = size_[0]; + int resize_h = size_[1]; + if (resize_h != mat->Height() || resize_w != mat->Width()) { + Resize::Run(mat, resize_w, resize_h); + } + // RGR2RGB + BGR2RGB::Run(mat); + + // Normalize + Normalize::Run(mat, mean_vals_, std_vals_); + + // Record output shape of preprocessed image + (*im_info)["output_shape"] = {mat->Height(), mat->Width()}; + + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + mat->ShareWithTensor(output); + output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c + return true; +} + +bool PIPNet::Postprocess( + std::vector& infer_result, FaceAlignmentResult* result, + const std::map>& im_info) { + FDASSERT(infer_result.at(0).shape[0] == 1, "Only support batch = 1 now."); + if (infer_result.at(0).dtype != FDDataType::FP32) { + FDERROR << "Only support post process with float32 data." << std::endl; + return false; + } + + auto iter_in = im_info.find("input_shape"); + FDASSERT(iter_in != im_info.end(), "Cannot find input_shape from im_info."); + int in_h = iter_in->second[0]; + int in_w = iter_in->second[1]; + GenerateLandmarks(infer_result, result, in_h, in_w); + + return true; +} + +bool PIPNet::Predict(cv::Mat* im, FaceAlignmentResult* result) { + Mat mat(*im); + std::vector input_tensors(1); + + std::map> im_info; + + // Record the shape of image and the shape of preprocessed image + im_info["input_shape"] = {mat.Height(), mat.Width()}; + im_info["output_shape"] = {mat.Height(), mat.Width()}; + + if (!Preprocess(&mat, &input_tensors[0], &im_info)) { + FDERROR << "Failed to preprocess input image." << std::endl; + return false; + } + input_tensors[0].name = InputInfoOfRuntime(0).name; + std::vector output_tensors; + if (!Infer(input_tensors, &output_tensors)) { + FDERROR << "Failed to inference." << std::endl; + return false; + } + + if (!Postprocess(output_tensors, result, im_info)) { + FDERROR << "Failed to post process." << std::endl; + return false; + } + return true; +} + +} // namespace facealign +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/facealign/contrib/pipnet.h b/fastdeploy/vision/facealign/contrib/pipnet.h new file mode 100644 index 00000000000..b0ea801f4df --- /dev/null +++ b/fastdeploy/vision/facealign/contrib/pipnet.h @@ -0,0 +1,127 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { + +namespace vision { + +namespace facealign { +/*! @brief PIPNet model object used when to load a PIPNet model exported by PIPNet. + */ +class FASTDEPLOY_DECL PIPNet : public FastDeployModel { + public: + /** \brief Set path of model file and the configuration of runtime. + * + * \param[in] model_file Path of model file, e.g ./pipnet.onnx + * \param[in] params_file Path of parameter file, e.g ppyoloe/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" + * \param[in] model_format Model format of the loaded model, default is ONNX format + */ + PIPNet(const std::string& model_file, const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::ONNX); + + std::string ModelName() const { return "PIPNet"; } + /** \brief Predict the face detection result for an input image + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output face detection result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(cv::Mat* im, FaceAlignmentResult* result); + + /** \brief Get the number of landmakrs + * + * \return Integer type, default num_landmarks = 19 + */ + int GetNumLandmarks() {return num_landmarks_; } + /** \brief Get the mean values for normalization + * + * \return Vector of float values, default mean_vals = {0.485f, 0.456f, 0.406f} + */ + std::vector GetMeanVals() { return mean_vals_; } + /** \brief Get the std values for normalization + * + * \return Vector of float values, default std_vals = {0.229f, 0.224f, 0.225f} + */ + std::vector GetStdVals() { return std_vals_; } + /** \brief Get the input size of image + * + * \return Vector of int values, default {256, 256} + */ + std::vector GetSize() { return size_; } + /** \brief Set the number of landmarks + * + * \param[in] num_landmarks Integer value which represents number of landmarks + */ + void SetNumLandmarks(const int& num_landmarks); + /** \brief Set the mean values for normalization + * + * \param[in] mean_vals Vector of float values whose length is equal to 3 + */ + void SetMeanVals(const std::vector& mean_vals) { + mean_vals_ = mean_vals; + } + /** \brief Set the std values for normalization + * + * \param[in] std_vals Vector of float values whose length is equal to 3 + */ + void SetStdVals(const std::vector& std_vals) { std_vals_ = std_vals; } + /** \brief Set the input size of image + * + * \param[in] size Vector of int values which represents {width, height} of image + */ + void SetSize(const std::vector& size) { size_ = size; } + + private: + bool Initialize(); + + bool Preprocess(Mat* mat, FDTensor* outputs, + std::map>* im_info); + + bool Postprocess(std::vector& infer_result, + FaceAlignmentResult* result, + const std::map>& im_info); + void GenerateLandmarks(std::vector& infer_result, + FaceAlignmentResult* result, + float img_height, float img_width); + std::map num_lms_map_; + std::map max_len_map_; + std::map> reverse_index1_map_; + std::map> reverse_index2_map_; + int num_nb_; + int net_stride_; + // Now PIPNet support num_landmarks in {19, 29, 68, 98} + std::vector supported_num_landmarks_; + // tuple of (width, height), default (256, 256) + std::vector size_; + + // Mean parameters for normalize, size should be the the same as channels, + // default mean_vals = {0.485f, 0.456f, 0.406f} + std::vector mean_vals_; + // Std parameters for normalize, size should be the the same as channels, + // default std_vals = {0.229f, 0.224f, 0.225f} + std::vector std_vals_; + // number of landmarks + int num_landmarks_; +}; + +} // namespace facealign +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/facealign/contrib/pipnet_pybind.cc b/fastdeploy/vision/facealign/contrib/pipnet_pybind.cc new file mode 100644 index 00000000000..025259ebc74 --- /dev/null +++ b/fastdeploy/vision/facealign/contrib/pipnet_pybind.cc @@ -0,0 +1,39 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindPIPNet(pybind11::module& m) { + pybind11::class_(m, "PIPNet") + .def(pybind11::init()) + .def("predict", + [](vision::facealign::PIPNet& self, pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::FaceAlignmentResult res; + self.Predict(&mat, &res); + return res; + }) + .def_property("size", &vision::facealign::PIPNet::GetSize, + &vision::facealign::PIPNet::SetSize) + .def_property("mean_vals", &vision::facealign::PIPNet::GetMeanVals, + &vision::facealign::PIPNet::SetMeanVals) + .def_property("std_vals", &vision::facealign::PIPNet::GetStdVals, + &vision::facealign::PIPNet::SetStdVals) + .def_property("num_landmarks", + &vision::facealign::PIPNet::GetNumLandmarks, + &vision::facealign::PIPNet::SetNumLandmarks); +} +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/facealign/facealign_pybind.cc b/fastdeploy/vision/facealign/facealign_pybind.cc index 60ccdf605a9..19067d56699 100644 --- a/fastdeploy/vision/facealign/facealign_pybind.cc +++ b/fastdeploy/vision/facealign/facealign_pybind.cc @@ -17,9 +17,14 @@ namespace fastdeploy { void BindPFLD(pybind11::module& m); +void BindFaceLandmark1000(pybind11::module& m); +void BindPIPNet(pybind11::module& m); void BindFaceAlign(pybind11::module& m) { auto facedet_module = m.def_submodule("facealign", "Face alignment models."); BindPFLD(facedet_module); + BindFaceLandmark1000(facedet_module); + BindPIPNet(facedet_module); + } } // namespace fastdeploy diff --git a/python/fastdeploy/vision/facealign/__init__.py b/python/fastdeploy/vision/facealign/__init__.py index be5b8c76cb0..c55585ff9ad 100644 --- a/python/fastdeploy/vision/facealign/__init__.py +++ b/python/fastdeploy/vision/facealign/__init__.py @@ -14,3 +14,5 @@ from __future__ import absolute_import from .contrib.pfld import PFLD +from .contrib.pipnet import PIPNet +from .contrib.face_landmark_1000 import FaceLandmark1000 diff --git a/python/fastdeploy/vision/facealign/contrib/face_landmark_1000.py b/python/fastdeploy/vision/facealign/contrib/face_landmark_1000.py new file mode 100644 index 00000000000..67f980f4099 --- /dev/null +++ b/python/fastdeploy/vision/facealign/contrib/face_landmark_1000.py @@ -0,0 +1,68 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import logging +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class FaceLandmark1000(FastDeployModel): + def __init__(self, + model_file, + params_file="", + runtime_option=None, + model_format=ModelFormat.ONNX): + """Load a face alignment model exported by FaceLandmark1000. + + :param model_file: (str)Path of model file, e.g ./FaceLandmark1000.onnx + :param params_file: (str)Path of parameters file, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model, default is ONNX + """ + + super(FaceLandmark1000, self).__init__(runtime_option) + + assert model_format == ModelFormat.ONNX, "FaceLandmark1000 only support model format of ModelFormat.ONNX now." + self._model = C.vision.facealign.FaceLandmark1000( + model_file, params_file, self._runtime_option, model_format) + assert self.initialized, "FaceLandmark1000 initialize failed." + + def predict(self, input_image): + """Detect an input image landmarks + + :param im: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :return: FaceAlignmentResult + """ + + return self._model.predict(input_image) + + @property + def size(self): + """ + Returns the preprocess image size, default (128, 128) + """ + return self._model.size + + @size.setter + def size(self, wh): + """ + Set the preprocess image size, default (128, 128) + """ + assert isinstance(wh, (list, tuple)),\ + "The value to set `size` must be type of tuple or list." + assert len(wh) == 2,\ + "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( + len(wh)) + self._model.size = wh diff --git a/python/fastdeploy/vision/facealign/contrib/pipnet.py b/python/fastdeploy/vision/facealign/contrib/pipnet.py new file mode 100644 index 00000000000..6ba3fc71149 --- /dev/null +++ b/python/fastdeploy/vision/facealign/contrib/pipnet.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +import logging +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class PIPNet(FastDeployModel): + def __init__(self, + model_file, + params_file="", + runtime_option=None, + model_format=ModelFormat.ONNX): + """Load a face alignment model exported by PIPNet. + + :param model_file: (str)Path of model file, e.g ./PIPNet.onnx + :param params_file: (str)Path of parameters file, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model, default is ONNX + """ + + super(PIPNet, self).__init__(runtime_option) + + assert model_format == ModelFormat.ONNX, "PIPNet only support model format of ModelFormat.ONNX now." + self._model = C.vision.facealign.PIPNet( + model_file, params_file, self._runtime_option, model_format) + assert self.initialized, "PIPNet initialize failed." + + def predict(self, input_image): + """Detect an input image landmarks + + :param im: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :return: FaceAlignmentResult + """ + + return self._model.predict(input_image) + + @property + def size(self): + """ + Returns the preprocess image size, default (256, 256) + """ + return self._model.size + + @property + def mean_vals(self): + """ + Returns the mean value of normlization, default mean_vals = [0.485f, 0.456f, 0.406f]; + """ + return self._model.mean_vals + + @property + def std_vals(self): + """ + Returns the std value of normlization, default std_vals = [0.229f, 0.224f, 0.225f]; + """ + return self._model.std_vals + + @property + def num_landmarks(self): + """ + Returns the number of landmarks + """ + return self._model.num_landmarks + + @size.setter + def size(self, wh): + """ + Set the preprocess image size, default (256, 256) + """ + assert isinstance(wh, (list, tuple)),\ + "The value to set `size` must be type of tuple or list." + assert len(wh) == 2,\ + "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( + len(wh)) + self._model.size = wh + + @mean_vals.setter + def mean_vals(self, value): + assert isinstance( + value, list), "The value to set `mean_vals` must be type of list." + self._model.mean_vals = value + + @std_vals.setter + def std_vals(self, value): + assert isinstance( + value, list), "The value to set `std_vals` must be type of list." + self._model.std_vals = value + + @num_landmarks.setter + def num_landmarks(self, value): + assert isinstance( + value, int), "The value to set `std_vals` must be type of int." + self._model.num_landmarks = value diff --git a/tests/eval_example/test_facelandmark1000.py b/tests/eval_example/test_facelandmark1000.py new file mode 100644 index 00000000000..5fc2f87302c --- /dev/null +++ b/tests/eval_example/test_facelandmark1000.py @@ -0,0 +1,43 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np + + +def test_facealignment_pipnet(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/FaceLandmark1000.onnx" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png" + output_url = "https://bj.bcebos.com/paddlehub/fastdeploy/tests/facelandmark1000_result_landmarks.npy" + fd.download(model_url, ".") + fd.download(input_url, ".") + fd.download(output_url, ".") + model_path = "FaceLandmark1000.onnx" + # use ORT + runtime_option = fd.RuntimeOption() + runtime_option.use_ort_backend() + model = fd.vision.facealign.FaceLandmark1000( + model_path, runtime_option=runtime_option) + + # compare diff + im = cv2.imread("./facealign_input.png") + result = model.predict(im.copy()) + expect = np.load("./facelandmark1000_result_landmarks.npy") + + diff = np.fabs(np.array(result.landmarks) - expect) + thres = 1e-04 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) diff --git a/tests/eval_example/test_pipnet.py b/tests/eval_example/test_pipnet.py new file mode 100644 index 00000000000..012ae675861 --- /dev/null +++ b/tests/eval_example/test_pipnet.py @@ -0,0 +1,43 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np + + +def test_facealignment_pipnet(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/pipnet_resnet18_10x19x32x256_aflw.onnx" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/facealign_input.png" + output_url = "https://bj.bcebos.com/paddlehub/fastdeploy/tests/pipnet_result_landmarks.npy" + fd.download(model_url, ".") + fd.download(input_url, ".") + fd.download(output_url, ".") + model_path = "pipnet_resnet18_10x19x32x256_aflw.onnx" + # use ORT + runtime_option = fd.RuntimeOption() + runtime_option.use_ort_backend() + model = fd.vision.facealign.PIPNet( + model_path, runtime_option=runtime_option) + + # compare diff + im = cv2.imread("./facealign_input.png") + result = model.predict(im.copy()) + expect = np.load("./pipnet_result_landmarks.npy") + + diff = np.fabs(np.array(result.landmarks) - expect) + thres = 1e-04 + assert diff.max() < thres, "The diff is %f, which is bigger than %f" % ( + diff.max(), thres) diff --git a/tests/vision_preprocess/test_vision_colorspace_convert.cc b/tests/vision_preprocess/test_vision_colorspace_convert.cc index f1253d959ca..d9cdc823150 100644 --- a/tests/vision_preprocess/test_vision_colorspace_convert.cc +++ b/tests/vision_preprocess/test_vision_colorspace_convert.cc @@ -71,6 +71,56 @@ TEST(fastdeploy, flycv_rgb2bgr) { check_data(reinterpret_cast(opencv.Data()), reinterpret_cast(flycv.Data()), opencv.Numel()); check_type(opencv.dtype, flycv.dtype); } + +TEST(fastdeploy, flycv_rgb2gray) { + CheckShape check_shape; + CheckData check_data; + CheckType check_type; + + cv::Mat mat(64, 64, CV_8UC3); + cv::randu(mat, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::Mat mat1 = mat.clone(); + + vision::Mat mat_opencv(mat); + vision::Mat mat_flycv(mat1); + vision::RGB2GRAY::Run(&mat_opencv, vision::ProcLib::OPENCV); + vision::RGB2GRAY::Run(&mat_flycv, vision::ProcLib::FLYCV); + + FDTensor opencv; + FDTensor flycv; + + mat_opencv.ShareWithTensor(&opencv); + mat_flycv.ShareWithTensor(&flycv); + + check_shape(opencv.shape, flycv.shape); + check_data(reinterpret_cast(opencv.Data()), reinterpret_cast(flycv.Data()), opencv.Numel()); + check_type(opencv.dtype, flycv.dtype); +} + +TEST(fastdeploy, flycv_bgr2gray) { + CheckShape check_shape; + CheckData check_data; + CheckType check_type; + + cv::Mat mat(64, 64, CV_8UC3); + cv::randu(mat, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::Mat mat1 = mat.clone(); + + vision::Mat mat_opencv(mat); + vision::Mat mat_flycv(mat1); + vision::BGR2GRAY::Run(&mat_opencv, vision::ProcLib::OPENCV); + vision::BGR2GRAY::Run(&mat_flycv, vision::ProcLib::FLYCV); + + FDTensor opencv; + FDTensor flycv; + + mat_opencv.ShareWithTensor(&opencv); + mat_flycv.ShareWithTensor(&flycv); + + check_shape(opencv.shape, flycv.shape); + check_data(reinterpret_cast(opencv.Data()), reinterpret_cast(flycv.Data()), opencv.Numel()); + check_type(opencv.dtype, flycv.dtype); +} #endif } // namespace fastdeploy From f5ae0d2f8eea73acf8086986131dd3fa06f0b199 Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Wed, 16 Nov 2022 09:58:01 +0000 Subject: [PATCH 040/136] modify docs --- docs/api_docs/python/conf.py | 3 ++- docs/api_docs/python/face_alignment.md | 2 +- docs/api_docs/python/face_detection.md | 2 +- docs/api_docs/python/face_recognition.md | 2 +- docs/api_docs/python/headpose.md | 2 +- docs/api_docs/python/image_classification.md | 2 +- docs/api_docs/python/index.rst | 15 ++++++++++++++- docs/api_docs/python/keypoint_detection.md | 2 +- docs/api_docs/python/matting.md | 2 +- docs/api_docs/python/object_detection.md | 10 +--------- docs/api_docs/python/ocr.md | 2 +- docs/api_docs/python/runtime.md | 2 +- docs/api_docs/python/runtime_option.md | 2 +- docs/api_docs/python/semantic_segmentation.md | 2 +- 14 files changed, 28 insertions(+), 22 deletions(-) diff --git a/docs/api_docs/python/conf.py b/docs/api_docs/python/conf.py index f22b0fd169d..f3bb44d1f02 100644 --- a/docs/api_docs/python/conf.py +++ b/docs/api_docs/python/conf.py @@ -66,7 +66,8 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'furo' +# html_theme = 'furo' +html_theme = 'sphinx_book_theme' # -- Extension configuration ------------------------------------------------- def setup(app): diff --git a/docs/api_docs/python/face_alignment.md b/docs/api_docs/python/face_alignment.md index f0369b55af6..85f79e2f052 100644 --- a/docs/api_docs/python/face_alignment.md +++ b/docs/api_docs/python/face_alignment.md @@ -1,4 +1,4 @@ -# Face Alignment API +# Face Alignment(人脸对齐) ## fastdeploy.vision.facealign.PFLD diff --git a/docs/api_docs/python/face_detection.md b/docs/api_docs/python/face_detection.md index 3e8d5762ca8..93068a4d189 100644 --- a/docs/api_docs/python/face_detection.md +++ b/docs/api_docs/python/face_detection.md @@ -1,4 +1,4 @@ -# Face Detection API +# Face Detection(人脸检测) ## fastdeploy.vision.facedet.RetinaFace diff --git a/docs/api_docs/python/face_recognition.md b/docs/api_docs/python/face_recognition.md index 344aab79687..1f0c507ae47 100644 --- a/docs/api_docs/python/face_recognition.md +++ b/docs/api_docs/python/face_recognition.md @@ -1,4 +1,4 @@ -# Face Recognition API +# Face Recognition(人脸识别) ## fastdeploy.vision.faceid.AdaFace diff --git a/docs/api_docs/python/headpose.md b/docs/api_docs/python/headpose.md index d1fba74f927..fb2b9080fb3 100644 --- a/docs/api_docs/python/headpose.md +++ b/docs/api_docs/python/headpose.md @@ -1,4 +1,4 @@ -# Headpose API +# Headpose(头部姿态) ## fastdeploy.vision.headpose.FSANet diff --git a/docs/api_docs/python/image_classification.md b/docs/api_docs/python/image_classification.md index ad284ca66db..6b18f5c09f4 100755 --- a/docs/api_docs/python/image_classification.md +++ b/docs/api_docs/python/image_classification.md @@ -1,4 +1,4 @@ -# Image Classification API +# Image Classification(图像分类) ## fastdeploy.vision.classification.PaddleClasModel diff --git a/docs/api_docs/python/index.rst b/docs/api_docs/python/index.rst index 69b65b3b101..77d7d64ea3d 100644 --- a/docs/api_docs/python/index.rst +++ b/docs/api_docs/python/index.rst @@ -8,7 +8,7 @@ FastDeploy * Contact Us: fastdeploy@baidu.com .. toctree:: - :caption: 文档目录 + :caption: Vision Task Model API (视觉任务模型API) :maxdepth: 2 :titlesonly: @@ -22,6 +22,19 @@ FastDeploy face_detection.md face_alignment.md headpose.md + + +.. toctree:: + :caption: Predict Result Struct (预测结果结构体) + :maxdepth: 3 + :titlesonly: + vision_results_en.md + +.. toctree:: + :caption: FastDeploy Runtime API (多后端推理引擎API) + :maxdepth: 2 + :titlesonly: + runtime.md runtime_option.md diff --git a/docs/api_docs/python/keypoint_detection.md b/docs/api_docs/python/keypoint_detection.md index b2ee837046a..b0510e1b947 100644 --- a/docs/api_docs/python/keypoint_detection.md +++ b/docs/api_docs/python/keypoint_detection.md @@ -1,4 +1,4 @@ -# Keypoint Detection API +# Keypoint Detection(关键点检测) ## fastdeploy.vision.keypointdetection.PPTinyPose diff --git a/docs/api_docs/python/matting.md b/docs/api_docs/python/matting.md index 5e9c2a22732..5998f132df5 100644 --- a/docs/api_docs/python/matting.md +++ b/docs/api_docs/python/matting.md @@ -1,4 +1,4 @@ -# Matting API +# Matting(图像抠图) ## fastdeploy.vision.matting.MODNet diff --git a/docs/api_docs/python/object_detection.md b/docs/api_docs/python/object_detection.md index 6bab167cf06..efe7b5923f0 100644 --- a/docs/api_docs/python/object_detection.md +++ b/docs/api_docs/python/object_detection.md @@ -1,4 +1,4 @@ -# Object Detection API +# Object Detection(目标检测) ## fastdeploy.vision.detection.PPYOLOE @@ -16,14 +16,6 @@ :inherited-members: ``` -## fastdeploy.vision.detection.PPYOLOv2 - -```{eval-rst} -.. autoclass:: fastdeploy.vision.detection.PPYOLOv2 - :members: - :inherited-members: -``` - ## fastdeploy.vision.detection.PicoDet ```{eval-rst} diff --git a/docs/api_docs/python/ocr.md b/docs/api_docs/python/ocr.md index 62291820370..a197263d50c 100644 --- a/docs/api_docs/python/ocr.md +++ b/docs/api_docs/python/ocr.md @@ -1,4 +1,4 @@ -# OCR API +# OCR(文字识别) ## fastdeploy.vision.ocr.DBDetector diff --git a/docs/api_docs/python/runtime.md b/docs/api_docs/python/runtime.md index 4a519ee7ee1..6e7b8888ed2 100644 --- a/docs/api_docs/python/runtime.md +++ b/docs/api_docs/python/runtime.md @@ -1,4 +1,4 @@ -# Runtime API +# Runtime(模型推理Runtime) ## fastdeploy.Runtime diff --git a/docs/api_docs/python/runtime_option.md b/docs/api_docs/python/runtime_option.md index 96eff8672d4..3f0f8dde973 100644 --- a/docs/api_docs/python/runtime_option.md +++ b/docs/api_docs/python/runtime_option.md @@ -1,4 +1,4 @@ -# Runtime Option API +# Runtime Option(Runtime配置) ## fastdeploy.RuntimeOption diff --git a/docs/api_docs/python/semantic_segmentation.md b/docs/api_docs/python/semantic_segmentation.md index dd9a2a5ffa2..6d224e08595 100644 --- a/docs/api_docs/python/semantic_segmentation.md +++ b/docs/api_docs/python/semantic_segmentation.md @@ -1,4 +1,4 @@ -# Semantic Segmentation API +# Semantic Segmentation(语义分割) ## fastdeploy.vision.segmentation.PaddleSegModel From 19df9639e00e29bfd330f12e4e16399ed06e17ef Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 16 Nov 2022 17:32:53 +0800 Subject: [PATCH 041/136] Update download_prebuilt_libraries.md --- docs/en/build_and_install/download_prebuilt_libraries.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/build_and_install/download_prebuilt_libraries.md b/docs/en/build_and_install/download_prebuilt_libraries.md index 58edd1d56eb..99f4f5ea854 100755 --- a/docs/en/build_and_install/download_prebuilt_libraries.md +++ b/docs/en/build_and_install/download_prebuilt_libraries.md @@ -44,13 +44,13 @@ Install the released version（Latest 0.6.0） | Platform | File | Description | |:----------- |:--------------------------------------------------------------------------------------------------------------------- |:--------------------------------------------------------- | | Linux x64 | [fastdeploy-linux-x64-gpu-0.6.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz) | Compiled from g++ 8.2, CUDA 11.2, cuDNN 8.2 | -| Windows x64 | [fastdeploy-win-x64-gpu-0.5.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-gpu-0.6.0.zip) | Compiled from Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | +| Windows x64 | [fastdeploy-win-x64-gpu-0.6.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-gpu-0.6.0.zip) | Compiled from Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | Install the Develop version（Nightly build） | Platform | File | Description | |:----------- |:--------------------------------------------------------------------------------------------------------------------- |:--------------------------------------------------------- | -| Linux x64 | [fastdeploy-linux-x64-gpu-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-gpu-0.0.0.tgz | Compiled from g++ 8.2, CUDA 11.2, cuDNN 8.2 | +| Linux x64 | [fastdeploy-linux-x64-gpu-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-gpu-0.0.0.tgz) | Compiled from g++ 8.2, CUDA 11.2, cuDNN 8.2 | | Windows x64 | [fastdeploy-win-x64-gpu-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-gpu-0.0.0.zip) | Compiled from Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | ## CPU Deployment Environment @@ -93,7 +93,7 @@ Install the Develop version（Nightly build） | Platform | File | Description | |:------------- |:--------------------------------------------------------------------------------------------------------------------- |:------------------------------ | -| Linux x64 | [fastdeploy-linux-x64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-0.0.0.tgz) | Compiled from g++ 8.2 | +| Linux x64 | [fastdeploy-linux-x64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-0.0.0.tgz) | Compiled from g++ 8.2 | | Windows x64 | [fastdeploy-win-x64-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-0.0.0.zip) | Compiled from Visual Studio 16 | | Mac OSX x64 | - | - | | Mac OSX arm64 | [fastdeploy-osx-arm64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-osx-arm64-0.0.0.tgz) | - | From 83bf2d8e10e2bd53d0136016e035fc2b412b6fb3 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Wed, 16 Nov 2022 17:56:18 +0800 Subject: [PATCH 042/136] [Android] Add facedet Android app example (#614) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android * [Android] Run button shutter in sub Ui Thread * [Android] Update CameraSurfaceView * [Android] Update Android SDK usage docs * [Android] Add facedet Android app example --- java/android/README.md | 10 +- java/android/app/src/main/AndroidManifest.xml | 6 +- .../detection/DetectionMainActivity.java | 67 +-- .../examples/facedet/FaceDetMainActivity.java | 465 ++++++++++++++++++ .../facedet/FaceDetSettingsActivity.java | 181 +++++++ .../SegmentationMainActivity.java | 4 + .../SegmentationSettingsActivity.java | 4 + .../main/res/layout/facedet_activity_main.xml | 14 + .../main/res/layout/facedet_camera_page.xml | 159 ++++++ .../main/res/layout/facedet_result_page.xml | 160 ++++++ .../res/layout/facedet_result_page_item.xml | 26 + .../res/layout/segmentation_activity_main.xml | 6 + .../res/layout/segmentation_camera_page.xml | 6 + .../res/layout/segmentation_result_page.xml | 6 + .../layout/segmentation_result_page_item.xml | 6 + .../app/src/main/res/values/strings.xml | 7 + .../app/src/main/res/xml/facedet_setting.xml | 41 ++ .../src/main/res/xml/segmentation_setting.xml | 4 + .../vision/facedet/scrfd_jni.cc | 5 +- .../vision/facedet/yolov5face_jni.cc | 5 +- .../fastdeploy/vision/facedet/SCRFD.java | 33 +- .../fastdeploy/vision/facedet/YOLOv5Face.java | 33 +- 22 files changed, 1194 insertions(+), 54 deletions(-) create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetSettingsActivity.java create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java create mode 100644 java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java create mode 100644 java/android/app/src/main/res/layout/facedet_activity_main.xml create mode 100644 java/android/app/src/main/res/layout/facedet_camera_page.xml create mode 100644 java/android/app/src/main/res/layout/facedet_result_page.xml create mode 100644 java/android/app/src/main/res/layout/facedet_result_page_item.xml create mode 100644 java/android/app/src/main/res/layout/segmentation_activity_main.xml create mode 100644 java/android/app/src/main/res/layout/segmentation_camera_page.xml create mode 100644 java/android/app/src/main/res/layout/segmentation_result_page.xml create mode 100644 java/android/app/src/main/res/layout/segmentation_result_page_item.xml create mode 100644 java/android/app/src/main/res/xml/facedet_setting.xml create mode 100644 java/android/app/src/main/res/xml/segmentation_setting.xml diff --git a/java/android/README.md b/java/android/README.md index aae25b68008..e73443506fc 100644 --- a/java/android/README.md +++ b/java/android/README.md @@ -188,9 +188,10 @@ public boolean init(String modelFile, String paramsFile, RuntimeOption option); ```java // 直接预测：不保存图片以及不渲染结果到Bitmap上 public FaceDetectionResult predict(Bitmap ARGB8888Bitmap)； +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, float confThreshold, float nmsIouThreshold)； // 设置置信度阈值和NMS阈值 // 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 -public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold); -public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float scoreThreshold); // 只渲染 不保存图片 +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float confThreshold, float nmsIouThreshold); +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float confThreshold, float nmsIouThreshold); // 只渲染 不保存图片 ``` - 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 ```java @@ -216,9 +217,10 @@ public boolean init(String modelFile, String paramsFile, RuntimeOption option); ```java // 直接预测：不保存图片以及不渲染结果到Bitmap上 public FaceDetectionResult predict(Bitmap ARGB8888Bitmap)； +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, float confThreshold, float nmsIouThreshold)； // 设置置信度阈值和NMS阈值 // 预测并且可视化：预测结果以及可视化，并将可视化后的图片保存到指定的途径，以及将可视化结果渲染在Bitmap上 -public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold); -public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float scoreThreshold); // 只渲染 不保存图片 +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float confThreshold, float nmsIouThreshold); +public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, boolean rendering, float confThreshold, float nmsIouThreshold); // 只渲染 不保存图片 ``` - 模型资源释放 API：调用 release() API 可以释放模型资源，返回true表示释放成功，false表示失败；调用 initialized() 可以判断模型是否初始化成功，true表示初始化成功，false表示失败。 ```java diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index 1cdc519e053..316b35c034e 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -11,18 +11,18 @@ - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 0b2ca905c7a..2b654e8a8c0 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -1,8 +1,5 @@ package com.baidu.paddle.fastdeploy.app.examples.detection; -import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; -import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; - import android.Manifest; import android.annotation.SuppressLint; import android.app.Activity; @@ -34,6 +31,7 @@ import com.baidu.paddle.fastdeploy.RuntimeOption; import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.examples.facedet.FaceDetMainActivity; import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; import com.baidu.paddle.fastdeploy.app.ui.Utils; @@ -42,13 +40,14 @@ import com.baidu.paddle.fastdeploy.vision.DetectionResult; import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; + import java.io.File; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; -import javax.microedition.khronos.opengles.GL10; - public class DetectionMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { private static final String TAG = DetectionMainActivity.class.getSimpleName(); @@ -122,7 +121,7 @@ public void onClick(View v) { break; case R.id.btn_shutter: TYPE = BTN_SHUTTER; - runOnShutterUiThread(); + shutterAndPauseCamera(); break; case R.id.btn_settings: startActivity(new Intent(DetectionMainActivity.this, DetectionSettingsActivity.class)); @@ -154,32 +153,39 @@ public void onClick(View v) { } } - private void runOnShutterUiThread() { - runOnUiThread(new Runnable() { - @SuppressLint("SetTextI18n") + private void shutterAndPauseCamera() { + new Thread(new Runnable() { + @Override public void run() { try { - Thread.sleep(TIME_SLEEP_INTERVAL * 2); - - svPreview.onPause(); - cameraPageView.setVisibility(View.GONE); - resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultNum + ""); - confidenceSeekbar.setProgress((int) (resultNum * 100)); - if (shutterBitmap != null && !shutterBitmap.isRecycled()) { - resultImage.setImageBitmap(shutterBitmap); - } else { - new AlertDialog.Builder(DetectionMainActivity.this) - .setTitle("Empty Result!") - .setMessage("Current picture is empty, please shutting it again!") - .setCancelable(true) - .show(); - } + // Sleep some times to ensure picture has been correctly shut. + Thread.sleep(TIME_SLEEP_INTERVAL * 2); // 100ms } catch (InterruptedException e) { e.printStackTrace(); } + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + // These code will run in main thread. + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + resultImage.setImageBitmap(shutterBitmap); + } else { + new AlertDialog.Builder(DetectionMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } + }); + } - }); + }).start(); } private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { @@ -191,6 +197,7 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } + SystemClock.sleep(TIME_SLEEP_INTERVAL); // 50ms } } @@ -233,11 +240,9 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { - synchronized (this) { - if (TYPE == BTN_SHUTTER) { - copyBitmapFromCamera(ARGB8888ImageBitmap); - return false; - } + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; } String savedImagePath = ""; diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java new file mode 100644 index 00000000000..a5b5aa839da --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -0,0 +1,465 @@ +package com.baidu.paddle.fastdeploy.app.examples.facedet; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.AlertDialog; +import android.content.DialogInterface; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.PackageManager; +import android.database.Cursor; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.net.Uri; +import android.os.Bundle; +import android.os.SystemClock; +import android.preference.PreferenceManager; +import android.provider.MediaStore; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.util.Log; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; +import android.widget.ImageButton; +import android.widget.ImageView; +import android.widget.SeekBar; +import android.widget.TextView; + +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; +import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; +import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; +import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.FaceDetectionResult; +import com.baidu.paddle.fastdeploy.vision.facedet.SCRFD; + +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; + +import java.io.File; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +public class FaceDetMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { + private static final String TAG = FaceDetMainActivity.class.getSimpleName(); + + CameraSurfaceView svPreview; + TextView tvStatus; + ImageButton btnSwitch; + ImageButton btnShutter; + ImageButton btnSettings; + ImageView realtimeToggleButton; + boolean isRealtimeStatusRunning = false; + ImageView backInPreview; + private ImageView albumSelectButton; + private View cameraPageView; + private ViewGroup resultPageView; + private ImageView resultImage; + private ImageView backInResult; + private SeekBar confidenceSeekbar; + private TextView seekbarText; + private float resultConfThreshold = 1.0f; + private ResultListView detectResultView; + private Bitmap shutterBitmap; + private Bitmap originShutterBitmap; + private Bitmap picBitmap; + private Bitmap originPicBitmap; + + public static final int TYPE_UNKNOWN = -1; + public static final int BTN_SHUTTER = 0; + public static final int ALBUM_SELECT = 1; + public static final int REALTIME_DETECT = 2; + private static int TYPE = REALTIME_DETECT; + + private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; + private static final int INTENT_CODE_PICK_IMAGE = 100; + private static final int TIME_SLEEP_INTERVAL = 50; // ms + + String savedImagePath = "result.jpg"; + int lastFrameIndex = 0; + long lastFrameTime; + + // Call 'init' and 'release' manually later + SCRFD predictor = new SCRFD(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + // Fullscreen + requestWindowFeature(Window.FEATURE_NO_TITLE); + getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); + + setContentView(R.layout.facedet_activity_main); + + // Clear all setting items to avoid app crashing due to the incorrect settings + initSettings(); + + // Init the camera preview and UI components + initView(); + + // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions + if (!checkAllPermissions()) { + requestAllPermissions(); + } + } + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View v) { + switch (v.getId()) { + case R.id.btn_switch: + svPreview.switchCamera(); + break; + case R.id.btn_shutter: + TYPE = BTN_SHUTTER; + shutterAndPauseCamera(); + break; + case R.id.btn_settings: + startActivity(new Intent(FaceDetMainActivity.this, FaceDetSettingsActivity.class)); + break; + case R.id.realtime_toggle_btn: + toggleRealtimeStyle(); + break; + case R.id.back_in_preview: + finish(); + break; + case R.id.album_select: + TYPE = ALBUM_SELECT; + // Judge whether authority has been granted. + if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + // If this permission was requested before the application but the user refused the request, this method will return true. + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_PERMISSION_CODE_STORAGE); + } else { + Intent intent = new Intent(Intent.ACTION_PICK); + intent.setType("image/*"); + startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); + } + break; + case R.id.back_in_result: + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + svPreview.onResume(); + break; + } + } + + private void shutterAndPauseCamera() { + new Thread(new Runnable() { + @Override + public void run() { + try { + // Sleep some times to ensure picture has been correctly shut. + Thread.sleep(TIME_SLEEP_INTERVAL * 2); // 100ms + } catch (InterruptedException e) { + e.printStackTrace(); + } + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + // These codes will run in main thread. + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultConfThreshold + ""); + confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + resultImage.setImageBitmap(shutterBitmap); + } else { + new AlertDialog.Builder(FaceDetMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } + }); + + } + }).start(); + } + + private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { + if (ARGB8888ImageBitmap == null) { + return; + } + if (!ARGB8888ImageBitmap.isRecycled()) { + synchronized (this) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + SystemClock.sleep(TIME_SLEEP_INTERVAL); + } + } + + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (requestCode == INTENT_CODE_PICK_IMAGE) { + if (resultCode == Activity.RESULT_OK) { + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultConfThreshold + ""); + confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + Uri uri = data.getData(); + String path = getRealPathFromURI(this, uri); + picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); + resultImage.setImageBitmap(picBitmap); + } + } + } + + private void toggleRealtimeStyle() { + if (isRealtimeStatusRunning) { + isRealtimeStatusRunning = false; + realtimeToggleButton.setImageResource(R.drawable.realtime_stop_btn); + svPreview.setOnTextureChangedListener(this); + tvStatus.setVisibility(View.VISIBLE); + } else { + isRealtimeStatusRunning = true; + realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); + tvStatus.setVisibility(View.GONE); + svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + return false; + } + }); + } + } + + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; + } + + String savedImagePath = ""; + synchronized (this) { + savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; + } + + boolean modified = false; + FaceDetectionResult result = predictor.predict( + ARGB8888ImageBitmap, true, FaceDetSettingsActivity.scoreThreshold, 0.4f); + modified = result.initialized(); + if (!savedImagePath.isEmpty()) { + synchronized (this) { + FaceDetMainActivity.this.savedImagePath = "result.jpg"; + } + } + lastFrameIndex++; + if (lastFrameIndex >= 30) { + final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + tvStatus.setText(Integer.toString(fps) + "fps"); + } + }); + lastFrameIndex = 0; + lastFrameTime = System.nanoTime(); + } + return modified; + } + + @Override + protected void onResume() { + super.onResume(); + // Reload settings and re-initialize the predictor + checkAndUpdateSettings(); + // Open camera until the permissions have been granted + if (!checkAllPermissions()) { + svPreview.disableCamera(); + } + svPreview.onResume(); + } + + @Override + protected void onPause() { + super.onPause(); + svPreview.onPause(); + } + + @Override + protected void onDestroy() { + if (predictor != null) { + predictor.release(); + } + super.onDestroy(); + } + + public void initView() { + TYPE = REALTIME_DETECT; + svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); + svPreview.setOnTextureChangedListener(this); + tvStatus = (TextView) findViewById(R.id.tv_status); + btnSwitch = (ImageButton) findViewById(R.id.btn_switch); + btnSwitch.setOnClickListener(this); + btnShutter = (ImageButton) findViewById(R.id.btn_shutter); + btnShutter.setOnClickListener(this); + btnSettings = (ImageButton) findViewById(R.id.btn_settings); + btnSettings.setOnClickListener(this); + realtimeToggleButton = findViewById(R.id.realtime_toggle_btn); + realtimeToggleButton.setOnClickListener(this); + backInPreview = findViewById(R.id.back_in_preview); + backInPreview.setOnClickListener(this); + albumSelectButton = findViewById(R.id.album_select); + albumSelectButton.setOnClickListener(this); + cameraPageView = findViewById(R.id.camera_page); + resultPageView = findViewById(R.id.result_page); + resultImage = findViewById(R.id.result_image); + backInResult = findViewById(R.id.back_in_result); + backInResult.setOnClickListener(this); + confidenceSeekbar = findViewById(R.id.confidence_seekbar); + seekbarText = findViewById(R.id.seekbar_text); + detectResultView = findViewById(R.id.result_list_view); + + List results = new ArrayList<>(); + // TODO: add model results from FaceDetectionResult instead of using fake data. + results.add(new BaseResultModel(1, "face", 0.4f)); + results.add(new BaseResultModel(2, "face", 0.6f)); + results.add(new BaseResultModel(3, "face", 1.0f)); + final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.facedet_result_page_item, results); + detectResultView.setAdapter(adapter); + detectResultView.invalidate(); + + confidenceSeekbar.setMax(100); + confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { + @Override + public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { + float resultConfidence = seekBar.getProgress() / 100f; + BigDecimal bd = new BigDecimal(resultConfidence); + resultConfThreshold = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); + seekbarText.setText(resultConfThreshold + ""); + confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + } + + @Override + public void onStartTrackingTouch(SeekBar seekBar) { + + } + + @Override + public void onStopTrackingTouch(SeekBar seekBar) { + runOnUiThread(new Runnable() { + @Override + public void run() { + if (TYPE == ALBUM_SELECT) { + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + if (!picBitmap.isRecycled()) { + predictor.predict(picBitmap, true, resultConfThreshold, 0.4f); + resultImage.setImageBitmap(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + resultConfThreshold = 1.0f; + } else { + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + if (!shutterBitmap.isRecycled()) { + predictor.predict(shutterBitmap, true, resultConfThreshold, 0.4f); + resultImage.setImageBitmap(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + resultConfThreshold = 1.0f; + } + } + }); + } + }); + } + + @SuppressLint("ApplySharedPref") + public void initSettings() { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.clear(); + editor.commit(); + FaceDetSettingsActivity.resetSettings(); + } + + public void checkAndUpdateSettings() { + if (FaceDetSettingsActivity.checkAndUpdateSettings(this)) { + String realModelDir = getCacheDir() + "/" + FaceDetSettingsActivity.modelDir; + Utils.copyDirectoryFromAssets(this, FaceDetSettingsActivity.modelDir, realModelDir); + + String modelFile = realModelDir + "/" + "model.pdmodel"; + String paramsFile = realModelDir + "/" + "model.pdiparams"; + RuntimeOption option = new RuntimeOption(); + option.setCpuThreadNum(FaceDetSettingsActivity.cpuThreadNum); + option.setLitePowerMode(FaceDetSettingsActivity.cpuPowerMode); + if (Boolean.parseBoolean(FaceDetSettingsActivity.enableLiteFp16)) { + option.enableLiteFp16(); + } + predictor.init(modelFile, paramsFile, option); + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, + @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (grantResults[0] != PackageManager.PERMISSION_GRANTED || grantResults[1] != PackageManager.PERMISSION_GRANTED) { + new AlertDialog.Builder(FaceDetMainActivity.this) + .setTitle("Permission denied") + .setMessage("Click to force quit the app, then open Settings->Apps & notifications->Target " + + "App->Permissions to grant all of the permissions.") + .setCancelable(false) + .setPositiveButton("Exit", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + FaceDetMainActivity.this.finish(); + } + }).show(); + } + } + + private void requestAllPermissions() { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.CAMERA}, 0); + } + + private boolean checkAllPermissions() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED + && ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } + +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetSettingsActivity.java new file mode 100644 index 00000000000..ed82c5b59fa --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetSettingsActivity.java @@ -0,0 +1,181 @@ +package com.baidu.paddle.fastdeploy.app.examples.facedet; + +import android.annotation.SuppressLint; +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Bundle; +import android.preference.EditTextPreference; +import android.preference.ListPreference; +import android.preference.PreferenceManager; +import android.support.v7.app.ActionBar; + +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.AppCompatPreferenceActivity; + +import java.util.ArrayList; +import java.util.List; + +public class FaceDetSettingsActivity extends AppCompatPreferenceActivity implements + SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = FaceDetSettingsActivity.class.getSimpleName(); + + static public int selectedModelIdx = -1; + static public String modelDir = ""; + static public int cpuThreadNum = 2; + static public String cpuPowerMode = ""; + static public float scoreThreshold = 0.25f; + static public String enableLiteFp16 = "true"; + + ListPreference lpChoosePreInstalledModel = null; + EditTextPreference etModelDir = null; + ListPreference lpCPUThreadNum = null; + ListPreference lpCPUPowerMode = null; + EditTextPreference etScoreThreshold = null; + ListPreference lpEnableLiteFp16 = null; + + List preInstalledModelDirs = null; + List preInstalledCPUThreadNums = null; + List preInstalledCPUPowerModes = null; + List preInstalledScoreThresholds = null; + List preInstalledEnableLiteFp16s = null; + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + addPreferencesFromResource(R.xml.facedet_setting); + ActionBar supportActionBar = getSupportActionBar(); + if (supportActionBar != null) { + supportActionBar.setDisplayHomeAsUpEnabled(true); + } + + // Initialize pre-installed models + preInstalledModelDirs = new ArrayList(); + preInstalledCPUThreadNums = new ArrayList(); + preInstalledCPUPowerModes = new ArrayList(); + preInstalledScoreThresholds = new ArrayList(); + preInstalledEnableLiteFp16s = new ArrayList(); + preInstalledModelDirs.add(getString(R.string.FACEDET_MODEL_DIR_DEFAULT)); + preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); + preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); + preInstalledScoreThresholds.add(getString(R.string.SCORE_THRESHOLD_FACEDET)); + preInstalledEnableLiteFp16s.add(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + // Setup UI components + lpChoosePreInstalledModel = + (ListPreference) findPreference(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY)); + String[] preInstalledModelNames = new String[preInstalledModelDirs.size()]; + for (int i = 0; i < preInstalledModelDirs.size(); i++) { + preInstalledModelNames[i] = preInstalledModelDirs.get(i).substring(preInstalledModelDirs.get(i).lastIndexOf("/") + 1); + } + lpChoosePreInstalledModel.setEntries(preInstalledModelNames); + lpChoosePreInstalledModel.setEntryValues(preInstalledModelDirs.toArray(new String[preInstalledModelDirs.size()])); + lpCPUThreadNum = (ListPreference) findPreference(getString(R.string.CPU_THREAD_NUM_KEY)); + lpCPUPowerMode = (ListPreference) findPreference(getString(R.string.CPU_POWER_MODE_KEY)); + etModelDir = (EditTextPreference) findPreference(getString(R.string.MODEL_DIR_KEY)); + etModelDir.setTitle("Model dir (SDCard: " + Utils.getSDCardDirectory() + ")"); + etScoreThreshold = (EditTextPreference) findPreference(getString(R.string.SCORE_THRESHOLD_KEY)); + lpEnableLiteFp16 = (ListPreference) findPreference(getString(R.string.ENABLE_LITE_FP16_MODE_KEY)); + } + + @SuppressLint("ApplySharedPref") + private void reloadSettingsAndUpdateUI() { + SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); + + String selected_model_dir = sharedPreferences.getString(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY), + getString(R.string.FACEDET_MODEL_DIR_DEFAULT)); + int selected_model_idx = lpChoosePreInstalledModel.findIndexOfValue(selected_model_dir); + if (selected_model_idx >= 0 && selected_model_idx < preInstalledModelDirs.size() && selected_model_idx != selectedModelIdx) { + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.putString(getString(R.string.MODEL_DIR_KEY), preInstalledModelDirs.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_THREAD_NUM_KEY), preInstalledCPUThreadNums.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_POWER_MODE_KEY), preInstalledCPUPowerModes.get(selected_model_idx)); + editor.putString(getString(R.string.SCORE_THRESHOLD_KEY), preInstalledScoreThresholds.get(selected_model_idx)); + editor.putString(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT), preInstalledEnableLiteFp16s.get(selected_model_idx)); + editor.commit(); + lpChoosePreInstalledModel.setSummary(selected_model_dir); + selectedModelIdx = selected_model_idx; + } + + String model_dir = sharedPreferences.getString(getString(R.string.MODEL_DIR_KEY), + getString(R.string.FACEDET_MODEL_DIR_DEFAULT)); + String cpu_thread_num = sharedPreferences.getString(getString(R.string.CPU_THREAD_NUM_KEY), + getString(R.string.CPU_THREAD_NUM_DEFAULT)); + String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), + getString(R.string.CPU_POWER_MODE_DEFAULT)); + String score_threshold = sharedPreferences.getString(getString(R.string.SCORE_THRESHOLD_KEY), + getString(R.string.SCORE_THRESHOLD_FACEDET)); + String enable_lite_fp16 = sharedPreferences.getString(getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + etModelDir.setSummary(model_dir); + lpCPUThreadNum.setValue(cpu_thread_num); + lpCPUThreadNum.setSummary(cpu_thread_num); + lpCPUPowerMode.setValue(cpu_power_mode); + lpCPUPowerMode.setSummary(cpu_power_mode); + etScoreThreshold.setSummary(score_threshold); + etScoreThreshold.setText(score_threshold); + lpEnableLiteFp16.setValue(enable_lite_fp16); + lpEnableLiteFp16.setSummary(enable_lite_fp16); + + } + + static boolean checkAndUpdateSettings(Context ctx) { + boolean settingsChanged = false; + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(ctx); + + String model_dir = sharedPreferences.getString(ctx.getString(R.string.MODEL_DIR_KEY), + ctx.getString(R.string.FACEDET_MODEL_DIR_DEFAULT)); + settingsChanged |= !modelDir.equalsIgnoreCase(model_dir); + modelDir = model_dir; + + String cpu_thread_num = sharedPreferences.getString(ctx.getString(R.string.CPU_THREAD_NUM_KEY), + ctx.getString(R.string.CPU_THREAD_NUM_DEFAULT)); + settingsChanged |= cpuThreadNum != Integer.parseInt(cpu_thread_num); + cpuThreadNum = Integer.parseInt(cpu_thread_num); + + String cpu_power_mode = sharedPreferences.getString(ctx.getString(R.string.CPU_POWER_MODE_KEY), + ctx.getString(R.string.CPU_POWER_MODE_DEFAULT)); + settingsChanged |= !cpuPowerMode.equalsIgnoreCase(cpu_power_mode); + cpuPowerMode = cpu_power_mode; + + String score_threshold = sharedPreferences.getString(ctx.getString(R.string.SCORE_THRESHOLD_KEY), + ctx.getString(R.string.SCORE_THRESHOLD_FACEDET)); + settingsChanged |= scoreThreshold != Float.parseFloat(score_threshold); + scoreThreshold = Float.parseFloat(score_threshold); + + String enable_lite_fp16 = sharedPreferences.getString(ctx.getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + ctx.getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + settingsChanged |= !enableLiteFp16.equalsIgnoreCase(enable_lite_fp16); + enableLiteFp16 = enable_lite_fp16; + + return settingsChanged; + } + + static void resetSettings() { + selectedModelIdx = -1; + modelDir = ""; + cpuThreadNum = 2; + cpuPowerMode = ""; + scoreThreshold = 0.25f; // confThreshold + enableLiteFp16 = "true"; + } + + @Override + protected void onResume() { + super.onResume(); + getPreferenceScreen().getSharedPreferences().registerOnSharedPreferenceChangeListener(this); + reloadSettingsAndUpdateUI(); + } + + @Override + protected void onPause() { + super.onPause(); + getPreferenceScreen().getSharedPreferences().unregisterOnSharedPreferenceChangeListener(this); + } + + @Override + public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key) { + reloadSettingsAndUpdateUI(); + } +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java new file mode 100644 index 00000000000..1b6f28774f1 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -0,0 +1,4 @@ +package com.baidu.paddle.fastdeploy.app.examples.segmentation; + +public class SegmentationMainActivity { +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java new file mode 100644 index 00000000000..1cbc0f5cca5 --- /dev/null +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java @@ -0,0 +1,4 @@ +package com.baidu.paddle.fastdeploy.app.examples.segmentation; + +public class SegmentationSettingsActivity { +} diff --git a/java/android/app/src/main/res/layout/facedet_activity_main.xml b/java/android/app/src/main/res/layout/facedet_activity_main.xml new file mode 100644 index 00000000000..51b548df4e6 --- /dev/null +++ b/java/android/app/src/main/res/layout/facedet_activity_main.xml @@ -0,0 +1,14 @@ + + + + + + + diff --git a/java/android/app/src/main/res/layout/facedet_camera_page.xml b/java/android/app/src/main/res/layout/facedet_camera_page.xml new file mode 100644 index 00000000000..f6b43ad0b5b --- /dev/null +++ b/java/android/app/src/main/res/layout/facedet_camera_page.xml @@ -0,0 +1,159 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/facedet_result_page.xml b/java/android/app/src/main/res/layout/facedet_result_page.xml new file mode 100644 index 00000000000..958a8594014 --- /dev/null +++ b/java/android/app/src/main/res/layout/facedet_result_page.xml @@ -0,0 +1,160 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/facedet_result_page_item.xml b/java/android/app/src/main/res/layout/facedet_result_page_item.xml new file mode 100644 index 00000000000..6a2b09ebff1 --- /dev/null +++ b/java/android/app/src/main/res/layout/facedet_result_page_item.xml @@ -0,0 +1,26 @@ + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_activity_main.xml b/java/android/app/src/main/res/layout/segmentation_activity_main.xml new file mode 100644 index 00000000000..ae843ab710c --- /dev/null +++ b/java/android/app/src/main/res/layout/segmentation_activity_main.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_camera_page.xml b/java/android/app/src/main/res/layout/segmentation_camera_page.xml new file mode 100644 index 00000000000..ae843ab710c --- /dev/null +++ b/java/android/app/src/main/res/layout/segmentation_camera_page.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_result_page.xml b/java/android/app/src/main/res/layout/segmentation_result_page.xml new file mode 100644 index 00000000000..ae843ab710c --- /dev/null +++ b/java/android/app/src/main/res/layout/segmentation_result_page.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_result_page_item.xml b/java/android/app/src/main/res/layout/segmentation_result_page_item.xml new file mode 100644 index 00000000000..ae843ab710c --- /dev/null +++ b/java/android/app/src/main/res/layout/segmentation_result_page_item.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/values/strings.xml b/java/android/app/src/main/res/values/strings.xml index 9ae5341db30..b5f9cd2e8a2 100644 --- a/java/android/app/src/main/res/values/strings.xml +++ b/java/android/app/src/main/res/values/strings.xml @@ -5,6 +5,8 @@ EasyEdge EasyEdge EasyEdge + EasyEdge + EasyEdge CHOOSE_INSTALLED_MODEL_KEY MODEL_DIR_KEY @@ -18,6 +20,7 @@ LITE_POWER_HIGH 0.4 0.1 + 0.25 true @@ -29,6 +32,10 @@ models/MobileNetV1_x0_25_infer labels/imagenet1k_label_list.txt + + models/scrfd_500m_bnkps_shape320x320_pd + + models/Portrait_PP_HumanSegV2_Lite_256x144_infer 拍照识别 实时识别 diff --git a/java/android/app/src/main/res/xml/facedet_setting.xml b/java/android/app/src/main/res/xml/facedet_setting.xml new file mode 100644 index 00000000000..cab7f937a9c --- /dev/null +++ b/java/android/app/src/main/res/xml/facedet_setting.xml @@ -0,0 +1,41 @@ + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/xml/segmentation_setting.xml b/java/android/app/src/main/res/xml/segmentation_setting.xml new file mode 100644 index 00000000000..624ed13aec2 --- /dev/null +++ b/java/android/app/src/main/res/xml/segmentation_setting.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/scrfd_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/scrfd_jni.cc index f45e554569b..b1b313bf130 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/scrfd_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/scrfd_jni.cc @@ -57,7 +57,8 @@ Java_com_baidu_paddle_fastdeploy_vision_facedet_SCRFD_bindNative( JNIEXPORT jobject JNICALL Java_com_baidu_paddle_fastdeploy_vision_facedet_SCRFD_predictNative( JNIEnv *env, jobject thiz, jlong cxx_context, - jobject argb8888_bitmap, jboolean save_image, + jobject argb8888_bitmap, jfloat conf_threshold, + jfloat nms_iou_threshold, jboolean save_image, jstring save_path, jboolean rendering) { if (cxx_context == 0) { return NULL; @@ -69,7 +70,7 @@ Java_com_baidu_paddle_fastdeploy_vision_facedet_SCRFD_predictNative( auto c_model_ptr = reinterpret_cast(cxx_context); vision::FaceDetectionResult c_result; auto t = fni::GetCurrentTime(); - c_model_ptr->Predict(&c_bgr, &c_result); + c_model_ptr->Predict(&c_bgr, &c_result, conf_threshold, nms_iou_threshold); PERF_TIME_OF_RUNTIME(c_model_ptr, t) if (rendering) { fni::RenderingFaceDetection(env, c_bgr, c_result, argb8888_bitmap, diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/yolov5face_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/yolov5face_jni.cc index c764728ea54..e08db0c9a9d 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/yolov5face_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/facedet/yolov5face_jni.cc @@ -57,7 +57,8 @@ Java_com_baidu_paddle_fastdeploy_vision_facedet_YOLOv5Face_bindNative( JNIEXPORT jobject JNICALL Java_com_baidu_paddle_fastdeploy_vision_facedet_YOLOv5Face_predictNative( JNIEnv *env, jobject thiz, jlong cxx_context, - jobject argb8888_bitmap, jboolean save_image, + jobject argb8888_bitmap, jfloat conf_threshold, + jfloat nms_iou_threshold, jboolean save_image, jstring save_path, jboolean rendering) { if (cxx_context == 0) { return NULL; @@ -69,7 +70,7 @@ Java_com_baidu_paddle_fastdeploy_vision_facedet_YOLOv5Face_predictNative( auto c_model_ptr = reinterpret_cast(cxx_context); vision::FaceDetectionResult c_result; auto t = fni::GetCurrentTime(); - c_model_ptr->Predict(&c_bgr, &c_result); + c_model_ptr->Predict(&c_bgr, &c_result, conf_threshold, nms_iou_threshold); PERF_TIME_OF_RUNTIME(c_model_ptr, t) if (rendering) { fni::RenderingFaceDetection(env, c_bgr, c_result, argb8888_bitmap, diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/SCRFD.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/SCRFD.java index 7a042133db2..200ddaf9509 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/SCRFD.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/SCRFD.java @@ -54,7 +54,7 @@ public FaceDetectionResult predict(Bitmap ARGB8888Bitmap) { } // Only support ARGB8888 bitmap in native now. FaceDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, - false, "", false); + 0.25f, 0.4f, false, "", false); if (result == null) { return new FaceDetectionResult(); } @@ -62,13 +62,30 @@ public FaceDetectionResult predict(Bitmap ARGB8888Bitmap) { } public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, - boolean rendering) { + float confThreshold, + float nmsIouThreshold) { if (mCxxContext == 0) { return new FaceDetectionResult(); } // Only support ARGB8888 bitmap in native now. FaceDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, - false, "", rendering); + confThreshold, nmsIouThreshold, false, "", false); + if (result == null) { + return new FaceDetectionResult(); + } + return result; + } + + public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, + boolean rendering, + float confThreshold, + float nmsIouThreshold) { + if (mCxxContext == 0) { + return new FaceDetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + FaceDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, + confThreshold, nmsIouThreshold, false, "", rendering); if (result == null) { return new FaceDetectionResult(); } @@ -77,15 +94,17 @@ public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, // Predict with image saving and bitmap rendering (will cost more times) public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, - String savedImagePath) { + String savedImagePath, + float confThreshold, + float nmsIouThreshold) { // scoreThreshold is for visualizing only. if (mCxxContext == 0) { return new FaceDetectionResult(); } // Only support ARGB8888 bitmap in native now. FaceDetectionResult result = predictNative( - mCxxContext, ARGB8888Bitmap, true, - savedImagePath, true); + mCxxContext, ARGB8888Bitmap, confThreshold, nmsIouThreshold, + true, savedImagePath, true); if (result == null) { return new FaceDetectionResult(); } @@ -128,6 +147,8 @@ private native long bindNative(String modelFile, // Call prediction from native context with rendering. private native FaceDetectionResult predictNative(long CxxContext, Bitmap ARGB8888Bitmap, + float confThreshold, + float nmsIouThreshold, boolean saveImage, String savePath, boolean rendering); diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/YOLOv5Face.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/YOLOv5Face.java index 49464c8686f..3446f82cd1c 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/YOLOv5Face.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/facedet/YOLOv5Face.java @@ -53,7 +53,7 @@ public FaceDetectionResult predict(Bitmap ARGB8888Bitmap) { } // Only support ARGB8888 bitmap in native now. FaceDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, - false, "", false); + 0.25f, 0.4f, false, "", false); if (result == null) { return new FaceDetectionResult(); } @@ -61,13 +61,30 @@ public FaceDetectionResult predict(Bitmap ARGB8888Bitmap) { } public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, - boolean rendering) { + float confThreshold, + float nmsIouThreshold) { if (mCxxContext == 0) { return new FaceDetectionResult(); } // Only support ARGB8888 bitmap in native now. FaceDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, - false, "", rendering); + confThreshold, nmsIouThreshold, false, "", false); + if (result == null) { + return new FaceDetectionResult(); + } + return result; + } + + public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, + boolean rendering, + float confThreshold, + float nmsIouThreshold) { + if (mCxxContext == 0) { + return new FaceDetectionResult(); + } + // Only support ARGB8888 bitmap in native now. + FaceDetectionResult result = predictNative(mCxxContext, ARGB8888Bitmap, + confThreshold, nmsIouThreshold, false, "", rendering); if (result == null) { return new FaceDetectionResult(); } @@ -76,15 +93,17 @@ public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, // Predict with image saving and bitmap rendering (will cost more times) public FaceDetectionResult predict(Bitmap ARGB8888Bitmap, - String savedImagePath) { + String savedImagePath, + float confThreshold, + float nmsIouThreshold) { // scoreThreshold is for visualizing only. if (mCxxContext == 0) { return new FaceDetectionResult(); } // Only support ARGB8888 bitmap in native now. FaceDetectionResult result = predictNative( - mCxxContext, ARGB8888Bitmap, true, - savedImagePath, true); + mCxxContext, ARGB8888Bitmap, confThreshold, nmsIouThreshold, + true, savedImagePath, true); if (result == null) { return new FaceDetectionResult(); } @@ -127,6 +146,8 @@ private native long bindNative(String modelFile, // Call prediction from native context with rendering. private native FaceDetectionResult predictNative(long CxxContext, Bitmap ARGB8888Bitmap, + float confThreshold, + float nmsIouThreshold, boolean saveImage, String savePath, boolean rendering); From a59fbb732bca5c929f27f64d51cf64ec1a4c10da Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Wed, 16 Nov 2022 22:33:04 +0800 Subject: [PATCH 043/136] [Bug Fix] fix FastDeploy.cmake.in errors for Android (#616) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android * [Android] Run button shutter in sub Ui Thread * [Android] Update CameraSurfaceView * [Android] Update Android SDK usage docs * [Android] Add facedet Android app example * [cmake] fix FastDeploy.cmake.in errors for Android * [Doc] update SetProcLibCpuNumThreads API doc --- FastDeploy.cmake.in | 16 +++---------- cmake/summary.cmake | 1 + .../vision/classification/ppcls/model.cc | 1 - fastdeploy/vision/common/processors/base.h | 3 +-- .../common/processors/color_space_convert.cc | 8 +++---- .../common/processors/color_space_convert.h | 8 +++---- fastdeploy/vision/facedet/contrib/scrfd.cc | 2 +- .../vision/facedet/contrib/yolov5face.cc | 1 + java/android/app/src/main/AndroidManifest.xml | 4 ++-- .../ClassificationMainActivity.java | 20 ++++++++++------ .../detection/DetectionMainActivity.java | 21 ++++++++++------- .../examples/facedet/FaceDetMainActivity.java | 23 +++++++++++-------- 12 files changed, 57 insertions(+), 51 deletions(-) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 1074c86ad99..92439af613e 100755 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -18,8 +18,6 @@ set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@) set(OPENVINO_VERSION @OPENVINO_VERSION@) set(WITH_LITE_STATIC @WITH_LITE_STATIC@) set(WITH_OPENCV_STATIC @WITH_OPENCV_STATIC@) -set(WITH_LITE_FULL_API @WITH_LITE_FULL_API@) -set(WITH_LITE_FP16 @WITH_LITE_FP16@) # set(ENABLE_OPENCV_CUDA @ENABLE_OPENCV_CUDA@) set(OPENCV_FILENAME @OPENCV_FILENAME@) set(OPENVINO_FILENAME @OPENVINO_FILENAME@) @@ -106,15 +104,9 @@ if(ENABLE_LITE_BACKEND) set(LITE_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/${PADDLELITE_FILENAME}) if(ANDROID) if(NOT WITH_LITE_STATIC) - if(WITH_LITE_FULL_API) - add_library(paddle_full_api_shared STATIC IMPORTED GLOBAL) - set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_full_api_shared.so) - list(APPEND FASTDEPLOY_LIBS paddle_full_api_shared) - else() - add_library(paddle_light_api_shared STATIC IMPORTED GLOBAL) - set_property(TARGET paddle_light_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_light_api_shared.so) - list(APPEND FASTDEPLOY_LIBS paddle_light_api_shared) - endif() + add_library(paddle_full_api_shared STATIC IMPORTED GLOBAL) + set_property(TARGET paddle_full_api_shared PROPERTY IMPORTED_LOCATION ${LITE_DIR}/lib/${ANDROID_ABI}/libpaddle_full_api_shared.so) + list(APPEND FASTDEPLOY_LIBS paddle_full_api_shared) endif() else() # Linux/Mac/Win/... @@ -285,8 +277,6 @@ if(ANDROID) message(STATUS " WITH_OPENCV_STATIC: : ${WITH_OPENCV_STATIC}") if(ENABLE_LITE_BACKEND) message(STATUS " WITH_LITE_STATIC : ${WITH_LITE_STATIC}") - message(STATUS " WITH_LITE_FULL_API : ${WITH_LITE_FULL_API}") - message(STATUS " WITH_LITE_FP16 : ${WITH_LITE_FP16}") endif() endif() message(STATUS " DEPENDENCY_LIBS : ${FASTDEPLOY_LIBS}") diff --git a/cmake/summary.cmake b/cmake/summary.cmake index 9a545441f4a..7a729484e3c 100755 --- a/cmake/summary.cmake +++ b/cmake/summary.cmake @@ -33,6 +33,7 @@ function(fastdeploy_summary) message(STATUS " ENABLE_ORT_BACKEND : ${ENABLE_ORT_BACKEND}") message(STATUS " ENABLE_RKNPU2_BACKEND : ${ENABLE_RKNPU2_BACKEND}") message(STATUS " ENABLE_PADDLE_BACKEND : ${ENABLE_PADDLE_BACKEND}") + message(STATUS " ENABLE_LITE_BACKEND : ${ENABLE_LITE_BACKEND}") message(STATUS " ENABLE_POROS_BACKEND : ${ENABLE_POROS_BACKEND}") message(STATUS " ENABLE_TRT_BACKEND : ${ENABLE_TRT_BACKEND}") message(STATUS " ENABLE_OPENVINO_BACKEND : ${ENABLE_OPENVINO_BACKEND}") diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index 94ee0e11946..935336a26fd 100644 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -72,7 +72,6 @@ bool PaddleClasModel::BatchPredict(const std::vector& images, std::vect FDERROR << "Failed to preprocess the input image." << std::endl; return false; } - reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { FDERROR << "Failed to inference by runtime." << std::endl; diff --git a/fastdeploy/vision/common/processors/base.h b/fastdeploy/vision/common/processors/base.h index c833ef8a379..00bd9c82ffd 100644 --- a/fastdeploy/vision/common/processors/base.h +++ b/fastdeploy/vision/common/processors/base.h @@ -32,8 +32,7 @@ FASTDEPLOY_DECL void EnableFlyCV(); /// Disable using FlyCV to process image while deploy vision models. FASTDEPLOY_DECL void DisableFlyCV(); -/*! @brief Set the cpu num threads of ProcLib. The cpu num threads - * of FlyCV and OpenCV is 2 by default. +/*! @brief Set the cpu num threads of ProcLib. */ FASTDEPLOY_DECL void SetProcLibCpuNumThreads(int threads); diff --git a/fastdeploy/vision/common/processors/color_space_convert.cc b/fastdeploy/vision/common/processors/color_space_convert.cc index c486c537940..5d2158063e4 100644 --- a/fastdeploy/vision/common/processors/color_space_convert.cc +++ b/fastdeploy/vision/common/processors/color_space_convert.cc @@ -72,8 +72,8 @@ bool BGR2GRAY::ImplByOpenCV(FDMat* mat) { } #ifdef ENABLE_FLYCV -bool BGR2GRAY::ImplByFalconCV(FDMat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool BGR2GRAY::ImplByFlyCV(FDMat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { FDERROR << "[BGR2GRAY] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl; return false; @@ -94,8 +94,8 @@ bool RGB2GRAY::ImplByOpenCV(FDMat* mat) { } #ifdef ENABLE_FLYCV -bool RGB2GRAY::ImplByFalconCV(FDMat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool RGB2GRAY::ImplByFlyCV(FDMat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { FDERROR << "[RGB2GRAY] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl; return false; diff --git a/fastdeploy/vision/common/processors/color_space_convert.h b/fastdeploy/vision/common/processors/color_space_convert.h index f9cb10a1905..490034868f7 100644 --- a/fastdeploy/vision/common/processors/color_space_convert.h +++ b/fastdeploy/vision/common/processors/color_space_convert.h @@ -45,22 +45,22 @@ class FASTDEPLOY_DECL BGR2GRAY : public Processor { public: bool ImplByOpenCV(FDMat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(FDMat* mat); + bool ImplByFlyCV(FDMat* mat); #endif virtual std::string Name() { return "BGR2GRAY"; } - static bool Run(FDMat* mat, ProcLib lib = ProcLib::OPENCV); + static bool Run(FDMat* mat, ProcLib lib = ProcLib::DEFAULT); }; class FASTDEPLOY_DECL RGB2GRAY : public Processor { public: bool ImplByOpenCV(FDMat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(FDMat* mat); + bool ImplByFlyCV(FDMat* mat); #endif std::string Name() { return "RGB2GRAY"; } - static bool Run(FDMat* mat, ProcLib lib = ProcLib::OPENCV); + static bool Run(FDMat* mat, ProcLib lib = ProcLib::DEFAULT); }; diff --git a/fastdeploy/vision/facedet/contrib/scrfd.cc b/fastdeploy/vision/facedet/contrib/scrfd.cc index dcb590b828e..257db57b399 100644 --- a/fastdeploy/vision/facedet/contrib/scrfd.cc +++ b/fastdeploy/vision/facedet/contrib/scrfd.cc @@ -153,7 +153,7 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output, HWC2CHW::Run(mat); Cast::Run(mat, "float"); } - + // Record output shape of preprocessed image (*im_info)["output_shape"] = {static_cast(mat->Height()), static_cast(mat->Width())}; diff --git a/fastdeploy/vision/facedet/contrib/yolov5face.cc b/fastdeploy/vision/facedet/contrib/yolov5face.cc index ed507363e18..094ab577b7b 100644 --- a/fastdeploy/vision/facedet/contrib/yolov5face.cc +++ b/fastdeploy/vision/facedet/contrib/yolov5face.cc @@ -149,6 +149,7 @@ bool YOLOv5Face::Preprocess( HWC2CHW::Run(mat); Cast::Run(mat, "float"); + mat->ShareWithTensor(output); output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c return true; diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index 316b35c034e..aaad8ac8585 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -15,14 +15,14 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/AppTheme"> - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java index c78f8f39783..14c1c329f28 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java @@ -77,8 +77,8 @@ public class ClassificationMainActivity extends Activity implements View.OnClick private static final int INTENT_CODE_PICK_IMAGE = 100; String savedImagePath = "result.jpg"; - int lastFrameIndex = 0; - long lastFrameTime; + long timeElapsed = 0; + long frameCounter = 0; // Call 'init' and 'release' manually later PaddleClasModel predictor = new PaddleClasModel(); @@ -203,25 +203,31 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { originShutterBitmap = ARGB8888ImageBitmap; } boolean modified = false; + + long tc = System.currentTimeMillis(); ClassifyResult result = predictor.predict( ARGB8888ImageBitmap, true, ClassificationSettingsActivity.scoreThreshold); + + timeElapsed += (System.currentTimeMillis() - tc); + frameCounter++; + modified = result.initialized(); if (!savedImagePath.isEmpty()) { synchronized (this) { ClassificationMainActivity.this.savedImagePath = "result.jpg"; } } - lastFrameIndex++; - if (lastFrameIndex >= 30) { - final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + + if (frameCounter >= 30) { + final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @SuppressLint("SetTextI18n") public void run() { tvStatus.setText(Integer.toString(fps) + "fps"); } }); - lastFrameIndex = 0; - lastFrameTime = System.nanoTime(); + frameCounter = 0; + timeElapsed = 0; } return modified; } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 2b654e8a8c0..e890b42f0a7 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -84,8 +84,8 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private static final int TIME_SLEEP_INTERVAL = 50; // ms String savedImagePath = "result.jpg"; - int lastFrameIndex = 0; - long lastFrameTime; + long timeElapsed = 0; + long frameCounter = 0; // Call 'init' and 'release' manually later PicoDet predictor = new PicoDet(); @@ -159,7 +159,7 @@ private void shutterAndPauseCamera() { public void run() { try { // Sleep some times to ensure picture has been correctly shut. - Thread.sleep(TIME_SLEEP_INTERVAL * 2); // 100ms + Thread.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms } catch (InterruptedException e) { e.printStackTrace(); } @@ -251,25 +251,30 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { } boolean modified = false; + + long tc = System.currentTimeMillis(); DetectionResult result = predictor.predict( ARGB8888ImageBitmap, true, DetectionSettingsActivity.scoreThreshold); + timeElapsed += (System.currentTimeMillis() - tc); + frameCounter++; + modified = result.initialized(); if (!savedImagePath.isEmpty()) { synchronized (this) { DetectionMainActivity.this.savedImagePath = "result.jpg"; } } - lastFrameIndex++; - if (lastFrameIndex >= 30) { - final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + + if (frameCounter >= 30) { + final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @SuppressLint("SetTextI18n") public void run() { tvStatus.setText(Integer.toString(fps) + "fps"); } }); - lastFrameIndex = 0; - lastFrameTime = System.nanoTime(); + frameCounter = 0; + timeElapsed = 0; } return modified; } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index a5b5aa839da..56cc8e81580 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -36,7 +36,6 @@ import com.baidu.paddle.fastdeploy.app.ui.Utils; import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; -import com.baidu.paddle.fastdeploy.vision.DetectionResult; import com.baidu.paddle.fastdeploy.vision.FaceDetectionResult; import com.baidu.paddle.fastdeploy.vision.facedet.SCRFD; @@ -84,8 +83,8 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene private static final int TIME_SLEEP_INTERVAL = 50; // ms String savedImagePath = "result.jpg"; - int lastFrameIndex = 0; - long lastFrameTime; + long timeElapsed = 0; + long frameCounter = 0; // Call 'init' and 'release' manually later SCRFD predictor = new SCRFD(); @@ -159,7 +158,7 @@ private void shutterAndPauseCamera() { public void run() { try { // Sleep some times to ensure picture has been correctly shut. - Thread.sleep(TIME_SLEEP_INTERVAL * 2); // 100ms + Thread.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms } catch (InterruptedException e) { e.printStackTrace(); } @@ -251,25 +250,31 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { } boolean modified = false; + + long tc = System.currentTimeMillis(); FaceDetectionResult result = predictor.predict( ARGB8888ImageBitmap, true, FaceDetSettingsActivity.scoreThreshold, 0.4f); + + timeElapsed += (System.currentTimeMillis() - tc); + frameCounter++; + modified = result.initialized(); if (!savedImagePath.isEmpty()) { synchronized (this) { FaceDetMainActivity.this.savedImagePath = "result.jpg"; } } - lastFrameIndex++; - if (lastFrameIndex >= 30) { - final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + + if (frameCounter >= 30) { + final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @SuppressLint("SetTextI18n") public void run() { tvStatus.setText(Integer.toString(fps) + "fps"); } }); - lastFrameIndex = 0; - lastFrameTime = System.nanoTime(); + frameCounter = 0; + timeElapsed = 0; } return modified; } From 9bd9dcc48bbcaad82528f12946d8422b6b6b5ad9 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 17 Nov 2022 10:09:30 +0800 Subject: [PATCH 044/136] Update README_CN.md --- README_CN.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index 00a6fe1eda7..4f3a8b13608 100644 --- a/README_CN.md +++ b/README_CN.md @@ -3,7 +3,7 @@ ![⚡️FastDeploy](https://user-images.githubusercontent.com/31974251/185771818-5d4423cd-c94c-4a49-9894-bc7a8d1c29d0.png)

- +

@@ -16,6 +16,14 @@

+

+ 使用文档 + | + API文档 + | + 更新日志 +

+ **⚡️FastDeploy**是一款**全场景**、**易用灵活**、**极致高效**的AI推理部署套件。提供📦**开箱即用**的**云边端**部署体验, 支持超过 🔥150+ **Text**, **Vision**, **Speech**和**跨模态**模型，并实现🔚**端到端**的推理性能优化。包括图像分类、物体检测、图像分割、人脸检测、人脸识别、关键点检测、抠图、OCR、NLP、TTS等任务，满足开发者**多场景、多硬件、多平台**的产业部署需求。 | [Image Classification](examples/vision/classification) | [Object Detection](examples/vision/detection) | [Semantic Segmentation](examples/vision/segmentation/paddleseg) | [Potrait Segmentation](examples/vision/segmentation/paddleseg) | From cb5f3a4e52abd364ec845557df21d467c35edbe0 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 17 Nov 2022 10:10:43 +0800 Subject: [PATCH 045/136] Update README_EN.md --- README_EN.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README_EN.md b/README_EN.md index b439b2c1b82..7b3a451533e 100644 --- a/README_EN.md +++ b/README_EN.md @@ -16,7 +16,13 @@ English | [简体中文](README_CN.md)

- +

+ Documents + | + API Docs + | + Release Notes +

**⚡️FastDeploy** is an **Easy-to-use** and **High Performance** AI model deployment toolkit for Cloud, Mobile and Edge with 📦**out-of-the-box and unified experience**, 🔚**end-to-end optimization** for over **🔥150+ Text, Vision, Speech and Cross-modal AI models**. Including image classification, object detection, image segmentation, face detection, face recognition, keypoint detection, matting, OCR, NLP, TTS and other tasks to meet developers' industrial deployment needs for **multi-scenario**, **multi-hardware** and **multi-platform**. From 828b44a0fb215692754320745d0f2c15bd94588d Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Thu, 17 Nov 2022 10:59:47 +0800 Subject: [PATCH 046/136] [Bug Fix] fix CopyShutterBitmap and async camera pause (#617) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android * [Android] Run button shutter in sub Ui Thread * [Android] Update CameraSurfaceView * [Android] Update Android SDK usage docs * [Android] Add facedet Android app example * [cmake] fix FastDeploy.cmake.in errors for Android * [Doc] update SetProcLibCpuNumThreads API doc * [Bug Fix] fix CopyShutterBitmap and async camera pause --- .../detection/DetectionMainActivity.java | 15 +- .../examples/facedet/FaceDetMainActivity.java | 37 ++-- .../res/layout/segmentation_activity_main.xml | 12 +- .../res/layout/segmentation_camera_page.xml | 155 ++++++++++++++++- .../res/layout/segmentation_result_page.xml | 158 +++++++++++++++++- .../layout/segmentation_result_page_item.xml | 26 ++- .../src/main/res/xml/segmentation_setting.xml | 39 ++++- 7 files changed, 413 insertions(+), 29 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index e890b42f0a7..e3da008bb2d 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -38,6 +38,7 @@ import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; @@ -72,6 +73,7 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private Bitmap originShutterBitmap; private Bitmap picBitmap; private Bitmap originPicBitmap; + private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; @@ -148,6 +150,7 @@ public void onClick(View v) { resultPageView.setVisibility(View.GONE); cameraPageView.setVisibility(View.VISIBLE); TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; svPreview.onResume(); break; } @@ -189,7 +192,7 @@ public void run() { } private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { - if (ARGB8888ImageBitmap == null) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { return; } if (!ARGB8888ImageBitmap.isRecycled()) { @@ -197,7 +200,8 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } - SystemClock.sleep(TIME_SLEEP_INTERVAL); // 50ms + SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; } } @@ -253,10 +257,10 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { boolean modified = false; long tc = System.currentTimeMillis(); - DetectionResult result = predictor.predict( - ARGB8888ImageBitmap, true, DetectionSettingsActivity.scoreThreshold); + DetectionResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); - frameCounter++; + + Visualize.visDetection(ARGB8888ImageBitmap, result, DetectionSettingsActivity.scoreThreshold); modified = result.initialized(); if (!savedImagePath.isEmpty()) { @@ -265,6 +269,7 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { } } + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index 56cc8e81580..c7c270faa97 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -37,6 +37,7 @@ import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.FaceDetectionResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.facedet.SCRFD; import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; @@ -65,12 +66,13 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene private ImageView backInResult; private SeekBar confidenceSeekbar; private TextView seekbarText; - private float resultConfThreshold = 1.0f; + private float resultNum = 1.0f; private ResultListView detectResultView; private Bitmap shutterBitmap; private Bitmap originShutterBitmap; private Bitmap picBitmap; private Bitmap originPicBitmap; + private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; @@ -147,6 +149,7 @@ public void onClick(View v) { resultPageView.setVisibility(View.GONE); cameraPageView.setVisibility(View.VISIBLE); TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; svPreview.onResume(); break; } @@ -169,8 +172,8 @@ public void run() { svPreview.onPause(); cameraPageView.setVisibility(View.GONE); resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultConfThreshold + ""); - confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); if (shutterBitmap != null && !shutterBitmap.isRecycled()) { resultImage.setImageBitmap(shutterBitmap); } else { @@ -188,7 +191,7 @@ public void run() { } private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { - if (ARGB8888ImageBitmap == null) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { return; } if (!ARGB8888ImageBitmap.isRecycled()) { @@ -197,6 +200,7 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; } } @@ -207,8 +211,8 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { if (resultCode == Activity.RESULT_OK) { cameraPageView.setVisibility(View.GONE); resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultConfThreshold + ""); - confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); @@ -228,6 +232,7 @@ private void toggleRealtimeStyle() { isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + // Camera is still working but detecting loop is on pause. svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @@ -253,10 +258,11 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { long tc = System.currentTimeMillis(); FaceDetectionResult result = predictor.predict( - ARGB8888ImageBitmap, true, FaceDetSettingsActivity.scoreThreshold, 0.4f); + ARGB8888ImageBitmap, FaceDetSettingsActivity.scoreThreshold, 0.4f); timeElapsed += (System.currentTimeMillis() - tc); - frameCounter++; + + Visualize.visFaceDetection(ARGB8888ImageBitmap, result); modified = result.initialized(); if (!savedImagePath.isEmpty()) { @@ -265,6 +271,7 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { } } + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @@ -346,9 +353,9 @@ public void initView() { public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { float resultConfidence = seekBar.getProgress() / 100f; BigDecimal bd = new BigDecimal(resultConfidence); - resultConfThreshold = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); - seekbarText.setText(resultConfThreshold + ""); - confidenceSeekbar.setProgress((int) (resultConfThreshold * 100)); + resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); } @Override @@ -364,19 +371,19 @@ public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms if (!picBitmap.isRecycled()) { - predictor.predict(picBitmap, true, resultConfThreshold, 0.4f); + predictor.predict(picBitmap, true, resultNum, 0.4f); resultImage.setImageBitmap(picBitmap); picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); } - resultConfThreshold = 1.0f; + resultNum = 1.0f; } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms if (!shutterBitmap.isRecycled()) { - predictor.predict(shutterBitmap, true, resultConfThreshold, 0.4f); + predictor.predict(shutterBitmap, true, resultNum, 0.4f); resultImage.setImageBitmap(shutterBitmap); shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); } - resultConfThreshold = 1.0f; + resultNum = 1.0f; } } }); diff --git a/java/android/app/src/main/res/layout/segmentation_activity_main.xml b/java/android/app/src/main/res/layout/segmentation_activity_main.xml index ae843ab710c..2c5ab44f0ad 100644 --- a/java/android/app/src/main/res/layout/segmentation_activity_main.xml +++ b/java/android/app/src/main/res/layout/segmentation_activity_main.xml @@ -1,6 +1,14 @@ - - \ No newline at end of file + + + + diff --git a/java/android/app/src/main/res/layout/segmentation_camera_page.xml b/java/android/app/src/main/res/layout/segmentation_camera_page.xml index ae843ab710c..baaedf594ea 100644 --- a/java/android/app/src/main/res/layout/segmentation_camera_page.xml +++ b/java/android/app/src/main/res/layout/segmentation_camera_page.xml @@ -1,6 +1,159 @@ + android:layout_height="match_parent" + android:keepScreenOn="true" + tools:context=".segmentation.SegmentationMainActivity"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_result_page.xml b/java/android/app/src/main/res/layout/segmentation_result_page.xml index ae843ab710c..958a8594014 100644 --- a/java/android/app/src/main/res/layout/segmentation_result_page.xml +++ b/java/android/app/src/main/res/layout/segmentation_result_page.xml @@ -1,6 +1,160 @@ - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/layout/segmentation_result_page_item.xml b/java/android/app/src/main/res/layout/segmentation_result_page_item.xml index ae843ab710c..6a2b09ebff1 100644 --- a/java/android/app/src/main/res/layout/segmentation_result_page_item.xml +++ b/java/android/app/src/main/res/layout/segmentation_result_page_item.xml @@ -1,6 +1,26 @@ - + android:layout_height="wrap_content" + android:background="@drawable/result_page_border_section_bk"> - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/java/android/app/src/main/res/xml/segmentation_setting.xml b/java/android/app/src/main/res/xml/segmentation_setting.xml index 624ed13aec2..91082dd8f14 100644 --- a/java/android/app/src/main/res/xml/segmentation_setting.xml +++ b/java/android/app/src/main/res/xml/segmentation_setting.xml @@ -1,4 +1,41 @@ - + + + + + + \ No newline at end of file From 056fdca882ea8b3b235a6aacdd84bc22ab941180 Mon Sep 17 00:00:00 2001 From: felixhjh <852142024@qq.com> Date: Wed, 23 Nov 2022 21:48:37 +0800 Subject: [PATCH 047/136] Fix precision diff for osx-arm64 --- tests/release_task/compare_with_gt.py | 16 ++++++++-------- tests/release_task/cpp_run.sh | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py index ba030999c8e..57258f59b33 100644 --- a/tests/release_task/compare_with_gt.py +++ b/tests/release_task/compare_with_gt.py @@ -3,31 +3,31 @@ diff_score_threshold = { "linux-x64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 1e-4, "boxes_diff": 1e-3 }, "linux-aarch64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 1e-4, "boxes_diff": 1e-3 }, "osx-x86_64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 1e-4, "boxes_diff_ratio": 2e-4, "boxes_diff": 1e-3 }, "osx-arm64": { - "label_diff": 1e-02, - "score_diff": 1e-4, - "boxes_diff_ratio": 2e-4, + "label_diff": 0, + "score_diff": 1e-3, + "boxes_diff_ratio": 5e-4, "boxes_diff": 1e-3 }, "win-x64": { - "label_diff": 1e-02, + "label_diff": 0, "score_diff": 5e-4, "boxes_diff_ratio": 1e-3, "boxes_diff": 1e-3 @@ -133,7 +133,7 @@ def check_result(gt_result, infer_result, args): is_diff = True boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt" boxes_diff_ratio_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio.txt" - boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool" + boxes_diff_ratio_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_ratio_bool.txt" save_numpy_result(boxes_diff_bool_file, boxes_diff > boxes_diff_threshold) save_numpy_result(boxes_diff_ratio_file, boxes_diff_ratio) diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh index 9a160b6e2bc..954a2a38e57 100644 --- a/tests/release_task/cpp_run.sh +++ b/tests/release_task/cpp_run.sh @@ -19,7 +19,7 @@ LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt') LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino') #LINUX_AARCH_CPU_CASE=('ort' 'openvino') LINUX_AARCH_CPU_CASE=('ort') -MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino') +MACOS_INTEL_CPU_CASE=('ort' 'openvino') MACOS_ARM64_CPU_CASE=('default') wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz wget -q https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg From 512f163896aa613c3378df659e689422445afa93 Mon Sep 17 00:00:00 2001 From: heliqi <1101791222@qq.com> Date: Thu, 17 Nov 2022 14:39:31 +0800 Subject: [PATCH 048/136] [Serving]support uie model (#599) * serving support uie model * serving support uie model * delete comment --- examples/text/uie/README.md | 1 + examples/text/uie/serving/README.md | 139 ++++++++++++++++ examples/text/uie/serving/grpc_client.py | 151 +++++++++++++++++ .../text/uie/serving/models/uie/1/model.py | 156 ++++++++++++++++++ .../text/uie/serving/models/uie/config.pbtxt | 46 ++++++ 5 files changed, 493 insertions(+) create mode 100644 examples/text/uie/serving/README.md create mode 100755 examples/text/uie/serving/grpc_client.py create mode 100644 examples/text/uie/serving/models/uie/1/model.py create mode 100644 examples/text/uie/serving/models/uie/config.pbtxt diff --git a/examples/text/uie/README.md b/examples/text/uie/README.md index 592fbca3afb..9dbf8af75fe 100644 --- a/examples/text/uie/README.md +++ b/examples/text/uie/README.md @@ -37,3 +37,4 @@ - [Python部署](python) - [C++部署](cpp) +- [服务化部署](serving) diff --git a/examples/text/uie/serving/README.md b/examples/text/uie/serving/README.md new file mode 100644 index 00000000000..f33a6e41c09 --- /dev/null +++ b/examples/text/uie/serving/README.md @@ -0,0 +1,139 @@ +# UIE 服务化部署示例 + +## 准备模型 + +下载UIE-Base模型(如果有已训练好的模型，跳过此步骤): +```bash +# 下载UIE模型文件和词表，以uie-base模型为例 +wget https://bj.bcebos.com/fastdeploy/models/uie/uie-base.tgz +tar -xvfz uie-base.tgz + +# 将下载的模型移动到模型仓库目录 +mv uie-base/* models/uie/1/ +``` + +模型下载移动好之后，目录结构如下: +``` +models +└── uie + ├── 1 + │   ├── inference.pdiparams + │   ├── inference.pdmodel + │   ├── model.py + │   └── vocab.txt + └── config.pbtxt +``` + +## 拉取并运行镜像 +```bash +# CPU镜像, 仅支持Paddle/ONNX模型在CPU上进行服务化部署，支持的推理后端包括OpenVINO、Paddle Inference和ONNX Runtime +docker pull paddlepaddle/fastdeploy:0.6.0-cpu-only-21.10 + +# GPU 镜像, 支持Paddle/ONNX模型在GPU/CPU上进行服务化部署，支持的推理后端包括OpenVINO、TensorRT、Paddle Inference和ONNX Runtime +docker pull paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 + +# 运行容器.容器名字为 fd_serving, 并挂载当前目录为容器的 /uie_serving 目录 +docker run -it --net=host --name fastdeploy_server --shm-size="1g" -v `pwd`/:/uie_serving paddlepaddle/fastdeploy:0.6.0-gpu-cuda11.4-trt8.4-21.10 bash + +# 启动服务(不设置CUDA_VISIBLE_DEVICES环境变量，会拥有所有GPU卡的调度权限) +CUDA_VISIBLE_DEVICES=0 fastdeployserver --model-repository=/uie_serving/models --backend-config=python,shm-default-byte-size=10485760 +``` + +>> **注意**: 当出现"Address already in use", 请使用`--grpc-port`指定端口号来启动服务，同时更改grpc_client.py中的请求端口号 + +服务启动成功后， 会有以下输出: +``` +...... +I0928 04:51:15.784517 206 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I0928 04:51:15.785177 206 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I0928 04:51:15.826578 206 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +``` + + +## 客户端请求 +客户端请求可以在本地执行脚本请求；也可以在容器中执行。 + +本地执行脚本需要先安装依赖: +``` +pip install grpcio +pip install tritonclient[all] + +# 如果bash无法识别括号，可以使用如下指令安装: +pip install tritonclient\[all\] + +# 发送请求 +python3 grpc_client.py +``` + +发送请求成功后，会返回结果并打印输出: +``` +1. Named Entity Recognition Task-------------- +The extraction schema: ['时间', '选手', '赛事名称'] +text= ['2月8日上午北京冬奥会自由式滑雪女子大跳台决赛中中国选手谷爱凌以188.25分获得金牌！'] +results: +{'时间': {'end': 6, + 'probability': 0.9857379794120789, + 'start': 0, + 'text': '2月8日上午'}, + '赛事名称': {'end': 23, + 'probability': 0.8503087162971497, + 'start': 6, + 'text': '北京冬奥会自由式滑雪女子大跳台决赛'}, + '选手': {'end': 31, + 'probability': 0.8981545567512512, + 'start': 28, + 'text': '谷爱凌'}} +================================================ +text= ['2月7日北京冬奥会短道速滑男子1000米决赛中任子威获得冠军！'] +results: +{'时间': {'end': 4, + 'probability': 0.9921242594718933, + 'start': 0, + 'text': '2月7日'}, + '赛事名称': {'end': 22, + 'probability': 0.8171929121017456, + 'start': 4, + 'text': '北京冬奥会短道速滑男子1000米决赛'}, + '选手': {'end': 26, + 'probability': 0.9821093678474426, + 'start': 23, + 'text': '任子威'}} + +2. Relation Extraction Task +The extraction schema: {'竞赛名称': ['主办方', '承办方', '已举办次数']} +text= ['2022语言与智能技术竞赛由中国中文信息学会和中国计算机学会联合主办，百度公司、中国中文信息学会评测工作委员会和中国计算机学会自然语言处理专委会承办，已连续举办4届，成为全球最热门的中文NLP赛事之一。'] +results: +{'竞赛名称': {'end': 13, + 'probability': 0.7825395464897156, + 'relation': {'主办方': [{'end': 22, + 'probability': 0.8421710729598999, + 'start': 14, + 'text': '中国中文信息学会'}, + {'end': 30, + 'probability': 0.7580801248550415, + 'start': 23, + 'text': '中国计算机学会'}], + '已举办次数': [{'end': 82, + 'probability': 0.4671308398246765, + 'start': 80, + 'text': '4届'}], + '承办方': [{'end': 39, + 'probability': 0.8292703628540039, + 'start': 35, + 'text': '百度公司'}, + {'end': 55, + 'probability': 0.7000497579574585, + 'start': 40, + 'text': '中国中文信息学会评测工作委员会'}, + {'end': 72, + 'probability': 0.6193480491638184, + 'start': 56, + 'text': '中国计算机学会自然语言处理专委会'}]}, + 'start': 0, + 'text': '2022语言与智能技术竞赛'}} +``` + + +## 配置修改 + +当前默认配置在GPU上运行Paddle引擎,如果要在CPU/GPU或其他推理引擎上运行, 需要修改配置，详情请参考[配置文档](../../../../serving/docs/zh_CN/model_configuration.md) diff --git a/examples/text/uie/serving/grpc_client.py b/examples/text/uie/serving/grpc_client.py new file mode 100755 index 00000000000..fbe59f2475c --- /dev/null +++ b/examples/text/uie/serving/grpc_client.py @@ -0,0 +1,151 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import numpy as np +from typing import Optional +import json +import ast + +from pprint import pprint +from tritonclient import utils as client_utils +from tritonclient.grpc import InferenceServerClient, InferInput, InferRequestedOutput, service_pb2_grpc, service_pb2 + +LOGGER = logging.getLogger("run_inference_on_triton") + + +class SyncGRPCTritonRunner: + DEFAULT_MAX_RESP_WAIT_S = 120 + + def __init__( + self, + server_url: str, + model_name: str, + model_version: str, + *, + verbose=False, + resp_wait_s: Optional[float]=None, ): + self._server_url = server_url + self._model_name = model_name + self._model_version = model_version + self._verbose = verbose + self._response_wait_t = self.DEFAULT_MAX_RESP_WAIT_S if resp_wait_s is None else resp_wait_s + + self._client = InferenceServerClient( + self._server_url, verbose=self._verbose) + error = self._verify_triton_state(self._client) + if error: + raise RuntimeError( + f"Could not communicate to Triton Server: {error}") + + LOGGER.debug( + f"Triton server {self._server_url} and model {self._model_name}:{self._model_version} " + f"are up and ready!") + + model_config = self._client.get_model_config(self._model_name, + self._model_version) + model_metadata = self._client.get_model_metadata(self._model_name, + self._model_version) + LOGGER.info(f"Model config {model_config}") + LOGGER.info(f"Model metadata {model_metadata}") + + self._inputs = {tm.name: tm for tm in model_metadata.inputs} + self._input_names = list(self._inputs) + self._outputs = {tm.name: tm for tm in model_metadata.outputs} + self._output_names = list(self._outputs) + self._outputs_req = [ + InferRequestedOutput(name) for name in self._outputs + ] + + def Run(self, inputs): + """ + Args: + inputs: list, Each value corresponds to an input name of self._input_names + Returns: + results: dict, {name : numpy.array} + """ + infer_inputs = [] + for idx, data in enumerate(inputs): + data = json.dumps(data) + data = np.array([[data], ], dtype=np.object_) + infer_input = InferInput(self._input_names[idx], data.shape, + "BYTES") + infer_input.set_data_from_numpy(data) + infer_inputs.append(infer_input) + + results = self._client.infer( + model_name=self._model_name, + model_version=self._model_version, + inputs=infer_inputs, + outputs=self._outputs_req, + client_timeout=self._response_wait_t, ) + # only one output + results = results.as_numpy(self._output_names[0]) + return results + + def _verify_triton_state(self, triton_client): + if not triton_client.is_server_live(): + return f"Triton server {self._server_url} is not live" + elif not triton_client.is_server_ready(): + return f"Triton server {self._server_url} is not ready" + elif not triton_client.is_model_ready(self._model_name, + self._model_version): + return f"Model {self._model_name}:{self._model_version} is not ready" + return None + + +if __name__ == "__main__": + model_name = "uie" + model_version = "1" + url = "localhost:8001" + runner = SyncGRPCTritonRunner(url, model_name, model_version) + + print("1. Named Entity Recognition Task--------------") + schema = ["时间", "选手", "赛事名称"] + print(f"The extraction schema: {schema}") + text = ["2月8日上午北京冬奥会自由式滑雪女子大跳台决赛中中国选手谷爱凌以188.25分获得金牌！"] + print("text=", text) + print("results:") + results = runner.Run([text, schema]) + for result in results: + result = result.decode('utf-8') + result = ast.literal_eval(result) + pprint(result) + + print("================================================") + text = ["2月7日北京冬奥会短道速滑男子1000米决赛中任子威获得冠军！"] + print("text=", text) + # while schema is empty, use the schema set up last time. + schema = [] + results = runner.Run([text, schema]) + print("results:") + for result in results: + result = result.decode('utf-8') + result = ast.literal_eval(result) + pprint(result) + + print("\n2. Relation Extraction Task") + schema = {"竞赛名称": ["主办方", "承办方", "已举办次数"]} + print(f"The extraction schema: {schema}") + text = [ + "2022语言与智能技术竞赛由中国中文信息学会和中国计算机学会联合主办，百度公司、中国中文信息学会评测工作" + "委员会和中国计算机学会自然语言处理专委会承办，已连续举办4届，成为全球最热门的中文NLP赛事之一。" + ] + print("text=", text) + print("results:") + results = runner.Run([text, schema]) + for result in results: + result = result.decode('utf-8') + result = ast.literal_eval(result) + pprint(result) diff --git a/examples/text/uie/serving/models/uie/1/model.py b/examples/text/uie/serving/models/uie/1/model.py new file mode 100644 index 00000000000..42495faf263 --- /dev/null +++ b/examples/text/uie/serving/models/uie/1/model.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import numpy as np +import os + +import fastdeploy +from fastdeploy.text import UIEModel, SchemaLanguage + +# triton_python_backend_utils is available in every Triton Python model. You +# need to use this module to create inference requests and responses. It also +# contains some utility functions for extracting information from model_config +# and converting Triton input/output types to numpy types. +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + """Your Python model must use the same class name. Every Python model + that is created must have "TritonPythonModel" as the class name. + """ + + def initialize(self, args): + """`initialize` is called only once when the model is being loaded. + Implementing `initialize` function is optional. This function allows + the model to intialize any state associated with this model. + Parameters + ---------- + args : dict + Both keys and values are strings. The dictionary keys and values are: + * model_config: A JSON string containing the model configuration + * model_instance_kind: A string containing model instance kind + * model_instance_device_id: A string containing model instance device ID + * model_repository: Model repository path + * model_version: Model version + * model_name: Model name + """ + # You must parse model_config. JSON string is not parsed here + self.model_config = json.loads(args['model_config']) + print("model_config:", self.model_config) + + self.input_names = [] + for input_config in self.model_config["input"]: + self.input_names.append(input_config["name"]) + print("input:", self.input_names) + + self.output_names = [] + self.output_dtype = [] + for output_config in self.model_config["output"]: + self.output_names.append(output_config["name"]) + dtype = pb_utils.triton_string_to_numpy(output_config["data_type"]) + self.output_dtype.append(dtype) + print("output:", self.output_names) + + #Init fastdeploy.RuntimeOption + runtime_option = fastdeploy.RuntimeOption() + options = None + if (args['model_instance_kind'] == 'GPU'): + runtime_option.use_gpu(int(args['model_instance_device_id'])) + options = self.model_config['optimization'][ + 'execution_accelerators']['gpu_execution_accelerator'] + else: + runtime_option.use_cpu() + options = self.model_config['optimization'][ + 'execution_accelerators']['cpu_execution_accelerator'] + + for option in options: + if option['name'] == 'paddle': + runtime_option.use_paddle_backend() + elif option['name'] == 'onnxruntime': + runtime_option.use_ort_backend() + elif option['name'] == 'openvino': + runtime_option.use_openvino_backend() + + if option['parameters']: + if 'cpu_threads' in option['parameters']: + runtime_option.set_cpu_thread_num( + int(option['parameters']['cpu_threads'])) + + model_path = os.path.abspath(os.path.dirname( + __file__)) + "/inference.pdmodel" + param_path = os.path.abspath(os.path.dirname( + __file__)) + "/inference.pdiparams" + vocab_path = os.path.abspath(os.path.dirname(__file__)) + "/vocab.txt" + schema = [] + # init UIE model + self.uie_model_ = UIEModel( + model_path, + param_path, + vocab_path, + position_prob=0.5, + max_length=128, + schema=schema, + runtime_option=runtime_option, + schema_language=SchemaLanguage.ZH) + + def execute(self, requests): + """`execute` must be implemented in every Python model. `execute` + function receives a list of pb_utils.InferenceRequest as the only + argument. This function is called when an inference is requested + for this model. Depending on the batching configuration (e.g. Dynamic + Batching) used, `requests` may contain multiple requests. Every + Python model, must create one pb_utils.InferenceResponse for every + pb_utils.InferenceRequest in `requests`. If there is an error, you can + set the error argument when creating a pb_utils.InferenceResponse. + Parameters + ---------- + requests : list + A list of pb_utils.InferenceRequest + Returns + ------- + list + A list of pb_utils.InferenceResponse. The length of this list must + be the same as `requests` + """ + responses = [] + for request in requests: + texts = pb_utils.get_input_tensor_by_name(request, + self.input_names[0]) + schema = pb_utils.get_input_tensor_by_name(request, + self.input_names[1]) + texts = texts.as_numpy() + schema = schema.as_numpy() + # not support batch predict + texts = json.loads(texts[0][0]) + schema = json.loads(schema[0][0]) + + if schema: + self.uie_model_.set_schema(schema) + results = self.uie_model_.predict(texts, return_dict=True) + + results = np.array(results, dtype=np.object) + out_tensor = pb_utils.Tensor(self.output_names[0], results) + inference_response = pb_utils.InferenceResponse( + output_tensors=[out_tensor, ]) + responses.append(inference_response) + return responses + + def finalize(self): + """`finalize` is called only once when the model is being unloaded. + Implementing `finalize` function is optional. This function allows + the model to perform any necessary clean ups before exit. + """ + print('Cleaning up...') diff --git a/examples/text/uie/serving/models/uie/config.pbtxt b/examples/text/uie/serving/models/uie/config.pbtxt new file mode 100644 index 00000000000..1519b392b3d --- /dev/null +++ b/examples/text/uie/serving/models/uie/config.pbtxt @@ -0,0 +1,46 @@ +name: "uie" +backend: "python" +max_batch_size: 1 + +input [ + { + name: "INPUT_0" + data_type: TYPE_STRING + dims: [ 1 ] + }, + { + name: "INPUT_1" + data_type: TYPE_STRING + dims: [ 1 ] + } +] + +output [ + { + name: "OUTPUT_0" + data_type: TYPE_STRING + dims: [ 1 ] + } +] + +instance_group [ + { + count: 1 + # Use GPU, CPU inference option is:KIND_CPU + kind: KIND_GPU + # The instance is deployed on the 0th GPU card + gpus: [0] + } +] + +optimization { + execution_accelerators { + gpu_execution_accelerator : [ + { + # use paddle backend + name: "paddle" + parameters { key: "cpu_threads" value: "12" } + } + ] + } +} From 42eeec6e92058139eedb068a1b9de471597cfbce Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 17 Nov 2022 15:35:27 +0800 Subject: [PATCH 049/136] Update README_EN.md --- docs/README_EN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/README_EN.md b/docs/README_EN.md index de2e2bef842..e995e302457 100644 --- a/docs/README_EN.md +++ b/docs/README_EN.md @@ -18,7 +18,7 @@ - [A Quick Start on Runtime Python](en/quick_start/runtime/python.md) - [A Quick Start on Runtime C++](en/quick_start/runtime/cpp.md) -## API (To be continued) +## API - [Python API](https://baidu-paddle.github.io/fastdeploy-api/python/html/) - [C++ API](https://baidu-paddle.github.io/fastdeploy-api/cpp/html/) @@ -31,9 +31,9 @@ - [1. How to Change Inference Backends](en/faq/how_to_change_backend.md) - [2. How to Use FastDeploy C++ SDK on Windows Platform](en/faq/use_sdk_on_windows.md) -- [3. How to Use FastDeploy C++ SDK on Android Platform](en/faq/use_sdk_on_android.md)(To be Continued) +- [3. How to Use FastDeploy C++ SDK on Android Platform](en/faq/use_sdk_on_android.md) - [4. Tricks of TensorRT](en/faq/tensorrt_tricks.md) -- [5. How to Develop a New Model](en/faq/develop_a_new_model.md)(To be Continued) +- [5. How to Develop a New Model](en/faq/develop_a_new_model.md) ## More FastDeploy Deployment Module From 4c4397ab454d5d0bbdb9717c98fa43b1765f5118 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Thu, 17 Nov 2022 16:12:07 +0800 Subject: [PATCH 050/136] [Android] Add PaddleSeg Android app example (#620) [Android] Add PaddleSeg Android app example --- java/android/app/src/main/AndroidManifest.xml | 6 +- .../SegmentationMainActivity.java | 477 +++++++++++++++++- .../SegmentationSettingsActivity.java | 164 +++++- .../app/ui/view/CameraSurfaceView.java | 4 +- .../app/src/main/res/values/strings.xml | 2 +- .../src/main/res/xml/segmentation_setting.xml | 4 - .../cpp/fastdeploy_jni/vision/results_jni.cc | 56 +- .../segmentation/paddleseg_model_jni.cc | 1 - .../fastdeploy/vision/SegmentationResult.java | 4 +- .../vision/segmentation/PaddleSegModel.java | 1 - 10 files changed, 675 insertions(+), 44 deletions(-) diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index aaad8ac8585..bb2df4ce4f4 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -11,18 +11,18 @@ - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 1b6f28774f1..404252814de 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -1,4 +1,479 @@ package com.baidu.paddle.fastdeploy.app.examples.segmentation; -public class SegmentationMainActivity { +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.AlertDialog; +import android.content.DialogInterface; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.net.Uri; +import android.os.Bundle; +import android.os.SystemClock; +import android.preference.PreferenceManager; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.util.Log; +import android.view.View; +import android.view.ViewGroup; +import android.view.Window; +import android.view.WindowManager; +import android.widget.ImageButton; +import android.widget.ImageView; +import android.widget.SeekBar; +import android.widget.TextView; + +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; +import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; +import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; +import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; +import com.baidu.paddle.fastdeploy.vision.SegmentationResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; +import com.baidu.paddle.fastdeploy.vision.segmentation.PaddleSegModel; + +import java.io.File; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; + +public class SegmentationMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { + private static final String TAG = SegmentationMainActivity.class.getSimpleName(); + + CameraSurfaceView svPreview; + TextView tvStatus; + ImageButton btnSwitch; + ImageButton btnShutter; + ImageButton btnSettings; + ImageView realtimeToggleButton; + boolean isRealtimeStatusRunning = false; + ImageView backInPreview; + private ImageView albumSelectButton; + private View cameraPageView; + private ViewGroup resultPageView; + private ImageView resultImage; + private ImageView backInResult; + private SeekBar confidenceSeekbar; + private TextView seekbarText; + private float resultNum = 1.0f; + private ResultListView detectResultView; + private Bitmap shutterBitmap; + private Bitmap originShutterBitmap; + private Bitmap picBitmap; + private Bitmap originPicBitmap; + private boolean isShutterBitmapCopied = false; + + public static final int TYPE_UNKNOWN = -1; + public static final int BTN_SHUTTER = 0; + public static final int ALBUM_SELECT = 1; + public static final int REALTIME_DETECT = 2; + private static int TYPE = REALTIME_DETECT; + + private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; + private static final int INTENT_CODE_PICK_IMAGE = 100; + private static final int TIME_SLEEP_INTERVAL = 50; // ms + + String savedImagePath = "result.jpg"; + long timeElapsed = 0; + long frameCounter = 0; + + // Call 'init' and 'release' manually later + PaddleSegModel predictor = new PaddleSegModel(); + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + // Fullscreen + requestWindowFeature(Window.FEATURE_NO_TITLE); + getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN, WindowManager.LayoutParams.FLAG_FULLSCREEN); + + setContentView(R.layout.segmentation_activity_main); + + // Clear all setting items to avoid app crashing due to the incorrect settings + initSettings(); + + // Init the camera preview and UI components + initView(); + + // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions + if (!checkAllPermissions()) { + requestAllPermissions(); + } + } + + @SuppressLint("NonConstantResourceId") + @Override + public void onClick(View v) { + switch (v.getId()) { + case R.id.btn_switch: + svPreview.switchCamera(); + break; + case R.id.btn_shutter: + TYPE = BTN_SHUTTER; + shutterAndPauseCamera(); + break; + case R.id.btn_settings: + startActivity(new Intent(SegmentationMainActivity.this, SegmentationSettingsActivity.class)); + break; + case R.id.realtime_toggle_btn: + toggleRealtimeStyle(); + break; + case R.id.back_in_preview: + finish(); + break; + case R.id.album_select: + TYPE = ALBUM_SELECT; + // Judge whether authority has been granted. + if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + // If this permission was requested before the application but the user refused the request, this method will return true. + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_PERMISSION_CODE_STORAGE); + } else { + Intent intent = new Intent(Intent.ACTION_PICK); + intent.setType("image/*"); + startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); + } + break; + case R.id.back_in_result: + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + break; + } + } + + private void shutterAndPauseCamera() { + new Thread(new Runnable() { + @Override + public void run() { + try { + // Sleep some times to ensure picture has been correctly shut. + Thread.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + } catch (InterruptedException e) { + e.printStackTrace(); + } + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + // These codes will run in main thread. + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + resultImage.setImageBitmap(shutterBitmap); + } else { + new AlertDialog.Builder(SegmentationMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } + }); + } + }).start(); + } + + private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { + return; + } + if (!ARGB8888ImageBitmap.isRecycled()) { + synchronized (this) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; + } + } + + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (requestCode == INTENT_CODE_PICK_IMAGE) { + if (resultCode == Activity.RESULT_OK) { + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + Uri uri = data.getData(); + String path = getRealPathFromURI(this, uri); + picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); + resultImage.setImageBitmap(picBitmap); + } + } + } + + private void toggleRealtimeStyle() { + if (isRealtimeStatusRunning) { + isRealtimeStatusRunning = false; + realtimeToggleButton.setImageResource(R.drawable.realtime_stop_btn); + svPreview.setOnTextureChangedListener(this); + tvStatus.setVisibility(View.VISIBLE); + } else { + isRealtimeStatusRunning = true; + realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); + tvStatus.setVisibility(View.GONE); + // Camera is still working but detecting loop is on pause. + svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + return false; + } + }); + } + } + + @Override + public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; + } + + String savedImagePath = ""; + synchronized (this) { + savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; + } + + boolean modified = false; + + long tc = System.currentTimeMillis(); + SegmentationResult result = predictor.predict(ARGB8888ImageBitmap); + timeElapsed += (System.currentTimeMillis() - tc); + + Visualize.visSegmentation(ARGB8888ImageBitmap, result); + + modified = result.initialized(); + if (!savedImagePath.isEmpty()) { + synchronized (this) { + SegmentationMainActivity.this.savedImagePath = "result.jpg"; + } + } + + frameCounter++; + if (frameCounter >= 30) { + final int fps = (int) (1000 / (timeElapsed / 30)); + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + tvStatus.setText(Integer.toString(fps) + "fps"); + } + }); + frameCounter = 0; + timeElapsed = 0; + } + return modified; + } + + @Override + protected void onResume() { + super.onResume(); + // Reload settings and re-initialize the predictor + checkAndUpdateSettings(); + // Open camera until the permissions have been granted + if (!checkAllPermissions()) { + svPreview.disableCamera(); + } + svPreview.onResume(); + } + + @Override + protected void onPause() { + super.onPause(); + svPreview.onPause(); + } + + @Override + protected void onDestroy() { + if (predictor != null) { + predictor.release(); + } + super.onDestroy(); + } + + public void initView() { + TYPE = REALTIME_DETECT; + // For front face camera and human seg, the smaller width and height + // may get both better result and performance. EXPECTED_PREVIEW_HEIGHT + // should be 'width' and EXPECTED_PREVIEW_WIDTH should be 'height' if + // your camera display orientation is rotate (degree == 90 || degree == 270). + // The transformation will auto process in camera. + CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 360; + CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; + svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); + svPreview.setOnTextureChangedListener(this); + svPreview.switchCamera(); // switch to front camera for human seg + tvStatus = (TextView) findViewById(R.id.tv_status); + btnSwitch = (ImageButton) findViewById(R.id.btn_switch); + btnSwitch.setOnClickListener(this); + btnShutter = (ImageButton) findViewById(R.id.btn_shutter); + btnShutter.setOnClickListener(this); + btnSettings = (ImageButton) findViewById(R.id.btn_settings); + btnSettings.setOnClickListener(this); + realtimeToggleButton = findViewById(R.id.realtime_toggle_btn); + realtimeToggleButton.setOnClickListener(this); + backInPreview = findViewById(R.id.back_in_preview); + backInPreview.setOnClickListener(this); + albumSelectButton = findViewById(R.id.album_select); + albumSelectButton.setOnClickListener(this); + cameraPageView = findViewById(R.id.camera_page); + resultPageView = findViewById(R.id.result_page); + resultImage = findViewById(R.id.result_image); + backInResult = findViewById(R.id.back_in_result); + backInResult.setOnClickListener(this); + confidenceSeekbar = findViewById(R.id.confidence_seekbar); + seekbarText = findViewById(R.id.seekbar_text); + detectResultView = findViewById(R.id.result_list_view); + + List results = new ArrayList<>(); + // TODO: add model results from SegmentationResult instead of using fake data. + results.add(new BaseResultModel(1, "human", 1.0f)); + results.add(new BaseResultModel(2, "human", 1.0f)); + results.add(new BaseResultModel(3, "human", 1.0f)); + final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.facedet_result_page_item, results); + detectResultView.setAdapter(adapter); + detectResultView.invalidate(); + + confidenceSeekbar.setMax(100); + confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { + @Override + public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { + float resultConfidence = seekBar.getProgress() / 100f; + BigDecimal bd = new BigDecimal(resultConfidence); + resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + } + + @Override + public void onStartTrackingTouch(SeekBar seekBar) { + + } + + @Override + public void onStopTrackingTouch(SeekBar seekBar) { + runOnUiThread(new Runnable() { + @Override + public void run() { + if (TYPE == ALBUM_SELECT) { + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + if (!picBitmap.isRecycled()) { + predictor.predict(picBitmap, true, resultNum); + resultImage.setImageBitmap(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + resultNum = 1.0f; + } else { + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + if (!shutterBitmap.isRecycled()) { + predictor.predict(shutterBitmap, true, resultNum); + resultImage.setImageBitmap(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + resultNum = 1.0f; + } + } + }); + } + }); + } + + @SuppressLint("ApplySharedPref") + public void initSettings() { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.clear(); + editor.commit(); + SegmentationSettingsActivity.resetSettings(); + } + + public void checkAndUpdateSettings() { + if (SegmentationSettingsActivity.checkAndUpdateSettings(this)) { + String realModelDir = getCacheDir() + "/" + SegmentationSettingsActivity.modelDir; + Utils.copyDirectoryFromAssets(this, SegmentationSettingsActivity.modelDir, realModelDir); + + String modelFile = realModelDir + "/" + "model.pdmodel"; + String paramsFile = realModelDir + "/" + "model.pdiparams"; + String configFile = realModelDir + "/" + "deploy.yaml"; + RuntimeOption option = new RuntimeOption(); + option.setCpuThreadNum(SegmentationSettingsActivity.cpuThreadNum); + option.setLitePowerMode(SegmentationSettingsActivity.cpuPowerMode); + if (Boolean.parseBoolean(SegmentationSettingsActivity.enableLiteFp16)) { + option.enableLiteFp16(); + } + predictor.init(modelFile, paramsFile, configFile, option); + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, + @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (grantResults[0] != PackageManager.PERMISSION_GRANTED || grantResults[1] != PackageManager.PERMISSION_GRANTED) { + new AlertDialog.Builder(SegmentationMainActivity.this) + .setTitle("Permission denied") + .setMessage("Click to force quit the app, then open Settings->Apps & notifications->Target " + + "App->Permissions to grant all of the permissions.") + .setCancelable(false) + .setPositiveButton("Exit", new DialogInterface.OnClickListener() { + @Override + public void onClick(DialogInterface dialog, int which) { + SegmentationMainActivity.this.finish(); + } + }).show(); + } + } + + private void requestAllPermissions() { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.CAMERA}, 0); + } + + private boolean checkAllPermissions() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED + && ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } } + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java index 1cbc0f5cca5..866c2c712f3 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationSettingsActivity.java @@ -1,4 +1,166 @@ package com.baidu.paddle.fastdeploy.app.examples.segmentation; -public class SegmentationSettingsActivity { +import android.annotation.SuppressLint; +import android.content.Context; +import android.content.SharedPreferences; +import android.os.Bundle; +import android.preference.EditTextPreference; +import android.preference.ListPreference; +import android.preference.PreferenceManager; +import android.support.v7.app.ActionBar; + +import com.baidu.paddle.fastdeploy.app.examples.R; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.AppCompatPreferenceActivity; + +import java.util.ArrayList; +import java.util.List; + +public class SegmentationSettingsActivity extends AppCompatPreferenceActivity implements + SharedPreferences.OnSharedPreferenceChangeListener { + private static final String TAG = SegmentationSettingsActivity.class.getSimpleName(); + + static public int selectedModelIdx = -1; + static public String modelDir = ""; + static public int cpuThreadNum = 2; + static public String cpuPowerMode = ""; + static public String enableLiteFp16 = "true"; + + ListPreference lpChoosePreInstalledModel = null; + EditTextPreference etModelDir = null; + ListPreference lpCPUThreadNum = null; + ListPreference lpCPUPowerMode = null; + ListPreference lpEnableLiteFp16 = null; + + List preInstalledModelDirs = null; + List preInstalledCPUThreadNums = null; + List preInstalledCPUPowerModes = null; + List preInstalledEnableLiteFp16s = null; + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + addPreferencesFromResource(R.xml.segmentation_setting); + ActionBar supportActionBar = getSupportActionBar(); + if (supportActionBar != null) { + supportActionBar.setDisplayHomeAsUpEnabled(true); + } + + // Initialize pre-installed models + preInstalledModelDirs = new ArrayList(); + preInstalledCPUThreadNums = new ArrayList(); + preInstalledCPUPowerModes = new ArrayList(); + preInstalledEnableLiteFp16s = new ArrayList(); + preInstalledModelDirs.add(getString(R.string.SEGMENTATION_MODEL_DIR_DEFAULT)); + preInstalledCPUThreadNums.add(getString(R.string.CPU_THREAD_NUM_DEFAULT)); + preInstalledCPUPowerModes.add(getString(R.string.CPU_POWER_MODE_DEFAULT)); + preInstalledEnableLiteFp16s.add(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + // Setup UI components + lpChoosePreInstalledModel = + (ListPreference) findPreference(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY)); + String[] preInstalledModelNames = new String[preInstalledModelDirs.size()]; + for (int i = 0; i < preInstalledModelDirs.size(); i++) { + preInstalledModelNames[i] = preInstalledModelDirs.get(i).substring(preInstalledModelDirs.get(i).lastIndexOf("/") + 1); + } + lpChoosePreInstalledModel.setEntries(preInstalledModelNames); + lpChoosePreInstalledModel.setEntryValues(preInstalledModelDirs.toArray(new String[preInstalledModelDirs.size()])); + lpCPUThreadNum = (ListPreference) findPreference(getString(R.string.CPU_THREAD_NUM_KEY)); + lpCPUPowerMode = (ListPreference) findPreference(getString(R.string.CPU_POWER_MODE_KEY)); + etModelDir = (EditTextPreference) findPreference(getString(R.string.MODEL_DIR_KEY)); + etModelDir.setTitle("Model dir (SDCard: " + Utils.getSDCardDirectory() + ")"); + lpEnableLiteFp16 = (ListPreference) findPreference(getString(R.string.ENABLE_LITE_FP16_MODE_KEY)); + } + + @SuppressLint("ApplySharedPref") + private void reloadSettingsAndUpdateUI() { + SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); + + String selected_model_dir = sharedPreferences.getString(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY), + getString(R.string.SEGMENTATION_MODEL_DIR_DEFAULT)); + int selected_model_idx = lpChoosePreInstalledModel.findIndexOfValue(selected_model_dir); + if (selected_model_idx >= 0 && selected_model_idx < preInstalledModelDirs.size() && selected_model_idx != selectedModelIdx) { + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.putString(getString(R.string.MODEL_DIR_KEY), preInstalledModelDirs.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_THREAD_NUM_KEY), preInstalledCPUThreadNums.get(selected_model_idx)); + editor.putString(getString(R.string.CPU_POWER_MODE_KEY), preInstalledCPUPowerModes.get(selected_model_idx)); + editor.putString(getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT), preInstalledEnableLiteFp16s.get(selected_model_idx)); + editor.commit(); + lpChoosePreInstalledModel.setSummary(selected_model_dir); + selectedModelIdx = selected_model_idx; + } + + String model_dir = sharedPreferences.getString(getString(R.string.MODEL_DIR_KEY), + getString(R.string.SEGMENTATION_MODEL_DIR_DEFAULT)); + String cpu_thread_num = sharedPreferences.getString(getString(R.string.CPU_THREAD_NUM_KEY), + getString(R.string.CPU_THREAD_NUM_DEFAULT)); + String cpu_power_mode = sharedPreferences.getString(getString(R.string.CPU_POWER_MODE_KEY), + getString(R.string.CPU_POWER_MODE_DEFAULT)); + String score_threshold = sharedPreferences.getString(getString(R.string.SCORE_THRESHOLD_KEY), + getString(R.string.SCORE_THRESHOLD_FACEDET)); + String enable_lite_fp16 = sharedPreferences.getString(getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + + etModelDir.setSummary(model_dir); + lpCPUThreadNum.setValue(cpu_thread_num); + lpCPUThreadNum.setSummary(cpu_thread_num); + lpCPUPowerMode.setValue(cpu_power_mode); + lpCPUPowerMode.setSummary(cpu_power_mode); + lpEnableLiteFp16.setValue(enable_lite_fp16); + lpEnableLiteFp16.setSummary(enable_lite_fp16); + + } + + static boolean checkAndUpdateSettings(Context ctx) { + boolean settingsChanged = false; + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(ctx); + + String model_dir = sharedPreferences.getString(ctx.getString(R.string.MODEL_DIR_KEY), + ctx.getString(R.string.SEGMENTATION_MODEL_DIR_DEFAULT)); + settingsChanged |= !modelDir.equalsIgnoreCase(model_dir); + modelDir = model_dir; + + String cpu_thread_num = sharedPreferences.getString(ctx.getString(R.string.CPU_THREAD_NUM_KEY), + ctx.getString(R.string.CPU_THREAD_NUM_DEFAULT)); + settingsChanged |= cpuThreadNum != Integer.parseInt(cpu_thread_num); + cpuThreadNum = Integer.parseInt(cpu_thread_num); + + String cpu_power_mode = sharedPreferences.getString(ctx.getString(R.string.CPU_POWER_MODE_KEY), + ctx.getString(R.string.CPU_POWER_MODE_DEFAULT)); + settingsChanged |= !cpuPowerMode.equalsIgnoreCase(cpu_power_mode); + cpuPowerMode = cpu_power_mode; + + String enable_lite_fp16 = sharedPreferences.getString(ctx.getString(R.string.ENABLE_LITE_FP16_MODE_KEY), + ctx.getString(R.string.ENABLE_LITE_FP16_MODE_DEFAULT)); + settingsChanged |= !enableLiteFp16.equalsIgnoreCase(enable_lite_fp16); + enableLiteFp16 = enable_lite_fp16; + + return settingsChanged; + } + + static void resetSettings() { + selectedModelIdx = -1; + modelDir = ""; + cpuThreadNum = 2; + cpuPowerMode = ""; + enableLiteFp16 = "true"; + } + + @Override + protected void onResume() { + super.onResume(); + getPreferenceScreen().getSharedPreferences().registerOnSharedPreferenceChangeListener(this); + reloadSettingsAndUpdateUI(); + } + + @Override + protected void onPause() { + super.onPause(); + getPreferenceScreen().getSharedPreferences().unregisterOnSharedPreferenceChangeListener(this); + } + + @Override + public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key) { + reloadSettingsAndUpdateUI(); + } } diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java index b4456e8a4db..9269a48284a 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java @@ -30,8 +30,8 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer, SurfaceTexture.OnFrameAvailableListener { private static final String TAG = CameraSurfaceView.class.getSimpleName(); - public static final int EXPECTED_PREVIEW_WIDTH = 1280; - public static final int EXPECTED_PREVIEW_HEIGHT = 720; + public static int EXPECTED_PREVIEW_WIDTH = 1280; + public static int EXPECTED_PREVIEW_HEIGHT = 720; protected int numberOfCameras; protected int selectedCameraId; diff --git a/java/android/app/src/main/res/values/strings.xml b/java/android/app/src/main/res/values/strings.xml index b5f9cd2e8a2..267871056fe 100644 --- a/java/android/app/src/main/res/values/strings.xml +++ b/java/android/app/src/main/res/values/strings.xml @@ -35,7 +35,7 @@ models/scrfd_500m_bnkps_shape320x320_pd - models/Portrait_PP_HumanSegV2_Lite_256x144_infer + models/portrait_pp_humansegv2_lite_256x144_inference_model 拍照识别 实时识别 diff --git a/java/android/app/src/main/res/xml/segmentation_setting.xml b/java/android/app/src/main/res/xml/segmentation_setting.xml index 91082dd8f14..56dd4152816 100644 --- a/java/android/app/src/main/res/xml/segmentation_setting.xml +++ b/java/android/app/src/main/res/xml/segmentation_setting.xml @@ -26,10 +26,6 @@ android:negativeButtonText="@null" android:positiveButtonText="@null" android:title="CPU Power Mode" /> - FindClass( "com/baidu/paddle/fastdeploy/vision/SegmentationResult"); const jfieldID j_seg_label_map_id = env->GetFieldID( - j_seg_result_clazz, "mLabelMap", "[I"); + j_seg_result_clazz, "mLabelMap", "[B"); const jfieldID j_seg_shape_id = env->GetFieldID( j_seg_result_clazz, "mShape", "[J"); const jfieldID j_seg_contain_shape_map_id = env->GetFieldID( @@ -292,10 +292,10 @@ bool AllocateJavaSegmentationResultFromCxx( // mLabelMap int[] shape (n): [I const auto &label_map_uint8 = c_result_ptr->label_map; - std::vector label_map; // cast uint8 -> int32 - label_map.assign(label_map_uint8.begin(), label_map_uint8.end()); - jintArray j_seg_label_map_int_arr = env->NewIntArray(len); - env->SetIntArrayRegion(j_seg_label_map_int_arr, 0, len, label_map.data()); + jbyteArray j_seg_label_map_byte_arr = env->NewByteArray(len); + env->SetByteArrayRegion(j_seg_label_map_byte_arr, 0, len, + reinterpret_cast(const_cast( + label_map_uint8.data()))); // mShape long[] shape (2) (H,W): [J const auto &shape = c_result_ptr->shape; @@ -319,12 +319,13 @@ bool AllocateJavaSegmentationResultFromCxx( } // Set object fields - env->SetObjectField(j_seg_result_obj, j_seg_label_map_id, j_seg_label_map_int_arr); + env->SetObjectField(j_seg_result_obj, j_seg_label_map_id, j_seg_label_map_byte_arr); env->SetObjectField(j_seg_result_obj, j_seg_shape_id, j_seg_shape_long_arr); env->SetBooleanField(j_seg_result_obj, j_seg_initialized_id, JNI_TRUE); // Release local Refs - env->DeleteLocalRef(j_seg_label_map_int_arr); + // env->DeleteLocalRef(j_seg_label_map_int_arr); + env->DeleteLocalRef(j_seg_label_map_byte_arr); env->DeleteLocalRef(j_seg_shape_long_arr); env->DeleteLocalRef(j_seg_result_clazz); @@ -421,7 +422,6 @@ bool AllocateJavaFaceDetectionResultFromCxx( return true; } - bool AllocateJavaResultFromCxx( JNIEnv *env, jobject j_result_obj, void *cxx_result, vision::ResultType type) { @@ -825,7 +825,7 @@ bool AllocateSegmentationResultFromJava( const jclass j_seg_result_clazz_cc = env->FindClass( "com/baidu/paddle/fastdeploy/vision/SegmentationResult"); const jfieldID j_seg_label_map_id_cc = env->GetFieldID( - j_seg_result_clazz_cc, "mLabelMap", "[I"); + j_seg_result_clazz_cc, "mLabelMap", "[B"); const jfieldID j_seg_shape_id_cc = env->GetFieldID( j_seg_result_clazz_cc, "mShape", "[J"); const jfieldID j_seg_contain_shape_map_id_cc = env->GetFieldID( @@ -846,34 +846,25 @@ bool AllocateSegmentationResultFromJava( return false; } - jintArray j_seg_label_map_int_arr = reinterpret_cast( + jbyteArray j_seg_label_map_byte_arr = reinterpret_cast( env->GetObjectField(j_seg_result_obj, j_seg_label_map_id_cc)); jlongArray j_seg_shape_long_arr = reinterpret_cast( env->GetObjectField(j_seg_result_obj, j_seg_shape_id_cc)); jboolean j_seg_contain_score_map = env->GetBooleanField(j_seg_result_obj, j_seg_contain_shape_map_id_cc); - jfloatArray j_seg_score_map_float_arr = reinterpret_cast( - env->GetObjectField(j_seg_result_obj, j_seg_score_map_id_cc)); // Init cxx result c_result_ptr->Clear(); - const int label_len = env->GetArrayLength(j_seg_label_map_int_arr); // HxW + const int label_len = env->GetArrayLength(j_seg_label_map_byte_arr); // HxW const int shape_len = env->GetArrayLength(j_seg_shape_long_arr); // 2 - const int score_len = env->GetArrayLength(j_seg_score_map_float_arr); // 0 | HxW c_result_ptr->label_map.resize(label_len); c_result_ptr->shape.resize(shape_len); - if (j_seg_contain_score_map) { - c_result_ptr->contain_score_map = true; - c_result_ptr->score_map.resize(score_len); - } // mLabelMap int[] shape (n): [I - std::vector label_map_int8; // cast int32 -> uint8_t - jint *j_seg_label_map_int_ptr = - env->GetIntArrayElements(j_seg_label_map_int_arr, nullptr); - label_map_int8.assign(j_seg_label_map_int_ptr,j_seg_label_map_int_ptr + label_len); - std::memcpy(c_result_ptr->label_map.data(), label_map_int8.data(), label_len * sizeof(int)); - env->ReleaseIntArrayElements(j_seg_label_map_int_arr, j_seg_label_map_int_ptr,0); + jbyte *j_seg_label_map_byte_ptr = + env->GetByteArrayElements(j_seg_label_map_byte_arr, nullptr); + std::memcpy(c_result_ptr->label_map.data(), j_seg_label_map_byte_ptr, label_len * sizeof(jbyte)); + env->ReleaseByteArrayElements(j_seg_label_map_byte_arr, j_seg_label_map_byte_ptr,0); // mShape long[] shape (2) (H,W): [J jlong *j_seg_shape_long_ptr = @@ -883,10 +874,19 @@ bool AllocateSegmentationResultFromJava( // mScoreMap float[] shape (n): [F if (j_seg_contain_score_map) { - jfloat *j_seg_score_map_float_ptr = - env->GetFloatArrayElements(j_seg_score_map_float_arr, nullptr); - std::memcpy(c_result_ptr->score_map.data(), j_seg_score_map_float_ptr, score_len * sizeof(float)); - env->ReleaseFloatArrayElements(j_seg_score_map_float_arr, j_seg_score_map_float_ptr, 0); + jfloatArray j_seg_score_map_float_arr = reinterpret_cast( + env->GetObjectField(j_seg_result_obj, j_seg_score_map_id_cc)); + + if (j_seg_score_map_float_arr != NULL) { + const int score_len = env->GetArrayLength(j_seg_score_map_float_arr); // 0 | HxW + + c_result_ptr->contain_score_map = true; + c_result_ptr->score_map.resize(score_len); + jfloat *j_seg_score_map_float_ptr = + env->GetFloatArrayElements(j_seg_score_map_float_arr, nullptr); + std::memcpy(c_result_ptr->score_map.data(), j_seg_score_map_float_ptr, score_len * sizeof(float)); + env->ReleaseFloatArrayElements(j_seg_score_map_float_arr, j_seg_score_map_float_ptr, 0); + } } // Release local Refs diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc index 63d9faf6e06..0bdad7a4dba 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/segmentation/paddleseg_model_jni.cc @@ -66,7 +66,6 @@ Java_com_baidu_paddle_fastdeploy_vision_segmentation_PaddleSegModel_predictNativ fni::RenderingSegmentation(env, c_bgr, c_result, argb8888_bitmap, save_image, weight, save_path); } - return fni::NewJavaResultFromCxx(env, reinterpret_cast(&c_result), vision::ResultType::SEGMENTATION); } diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java index 0663c7330e5..974abeb6632 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/SegmentationResult.java @@ -4,7 +4,7 @@ public class SegmentationResult { // Init from native - public int[] mLabelMap; + public byte[] mLabelMap; public float[] mScoreMap; public long[] mShape; public boolean mContainScoreMap = false; @@ -18,7 +18,7 @@ public boolean initialized() { return mInitialized; } - public void setLabelMap(@NonNull int[] labelMapBuffer) { + public void setLabelMap(@NonNull byte[] labelMapBuffer) { if (labelMapBuffer.length > 0) { mLabelMap = labelMapBuffer.clone(); } diff --git a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java index c7c74cd6cb4..d2bd9f7aeac 100644 --- a/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java +++ b/java/android/fastdeploy/src/main/java/com/baidu/paddle/fastdeploy/vision/segmentation/PaddleSegModel.java @@ -81,7 +81,6 @@ public SegmentationResult predict(Bitmap ARGB8888Bitmap, public SegmentationResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float weight) { - // scoreThreshold is for visualizing only. if (mCxxContext == 0) { return new SegmentationResult(); } From ff02ff03c1877672cd157b9201c2964a8b9f2e45 Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 17 Nov 2022 17:08:49 +0800 Subject: [PATCH 051/136] [Backend] Support Intel GPU with OpenVINO (#472) * Update ov_backend.cc * Update ov_backend.cc * support set openvino device --- fastdeploy/backends/openvino/ov_backend.cc | 6 ++++-- fastdeploy/backends/openvino/ov_backend.h | 1 + fastdeploy/pybind/runtime.cc | 1 + fastdeploy/runtime.cc | 5 +++++ fastdeploy/runtime.h | 8 ++++++++ fastdeploy/vision/common/processors/transform.h | 1 - python/fastdeploy/runtime.py | 5 +++++ 7 files changed, 24 insertions(+), 3 deletions(-) diff --git a/fastdeploy/backends/openvino/ov_backend.cc b/fastdeploy/backends/openvino/ov_backend.cc index 5a664fc8772..608e9199ff2 100644 --- a/fastdeploy/backends/openvino/ov_backend.cc +++ b/fastdeploy/backends/openvino/ov_backend.cc @@ -158,7 +158,8 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file, } else if (option_.ov_num_streams > 0) { properties["NUM_STREAMS"] = option_.ov_num_streams; } - compiled_model_ = core_.compile_model(model, "CPU", properties); + FDINFO << "Compile OpenVINO model on device_name:" << option.device << "." << std::endl; + compiled_model_ = core_.compile_model(model, option.device, properties); request_ = compiled_model_.create_infer_request(); initialized_ = true; @@ -255,7 +256,8 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file, } else if (option_.ov_num_streams > 0) { properties["NUM_STREAMS"] = option_.ov_num_streams; } - compiled_model_ = core_.compile_model(model, "CPU", properties); + FDINFO << "Compile OpenVINO model on device_name:" << option.device << "." << std::endl; + compiled_model_ = core_.compile_model(model, option.device, properties); request_ = compiled_model_.create_infer_request(); diff --git a/fastdeploy/backends/openvino/ov_backend.h b/fastdeploy/backends/openvino/ov_backend.h index b7d77e58ffb..f28459f030a 100644 --- a/fastdeploy/backends/openvino/ov_backend.h +++ b/fastdeploy/backends/openvino/ov_backend.h @@ -26,6 +26,7 @@ namespace fastdeploy { struct OpenVINOBackendOption { + std::string device = "CPU"; int cpu_thread_num = -1; int ov_num_streams = 1; std::map> shape_infos; diff --git a/fastdeploy/pybind/runtime.cc b/fastdeploy/pybind/runtime.cc index 759c555309f..c46f83dbc2e 100644 --- a/fastdeploy/pybind/runtime.cc +++ b/fastdeploy/pybind/runtime.cc @@ -33,6 +33,7 @@ void BindRuntime(pybind11::module& m) { .def("use_openvino_backend", &RuntimeOption::UseOpenVINOBackend) .def("use_lite_backend", &RuntimeOption::UseLiteBackend) .def("set_paddle_mkldnn", &RuntimeOption::SetPaddleMKLDNN) + .def("set_openvino_device", &RuntimeOption::SetOpenVINODevice) .def("enable_paddle_log_info", &RuntimeOption::EnablePaddleLogInfo) .def("disable_paddle_log_info", &RuntimeOption::DisablePaddleLogInfo) .def("set_paddle_mkldnn_cache_size", diff --git a/fastdeploy/runtime.cc b/fastdeploy/runtime.cc index 3077afb51d6..6a8be7dd7b4 100755 --- a/fastdeploy/runtime.cc +++ b/fastdeploy/runtime.cc @@ -332,6 +332,10 @@ void RuntimeOption::SetPaddleMKLDNNCacheSize(int size) { pd_mkldnn_cache_size = size; } +void RuntimeOption::SetOpenVINODevice(const std::string& name) { + openvino_device = name; +} + void RuntimeOption::EnableLiteFP16() { lite_enable_fp16 = true; } @@ -641,6 +645,7 @@ void Runtime::CreateOpenVINOBackend() { #ifdef ENABLE_OPENVINO_BACKEND auto ov_option = OpenVINOBackendOption(); ov_option.cpu_thread_num = option.cpu_thread_num; + ov_option.device = option.openvino_device; ov_option.ov_num_streams = option.ov_num_streams; FDASSERT(option.model_format == ModelFormat::PADDLE || option.model_format == ModelFormat::ONNX, diff --git a/fastdeploy/runtime.h b/fastdeploy/runtime.h index 8822308ec76..9c8a1297692 100755 --- a/fastdeploy/runtime.h +++ b/fastdeploy/runtime.h @@ -168,6 +168,11 @@ struct FASTDEPLOY_DECL RuntimeOption { */ void SetPaddleMKLDNNCacheSize(int size); + /** + * @brief Set device name for OpenVINO, default 'CPU', can also be 'AUTO', 'GPU', 'GPU.1'.... + */ + void SetOpenVINODevice(const std::string& name = "CPU"); + /** * @brief Set optimzed model dir for Paddle Lite backend. */ @@ -344,6 +349,9 @@ struct FASTDEPLOY_DECL RuntimeOption { size_t trt_max_batch_size = 32; size_t trt_max_workspace_size = 1 << 30; + // ======Only for OpenVINO Backend====== + std::string openvino_device = "CPU"; + // ======Only for Poros Backend======= bool is_dynamic = false; bool long_to_int = true; diff --git a/fastdeploy/vision/common/processors/transform.h b/fastdeploy/vision/common/processors/transform.h index e3f71463657..ce7612a378f 100755 --- a/fastdeploy/vision/common/processors/transform.h +++ b/fastdeploy/vision/common/processors/transform.h @@ -37,7 +37,6 @@ namespace fastdeploy { namespace vision { void FuseTransforms(std::vector>* processors); - // Fuse Normalize + Cast(Float) to Normalize void FuseNormalizeCast(std::vector>* processors); // Fuse Normalize + HWC2CHW to NormalizeAndPermute diff --git a/python/fastdeploy/runtime.py b/python/fastdeploy/runtime.py index d6a16adaaea..27b99b760f1 100755 --- a/python/fastdeploy/runtime.py +++ b/python/fastdeploy/runtime.py @@ -269,6 +269,11 @@ def set_paddle_mkldnn(self, use_mkldnn=True): """ return self._option.set_paddle_mkldnn(use_mkldnn) + def set_openvino_device(self, name="CPU"): + """Set device name for OpenVINO, default 'CPU', can also be 'AUTO', 'GPU', 'GPU.1'.... + """ + return self._option.set_openvino_device(name) + def enable_paddle_log_info(self): """Enable print out the debug log information while using Paddle Inference backend, the log information is disabled by default. """ From 79e0a0059dd44749a5200c70d367e399ad465bad Mon Sep 17 00:00:00 2001 From: heliqi <1101791222@qq.com> Date: Thu, 17 Nov 2022 17:16:32 +0800 Subject: [PATCH 052/136] [Serving]ppcls preprocessor support gpu (#615) * serving ppcls support gpu * serving ppcls preprocessor use cpu --- examples/vision/classification/paddleclas/README.md | 1 + .../paddleclas/serving/models/preprocess/1/model.py | 3 +++ .../paddleclas/serving/models/preprocess/config.pbtxt | 8 ++++++-- fastdeploy/pybind/fd_tensor.cc | 3 +-- fastdeploy/vision/classification/ppcls/ppcls_pybind.cc | 6 ++++-- fastdeploy/vision/classification/ppcls/preprocessor.h | 2 ++ 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/examples/vision/classification/paddleclas/README.md b/examples/vision/classification/paddleclas/README.md index d942e626162..b0dd5ebaaf3 100644 --- a/examples/vision/classification/paddleclas/README.md +++ b/examples/vision/classification/paddleclas/README.md @@ -54,3 +54,4 @@ PaddleClas模型导出，请参考其文档说明[模型导出](https://github.c - [Python部署](python) - [C++部署](cpp) +- [服务化部署](serving) diff --git a/examples/vision/classification/paddleclas/serving/models/preprocess/1/model.py b/examples/vision/classification/paddleclas/serving/models/preprocess/1/model.py index f65775b196d..c05ef7b6510 100644 --- a/examples/vision/classification/paddleclas/serving/models/preprocess/1/model.py +++ b/examples/vision/classification/paddleclas/serving/models/preprocess/1/model.py @@ -68,6 +68,9 @@ def initialize(self, args): __file__)) + "/inference_cls.yaml" self.preprocess_ = fd.vision.classification.PaddleClasPreprocessor( yaml_path) + if args['model_instance_kind'] == 'GPU': + device_id = int(args['model_instance_device_id']) + self.preprocess_.use_gpu(device_id) def execute(self, requests): """`execute` must be implemented in every Python model. `execute` diff --git a/examples/vision/classification/paddleclas/serving/models/preprocess/config.pbtxt b/examples/vision/classification/paddleclas/serving/models/preprocess/config.pbtxt index 8bb740fd223..a6f5d92db07 100644 --- a/examples/vision/classification/paddleclas/serving/models/preprocess/config.pbtxt +++ b/examples/vision/classification/paddleclas/serving/models/preprocess/config.pbtxt @@ -20,7 +20,11 @@ output [ instance_group [ { - count: 1 - kind: KIND_CPU + # The number of instances is 1 + count: 1 + # Use CPU, GPU inference option is:KIND_GPU + kind: KIND_CPU + # The instance is deployed on the 0th GPU card + # gpus: [0] } ] \ No newline at end of file diff --git a/fastdeploy/pybind/fd_tensor.cc b/fastdeploy/pybind/fd_tensor.cc index 28c5e2b5353..93b8cbe9a73 100644 --- a/fastdeploy/pybind/fd_tensor.cc +++ b/fastdeploy/pybind/fd_tensor.cc @@ -151,8 +151,7 @@ pybind11::capsule FDTensorToDLPack(FDTensor& fd_tensor) { dlpack_tensor->dl_tensor.dtype = FDToDlpackType(fd_tensor.dtype); - // TODO(liqi): FDTensor add device_id - dlpack_tensor->dl_tensor.device.device_id = 0; + dlpack_tensor->dl_tensor.device.device_id = fd_tensor.device_id; if(fd_tensor.device == Device::GPU) { if (fd_tensor.is_pinned_memory) { dlpack_tensor->dl_tensor.device.device_type = DLDeviceType::kDLCUDAHost; diff --git a/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc b/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc index 267042ce054..65176ca156e 100644 --- a/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc +++ b/fastdeploy/vision/classification/ppcls/ppcls_pybind.cc @@ -27,8 +27,10 @@ void BindPaddleClas(pybind11::module& m) { if (!self.Run(&images, &outputs)) { pybind11::eval("raise Exception('Failed to preprocess the input data in PaddleClasPreprocessor.')"); } - for (size_t i = 0; i < outputs.size(); ++i) { - outputs[i].StopSharing(); + if (!self.WithGpu()) { + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].StopSharing(); + } } return outputs; }) diff --git a/fastdeploy/vision/classification/ppcls/preprocessor.h b/fastdeploy/vision/classification/ppcls/preprocessor.h index 4e6a0f9aff8..54c5e669d15 100644 --- a/fastdeploy/vision/classification/ppcls/preprocessor.h +++ b/fastdeploy/vision/classification/ppcls/preprocessor.h @@ -44,6 +44,8 @@ class FASTDEPLOY_DECL PaddleClasPreprocessor { */ void UseGpu(int gpu_id = -1); + bool WithGpu() { return use_cuda_; } + private: bool BuildPreprocessPipelineFromConfig(const std::string& config_file); std::vector> processors_; From f06cc5d76275d688c9ea65263a2d71ced8bce9a3 Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Thu, 17 Nov 2022 17:17:19 +0800 Subject: [PATCH 053/136] [Model] Add move assignment for DetectionResult (#600) * add onnx_ort_runtime demo * rm in requirements * support batch eval * fixed MattingResults bug * move assignment for DetectionResult --- fastdeploy/vision/common/result.cc | 14 ++++++++++++++ fastdeploy/vision/common/result.h | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index ddb5f616a31..ee137604891 100755 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -80,6 +80,20 @@ DetectionResult::DetectionResult(const DetectionResult& res) { } } +DetectionResult& DetectionResult::operator=(DetectionResult&& other) { + if (&other != this) { + boxes = std::move(other.boxes); + scores = std::move(other.scores); + label_ids = std::move(other.label_ids); + contain_masks = std::move(other.contain_masks); + if (contain_masks) { + masks.clear(); + masks = std::move(other.masks); + } + } + return *this; +} + void DetectionResult::Clear() { std::vector>().swap(boxes); std::vector().swap(scores); diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index 59690ab6d88..27f6a977094 100755 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -95,6 +95,7 @@ struct FASTDEPLOY_DECL Mask : public BaseResult { /*! @brief Detection result structure for all the object detection models and instance segmentation models */ struct FASTDEPLOY_DECL DetectionResult : public BaseResult { + DetectionResult() = default; /** \brief All the detected object boxes for an input image, the size of `boxes` is the number of detected objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax] */ std::vector> boxes; @@ -111,8 +112,10 @@ struct FASTDEPLOY_DECL DetectionResult : public BaseResult { ResultType type = ResultType::DETECTION; - DetectionResult() {} + /// Copy constructor DetectionResult(const DetectionResult& res); + /// Move assignment + DetectionResult& operator=(DetectionResult&& other); /// Clear detection result void Clear(); From 698dc45739abf61ded519dd074604c3316e957d7 Mon Sep 17 00:00:00 2001 From: Winter <1558270516@qq.com> Date: Thu, 17 Nov 2022 19:18:47 +0800 Subject: [PATCH 054/136] [Android] Configure face recognition and segmentation details page (#619) * 1.Classification Code submission; 2.Specification code format. * Specification code format. * Update strings.xml * 1.Display ocr, detection and classification results. 2.Specification code format. * Configure face recognition and semantic segmentation details page. Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> --- .../ClassificationMainActivity.java | 186 ++++++++---- .../detection/DetectionMainActivity.java | 124 ++++---- .../examples/facedet/FaceDetMainActivity.java | 107 +++---- .../app/examples/ocr/OcrMainActivity.java | 267 +++++++++++++++--- .../SegmentationMainActivity.java | 154 ++++------ .../baidu/paddle/fastdeploy/app/ui/Utils.java | 24 ++ .../res/layout/classification_camera_page.xml | 2 +- .../main/res/layout/detection_camera_page.xml | 2 +- .../src/main/res/layout/ocr_camera_page.xml | 5 +- 9 files changed, 565 insertions(+), 306 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java index 14c1c329f28..e052df7a844 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java @@ -2,6 +2,7 @@ import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.readTxt; import android.Manifest; import android.annotation.SuppressLint; @@ -36,9 +37,9 @@ import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.ClassifyResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.classification.PaddleClasModel; -import java.io.File; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -62,27 +63,33 @@ public class ClassificationMainActivity extends Activity implements View.OnClick private SeekBar confidenceSeekbar; private TextView seekbarText; private float resultNum = 1.0f; - private ResultListView detectResultView; + private ResultListView resultView; private Bitmap shutterBitmap; - private Bitmap originShutterBitmap; private Bitmap picBitmap; - private Bitmap originPicBitmap; + private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; public static final int BTN_SHUTTER = 0; public static final int ALBUM_SELECT = 1; - private static int TYPE = TYPE_UNKNOWN; + public static final int REALTIME_DETECT = 2; + private static int TYPE = REALTIME_DETECT; private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; private static final int INTENT_CODE_PICK_IMAGE = 100; + private static final int TIME_SLEEP_INTERVAL = 50; // ms - String savedImagePath = "result.jpg"; long timeElapsed = 0; long frameCounter = 0; // Call 'init' and 'release' manually later PaddleClasModel predictor = new PaddleClasModel(); + private float[] scores; + private int[] labelId; + private boolean initialized; + private List labelText; + private List results = new ArrayList<>(); + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -114,12 +121,8 @@ public void onClick(View v) { break; case R.id.btn_shutter: TYPE = BTN_SHUTTER; - svPreview.onPause(); - cameraPageView.setVisibility(View.GONE); - resultPageView.setVisibility(View.VISIBLE); - seekbarText.setText(resultNum + ""); - confidenceSeekbar.setProgress((int) (resultNum * 100)); - resultImage.setImageBitmap(shutterBitmap); + shutterAndPauseCamera(); + resultView.setAdapter(null); break; case R.id.btn_settings: startActivity(new Intent(ClassificationMainActivity.this, ClassificationSettingsActivity.class)); @@ -130,7 +133,7 @@ public void onClick(View v) { case R.id.back_in_preview: finish(); break; - case R.id.albumSelect: + case R.id.iv_select: TYPE = ALBUM_SELECT; // Judge whether authority has been granted. if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { @@ -141,16 +144,84 @@ public void onClick(View v) { intent.setType("image/*"); startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); } + resultView.setAdapter(null); break; case R.id.back_in_result: - resultPageView.setVisibility(View.GONE); - cameraPageView.setVisibility(View.VISIBLE); - svPreview.onResume(); + back(); break; } } + @Override + public void onBackPressed() { + super.onBackPressed(); + back(); + } + + private void back() { + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + results.clear(); + if (scores != null) { + scores = null; + } + if (labelId != null) { + labelId = null; + } + } + + private void shutterAndPauseCamera() { + new Thread(new Runnable() { + @Override + public void run() { + try { + // Sleep some times to ensure picture has been correctly shut. + Thread.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + } catch (InterruptedException e) { + e.printStackTrace(); + } + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + // These code will run in main thread. + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + resultImage.setImageBitmap(shutterBitmap); + } else { + new AlertDialog.Builder(ClassificationMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } + }); + + } + }).start(); + } + + private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { + return; + } + if (!ARGB8888ImageBitmap.isRecycled()) { + synchronized (this) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; + } + } + @Override protected void onActivityResult(int requestCode, int resultCode, Intent data) { super.onActivityResult(requestCode, resultCode, data); @@ -163,7 +234,6 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); - originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -190,34 +260,17 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { - String savedImagePath = ""; - synchronized (this) { - savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; - } if (TYPE == BTN_SHUTTER) { - shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - } else { - // Only reference in predict loops. - shutterBitmap = ARGB8888ImageBitmap; - originShutterBitmap = ARGB8888ImageBitmap; + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; } boolean modified = false; - long tc = System.currentTimeMillis(); - ClassifyResult result = predictor.predict( - ARGB8888ImageBitmap, true, ClassificationSettingsActivity.scoreThreshold); - + ClassifyResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); - frameCounter++; - + Visualize.visClassification(ARGB8888ImageBitmap, result, resultNum, 12); modified = result.initialized(); - if (!savedImagePath.isEmpty()) { - synchronized (this) { - ClassificationMainActivity.this.savedImagePath = "result.jpg"; - } - } - + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @@ -259,7 +312,7 @@ protected void onDestroy() { } public void initView() { - TYPE = BTN_SHUTTER; + TYPE = REALTIME_DETECT; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); tvStatus = (TextView) findViewById(R.id.tv_status); @@ -273,7 +326,7 @@ public void initView() { realtimeToggleButton.setOnClickListener(this); backInPreview = findViewById(R.id.back_in_preview); backInPreview.setOnClickListener(this); - albumSelectButton = findViewById(R.id.albumSelect); + albumSelectButton = findViewById(R.id.iv_select); albumSelectButton.setOnClickListener(this); cameraPageView = findViewById(R.id.camera_page); resultPageView = findViewById(R.id.result_page); @@ -282,15 +335,7 @@ public void initView() { backInResult.setOnClickListener(this); confidenceSeekbar = findViewById(R.id.confidence_seekbar); seekbarText = findViewById(R.id.seekbar_text); - detectResultView = findViewById(R.id.result_list_view); - - List results = new ArrayList<>(); - results.add(new BaseResultModel(1, "cup", 0.4f)); - results.add(new BaseResultModel(2, "pen", 0.6f)); - results.add(new BaseResultModel(3, "tang", 1.0f)); - final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.classification_result_page_item, results); - detectResultView.setAdapter(adapter); - detectResultView.invalidate(); + resultView = findViewById(R.id.result_list_view); confidenceSeekbar.setMax(100); confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { @@ -301,6 +346,7 @@ public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); + results.clear(); } @Override @@ -315,16 +361,11 @@ public void onStopTrackingTouch(SeekBar seekBar) { public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(500); - predictor.predict(picBitmap, savedImagePath, resultNum); - resultImage.setImageBitmap(picBitmap); - picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); - resultNum = 1.0f; + detail(picBitmap); } else { SystemClock.sleep(500); - predictor.predict(shutterBitmap, savedImagePath, resultNum); - resultImage.setImageBitmap(shutterBitmap); - shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); - resultNum = 1.0f; + svPreview.onPause(); + detail(shutterBitmap); } } }); @@ -332,6 +373,34 @@ public void run() { }); } + private void detail(Bitmap bitmap) { + ClassifyResult result = predictor.predict(bitmap, true, ClassificationSettingsActivity.scoreThreshold); + if (scores == null) { + scores = result.mScores; + } + if (labelId == null) { + labelId = result.mLabelIds; + } + initialized = result.initialized(); + if (initialized) { + for (int i = 0; i < labelId.length; i++) { + for (int j = 0; j < labelText.size(); j++) { + if (scores[i] > resultNum) { + if (labelId[i] == Integer.parseInt(labelText.get(j).substring(0, labelText.get(j).indexOf(" ")))) { + results.add(new BaseResultModel(labelId[i], labelText.get(j), scores[i])); + } + } + } + } + } + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + resultView.setAdapter(adapter); + resultView.invalidate(); + + resultImage.setImageBitmap(bitmap); + resultNum = 1.0f; + } + @SuppressLint("ApplySharedPref") public void initSettings() { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); @@ -352,6 +421,7 @@ public void checkAndUpdateSettings() { String paramsFile = realModelDir + "/" + "inference.pdiparams"; String configFile = realModelDir + "/" + "inference_cls.yaml"; String labelFile = realLabelPath; + labelText = readTxt(labelFile); RuntimeOption option = new RuntimeOption(); option.setCpuThreadNum(ClassificationSettingsActivity.cpuThreadNum); option.setLitePowerMode(ClassificationSettingsActivity.cpuPowerMode); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index e3da008bb2d..3b8c750113a 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -8,18 +8,14 @@ import android.content.Intent; import android.content.SharedPreferences; import android.content.pm.PackageManager; -import android.database.Cursor; import android.graphics.Bitmap; -import android.graphics.BitmapFactory; import android.net.Uri; import android.os.Bundle; import android.os.SystemClock; import android.preference.PreferenceManager; -import android.provider.MediaStore; import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; -import android.util.Log; import android.view.View; import android.view.ViewGroup; import android.view.Window; @@ -31,7 +27,6 @@ import com.baidu.paddle.fastdeploy.RuntimeOption; import com.baidu.paddle.fastdeploy.app.examples.R; -import com.baidu.paddle.fastdeploy.app.examples.facedet.FaceDetMainActivity; import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; import com.baidu.paddle.fastdeploy.app.ui.Utils; @@ -43,8 +38,8 @@ import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.readTxt; -import java.io.File; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -68,11 +63,9 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private SeekBar confidenceSeekbar; private TextView seekbarText; private float resultNum = 1.0f; - private ResultListView detectResultView; + private ResultListView resultView; private Bitmap shutterBitmap; - private Bitmap originShutterBitmap; private Bitmap picBitmap; - private Bitmap originPicBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -85,13 +78,18 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private static final int INTENT_CODE_PICK_IMAGE = 100; private static final int TIME_SLEEP_INTERVAL = 50; // ms - String savedImagePath = "result.jpg"; long timeElapsed = 0; long frameCounter = 0; // Call 'init' and 'release' manually later PicoDet predictor = new PicoDet(); + private float[] scores; + private int[] labelId; + private boolean initialized; + private List labelText; + private List results = new ArrayList<>(); + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -124,6 +122,7 @@ public void onClick(View v) { case R.id.btn_shutter: TYPE = BTN_SHUTTER; shutterAndPauseCamera(); + resultView.setAdapter(null); break; case R.id.btn_settings: startActivity(new Intent(DetectionMainActivity.this, DetectionSettingsActivity.class)); @@ -134,7 +133,7 @@ public void onClick(View v) { case R.id.back_in_preview: finish(); break; - case R.id.albumSelect: + case R.id.iv_select: TYPE = ALBUM_SELECT; // Judge whether authority has been granted. if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { @@ -145,17 +144,35 @@ public void onClick(View v) { intent.setType("image/*"); startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); } + resultView.setAdapter(null); break; case R.id.back_in_result: - resultPageView.setVisibility(View.GONE); - cameraPageView.setVisibility(View.VISIBLE); - TYPE = REALTIME_DETECT; - isShutterBitmapCopied = false; - svPreview.onResume(); + back(); break; } } + @Override + public void onBackPressed() { + super.onBackPressed(); + back(); + } + + private void back() { + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + results.clear(); + if (scores != null) { + scores = null; + } + if (labelId != null) { + labelId = null; + } + } + private void shutterAndPauseCamera() { new Thread(new Runnable() { @Override @@ -198,7 +215,6 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -217,7 +233,6 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); - originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -248,27 +263,12 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } - - String savedImagePath = ""; - synchronized (this) { - savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; - } - boolean modified = false; - long tc = System.currentTimeMillis(); DetectionResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); - Visualize.visDetection(ARGB8888ImageBitmap, result, DetectionSettingsActivity.scoreThreshold); - modified = result.initialized(); - if (!savedImagePath.isEmpty()) { - synchronized (this) { - DetectionMainActivity.this.savedImagePath = "result.jpg"; - } - } - frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -325,7 +325,7 @@ public void initView() { realtimeToggleButton.setOnClickListener(this); backInPreview = findViewById(R.id.back_in_preview); backInPreview.setOnClickListener(this); - albumSelectButton = findViewById(R.id.albumSelect); + albumSelectButton = findViewById(R.id.iv_select); albumSelectButton.setOnClickListener(this); cameraPageView = findViewById(R.id.camera_page); resultPageView = findViewById(R.id.result_page); @@ -334,15 +334,7 @@ public void initView() { backInResult.setOnClickListener(this); confidenceSeekbar = findViewById(R.id.confidence_seekbar); seekbarText = findViewById(R.id.seekbar_text); - detectResultView = findViewById(R.id.result_list_view); - - List results = new ArrayList<>(); - results.add(new BaseResultModel(1, "cup", 0.4f)); - results.add(new BaseResultModel(2, "pen", 0.6f)); - results.add(new BaseResultModel(3, "tang", 1.0f)); - final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.detection_result_page_item, results); - detectResultView.setAdapter(adapter); - detectResultView.invalidate(); + resultView = findViewById(R.id.result_list_view); confidenceSeekbar.setMax(100); confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { @@ -353,6 +345,7 @@ public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); + results.clear(); } @Override @@ -367,20 +360,11 @@ public void onStopTrackingTouch(SeekBar seekBar) { public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); - if (!picBitmap.isRecycled()) { - predictor.predict(picBitmap, true, resultNum); - resultImage.setImageBitmap(picBitmap); - picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); - } - resultNum = 1.0f; + detail(picBitmap); } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); - if (!shutterBitmap.isRecycled()) { - predictor.predict(shutterBitmap, true, resultNum); - resultImage.setImageBitmap(shutterBitmap); - shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); - } - resultNum = 1.0f; + svPreview.onPause(); + detail(shutterBitmap); } } }); @@ -388,6 +372,35 @@ public void run() { }); } + private void detail(Bitmap bitmap) { + DetectionResult result = predictor.predict( + bitmap, true, DetectionSettingsActivity.scoreThreshold); + if (scores == null) { + scores = result.mScores; + } + if (labelId == null) { + labelId = result.mLabelIds; + } + initialized = result.initialized(); + if (initialized) { + for (int i = 0; i < labelId.length; i++) { + for (int j = 0; j < labelText.size(); j++) { + if (scores[i] > resultNum) { + if (labelId[i] == j) { + results.add(new BaseResultModel(labelId[i], labelText.get(j), scores[i])); + } + } + } + } + } + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + resultView.setAdapter(adapter); + resultView.invalidate(); + + resultImage.setImageBitmap(bitmap); + resultNum = 1.0f; + } + @SuppressLint("ApplySharedPref") public void initSettings() { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); @@ -408,6 +421,7 @@ public void checkAndUpdateSettings() { String paramsFile = realModelDir + "/" + "model.pdiparams"; String configFile = realModelDir + "/" + "infer_cfg.yml"; String labelFile = realLabelPath; + labelText = readTxt(labelFile); RuntimeOption option = new RuntimeOption(); option.setCpuThreadNum(DetectionSettingsActivity.cpuThreadNum); option.setLitePowerMode(DetectionSettingsActivity.cpuPowerMode); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index c7c270faa97..934f568b3af 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -8,14 +8,11 @@ import android.content.Intent; import android.content.SharedPreferences; import android.content.pm.PackageManager; -import android.database.Cursor; import android.graphics.Bitmap; -import android.graphics.BitmapFactory; import android.net.Uri; import android.os.Bundle; import android.os.SystemClock; import android.preference.PreferenceManager; -import android.provider.MediaStore; import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; @@ -43,7 +40,6 @@ import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; -import java.io.File; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -67,11 +63,9 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene private SeekBar confidenceSeekbar; private TextView seekbarText; private float resultNum = 1.0f; - private ResultListView detectResultView; + private ResultListView resultView; private Bitmap shutterBitmap; - private Bitmap originShutterBitmap; private Bitmap picBitmap; - private Bitmap originPicBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -84,13 +78,16 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene private static final int INTENT_CODE_PICK_IMAGE = 100; private static final int TIME_SLEEP_INTERVAL = 50; // ms - String savedImagePath = "result.jpg"; long timeElapsed = 0; long frameCounter = 0; // Call 'init' and 'release' manually later SCRFD predictor = new SCRFD(); + public float[] scores; // [n] + public boolean initialized = false; + private List results = new ArrayList<>(); + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -123,6 +120,7 @@ public void onClick(View v) { case R.id.btn_shutter: TYPE = BTN_SHUTTER; shutterAndPauseCamera(); + resultView.setAdapter(null); break; case R.id.btn_settings: startActivity(new Intent(FaceDetMainActivity.this, FaceDetSettingsActivity.class)); @@ -144,17 +142,32 @@ public void onClick(View v) { intent.setType("image/*"); startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); } + resultView.setAdapter(null); break; case R.id.back_in_result: - resultPageView.setVisibility(View.GONE); - cameraPageView.setVisibility(View.VISIBLE); - TYPE = REALTIME_DETECT; - isShutterBitmapCopied = false; - svPreview.onResume(); + back(); break; } } + @Override + public void onBackPressed() { + super.onBackPressed(); + back(); + } + + private void back() { + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + results.clear(); + if (scores != null) { + scores = null; + } + } + private void shutterAndPauseCamera() { new Thread(new Runnable() { @Override @@ -197,7 +210,6 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -216,7 +228,6 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); - originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -248,29 +259,14 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } - - String savedImagePath = ""; - synchronized (this) { - savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; - } - boolean modified = false; - long tc = System.currentTimeMillis(); + SystemClock.sleep(TIME_SLEEP_INTERVAL); FaceDetectionResult result = predictor.predict( ARGB8888ImageBitmap, FaceDetSettingsActivity.scoreThreshold, 0.4f); - timeElapsed += (System.currentTimeMillis() - tc); - Visualize.visFaceDetection(ARGB8888ImageBitmap, result); - modified = result.initialized(); - if (!savedImagePath.isEmpty()) { - synchronized (this) { - FaceDetMainActivity.this.savedImagePath = "result.jpg"; - } - } - frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -336,16 +332,7 @@ public void initView() { backInResult.setOnClickListener(this); confidenceSeekbar = findViewById(R.id.confidence_seekbar); seekbarText = findViewById(R.id.seekbar_text); - detectResultView = findViewById(R.id.result_list_view); - - List results = new ArrayList<>(); - // TODO: add model results from FaceDetectionResult instead of using fake data. - results.add(new BaseResultModel(1, "face", 0.4f)); - results.add(new BaseResultModel(2, "face", 0.6f)); - results.add(new BaseResultModel(3, "face", 1.0f)); - final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.facedet_result_page_item, results); - detectResultView.setAdapter(adapter); - detectResultView.invalidate(); + resultView = findViewById(R.id.result_list_view); confidenceSeekbar.setMax(100); confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { @@ -356,6 +343,7 @@ public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); + results.clear(); } @Override @@ -370,20 +358,11 @@ public void onStopTrackingTouch(SeekBar seekBar) { public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms - if (!picBitmap.isRecycled()) { - predictor.predict(picBitmap, true, resultNum, 0.4f); - resultImage.setImageBitmap(picBitmap); - picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); - } - resultNum = 1.0f; + detail(picBitmap); } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms - if (!shutterBitmap.isRecycled()) { - predictor.predict(shutterBitmap, true, resultNum, 0.4f); - resultImage.setImageBitmap(shutterBitmap); - shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); - } - resultNum = 1.0f; + svPreview.onPause(); + detail(shutterBitmap); } } }); @@ -391,6 +370,28 @@ public void run() { }); } + private void detail(Bitmap bitmap) { + FaceDetectionResult result = predictor.predict(bitmap, true, FaceDetSettingsActivity.scoreThreshold, 0.4f); + if (scores == null) { + scores = result.mScores; + } + initialized = result.initialized(); + Log.e("GBD", initialized + "---initialized"); + if (initialized) { + for (int i = 0; i < scores.length; i++) { + if (scores[i] > resultNum) { + results.add(new BaseResultModel(i + 1, "face", scores[i])); + } + } + } + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + resultView.setAdapter(adapter); + resultView.invalidate(); + + resultImage.setImageBitmap(bitmap); + resultNum = 1.0f; + } + @SuppressLint("ApplySharedPref") public void initSettings() { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java index 55f7294828e..321b03ecbea 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java @@ -1,5 +1,7 @@ package com.baidu.paddle.fastdeploy.app.examples.ocr; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; import android.Manifest; import android.annotation.SuppressLint; @@ -10,32 +12,39 @@ import android.content.SharedPreferences; import android.content.pm.PackageManager; import android.graphics.Bitmap; +import android.net.Uri; import android.os.Bundle; +import android.os.SystemClock; import android.preference.PreferenceManager; import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; import android.view.View; +import android.view.ViewGroup; import android.view.Window; import android.view.WindowManager; import android.widget.ImageButton; import android.widget.ImageView; +import android.widget.SeekBar; import android.widget.TextView; -import android.widget.Toast; import com.baidu.paddle.fastdeploy.RuntimeOption; import com.baidu.paddle.fastdeploy.app.examples.R; -import com.baidu.paddle.fastdeploy.app.ui.Utils; import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; -import com.baidu.paddle.fastdeploy.vision.OCRResult; +import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; +import com.baidu.paddle.fastdeploy.app.ui.Utils; +import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; +import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.pipeline.PPOCRv2; +import com.baidu.paddle.fastdeploy.vision.OCRResult; +import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.ocr.Classifier; import com.baidu.paddle.fastdeploy.vision.ocr.DBDetector; import com.baidu.paddle.fastdeploy.vision.ocr.Recognizer; -import java.io.File; -import java.text.SimpleDateFormat; -import java.util.Date; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; public class OcrMainActivity extends Activity implements View.OnClickListener, CameraSurfaceView.OnTextureChangedListener { private static final String TAG = OcrMainActivity.class.getSimpleName(); @@ -48,14 +57,40 @@ public class OcrMainActivity extends Activity implements View.OnClickListener, C ImageView realtimeToggleButton; boolean isRealtimeStatusRunning = false; ImageView backInPreview; + private ImageView albumSelectButton; + private View cameraPageView; + private ViewGroup resultPageView; + private ImageView resultImage; + private ImageView backInResult; + private SeekBar confidenceSeekbar; + private TextView seekbarText; + private float resultNum = 1.0f; + private ResultListView resultView; + private Bitmap shutterBitmap; + private Bitmap picBitmap; + private boolean isShutterBitmapCopied = false; - String savedImagePath = "result.jpg"; - int lastFrameIndex = 0; - long lastFrameTime; + public static final int TYPE_UNKNOWN = -1; + public static final int BTN_SHUTTER = 0; + public static final int ALBUM_SELECT = 1; + public static final int REALTIME_DETECT = 2; + private static int TYPE = REALTIME_DETECT; + + private static final int REQUEST_PERMISSION_CODE_STORAGE = 101; + private static final int INTENT_CODE_PICK_IMAGE = 100; + private static final int TIME_SLEEP_INTERVAL = 50; // ms + + long timeElapsed = 0; + long frameCounter = 0; // Call 'init' and 'release' manually later PPOCRv2 predictor = new PPOCRv2(); + private String[] texts; + private float[] recScores; + private boolean initialized; + private List results = new ArrayList<>(); + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -86,12 +121,9 @@ public void onClick(View v) { svPreview.switchCamera(); break; case R.id.btn_shutter: - @SuppressLint("SimpleDateFormat") - SimpleDateFormat date = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss"); - synchronized (this) { - savedImagePath = Utils.getDCIMDirectory() + File.separator + date.format(new Date()).toString() + ".png"; - } - Toast.makeText(OcrMainActivity.this, "Save snapshot to " + savedImagePath, Toast.LENGTH_SHORT).show(); + TYPE = BTN_SHUTTER; + shutterAndPauseCamera(); + resultView.setAdapter(null); break; case R.id.btn_settings: startActivity(new Intent(OcrMainActivity.this, OcrSettingsActivity.class)); @@ -102,6 +134,109 @@ public void onClick(View v) { case R.id.back_in_preview: finish(); break; + case R.id.iv_select: + TYPE = ALBUM_SELECT; + // Judge whether authority has been granted. + if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED) { + // If this permission was requested before the application but the user refused the request, this method will return true. + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE}, REQUEST_PERMISSION_CODE_STORAGE); + } else { + Intent intent = new Intent(Intent.ACTION_PICK); + intent.setType("image/*"); + startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); + } + resultView.setAdapter(null); + break; + case R.id.back_in_result: + back(); + break; + } + } + + @Override + public void onBackPressed() { + super.onBackPressed(); + back(); + } + + private void back() { + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + results.clear(); + if (texts != null) { + texts = null; + } + if (recScores != null) { + recScores = null; + } + } + + private void shutterAndPauseCamera() { + new Thread(new Runnable() { + @Override + public void run() { + try { + // Sleep some times to ensure picture has been correctly shut. + Thread.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + } catch (InterruptedException e) { + e.printStackTrace(); + } + runOnUiThread(new Runnable() { + @SuppressLint("SetTextI18n") + public void run() { + // These code will run in main thread. + svPreview.onPause(); + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + if (shutterBitmap != null && !shutterBitmap.isRecycled()) { + resultImage.setImageBitmap(shutterBitmap); + } else { + new AlertDialog.Builder(OcrMainActivity.this) + .setTitle("Empty Result!") + .setMessage("Current picture is empty, please shutting it again!") + .setCancelable(true) + .show(); + } + } + }); + + } + }).start(); + } + + private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { + if (isShutterBitmapCopied || ARGB8888ImageBitmap == null) { + return; + } + if (!ARGB8888ImageBitmap.isRecycled()) { + synchronized (this) { + shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + } + SystemClock.sleep(TIME_SLEEP_INTERVAL); + isShutterBitmapCopied = true; + } + } + + + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (requestCode == INTENT_CODE_PICK_IMAGE) { + if (resultCode == Activity.RESULT_OK) { + cameraPageView.setVisibility(View.GONE); + resultPageView.setVisibility(View.VISIBLE); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + Uri uri = data.getData(); + String path = getRealPathFromURI(this, uri); + picBitmap = decodeBitmap(path, 720, 1280); + resultImage.setImageBitmap(picBitmap); + } } } @@ -126,29 +261,27 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { - String savedImagePath = ""; - synchronized (this) { - savedImagePath = OcrMainActivity.this.savedImagePath; + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + return false; } boolean modified = false; - OCRResult result = predictor.predict(ARGB8888ImageBitmap, savedImagePath); + long tc = System.currentTimeMillis(); + OCRResult result = predictor.predict(ARGB8888ImageBitmap, true); + timeElapsed += (System.currentTimeMillis() - tc); + Visualize.visOcr(ARGB8888ImageBitmap, result); modified = result.initialized(); - if (!savedImagePath.isEmpty()) { - synchronized (this) { - OcrMainActivity.this.savedImagePath = "result.jpg"; - } - } - lastFrameIndex++; - if (lastFrameIndex >= 30) { - final int fps = (int) (lastFrameIndex * 1e9 / (System.nanoTime() - lastFrameTime)); + frameCounter++; + if (frameCounter >= 30) { + final int fps = (int) (1000 / (timeElapsed / 30)); runOnUiThread(new Runnable() { @SuppressLint("SetTextI18n") public void run() { tvStatus.setText(Integer.toString(fps) + "fps"); } }); - lastFrameIndex = 0; - lastFrameTime = System.nanoTime(); + frameCounter = 0; + timeElapsed = 0; } return modified; } @@ -180,6 +313,7 @@ protected void onDestroy() { } public void initView() { + TYPE = REALTIME_DETECT; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); tvStatus = (TextView) findViewById(R.id.tv_status); @@ -193,6 +327,75 @@ public void initView() { realtimeToggleButton.setOnClickListener(this); backInPreview = findViewById(R.id.back_in_preview); backInPreview.setOnClickListener(this); + albumSelectButton = findViewById(R.id.iv_select); + albumSelectButton.setOnClickListener(this); + cameraPageView = findViewById(R.id.camera_page); + resultPageView = findViewById(R.id.result_page); + resultImage = findViewById(R.id.result_image); + backInResult = findViewById(R.id.back_in_result); + backInResult.setOnClickListener(this); + confidenceSeekbar = findViewById(R.id.confidence_seekbar); + seekbarText = findViewById(R.id.seekbar_text); + resultView = findViewById(R.id.result_list_view); + + confidenceSeekbar.setMax(100); + confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { + @Override + public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { + float resultConfidence = seekBar.getProgress() / 100f; + BigDecimal bd = new BigDecimal(resultConfidence); + resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); + seekbarText.setText(resultNum + ""); + confidenceSeekbar.setProgress((int) (resultNum * 100)); + results.clear(); + } + + @Override + public void onStartTrackingTouch(SeekBar seekBar) { + + } + + @Override + public void onStopTrackingTouch(SeekBar seekBar) { + runOnUiThread(new Runnable() { + @Override + public void run() { + if (TYPE == ALBUM_SELECT) { + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); + detail(picBitmap); + } else { + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); + svPreview.onPause(); + detail(shutterBitmap); + } + } + }); + } + }); + } + + private void detail(Bitmap bitmap) { + OCRResult result = predictor.predict(bitmap, true); + if (texts == null) { + texts = result.mText; + } + if (recScores == null) { + recScores = result.mRecScores; + } + initialized = result.initialized(); + if (initialized) { + for (int i = 0; i < texts.length; i++) { + if (recScores[i] > resultNum) { + results.add(new BaseResultModel(i + 1, texts[i], recScores[i])); + } + } + } + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + resultView.setAdapter(adapter); + resultView.invalidate(); + + resultImage.setImageBitmap(bitmap); + resultNum = 1.0f; } @SuppressLint("ApplySharedPref") @@ -215,9 +418,9 @@ public void checkAndUpdateSettings() { String realDetModelDir = realModelDir + "/" + detModelName; String realClsModelDir = realModelDir + "/" + clsModelName; String realRecModelDir = realModelDir + "/" + recModelName; - String srcDetModelDir = OcrSettingsActivity.modelDir + "/" + detModelName; - String srcClsModelDir = OcrSettingsActivity.modelDir + "/" + clsModelName; - String srcRecModelDir = OcrSettingsActivity.modelDir + "/" + recModelName; + String srcDetModelDir = OcrSettingsActivity.modelDir + "/" + detModelName; + String srcClsModelDir = OcrSettingsActivity.modelDir + "/" + clsModelName; + String srcRecModelDir = OcrSettingsActivity.modelDir + "/" + recModelName; Utils.copyDirectoryFromAssets(this, srcDetModelDir, realDetModelDir); Utils.copyDirectoryFromAssets(this, srcClsModelDir, realClsModelDir); Utils.copyDirectoryFromAssets(this, srcRecModelDir, realRecModelDir); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 404252814de..57624477ab8 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -1,8 +1,5 @@ package com.baidu.paddle.fastdeploy.app.examples.segmentation; -import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; -import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; - import android.Manifest; import android.annotation.SuppressLint; import android.app.Activity; @@ -31,16 +28,17 @@ import com.baidu.paddle.fastdeploy.RuntimeOption; import com.baidu.paddle.fastdeploy.app.examples.R; -import com.baidu.paddle.fastdeploy.app.ui.Utils; import com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView; import com.baidu.paddle.fastdeploy.app.ui.view.ResultListView; -import com.baidu.paddle.fastdeploy.app.ui.view.adapter.BaseResultAdapter; +import com.baidu.paddle.fastdeploy.app.ui.Utils; import com.baidu.paddle.fastdeploy.app.ui.view.model.BaseResultModel; import com.baidu.paddle.fastdeploy.vision.SegmentationResult; import com.baidu.paddle.fastdeploy.vision.Visualize; import com.baidu.paddle.fastdeploy.vision.segmentation.PaddleSegModel; -import java.io.File; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.decodeBitmap; +import static com.baidu.paddle.fastdeploy.app.ui.Utils.getRealPathFromURI; + import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -64,11 +62,9 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi private SeekBar confidenceSeekbar; private TextView seekbarText; private float resultNum = 1.0f; - private ResultListView detectResultView; + private ResultListView resultView; private Bitmap shutterBitmap; - private Bitmap originShutterBitmap; private Bitmap picBitmap; - private Bitmap originPicBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -81,12 +77,12 @@ public class SegmentationMainActivity extends Activity implements View.OnClickLi private static final int INTENT_CODE_PICK_IMAGE = 100; private static final int TIME_SLEEP_INTERVAL = 50; // ms - String savedImagePath = "result.jpg"; long timeElapsed = 0; long frameCounter = 0; // Call 'init' and 'release' manually later PaddleSegModel predictor = new PaddleSegModel(); + private List results = new ArrayList<>(); @Override protected void onCreate(Bundle savedInstanceState) { @@ -120,6 +116,7 @@ public void onClick(View v) { case R.id.btn_shutter: TYPE = BTN_SHUTTER; shutterAndPauseCamera(); + resultView.setAdapter(null); break; case R.id.btn_settings: startActivity(new Intent(SegmentationMainActivity.this, SegmentationSettingsActivity.class)); @@ -141,17 +138,29 @@ public void onClick(View v) { intent.setType("image/*"); startActivityForResult(intent, INTENT_CODE_PICK_IMAGE); } + resultView.setAdapter(null); break; case R.id.back_in_result: - resultPageView.setVisibility(View.GONE); - cameraPageView.setVisibility(View.VISIBLE); - TYPE = REALTIME_DETECT; - isShutterBitmapCopied = false; - svPreview.onResume(); + back(); break; } } + @Override + public void onBackPressed() { + super.onBackPressed(); + back(); + } + + private void back() { + resultPageView.setVisibility(View.GONE); + cameraPageView.setVisibility(View.VISIBLE); + TYPE = REALTIME_DETECT; + isShutterBitmapCopied = false; + svPreview.onResume(); + results.clear(); + } + private void shutterAndPauseCamera() { new Thread(new Runnable() { @Override @@ -172,7 +181,7 @@ public void run() { seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); if (shutterBitmap != null && !shutterBitmap.isRecycled()) { - resultImage.setImageBitmap(shutterBitmap); + detail(shutterBitmap); } else { new AlertDialog.Builder(SegmentationMainActivity.this) .setTitle("Empty Result!") @@ -193,7 +202,6 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); - originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -211,9 +219,10 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { confidenceSeekbar.setProgress((int) (resultNum * 100)); Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); - picBitmap = decodeBitmap(path, 720, 1280); - originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); - resultImage.setImageBitmap(picBitmap); + Bitmap bitmap = decodeBitmap(path, 720, 1280); + picBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true); + SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms + detail(picBitmap); } } } @@ -244,27 +253,12 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } - - String savedImagePath = ""; - synchronized (this) { - savedImagePath = Utils.getDCIMDirectory() + File.separator + "result.jpg"; - } - boolean modified = false; - long tc = System.currentTimeMillis(); SegmentationResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); - Visualize.visSegmentation(ARGB8888ImageBitmap, result); - modified = result.initialized(); - if (!savedImagePath.isEmpty()) { - synchronized (this) { - SegmentationMainActivity.this.savedImagePath = "result.jpg"; - } - } - frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -308,16 +302,8 @@ protected void onDestroy() { public void initView() { TYPE = REALTIME_DETECT; - // For front face camera and human seg, the smaller width and height - // may get both better result and performance. EXPECTED_PREVIEW_HEIGHT - // should be 'width' and EXPECTED_PREVIEW_WIDTH should be 'height' if - // your camera display orientation is rotate (degree == 90 || degree == 270). - // The transformation will auto process in camera. - CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 360; - CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); - svPreview.switchCamera(); // switch to front camera for human seg tvStatus = (TextView) findViewById(R.id.tv_status); btnSwitch = (ImageButton) findViewById(R.id.btn_switch); btnSwitch.setOnClickListener(this); @@ -338,16 +324,7 @@ public void initView() { backInResult.setOnClickListener(this); confidenceSeekbar = findViewById(R.id.confidence_seekbar); seekbarText = findViewById(R.id.seekbar_text); - detectResultView = findViewById(R.id.result_list_view); - - List results = new ArrayList<>(); - // TODO: add model results from SegmentationResult instead of using fake data. - results.add(new BaseResultModel(1, "human", 1.0f)); - results.add(new BaseResultModel(2, "human", 1.0f)); - results.add(new BaseResultModel(3, "human", 1.0f)); - final BaseResultAdapter adapter = new BaseResultAdapter(this, R.layout.facedet_result_page_item, results); - detectResultView.setAdapter(adapter); - detectResultView.invalidate(); + resultView = findViewById(R.id.result_list_view); confidenceSeekbar.setMax(100); confidenceSeekbar.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() { @@ -358,6 +335,7 @@ public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { resultNum = bd.setScale(1, BigDecimal.ROUND_HALF_UP).floatValue(); seekbarText.setText(resultNum + ""); confidenceSeekbar.setProgress((int) (resultNum * 100)); + results.clear(); } @Override @@ -367,32 +345,29 @@ public void onStartTrackingTouch(SeekBar seekBar) { @Override public void onStopTrackingTouch(SeekBar seekBar) { - runOnUiThread(new Runnable() { - @Override - public void run() { - if (TYPE == ALBUM_SELECT) { - SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms - if (!picBitmap.isRecycled()) { - predictor.predict(picBitmap, true, resultNum); - resultImage.setImageBitmap(picBitmap); - picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); - } - resultNum = 1.0f; - } else { - SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms - if (!shutterBitmap.isRecycled()) { - predictor.predict(shutterBitmap, true, resultNum); - resultImage.setImageBitmap(shutterBitmap); - shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); - } - resultNum = 1.0f; - } - } - }); +// runOnUiThread(new Runnable() { +// @Override +// public void run() { +// if (TYPE == ALBUM_SELECT) { +// SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms +// detail(picBitmap); +// } else { +// SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms +// svPreview.onPause(); +// detail(shutterBitmap); +// } +// } +// }); } }); } + private void detail(Bitmap bitmap) { + predictor.predict(bitmap, true, 0.6f); + resultImage.setImageBitmap(bitmap); + resultNum = 1.0f; + } + @SuppressLint("ApplySharedPref") public void initSettings() { SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); @@ -448,32 +423,5 @@ private boolean checkAllPermissions() { return ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED && ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - +} diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java index 427da7f2456..6da24c2f6af 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java @@ -16,12 +16,16 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; +import java.util.ArrayList; import java.util.List; public class Utils { @@ -286,4 +290,24 @@ public static String getRealPathFromURI(Context context, Uri contentURI) { } return result; } + + public static List readTxt(String txtPath) { + File file = new File(txtPath); + if (file.isFile() && file.exists()) { + try { + FileInputStream fileInputStream = new FileInputStream(file); + InputStreamReader inputStreamReader = new InputStreamReader(fileInputStream); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + String text; + List labels = new ArrayList<>(); + while ((text = bufferedReader.readLine()) != null) { + labels.add(text); + } + return labels; + } catch (Exception e) { + e.printStackTrace(); + } + } + return null; + } } diff --git a/java/android/app/src/main/res/layout/classification_camera_page.xml b/java/android/app/src/main/res/layout/classification_camera_page.xml index 15b009e2449..441c3e59112 100644 --- a/java/android/app/src/main/res/layout/classification_camera_page.xml +++ b/java/android/app/src/main/res/layout/classification_camera_page.xml @@ -64,7 +64,7 @@ android:layout_centerInParent="true" /> - Date: Thu, 17 Nov 2022 19:21:58 +0800 Subject: [PATCH 055/136] [Doc] Fix ResNet PIPNet and Facelandmark1000 comments (#622) * first commit for yolov7 * pybind for yolov7 * CPP README.md * CPP README.md * modified yolov7.cc * README.md * python file modify * delete license in fastdeploy/ * repush the conflict part * README.md modified * README.md modified * file path modified * file path modified * file path modified * file path modified * file path modified * README modified * README modified * move some helpers to private * add examples for yolov7 * api.md modified * api.md modified * api.md modified * YOLOv7 * yolov7 release link * yolov7 release link * yolov7 release link * copyright * change some helpers to private * change variables to const and fix documents. * gitignore * Transfer some funtions to private member of class * Transfer some funtions to private member of class * Merge from develop (#9) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * first commit for yolor * for merge * Develop (#11) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Yolor (#16) * Develop (#11) (#12) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * Develop (#13) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * documents * Develop (#14) * Fix compile problem in different python version (#26) * fix some usage problem in linux * Fix compile problem Co-authored-by: root * Add PaddleDetetion/PPYOLOE model support (#22) * add ppdet/ppyoloe * Add demo code and documents * add convert processor to vision (#27) * update .gitignore * Added checking for cmake include dir * fixed missing trt_backend option bug when init from trt * remove un-need data layout and add pre-check for dtype * changed RGB2BRG to BGR2RGB in ppcls model * add model_zoo yolov6 c++/python demo * fixed CMakeLists.txt typos * update yolov6 cpp/README.md * add yolox c++/pybind and model_zoo demo * move some helpers to private * fixed CMakeLists.txt typos * add normalize with alpha and beta * add version notes for yolov5/yolov6/yolox * add copyright to yolov5.cc * revert normalize * fixed some bugs in yolox * fixed examples/CMakeLists.txt to avoid conflicts * add convert processor to vision * format examples/CMakeLists summary * Fix bug while the inference result is empty with YOLOv5 (#29) * Add multi-label function for yolov5 * Update README.md Update doc * Update fastdeploy_runtime.cc fix variable option.trt_max_shape wrong name * Update runtime_option.md Update resnet model dynamic shape setting name from images to x * Fix bug when inference result boxes are empty * Delete detection.py Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> * add is_dynamic for YOLO series (#22) * modify ppmatting backend and docs * modify ppmatting docs * fix the PPMatting size problem * fix LimitShort's log * retrigger ci * modify PPMatting docs * modify the way for dealing with LimitShort * add python comments for external models * modify resnet c++ comments * modify C++ comments for external models * modify python comments and add result class comments * fix comments compile error * modify result.h comments * fix resnet, pipnet and facelandmark1000 comments Co-authored-by: Jason Co-authored-by: root Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: Jason <928090362@qq.com> --- docs/api_docs/python/face_alignment.md | 16 ++++++++++++++++ .../vision/classification/contrib/resnet.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/api_docs/python/face_alignment.md b/docs/api_docs/python/face_alignment.md index 85f79e2f052..9cb06ec26f9 100644 --- a/docs/api_docs/python/face_alignment.md +++ b/docs/api_docs/python/face_alignment.md @@ -7,3 +7,19 @@ :members: :inherited-members: ``` + +## fastdeploy.vision.facealign.FaceLandmark1000 + +```{eval-rst} +.. autoclass:: fastdeploy.vision.facealign.FaceLandmark1000 + :members: + :inherited-members: +``` + +## fastdeploy.vision.facealign.PIPNet + +```{eval-rst} +.. autoclass:: fastdeploy.vision.facealign.PIPNet + :members: + :inherited-members: +``` diff --git a/python/fastdeploy/vision/classification/contrib/resnet.py b/python/fastdeploy/vision/classification/contrib/resnet.py index 46383c15916..19f200d7897 100644 --- a/python/fastdeploy/vision/classification/contrib/resnet.py +++ b/python/fastdeploy/vision/classification/contrib/resnet.py @@ -24,7 +24,7 @@ def __init__(self, params_file="", runtime_option=None, model_format=ModelFormat.ONNX): - """Load a image classification model exported by ResNet. + """Load a image classification model exported by torchvision.ResNet. :param model_file: (str)Path of model file, e.g resnet/resnet50.onnx :param params_file: (str)Path of parameters file, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string From dde0bf2afb898cf32d9c6e0ad4c7f494bd10010e Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 17 Nov 2022 21:21:46 +0800 Subject: [PATCH 056/136] [Doc] update version in docs (#623) update version in docs --- .../download_prebuilt_libraries.md | 24 ++++++------ .../download_prebuilt_libraries.md | 38 +++++++++---------- examples/text/ernie-3.0/cpp/README.md | 8 ++-- examples/text/uie/cpp/README.md | 8 ++-- .../classification/paddleclas/cpp/README.md | 8 ++-- .../paddleclas/quantize/cpp/README.md | 6 +-- .../classification/yolov5cls/cpp/README.md | 6 +-- .../detection/nanodet_plus/cpp/README.md | 6 +-- .../detection/paddledetection/cpp/README.md | 8 ++-- .../paddledetection/quantize/cpp/README.md | 6 +-- .../detection/scaledyolov4/cpp/README.md | 6 +-- examples/vision/detection/yolor/cpp/README.md | 6 +-- .../vision/detection/yolov5/cpp/README.md | 6 +-- .../detection/yolov5/quantize/cpp/README.md | 6 +-- .../vision/detection/yolov5lite/cpp/README.md | 6 +-- .../vision/detection/yolov6/cpp/README.md | 6 +-- .../detection/yolov6/quantize/cpp/README.md | 6 +-- .../vision/detection/yolov7/cpp/README.md | 6 +-- .../detection/yolov7/quantize/cpp/README.md | 6 +-- .../detection/yolov7end2end_ort/cpp/README.md | 6 +-- .../detection/yolov7end2end_trt/cpp/README.md | 6 +-- examples/vision/detection/yolox/cpp/README.md | 6 +-- .../vision/facedet/retinaface/cpp/README.md | 6 +-- examples/vision/facedet/scrfd/cpp/README.md | 6 +-- .../vision/facedet/ultraface/cpp/README.md | 6 +-- .../vision/facedet/yolov5face/cpp/README.md | 6 +-- examples/vision/faceid/adaface/cpp/README.md | 6 +-- .../vision/faceid/insightface/cpp/README.md | 6 +-- .../det_keypoint_unite/cpp/README.md | 8 ++-- .../keypointdetection/tiny_pose/cpp/README.md | 8 ++-- examples/vision/matting/modnet/cpp/README.md | 6 +-- .../vision/matting/ppmatting/cpp/README.md | 8 ++-- examples/vision/matting/rvm/cpp/README.md | 8 ++-- examples/vision/ocr/PP-OCRv2/cpp/README.md | 6 +-- examples/vision/ocr/PP-OCRv3/cpp/README.md | 6 +-- .../segmentation/paddleseg/cpp/README.md | 8 ++-- .../paddleseg/quantize/cpp/README.md | 6 +-- .../vision/tracking/pptracking/cpp/README.md | 8 ++-- 38 files changed, 149 insertions(+), 149 deletions(-) diff --git a/docs/cn/build_and_install/download_prebuilt_libraries.md b/docs/cn/build_and_install/download_prebuilt_libraries.md index 28db5510487..f296363e6b7 100755 --- a/docs/cn/build_and_install/download_prebuilt_libraries.md +++ b/docs/cn/build_and_install/download_prebuilt_libraries.md @@ -18,7 +18,7 @@ FastDeploy提供各平台预编译库，供开发者直接下载安装使用。 ### Python安装 -Release版本（当前最新0.6.0）安装 +Release版本（当前最新0.7.0）安装 ```bash pip install fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html ``` @@ -35,12 +35,12 @@ conda config --add channels conda-forge && conda install cudatoolkit=11.2 cudnn= ### C++ SDK安装 -Release版本（当前最新0.6.0） +Release版本（当前最新0.7.0） | 平台 | 文件 | 说明 | | :--- | :--- | :---- | -| Linux x64 | [fastdeploy-linux-x64-gpu-0.6.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz) | g++ 8.2, CUDA 11.2, cuDNN 8.2编译产出 | -| Windows x64 | [fastdeploy-win-x64-gpu-0.6.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-gpu-0.6.0.zip) | Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2编译产出 | +| Linux x64 | [fastdeploy-linux-x64-gpu-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz) | g++ 8.2, CUDA 11.2, cuDNN 8.2编译产出 | +| Windows x64 | [fastdeploy-win-x64-gpu-0.7.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-gpu-0.7.0.zip) | Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2编译产出 | Develop版本（Nightly build） @@ -59,7 +59,7 @@ Develop版本（Nightly build） ### Python安装 -Release版本（当前最新0.6.0）安装 +Release版本（当前最新0.7.0）安装 ```bash pip install fastdeploy-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html ``` @@ -71,15 +71,15 @@ pip install fastdeploy-python==0.0.0 -f https://www.paddlepaddle.org.cn/whl/fast ## C++ SDK安装 -Release版本（当前最新0.6.0，Android为0.4.0 pre-release） +Release版本（当前最新0.7.0，Android为0.4.0 pre-release） | 平台 | 文件 | 说明 | | :--- | :--- | :---- | -| Linux x64 | [fastdeploy-linux-x64-0.6.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz) | g++ 8.2编译产出 | -| Windows x64 | [fastdeploy-win-x64-0.6.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-0.6.0.zip) | Visual Studio 16 2019编译产出 | -| Mac OSX x64 | [fastdeploy-osx-x86_64-0.4.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-x86_64-0.4.0.tgz) | clang++ 10.0.0编译产出| -| Mac OSX arm64 | [fastdeploy-osx-arm64-0.4.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-arm64-0.4.0.tgz) | clang++ 13.0.0编译产出 | -| Linux aarch64 | [fastdeploy-linux-aarch64-0.4.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-aarch64-0.4.0.tgz) | g++ 6.3.0编译产出 | +| Linux x64 | [fastdeploy-linux-x64-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz) | g++ 8.2编译产出 | +| Windows x64 | [fastdeploy-win-x64-0.7.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-0.7.0.zip) | Visual Studio 16 2019编译产出 | +| Mac OSX x64 | [fastdeploy-osx-x86_64-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-x86_64-0.7.0.tgz) | clang++ 10.0.0编译产出| +| Mac OSX arm64 | [fastdeploy-osx-arm64-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-arm64-0.7.0.tgz) | clang++ 13.0.0编译产出 | +| Linux aarch64 | - | 自行编译，可集成ONNX Runtime、Paddle Lite后端 | | Android armv7&v8 | [fastdeploy-android-0.4.0-shared.tgz](https://bj.bcebos.com/fastdeploy/release/android/fastdeploy-android-0.4.0-shared.tgz) | NDK 25及clang++编译产出, 支持arm64-v8a及armeabi-v7a | Develop版本（Nightly build） @@ -88,7 +88,7 @@ Develop版本（Nightly build） | :--- | :--- | :---- | | Linux x64 | [fastdeploy-linux-x64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-0.0.0.tgz) | g++ 8.2编译产出 | | Windows x64 | [fastdeploy-win-x64-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-0.0.0.zip) | Visual Studio 16 2019编译产出 | -| Mac OSX x64 | - | - | +| Mac OSX x64 | [fastdeploy-osx-arm64-0.0.0.tgz](https://bj.bcebos.com/fastdeploy/dev/cpp/fastdeploy-osx-arm64-0.0.0.tgz) | - | | Mac OSX arm64 | [fastdeploy-osx-arm64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-osx-arm64-0.0.0.tgz) | clang++ 13.0.0编译产出 | | Linux aarch64 | - | - | | Android armv7&v8 | - | - | diff --git a/docs/en/build_and_install/download_prebuilt_libraries.md b/docs/en/build_and_install/download_prebuilt_libraries.md index 99f4f5ea854..565d39d93d9 100755 --- a/docs/en/build_and_install/download_prebuilt_libraries.md +++ b/docs/en/build_and_install/download_prebuilt_libraries.md @@ -19,7 +19,7 @@ FastDeploy supports Computer Vision, Text and NLP model deployment on CPU and Nv ### Python SDK -Install the released version（the newest 0.6.0 for now） +Install the released version（the newest 0.7.0 for now） ``` pip install fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html @@ -39,19 +39,19 @@ conda config --add channels conda-forge && conda install cudatoolkit=11.2 cudnn= ### C++ SDK -Install the released version（Latest 0.6.0） +Install the released version（Latest 0.7.0） | Platform | File | Description | |:----------- |:--------------------------------------------------------------------------------------------------------------------- |:--------------------------------------------------------- | -| Linux x64 | [fastdeploy-linux-x64-gpu-0.6.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz) | Compiled from g++ 8.2, CUDA 11.2, cuDNN 8.2 | -| Windows x64 | [fastdeploy-win-x64-gpu-0.6.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-gpu-0.6.0.zip) | Compiled from Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | +| Linux x64 | [fastdeploy-linux-x64-gpu-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz) | g++ 8.2, CUDA 11.2, cuDNN 8.2 | +| Windows x64 | [fastdeploy-win-x64-gpu-0.7.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-gpu-0.7.0.zip) | Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | Install the Develop version（Nightly build） | Platform | File | Description | |:----------- |:--------------------------------------------------------------------------------------------------------------------- |:--------------------------------------------------------- | -| Linux x64 | [fastdeploy-linux-x64-gpu-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-gpu-0.0.0.tgz) | Compiled from g++ 8.2, CUDA 11.2, cuDNN 8.2 | -| Windows x64 | [fastdeploy-win-x64-gpu-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-gpu-0.0.0.zip) | Compiled from Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | +| Linux x64 | [fastdeploy-linux-x64-gpu-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-gpu-0.0.0.tgz) | g++ 8.2, CUDA 11.2, cuDNN 8.2 | +| Windows x64 | [fastdeploy-win-x64-gpu-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-gpu-0.0.0.zip) | Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | ## CPU Deployment Environment @@ -64,7 +64,7 @@ FastDeploy supports computer vision, text and NLP model deployment on CPU with P ### Python SDK -Install the released version（Latest 0.6.0 for now） +Install the released version（Latest 0.7.0 for now） ``` pip install fastdeploy-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html @@ -78,24 +78,24 @@ pip install fastdeploy-python==0.0.0 -f https://www.paddlepaddle.org.cn/whl/fast ### C++ SDK -Install the released version（Latest 0.6.0 for now, Android is 0.4.0 pre-release） +Install the released version（Latest 0.7.0 for now, Android is 0.4.0 pre-release） | Platform | File | Description | |:------------- |:--------------------------------------------------------------------------------------------------------------------- |:------------------------------ | -| Linux x64 | [fastdeploy-linux-x64-0.6.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz) | Compiled from g++ 8.2 | -| Windows x64 | [fastdeploy-win-x64-0.6.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-0.6.0.zip) | Compiled from Visual Studio 16 | -| Mac OSX x64 | [fastdeploy-osx-x86_64-0.4.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-x86_64-0.4.0.tgz) | - | -| Mac OSX arm64 | [fastdeploy-osx-arm64-0.4.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-arm64-0.4.0.tgz) | - | -| Linux aarch64 | [fastdeploy-linux-aarch64-0.4.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-aarch64-0.4.0.tgz) | Compiled from g++ 6.3.0 | -| Android armv7&v8 | [fastdeploy-android-0.4.0-shared.tgz](https://bj.bcebos.com/fastdeploy/release/android/fastdeploy-android-0.4.0-shared.tgz) | Compiled from NDK 25 and clang++, support arm64-v8a and armeabi-v7a | +| Linux x64 | [fastdeploy-linux-x64-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz) | g++ 8.2 | +| Windows x64 | [fastdeploy-win-x64-0.7.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-0.7.0.zip) | Visual Studio 16 2019 | +| Mac OSX x64 | [fastdeploy-osx-x86_64-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-x86_64-0.7.0.tgz) | clang++ 10.0.0| +| Mac OSX arm64 | [fastdeploy-osx-arm64-0.7.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-arm64-0.7.0.tgz) | clang++ 13.0.0 | +| Linux aarch64 | - | - | +| Android armv7&v8 | [fastdeploy-android-0.4.0-shared.tgz](https://bj.bcebos.com/fastdeploy/release/android/fastdeploy-android-0.4.0-shared.tgz) | NDK 25, clang++, support arm64-v8a及armeabi-v7a | Install the Develop version（Nightly build） | Platform | File | Description | |:------------- |:--------------------------------------------------------------------------------------------------------------------- |:------------------------------ | -| Linux x64 | [fastdeploy-linux-x64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-0.0.0.tgz) | Compiled from g++ 8.2 | -| Windows x64 | [fastdeploy-win-x64-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-0.0.0.zip) | Compiled from Visual Studio 16 | -| Mac OSX x64 | - | - | -| Mac OSX arm64 | [fastdeploy-osx-arm64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-osx-arm64-0.0.0.tgz) | - | -| Linux aarch64 | - | - | +| Linux x64 | [fastdeploy-linux-x64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-0.0.0.tgz) | g++ 8.2 | +| Windows x64 | [fastdeploy-win-x64-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-0.0.0.zip) | Visual Studio 16 2019 | +| Mac OSX x64 | [fastdeploy-osx-arm64-0.0.0.tgz](https://bj.bcebos.com/fastdeploy/dev/cpp/fastdeploy-osx-arm64-0.0.0.tgz) | - | +| Mac OSX arm64 | [fastdeploy-osx-arm64-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-osx-arm64-0.0.0.tgz) | clang++ 13.0.0编译产出 | +| Linux aarch64 | - | - | | Android armv7&v8 | - | - | diff --git a/examples/text/ernie-3.0/cpp/README.md b/examples/text/ernie-3.0/cpp/README.md index a172b06e3a0..0882a907076 100644 --- a/examples/text/ernie-3.0/cpp/README.md +++ b/examples/text/ernie-3.0/cpp/README.md @@ -16,14 +16,14 @@ ```bash # 下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/text/ernie-3.0/cpp +cd fastdeploy-linux-x64-gpu-0.7.0/examples/text/ernie-3.0/cpp mkdir build cd build # 执行cmake，需要指定FASTDEPLOY_INSTALL_DIR为FastDeploy SDK的目录。 -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载AFQMC数据集的微调后的ERNIE 3.0模型以及词表 diff --git a/examples/text/uie/cpp/README.md b/examples/text/uie/cpp/README.md index c4fcc7f4f5f..cfb045d8a91 100644 --- a/examples/text/uie/cpp/README.md +++ b/examples/text/uie/cpp/README.md @@ -12,13 +12,13 @@ ``` #下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/text/uie/cpp +cd fastdeploy-linux-x64-gpu-0.7.0/examples/text/uie/cpp mkdir build cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载uie-base模型以及词表 diff --git a/examples/vision/classification/paddleclas/cpp/README.md b/examples/vision/classification/paddleclas/cpp/README.md index 688ec80a209..7d71f472370 100644 --- a/examples/vision/classification/paddleclas/cpp/README.md +++ b/examples/vision/classification/paddleclas/cpp/README.md @@ -11,12 +11,12 @@ ```bash #下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/classification/paddleclas/cpp +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/classification/paddleclas/cpp mkdir build cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载ResNet50_vd模型文件和测试图片 diff --git a/examples/vision/classification/paddleclas/quantize/cpp/README.md b/examples/vision/classification/paddleclas/quantize/cpp/README.md index cb68fcd5ed2..681d1e3f704 100644 --- a/examples/vision/classification/paddleclas/quantize/cpp/README.md +++ b/examples/vision/classification/paddleclas/quantize/cpp/README.md @@ -15,9 +15,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载FastDeloy提供的ResNet50_Vd量化模型文件和测试图片 diff --git a/examples/vision/classification/yolov5cls/cpp/README.md b/examples/vision/classification/yolov5cls/cpp/README.md index 56f994a66e5..e56a6c1af13 100755 --- a/examples/vision/classification/yolov5cls/cpp/README.md +++ b/examples/vision/classification/yolov5cls/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的yolov5模型文件和测试图片 diff --git a/examples/vision/detection/nanodet_plus/cpp/README.md b/examples/vision/detection/nanodet_plus/cpp/README.md index 81f1c37fe2a..5d575457bb0 100644 --- a/examples/vision/detection/nanodet_plus/cpp/README.md +++ b/examples/vision/detection/nanodet_plus/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的NanoDetPlus模型文件和测试图片 diff --git a/examples/vision/detection/paddledetection/cpp/README.md b/examples/vision/detection/paddledetection/cpp/README.md index 43af2ddfdf0..95b640eead3 100644 --- a/examples/vision/detection/paddledetection/cpp/README.md +++ b/examples/vision/detection/paddledetection/cpp/README.md @@ -13,11 +13,11 @@ 以ppyoloe为例进行推理部署 #下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/detection/paddledetection/cpp +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/detection/paddledetection/cpp mkdir build && cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载PPYOLOE模型文件和测试图片 diff --git a/examples/vision/detection/paddledetection/quantize/cpp/README.md b/examples/vision/detection/paddledetection/quantize/cpp/README.md index b1eca083485..7e321003fcb 100644 --- a/examples/vision/detection/paddledetection/quantize/cpp/README.md +++ b/examples/vision/detection/paddledetection/quantize/cpp/README.md @@ -16,9 +16,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载FastDeloy提供的ppyoloe_crn_l_300e_coco量化模型文件和测试图片 diff --git a/examples/vision/detection/scaledyolov4/cpp/README.md b/examples/vision/detection/scaledyolov4/cpp/README.md index bda9ea615b9..640eefb9398 100644 --- a/examples/vision/detection/scaledyolov4/cpp/README.md +++ b/examples/vision/detection/scaledyolov4/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的ScaledYOLOv4模型文件和测试图片 diff --git a/examples/vision/detection/yolor/cpp/README.md b/examples/vision/detection/yolor/cpp/README.md index f0c7c035474..d2459994240 100644 --- a/examples/vision/detection/yolor/cpp/README.md +++ b/examples/vision/detection/yolor/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的YOLOR模型文件和测试图片 diff --git a/examples/vision/detection/yolov5/cpp/README.md b/examples/vision/detection/yolov5/cpp/README.md index a8e12715f2d..cc88e3c351d 100644 --- a/examples/vision/detection/yolov5/cpp/README.md +++ b/examples/vision/detection/yolov5/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的yolov5模型文件和测试图片 diff --git a/examples/vision/detection/yolov5/quantize/cpp/README.md b/examples/vision/detection/yolov5/quantize/cpp/README.md index 7adcf7eafc0..ebe941a9348 100644 --- a/examples/vision/detection/yolov5/quantize/cpp/README.md +++ b/examples/vision/detection/yolov5/quantize/cpp/README.md @@ -16,9 +16,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载FastDeloy提供的yolov5s量化模型文件和测试图片 diff --git a/examples/vision/detection/yolov5lite/cpp/README.md b/examples/vision/detection/yolov5lite/cpp/README.md index a3b150bd0dc..cd1c46d7838 100644 --- a/examples/vision/detection/yolov5lite/cpp/README.md +++ b/examples/vision/detection/yolov5lite/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的YOLOv5Lite模型文件和测试图片 diff --git a/examples/vision/detection/yolov6/cpp/README.md b/examples/vision/detection/yolov6/cpp/README.md index 7804bc05f89..00303ffa567 100644 --- a/examples/vision/detection/yolov6/cpp/README.md +++ b/examples/vision/detection/yolov6/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的YOLOv6模型文件和测试图片 diff --git a/examples/vision/detection/yolov6/quantize/cpp/README.md b/examples/vision/detection/yolov6/quantize/cpp/README.md index e28d8f804d5..cc6f7c9b284 100644 --- a/examples/vision/detection/yolov6/quantize/cpp/README.md +++ b/examples/vision/detection/yolov6/quantize/cpp/README.md @@ -16,9 +16,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载FastDeloy提供的yolov6s量化模型文件和测试图片 diff --git a/examples/vision/detection/yolov7/cpp/README.md b/examples/vision/detection/yolov7/cpp/README.md index 21b62b38667..08c59ac7635 100644 --- a/examples/vision/detection/yolov7/cpp/README.md +++ b/examples/vision/detection/yolov7/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的yolov7模型文件和测试图片 diff --git a/examples/vision/detection/yolov7/quantize/cpp/README.md b/examples/vision/detection/yolov7/quantize/cpp/README.md index 5c76eecdfd2..bad31d532e6 100644 --- a/examples/vision/detection/yolov7/quantize/cpp/README.md +++ b/examples/vision/detection/yolov7/quantize/cpp/README.md @@ -16,9 +16,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载FastDeloy提供的yolov7量化模型文件和测试图片 diff --git a/examples/vision/detection/yolov7end2end_ort/cpp/README.md b/examples/vision/detection/yolov7end2end_ort/cpp/README.md index bbe5ab71cc0..63747cb32d1 100644 --- a/examples/vision/detection/yolov7end2end_ort/cpp/README.md +++ b/examples/vision/detection/yolov7end2end_ort/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-gpu-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-gpu-0.7.0 make -j # 如果预编译库还没有支持该模型，请从develop分支源码编译最新的SDK diff --git a/examples/vision/detection/yolov7end2end_trt/cpp/README.md b/examples/vision/detection/yolov7end2end_trt/cpp/README.md index f39d779ab0b..24563efbaf7 100644 --- a/examples/vision/detection/yolov7end2end_trt/cpp/README.md +++ b/examples/vision/detection/yolov7end2end_trt/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-gpu-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-gpu-0.7.0 make -j # 若预编译库没有支持该类 则请自行从源码develop分支编译最新的FastDeploy C++ SDK diff --git a/examples/vision/detection/yolox/cpp/README.md b/examples/vision/detection/yolox/cpp/README.md index a6cbecd1d3c..e9ee5923893 100644 --- a/examples/vision/detection/yolox/cpp/README.md +++ b/examples/vision/detection/yolox/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的YOLOX模型文件和测试图片 diff --git a/examples/vision/facedet/retinaface/cpp/README.md b/examples/vision/facedet/retinaface/cpp/README.md index 1cf4e0c93ea..9ab11da9a76 100644 --- a/examples/vision/facedet/retinaface/cpp/README.md +++ b/examples/vision/facedet/retinaface/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的RetinaFace模型文件和测试图片 diff --git a/examples/vision/facedet/scrfd/cpp/README.md b/examples/vision/facedet/scrfd/cpp/README.md index c491f0ae6e6..669da8c7bdc 100644 --- a/examples/vision/facedet/scrfd/cpp/README.md +++ b/examples/vision/facedet/scrfd/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的SCRFD模型文件和测试图片 diff --git a/examples/vision/facedet/ultraface/cpp/README.md b/examples/vision/facedet/ultraface/cpp/README.md index f7382952030..10fa2e95f5c 100644 --- a/examples/vision/facedet/ultraface/cpp/README.md +++ b/examples/vision/facedet/ultraface/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的UltraFace模型文件和测试图片 diff --git a/examples/vision/facedet/yolov5face/cpp/README.md b/examples/vision/facedet/yolov5face/cpp/README.md index 08a9bdd692a..5be8d9ecda0 100644 --- a/examples/vision/facedet/yolov5face/cpp/README.md +++ b/examples/vision/facedet/yolov5face/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://https://bj.bcebos.com/paddlehub/fastdeploy/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://https://bj.bcebos.com/paddlehub/fastdeploy/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的YOLOv5Face模型文件和测试图片 diff --git a/examples/vision/faceid/adaface/cpp/README.md b/examples/vision/faceid/adaface/cpp/README.md index f26b52089e0..95b939a2edd 100644 --- a/examples/vision/faceid/adaface/cpp/README.md +++ b/examples/vision/faceid/adaface/cpp/README.md @@ -14,9 +14,9 @@ # “如果预编译库不包含本模型，请从最新代码编译SDK” mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载测试图片 diff --git a/examples/vision/faceid/insightface/cpp/README.md b/examples/vision/faceid/insightface/cpp/README.md index e4492ef1a06..71a9261cc03 100644 --- a/examples/vision/faceid/insightface/cpp/README.md +++ b/examples/vision/faceid/insightface/cpp/README.md @@ -12,9 +12,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的ArcFace模型文件和测试图片 diff --git a/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md b/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md index 93bcc953000..a7ccba90354 100644 --- a/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md +++ b/examples/vision/keypointdetection/det_keypoint_unite/cpp/README.md @@ -12,12 +12,12 @@ 以Linux上推理为例，在本目录执行如下命令即可完成编译测试 ```bash -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/keypointdetection/tiny_pose/cpp/ +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/keypointdetection/tiny_pose/cpp/ mkdir build cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载PP-TinyPose和PP-PicoDet模型文件和测试图片 diff --git a/examples/vision/keypointdetection/tiny_pose/cpp/README.md b/examples/vision/keypointdetection/tiny_pose/cpp/README.md index 03c3dbdecda..d2a559c341e 100644 --- a/examples/vision/keypointdetection/tiny_pose/cpp/README.md +++ b/examples/vision/keypointdetection/tiny_pose/cpp/README.md @@ -12,12 +12,12 @@ 以Linux上推理为例，在本目录执行如下命令即可完成编译测试 ```bash -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/keypointdetection/tiny_pose/cpp/ +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/keypointdetection/tiny_pose/cpp/ mkdir build cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载PP-TinyPose模型文件和测试图片 diff --git a/examples/vision/matting/modnet/cpp/README.md b/examples/vision/matting/modnet/cpp/README.md index 89bbdf50bec..0fd773361a4 100644 --- a/examples/vision/matting/modnet/cpp/README.md +++ b/examples/vision/matting/modnet/cpp/README.md @@ -10,11 +10,11 @@ 以Linux上CPU推理为例，在本目录执行如下命令即可完成编译测试 ```bash -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz mkdir build && cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载官方转换好的MODNet模型文件和测试图片 diff --git a/examples/vision/matting/ppmatting/cpp/README.md b/examples/vision/matting/ppmatting/cpp/README.md index 470faf077f1..4ec03d2bf16 100644 --- a/examples/vision/matting/ppmatting/cpp/README.md +++ b/examples/vision/matting/ppmatting/cpp/README.md @@ -11,11 +11,11 @@ ```bash #下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/matting/ppmatting/cpp/ +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/matting/ppmatting/cpp/ mkdir build && cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载PP-Matting模型文件和测试图片 diff --git a/examples/vision/matting/rvm/cpp/README.md b/examples/vision/matting/rvm/cpp/README.md index 1ec399df6d3..c236a717b58 100755 --- a/examples/vision/matting/rvm/cpp/README.md +++ b/examples/vision/matting/rvm/cpp/README.md @@ -11,11 +11,11 @@ ```bash #下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/matting/rvm/cpp/ +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/matting/rvm/cpp/ mkdir build && cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载RobustVideoMatting模型文件和测试图片以及视频 diff --git a/examples/vision/ocr/PP-OCRv2/cpp/README.md b/examples/vision/ocr/PP-OCRv2/cpp/README.md index 45a7e332d69..9f034c09e6f 100644 --- a/examples/vision/ocr/PP-OCRv2/cpp/README.md +++ b/examples/vision/ocr/PP-OCRv2/cpp/README.md @@ -12,9 +12,9 @@ ``` mkdir build cd build -wget https://https://bj.bcebos.com/paddlehub/fastdeploy/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://https://bj.bcebos.com/paddlehub/fastdeploy/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j diff --git a/examples/vision/ocr/PP-OCRv3/cpp/README.md b/examples/vision/ocr/PP-OCRv3/cpp/README.md index a5f2b9ccb21..0d397d4cf7b 100644 --- a/examples/vision/ocr/PP-OCRv3/cpp/README.md +++ b/examples/vision/ocr/PP-OCRv3/cpp/README.md @@ -12,9 +12,9 @@ ``` mkdir build cd build -wget https://https://bj.bcebos.com/paddlehub/fastdeploy/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://https://bj.bcebos.com/paddlehub/fastdeploy/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j diff --git a/examples/vision/segmentation/paddleseg/cpp/README.md b/examples/vision/segmentation/paddleseg/cpp/README.md index 59659463613..01ff7b33fcf 100644 --- a/examples/vision/segmentation/paddleseg/cpp/README.md +++ b/examples/vision/segmentation/paddleseg/cpp/README.md @@ -12,12 +12,12 @@ 以Linux上推理为例，在本目录执行如下命令即可完成编译测试 ```bash -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/segmentation/paddleseg/cpp/ +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/segmentation/paddleseg/cpp/ mkdir build cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载Unet模型文件和测试图片 diff --git a/examples/vision/segmentation/paddleseg/quantize/cpp/README.md b/examples/vision/segmentation/paddleseg/quantize/cpp/README.md index 2ff85d3d020..70e6a839dad 100644 --- a/examples/vision/segmentation/paddleseg/quantize/cpp/README.md +++ b/examples/vision/segmentation/paddleseg/quantize/cpp/README.md @@ -15,9 +15,9 @@ ```bash mkdir build cd build -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.6.0.tgz -tar xvf fastdeploy-linux-x64-0.6.0.tgz -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.6.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-0.7.0.tgz +tar xvf fastdeploy-linux-x64-0.7.0.tgz +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/fastdeploy-linux-x64-0.7.0 make -j #下载FastDeloy提供的PP_LiteSeg_T_STDC1_cityscapes量化模型文件和测试图片 diff --git a/examples/vision/tracking/pptracking/cpp/README.md b/examples/vision/tracking/pptracking/cpp/README.md index af209532835..45b26d0ca55 100644 --- a/examples/vision/tracking/pptracking/cpp/README.md +++ b/examples/vision/tracking/pptracking/cpp/README.md @@ -11,11 +11,11 @@ ```bash #下载SDK，编译模型examples代码（SDK中包含了examples代码） -wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.6.0.tgz -tar xvf fastdeploy-linux-x64-gpu-0.6.0.tgz -cd fastdeploy-linux-x64-gpu-0.6.0/examples/vision/tracking/pptracking/cpp/ +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/tracking/pptracking/cpp/ mkdir build && cd build -cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.6.0 +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 make -j # 下载PP-Tracking模型文件和测试视频 From 1e7ab438f339f4bbbd131052afab6efaf24d07d2 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Thu, 17 Nov 2022 21:37:58 +0800 Subject: [PATCH 057/136] [Bug Fix] fix android app detail page errors (#624) --- java/android/app/src/main/AndroidManifest.xml | 6 +-- .../ClassificationMainActivity.java | 51 +++++++++++-------- .../detection/DetectionMainActivity.java | 48 +++++++++-------- .../examples/facedet/FaceDetMainActivity.java | 34 ++++++++----- .../app/examples/ocr/OcrMainActivity.java | 31 ++++++----- .../SegmentationMainActivity.java | 14 +++-- 6 files changed, 113 insertions(+), 71 deletions(-) diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index bb2df4ce4f4..95511824f5c 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -11,18 +11,18 @@ - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java index e052df7a844..07a462d67de 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java @@ -20,6 +20,7 @@ import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; +import android.util.Log; import android.view.View; import android.view.ViewGroup; import android.view.Window; @@ -64,8 +65,10 @@ public class ClassificationMainActivity extends Activity implements View.OnClick private TextView seekbarText; private float resultNum = 1.0f; private ResultListView resultView; - private Bitmap shutterBitmap; private Bitmap picBitmap; + private Bitmap shutterBitmap; + private Bitmap originPicBitmap; + private Bitmap originShutterBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -103,13 +106,13 @@ protected void onCreate(Bundle savedInstanceState) { // Clear all setting items to avoid app crashing due to the incorrect settings initSettings(); - // Init the camera preview and UI components - initView(); - // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions if (!checkAllPermissions()) { requestAllPermissions(); } + + // Init the camera preview and UI components + initView(); } @SuppressLint("NonConstantResourceId") @@ -216,6 +219,7 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -234,6 +238,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -264,12 +269,16 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } + boolean modified = false; + long tc = System.currentTimeMillis(); - ClassifyResult result = predictor.predict(ARGB8888ImageBitmap); + ClassifyResult result = predictor.predict(ARGB8888ImageBitmap, + true, ClassificationSettingsActivity.scoreThreshold); timeElapsed += (System.currentTimeMillis() - tc); - Visualize.visClassification(ARGB8888ImageBitmap, result, resultNum, 12); + modified = result.initialized(); + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -362,10 +371,11 @@ public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(500); detail(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); } else { SystemClock.sleep(500); - svPreview.onPause(); detail(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); } } }); @@ -374,26 +384,25 @@ public void run() { } private void detail(Bitmap bitmap) { - ClassifyResult result = predictor.predict(bitmap, true, ClassificationSettingsActivity.scoreThreshold); - if (scores == null) { - scores = result.mScores; - } - if (labelId == null) { - labelId = result.mLabelIds; - } + + ClassifyResult result = predictor.predict(bitmap, true, resultNum); + + scores = result.mScores; + labelId = result.mLabelIds; + initialized = result.initialized(); if (initialized) { for (int i = 0; i < labelId.length; i++) { - for (int j = 0; j < labelText.size(); j++) { - if (scores[i] > resultNum) { - if (labelId[i] == Integer.parseInt(labelText.get(j).substring(0, labelText.get(j).indexOf(" ")))) { - results.add(new BaseResultModel(labelId[i], labelText.get(j), scores[i])); - } - } + if (scores[i] > resultNum) { + Log.d(TAG, "[fastdeploy] detail: scores[i]: " + scores[i]); + int idx = labelId[i]; + String text = labelText.get(idx); + text = text.substring(text.indexOf(" ")); + results.add(new BaseResultModel(idx, text, scores[i])); } } } - BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.classification_result_page_item, results); resultView.setAdapter(adapter); resultView.invalidate(); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 3b8c750113a..5a19004ba3e 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -16,6 +16,7 @@ import android.support.annotation.NonNull; import android.support.v4.app.ActivityCompat; import android.support.v4.content.ContextCompat; +import android.util.Log; import android.view.View; import android.view.ViewGroup; import android.view.Window; @@ -64,8 +65,10 @@ public class DetectionMainActivity extends Activity implements View.OnClickListe private TextView seekbarText; private float resultNum = 1.0f; private ResultListView resultView; - private Bitmap shutterBitmap; private Bitmap picBitmap; + private Bitmap shutterBitmap; + private Bitmap originPicBitmap; + private Bitmap originShutterBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -103,13 +106,13 @@ protected void onCreate(Bundle savedInstanceState) { // Clear all setting items to avoid app crashing due to the incorrect settings initSettings(); - // Init the camera preview and UI components - initView(); - // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions if (!checkAllPermissions()) { requestAllPermissions(); } + + // Init the camera preview and UI components + initView(); } @SuppressLint("NonConstantResourceId") @@ -215,6 +218,7 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -233,6 +237,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -263,12 +268,16 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } + boolean modified = false; + long tc = System.currentTimeMillis(); DetectionResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); + Visualize.visDetection(ARGB8888ImageBitmap, result, DetectionSettingsActivity.scoreThreshold); modified = result.initialized(); + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -361,10 +370,12 @@ public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); detail(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); - svPreview.onPause(); + // svPreview.onPause(); detail(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); } } }); @@ -373,27 +384,24 @@ public void run() { } private void detail(Bitmap bitmap) { - DetectionResult result = predictor.predict( - bitmap, true, DetectionSettingsActivity.scoreThreshold); - if (scores == null) { - scores = result.mScores; - } - if (labelId == null) { - labelId = result.mLabelIds; - } + + DetectionResult result = predictor.predict(bitmap, true, resultNum); + + scores = result.mScores; + labelId = result.mLabelIds; + initialized = result.initialized(); if (initialized) { for (int i = 0; i < labelId.length; i++) { - for (int j = 0; j < labelText.size(); j++) { - if (scores[i] > resultNum) { - if (labelId[i] == j) { - results.add(new BaseResultModel(labelId[i], labelText.get(j), scores[i])); - } - } + if (scores[i] > resultNum) { + Log.d(TAG, "[fastdeploy] detail: scores[i]: " + scores[i]); + int idx = labelId[i]; + String text = labelText.get(idx); + results.add(new BaseResultModel(idx, text, scores[i])); } } } - BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.detection_result_page_item, results); resultView.setAdapter(adapter); resultView.invalidate(); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index 934f568b3af..e2883baa3ca 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -64,8 +64,10 @@ public class FaceDetMainActivity extends Activity implements View.OnClickListene private TextView seekbarText; private float resultNum = 1.0f; private ResultListView resultView; - private Bitmap shutterBitmap; private Bitmap picBitmap; + private Bitmap shutterBitmap; + private Bitmap originPicBitmap; + private Bitmap originShutterBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -101,13 +103,13 @@ protected void onCreate(Bundle savedInstanceState) { // Clear all setting items to avoid app crashing due to the incorrect settings initSettings(); - // Init the camera preview and UI components - initView(); - // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions if (!checkAllPermissions()) { requestAllPermissions(); } + + // Init the camera preview and UI components + initView(); } @SuppressLint("NonConstantResourceId") @@ -210,6 +212,7 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -228,6 +231,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -259,12 +263,14 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } + boolean modified = false; + long tc = System.currentTimeMillis(); - SystemClock.sleep(TIME_SLEEP_INTERVAL); FaceDetectionResult result = predictor.predict( ARGB8888ImageBitmap, FaceDetSettingsActivity.scoreThreshold, 0.4f); timeElapsed += (System.currentTimeMillis() - tc); + Visualize.visFaceDetection(ARGB8888ImageBitmap, result); modified = result.initialized(); frameCounter++; @@ -310,8 +316,12 @@ protected void onDestroy() { public void initView() { TYPE = REALTIME_DETECT; + CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; + CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 360; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); + svPreview.switchCamera(); // Front camera for HumanSeg + tvStatus = (TextView) findViewById(R.id.tv_status); btnSwitch = (ImageButton) findViewById(R.id.btn_switch); btnSwitch.setOnClickListener(this); @@ -359,10 +369,11 @@ public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms detail(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms - svPreview.onPause(); detail(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); } } }); @@ -371,12 +382,11 @@ public void run() { } private void detail(Bitmap bitmap) { - FaceDetectionResult result = predictor.predict(bitmap, true, FaceDetSettingsActivity.scoreThreshold, 0.4f); - if (scores == null) { - scores = result.mScores; - } + FaceDetectionResult result = predictor.predict(bitmap, true, resultNum, 0.4f); + + scores = result.mScores; + initialized = result.initialized(); - Log.e("GBD", initialized + "---initialized"); if (initialized) { for (int i = 0; i < scores.length; i++) { if (scores[i] > resultNum) { @@ -384,7 +394,7 @@ private void detail(Bitmap bitmap) { } } } - BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.ocr_result_page_item, results); + BaseResultAdapter adapter = new BaseResultAdapter(getBaseContext(), R.layout.facedet_result_page_item, results); resultView.setAdapter(adapter); resultView.invalidate(); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java index 321b03ecbea..c515abf773d 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java @@ -66,8 +66,10 @@ public class OcrMainActivity extends Activity implements View.OnClickListener, C private TextView seekbarText; private float resultNum = 1.0f; private ResultListView resultView; - private Bitmap shutterBitmap; private Bitmap picBitmap; + private Bitmap shutterBitmap; + private Bitmap originPicBitmap; + private Bitmap originShutterBitmap; private boolean isShutterBitmapCopied = false; public static final int TYPE_UNKNOWN = -1; @@ -104,13 +106,13 @@ protected void onCreate(Bundle savedInstanceState) { // Clear all setting items to avoid app crashing due to the incorrect settings initSettings(); - // Init the camera preview and UI components - initView(); - // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions if (!checkAllPermissions()) { requestAllPermissions(); } + + // Init the camera preview and UI components + initView(); } @SuppressLint("NonConstantResourceId") @@ -216,6 +218,7 @@ private void copyBitmapFromCamera(Bitmap ARGB8888ImageBitmap) { if (!ARGB8888ImageBitmap.isRecycled()) { synchronized (this) { shutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); + originShutterBitmap = ARGB8888ImageBitmap.copy(Bitmap.Config.ARGB_8888, true); } SystemClock.sleep(TIME_SLEEP_INTERVAL); isShutterBitmapCopied = true; @@ -235,6 +238,7 @@ protected void onActivityResult(int requestCode, int resultCode, Intent data) { Uri uri = data.getData(); String path = getRealPathFromURI(this, uri); picBitmap = decodeBitmap(path, 720, 1280); + originPicBitmap = picBitmap.copy(Bitmap.Config.ARGB_8888, true); resultImage.setImageBitmap(picBitmap); } } @@ -265,12 +269,16 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } + boolean modified = false; + long tc = System.currentTimeMillis(); - OCRResult result = predictor.predict(ARGB8888ImageBitmap, true); + OCRResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); + Visualize.visOcr(ARGB8888ImageBitmap, result); modified = result.initialized(); + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -363,10 +371,11 @@ public void run() { if (TYPE == ALBUM_SELECT) { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); detail(picBitmap); + picBitmap = originPicBitmap.copy(Bitmap.Config.ARGB_8888, true); } else { SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); - svPreview.onPause(); detail(shutterBitmap); + shutterBitmap = originShutterBitmap.copy(Bitmap.Config.ARGB_8888, true); } } }); @@ -376,12 +385,10 @@ public void run() { private void detail(Bitmap bitmap) { OCRResult result = predictor.predict(bitmap, true); - if (texts == null) { - texts = result.mText; - } - if (recScores == null) { - recScores = result.mRecScores; - } + + texts = result.mText; + recScores = result.mRecScores; + initialized = result.initialized(); if (initialized) { for (int i = 0; i < texts.length; i++) { diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 57624477ab8..07b9bc7a86c 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -97,13 +97,13 @@ protected void onCreate(Bundle savedInstanceState) { // Clear all setting items to avoid app crashing due to the incorrect settings initSettings(); - // Init the camera preview and UI components - initView(); - // Check and request CAMERA and WRITE_EXTERNAL_STORAGE permissions if (!checkAllPermissions()) { requestAllPermissions(); } + + // Init the camera preview and UI components + initView(); } @SuppressLint("NonConstantResourceId") @@ -253,12 +253,16 @@ public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { copyBitmapFromCamera(ARGB8888ImageBitmap); return false; } + boolean modified = false; + long tc = System.currentTimeMillis(); SegmentationResult result = predictor.predict(ARGB8888ImageBitmap); timeElapsed += (System.currentTimeMillis() - tc); + Visualize.visSegmentation(ARGB8888ImageBitmap, result); modified = result.initialized(); + frameCounter++; if (frameCounter >= 30) { final int fps = (int) (1000 / (timeElapsed / 30)); @@ -302,8 +306,12 @@ protected void onDestroy() { public void initView() { TYPE = REALTIME_DETECT; + CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; + CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 360; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); + svPreview.switchCamera(); // Front camera for HumanSeg + tvStatus = (TextView) findViewById(R.id.tv_status); btnSwitch = (ImageButton) findViewById(R.id.btn_switch); btnSwitch.setOnClickListener(this); From 06b5fccf09dc942cd4b824c0914dad1df1dea33c Mon Sep 17 00:00:00 2001 From: Jason Date: Thu, 17 Nov 2022 21:48:51 +0800 Subject: [PATCH 058/136] move tests --- tests/{eval_example => models}/test_facelandmark1000.py | 0 tests/{eval_example => models}/test_pipnet.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/{eval_example => models}/test_facelandmark1000.py (100%) rename tests/{eval_example => models}/test_pipnet.py (100%) diff --git a/tests/eval_example/test_facelandmark1000.py b/tests/models/test_facelandmark1000.py similarity index 100% rename from tests/eval_example/test_facelandmark1000.py rename to tests/models/test_facelandmark1000.py diff --git a/tests/eval_example/test_pipnet.py b/tests/models/test_pipnet.py similarity index 100% rename from tests/eval_example/test_pipnet.py rename to tests/models/test_pipnet.py From 8529590a844d0ebb90423bfcf7eed1456d5941fd Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Thu, 17 Nov 2022 23:46:59 +0800 Subject: [PATCH 059/136] [Bug Fix] fix realtime camera mode and shutter (#625) * [Bug Fix] fix android app detail page errors * [Android] fix realtime camera mode and shutter --- .../examples/classification/ClassificationMainActivity.java | 5 ++++- .../app/examples/detection/DetectionMainActivity.java | 5 ++++- .../fastdeploy/app/examples/facedet/FaceDetMainActivity.java | 4 ++++ .../paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java | 4 ++++ .../app/examples/segmentation/SegmentationMainActivity.java | 4 ++++ 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java index 07a462d67de..7e7fce004ff 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/classification/ClassificationMainActivity.java @@ -254,9 +254,13 @@ private void toggleRealtimeStyle() { isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + isShutterBitmapCopied = false; svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + } return false; } }); @@ -394,7 +398,6 @@ private void detail(Bitmap bitmap) { if (initialized) { for (int i = 0; i < labelId.length; i++) { if (scores[i] > resultNum) { - Log.d(TAG, "[fastdeploy] detail: scores[i]: " + scores[i]); int idx = labelId[i]; String text = labelText.get(idx); text = text.substring(text.indexOf(" ")); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java index 5a19004ba3e..dbb76df83ec 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/detection/DetectionMainActivity.java @@ -253,9 +253,13 @@ private void toggleRealtimeStyle() { isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + isShutterBitmapCopied = false; svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + } return false; } }); @@ -394,7 +398,6 @@ private void detail(Bitmap bitmap) { if (initialized) { for (int i = 0; i < labelId.length; i++) { if (scores[i] > resultNum) { - Log.d(TAG, "[fastdeploy] detail: scores[i]: " + scores[i]); int idx = labelId[i]; String text = labelText.get(idx); results.add(new BaseResultModel(idx, text, scores[i])); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index e2883baa3ca..eaf3d3348a5 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -247,10 +247,14 @@ private void toggleRealtimeStyle() { isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + isShutterBitmapCopied = false; // Camera is still working but detecting loop is on pause. svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + } return false; } }); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java index c515abf773d..c4b69c5209a 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java @@ -254,9 +254,13 @@ private void toggleRealtimeStyle() { isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + isShutterBitmapCopied = false; svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + } return false; } }); diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index 07b9bc7a86c..f1809d8105c 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -237,10 +237,14 @@ private void toggleRealtimeStyle() { isRealtimeStatusRunning = true; realtimeToggleButton.setImageResource(R.drawable.realtime_start_btn); tvStatus.setVisibility(View.GONE); + isShutterBitmapCopied = false; // Camera is still working but detecting loop is on pause. svPreview.setOnTextureChangedListener(new CameraSurfaceView.OnTextureChangedListener() { @Override public boolean onTextureChanged(Bitmap ARGB8888ImageBitmap) { + if (TYPE == BTN_SHUTTER) { + copyBitmapFromCamera(ARGB8888ImageBitmap); + } return false; } }); From a3790f7203cdab753b51e58d61e3500635175294 Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Fri, 18 Nov 2022 10:52:02 +0800 Subject: [PATCH 060/136] [Model] Refactor YOLOv7 module (#611) * add paddle_trt in benchmark * update benchmark in device * update benchmark * update result doc * fixed for CI * update python api_docs * update index.rst * add runtime cpp examples * deal with comments * Update infer_paddle_tensorrt.py * Add runtime quick start * deal with comments * fixed reused_input_tensors&&reused_output_tensors * fixed docs * fixed headpose typo * fixed typo * refactor yolov5 * update model infer * refactor pybind for yolov5 * rm origin yolov5 * fixed bugs * rm cuda preprocess * fixed bugs * fixed bugs * fixed bug * fixed bug * fix pybind * rm useless code * add convert_and_permute * fixed bugs * fixed im_info for bs_predict * fixed bug * add bs_predict for yolov5 * Add runtime test and batch eval * deal with comments * fixed bug * update testcase * fixed batch eval bug * fixed preprocess bug * refactor yolov7 * add yolov7 testcase * rm resize_after_load and add is_scale_up * fixed bug * set multi_label true Co-authored-by: Jason <928090362@qq.com> --- fastdeploy/vision.h | 2 +- .../detection/contrib/yolov5/postprocessor.cc | 4 +- .../detection/contrib/yolov5/postprocessor.h | 2 +- .../detection/contrib/yolov5/preprocessor.cc | 19 +- .../detection/contrib/yolov5/preprocessor.h | 9 + .../detection/contrib/yolov5/yolov5_pybind.cc | 3 +- fastdeploy/vision/detection/contrib/yolov7.cc | 344 ------------------ fastdeploy/vision/detection/contrib/yolov7.h | 113 ------ .../detection/contrib/yolov7/postprocessor.cc | 103 ++++++ .../detection/contrib/yolov7/postprocessor.h | 66 ++++ .../detection/contrib/yolov7/preprocessor.cc | 116 ++++++ .../detection/contrib/yolov7/preprocessor.h | 96 +++++ .../vision/detection/contrib/yolov7/yolov7.cc | 89 +++++ .../vision/detection/contrib/yolov7/yolov7.h | 88 +++++ .../detection/contrib/yolov7/yolov7_pybind.cc | 87 +++++ .../vision/detection/contrib/yolov7_pybind.cc | 42 --- .../fastdeploy/vision/detection/__init__.py | 2 +- .../vision/detection/contrib/yolov5.py | 19 +- .../vision/detection/contrib/yolov7.py | 203 +++++++---- tests/models/test_yolov7.py | 165 +++++++++ 20 files changed, 971 insertions(+), 601 deletions(-) delete mode 100755 fastdeploy/vision/detection/contrib/yolov7.cc delete mode 100644 fastdeploy/vision/detection/contrib/yolov7.h create mode 100755 fastdeploy/vision/detection/contrib/yolov7/postprocessor.cc create mode 100755 fastdeploy/vision/detection/contrib/yolov7/postprocessor.h create mode 100755 fastdeploy/vision/detection/contrib/yolov7/preprocessor.cc create mode 100755 fastdeploy/vision/detection/contrib/yolov7/preprocessor.h create mode 100755 fastdeploy/vision/detection/contrib/yolov7/yolov7.cc create mode 100755 fastdeploy/vision/detection/contrib/yolov7/yolov7.h create mode 100755 fastdeploy/vision/detection/contrib/yolov7/yolov7_pybind.cc delete mode 100644 fastdeploy/vision/detection/contrib/yolov7_pybind.cc create mode 100755 tests/models/test_yolov7.py diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index c9719cc9ea7..82f06e0036b 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -24,7 +24,7 @@ #include "fastdeploy/vision/detection/contrib/yolov5/yolov5.h" #include "fastdeploy/vision/detection/contrib/yolov5lite.h" #include "fastdeploy/vision/detection/contrib/yolov6.h" -#include "fastdeploy/vision/detection/contrib/yolov7.h" +#include "fastdeploy/vision/detection/contrib/yolov7/yolov7.h" #include "fastdeploy/vision/detection/contrib/yolov7end2end_ort.h" #include "fastdeploy/vision/detection/contrib/yolov7end2end_trt.h" #include "fastdeploy/vision/detection/contrib/yolox.h" diff --git a/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc index dd61efb0023..4fe01dfeb86 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc +++ b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.cc @@ -103,9 +103,9 @@ bool YOLOv5Postprocessor::Run(const std::vector& tensors, std::vector< float ipt_h = iter_ipt->second[0]; float ipt_w = iter_ipt->second[1]; float scale = std::min(out_h / ipt_h, out_w / ipt_w); + float pad_h = (out_h - ipt_h * scale) / 2; + float pad_w = (out_w - ipt_w * scale) / 2; for (size_t i = 0; i < (*results)[bs].boxes.size(); ++i) { - float pad_h = (out_h - ipt_h * scale) / 2; - float pad_w = (out_w - ipt_w * scale) / 2; int32_t label_id = ((*results)[bs].label_ids)[i]; // clip box (*results)[bs].boxes[i][0] = (*results)[bs].boxes[i][0] - max_wh_ * label_id; diff --git a/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h index a1479dd9403..88f9400fa20 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h +++ b/fastdeploy/vision/detection/contrib/yolov5/postprocessor.h @@ -55,7 +55,7 @@ class FASTDEPLOY_DECL YOLOv5Postprocessor { /// Get nms_threshold, default 0.5 float GetNMSThreshold() const { return nms_threshold_; } - /// Set multi_label, default true + /// Set multi_label, set true for eval, default true void SetMultiLabel(bool multi_label) { multi_label_ = multi_label; } diff --git a/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc index 112a4d4d5da..846e2513163 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc +++ b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.cc @@ -24,7 +24,7 @@ YOLOv5Preprocessor::YOLOv5Preprocessor() { padding_value_ = {114.0, 114.0, 114.0}; is_mini_pad_ = false; is_no_pad_ = false; - is_scale_up_ = false; + is_scale_up_ = true; stride_ = 32; max_wh_ = 7680.0; } @@ -50,7 +50,9 @@ void YOLOv5Preprocessor::LetterBox(FDMat* mat) { resize_h = size_[1]; resize_w = size_[0]; } - Resize::Run(mat, resize_w, resize_h); + if (std::fabs(scale - 1.0f) > 1e-06) { + Resize::Run(mat, resize_w, resize_h); + } if (pad_h > 0 || pad_w > 0) { float half_h = pad_h * 1.0 / 2; int top = int(round(half_h - 0.1)); @@ -67,19 +69,6 @@ bool YOLOv5Preprocessor::Preprocess(FDMat* mat, FDTensor* output, // Record the shape of image and the shape of preprocessed image (*im_info)["input_shape"] = {static_cast(mat->Height()), static_cast(mat->Width())}; - - // process after image load - double ratio = (size_[0] * 1.0) / std::max(static_cast(mat->Height()), - static_cast(mat->Width())); - if (std::fabs(ratio - 1.0f) > 1e-06) { - int interp = cv::INTER_AREA; - if (ratio > 1.0) { - interp = cv::INTER_LINEAR; - } - int resize_h = int(mat->Height() * ratio); - int resize_w = int(mat->Width() * ratio); - Resize::Run(mat, resize_w, resize_h, -1, -1, interp); - } // yolov5's preprocess steps // 1. letterbox // 2. convert_and_permute(swap_rb=true) diff --git a/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h index b3559685db7..f0cf438df0c 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h +++ b/fastdeploy/vision/detection/contrib/yolov5/preprocessor.h @@ -52,6 +52,15 @@ class FASTDEPLOY_DECL YOLOv5Preprocessor { /// Get padding value, size should be the same as channels std::vector GetPaddingValue() const { return padding_value_; } + /// Set is_scale_up, if is_scale_up is false, the input image only + /// can be zoom out, the maximum resize scale cannot exceed 1.0, default true + void SetScaleUp(bool is_scale_up) { + is_scale_up_ = is_scale_up; + } + + /// Get is_scale_up, default true + bool GetScaleUp() const { return is_scale_up_; } + protected: bool Preprocess(FDMat* mat, FDTensor* output, std::map>* im_info); diff --git a/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc b/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc index f44891d9846..7b1574401fb 100755 --- a/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc +++ b/fastdeploy/vision/detection/contrib/yolov5/yolov5_pybind.cc @@ -35,7 +35,8 @@ void BindYOLOv5(pybind11::module& m) { return make_pair(outputs, ims_info); }) .def_property("size", &vision::detection::YOLOv5Preprocessor::GetSize, &vision::detection::YOLOv5Preprocessor::SetSize) - .def_property("padding_value", &vision::detection::YOLOv5Preprocessor::GetPaddingValue, &vision::detection::YOLOv5Preprocessor::SetPaddingValue); + .def_property("padding_value", &vision::detection::YOLOv5Preprocessor::GetPaddingValue, &vision::detection::YOLOv5Preprocessor::SetPaddingValue) + .def_property("is_scale_up", &vision::detection::YOLOv5Preprocessor::GetScaleUp, &vision::detection::YOLOv5Preprocessor::SetScaleUp); pybind11::class_( m, "YOLOv5Postprocessor") diff --git a/fastdeploy/vision/detection/contrib/yolov7.cc b/fastdeploy/vision/detection/contrib/yolov7.cc deleted file mode 100755 index 9185e16ed0e..00000000000 --- a/fastdeploy/vision/detection/contrib/yolov7.cc +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/detection/contrib/yolov7.h" - -#include "fastdeploy/utils/perf.h" -#include "fastdeploy/vision/utils/utils.h" -#ifdef ENABLE_CUDA_PREPROCESS -#include "fastdeploy/vision/utils/cuda_utils.h" -#endif // ENABLE_CUDA_PREPROCESS - -namespace fastdeploy { -namespace vision { -namespace detection { - -void YOLOv7::LetterBox(Mat* mat, const std::vector& size, - const std::vector& color, bool _auto, - bool scale_fill, bool scale_up, int stride) { - float scale = - std::min(size[1] * 1.0 / mat->Height(), size[0] * 1.0 / mat->Width()); - if (!scale_up) { - scale = std::min(scale, 1.0f); - } - - int resize_h = int(round(mat->Height() * scale)); - int resize_w = int(round(mat->Width() * scale)); - - int pad_w = size[0] - resize_w; - int pad_h = size[1] - resize_h; - if (_auto) { - pad_h = pad_h % stride; - pad_w = pad_w % stride; - } else if (scale_fill) { - pad_h = 0; - pad_w = 0; - resize_h = size[1]; - resize_w = size[0]; - } - if (resize_h != mat->Height() || resize_w != mat->Width()) { - Resize::Run(mat, resize_w, resize_h); - } - if (pad_h > 0 || pad_w > 0) { - float half_h = pad_h * 1.0 / 2; - int top = int(round(half_h - 0.1)); - int bottom = int(round(half_h + 0.1)); - float half_w = pad_w * 1.0 / 2; - int left = int(round(half_w - 0.1)); - int right = int(round(half_w + 0.1)); - Pad::Run(mat, top, bottom, left, right, color); - } -} - -YOLOv7::YOLOv7(const std::string& model_file, const std::string& params_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { - if (model_format == ModelFormat::ONNX) { - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; - valid_gpu_backends = {Backend::ORT, Backend::TRT}; - } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; - } - runtime_option = custom_option; - runtime_option.model_format = model_format; - runtime_option.model_file = model_file; - runtime_option.params_file = params_file; -#ifdef ENABLE_CUDA_PREPROCESS - cudaSetDevice(runtime_option.device_id); - cudaStream_t stream; - CUDA_CHECK(cudaStreamCreate(&stream)); - cuda_stream_ = reinterpret_cast(stream); - runtime_option.SetExternalStream(cuda_stream_); -#endif // ENABLE_CUDA_PREPROCESS - initialized = Initialize(); -} - -bool YOLOv7::Initialize() { - // parameters for preprocess - size = {640, 640}; - padding_value = {114.0, 114.0, 114.0}; - is_mini_pad = false; - is_no_pad = false; - is_scale_up = false; - stride = 32; - max_wh = 7680.0; - reused_input_tensors_.resize(1); - - if (!InitRuntime()) { - FDERROR << "Failed to initialize fastdeploy backend." << std::endl; - return false; - } - // Check if the input shape is dynamic after Runtime already initialized, - // Note that, We need to force is_mini_pad 'false' to keep static - // shape after padding (LetterBox) when the is_dynamic_shape is 'false'. - is_dynamic_input_ = false; - auto shape = InputInfoOfRuntime(0).shape; - for (int i = 0; i < shape.size(); ++i) { - // if height or width is dynamic - if (i >= 2 && shape[i] <= 0) { - is_dynamic_input_ = true; - break; - } - } - if (!is_dynamic_input_) { - is_mini_pad = false; - } - return true; -} - -YOLOv7::~YOLOv7() { -#ifdef ENABLE_CUDA_PREPROCESS - if (use_cuda_preprocessing_) { - CUDA_CHECK(cudaFreeHost(input_img_cuda_buffer_host_)); - CUDA_CHECK(cudaFree(input_img_cuda_buffer_device_)); - CUDA_CHECK(cudaFree(input_tensor_cuda_buffer_device_)); - CUDA_CHECK(cudaStreamDestroy(reinterpret_cast(cuda_stream_))); - } -#endif // ENABLE_CUDA_PREPROCESS -} - -bool YOLOv7::Preprocess(Mat* mat, FDTensor* output, - std::map>* im_info) { - // process after image load - float ratio = std::min(size[1] * 1.0f / static_cast(mat->Height()), - size[0] * 1.0f / static_cast(mat->Width())); - if (std::fabs(ratio - 1.0f) > 1e-06) { - int interp = cv::INTER_AREA; - if (ratio > 1.0) { - interp = cv::INTER_LINEAR; - } - int resize_h = int(mat->Height() * ratio); - int resize_w = int(mat->Width() * ratio); - Resize::Run(mat, resize_w, resize_h, -1, -1, interp); - } - // yolov7's preprocess steps - // 1. letterbox - // 2. BGR->RGB - // 3. HWC->CHW - YOLOv7::LetterBox(mat, size, padding_value, is_mini_pad, is_no_pad, - is_scale_up, stride); - BGR2RGB::Run(mat); - // Normalize::Run(mat, std::vector(mat->Channels(), 0.0), - // std::vector(mat->Channels(), 1.0)); - // Compute `result = mat * alpha + beta` directly by channel - std::vector alpha = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; - std::vector beta = {0.0f, 0.0f, 0.0f}; - Convert::Run(mat, alpha, beta); - - // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); - mat->ShareWithTensor(output); - output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c - return true; -} - -void YOLOv7::UseCudaPreprocessing(int max_image_size) { -#ifdef ENABLE_CUDA_PREPROCESS - use_cuda_preprocessing_ = true; - is_scale_up = true; - if (input_img_cuda_buffer_host_ == nullptr) { - // prepare input data cache in GPU pinned memory - CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, - max_image_size * 3)); - // prepare input data cache in GPU device memory - CUDA_CHECK( - cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); - CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, - 3 * size[0] * size[1] * sizeof(float))); - } -#else - FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON." - << std::endl; - use_cuda_preprocessing_ = false; -#endif -} - -bool YOLOv7::CudaPreprocess( - Mat* mat, FDTensor* output, - std::map>* im_info) { -#ifdef ENABLE_CUDA_PREPROCESS - if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) { - FDERROR << "Preprocessing with CUDA is only available when the arguments " - "satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." - << std::endl; - return false; - } - - // Record the shape of image and the shape of preprocessed image - (*im_info)["input_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - (*im_info)["output_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - - cudaStream_t stream = reinterpret_cast(cuda_stream_); - int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels(); - memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size); - CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_, - input_img_cuda_buffer_host_, src_img_buf_size, - cudaMemcpyHostToDevice, stream)); - utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(), - mat->Height(), input_tensor_cuda_buffer_device_, - size[0], size[1], padding_value, stream); - - // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(size[0]), - static_cast(size[1])}; - - output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32, - input_tensor_cuda_buffer_device_); - output->device = Device::GPU; - output->shape.insert(output->shape.begin(), 1); // reshape to n, h, w, c - return true; -#else - FDERROR << "CUDA src code was not enabled." << std::endl; - return false; -#endif // ENABLE_CUDA_PREPROCESS -} - -bool YOLOv7::Postprocess( - FDTensor& infer_result, DetectionResult* result, - const std::map>& im_info, - float conf_threshold, float nms_iou_threshold) { - FDASSERT(infer_result.shape[0] == 1, "Only support batch =1 now."); - result->Clear(); - result->Reserve(infer_result.shape[1]); - if (infer_result.dtype != FDDataType::FP32) { - FDERROR << "Only support post process with float32 data." << std::endl; - return false; - } - float* data = static_cast(infer_result.Data()); - for (size_t i = 0; i < infer_result.shape[1]; ++i) { - int s = i * infer_result.shape[2]; - float confidence = data[s + 4]; - float* max_class_score = - std::max_element(data + s + 5, data + s + infer_result.shape[2]); - confidence *= (*max_class_score); - // filter boxes by conf_threshold - if (confidence <= conf_threshold) { - continue; - } - int32_t label_id = std::distance(data + s + 5, max_class_score); - // convert from [x, y, w, h] to [x1, y1, x2, y2] - result->boxes.emplace_back(std::array{ - data[s] - data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh, - data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh, - data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh}); - result->label_ids.push_back(label_id); - result->scores.push_back(confidence); - } - utils::NMS(result, nms_iou_threshold); - - // scale the boxes to the origin image shape - auto iter_out = im_info.find("output_shape"); - auto iter_ipt = im_info.find("input_shape"); - FDASSERT(iter_out != im_info.end() && iter_ipt != im_info.end(), - "Cannot find input_shape or output_shape from im_info."); - float out_h = iter_out->second[0]; - float out_w = iter_out->second[1]; - float ipt_h = iter_ipt->second[0]; - float ipt_w = iter_ipt->second[1]; - float scale = std::min(out_h / ipt_h, out_w / ipt_w); - float pad_h = (out_h - ipt_h * scale) / 2.0f; - float pad_w = (out_w - ipt_w * scale) / 2.0f; - if (is_mini_pad) { - pad_h = static_cast(static_cast(pad_h) % stride); - pad_w = static_cast(static_cast(pad_w) % stride); - } - for (size_t i = 0; i < result->boxes.size(); ++i) { - int32_t label_id = (result->label_ids)[i]; - // clip box - result->boxes[i][0] = result->boxes[i][0] - max_wh * label_id; - result->boxes[i][1] = result->boxes[i][1] - max_wh * label_id; - result->boxes[i][2] = result->boxes[i][2] - max_wh * label_id; - result->boxes[i][3] = result->boxes[i][3] - max_wh * label_id; - result->boxes[i][0] = std::max((result->boxes[i][0] - pad_w) / scale, 0.0f); - result->boxes[i][1] = std::max((result->boxes[i][1] - pad_h) / scale, 0.0f); - result->boxes[i][2] = std::max((result->boxes[i][2] - pad_w) / scale, 0.0f); - result->boxes[i][3] = std::max((result->boxes[i][3] - pad_h) / scale, 0.0f); - result->boxes[i][0] = std::min(result->boxes[i][0], ipt_w - 1.0f); - result->boxes[i][1] = std::min(result->boxes[i][1], ipt_h - 1.0f); - result->boxes[i][2] = std::min(result->boxes[i][2], ipt_w - 1.0f); - result->boxes[i][3] = std::min(result->boxes[i][3], ipt_h - 1.0f); - } - return true; -} - -bool YOLOv7::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, - float nms_iou_threshold) { - Mat mat(*im); - - std::map> im_info; - - // Record the shape of image and the shape of preprocessed image - im_info["input_shape"] = {static_cast(mat.Height()), - static_cast(mat.Width())}; - im_info["output_shape"] = {static_cast(mat.Height()), - static_cast(mat.Width())}; - - if (use_cuda_preprocessing_) { - if (!CudaPreprocess(&mat, &reused_input_tensors_[0], &im_info)) { - FDERROR << "Failed to preprocess input image." << std::endl; - return false; - } - } else { - if (!Preprocess(&mat, &reused_input_tensors_[0], &im_info)) { - FDERROR << "Failed to preprocess input image." << std::endl; - return false; - } - } - - reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; - if (!Infer()) { - FDERROR << "Failed to inference." << std::endl; - return false; - } - - if (!Postprocess(reused_output_tensors_[0], result, im_info, conf_threshold, - nms_iou_threshold)) { - FDERROR << "Failed to post process." << std::endl; - return false; - } - - return true; -} - -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7.h b/fastdeploy/vision/detection/contrib/yolov7.h deleted file mode 100644 index b9d637ed9ab..00000000000 --- a/fastdeploy/vision/detection/contrib/yolov7.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. //NOLINT -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include "fastdeploy/fastdeploy_model.h" -#include "fastdeploy/vision/common/processors/transform.h" -#include "fastdeploy/vision/common/result.h" - -namespace fastdeploy { -namespace vision { -namespace detection { -/*! @brief YOLOv7 model object used when to load a YOLOv7 model exported by YOLOv7. - */ -class FASTDEPLOY_DECL YOLOv7 : public FastDeployModel { - public: - /** \brief Set path of model file and the configuration of runtime. - * - * \param[in] model_file Path of model file, e.g ./yolov7.onnx - * \param[in] params_file Path of parameter file, e.g ppyoloe/model.pdiparams, if the model format is ONNX, this parameter will be ignored - * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" - * \param[in] model_format Model format of the loaded model, default is ONNX format - */ - YOLOv7(const std::string& model_file, const std::string& params_file = "", - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::ONNX); - - ~YOLOv7(); - - virtual std::string ModelName() const { return "yolov7"; } - /** \brief Predict the detection result for an input image - * - * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format - * \param[in] result The output detection result will be writen to this structure - * \param[in] conf_threshold confidence threashold for postprocessing, default is 0.25 - * \param[in] nms_iou_threshold iou threashold for NMS, default is 0.5 - * \return true if the prediction successed, otherwise false - */ - virtual bool Predict(cv::Mat* im, DetectionResult* result, - float conf_threshold = 0.25, - float nms_iou_threshold = 0.5); - - - void UseCudaPreprocessing(int max_img_size = 3840 * 2160); - - /*! @brief - Argument for image preprocessing step, tuple of (width, height), decide the target size after resize, default size = {640, 640} - */ - std::vector size; - // padding value, size should be the same as channels - - std::vector padding_value; - // only pad to the minimum rectange which height and width is times of stride - bool is_mini_pad; - // while is_mini_pad = false and is_no_pad = true, - // will resize the image to the set size - bool is_no_pad; - // if is_scale_up is false, the input image only can be zoom out, - // the maximum resize scale cannot exceed 1.0 - bool is_scale_up; - // padding stride, for is_mini_pad - int stride; - // for offseting the boxes by classes when using NMS - float max_wh; - - private: - bool Initialize(); - - bool Preprocess(Mat* mat, FDTensor* output, - std::map>* im_info); - - bool CudaPreprocess(Mat* mat, FDTensor* output, - std::map>* im_info); - - bool Postprocess(FDTensor& infer_result, DetectionResult* result, - const std::map>& im_info, - float conf_threshold, float nms_iou_threshold); - - void LetterBox(Mat* mat, const std::vector& size, - const std::vector& color, bool _auto, - bool scale_fill = false, bool scale_up = true, - int stride = 32); - - // whether to inference with dynamic shape (e.g ONNX export with dynamic shape - // or not.) - // while is_dynamic_shape if 'false', is_mini_pad will force 'false'. This - // value will - // auto check by fastdeploy after the internal Runtime already initialized. - bool is_dynamic_input_; - // CUDA host buffer for input image - uint8_t* input_img_cuda_buffer_host_ = nullptr; - // CUDA device buffer for input image - uint8_t* input_img_cuda_buffer_device_ = nullptr; - // CUDA device buffer for TRT input tensor - float* input_tensor_cuda_buffer_device_ = nullptr; - // Whether to use CUDA preprocessing - bool use_cuda_preprocessing_ = false; - // CUDA stream - void* cuda_stream_ = nullptr; -}; -} // namespace detection -} // namespace vision -} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/postprocessor.cc b/fastdeploy/vision/detection/contrib/yolov7/postprocessor.cc new file mode 100755 index 00000000000..01d657adb30 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/postprocessor.cc @@ -0,0 +1,103 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/contrib/yolov7/postprocessor.h" +#include "fastdeploy/vision/utils/utils.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +YOLOv7Postprocessor::YOLOv7Postprocessor() { + conf_threshold_ = 0.25; + nms_threshold_ = 0.5; + max_wh_ = 7680.0; +} + +bool YOLOv7Postprocessor::Run(const std::vector& tensors, std::vector* results, + const std::vector>>& ims_info) { + int batch = tensors[0].shape[0]; + + results->resize(batch); + + for (size_t bs = 0; bs < batch; ++bs) { + (*results)[bs].Clear(); + (*results)[bs].Reserve(tensors[0].shape[1]); + if (tensors[0].dtype != FDDataType::FP32) { + FDERROR << "Only support post process with float32 data." << std::endl; + return false; + } + const float* data = reinterpret_cast(tensors[0].Data()) + bs * tensors[0].shape[1] * tensors[0].shape[2]; + for (size_t i = 0; i < tensors[0].shape[1]; ++i) { + int s = i * tensors[0].shape[2]; + float confidence = data[s + 4]; + const float* max_class_score = + std::max_element(data + s + 5, data + s + tensors[0].shape[2]); + confidence *= (*max_class_score); + // filter boxes by conf_threshold + if (confidence <= conf_threshold_) { + continue; + } + int32_t label_id = std::distance(data + s + 5, max_class_score); + // convert from [x, y, w, h] to [x1, y1, x2, y2] + (*results)[bs].boxes.emplace_back(std::array{ + data[s] - data[s + 2] / 2.0f + label_id * max_wh_, + data[s + 1] - data[s + 3] / 2.0f + label_id * max_wh_, + data[s + 0] + data[s + 2] / 2.0f + label_id * max_wh_, + data[s + 1] + data[s + 3] / 2.0f + label_id * max_wh_}); + (*results)[bs].label_ids.push_back(label_id); + (*results)[bs].scores.push_back(confidence); + } + + if ((*results)[bs].boxes.size() == 0) { + return true; + } + + utils::NMS(&((*results)[bs]), nms_threshold_); + + // scale the boxes to the origin image shape + auto iter_out = ims_info[bs].find("output_shape"); + auto iter_ipt = ims_info[bs].find("input_shape"); + FDASSERT(iter_out != ims_info[bs].end() && iter_ipt != ims_info[bs].end(), + "Cannot find input_shape or output_shape from im_info."); + float out_h = iter_out->second[0]; + float out_w = iter_out->second[1]; + float ipt_h = iter_ipt->second[0]; + float ipt_w = iter_ipt->second[1]; + float scale = std::min(out_h / ipt_h, out_w / ipt_w); + float pad_h = (out_h - ipt_h * scale) / 2; + float pad_w = (out_w - ipt_w * scale) / 2; + for (size_t i = 0; i < (*results)[bs].boxes.size(); ++i) { + int32_t label_id = ((*results)[bs].label_ids)[i]; + // clip box + (*results)[bs].boxes[i][0] = (*results)[bs].boxes[i][0] - max_wh_ * label_id; + (*results)[bs].boxes[i][1] = (*results)[bs].boxes[i][1] - max_wh_ * label_id; + (*results)[bs].boxes[i][2] = (*results)[bs].boxes[i][2] - max_wh_ * label_id; + (*results)[bs].boxes[i][3] = (*results)[bs].boxes[i][3] - max_wh_ * label_id; + (*results)[bs].boxes[i][0] = std::max(((*results)[bs].boxes[i][0] - pad_w) / scale, 0.0f); + (*results)[bs].boxes[i][1] = std::max(((*results)[bs].boxes[i][1] - pad_h) / scale, 0.0f); + (*results)[bs].boxes[i][2] = std::max(((*results)[bs].boxes[i][2] - pad_w) / scale, 0.0f); + (*results)[bs].boxes[i][3] = std::max(((*results)[bs].boxes[i][3] - pad_h) / scale, 0.0f); + (*results)[bs].boxes[i][0] = std::min((*results)[bs].boxes[i][0], ipt_w - 1.0f); + (*results)[bs].boxes[i][1] = std::min((*results)[bs].boxes[i][1], ipt_h - 1.0f); + (*results)[bs].boxes[i][2] = std::min((*results)[bs].boxes[i][2], ipt_w - 1.0f); + (*results)[bs].boxes[i][3] = std::min((*results)[bs].boxes[i][3], ipt_h - 1.0f); + } + } + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/postprocessor.h b/fastdeploy/vision/detection/contrib/yolov7/postprocessor.h new file mode 100755 index 00000000000..5ece87eb8b7 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/postprocessor.h @@ -0,0 +1,66 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace detection { +/*! @brief Postprocessor object for YOLOv7 serials model. + */ +class FASTDEPLOY_DECL YOLOv7Postprocessor { + public: + /** \brief Create a postprocessor instance for YOLOv7 serials model + */ + YOLOv7Postprocessor(); + + /** \brief Process the result of runtime and fill to DetectionResult structure + * + * \param[in] tensors The inference result from runtime + * \param[in] result The output result of detection + * \param[in] ims_info The shape info list, record input_shape and output_shape + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector* results, + const std::vector>>& ims_info); + + /// Set conf_threshold, default 0.25 + void SetConfThreshold(const float& conf_threshold) { + conf_threshold_ = conf_threshold; + } + + /// Get conf_threshold, default 0.25 + float GetConfThreshold() const { return conf_threshold_; } + + /// Set nms_threshold, default 0.5 + void SetNMSThreshold(const float& nms_threshold) { + nms_threshold_ = nms_threshold; + } + + /// Get nms_threshold, default 0.5 + float GetNMSThreshold() const { return nms_threshold_; } + + protected: + float conf_threshold_; + float nms_threshold_; + float max_wh_; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/preprocessor.cc b/fastdeploy/vision/detection/contrib/yolov7/preprocessor.cc new file mode 100755 index 00000000000..91e22f32b4b --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/preprocessor.cc @@ -0,0 +1,116 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/contrib/yolov7/preprocessor.h" +#include "fastdeploy/function/concat.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +YOLOv7Preprocessor::YOLOv7Preprocessor() { + size_ = {640, 640}; + padding_value_ = {114.0, 114.0, 114.0}; + is_mini_pad_ = false; + is_no_pad_ = false; + is_scale_up_ = true; + stride_ = 32; + max_wh_ = 7680.0; +} + +void YOLOv7Preprocessor::LetterBox(FDMat* mat) { + float scale = + std::min(size_[1] * 1.0 / mat->Height(), size_[0] * 1.0 / mat->Width()); + if (!is_scale_up_) { + scale = std::min(scale, 1.0f); + } + + int resize_h = int(round(mat->Height() * scale)); + int resize_w = int(round(mat->Width() * scale)); + + int pad_w = size_[0] - resize_w; + int pad_h = size_[1] - resize_h; + if (is_mini_pad_) { + pad_h = pad_h % stride_; + pad_w = pad_w % stride_; + } else if (is_no_pad_) { + pad_h = 0; + pad_w = 0; + resize_h = size_[1]; + resize_w = size_[0]; + } + if (std::fabs(scale - 1.0f) > 1e-06) { + Resize::Run(mat, resize_w, resize_h); + } + if (pad_h > 0 || pad_w > 0) { + float half_h = pad_h * 1.0 / 2; + int top = int(round(half_h - 0.1)); + int bottom = int(round(half_h + 0.1)); + float half_w = pad_w * 1.0 / 2; + int left = int(round(half_w - 0.1)); + int right = int(round(half_w + 0.1)); + Pad::Run(mat, top, bottom, left, right, padding_value_); + } +} + +bool YOLOv7Preprocessor::Preprocess(FDMat* mat, FDTensor* output, + std::map>* im_info) { + // Record the shape of image and the shape of preprocessed image + (*im_info)["input_shape"] = {static_cast(mat->Height()), + static_cast(mat->Width())}; + // yolov7's preprocess steps + // 1. letterbox + // 2. convert_and_permute(swap_rb=true) + LetterBox(mat); + std::vector alpha = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; + std::vector beta = {0.0f, 0.0f, 0.0f}; + ConvertAndPermute::Run(mat, alpha, beta, true); + + // Record output shape of preprocessed image + (*im_info)["output_shape"] = {static_cast(mat->Height()), + static_cast(mat->Width())}; + + mat->ShareWithTensor(output); + output->ExpandDim(0); // reshape to n, h, w, c + return true; +} + +bool YOLOv7Preprocessor::Run(std::vector* images, std::vector* outputs, + std::vector>>* ims_info) { + if (images->size() == 0) { + FDERROR << "The size of input images should be greater than 0." << std::endl; + return false; + } + ims_info->resize(images->size()); + outputs->resize(1); + // Concat all the preprocessed data to a batch tensor + std::vector tensors(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + if (!Preprocess(&(*images)[i], &tensors[i], &(*ims_info)[i])) { + FDERROR << "Failed to preprocess input image." << std::endl; + return false; + } + } + + if (tensors.size() == 1) { + (*outputs)[0] = std::move(tensors[0]); + } else { + function::Concat(tensors, &((*outputs)[0]), 0); + } + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/preprocessor.h b/fastdeploy/vision/detection/contrib/yolov7/preprocessor.h new file mode 100755 index 00000000000..ff6c6cad55e --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/preprocessor.h @@ -0,0 +1,96 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace detection { +/*! @brief Preprocessor object for YOLOv7 serials model. + */ +class FASTDEPLOY_DECL YOLOv7Preprocessor { + public: + /** \brief Create a preprocessor instance for YOLOv7 serials model + */ + YOLOv7Preprocessor(); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime + * \param[in] ims_info The shape info list, record input_shape and output_shape + * \return true if the preprocess successed, otherwise false + */ + bool Run(std::vector* images, std::vector* outputs, + std::vector>>* ims_info); + + /// Set target size, tuple of (width, height), default size = {640, 640} + void SetSize(const std::vector& size) { size_ = size; } + + /// Get target size, tuple of (width, height), default size = {640, 640} + std::vector GetSize() const { return size_; } + + /// Set padding value, size should be the same as channels + void SetPaddingValue(const std::vector& padding_value) { + padding_value_ = padding_value; + } + + /// Get padding value, size should be the same as channels + std::vector GetPaddingValue() const { return padding_value_; } + + /// Set is_scale_up, if is_scale_up is false, the input image only + /// can be zoom out, the maximum resize scale cannot exceed 1.0, default true + void SetScaleUp(bool is_scale_up) { + is_scale_up_ = is_scale_up; + } + + /// Get is_scale_up, default true + bool GetScaleUp() const { return is_scale_up_; } + + protected: + bool Preprocess(FDMat* mat, FDTensor* output, + std::map>* im_info); + + void LetterBox(FDMat* mat); + + // target size, tuple of (width, height), default size = {640, 640} + std::vector size_; + + // padding value, size should be the same as channels + std::vector padding_value_; + + // only pad to the minimum rectange which height and width is times of stride + bool is_mini_pad_; + + // while is_mini_pad = false and is_no_pad = true, + // will resize the image to the set size + bool is_no_pad_; + + // if is_scale_up is false, the input image only can be zoom out, + // the maximum resize scale cannot exceed 1.0 + bool is_scale_up_; + + // padding stride, for is_mini_pad + int stride_; + + // for offseting the boxes by classes when using NMS + float max_wh_; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/yolov7.cc b/fastdeploy/vision/detection/contrib/yolov7/yolov7.cc new file mode 100755 index 00000000000..513351a095e --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/yolov7.cc @@ -0,0 +1,89 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/detection/contrib/yolov7/yolov7.h" + +namespace fastdeploy { +namespace vision { +namespace detection { + +YOLOv7::YOLOv7(const std::string& model_file, const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) { + if (model_format == ModelFormat::ONNX) { + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else { + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + initialized = Initialize(); +} + +bool YOLOv7::Initialize() { + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + return true; +} + +bool YOLOv7::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, float nms_threshold) { + postprocessor_.SetConfThreshold(conf_threshold); + postprocessor_.SetNMSThreshold(nms_threshold); + if (!Predict(*im, result)) { + return false; + } + return true; +} + +bool YOLOv7::Predict(const cv::Mat& im, DetectionResult* result) { + std::vector results; + if (!BatchPredict({im}, &results)) { + return false; + } + *result = std::move(results[0]); + return true; +} + +bool YOLOv7::BatchPredict(const std::vector& images, std::vector* results) { + std::vector>> ims_info; + std::vector fd_images = WrapMat(images); + + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_, &ims_info)) { + FDERROR << "Failed to preprocess the input image." << std::endl; + return false; + } + + reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; + return false; + } + + if (!postprocessor_.Run(reused_output_tensors_, results, ims_info)) { + FDERROR << "Failed to postprocess the inference results by runtime." << std::endl; + return false; + } + + return true; +} + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/yolov7.h b/fastdeploy/vision/detection/contrib/yolov7/yolov7.h new file mode 100755 index 00000000000..2c36fd0c809 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/yolov7.h @@ -0,0 +1,88 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. //NOLINT +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/detection/contrib/yolov7/preprocessor.h" +#include "fastdeploy/vision/detection/contrib/yolov7/postprocessor.h" + +namespace fastdeploy { +namespace vision { +namespace detection { +/*! @brief YOLOv7 model object used when to load a YOLOv7 model exported by YOLOv7. + */ +class FASTDEPLOY_DECL YOLOv7 : public FastDeployModel { + public: + /** \brief Set path of model file and the configuration of runtime. + * + * \param[in] model_file Path of model file, e.g ./yolov7.onnx + * \param[in] params_file Path of parameter file, e.g ppyoloe/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in "valid_cpu_backends" + * \param[in] model_format Model format of the loaded model, default is ONNX format + */ + YOLOv7(const std::string& model_file, const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::ONNX); + + std::string ModelName() const { return "yolov7"; } + + /** \brief DEPRECATED Predict the detection result for an input image, remove at 1.0 version + * + * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output detection result will be writen to this structure + * \param[in] conf_threshold confidence threashold for postprocessing, default is 0.25 + * \param[in] nms_threshold iou threashold for NMS, default is 0.5 + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(cv::Mat* im, DetectionResult* result, + float conf_threshold = 0.25, + float nms_threshold = 0.5); + + /** \brief Predict the detection result for an input image + * + * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format + * \param[in] result The output detection result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ + virtual bool Predict(const cv::Mat& img, DetectionResult* result); + + /** \brief Predict the detection results for a batch of input images + * + * \param[in] imgs, The input image list, each element comes from cv::imread() + * \param[in] results The output detection result list + * \return true if the prediction successed, otherwise false + */ + virtual bool BatchPredict(const std::vector& imgs, + std::vector* results); + + /// Get preprocessor reference of YOLOv7 + virtual YOLOv7Preprocessor& GetPreprocessor() { + return preprocessor_; + } + + /// Get postprocessor reference of YOLOv7 + virtual YOLOv7Postprocessor& GetPostprocessor() { + return postprocessor_; + } + + protected: + bool Initialize(); + YOLOv7Preprocessor preprocessor_; + YOLOv7Postprocessor postprocessor_; +}; + +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7/yolov7_pybind.cc b/fastdeploy/vision/detection/contrib/yolov7/yolov7_pybind.cc new file mode 100755 index 00000000000..6899faa9167 --- /dev/null +++ b/fastdeploy/vision/detection/contrib/yolov7/yolov7_pybind.cc @@ -0,0 +1,87 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindYOLOv7(pybind11::module& m) { + pybind11::class_( + m, "YOLOv7Preprocessor") + .def(pybind11::init<>()) + .def("run", [](vision::detection::YOLOv7Preprocessor& self, std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + std::vector>> ims_info; + if (!self.Run(&images, &outputs, &ims_info)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in PaddleClasPreprocessor.')"); + } + for (size_t i = 0; i < outputs.size(); ++i) { + outputs[i].StopSharing(); + } + return make_pair(outputs, ims_info); + }) + .def_property("size", &vision::detection::YOLOv7Preprocessor::GetSize, &vision::detection::YOLOv7Preprocessor::SetSize) + .def_property("padding_value", &vision::detection::YOLOv7Preprocessor::GetPaddingValue, &vision::detection::YOLOv7Preprocessor::SetPaddingValue) + .def_property("is_scale_up", &vision::detection::YOLOv7Preprocessor::GetScaleUp, &vision::detection::YOLOv7Preprocessor::SetScaleUp); + + pybind11::class_( + m, "YOLOv7Postprocessor") + .def(pybind11::init<>()) + .def("run", [](vision::detection::YOLOv7Postprocessor& self, std::vector& inputs, + const std::vector>>& ims_info) { + std::vector results; + if (!self.Run(inputs, &results, ims_info)) { + pybind11::eval("raise Exception('Failed to postprocess the runtime result in YOLOv7Postprocessor.')"); + } + return results; + }) + .def("run", [](vision::detection::YOLOv7Postprocessor& self, std::vector& input_array, + const std::vector>>& ims_info) { + std::vector results; + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + if (!self.Run(inputs, &results, ims_info)) { + pybind11::eval("raise Exception('Failed to postprocess the runtime result in YOLOv7Postprocessor.')"); + } + return results; + }) + .def_property("conf_threshold", &vision::detection::YOLOv7Postprocessor::GetConfThreshold, &vision::detection::YOLOv7Postprocessor::SetConfThreshold) + .def_property("nms_threshold", &vision::detection::YOLOv7Postprocessor::GetNMSThreshold, &vision::detection::YOLOv7Postprocessor::SetNMSThreshold); + + pybind11::class_(m, "YOLOv7") + .def(pybind11::init()) + .def("predict", + [](vision::detection::YOLOv7& self, pybind11::array& data) { + auto mat = PyArrayToCvMat(data); + vision::DetectionResult res; + self.Predict(mat, &res); + return res; + }) + .def("batch_predict", [](vision::detection::YOLOv7& self, std::vector& data) { + std::vector images; + for (size_t i = 0; i < data.size(); ++i) { + images.push_back(PyArrayToCvMat(data[i])); + } + std::vector results; + self.BatchPredict(images, &results); + return results; + }) + .def_property_readonly("preprocessor", &vision::detection::YOLOv7::GetPreprocessor) + .def_property_readonly("postprocessor", &vision::detection::YOLOv7::GetPostprocessor); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/contrib/yolov7_pybind.cc b/fastdeploy/vision/detection/contrib/yolov7_pybind.cc deleted file mode 100644 index d7ab993401d..00000000000 --- a/fastdeploy/vision/detection/contrib/yolov7_pybind.cc +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/pybind/main.h" - -namespace fastdeploy { -void BindYOLOv7(pybind11::module& m) { - pybind11::class_(m, "YOLOv7") - .def(pybind11::init()) - .def("predict", - [](vision::detection::YOLOv7& self, pybind11::array& data, - float conf_threshold, float nms_iou_threshold) { - auto mat = PyArrayToCvMat(data); - vision::DetectionResult res; - self.Predict(&mat, &res, conf_threshold, nms_iou_threshold); - return res; - }) - .def("use_cuda_preprocessing", - [](vision::detection::YOLOv7& self, int max_image_size) { - self.UseCudaPreprocessing(max_image_size); - }) - .def_readwrite("size", &vision::detection::YOLOv7::size) - .def_readwrite("padding_value", &vision::detection::YOLOv7::padding_value) - .def_readwrite("is_mini_pad", &vision::detection::YOLOv7::is_mini_pad) - .def_readwrite("is_no_pad", &vision::detection::YOLOv7::is_no_pad) - .def_readwrite("is_scale_up", &vision::detection::YOLOv7::is_scale_up) - .def_readwrite("stride", &vision::detection::YOLOv7::stride) - .def_readwrite("max_wh", &vision::detection::YOLOv7::max_wh); -} -} // namespace fastdeploy diff --git a/python/fastdeploy/vision/detection/__init__.py b/python/fastdeploy/vision/detection/__init__.py index 6de4a3fa634..b5f01f3a77e 100755 --- a/python/fastdeploy/vision/detection/__init__.py +++ b/python/fastdeploy/vision/detection/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import absolute_import -from .contrib.yolov7 import YOLOv7 +from .contrib.yolov7 import * from .contrib.yolor import YOLOR from .contrib.scaled_yolov4 import ScaledYOLOv4 from .contrib.nanodet_plus import NanoDetPlus diff --git a/python/fastdeploy/vision/detection/contrib/yolov5.py b/python/fastdeploy/vision/detection/contrib/yolov5.py index 42eccb88d4a..b8113f3b83d 100644 --- a/python/fastdeploy/vision/detection/contrib/yolov5.py +++ b/python/fastdeploy/vision/detection/contrib/yolov5.py @@ -41,9 +41,19 @@ def size(self): @property def padding_value(self): + """ + padding value for preprocessing, default [114.0, 114.0, 114.0] + """ # padding value, size should be the same as channels return self._preprocessor.padding_value + @property + def is_scale_up(self): + """ + is_scale_up for preprocessing, the input image only can be zoom out, the maximum resize scale cannot exceed 1.0, default true + """ + return self._preprocessor.is_scale_up + @size.setter def size(self, wh): assert isinstance(wh, (list, tuple)),\ @@ -60,6 +70,13 @@ def padding_value(self, value): list), "The value to set `padding_value` must be type of list." self._preprocessor.padding_value = value + @is_scale_up.setter + def is_scale_up(self, value): + assert isinstance( + value, + bool), "The value to set `is_scale_up` must be type of bool." + self._preprocessor.is_scale_up = value + class YOLOv5Postprocessor: def __init__(self): @@ -93,7 +110,7 @@ def nms_threshold(self): @property def multi_label(self): """ - multi_label for postprocessing, default is true + multi_label for postprocessing, set true for eval, default is True """ return self._postprocessor.multi_label diff --git a/python/fastdeploy/vision/detection/contrib/yolov7.py b/python/fastdeploy/vision/detection/contrib/yolov7.py index 0334504851b..510b72ed653 100644 --- a/python/fastdeploy/vision/detection/contrib/yolov7.py +++ b/python/fastdeploy/vision/detection/contrib/yolov7.py @@ -18,77 +18,41 @@ from .... import c_lib_wrap as C -class YOLOv7(FastDeployModel): - def __init__(self, - model_file, - params_file="", - runtime_option=None, - model_format=ModelFormat.ONNX): - """Load a YOLOv7 model exported by YOLOv7. - - :param model_file: (str)Path of model file, e.g ./yolov7.onnx - :param params_file: (str)Path of parameters file, e.g yolox/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string - :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU - :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model +class YOLOv7Preprocessor: + def __init__(self): + """Create a preprocessor for YOLOv7 """ - # 调用基函数进行backend_option的初始化 - # 初始化后的option保存在self._runtime_option - super(YOLOv7, self).__init__(runtime_option) - - self._model = C.vision.detection.YOLOv7( - model_file, params_file, self._runtime_option, model_format) - # 通过self.initialized判断整个模型的初始化是否成功 - assert self.initialized, "YOLOv7 initialize failed." + self._preprocessor = C.vision.detection.YOLOv7Preprocessor() - def predict(self, input_image, conf_threshold=0.25, nms_iou_threshold=0.5): - """Detect an input image + def run(self, input_ims): + """Preprocess input images for YOLOv7 - :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format - :param conf_threshold: confidence threashold for postprocessing, default is 0.25 - :param nms_iou_threshold: iou threashold for NMS, default is 0.5 - :return: DetectionResult + :param: input_ims: (list of numpy.ndarray)The input image + :return: list of FDTensor """ - return self._model.predict(input_image, conf_threshold, - nms_iou_threshold) + return self._preprocessor.run(input_ims) - # 一些跟YOLOv7模型有关的属性封装 - # 多数是预处理相关，可通过修改如model.size = [1280, 1280]改变预处理时resize的大小（前提是模型支持） @property def size(self): """ Argument for image preprocessing step, the preprocess image size, tuple of (width, height), default size = [640, 640] """ - return self._model.size + return self._preprocessor.size @property def padding_value(self): + """ + padding value for preprocessing, default [114.0, 114.0, 114.0] + """ # padding value, size should be the same as channels - return self._model.padding_value - - @property - def is_no_pad(self): - # while is_mini_pad = false and is_no_pad = true, will resize the image to the set size - return self._model.is_no_pad - - @property - def is_mini_pad(self): - # only pad to the minimum rectange which height and width is times of stride - return self._model.is_mini_pad + return self._preprocessor.padding_value @property def is_scale_up(self): - # if is_scale_up is false, the input image only can be zoom out, the maximum resize scale cannot exceed 1.0 - return self._model.is_scale_up - - @property - def stride(self): - # padding stride, for is_mini_pad - return self._model.stride - - @property - def max_wh(self): - # for offseting the boxes by classes when using NMS - return self._model.max_wh + """ + is_scale_up for preprocessing, the input image only can be zoom out, the maximum resize scale cannot exceed 1.0, default true + """ + return self._preprocessor.is_scale_up @size.setter def size(self, wh): @@ -97,43 +61,122 @@ def size(self, wh): assert len(wh) == 2,\ "The value to set `size` must contatins 2 elements means [width, height], but now it contains {} elements.".format( len(wh)) - self._model.size = wh + self._preprocessor.size = wh @padding_value.setter def padding_value(self, value): assert isinstance( value, list), "The value to set `padding_value` must be type of list." - self._model.padding_value = value - - @is_no_pad.setter - def is_no_pad(self, value): - assert isinstance( - value, bool), "The value to set `is_no_pad` must be type of bool." - self._model.is_no_pad = value - - @is_mini_pad.setter - def is_mini_pad(self, value): - assert isinstance( - value, - bool), "The value to set `is_mini_pad` must be type of bool." - self._model.is_mini_pad = value + self._preprocessor.padding_value = value @is_scale_up.setter def is_scale_up(self, value): assert isinstance( value, bool), "The value to set `is_scale_up` must be type of bool." - self._model.is_scale_up = value + self._preprocessor.is_scale_up = value - @stride.setter - def stride(self, value): - assert isinstance( - value, int), "The value to set `stride` must be type of int." - self._model.stride = value - @max_wh.setter - def max_wh(self, value): - assert isinstance( - value, float), "The value to set `max_wh` must be type of float." - self._model.max_wh = value +class YOLOv7Postprocessor: + def __init__(self): + """Create a postprocessor for YOLOv7 + """ + self._postprocessor = C.vision.detection.YOLOv7Postprocessor() + + def run(self, runtime_results, ims_info): + """Postprocess the runtime results for YOLOv7 + + :param: runtime_results: (list of FDTensor)The output FDTensor results from runtime + :param: ims_info: (list of dict)Record input_shape and output_shape + :return: list of DetectionResult(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results, ims_info) + + @property + def conf_threshold(self): + """ + confidence threshold for postprocessing, default is 0.25 + """ + return self._postprocessor.conf_threshold + + @property + def nms_threshold(self): + """ + nms threshold for postprocessing, default is 0.5 + """ + return self._postprocessor.nms_threshold + + @conf_threshold.setter + def conf_threshold(self, conf_threshold): + assert isinstance(conf_threshold, float),\ + "The value to set `conf_threshold` must be type of float." + self._postprocessor.conf_threshold = conf_threshold + + @nms_threshold.setter + def nms_threshold(self, nms_threshold): + assert isinstance(nms_threshold, float),\ + "The value to set `nms_threshold` must be type of float." + self._postprocessor.nms_threshold = nms_threshold + + +class YOLOv7(FastDeployModel): + def __init__(self, + model_file, + params_file="", + runtime_option=None, + model_format=ModelFormat.ONNX): + """Load a YOLOv7 model exported by YOLOv7. + + :param model_file: (str)Path of model file, e.g ./yolov7.onnx + :param params_file: (str)Path of parameters file, e.g yolox/model.pdiparams, if the model_fomat is ModelFormat.ONNX, this param will be ignored, can be set as empty string + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + # 调用基函数进行backend_option的初始化 + # 初始化后的option保存在self._runtime_option + super(YOLOv7, self).__init__(runtime_option) + + assert model_format == ModelFormat.ONNX, "YOLOv7 only support model format of ModelFormat.ONNX now." + self._model = C.vision.detection.YOLOv7( + model_file, params_file, self._runtime_option, model_format) + # 通过self.initialized判断整个模型的初始化是否成功 + assert self.initialized, "YOLOv7 initialize failed." + + def predict(self, input_image, conf_threshold=0.25, nms_iou_threshold=0.5): + """Detect an input image + + :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :param conf_threshold: confidence threshold for postprocessing, default is 0.25 + :param nms_iou_threshold: iou threshold for NMS, default is 0.5 + :return: DetectionResult + """ + + self.postprocessor.conf_threshold = conf_threshold + self.postprocessor.nms_threshold = nms_iou_threshold + return self._model.predict(input_image) + + def batch_predict(self, images): + """Classify a batch of input image + + :param im: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return list of DetectionResult + """ + + return self._model.batch_predict(images) + + @property + def preprocessor(self): + """Get YOLOv7Preprocessor object of the loaded model + + :return YOLOv7Preprocessor + """ + return self._model.preprocessor + + @property + def postprocessor(self): + """Get YOLOv7Postprocessor object of the loaded model + + :return YOLOv7Postprocessor + """ + return self._model.postprocessor diff --git a/tests/models/test_yolov7.py b/tests/models/test_yolov7.py new file mode 100755 index 00000000000..ba08fbaf5bc --- /dev/null +++ b/tests/models/test_yolov7.py @@ -0,0 +1,165 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from fastdeploy import ModelFormat +import fastdeploy as fd +import cv2 +import os +import pickle +import numpy as np +import runtime_config as rc + + +def test_detection_yolov7(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov7.onnx" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + input_url2 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000570688.jpg" + result_url1 = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov7_result1.pkl" + result_url2 = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov7_result2.pkl" + fd.download(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(input_url2, "resources") + fd.download(result_url1, "resources") + fd.download(result_url2, "resources") + + model_file = "resources/yolov7.onnx" + model = fd.vision.detection.YOLOv7( + model_file, runtime_option=rc.test_option) + + with open("resources/yolov7_result1.pkl", "rb") as f: + expect1 = pickle.load(f) + + with open("resources/yolov7_result2.pkl", "rb") as f: + expect2 = pickle.load(f) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + im2 = cv2.imread("./resources/000000570688.jpg") + + for i in range(3): + # test single predict + result1 = model.predict(im1) + result2 = model.predict(im2) + + diff_boxes_1 = np.fabs( + np.array(result1.boxes) - np.array(expect1["boxes"])) + diff_boxes_2 = np.fabs( + np.array(result2.boxes) - np.array(expect2["boxes"])) + + diff_label_1 = np.fabs( + np.array(result1.label_ids) - np.array(expect1["label_ids"])) + diff_label_2 = np.fabs( + np.array(result2.label_ids) - np.array(expect2["label_ids"])) + + diff_scores_1 = np.fabs( + np.array(result1.scores) - np.array(expect1["scores"])) + diff_scores_2 = np.fabs( + np.array(result2.scores) - np.array(expect2["scores"])) + + assert diff_boxes_1.max( + ) < 1e-06, "There's difference in detection boxes 1." + assert diff_label_1.max( + ) < 1e-06, "There's difference in detection label 1." + assert diff_scores_1.max( + ) < 1e-05, "There's difference in detection score 1." + + assert diff_boxes_2.max( + ) < 1e-06, "There's difference in detection boxes 2." + assert diff_label_2.max( + ) < 1e-06, "There's difference in detection label 2." + assert diff_scores_2.max( + ) < 1e-05, "There's difference in detection score 2." + + # test batch predict + results = model.batch_predict([im1, im2]) + result1 = results[0] + result2 = results[1] + + diff_boxes_1 = np.fabs( + np.array(result1.boxes) - np.array(expect1["boxes"])) + diff_boxes_2 = np.fabs( + np.array(result2.boxes) - np.array(expect2["boxes"])) + + diff_label_1 = np.fabs( + np.array(result1.label_ids) - np.array(expect1["label_ids"])) + diff_label_2 = np.fabs( + np.array(result2.label_ids) - np.array(expect2["label_ids"])) + + diff_scores_1 = np.fabs( + np.array(result1.scores) - np.array(expect1["scores"])) + diff_scores_2 = np.fabs( + np.array(result2.scores) - np.array(expect2["scores"])) + assert diff_boxes_1.max( + ) < 1e-06, "There's difference in detection boxes 1." + assert diff_label_1.max( + ) < 1e-06, "There's difference in detection label 1." + assert diff_scores_1.max( + ) < 1e-05, "There's difference in detection score 1." + + assert diff_boxes_2.max( + ) < 1e-06, "There's difference in detection boxes 2." + assert diff_label_2.max( + ) < 1e-06, "There's difference in detection label 2." + assert diff_scores_2.max( + ) < 1e-05, "There's difference in detection score 2." + + +def test_detection_yolov7_runtime(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov7.onnx" + input_url1 = "https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg" + result_url1 = "https://bj.bcebos.com/paddlehub/fastdeploy/yolov7_result1.pkl" + fd.download(model_url, "resources") + fd.download(input_url1, "resources") + fd.download(result_url1, "resources") + + model_file = "resources/yolov7.onnx" + + preprocessor = fd.vision.detection.YOLOv7Preprocessor() + postprocessor = fd.vision.detection.YOLOv7Postprocessor() + + rc.test_option.set_model_path(model_file, model_format=ModelFormat.ONNX) + rc.test_option.use_openvino_backend() + runtime = fd.Runtime(rc.test_option) + + with open("resources/yolov7_result1.pkl", "rb") as f: + expect1 = pickle.load(f) + + # compare diff + im1 = cv2.imread("./resources/000000014439.jpg") + + for i in range(3): + # test runtime + input_tensors, ims_info = preprocessor.run([im1.copy()]) + output_tensors = runtime.infer({"images": input_tensors[0]}) + results = postprocessor.run(output_tensors, ims_info) + result1 = results[0] + + diff_boxes_1 = np.fabs( + np.array(result1.boxes) - np.array(expect1["boxes"])) + diff_label_1 = np.fabs( + np.array(result1.label_ids) - np.array(expect1["label_ids"])) + diff_scores_1 = np.fabs( + np.array(result1.scores) - np.array(expect1["scores"])) + + assert diff_boxes_1.max( + ) < 1e-04, "There's difference in detection boxes 1." + assert diff_label_1.max( + ) < 1e-06, "There's difference in detection label 1." + assert diff_scores_1.max( + ) < 1e-05, "There's difference in detection score 1." + + +if __name__ == "__main__": + test_detection_yolov7() + test_detection_yolov7_runtime() From 1310dd688dc914a27c740af1c639555d2c51c2e2 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Fri, 18 Nov 2022 11:12:16 +0800 Subject: [PATCH 061/136] [Bug Fix] fix AllocateFaceDetectionResultFromJava error (#628) * [Bug Fix] fix android app detail page errors * [Android] fix realtime camera mode and shutter * [Bug Fix] fix AllocateSegmentationResultFromJava error --- java/android/app/src/main/AndroidManifest.xml | 4 ++-- .../segmentation/SegmentationMainActivity.java | 16 +--------------- .../cpp/fastdeploy_jni/vision/results_jni.cc | 4 +++- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index 95511824f5c..0f963ffe416 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -15,14 +15,14 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/AppTheme"> - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index f1809d8105c..c1d9c72b68b 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -352,30 +352,16 @@ public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) { @Override public void onStartTrackingTouch(SeekBar seekBar) { - } @Override public void onStopTrackingTouch(SeekBar seekBar) { -// runOnUiThread(new Runnable() { -// @Override -// public void run() { -// if (TYPE == ALBUM_SELECT) { -// SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms -// detail(picBitmap); -// } else { -// SystemClock.sleep(TIME_SLEEP_INTERVAL * 10); // 500ms -// svPreview.onPause(); -// detail(shutterBitmap); -// } -// } -// }); } }); } private void detail(Bitmap bitmap) { - predictor.predict(bitmap, true, 0.6f); + predictor.predict(bitmap, true, 0.4f); resultImage.setImageBitmap(bitmap); resultNum = 1.0f; } diff --git a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc index e817731457d..9ae6054aaa6 100644 --- a/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc +++ b/java/android/fastdeploy/src/main/cpp/fastdeploy_jni/vision/results_jni.cc @@ -982,7 +982,9 @@ bool AllocateFaceDetectionResultFromJava( if (j_landmarks_per_face > 0) { jobjectArray j_face_det_landmarks_float_arr = reinterpret_cast( env->GetObjectField(j_face_det_result_obj, j_face_det_landmarks_id_cc)); - for (int i = 0; i < len; ++i) { + const int landmarks_len = env->GetArrayLength(j_face_det_landmarks_float_arr); + + for (int i = 0; i < landmarks_len; ++i) { auto j_landmark = reinterpret_cast( env->GetObjectArrayElement(j_face_det_landmarks_float_arr, i)); if (env->GetArrayLength(j_landmark) == 2) { From c4f627f8143ad8a32a9221e299399003032fbf1d Mon Sep 17 00:00:00 2001 From: Thomas Young <35565423+HexToString@users.noreply.github.com> Date: Fri, 18 Nov 2022 13:17:42 +0800 Subject: [PATCH 062/136] [Model] change ocr pre and post (#568) * change ocr pre and post * add pybind * change ocr * fix bug * fix bug * fix bug * fix bug * fix bug * fix bug * fix copy bug * fix code style * fix bug * add new function * fix windows ci bug --- fastdeploy/utils/utils.h | 1 + fastdeploy/vision.h | 1 + fastdeploy/vision/ocr/ppocr/classifier.cc | 91 +-------- fastdeploy/vision/ocr/ppocr/classifier.h | 27 ++- .../vision/ocr/ppocr/cls_postprocessor.cc | 65 +++++++ .../vision/ocr/ppocr/cls_postprocessor.h | 51 ++++++ .../vision/ocr/ppocr/cls_preprocessor.cc | 88 +++++++++ .../vision/ocr/ppocr/cls_preprocessor.h | 51 ++++++ fastdeploy/vision/ocr/ppocr/dbdetector.cc | 148 ++------------- fastdeploy/vision/ocr/ppocr/dbdetector.h | 48 +++-- .../vision/ocr/ppocr/det_postprocessor.cc | 110 +++++++++++ .../vision/ocr/ppocr/det_postprocessor.h | 62 +++++++ .../vision/ocr/ppocr/det_preprocessor.cc | 113 ++++++++++++ .../vision/ocr/ppocr/det_preprocessor.h | 54 ++++++ .../vision/ocr/ppocr/ocrmodel_pybind.cc | 166 +++++++++++++++-- fastdeploy/vision/ocr/ppocr/ppocr_pybind.cc | 18 ++ fastdeploy/vision/ocr/ppocr/ppocr_v2.cc | 116 ++++++------ fastdeploy/vision/ocr/ppocr/ppocr_v2.h | 13 +- fastdeploy/vision/ocr/ppocr/ppocr_v3.h | 4 +- .../vision/ocr/ppocr/rec_postprocessor.cc | 112 ++++++++++++ .../vision/ocr/ppocr/rec_postprocessor.h | 55 ++++++ .../vision/ocr/ppocr/rec_preprocessor.cc | 99 ++++++++++ .../vision/ocr/ppocr/rec_preprocessor.h | 52 ++++++ fastdeploy/vision/ocr/ppocr/recognizer.cc | 150 ++------------- fastdeploy/vision/ocr/ppocr/recognizer.h | 31 +--- .../ocr/ppocr/utils/ocr_postprocess_op.cc | 10 +- .../ocr/ppocr/utils/ocr_postprocess_op.h | 5 +- fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h | 4 +- .../vision/ocr/ppocr/utils/sorted_boxes.cc | 14 +- python/fastdeploy/vision/ocr/__init__.py | 8 +- .../fastdeploy/vision/ocr/ppocr/__init__.py | 172 +++++++++++++++--- 31 files changed, 1386 insertions(+), 553 deletions(-) mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/classifier.cc mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/classifier.h create mode 100644 fastdeploy/vision/ocr/ppocr/cls_postprocessor.cc create mode 100644 fastdeploy/vision/ocr/ppocr/cls_postprocessor.h create mode 100644 fastdeploy/vision/ocr/ppocr/cls_preprocessor.cc create mode 100644 fastdeploy/vision/ocr/ppocr/cls_preprocessor.h mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/dbdetector.cc mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/dbdetector.h create mode 100644 fastdeploy/vision/ocr/ppocr/det_postprocessor.cc create mode 100644 fastdeploy/vision/ocr/ppocr/det_postprocessor.h create mode 100644 fastdeploy/vision/ocr/ppocr/det_preprocessor.cc create mode 100644 fastdeploy/vision/ocr/ppocr/det_preprocessor.h mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/ocrmodel_pybind.cc mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/ppocr_pybind.cc mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/ppocr_v2.cc mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/ppocr_v2.h mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/ppocr_v3.h create mode 100644 fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc create mode 100644 fastdeploy/vision/ocr/ppocr/rec_postprocessor.h create mode 100644 fastdeploy/vision/ocr/ppocr/rec_preprocessor.cc create mode 100644 fastdeploy/vision/ocr/ppocr/rec_preprocessor.h mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/recognizer.cc mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/recognizer.h mode change 100644 => 100755 fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.cc mode change 100644 => 100755 python/fastdeploy/vision/ocr/__init__.py mode change 100644 => 100755 python/fastdeploy/vision/ocr/ppocr/__init__.py diff --git a/fastdeploy/utils/utils.h b/fastdeploy/utils/utils.h index 2c726976348..994ea9baa50 100644 --- a/fastdeploy/utils/utils.h +++ b/fastdeploy/utils/utils.h @@ -22,6 +22,7 @@ #include #include #include +#include #if defined(_WIN32) #ifdef FASTDEPLOY_LIB diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index 82f06e0036b..581d3b91ae5 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -48,6 +48,7 @@ #include "fastdeploy/vision/matting/ppmatting/ppmatting.h" #include "fastdeploy/vision/ocr/ppocr/classifier.h" #include "fastdeploy/vision/ocr/ppocr/dbdetector.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" #include "fastdeploy/vision/ocr/ppocr/ppocr_v2.h" #include "fastdeploy/vision/ocr/ppocr/ppocr_v3.h" #include "fastdeploy/vision/ocr/ppocr/recognizer.h" diff --git a/fastdeploy/vision/ocr/ppocr/classifier.cc b/fastdeploy/vision/ocr/ppocr/classifier.cc old mode 100644 new mode 100755 index 1fbd4cc364b..130329735e1 --- a/fastdeploy/vision/ocr/ppocr/classifier.cc +++ b/fastdeploy/vision/ocr/ppocr/classifier.cc @@ -41,16 +41,7 @@ Classifier::Classifier(const std::string& model_file, initialized = Initialize(); } -// Init bool Classifier::Initialize() { - // pre&post process parameters - cls_thresh = 0.9; - cls_image_shape = {3, 48, 192}; - cls_batch_num = 1; - mean = {0.5f, 0.5f, 0.5f}; - scale = {0.5f, 0.5f, 0.5f}; - is_scale = true; - if (!InitRuntime()) { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; return false; @@ -59,85 +50,23 @@ bool Classifier::Initialize() { return true; } -void OcrClassifierResizeImage(Mat* mat, - const std::vector& rec_image_shape) { - int imgC = rec_image_shape[0]; - int imgH = rec_image_shape[1]; - int imgW = rec_image_shape[2]; - - float ratio = float(mat->Width()) / float(mat->Height()); - - int resize_w; - if (ceilf(imgH * ratio) > imgW) - resize_w = imgW; - else - resize_w = int(ceilf(imgH * ratio)); - - Resize::Run(mat, resize_w, imgH); - - std::vector value = {0, 0, 0}; - if (resize_w < imgW) { - Pad::Run(mat, 0, 0, 0, imgW - resize_w, value); - } -} - -bool Classifier::Preprocess(Mat* mat, FDTensor* output) { - // 1. cls resizes - // 2. normalize - // 3. batch_permute - OcrClassifierResizeImage(mat, cls_image_shape); - - Normalize::Run(mat, mean, scale, true); - - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); - - mat->ShareWithTensor(output); - output->shape.insert(output->shape.begin(), 1); - - return true; -} - -bool Classifier::Postprocess(FDTensor& infer_result, - std::tuple* cls_result) { - std::vector output_shape = infer_result.shape; - FDASSERT(output_shape[0] == 1, "Only support batch =1 now."); - - float* out_data = static_cast(infer_result.Data()); - - int label = std::distance( - &out_data[0], std::max_element(&out_data[0], &out_data[output_shape[1]])); - - float score = - float(*std::max_element(&out_data[0], &out_data[output_shape[1]])); - - std::get<0>(*cls_result) = label; - std::get<1>(*cls_result) = score; - - return true; -} - -bool Classifier::Predict(cv::Mat* img, std::tuple* cls_result) { - Mat mat(*img); - std::vector input_tensors(1); - - if (!Preprocess(&mat, &input_tensors[0])) { - FDERROR << "Failed to preprocess input image." << std::endl; +bool Classifier::BatchPredict(const std::vector& images, + std::vector* cls_labels, std::vector* cls_scores) { + std::vector fd_images = WrapMat(images); + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_)) { + FDERROR << "Failed to preprocess the input image." << std::endl; return false; } - - input_tensors[0].name = InputInfoOfRuntime(0).name; - std::vector output_tensors; - if (!Infer(input_tensors, &output_tensors)) { - FDERROR << "Failed to inference." << std::endl; + reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; return false; } - if (!Postprocess(output_tensors[0], cls_result)) { - FDERROR << "Failed to post process." << std::endl; + if (!postprocessor_.Run(reused_output_tensors_, cls_labels, cls_scores)) { + FDERROR << "Failed to postprocess the inference cls_results by runtime." << std::endl; return false; } - return true; } diff --git a/fastdeploy/vision/ocr/ppocr/classifier.h b/fastdeploy/vision/ocr/ppocr/classifier.h old mode 100644 new mode 100755 index d87fec6fa6b..d3430e4e028 --- a/fastdeploy/vision/ocr/ppocr/classifier.h +++ b/fastdeploy/vision/ocr/ppocr/classifier.h @@ -17,6 +17,8 @@ #include "fastdeploy/vision/common/processors/transform.h" #include "fastdeploy/vision/common/result.h" #include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" +#include "fastdeploy/vision/ocr/ppocr/cls_postprocessor.h" +#include "fastdeploy/vision/ocr/ppocr/cls_preprocessor.h" namespace fastdeploy { namespace vision { @@ -41,29 +43,22 @@ class FASTDEPLOY_DECL Classifier : public FastDeployModel { const ModelFormat& model_format = ModelFormat::PADDLE); /// Get model's name std::string ModelName() const { return "ppocr/ocr_cls"; } - /** \brief Predict the input image and get OCR classification model result. + + /** \brief BatchPredict the input image and get OCR classification model cls_result. * - * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. - * \param[in] result The output of OCR classification model result will be writen to this structure. + * \param[in] images The list of input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] cls_results The output of OCR classification model cls_result will be writen to this structure. * \return true if the prediction is successed, otherwise false. */ - virtual bool Predict(cv::Mat* img, std::tuple* result); - - // Pre & Post parameters - float cls_thresh; - std::vector cls_image_shape; - int cls_batch_num; + virtual bool BatchPredict(const std::vector& images, + std::vector* cls_labels, + std::vector* cls_scores); - std::vector mean; - std::vector scale; - bool is_scale; + ClassifierPreprocessor preprocessor_; + ClassifierPostprocessor postprocessor_; private: bool Initialize(); - /// Preprocess the input data, and set the preprocessed results to `outputs` - bool Preprocess(Mat* img, FDTensor* output); - /// Postprocess the inferenced results, and set the final result to `result` - bool Postprocess(FDTensor& infer_result, std::tuple* result); }; } // namespace ocr diff --git a/fastdeploy/vision/ocr/ppocr/cls_postprocessor.cc b/fastdeploy/vision/ocr/ppocr/cls_postprocessor.cc new file mode 100644 index 00000000000..5eb6b5d690a --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/cls_postprocessor.cc @@ -0,0 +1,65 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/cls_postprocessor.h" +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +ClassifierPostprocessor::ClassifierPostprocessor() { + initialized_ = true; +} + +bool SingleBatchPostprocessor(const float* out_data, const size_t& length, int* cls_label, float* cls_score) { + + *cls_label = std::distance( + &out_data[0], std::max_element(&out_data[0], &out_data[length])); + + *cls_score = + float(*std::max_element(&out_data[0], &out_data[length])); + return true; +} + +bool ClassifierPostprocessor::Run(const std::vector& tensors, + std::vector* cls_labels, + std::vector* cls_scores) { + if (!initialized_) { + FDERROR << "Postprocessor is not initialized." << std::endl; + return false; + } + // Classifier have only 1 output tensor. + const FDTensor& tensor = tensors[0]; + + // For Classifier, the output tensor shape = [batch,2] + size_t batch = tensor.shape[0]; + size_t length = accumulate(tensor.shape.begin()+1, tensor.shape.end(), 1, std::multiplies()); + + cls_labels->resize(batch); + cls_scores->resize(batch); + const float* tensor_data = reinterpret_cast(tensor.Data()); + + for (int i_batch = 0; i_batch < batch; ++i_batch) { + if(!SingleBatchPostprocessor(tensor_data, length, &cls_labels->at(i_batch),&cls_scores->at(i_batch))) return false; + tensor_data = tensor_data + length; + } + + return true; +} + +} // namespace classification +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/cls_postprocessor.h b/fastdeploy/vision/ocr/ppocr/cls_postprocessor.h new file mode 100644 index 00000000000..15bf098c701 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/cls_postprocessor.h @@ -0,0 +1,51 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +namespace vision { + +namespace ocr { +/*! @brief Postprocessor object for Classifier serials model. + */ +class FASTDEPLOY_DECL ClassifierPostprocessor { + public: + /** \brief Create a postprocessor instance for Classifier serials model + * + */ + ClassifierPostprocessor(); + + /** \brief Process the result of runtime and fill to ClassifyResult structure + * + * \param[in] tensors The inference result from runtime + * \param[in] cls_labels The output result of classification + * \param[in] cls_scores The output result of classification + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector* cls_labels, std::vector* cls_scores); + + float cls_thresh_ = 0.9; + + private: + bool initialized_ = false; +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/cls_preprocessor.cc b/fastdeploy/vision/ocr/ppocr/cls_preprocessor.cc new file mode 100644 index 00000000000..1f09936905c --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/cls_preprocessor.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/cls_preprocessor.h" +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" +#include "fastdeploy/function/concat.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +ClassifierPreprocessor::ClassifierPreprocessor() { + initialized_ = true; +} + +void OcrClassifierResizeImage(FDMat* mat, + const std::vector& cls_image_shape) { + int imgC = cls_image_shape[0]; + int imgH = cls_image_shape[1]; + int imgW = cls_image_shape[2]; + + float ratio = float(mat->Width()) / float(mat->Height()); + + int resize_w; + if (ceilf(imgH * ratio) > imgW) + resize_w = imgW; + else + resize_w = int(ceilf(imgH * ratio)); + + Resize::Run(mat, resize_w, imgH); + + std::vector value = {0, 0, 0}; + if (resize_w < imgW) { + Pad::Run(mat, 0, 0, 0, imgW - resize_w, value); + } +} + +bool ClassifierPreprocessor::Run(std::vector* images, std::vector* outputs) { + if (!initialized_) { + FDERROR << "The preprocessor is not initialized." << std::endl; + return false; + } + if (images->size() == 0) { + FDERROR << "The size of input images should be greater than 0." << std::endl; + return false; + } + + for (size_t i = 0; i < images->size(); ++i) { + FDMat* mat = &(images->at(i)); + OcrClassifierResizeImage(mat, cls_image_shape_); + NormalizeAndPermute::Run(mat, mean_, scale_, is_scale_); + /* + Normalize::Run(mat, mean_, scale_, is_scale_); + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + */ + } + // Only have 1 output Tensor. + outputs->resize(1); + // Concat all the preprocessed data to a batch tensor + std::vector tensors(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + (*images)[i].ShareWithTensor(&(tensors[i])); + tensors[i].ExpandDim(0); + } + if (tensors.size() == 1) { + (*outputs)[0] = std::move(tensors[0]); + } else { + function::Concat(tensors, &((*outputs)[0]), 0); + } + return true; +} + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/cls_preprocessor.h b/fastdeploy/vision/ocr/ppocr/cls_preprocessor.h new file mode 100644 index 00000000000..a701e7e3ae6 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/cls_preprocessor.h @@ -0,0 +1,51 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace ocr { +/*! @brief Preprocessor object for Classifier serials model. + */ +class FASTDEPLOY_DECL ClassifierPreprocessor { + public: + /** \brief Create a preprocessor instance for Classifier serials model + * + */ + ClassifierPreprocessor(); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime + * \return true if the preprocess successed, otherwise false + */ + bool Run(std::vector* images, std::vector* outputs); + + std::vector mean_ = {0.5f, 0.5f, 0.5f}; + std::vector scale_ = {0.5f, 0.5f, 0.5f}; + bool is_scale_ = true; + std::vector cls_image_shape_ = {3, 48, 192}; + + private: + bool initialized_ = false; +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/dbdetector.cc b/fastdeploy/vision/ocr/ppocr/dbdetector.cc old mode 100644 new mode 100755 index 8ee44fddfaf..68a994afcbc --- a/fastdeploy/vision/ocr/ppocr/dbdetector.cc +++ b/fastdeploy/vision/ocr/ppocr/dbdetector.cc @@ -43,158 +43,50 @@ DBDetector::DBDetector(const std::string& model_file, // Init bool DBDetector::Initialize() { - // pre&post process parameters - max_side_len = 960; - - det_db_thresh = 0.3; - det_db_box_thresh = 0.6; - det_db_unclip_ratio = 1.5; - det_db_score_mode = "slow"; - use_dilation = false; - - mean = {0.485f, 0.456f, 0.406f}; - scale = {0.229f, 0.224f, 0.225f}; - is_scale = true; - if (!InitRuntime()) { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; return false; } - return true; } -void OcrDetectorResizeImage(Mat* img, int max_size_len, float* ratio_h, - float* ratio_w) { - int w = img->Width(); - int h = img->Height(); - - float ratio = 1.f; - int max_wh = w >= h ? w : h; - if (max_wh > max_size_len) { - if (h > w) { - ratio = float(max_size_len) / float(h); - } else { - ratio = float(max_size_len) / float(w); - } +bool DBDetector::Predict(cv::Mat* img, + std::vector>* boxes_result) { + if (!Predict(*img, boxes_result)) { + return false; } - - int resize_h = int(float(h) * ratio); - int resize_w = int(float(w) * ratio); - - resize_h = std::max(int(std::round(float(resize_h) / 32) * 32), 32); - resize_w = std::max(int(std::round(float(resize_w) / 32) * 32), 32); - - Resize::Run(img, resize_w, resize_h); - - *ratio_h = float(resize_h) / float(h); - *ratio_w = float(resize_w) / float(w); -} - -bool DBDetector::Preprocess( - Mat* mat, FDTensor* output, - std::map>* im_info) { - // Resize - OcrDetectorResizeImage(mat, max_side_len, &ratio_h, &ratio_w); - // Normalize - Normalize::Run(mat, mean, scale, true); - - (*im_info)["output_shape"] = {static_cast(mat->Height()), - static_cast(mat->Width())}; - //-CHW - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); - - mat->ShareWithTensor(output); - output->shape.insert(output->shape.begin(), 1); return true; } -bool DBDetector::Postprocess( - FDTensor& infer_result, std::vector>* boxes_result, - const std::map>& im_info) { - std::vector output_shape = infer_result.shape; - FDASSERT(output_shape[0] == 1, "Only support batch =1 now."); - int n2 = output_shape[2]; - int n3 = output_shape[3]; - int n = n2 * n3; - - float* out_data = static_cast(infer_result.Data()); - // prepare bitmap - std::vector pred(n, 0.0); - std::vector cbuf(n, ' '); - - for (int i = 0; i < n; i++) { - pred[i] = float(out_data[i]); - cbuf[i] = (unsigned char)((out_data[i]) * 255); - } - cv::Mat cbuf_map(n2, n3, CV_8UC1, (unsigned char*)cbuf.data()); - cv::Mat pred_map(n2, n3, CV_32F, (float*)pred.data()); - - const double threshold = det_db_thresh * 255; - const double maxvalue = 255; - cv::Mat bit_map; - cv::threshold(cbuf_map, bit_map, threshold, maxvalue, cv::THRESH_BINARY); - if (use_dilation) { - cv::Mat dila_ele = - cv::getStructuringElement(cv::MORPH_RECT, cv::Size(2, 2)); - cv::dilate(bit_map, bit_map, dila_ele); - } - - std::vector>> boxes; - - boxes = - post_processor_.BoxesFromBitmap(pred_map, bit_map, det_db_box_thresh, - det_db_unclip_ratio, det_db_score_mode); - - boxes = post_processor_.FilterTagDetRes(boxes, ratio_h, ratio_w, im_info); - - // boxes to boxes_result - for (int i = 0; i < boxes.size(); i++) { - std::array new_box; - int k = 0; - for (auto& vec : boxes[i]) { - for (auto& e : vec) { - new_box[k++] = e; - } - } - boxes_result->push_back(new_box); +bool DBDetector::Predict(const cv::Mat& img, + std::vector>* boxes_result) { + std::vector>> det_results; + if (!BatchPredict({img}, &det_results)) { + return false; } - + *boxes_result = std::move(det_results[0]); return true; } -bool DBDetector::Predict(cv::Mat* img, - std::vector>* boxes_result) { - Mat mat(*img); - - std::vector input_tensors(1); - - std::map> im_info; - - // Record the shape of image and the shape of preprocessed image - im_info["input_shape"] = {static_cast(mat.Height()), - static_cast(mat.Width())}; - im_info["output_shape"] = {static_cast(mat.Height()), - static_cast(mat.Width())}; - - if (!Preprocess(&mat, &input_tensors[0], &im_info)) { +bool DBDetector::BatchPredict(const std::vector& images, + std::vector>>* det_results) { + std::vector fd_images = WrapMat(images); + std::vector> batch_det_img_info; + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_, &batch_det_img_info)) { FDERROR << "Failed to preprocess input image." << std::endl; return false; } - input_tensors[0].name = InputInfoOfRuntime(0).name; - std::vector output_tensors; - if (!Infer(input_tensors, &output_tensors)) { - FDERROR << "Failed to inference." << std::endl; + reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; return false; } - if (!Postprocess(output_tensors[0], boxes_result, im_info)) { - FDERROR << "Failed to post process." << std::endl; + if (!postprocessor_.Run(reused_output_tensors_, det_results, batch_det_img_info)) { + FDERROR << "Failed to postprocess the inference cls_results by runtime." << std::endl; return false; } - return true; } diff --git a/fastdeploy/vision/ocr/ppocr/dbdetector.h b/fastdeploy/vision/ocr/ppocr/dbdetector.h old mode 100644 new mode 100755 index e0baf319cb3..d3b99d598f2 --- a/fastdeploy/vision/ocr/ppocr/dbdetector.h +++ b/fastdeploy/vision/ocr/ppocr/dbdetector.h @@ -17,6 +17,8 @@ #include "fastdeploy/vision/common/processors/transform.h" #include "fastdeploy/vision/common/result.h" #include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" +#include "fastdeploy/vision/ocr/ppocr/det_postprocessor.h" +#include "fastdeploy/vision/ocr/ppocr/det_preprocessor.h" namespace fastdeploy { namespace vision { @@ -44,40 +46,34 @@ class FASTDEPLOY_DECL DBDetector : public FastDeployModel { std::string ModelName() const { return "ppocr/ocr_det"; } /** \brief Predict the input image and get OCR detection model result. * - * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. * \param[in] boxes_result The output of OCR detection model result will be writen to this structure. * \return true if the prediction is successed, otherwise false. */ - virtual bool Predict(cv::Mat* im, + virtual bool Predict(cv::Mat* img, std::vector>* boxes_result); + /** \brief Predict the input image and get OCR detection model result. + * + * \param[in] img The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] boxes_result The output of OCR detection model result will be writen to this structure. + * \return true if the prediction is successed, otherwise false. + */ + virtual bool Predict(const cv::Mat& img, + std::vector>* boxes_result); + /** \brief BatchPredict the input image and get OCR detection model result. + * + * \param[in] images The list input of image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] det_results The output of OCR detection model result will be writen to this structure. + * \return true if the prediction is successed, otherwise false. + */ + virtual bool BatchPredict(const std::vector& images, + std::vector>>* det_results); - // Pre & Post process parameters - int max_side_len; - - float ratio_h{}; - float ratio_w{}; - - double det_db_thresh; - double det_db_box_thresh; - double det_db_unclip_ratio; - std::string det_db_score_mode; - bool use_dilation; - - std::vector mean; - std::vector scale; - bool is_scale; + DBDetectorPreprocessor preprocessor_; + DBDetectorPostprocessor postprocessor_; private: bool Initialize(); - /// Preprocess the input data, and set the preprocessed results to `outputs` - bool Preprocess(Mat* mat, FDTensor* outputs, - std::map>* im_info); - /*! @brief Postprocess the inferenced results, and set the final result to `boxes_result` - */ - bool Postprocess(FDTensor& infer_result, - std::vector>* boxes_result, - const std::map>& im_info); - PostProcessor post_processor_; }; } // namespace ocr diff --git a/fastdeploy/vision/ocr/ppocr/det_postprocessor.cc b/fastdeploy/vision/ocr/ppocr/det_postprocessor.cc new file mode 100644 index 00000000000..34a88c0111f --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/det_postprocessor.cc @@ -0,0 +1,110 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/det_postprocessor.h" +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +DBDetectorPostprocessor::DBDetectorPostprocessor() { + initialized_ = true; +} + +bool DBDetectorPostprocessor::SingleBatchPostprocessor( + const float* out_data, + int n2, + int n3, + const std::array& det_img_info, + std::vector>* boxes_result + ) { + int n = n2 * n3; + + // prepare bitmap + std::vector pred(n, 0.0); + std::vector cbuf(n, ' '); + + for (int i = 0; i < n; i++) { + pred[i] = float(out_data[i]); + cbuf[i] = (unsigned char)((out_data[i]) * 255); + } + cv::Mat cbuf_map(n2, n3, CV_8UC1, (unsigned char*)cbuf.data()); + cv::Mat pred_map(n2, n3, CV_32F, (float*)pred.data()); + + const double threshold = det_db_thresh_ * 255; + const double maxvalue = 255; + cv::Mat bit_map; + cv::threshold(cbuf_map, bit_map, threshold, maxvalue, cv::THRESH_BINARY); + if (use_dilation_) { + cv::Mat dila_ele = + cv::getStructuringElement(cv::MORPH_RECT, cv::Size(2, 2)); + cv::dilate(bit_map, bit_map, dila_ele); + } + + std::vector>> boxes; + + boxes = + post_processor_.BoxesFromBitmap(pred_map, bit_map, det_db_box_thresh_, + det_db_unclip_ratio_, det_db_score_mode_); + + boxes = post_processor_.FilterTagDetRes(boxes, det_img_info); + + // boxes to boxes_result + for (int i = 0; i < boxes.size(); i++) { + std::array new_box; + int k = 0; + for (auto& vec : boxes[i]) { + for (auto& e : vec) { + new_box[k++] = e; + } + } + boxes_result->push_back(new_box); + } + + return true; +} + +bool DBDetectorPostprocessor::Run(const std::vector& tensors, + std::vector>>* results, + const std::vector>& batch_det_img_info) { + if (!initialized_) { + FDERROR << "Postprocessor is not initialized." << std::endl; + return false; + } + // DBDetector have only 1 output tensor. + const FDTensor& tensor = tensors[0]; + + // For DBDetector, the output tensor shape = [batch, 1, ?, ?] + size_t batch = tensor.shape[0]; + size_t length = accumulate(tensor.shape.begin()+1, tensor.shape.end(), 1, std::multiplies()); + const float* tensor_data = reinterpret_cast(tensor.Data()); + + results->resize(batch); + for (int i_batch = 0; i_batch < batch; ++i_batch) { + if(!SingleBatchPostprocessor(tensor_data, + tensor.shape[2], + tensor.shape[3], + batch_det_img_info[i_batch], + &results->at(i_batch) + ))return false; + tensor_data = tensor_data + length; + } + return true; +} + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/det_postprocessor.h b/fastdeploy/vision/ocr/ppocr/det_postprocessor.h new file mode 100644 index 00000000000..f98b89b028b --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/det_postprocessor.h @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +namespace vision { + +namespace ocr { +/*! @brief Postprocessor object for DBDetector serials model. + */ +class FASTDEPLOY_DECL DBDetectorPostprocessor { + public: + /** \brief Create a postprocessor instance for DBDetector serials model + * + */ + DBDetectorPostprocessor(); + + /** \brief Process the result of runtime and fill to results structure + * + * \param[in] tensors The inference result from runtime + * \param[in] results The output result of detector + * \param[in] batch_det_img_info The detector_preprocess result + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector>>* results, + const std::vector>& batch_det_img_info); + + double det_db_thresh_ = 0.3; + double det_db_box_thresh_ = 0.6; + double det_db_unclip_ratio_ = 1.5; + std::string det_db_score_mode_ = "slow"; + bool use_dilation_ = false; + + private: + bool initialized_ = false; + PostProcessor post_processor_; + bool SingleBatchPostprocessor(const float* out_data, + int n2, + int n3, + const std::array& det_img_info, + std::vector>* boxes_result); +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/det_preprocessor.cc b/fastdeploy/vision/ocr/ppocr/det_preprocessor.cc new file mode 100644 index 00000000000..89a8d6d39ca --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/det_preprocessor.cc @@ -0,0 +1,113 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/det_preprocessor.h" +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" +#include "fastdeploy/function/concat.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +DBDetectorPreprocessor::DBDetectorPreprocessor() { + initialized_ = true; +} + +std::array OcrDetectorGetInfo(FDMat* img, int max_size_len) { + int w = img->Width(); + int h = img->Height(); + + float ratio = 1.f; + int max_wh = w >= h ? w : h; + if (max_wh > max_size_len) { + if (h > w) { + ratio = float(max_size_len) / float(h); + } else { + ratio = float(max_size_len) / float(w); + } + } + int resize_h = int(float(h) * ratio); + int resize_w = int(float(w) * ratio); + resize_h = std::max(int(std::round(float(resize_h) / 32) * 32), 32); + resize_w = std::max(int(std::round(float(resize_w) / 32) * 32), 32); + + return {w,h,resize_w,resize_h}; + /* + *ratio_h = float(resize_h) / float(h); + *ratio_w = float(resize_w) / float(w); + */ +} +bool OcrDetectorResizeImage(FDMat* img, + int resize_w, + int resize_h, + int max_resize_w, + int max_resize_h) { + Resize::Run(img, resize_w, resize_h); + std::vector value = {0, 0, 0}; + Pad::Run(img, 0, max_resize_h-resize_h, 0, max_resize_w - resize_w, value); + return true; +} + +bool DBDetectorPreprocessor::Run(std::vector* images, + std::vector* outputs, + std::vector>* batch_det_img_info_ptr) { + if (!initialized_) { + FDERROR << "The preprocessor is not initialized." << std::endl; + return false; + } + if (images->size() == 0) { + FDERROR << "The size of input images should be greater than 0." << std::endl; + return false; + } + int max_resize_w = 0; + int max_resize_h = 0; + std::vector>& batch_det_img_info = *batch_det_img_info_ptr; + batch_det_img_info.clear(); + batch_det_img_info.resize(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + FDMat* mat = &(images->at(i)); + batch_det_img_info[i] = OcrDetectorGetInfo(mat,max_side_len_); + max_resize_w = std::max(max_resize_w,batch_det_img_info[i][2]); + max_resize_h = std::max(max_resize_h,batch_det_img_info[i][3]); + } + for (size_t i = 0; i < images->size(); ++i) { + FDMat* mat = &(images->at(i)); + OcrDetectorResizeImage(mat, batch_det_img_info[i][2],batch_det_img_info[i][3],max_resize_w,max_resize_h); + NormalizeAndPermute::Run(mat, mean_, scale_, is_scale_); + /* + Normalize::Run(mat, mean_, scale_, is_scale_); + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + */ + } + // Only have 1 output Tensor. + outputs->resize(1); + // Concat all the preprocessed data to a batch tensor + std::vector tensors(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + (*images)[i].ShareWithTensor(&(tensors[i])); + tensors[i].ExpandDim(0); + } + if (tensors.size() == 1) { + (*outputs)[0] = std::move(tensors[0]); + } else { + function::Concat(tensors, &((*outputs)[0]), 0); + } + return true; +} + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/det_preprocessor.h b/fastdeploy/vision/ocr/ppocr/det_preprocessor.h new file mode 100644 index 00000000000..39c48691d76 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/det_preprocessor.h @@ -0,0 +1,54 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace ocr { +/*! @brief Preprocessor object for DBDetector serials model. + */ +class FASTDEPLOY_DECL DBDetectorPreprocessor { + public: + /** \brief Create a preprocessor instance for DBDetector serials model + * + */ + DBDetectorPreprocessor(); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime + * \param[in] batch_det_img_info_ptr The output of preprocess + * \return true if the preprocess successed, otherwise false + */ + bool Run(std::vector* images, + std::vector* outputs, + std::vector>* batch_det_img_info_ptr); + + int max_side_len_ = 960; + std::vector mean_ = {0.485f, 0.456f, 0.406f}; + std::vector scale_ = {0.229f, 0.224f, 0.225f}; + bool is_scale_ = true; + + private: + bool initialized_ = false; +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/ocrmodel_pybind.cc b/fastdeploy/vision/ocr/ppocr/ocrmodel_pybind.cc old mode 100644 new mode 100755 index 2e4e3e85ba2..96c3e177e3b --- a/fastdeploy/vision/ocr/ppocr/ocrmodel_pybind.cc +++ b/fastdeploy/vision/ocr/ppocr/ocrmodel_pybind.cc @@ -16,45 +16,171 @@ namespace fastdeploy { void BindPPOCRModel(pybind11::module& m) { + m.def("sort_boxes", [](std::vector>& boxes) { + vision::ocr::SortBoxes(&boxes); + return boxes; + }); // DBDetector pybind11::class_(m, "DBDetector") .def(pybind11::init()) .def(pybind11::init<>()) + .def_readwrite("preprocessor", &vision::ocr::DBDetector::preprocessor_) + .def_readwrite("postprocessor", &vision::ocr::DBDetector::postprocessor_); - .def_readwrite("max_side_len", &vision::ocr::DBDetector::max_side_len) - .def_readwrite("det_db_thresh", &vision::ocr::DBDetector::det_db_thresh) - .def_readwrite("det_db_box_thresh", - &vision::ocr::DBDetector::det_db_box_thresh) - .def_readwrite("det_db_unclip_ratio", - &vision::ocr::DBDetector::det_db_unclip_ratio) - .def_readwrite("det_db_score_mode", - &vision::ocr::DBDetector::det_db_score_mode) - .def_readwrite("use_dilation", &vision::ocr::DBDetector::use_dilation) - .def_readwrite("mean", &vision::ocr::DBDetector::mean) - .def_readwrite("scale", &vision::ocr::DBDetector::scale) - .def_readwrite("is_scale", &vision::ocr::DBDetector::is_scale); + pybind11::class_(m, "DBDetectorPreprocessor") + .def(pybind11::init<>()) + .def_readwrite("max_side_len", &vision::ocr::DBDetectorPreprocessor::max_side_len_) + .def_readwrite("mean", &vision::ocr::DBDetectorPreprocessor::mean_) + .def_readwrite("scale", &vision::ocr::DBDetectorPreprocessor::scale_) + .def_readwrite("is_scale", &vision::ocr::DBDetectorPreprocessor::is_scale_) + .def("run", [](vision::ocr::DBDetectorPreprocessor& self, std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + std::vector> batch_det_img_info; + self.Run(&images, &outputs, &batch_det_img_info); + for(size_t i = 0; i< outputs.size(); ++i){ + outputs[i].StopSharing(); + } + return make_pair(outputs, batch_det_img_info); + }); + + pybind11::class_(m, "DBDetectorPostprocessor") + .def(pybind11::init<>()) + .def_readwrite("det_db_thresh", &vision::ocr::DBDetectorPostprocessor::det_db_thresh_) + .def_readwrite("det_db_box_thresh", &vision::ocr::DBDetectorPostprocessor::det_db_box_thresh_) + .def_readwrite("det_db_unclip_ratio", &vision::ocr::DBDetectorPostprocessor::det_db_unclip_ratio_) + .def_readwrite("det_db_score_mode", &vision::ocr::DBDetectorPostprocessor::det_db_score_mode_) + .def_readwrite("use_dilation", &vision::ocr::DBDetectorPostprocessor::use_dilation_) + .def("run", [](vision::ocr::DBDetectorPostprocessor& self, + std::vector& inputs, + const std::vector>& batch_det_img_info) { + std::vector>> results; + + if (!self.Run(inputs, &results, batch_det_img_info)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in DBDetectorPostprocessor.')"); + } + return results; + }) + .def("run", [](vision::ocr::DBDetectorPostprocessor& self, + std::vector& input_array, + const std::vector>& batch_det_img_info) { + std::vector>> results; + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + if (!self.Run(inputs, &results, batch_det_img_info)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in DBDetectorPostprocessor.')"); + } + return results; + }); // Classifier pybind11::class_(m, "Classifier") .def(pybind11::init()) .def(pybind11::init<>()) + .def_readwrite("preprocessor", &vision::ocr::Classifier::preprocessor_) + .def_readwrite("postprocessor", &vision::ocr::Classifier::postprocessor_); + + pybind11::class_(m, "ClassifierPreprocessor") + .def(pybind11::init<>()) + .def_readwrite("cls_image_shape", &vision::ocr::ClassifierPreprocessor::cls_image_shape_) + .def_readwrite("mean", &vision::ocr::ClassifierPreprocessor::mean_) + .def_readwrite("scale", &vision::ocr::ClassifierPreprocessor::scale_) + .def_readwrite("is_scale", &vision::ocr::ClassifierPreprocessor::is_scale_) + .def("run", [](vision::ocr::ClassifierPreprocessor& self, std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + if (!self.Run(&images, &outputs)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in ClassifierPreprocessor.')"); + } + for(size_t i = 0; i< outputs.size(); ++i){ + outputs[i].StopSharing(); + } + return outputs; + }); + + pybind11::class_(m, "ClassifierPostprocessor") + .def(pybind11::init<>()) + .def_readwrite("cls_thresh", &vision::ocr::ClassifierPostprocessor::cls_thresh_) + .def("run", [](vision::ocr::ClassifierPostprocessor& self, + std::vector& inputs) { + std::vector cls_labels; + std::vector cls_scores; + if (!self.Run(inputs, &cls_labels, &cls_scores)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in ClassifierPostprocessor.')"); + } + return make_pair(cls_labels,cls_scores); + }) + .def("run", [](vision::ocr::ClassifierPostprocessor& self, + std::vector& input_array) { + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + std::vector cls_labels; + std::vector cls_scores; + if (!self.Run(inputs, &cls_labels, &cls_scores)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in ClassifierPostprocessor.')"); + } + return make_pair(cls_labels,cls_scores); + }); - .def_readwrite("cls_thresh", &vision::ocr::Classifier::cls_thresh) - .def_readwrite("cls_image_shape", - &vision::ocr::Classifier::cls_image_shape) - .def_readwrite("cls_batch_num", &vision::ocr::Classifier::cls_batch_num); // Recognizer pybind11::class_(m, "Recognizer") - .def(pybind11::init()) .def(pybind11::init<>()) + .def_readwrite("preprocessor", &vision::ocr::Recognizer::preprocessor_) + .def_readwrite("postprocessor", &vision::ocr::Recognizer::postprocessor_); + + pybind11::class_(m, "RecognizerPreprocessor") + .def(pybind11::init<>()) + .def_readwrite("rec_image_shape", &vision::ocr::RecognizerPreprocessor::rec_image_shape_) + .def_readwrite("mean", &vision::ocr::RecognizerPreprocessor::mean_) + .def_readwrite("scale", &vision::ocr::RecognizerPreprocessor::scale_) + .def_readwrite("is_scale", &vision::ocr::RecognizerPreprocessor::is_scale_) + .def("run", [](vision::ocr::RecognizerPreprocessor& self, std::vector& im_list) { + std::vector images; + for (size_t i = 0; i < im_list.size(); ++i) { + images.push_back(vision::WrapMat(PyArrayToCvMat(im_list[i]))); + } + std::vector outputs; + if (!self.Run(&images, &outputs)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in RecognizerPreprocessor.')"); + } + for(size_t i = 0; i< outputs.size(); ++i){ + outputs[i].StopSharing(); + } + return outputs; + }); - .def_readwrite("rec_img_h", &vision::ocr::Recognizer::rec_img_h) - .def_readwrite("rec_img_w", &vision::ocr::Recognizer::rec_img_w) - .def_readwrite("rec_batch_num", &vision::ocr::Recognizer::rec_batch_num); + pybind11::class_(m, "RecognizerPostprocessor") + .def(pybind11::init()) + .def("run", [](vision::ocr::RecognizerPostprocessor& self, + std::vector& inputs) { + std::vector texts; + std::vector rec_scores; + if (!self.Run(inputs, &texts, &rec_scores)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in RecognizerPostprocessor.')"); + } + return make_pair(texts, rec_scores); + }) + .def("run", [](vision::ocr::RecognizerPostprocessor& self, + std::vector& input_array) { + std::vector inputs; + PyArrayToTensorList(input_array, &inputs, /*share_buffer=*/true); + std::vector texts; + std::vector rec_scores; + if (!self.Run(inputs, &texts, &rec_scores)) { + pybind11::eval("raise Exception('Failed to preprocess the input data in RecognizerPostprocessor.')"); + } + return make_pair(texts, rec_scores); + }); } } // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_pybind.cc b/fastdeploy/vision/ocr/ppocr/ppocr_pybind.cc old mode 100644 new mode 100755 index a88ae2fc7f1..fcbbe022423 --- a/fastdeploy/vision/ocr/ppocr/ppocr_pybind.cc +++ b/fastdeploy/vision/ocr/ppocr/ppocr_pybind.cc @@ -31,6 +31,15 @@ void BindPPOCRv3(pybind11::module& m) { vision::OCRResult res; self.Predict(&mat, &res); return res; + }) + .def("batch_predict", [](pipeline::PPOCRv3& self, std::vector& data) { + std::vector images; + for (size_t i = 0; i < data.size(); ++i) { + images.push_back(PyArrayToCvMat(data[i])); + } + std::vector results; + self.BatchPredict(images, &results); + return results; }); } @@ -49,6 +58,15 @@ void BindPPOCRv2(pybind11::module& m) { vision::OCRResult res; self.Predict(&mat, &res); return res; + }) + .def("batch_predict", [](pipeline::PPOCRv2& self, std::vector& data) { + std::vector images; + for (size_t i = 0; i < data.size(); ++i) { + images.push_back(PyArrayToCvMat(data[i])); + } + std::vector results; + self.BatchPredict(images, &results); + return results; }); } diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc b/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc old mode 100644 new mode 100755 index 7ad1f105ca9..e6e89299b6a --- a/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v2.cc @@ -22,101 +22,95 @@ PPOCRv2::PPOCRv2(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Classifier* cls_model, fastdeploy::vision::ocr::Recognizer* rec_model) : detector_(det_model), classifier_(cls_model), recognizer_(rec_model) { - recognizer_->rec_image_shape[1] = 32; + Initialized(); + recognizer_->preprocessor_.rec_image_shape_[1] = 32; } PPOCRv2::PPOCRv2(fastdeploy::vision::ocr::DBDetector* det_model, fastdeploy::vision::ocr::Recognizer* rec_model) : detector_(det_model), recognizer_(rec_model) { - recognizer_->rec_image_shape[1] = 32; + Initialized(); + recognizer_->preprocessor_.rec_image_shape_[1] = 32; } bool PPOCRv2::Initialized() const { - if (detector_ != nullptr && !detector_->Initialized()){ + if (detector_ != nullptr && !detector_->Initialized()) { return false; } - if (classifier_ != nullptr && !classifier_->Initialized()){ + if (classifier_ != nullptr && !classifier_->Initialized()) { return false; } - if (recognizer_ != nullptr && !recognizer_->Initialized()){ + if (recognizer_ != nullptr && !recognizer_->Initialized()) { return false; } return true; } -bool PPOCRv2::Detect(cv::Mat* img, - fastdeploy::vision::OCRResult* result) { - if (!detector_->Predict(img, &(result->boxes))) { - FDERROR << "There's error while detecting image in PPOCR." << std::endl; - return false; - } - vision::ocr::SortBoxes(result); - return true; -} - -bool PPOCRv2::Recognize(cv::Mat* img, - fastdeploy::vision::OCRResult* result) { - std::tuple rec_result; - if (!recognizer_->Predict(img, &rec_result)) { - FDERROR << "There's error while recognizing image in PPOCR." << std::endl; - return false; - } - - result->text.push_back(std::get<0>(rec_result)); - result->rec_scores.push_back(std::get<1>(rec_result)); +bool PPOCRv2::Predict(cv::Mat* img, + fastdeploy::vision::OCRResult* result) { + std::vector batch_result(1); + BatchPredict({*img},&batch_result); + *result = std::move(batch_result[0]); return true; -} +}; -bool PPOCRv2::Classify(cv::Mat* img, - fastdeploy::vision::OCRResult* result) { - std::tuple cls_result; +bool PPOCRv2::BatchPredict(const std::vector& images, + std::vector* batch_result) { + batch_result->clear(); + batch_result->resize(images.size()); + std::vector>> batch_boxes(images.size()); - if (!classifier_->Predict(img, &cls_result)) { - FDERROR << "There's error while classifying image in PPOCR." << std::endl; + if (!detector_->BatchPredict(images, &batch_boxes)) { + FDERROR << "There's error while detecting image in PPOCR." << std::endl; return false; } - - result->cls_labels.push_back(std::get<0>(cls_result)); - result->cls_scores.push_back(std::get<1>(cls_result)); - return true; -} - -bool PPOCRv2::Predict(cv::Mat* img, - fastdeploy::vision::OCRResult* result) { - result->Clear(); - if (nullptr != detector_ && !Detect(img, result)) { - FDERROR << "Failed to detect image." << std::endl; - return false; + for(int i_batch = 0; i_batch < batch_boxes.size(); ++i_batch) { + vision::ocr::SortBoxes(&(batch_boxes[i_batch])); + (*batch_result)[i_batch].boxes = batch_boxes[i_batch]; } + + + for(int i_batch = 0; i_batch < images.size(); ++i_batch) { + fastdeploy::vision::OCRResult& ocr_result = (*batch_result)[i_batch]; + // Get croped images by detection result + const std::vector>& boxes = ocr_result.boxes; + const cv::Mat& img = images[i_batch]; + std::vector image_list; + if (boxes.size() == 0) { + image_list.emplace_back(img); + }else{ + image_list.resize(boxes.size()); + for (size_t i_box = 0; i_box < boxes.size(); ++i_box) { + image_list[i_box] = vision::ocr::GetRotateCropImage(img, boxes[i_box]); + } + } + std::vector* cls_labels_ptr = &ocr_result.cls_labels; + std::vector* cls_scores_ptr = &ocr_result.cls_scores; - // Get croped images by detection result - std::vector image_list; - for (size_t i = 0; i < result->boxes.size(); ++i) { - auto crop_im = vision::ocr::GetRotateCropImage(*img, (result->boxes)[i]); - image_list.push_back(crop_im); - } - if (result->boxes.size() == 0) { - image_list.push_back(*img); - } + std::vector* text_ptr = &ocr_result.text; + std::vector* rec_scores_ptr = &ocr_result.rec_scores; - for (size_t i = 0; i < image_list.size(); ++i) { - if (nullptr != classifier_ && !Classify(&(image_list[i]), result)) { - FDERROR << "Failed to classify croped image of index " << i << "." << std::endl; + if (!classifier_->BatchPredict(image_list, cls_labels_ptr, cls_scores_ptr)) { + FDERROR << "There's error while recognizing image in PPOCR." << std::endl; return false; + }else{ + for (size_t i_img = 0; i_img < image_list.size(); ++i_img) { + if(cls_labels_ptr->at(i_img) % 2 == 1 && cls_scores_ptr->at(i_img) > classifier_->postprocessor_.cls_thresh_) { + cv::rotate(image_list[i_img], image_list[i_img], 1); + } + } } - if (nullptr != classifier_ && result->cls_labels[i] % 2 == 1 && result->cls_scores[i] > classifier_->cls_thresh) { - cv::rotate(image_list[i], image_list[i], 1); - } - if (nullptr != recognizer_ && !Recognize(&(image_list[i]), result)) { - FDERROR << "Failed to recgnize croped image of index " << i << "." << std::endl; + + if (!recognizer_->BatchPredict(image_list, text_ptr, rec_scores_ptr)) { + FDERROR << "There's error while recognizing image in PPOCR." << std::endl; return false; } } return true; -}; +} } // namesapce pipeline } // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v2.h b/fastdeploy/vision/ocr/ppocr/ppocr_v2.h old mode 100644 new mode 100755 index bf53000202c..d021d6c32d8 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v2.h +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v2.h @@ -59,6 +59,14 @@ class FASTDEPLOY_DECL PPOCRv2 : public FastDeployModel { * \return true if the prediction successed, otherwise false. */ virtual bool Predict(cv::Mat* img, fastdeploy::vision::OCRResult* result); + /** \brief BatchPredict the input image and get OCR result. + * + * \param[in] images The list of input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] batch_result The output list of OCR result will be writen to this structure. + * \return true if the prediction successed, otherwise false. + */ + virtual bool BatchPredict(const std::vector& images, + std::vector* batch_result); bool Initialized() const override; protected: @@ -66,11 +74,6 @@ class FASTDEPLOY_DECL PPOCRv2 : public FastDeployModel { fastdeploy::vision::ocr::Classifier* classifier_ = nullptr; fastdeploy::vision::ocr::Recognizer* recognizer_ = nullptr; /// Launch the detection process in OCR. - virtual bool Detect(cv::Mat* img, fastdeploy::vision::OCRResult* result); - /// Launch the recognition process in OCR. - virtual bool Recognize(cv::Mat* img, fastdeploy::vision::OCRResult* result); - /// Launch the classification process in OCR. - virtual bool Classify(cv::Mat* img, fastdeploy::vision::OCRResult* result); }; namespace application { diff --git a/fastdeploy/vision/ocr/ppocr/ppocr_v3.h b/fastdeploy/vision/ocr/ppocr/ppocr_v3.h old mode 100644 new mode 100755 index e248eca75ed..ed9177d9232 --- a/fastdeploy/vision/ocr/ppocr/ppocr_v3.h +++ b/fastdeploy/vision/ocr/ppocr/ppocr_v3.h @@ -36,7 +36,7 @@ class FASTDEPLOY_DECL PPOCRv3 : public PPOCRv2 { fastdeploy::vision::ocr::Recognizer* rec_model) : PPOCRv2(det_model, cls_model, rec_model) { // The only difference between v2 and v3 - recognizer_->rec_image_shape[1] = 48; + recognizer_->preprocessor_.rec_image_shape_[1] = 48; } /** \brief Classification model is optional, so this function is set up the detection model path and recognition model path respectively. * @@ -47,7 +47,7 @@ class FASTDEPLOY_DECL PPOCRv3 : public PPOCRv2 { fastdeploy::vision::ocr::Recognizer* rec_model) : PPOCRv2(det_model, rec_model) { // The only difference between v2 and v3 - recognizer_->rec_image_shape[1] = 48; + recognizer_->preprocessor_.rec_image_shape_[1] = 48; } }; diff --git a/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc b/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc new file mode 100644 index 00000000000..cdc302e28c6 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/rec_postprocessor.cc @@ -0,0 +1,112 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/rec_postprocessor.h" +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +std::vector ReadDict(const std::string& path) { + std::ifstream in(path); + FDASSERT(in, "Cannot open file %s to read.", path.c_str()); + std::string line; + std::vector m_vec; + while (getline(in, line)) { + m_vec.push_back(line); + } + m_vec.insert(m_vec.begin(), "#"); // blank char for ctc + m_vec.push_back(" "); + return m_vec; +} + +RecognizerPostprocessor::RecognizerPostprocessor(){ + initialized_ = true; +} + +RecognizerPostprocessor::RecognizerPostprocessor(const std::string& label_path) { + // init label_lsit + label_list_ = ReadDict(label_path); + initialized_ = true; +} + +bool RecognizerPostprocessor::SingleBatchPostprocessor(const float* out_data, + const std::vector& output_shape, + std::string* text, float* rec_score) { + std::string& str_res = *text; + float& score = *rec_score; + score = 0.f; + int argmax_idx; + int last_index = 0; + int count = 0; + float max_value = 0.0f; + + for (int n = 0; n < output_shape[1]; n++) { + argmax_idx = int( + std::distance(&out_data[n * output_shape[2]], + std::max_element(&out_data[n * output_shape[2]], + &out_data[(n + 1) * output_shape[2]]))); + + max_value = float(*std::max_element(&out_data[n * output_shape[2]], + &out_data[(n + 1) * output_shape[2]])); + + if (argmax_idx > 0 && (!(n > 0 && argmax_idx == last_index))) { + score += max_value; + count += 1; + if(argmax_idx > label_list_.size()) { + FDERROR << "The output index: " << argmax_idx << " is larger than the size of label_list: " + << label_list_.size() << ". Please check the label file!" << std::endl; + return false; + } + str_res += label_list_[argmax_idx]; + } + last_index = argmax_idx; + } + score /= (count + 1e-6); + if (count == 0 || std::isnan(score)) { + score = 0.f; + } + return true; +} + +bool RecognizerPostprocessor::Run(const std::vector& tensors, + std::vector* texts, std::vector* rec_scores) { + if (!initialized_) { + FDERROR << "Postprocessor is not initialized." << std::endl; + return false; + } + // Recognizer have only 1 output tensor. + const FDTensor& tensor = tensors[0]; + // For Recognizer, the output tensor shape = [batch, ?, 6625] + size_t batch = tensor.shape[0]; + size_t length = accumulate(tensor.shape.begin()+1, tensor.shape.end(), 1, std::multiplies()); + + texts->resize(batch); + rec_scores->resize(batch); + const float* tensor_data = reinterpret_cast(tensor.Data()); + for (int i_batch = 0; i_batch < batch; ++i_batch) { + if(!SingleBatchPostprocessor(tensor_data, tensor.shape, &texts->at(i_batch), &rec_scores->at(i_batch))) { + return false; + } + tensor_data = tensor_data + length; + } + + return true; +} + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/rec_postprocessor.h b/fastdeploy/vision/ocr/ppocr/rec_postprocessor.h new file mode 100644 index 00000000000..d1aa0124b42 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/rec_postprocessor.h @@ -0,0 +1,55 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" + +namespace fastdeploy { +namespace vision { + +namespace ocr { +/*! @brief Postprocessor object for Recognizer serials model. + */ +class FASTDEPLOY_DECL RecognizerPostprocessor { + public: + RecognizerPostprocessor(); + /** \brief Create a postprocessor instance for Recognizer serials model + * + * \param[in] label_path The path of label_dict + */ + explicit RecognizerPostprocessor(const std::string& label_path); + + /** \brief Process the result of runtime and fill to ClassifyResult structure + * + * \param[in] tensors The inference result from runtime + * \param[in] texts The output result of recognizer + * \param[in] rec_scores The output result of recognizer + * \return true if the postprocess successed, otherwise false + */ + bool Run(const std::vector& tensors, + std::vector* texts, std::vector* rec_scores); + + private: + bool SingleBatchPostprocessor(const float* out_data, + const std::vector& output_shape, + std::string* text, float* rec_score); + bool initialized_ = false; + std::vector label_list_; +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/rec_preprocessor.cc b/fastdeploy/vision/ocr/ppocr/rec_preprocessor.cc new file mode 100644 index 00000000000..858578d6901 --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/rec_preprocessor.cc @@ -0,0 +1,99 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/ocr/ppocr/rec_preprocessor.h" +#include "fastdeploy/utils/perf.h" +#include "fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h" +#include "fastdeploy/function/concat.h" + +namespace fastdeploy { +namespace vision { +namespace ocr { + +RecognizerPreprocessor::RecognizerPreprocessor() { + initialized_ = true; +} + +void OcrRecognizerResizeImage(FDMat* mat, float max_wh_ratio, + const std::vector& rec_image_shape) { + int imgC, imgH, imgW; + imgC = rec_image_shape[0]; + imgH = rec_image_shape[1]; + imgW = rec_image_shape[2]; + + imgW = int(imgH * max_wh_ratio); + + float ratio = float(mat->Width()) / float(mat->Height()); + int resize_w; + if (ceilf(imgH * ratio) > imgW) { + resize_w = imgW; + }else{ + resize_w = int(ceilf(imgH * ratio)); + } + Resize::Run(mat, resize_w, imgH); + + std::vector value = {0, 0, 0}; + Pad::Run(mat, 0, 0, 0, int(imgW - mat->Width()), value); +} + +bool RecognizerPreprocessor::Run(std::vector* images, std::vector* outputs) { + if (!initialized_) { + FDERROR << "The preprocessor is not initialized." << std::endl; + return false; + } + if (images->size() == 0) { + FDERROR << "The size of input images should be greater than 0." << std::endl; + return false; + } + + int imgH = rec_image_shape_[1]; + int imgW = rec_image_shape_[2]; + float max_wh_ratio = imgW * 1.0 / imgH; + float ori_wh_ratio; + + for (size_t i = 0; i < images->size(); ++i) { + FDMat* mat = &(images->at(i)); + ori_wh_ratio = mat->Width() * 1.0 / mat->Height(); + max_wh_ratio = std::max(max_wh_ratio, ori_wh_ratio); + } + + for (size_t i = 0; i < images->size(); ++i) { + FDMat* mat = &(images->at(i)); + OcrRecognizerResizeImage(mat, max_wh_ratio, rec_image_shape_); + NormalizeAndPermute::Run(mat, mean_, scale_, is_scale_); + /* + Normalize::Run(mat, mean_, scale_, is_scale_); + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + */ + } + // Only have 1 output Tensor. + outputs->resize(1); + // Concat all the preprocessed data to a batch tensor + std::vector tensors(images->size()); + for (size_t i = 0; i < images->size(); ++i) { + (*images)[i].ShareWithTensor(&(tensors[i])); + tensors[i].ExpandDim(0); + } + if (tensors.size() == 1) { + (*outputs)[0] = std::move(tensors[0]); + } else { + function::Concat(tensors, &((*outputs)[0]), 0); + } + return true; +} + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/rec_preprocessor.h b/fastdeploy/vision/ocr/ppocr/rec_preprocessor.h new file mode 100644 index 00000000000..3e5c7de824c --- /dev/null +++ b/fastdeploy/vision/ocr/ppocr/rec_preprocessor.h @@ -0,0 +1,52 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/vision/common/result.h" + +namespace fastdeploy { +namespace vision { + +namespace ocr { +/*! @brief Preprocessor object for PaddleClas serials model. + */ +class FASTDEPLOY_DECL RecognizerPreprocessor { + public: + /** \brief Create a preprocessor instance for PaddleClas serials model + * + * \param[in] config_file Path of configuration file for deployment, e.g resnet/infer_cfg.yml + */ + RecognizerPreprocessor(); + + /** \brief Process the input image and prepare input tensors for runtime + * + * \param[in] images The input image data list, all the elements are returned by cv::imread() + * \param[in] outputs The output tensors which will feed in runtime + * \return true if the preprocess successed, otherwise false + */ + bool Run(std::vector* images, std::vector* outputs); + + std::vector rec_image_shape_ = {3, 48, 320}; + std::vector mean_ = {0.5f, 0.5f, 0.5f}; + std::vector scale_ = {0.5f, 0.5f, 0.5f}; + bool is_scale_ = true; + + private: + bool initialized_ = false; +}; + +} // namespace ocr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/ocr/ppocr/recognizer.cc b/fastdeploy/vision/ocr/ppocr/recognizer.cc old mode 100644 new mode 100755 index f0564ce3394..59d64a6c45b --- a/fastdeploy/vision/ocr/ppocr/recognizer.cc +++ b/fastdeploy/vision/ocr/ppocr/recognizer.cc @@ -20,29 +20,13 @@ namespace fastdeploy { namespace vision { namespace ocr { -std::vector ReadDict(const std::string& path) { - std::ifstream in(path); - std::string line; - std::vector m_vec; - if (in) { - while (getline(in, line)) { - m_vec.push_back(line); - } - } else { - std::cout << "no such label file: " << path << ", exit the program..." - << std::endl; - exit(1); - } - return m_vec; -} - Recognizer::Recognizer() {} Recognizer::Recognizer(const std::string& model_file, const std::string& params_file, const std::string& label_path, const RuntimeOption& custom_option, - const ModelFormat& model_format) { + const ModelFormat& model_format):postprocessor_(label_path) { if (model_format == ModelFormat::ONNX) { valid_cpu_backends = {Backend::ORT, Backend::OPENVINO}; @@ -56,27 +40,11 @@ Recognizer::Recognizer(const std::string& model_file, runtime_option.model_format = model_format; runtime_option.model_file = model_file; runtime_option.params_file = params_file; - initialized = Initialize(); - - // init label_lsit - label_list = ReadDict(label_path); - label_list.insert(label_list.begin(), "#"); // blank char for ctc - label_list.push_back(" "); } // Init bool Recognizer::Initialize() { - // pre&post process parameters - rec_batch_num = 1; - rec_img_h = 48; - rec_img_w = 320; - rec_image_shape = {3, rec_img_h, rec_img_w}; - - mean = {0.5f, 0.5f, 0.5f}; - scale = {0.5f, 0.5f, 0.5f}; - is_scale = true; - if (!InitRuntime()) { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; return false; @@ -85,119 +53,23 @@ bool Recognizer::Initialize() { return true; } -void OcrRecognizerResizeImage(Mat* mat, const float& wh_ratio, - const std::vector& rec_image_shape) { - int imgC, imgH, imgW; - imgC = rec_image_shape[0]; - imgH = rec_image_shape[1]; - imgW = rec_image_shape[2]; - - imgW = int(imgH * wh_ratio); - - float ratio = float(mat->Width()) / float(mat->Height()); - int resize_w; - if (ceilf(imgH * ratio) > imgW) - resize_w = imgW; - else - resize_w = int(ceilf(imgH * ratio)); - - Resize::Run(mat, resize_w, imgH); - - std::vector value = {127, 127, 127}; - Pad::Run(mat, 0, 0, 0, int(imgW - mat->Width()), value); -} - -bool Recognizer::Preprocess(Mat* mat, FDTensor* output, - const std::vector& rec_image_shape) { - int imgH = rec_image_shape[1]; - int imgW = rec_image_shape[2]; - float wh_ratio = imgW * 1.0 / imgH; - - float ori_wh_ratio = mat->Width() * 1.0 / mat->Height(); - wh_ratio = std::max(wh_ratio, ori_wh_ratio); - - OcrRecognizerResizeImage(mat, wh_ratio, rec_image_shape); - - Normalize::Run(mat, mean, scale, true); - - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); - - mat->ShareWithTensor(output); - output->shape.insert(output->shape.begin(), 1); - - return true; -} - -bool Recognizer::Postprocess(FDTensor& infer_result, - std::tuple* rec_result) { - std::vector output_shape = infer_result.shape; - FDASSERT(output_shape[0] == 1, "Only support batch =1 now."); - - float* out_data = static_cast(infer_result.Data()); - - std::string str_res; - int argmax_idx; - int last_index = 0; - float score = 0.f; - int count = 0; - float max_value = 0.0f; - - for (int n = 0; n < output_shape[1]; n++) { - argmax_idx = int( - std::distance(&out_data[n * output_shape[2]], - std::max_element(&out_data[n * output_shape[2]], - &out_data[(n + 1) * output_shape[2]]))); - - max_value = float(*std::max_element(&out_data[n * output_shape[2]], - &out_data[(n + 1) * output_shape[2]])); - - if (argmax_idx > 0 && (!(n > 0 && argmax_idx == last_index))) { - score += max_value; - count += 1; - if(argmax_idx > label_list.size()){ - FDERROR << "The output index: " << argmax_idx << " is larger than the size of label_list: " - << label_list.size() << ". Please check the label file!" << std::endl; - return false; - } - str_res += label_list[argmax_idx]; - } - last_index = argmax_idx; - } - score /= (count + 1e-6); - if (count == 0 || std::isnan(score)) { - score = 0.f; - } - std::get<0>(*rec_result) = str_res; - std::get<1>(*rec_result) = score; - - return true; -} - -bool Recognizer::Predict(cv::Mat* img, - std::tuple* rec_result) { - Mat mat(*img); - - std::vector input_tensors(1); - - if (!Preprocess(&mat, &input_tensors[0], rec_image_shape)) { - FDERROR << "Failed to preprocess input image." << std::endl; +bool Recognizer::BatchPredict(const std::vector& images, + std::vector* texts, std::vector* rec_scores) { + std::vector fd_images = WrapMat(images); + if (!preprocessor_.Run(&fd_images, &reused_input_tensors_)) { + FDERROR << "Failed to preprocess the input image." << std::endl; return false; } - - input_tensors[0].name = InputInfoOfRuntime(0).name; - std::vector output_tensors; - - if (!Infer(input_tensors, &output_tensors)) { - FDERROR << "Failed to inference." << std::endl; + reused_input_tensors_[0].name = InputInfoOfRuntime(0).name; + if (!Infer(reused_input_tensors_, &reused_output_tensors_)) { + FDERROR << "Failed to inference by runtime." << std::endl; return false; } - if (!Postprocess(output_tensors[0], rec_result)) { - FDERROR << "Failed to post process." << std::endl; + if (!postprocessor_.Run(reused_output_tensors_, texts, rec_scores)) { + FDERROR << "Failed to postprocess the inference cls_results by runtime." << std::endl; return false; } - return true; } diff --git a/fastdeploy/vision/ocr/ppocr/recognizer.h b/fastdeploy/vision/ocr/ppocr/recognizer.h old mode 100644 new mode 100755 index d3c5fcc9d40..1cd841eb453 --- a/fastdeploy/vision/ocr/ppocr/recognizer.h +++ b/fastdeploy/vision/ocr/ppocr/recognizer.h @@ -17,6 +17,8 @@ #include "fastdeploy/vision/common/processors/transform.h" #include "fastdeploy/vision/common/result.h" #include "fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h" +#include "fastdeploy/vision/ocr/ppocr/rec_preprocessor.h" +#include "fastdeploy/vision/ocr/ppocr/rec_postprocessor.h" namespace fastdeploy { namespace vision { @@ -43,35 +45,20 @@ class FASTDEPLOY_DECL Recognizer : public FastDeployModel { const ModelFormat& model_format = ModelFormat::PADDLE); /// Get model's name std::string ModelName() const { return "ppocr/ocr_rec"; } - /** \brief Predict the input image and get OCR recognition model result. + /** \brief BatchPredict the input image and get OCR recognition model result. * - * \param[in] im The input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. - * \param[in] rec_result The output of OCR recognition model result will be writen to this structure. + * \param[in] images The list of input image data, comes from cv::imread(), is a 3-D array with layout HWC, BGR format. + * \param[in] rec_results The output of OCR recognition model result will be writen to this structure. * \return true if the prediction is successed, otherwise false. */ - virtual bool Predict(cv::Mat* img, - std::tuple* rec_result); + virtual bool BatchPredict(const std::vector& images, + std::vector* texts, std::vector* rec_scores); - // Pre & Post parameters - std::vector label_list; - int rec_batch_num; - int rec_img_h; - int rec_img_w; - std::vector rec_image_shape; - - std::vector mean; - std::vector scale; - bool is_scale; + RecognizerPreprocessor preprocessor_; + RecognizerPostprocessor postprocessor_; private: bool Initialize(); - /// Preprocess the input data, and set the preprocessed results to `outputs` - bool Preprocess(Mat* img, FDTensor* outputs, - const std::vector& rec_image_shape); - /*! @brief Postprocess the inferenced results, and set the final result to `rec_result` - */ - bool Postprocess(FDTensor& infer_result, - std::tuple* rec_result); }; } // namespace ocr diff --git a/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.cc b/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.cc old mode 100644 new mode 100755 index 02e435f769d..7a8f387e233 --- a/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.cc +++ b/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.cc @@ -318,10 +318,12 @@ std::vector>> PostProcessor::BoxesFromBitmap( } std::vector>> PostProcessor::FilterTagDetRes( - std::vector>> boxes, float ratio_h, - float ratio_w, const std::map> &im_info) { - int oriimg_h = im_info.at("input_shape")[0]; - int oriimg_w = im_info.at("input_shape")[1]; + std::vector>> boxes, + const std::array& det_img_info) { + int oriimg_w = det_img_info[0]; + int oriimg_h = det_img_info[1]; + float ratio_w = float(det_img_info[2])/float(oriimg_w); + float ratio_h = float(det_img_info[3])/float(oriimg_h); std::vector>> root_points; for (int n = 0; n < boxes.size(); n++) { diff --git a/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h b/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h index 8422704fd5b..5900daea2b1 100644 --- a/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h +++ b/fastdeploy/vision/ocr/ppocr/utils/ocr_postprocess_op.h @@ -57,9 +57,8 @@ class PostProcessor { const float &det_db_unclip_ratio, const std::string &det_db_score_mode); std::vector>> FilterTagDetRes( - std::vector>> boxes, float ratio_h, - float ratio_w, - const std::map> &im_info); + std::vector>> boxes, + const std::array& det_img_info); private: static bool XsortInt(std::vector a, std::vector b); diff --git a/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h b/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h index 36994389ba9..0e5c040eb82 100644 --- a/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h +++ b/fastdeploy/vision/ocr/ppocr/utils/ocr_utils.h @@ -28,10 +28,10 @@ namespace fastdeploy { namespace vision { namespace ocr { -cv::Mat GetRotateCropImage(const cv::Mat& srcimage, +FASTDEPLOY_DECL cv::Mat GetRotateCropImage(const cv::Mat& srcimage, const std::array& box); -void SortBoxes(OCRResult* result); +FASTDEPLOY_DECL void SortBoxes(std::vector>* boxes); } // namespace ocr } // namespace vision diff --git a/fastdeploy/vision/ocr/ppocr/utils/sorted_boxes.cc b/fastdeploy/vision/ocr/ppocr/utils/sorted_boxes.cc index a6b723d947e..4705d9a5afc 100644 --- a/fastdeploy/vision/ocr/ppocr/utils/sorted_boxes.cc +++ b/fastdeploy/vision/ocr/ppocr/utils/sorted_boxes.cc @@ -29,17 +29,17 @@ bool CompareBox(const std::array& result1, } } -void SortBoxes(OCRResult* result) { - std::sort(result->boxes.begin(), result->boxes.end(), CompareBox); +void SortBoxes(std::vector>* boxes) { + std::sort(boxes->begin(), boxes->end(), CompareBox); - if (result->boxes.size() == 0) { + if (boxes->size() == 0) { return; } - for (int i = 0; i < result->boxes.size() - 1; i++) { - if (abs(result->boxes[i + 1][1] - result->boxes[i][1]) < 10 && - (result->boxes[i + 1][0] < result->boxes[i][0])) { - std::swap(result->boxes[i], result->boxes[i + 1]); + for (int i = 0; i < boxes->size() - 1; i++) { + if (abs((*boxes)[i + 1][1] - (*boxes)[i][1]) < 10 && + ((*boxes)[i + 1][0] < (*boxes)[i][0])) { + std::swap((*boxes)[i], (*boxes)[i + 1]); } } } diff --git a/python/fastdeploy/vision/ocr/__init__.py b/python/fastdeploy/vision/ocr/__init__.py old mode 100644 new mode 100755 index 98e210d3b0d..c83c6d678c6 --- a/python/fastdeploy/vision/ocr/__init__.py +++ b/python/fastdeploy/vision/ocr/__init__.py @@ -13,10 +13,4 @@ # limitations under the License. from __future__ import absolute_import -from .ppocr import PPOCRv3 -from .ppocr import PPOCRv2 -from .ppocr import PPOCRSystemv3 -from .ppocr import PPOCRSystemv2 -from .ppocr import DBDetector -from .ppocr import Classifier -from .ppocr import Recognizer +from .ppocr import * diff --git a/python/fastdeploy/vision/ocr/ppocr/__init__.py b/python/fastdeploy/vision/ocr/ppocr/__init__.py old mode 100644 new mode 100755 index e361a3a8acd..b8f5c81d11d --- a/python/fastdeploy/vision/ocr/ppocr/__init__.py +++ b/python/fastdeploy/vision/ocr/ppocr/__init__.py @@ -41,40 +41,11 @@ def __init__(self, assert self.initialized, "DBDetector initialize failed." # 一些跟DBDetector模型有关的属性封装 - @property - def max_side_len(self): - return self._model.max_side_len - + ''' @property def det_db_thresh(self): return self._model.det_db_thresh - @property - def det_db_box_thresh(self): - return self._model.det_db_box_thresh - - @property - def det_db_unclip_ratio(self): - return self._model.det_db_unclip_ratio - - @property - def det_db_score_mode(self): - return self._model.det_db_score_mode - - @property - def use_dilation(self): - return self._model.use_dilation - - @property - def is_scale(self): - return self._model.max_wh - - @max_side_len.setter - def max_side_len(self, value): - assert isinstance( - value, int), "The value to set `max_side_len` must be type of int." - self._model.max_side_len = value - @det_db_thresh.setter def det_db_thresh(self, value): assert isinstance( @@ -82,6 +53,10 @@ def det_db_thresh(self, value): float), "The value to set `det_db_thresh` must be type of float." self._model.det_db_thresh = value + @property + def det_db_box_thresh(self): + return self._model.det_db_box_thresh + @det_db_box_thresh.setter def det_db_box_thresh(self, value): assert isinstance( @@ -89,6 +64,10 @@ def det_db_box_thresh(self, value): ), "The value to set `det_db_box_thresh` must be type of float." self._model.det_db_box_thresh = value + @property + def det_db_unclip_ratio(self): + return self._model.det_db_unclip_ratio + @det_db_unclip_ratio.setter def det_db_unclip_ratio(self, value): assert isinstance( @@ -96,6 +75,10 @@ def det_db_unclip_ratio(self, value): ), "The value to set `det_db_unclip_ratio` must be type of float." self._model.det_db_unclip_ratio = value + @property + def det_db_score_mode(self): + return self._model.det_db_score_mode + @det_db_score_mode.setter def det_db_score_mode(self, value): assert isinstance( @@ -103,6 +86,10 @@ def det_db_score_mode(self, value): str), "The value to set `det_db_score_mode` must be type of str." self._model.det_db_score_mode = value + @property + def use_dilation(self): + return self._model.use_dilation + @use_dilation.setter def use_dilation(self, value): assert isinstance( @@ -110,11 +97,26 @@ def use_dilation(self, value): bool), "The value to set `use_dilation` must be type of bool." self._model.use_dilation = value + @property + def max_side_len(self): + return self._model.max_side_len + + @max_side_len.setter + def max_side_len(self, value): + assert isinstance( + value, int), "The value to set `max_side_len` must be type of int." + self._model.max_side_len = value + + @property + def is_scale(self): + return self._model.max_wh + @is_scale.setter def is_scale(self, value): assert isinstance( value, bool), "The value to set `is_scale` must be type of bool." self._model.is_scale = value + ''' class Classifier(FastDeployModel): @@ -139,6 +141,7 @@ def __init__(self, model_file, params_file, self._runtime_option, model_format) assert self.initialized, "Classifier initialize failed." + ''' @property def cls_thresh(self): return self._model.cls_thresh @@ -170,6 +173,7 @@ def cls_batch_num(self, value): value, int), "The value to set `cls_batch_num` must be type of int." self._model.cls_batch_num = value + ''' class Recognizer(FastDeployModel): @@ -197,6 +201,7 @@ def __init__(self, model_format) assert self.initialized, "Recognizer initialize failed." + ''' @property def rec_img_h(self): return self._model.rec_img_h @@ -227,6 +232,7 @@ def rec_batch_num(self, value): value, int), "The value to set `rec_batch_num` must be type of int." self._model.rec_batch_num = value + ''' class PPOCRv3(FastDeployModel): @@ -253,6 +259,14 @@ def predict(self, input_image): """ return self.system.predict(input_image) + def batch_predict(self, images): + """Predict a batch of input image + :param images: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return: OCRBatchResult + """ + + return self.system.batch_predict(images) + class PPOCRSystemv3(PPOCRv3): def __init__(self, det_model=None, cls_model=None, rec_model=None): @@ -289,6 +303,14 @@ def predict(self, input_image): """ return self.system.predict(input_image) + def batch_predict(self, images): + """Predict a batch of input image + :param images: (list of numpy.ndarray) The input image list, each element is a 3-D array with layout HWC, BGR format + :return: OCRBatchResult + """ + + return self.system.batch_predict(images) + class PPOCRSystemv2(PPOCRv2): def __init__(self, det_model=None, cls_model=None, rec_model=None): @@ -299,3 +321,93 @@ def __init__(self, det_model=None, cls_model=None, rec_model=None): def predict(self, input_image): return super(PPOCRSystemv2, self).predict(input_image) + + +class DBDetectorPreprocessor: + def __init__(self): + """Create a preprocessor for DBDetectorModel + """ + self._preprocessor = C.vision.ocr.DBDetectorPreprocessor() + + def run(self, input_ims): + """Preprocess input images for DBDetectorModel + :param: input_ims: (list of numpy.ndarray) The input image + :return: pair(list of FDTensor, list of std::array) + """ + return self._preprocessor.run(input_ims) + + +class DBDetectorPostprocessor: + def __init__(self): + """Create a postprocessor for DBDetectorModel + """ + self._postprocessor = C.vision.ocr.DBDetectorPostprocessor() + + def run(self, runtime_results, batch_det_img_info): + """Postprocess the runtime results for DBDetectorModel + :param: runtime_results: (list of FDTensor or list of pyArray)The output FDTensor results from runtime + :param: batch_det_img_info: (list of std::array)The output of det_preprocessor + :return: list of Result(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results, batch_det_img_info) + + +class RecognizerPreprocessor: + def __init__(self): + """Create a preprocessor for RecognizerModel + """ + self._preprocessor = C.vision.ocr.RecognizerPreprocessor() + + def run(self, input_ims): + """Preprocess input images for RecognizerModel + :param: input_ims: (list of numpy.ndarray)The input image + :return: list of FDTensor + """ + return self._preprocessor.run(input_ims) + + +class RecognizerPostprocessor: + def __init__(self, label_path): + """Create a postprocessor for RecognizerModel + :param label_path: (str)Path of label file + """ + self._postprocessor = C.vision.ocr.RecognizerPostprocessor(label_path) + + def run(self, runtime_results): + """Postprocess the runtime results for RecognizerModel + :param: runtime_results: (list of FDTensor or list of pyArray)The output FDTensor results from runtime + :return: list of Result(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results) + + +class ClassifierPreprocessor: + def __init__(self): + """Create a preprocessor for ClassifierModel + """ + self._preprocessor = C.vision.ocr.ClassifierPreprocessor() + + def run(self, input_ims): + """Preprocess input images for ClassifierModel + :param: input_ims: (list of numpy.ndarray)The input image + :return: list of FDTensor + """ + return self._preprocessor.run(input_ims) + + +class ClassifierPostprocessor: + def __init__(self): + """Create a postprocessor for ClassifierModel + """ + self._postprocessor = C.vision.ocr.ClassifierPostprocessor() + + def run(self, runtime_results): + """Postprocess the runtime results for ClassifierModel + :param: runtime_results: (list of FDTensor or list of pyArray)The output FDTensor results from runtime + :return: list of Result(If the runtime_results is predict by batched samples, the length of this list equals to the batch size) + """ + return self._postprocessor.run(runtime_results) + + +def sort_boxes(boxes): + return C.vision.ocr.sort_boxes(boxes) From c1eadccdc286f5a8c637c34795e0c877f6cb7179 Mon Sep 17 00:00:00 2001 From: jm12138 <2286040843@qq.com> Date: Fri, 18 Nov 2022 13:19:13 +0800 Subject: [PATCH 063/136] [Other] Update detection example codes using download_model api (#613) update detection example --- .../detection/nanodet_plus/python/infer.py | 18 +++++++++++----- .../python/infer_faster_rcnn.py | 21 +++++++++++++------ .../paddledetection/python/infer_mask_rcnn.py | 21 +++++++++++++------ .../paddledetection/python/infer_picodet.py | 21 +++++++++++++------ .../paddledetection/python/infer_ppyolo.py | 21 +++++++++++++------ .../paddledetection/python/infer_ppyoloe.py | 1 - .../paddledetection/python/infer_yolov3.py | 21 +++++++++++++------ .../paddledetection/python/infer_yolox.py | 21 +++++++++++++------ .../detection/scaledyolov4/python/infer.py | 18 +++++++++++----- .../vision/detection/yolor/python/infer.py | 1 - .../vision/detection/yolov5/python/infer.py | 17 +++++++++++---- .../detection/yolov5lite/python/infer.py | 18 +++++++++++----- .../vision/detection/yolov6/python/infer.py | 17 +++++++++++---- .../vision/detection/yolov7/python/infer.py | 17 +++++++++++---- .../vision/detection/yolox/python/infer.py | 17 +++++++++++---- 15 files changed, 181 insertions(+), 69 deletions(-) diff --git a/examples/vision/detection/nanodet_plus/python/infer.py b/examples/vision/detection/nanodet_plus/python/infer.py index a0667db0055..9e7bf184f26 100644 --- a/examples/vision/detection/nanodet_plus/python/infer.py +++ b/examples/vision/detection/nanodet_plus/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of nanodet_plus onnx model.") + "--model", default=None, help="Path of nanodet_plus onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,13 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='NanoDetPlus_320') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.NanoDetPlus( - args.model, runtime_option=runtime_option) +model = fd.vision.detection.NanoDetPlus(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/paddledetection/python/infer_faster_rcnn.py b/examples/vision/detection/paddledetection/python/infer_faster_rcnn.py index d75ea2d4db0..dc095df1002 100644 --- a/examples/vision/detection/paddledetection/python/infer_faster_rcnn.py +++ b/examples/vision/detection/paddledetection/python/infer_faster_rcnn.py @@ -9,10 +9,10 @@ def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument( "--model_dir", - required=True, + default=None, help="Path of PaddleDetection model directory") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -41,9 +41,14 @@ def build_option(args): args = parse_arguments() -model_file = os.path.join(args.model_dir, "model.pdmodel") -params_file = os.path.join(args.model_dir, "model.pdiparams") -config_file = os.path.join(args.model_dir, "infer_cfg.yml") +if args.model_dir is None: + model_dir = fd.download_model(name='faster_rcnn_r50_vd_fpn_2x_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") # 配置runtime，加载模型 runtime_option = build_option(args) @@ -51,7 +56,11 @@ def build_option(args): model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py b/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py index 8b6c49aa794..f39c4f76865 100644 --- a/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py +++ b/examples/vision/detection/paddledetection/python/infer_mask_rcnn.py @@ -9,10 +9,10 @@ def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument( "--model_dir", - required=True, + default=None, help="Path of PaddleDetection model directory") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -48,9 +48,14 @@ def build_option(args): args = parse_arguments() -model_file = os.path.join(args.model_dir, "model.pdmodel") -params_file = os.path.join(args.model_dir, "model.pdiparams") -config_file = os.path.join(args.model_dir, "infer_cfg.yml") +if args.model_dir is None: + model_dir = fd.download_model(name='mask_rcnn_r50_1x_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") # 配置runtime，加载模型 runtime_option = build_option(args) @@ -58,7 +63,11 @@ def build_option(args): model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/paddledetection/python/infer_picodet.py b/examples/vision/detection/paddledetection/python/infer_picodet.py index cff7fef7780..e4b4e79d28a 100644 --- a/examples/vision/detection/paddledetection/python/infer_picodet.py +++ b/examples/vision/detection/paddledetection/python/infer_picodet.py @@ -9,10 +9,10 @@ def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument( "--model_dir", - required=True, + default=None, help="Path of PaddleDetection model directory") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -39,9 +39,14 @@ def build_option(args): args = parse_arguments() -model_file = os.path.join(args.model_dir, "model.pdmodel") -params_file = os.path.join(args.model_dir, "model.pdiparams") -config_file = os.path.join(args.model_dir, "infer_cfg.yml") +if args.model_dir is None: + model_dir = fd.download_model(name='picodet_l_320_coco_lcnet') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") # 配置runtime，加载模型 runtime_option = build_option(args) @@ -49,7 +54,11 @@ def build_option(args): model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/paddledetection/python/infer_ppyolo.py b/examples/vision/detection/paddledetection/python/infer_ppyolo.py index 7650ebe8a81..92831ba8aea 100644 --- a/examples/vision/detection/paddledetection/python/infer_ppyolo.py +++ b/examples/vision/detection/paddledetection/python/infer_ppyolo.py @@ -9,10 +9,10 @@ def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument( "--model_dir", - required=True, + default=None, help="Path of PaddleDetection model directory") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -41,9 +41,14 @@ def build_option(args): args = parse_arguments() -model_file = os.path.join(args.model_dir, "model.pdmodel") -params_file = os.path.join(args.model_dir, "model.pdiparams") -config_file = os.path.join(args.model_dir, "infer_cfg.yml") +if args.model_dir is None: + model_dir = fd.download_model(name='ppyolo_r50vd_dcn_1x_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") # 配置runtime，加载模型 runtime_option = build_option(args) @@ -51,7 +56,11 @@ def build_option(args): model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/paddledetection/python/infer_ppyoloe.py b/examples/vision/detection/paddledetection/python/infer_ppyoloe.py index ff9e2724d48..1d3260b5378 100644 --- a/examples/vision/detection/paddledetection/python/infer_ppyoloe.py +++ b/examples/vision/detection/paddledetection/python/infer_ppyoloe.py @@ -2,7 +2,6 @@ import os import fastdeploy as fd -import fastdeploy.utils def parse_arguments(): diff --git a/examples/vision/detection/paddledetection/python/infer_yolov3.py b/examples/vision/detection/paddledetection/python/infer_yolov3.py index 776d3ddba8a..c6a7ec43c1e 100644 --- a/examples/vision/detection/paddledetection/python/infer_yolov3.py +++ b/examples/vision/detection/paddledetection/python/infer_yolov3.py @@ -9,10 +9,10 @@ def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument( "--model_dir", - required=True, + default=None, help="Path of PaddleDetection model directory") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -39,9 +39,14 @@ def build_option(args): args = parse_arguments() -model_file = os.path.join(args.model_dir, "model.pdmodel") -params_file = os.path.join(args.model_dir, "model.pdiparams") -config_file = os.path.join(args.model_dir, "infer_cfg.yml") +if args.model_dir is None: + model_dir = fd.download_model(name='yolov3_darknet53_270e_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") # 配置runtime，加载模型 runtime_option = build_option(args) @@ -49,7 +54,11 @@ def build_option(args): model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/paddledetection/python/infer_yolox.py b/examples/vision/detection/paddledetection/python/infer_yolox.py index eb89c4f93c0..fff5cd224f0 100644 --- a/examples/vision/detection/paddledetection/python/infer_yolox.py +++ b/examples/vision/detection/paddledetection/python/infer_yolox.py @@ -9,10 +9,10 @@ def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument( "--model_dir", - required=True, + default=None, help="Path of PaddleDetection model directory") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -39,9 +39,14 @@ def build_option(args): args = parse_arguments() -model_file = os.path.join(args.model_dir, "model.pdmodel") -params_file = os.path.join(args.model_dir, "model.pdiparams") -config_file = os.path.join(args.model_dir, "infer_cfg.yml") +if args.model_dir is None: + model_dir = fd.download_model(name='yolox_s_300e_coco') +else: + model_dir = args.model_dir + +model_file = os.path.join(model_dir, "model.pdmodel") +params_file = os.path.join(model_dir, "model.pdiparams") +config_file = os.path.join(model_dir, "infer_cfg.yml") # 配置runtime，加载模型 runtime_option = build_option(args) @@ -49,7 +54,11 @@ def build_option(args): model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/scaledyolov4/python/infer.py b/examples/vision/detection/scaledyolov4/python/infer.py index d3052381c49..9d990d77bfb 100644 --- a/examples/vision/detection/scaledyolov4/python/infer.py +++ b/examples/vision/detection/scaledyolov4/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of scaledyolov4 onnx model.") + "--model", default=None, help="Path of scaledyolov4 onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,13 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='ScaledYOLOv4-P5') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.ScaledYOLOv4( - args.model, runtime_option=runtime_option) +model = fd.vision.detection.ScaledYOLOv4(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/yolor/python/infer.py b/examples/vision/detection/yolor/python/infer.py index 85123420f43..d5ab3dd7f12 100644 --- a/examples/vision/detection/yolor/python/infer.py +++ b/examples/vision/detection/yolor/python/infer.py @@ -1,7 +1,6 @@ import cv2 import fastdeploy as fd -import fastdeploy.utils def parse_arguments(): diff --git a/examples/vision/detection/yolov5/python/infer.py b/examples/vision/detection/yolov5/python/infer.py index 2cf3a03849a..fdded06c994 100644 --- a/examples/vision/detection/yolov5/python/infer.py +++ b/examples/vision/detection/yolov5/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of yolov5 onnx model.") + "--model", default=None, help="Path of yolov5 onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,12 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='YOLOv5s') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.YOLOv5(args.model, runtime_option=runtime_option) +model = fd.vision.detection.YOLOv5(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/yolov5lite/python/infer.py b/examples/vision/detection/yolov5lite/python/infer.py index 0a4312c4516..b1ec6904665 100644 --- a/examples/vision/detection/yolov5lite/python/infer.py +++ b/examples/vision/detection/yolov5lite/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of yolov5lite onnx model.") + "--model", default=None, help="Path of yolov5lite onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,13 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='YOLOv5Lite-s') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.YOLOv5Lite( - args.model, runtime_option=runtime_option) +model = fd.vision.detection.YOLOv5Lite(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/yolov6/python/infer.py b/examples/vision/detection/yolov6/python/infer.py index 47ea4cd3a4a..0a0a163fb2b 100644 --- a/examples/vision/detection/yolov6/python/infer.py +++ b/examples/vision/detection/yolov6/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of yolov6 onnx model.") + "--model", default=None, help="Path of yolov6 onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,12 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='YOLOv6s') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.YOLOv6(args.model, runtime_option=runtime_option) +model = fd.vision.detection.YOLOv6(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) diff --git a/examples/vision/detection/yolov7/python/infer.py b/examples/vision/detection/yolov7/python/infer.py index aff8cbe3045..b0ece7e47e7 100644 --- a/examples/vision/detection/yolov7/python/infer.py +++ b/examples/vision/detection/yolov7/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of yolov7 onnx model.") + "--model", default=None, help="Path of yolov7 onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,12 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='YOLOv7') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.YOLOv7(args.model, runtime_option=runtime_option) +model = fd.vision.detection.YOLOv7(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) # 预测结果可视化 diff --git a/examples/vision/detection/yolox/python/infer.py b/examples/vision/detection/yolox/python/infer.py index 8af3d9c4c3f..69203010bbf 100644 --- a/examples/vision/detection/yolox/python/infer.py +++ b/examples/vision/detection/yolox/python/infer.py @@ -7,9 +7,9 @@ def parse_arguments(): import ast parser = argparse.ArgumentParser() parser.add_argument( - "--model", required=True, help="Path of yolox onnx model.") + "--model", default=None, help="Path of yolox onnx model.") parser.add_argument( - "--image", required=True, help="Path of test image file.") + "--image", default=None, help="Path of test image file.") parser.add_argument( "--device", type=str, @@ -37,12 +37,21 @@ def build_option(args): args = parse_arguments() +if args.model is None: + model = fd.download_model(name='YOLOX-s') +else: + model = args.model + # 配置runtime，加载模型 runtime_option = build_option(args) -model = fd.vision.detection.YOLOX(args.model, runtime_option=runtime_option) +model = fd.vision.detection.YOLOX(model, runtime_option=runtime_option) # 预测图片检测结果 -im = cv2.imread(args.image) +if args.image is None: + image = fd.utils.get_detection_test_image() +else: + image = args.image +im = cv2.imread(image) result = model.predict(im.copy()) print(result) # 预测结果可视化 From bafd3c1378db7b15cd94d631ae3081624f51cfd1 Mon Sep 17 00:00:00 2001 From: Xiue <30972246+Xiue233@users.noreply.github.com> Date: Sat, 19 Nov 2022 13:28:46 +0800 Subject: [PATCH 064/136] Fix a typing error (#633) --- docs/en/quick_start/models/python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/quick_start/models/python.md b/docs/en/quick_start/models/python.md index 684dab15684..090e0421af1 100644 --- a/docs/en/quick_start/models/python.md +++ b/docs/en/quick_start/models/python.md @@ -10,7 +10,7 @@ This document uses the PaddleDetection target detection model PPYOLOE as an exam import fastdeploy as fd model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz" -image_url - "https://bj.bcebos.com/fastdeploy/tests/test_det.jpg" +image_url = "https://bj.bcebos.com/fastdeploy/tests/test_det.jpg" fd.download_and_decompress(model_url, path=".") fd.download(image_url, path=".") ``` From bb9cb9a98827f8bc5eb6b436a49df3332dbe3966 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 20 Nov 2022 11:17:18 +0800 Subject: [PATCH 065/136] Update fastdeploy_init.sh --- scripts/fastdeploy_init.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/fastdeploy_init.sh b/scripts/fastdeploy_init.sh index af7e4c42955..92df0bc1040 100644 --- a/scripts/fastdeploy_init.sh +++ b/scripts/fastdeploy_init.sh @@ -32,8 +32,8 @@ if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/opencv ]; then fi if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/openvino ]; then - export LD_LIBRARY_PATH=${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/openvino/lib:${LD_LIBRARY_PATH} - echo "OpenVINO Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/openvino/lib" + export LD_LIBRARY_PATH=${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/openvino/runtime/lib:${LD_LIBRARY_PATH} + echo "OpenVINO Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/openvino/runtime/lib" fi if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/tensorrt ]; then From 400820fbb52d0bedbc4f8e5385f8bdc06eec1287 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 20 Nov 2022 11:28:53 +0800 Subject: [PATCH 066/136] Update fastdeploy_init.sh --- scripts/fastdeploy_init.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/fastdeploy_init.sh b/scripts/fastdeploy_init.sh index 92df0bc1040..22981b7ea72 100644 --- a/scripts/fastdeploy_init.sh +++ b/scripts/fastdeploy_init.sh @@ -60,5 +60,9 @@ if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/paddlelite ]; th echo "Paddle Lite Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/paddlelite/lib" fi +if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster-tokenizer ]; then + export LD_LIBRARY_PATH=${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer/lib:${LD_LIBRARY_PATH} + echo "Faster Tokenizer Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer/lib" +fi cd ${CURRENT_EXE_DIR} From 2f4f6e9e0a3e2558a2c418a9989ae40956997b9e Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 20 Nov 2022 11:32:30 +0800 Subject: [PATCH 067/136] Update fastdeploy_init.sh --- scripts/fastdeploy_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/fastdeploy_init.sh b/scripts/fastdeploy_init.sh index 22981b7ea72..1e5bc0bb355 100644 --- a/scripts/fastdeploy_init.sh +++ b/scripts/fastdeploy_init.sh @@ -60,7 +60,7 @@ if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/paddlelite ]; th echo "Paddle Lite Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/paddlelite/lib" fi -if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster-tokenizer ]; then +if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer ]; then export LD_LIBRARY_PATH=${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer/lib:${LD_LIBRARY_PATH} echo "Faster Tokenizer Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer/lib" fi From b2deec0794cd5506dedf97dd87d8c2ad2cf0b942 Mon Sep 17 00:00:00 2001 From: leiqing <54695910+leiqing1@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:51:38 +0800 Subject: [PATCH 068/136] Update README_CN.md --- README_CN.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README_CN.md b/README_CN.md index 4f3a8b13608..5a33f9b5309 100644 --- a/README_CN.md +++ b/README_CN.md @@ -46,15 +46,17 @@
-- 🔥 **2022.11.8：Release FastDeploy [release v0.6.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.6.0)** - - **🖥️ 服务端部署：支持推理速度更快的后端，支持更多的模型** - - 优化 YOLO系列、PaddleClas、PaddleDetection 前后处理内存创建逻辑； - - 融合视觉预处理操作，优化PaddleClas、PaddleDetection预处理性能，提升端到端推理性能； - - 服务化部署新增Clone接口支持，降低Paddle Inference/TensorRT/OpenVINO后端在多实例下内存/显存的使用； - - 增加[FSANet头部姿态识别](./examples/vision/headpose)、[PFLD人脸对齐](./examples/vision/facealign)、[ERNIE文本分类](./examples/text/ernie-3.0)等模型。 - - **📲 移动端和端侧部署：移动端后端能力升级，支持更多的CV模型** - - 集成 RKNPU2 后端，并提供与 Paddle Inference、Paddle Inference TensorRT、TensorRT、OpenVINO、ONNX Runtime、Paddle Lite 等推理后端一致的开发体验； - - 支持 [PP-HumanSeg](./examples/vision/segmentation/paddleseg/rknpu2)、[Unet](./examples/vision/segmentation/paddleseg/rknpu2)、[PicoDet](examples/vision/detection/paddledetection/rknpu2)、[SCRFD](./examples/vision/facedet/scrfd/rknpu2) 等在NPU高需求的特色模型。 +- 🔥 **2022.11.15：Release FastDeploy [release v0.7.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.7)** + - **🖥️ 服务端部署：支持更多的模型，推理性能进一步提升** + - 新增 [PaddleClas](./examples/vision/classification/paddleclas/serving) 模型服务化部署示例； + - 新增 [Stable Diffusion](./examples/multimodal/stable_diffusion) 模型部署示例； + - PaddleClas、PaddleDetection、YOLOv5 部署代码升级，支持 predict 及 batch_predict； + - 针对 FDTensor 增加 Pad function 操作符，支持在batch预测时，对输入进行 Padding； + - 针对 FDTensor 增加 Python API to_dlpack 接口，支持 FDTensor 在不同框架间的无拷贝传输； + - **📲 移动端和端侧部署：端侧能力升级，支持更多硬件** + - 集成 Paddle Lite TIM-VX 后端，用于支持瑞芯微RV1109、RV1126、RK1808、晶晨A311D等硬件，并提供与 Paddle Inference、Paddle Inference TensorRT、TensorRT、OpenVINO、ONNX Runtime、Paddle Lite、RKNPU2等推理后端一致的开发体验； + - 在瑞芯微 RV1126 硬件上，支持 图像分类模型 [ResNet50_vd](./examples/vision/classification/paddleclas/rk1126/cpp)； + - 在瑞芯微 RK3588、RK3568 等硬件上，支持 人脸检测模型 [SCRFD](./examples/vision/facedet/scrfd/rknpu2). - [**more releases information**](./releases) From d5ec6dde25820a2964513cd1ccb6a40f69ee397b Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:52:11 +0800 Subject: [PATCH 069/136] [Bug Fix] fix camera preview size setting problem (#638) * [Bug Fix] fix android app detail page errors * [Android] fix realtime camera mode and shutter * [Bug Fix] fix AllocateSegmentationResultFromJava error * [Bug Fix] fix camera preview size setting problem * [Model] use uint8 buffer instead of fp32 in ppseg postprocess * [Model] revert changes in ppseg * [Model] revert postprocess changes in ppseg --- fastdeploy/vision/segmentation/ppseg/model.cc | 2 +- java/android/app/src/main/AndroidManifest.xml | 4 +- .../examples/facedet/FaceDetMainActivity.java | 9 +++- .../SegmentationMainActivity.java | 9 +++- .../baidu/paddle/fastdeploy/app/ui/Utils.java | 2 +- .../app/ui/view/CameraSurfaceView.java | 50 +++++++++++-------- 6 files changed, 47 insertions(+), 29 deletions(-) diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc index 3d3b158a3a1..5fade8cb097 100644 --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -375,4 +375,4 @@ void PaddleSegModel::DisableNormalizeAndPermute() { } // namespace segmentation } // namespace vision -} // namespace fastdeploy +} // namespace fastdeploy \ No newline at end of file diff --git a/java/android/app/src/main/AndroidManifest.xml b/java/android/app/src/main/AndroidManifest.xml index 0f963ffe416..858ef04f620 100644 --- a/java/android/app/src/main/AndroidManifest.xml +++ b/java/android/app/src/main/AndroidManifest.xml @@ -15,14 +15,14 @@ android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true" android:theme="@style/AppTheme"> - + diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java index eaf3d3348a5..fa2f1aeb984 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/facedet/FaceDetMainActivity.java @@ -320,8 +320,13 @@ protected void onDestroy() { public void initView() { TYPE = REALTIME_DETECT; - CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; - CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 360; + // (1) EXPECTED_PREVIEW_WIDTH should mean 'height' and EXPECTED_PREVIEW_HEIGHT + // should mean 'width' if the camera display orientation is 90 | 270 degree + // (Hold the phone upright to record video) + // (2) Smaller resolution is more suitable for lite face detection + // on mobile phone. So, we set this preview size (480,480) here. + CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 480; + CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 480; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); svPreview.switchCamera(); // Front camera for HumanSeg diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java index c1d9c72b68b..3420581c290 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/segmentation/SegmentationMainActivity.java @@ -310,8 +310,13 @@ protected void onDestroy() { public void initView() { TYPE = REALTIME_DETECT; - CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 720; - CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 360; + // (1) EXPECTED_PREVIEW_WIDTH should mean 'height' and EXPECTED_PREVIEW_HEIGHT + // should mean 'width' if the camera display orientation is 90 | 270 degree + // (Hold the phone upright to record video) + // (2) Smaller resolution is more suitable for Lite Portrait HumanSeg. + // So, we set this preview size (480,480) here. + CameraSurfaceView.EXPECTED_PREVIEW_WIDTH = 480; + CameraSurfaceView.EXPECTED_PREVIEW_HEIGHT = 480; svPreview = (CameraSurfaceView) findViewById(R.id.sv_preview); svPreview.setOnTextureChangedListener(this); svPreview.switchCamera(); // Front camera for HumanSeg diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java index 6da24c2f6af..eabeb74f463 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/Utils.java @@ -126,7 +126,7 @@ public static String getDCIMDirectory() { } public static Camera.Size getOptimalPreviewSize(List sizes, int w, int h) { - final double ASPECT_TOLERANCE = 0.1; + final double ASPECT_TOLERANCE = 0.3; double targetRatio = (double) w / h; if (sizes == null) return null; diff --git a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java index 9269a48284a..11d9fcdcb27 100644 --- a/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java +++ b/java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/ui/view/CameraSurfaceView.java @@ -30,8 +30,8 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer, SurfaceTexture.OnFrameAvailableListener { private static final String TAG = CameraSurfaceView.class.getSimpleName(); - public static int EXPECTED_PREVIEW_WIDTH = 1280; - public static int EXPECTED_PREVIEW_HEIGHT = 720; + public static int EXPECTED_PREVIEW_WIDTH = 1280; // 1920 + public static int EXPECTED_PREVIEW_HEIGHT = 720; // 960 protected int numberOfCameras; protected int selectedCameraId; @@ -99,6 +99,16 @@ public class CameraSurfaceView extends GLSurfaceView implements Renderer, private int vcTex2Screen; private int tcTex2Screen; + public void setBitmapReleaseMode(boolean mode) { + synchronized (this) { + bitmapReleaseMode = mode; + } + } + + public Bitmap getBitmap() { + return ARGB8888ImageBitmap; // may null or recycled. + } + public interface OnTextureChangedListener { boolean onTextureChanged(Bitmap ARGB8888ImageBitmap); } @@ -236,16 +246,6 @@ public void onDrawFrame(GL10 gl) { GLES20.glFlush(); } - public void setBitmapReleaseMode(boolean mode) { - synchronized (this) { - bitmapReleaseMode = mode; - } - } - - public Bitmap getBitmap() { - return ARGB8888ImageBitmap; // may null or recycled. - } - private float[] transformTextureCoordinates(float[] coords, float[] matrix) { float[] result = new float[coords.length]; float[] vt = new float[4]; @@ -287,20 +287,28 @@ public void switchCamera() { public void openCamera() { if (disableCamera) return; camera = Camera.open(selectedCameraId); - List supportedPreviewSizes = camera.getParameters().getSupportedPreviewSizes(); - Size previewSize = Utils.getOptimalPreviewSize(supportedPreviewSizes, EXPECTED_PREVIEW_WIDTH, - EXPECTED_PREVIEW_HEIGHT); Camera.Parameters parameters = camera.getParameters(); + int degree = Utils.getCameraDisplayOrientation(context, selectedCameraId); + camera.setDisplayOrientation(degree); + boolean rotate = degree == 90 || degree == 270; + int adjusted_width = rotate ? EXPECTED_PREVIEW_HEIGHT : EXPECTED_PREVIEW_WIDTH; + int adjusted_height = rotate ? EXPECTED_PREVIEW_WIDTH : EXPECTED_PREVIEW_HEIGHT; + + List supportedPreviewSizes = camera.getParameters().getSupportedPreviewSizes(); + + Size previewSize = Utils.getOptimalPreviewSize( + supportedPreviewSizes, adjusted_width, adjusted_height); + + textureWidth = previewSize.width; + textureHeight = previewSize.height; + parameters.setPreviewSize(previewSize.width, previewSize.height); + camera.setParameters(parameters); + if (parameters.getSupportedFocusModes().contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); } - camera.setParameters(parameters); - int degree = Utils.getCameraDisplayOrientation(context, selectedCameraId); - camera.setDisplayOrientation(degree); - boolean rotate = degree == 90 || degree == 270; - textureWidth = rotate ? previewSize.height : previewSize.width; - textureHeight = rotate ? previewSize.width : previewSize.height; + // Destroy FBO and draw textures GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); GLES20.glDeleteFramebuffers(1, fbo, 0); From 1466b23f0c4980007b3dc6edd0fb8fedec9c4b42 Mon Sep 17 00:00:00 2001 From: leiqing <54695910+leiqing1@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:53:29 +0800 Subject: [PATCH 070/136] Update README_EN.md --- README_EN.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README_EN.md b/README_EN.md index 7b3a451533e..74e1d01b7f3 100644 --- a/README_EN.md +++ b/README_EN.md @@ -45,15 +45,17 @@ Including image classification, object detection, image segmentation, face detec
-- 🔥 **2022.11.8：Release FastDeploy [release v0.6.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.6.0)**
- - **🖥️ Server-side and Cloud Deployment: Support more backend, Support more CV models** - - Optimize preprocessing and postprocessing memory creation logic on YOLO series, PaddleClas, PaddleDetection; - - Integrate visual preprocessing operations, optimize the preprocessing performance of PaddleClas and PaddleDetection, and improve end-to-end performance; - - Add Clone interface support for service-based deployment, reducing the memory、GPU memory usage of Paddle Inference、TensorRT、OpenVINO backend in multiple instances - - Support [FSANet](./examples/vision/headpose) head pose recognition model, [PFLD](./examples/vision/facealign) face alignment model, [ERNIE](./examples/text/ernie-3.0) text classification model etc. +- 🔥 **2022.11.15：Release FastDeploy [release v0.7.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.7.0)**
+ - **🖥️ Server-side and Cloud Deployment: Support more CV models, improve deployment performance** + - Support [PaddleClas](./examples/vision/classification/paddleclas/serving) model service-based deployment； + - Support [Stable Diffusion](./examples/multimodal/stable_diffusion) model deployment； + - Upgrade PaddleClas、PaddleDetection、YOLOv5 deployment code to support predict and batch_predict; + - Add the Pad function operator for the FDTensor to support Padding of the input during batch prediction; + - Add Python API to_dlpack interface for FDTensor to support copyless transfer of FDTensor between frameworks.; - **📱 Mobile and Edge Device Deployment: support new backend，support more CV model** - - Support RKNPU2, and provide a seamless deployment experience with other inference engines include Paddle Inference、Paddle Inference TensorRT、Paddle Lite、TensorRT、OpenVINO、ONNX Runtime； - - Support [PP-HumanSeg、Unet](examples/vision/segmentation/paddleseg/rknpu2)、[PicoDet](./examples/vision/detection/paddledetection/rknpu2)、[SCRFD](./examples/vision/facedet/scrfd/rknpu2) and other popular models on NPU. + - Support Paddle Lite TIM-VX for supporting hardware such as Rockchip RV1109,RV1126, RK1808, Amlogic A311D, etc. And provide a seamless deployment experience with other inference engines include Paddle Inference、Paddle Inference TensorRT、Paddle Lite、TensorRT、OpenVINO、ONNX Runtime、RKNPU2; + - support Image classification model [ResNet50_vd](./examples/vision/classification/paddleclas/rk1126/cpp) on Rockchip RV1126; + - support Face detection model [SCRFD](./examples/vision/facedet/scrfd/rknpu2) on Rockchip RK3588, RK3568 and other hardware; - [**more releases information**](./releases) From a5d340d4db2776d038f7b6aaa049150d00d4dd5a Mon Sep 17 00:00:00 2001 From: leiqing <54695910+leiqing1@users.noreply.github.com> Date: Mon, 21 Nov 2022 10:58:17 +0800 Subject: [PATCH 071/136] Update README_EN.md --- README_EN.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README_EN.md b/README_EN.md index 74e1d01b7f3..d3f7f26d2e2 100644 --- a/README_EN.md +++ b/README_EN.md @@ -47,15 +47,15 @@ Including image classification, object detection, image segmentation, face detec - 🔥 **2022.11.15：Release FastDeploy [release v0.7.0](https://github.com/PaddlePaddle/FastDeploy/tree/release/0.7.0)**
- **🖥️ Server-side and Cloud Deployment: Support more CV models, improve deployment performance** - - Support [PaddleClas](./examples/vision/classification/paddleclas/serving) model service-based deployment； - - Support [Stable Diffusion](./examples/multimodal/stable_diffusion) model deployment； + - Support [PaddleClas](./examples/vision/classification/paddleclas/serving) model service-based deployment; + - Support [Stable Diffusion](./examples/multimodal/stable_diffusion) model deployment; - Upgrade PaddleClas、PaddleDetection、YOLOv5 deployment code to support predict and batch_predict; - Add the Pad function operator for the FDTensor to support Padding of the input during batch prediction; - - Add Python API to_dlpack interface for FDTensor to support copyless transfer of FDTensor between frameworks.; + - Add Python API to_dlpack interface for FDTensor to support copyless transfer of FDTensor between frameworks. - **📱 Mobile and Edge Device Deployment: support new backend，support more CV model** - Support Paddle Lite TIM-VX for supporting hardware such as Rockchip RV1109,RV1126, RK1808, Amlogic A311D, etc. And provide a seamless deployment experience with other inference engines include Paddle Inference、Paddle Inference TensorRT、Paddle Lite、TensorRT、OpenVINO、ONNX Runtime、RKNPU2; - - support Image classification model [ResNet50_vd](./examples/vision/classification/paddleclas/rk1126/cpp) on Rockchip RV1126; - - support Face detection model [SCRFD](./examples/vision/facedet/scrfd/rknpu2) on Rockchip RK3588, RK3568 and other hardware; + - Support Image classification model [ResNet50_vd](./examples/vision/classification/paddleclas/rk1126/cpp) on Rockchip RV1126; + - Support Face detection model [SCRFD](./examples/vision/facedet/scrfd/rknpu2) on Rockchip RK3588, RK3568 and other hardware. - [**more releases information**](./releases) From 2a2a0c89eedb90233a78872cdecd9222227942fe Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Mon, 21 Nov 2022 10:58:28 +0800 Subject: [PATCH 072/136] [Model] add vsr serials models (#518) * [Model] add vsr serials models Signed-off-by: ChaoII <849453582@qq.com> * [Model] add vsr serials models Signed-off-by: ChaoII <849453582@qq.com> * fix build problem Signed-off-by: ChaoII <849453582@qq.com> * fix code style Signed-off-by: ChaoII <849453582@qq.com> * modify according to review suggestions Signed-off-by: ChaoII <849453582@qq.com> * modify vsr trt example Signed-off-by: ChaoII <849453582@qq.com> * update sr directory * fix BindPPSR * add doxygen comment * add sr unit test * update model file url Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: Jason --- .gitignore | 4 +- examples/vision/sr/README.md | 9 + examples/vision/sr/basicvsr/README.md | 28 ++ .../vision/sr/basicvsr/cpp/CMakeLists.txt | 14 + examples/vision/sr/basicvsr/cpp/README.md | 74 +++++ examples/vision/sr/basicvsr/cpp/infer.cc | 297 +++++++++++++++++ examples/vision/sr/basicvsr/python/README.md | 61 ++++ examples/vision/sr/basicvsr/python/infer.py | 86 +++++ examples/vision/sr/edvr/README.md | 28 ++ examples/vision/sr/edvr/cpp/CMakeLists.txt | 14 + examples/vision/sr/edvr/cpp/README.md | 75 +++++ examples/vision/sr/edvr/cpp/infer.cc | 279 ++++++++++++++++ examples/vision/sr/edvr/python/README.md | 61 ++++ examples/vision/sr/edvr/python/infer.py | 89 ++++++ examples/vision/sr/ppmsvsr/README.md | 27 ++ examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt | 14 + examples/vision/sr/ppmsvsr/cpp/README.md | 75 +++++ examples/vision/sr/ppmsvsr/cpp/infer.cc | 298 ++++++++++++++++++ examples/vision/sr/ppmsvsr/python/README.md | 61 ++++ examples/vision/sr/ppmsvsr/python/infer.py | 86 +++++ .../vision/tracking/pptracking/cpp/infer.cc | 6 +- .../tracking/pptracking/python/README.md | 4 +- .../tracking/pptracking/python/infer.py | 3 +- fastdeploy/vision.h | 1 + fastdeploy/vision/sr/ppsr/basicvsr.cc | 38 +++ fastdeploy/vision/sr/ppsr/basicvsr.h | 41 +++ fastdeploy/vision/sr/ppsr/edvr.cc | 71 +++++ fastdeploy/vision/sr/ppsr/edvr.h | 45 +++ fastdeploy/vision/sr/ppsr/model.h | 18 ++ fastdeploy/vision/sr/ppsr/ppmsvsr.cc | 128 ++++++++ fastdeploy/vision/sr/ppsr/ppmsvsr.h | 62 ++++ fastdeploy/vision/sr/ppsr/ppsr_pybind.cc | 70 ++++ fastdeploy/vision/sr/sr_pybind.cc | 25 ++ fastdeploy/vision/vision_pybind.cc | 2 + python/fastdeploy/vision/__init__.py | 1 + python/fastdeploy/vision/sr/__init__.py | 15 + python/fastdeploy/vision/sr/ppsr/__init__.py | 107 +++++++ tests/models/test_basicvsr.py | 71 +++++ tests/models/test_edvr.py | 76 +++++ tests/models/test_ppmsvsr.py | 70 ++++ 40 files changed, 2526 insertions(+), 8 deletions(-) create mode 100644 examples/vision/sr/README.md create mode 100644 examples/vision/sr/basicvsr/README.md create mode 100644 examples/vision/sr/basicvsr/cpp/CMakeLists.txt create mode 100644 examples/vision/sr/basicvsr/cpp/README.md create mode 100644 examples/vision/sr/basicvsr/cpp/infer.cc create mode 100644 examples/vision/sr/basicvsr/python/README.md create mode 100644 examples/vision/sr/basicvsr/python/infer.py create mode 100644 examples/vision/sr/edvr/README.md create mode 100644 examples/vision/sr/edvr/cpp/CMakeLists.txt create mode 100644 examples/vision/sr/edvr/cpp/README.md create mode 100644 examples/vision/sr/edvr/cpp/infer.cc create mode 100644 examples/vision/sr/edvr/python/README.md create mode 100644 examples/vision/sr/edvr/python/infer.py create mode 100644 examples/vision/sr/ppmsvsr/README.md create mode 100644 examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt create mode 100644 examples/vision/sr/ppmsvsr/cpp/README.md create mode 100644 examples/vision/sr/ppmsvsr/cpp/infer.cc create mode 100644 examples/vision/sr/ppmsvsr/python/README.md create mode 100644 examples/vision/sr/ppmsvsr/python/infer.py create mode 100644 fastdeploy/vision/sr/ppsr/basicvsr.cc create mode 100644 fastdeploy/vision/sr/ppsr/basicvsr.h create mode 100644 fastdeploy/vision/sr/ppsr/edvr.cc create mode 100644 fastdeploy/vision/sr/ppsr/edvr.h create mode 100644 fastdeploy/vision/sr/ppsr/model.h create mode 100644 fastdeploy/vision/sr/ppsr/ppmsvsr.cc create mode 100644 fastdeploy/vision/sr/ppsr/ppmsvsr.h create mode 100644 fastdeploy/vision/sr/ppsr/ppsr_pybind.cc create mode 100644 fastdeploy/vision/sr/sr_pybind.cc create mode 100644 python/fastdeploy/vision/sr/__init__.py create mode 100644 python/fastdeploy/vision/sr/ppsr/__init__.py create mode 100644 tests/models/test_basicvsr.py create mode 100644 tests/models/test_edvr.py create mode 100644 tests/models/test_ppmsvsr.py diff --git a/.gitignore b/.gitignore index 2edbc3103c9..0bc8ba218c8 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,6 @@ coverage yalc.* .yalc examples/vision/collect_quantize_cc.sh -examples/vision/tests_quantize \ No newline at end of file +examples/vision/tests_quantize +fastdeploy/LICENSE +fastdeploy/ThirdPartyNotices.txt \ No newline at end of file diff --git a/examples/vision/sr/README.md b/examples/vision/sr/README.md new file mode 100644 index 00000000000..88f9e777739 --- /dev/null +++ b/examples/vision/sr/README.md @@ -0,0 +1,9 @@ +# sr 模型部署 + +FastDeploy目前支持如下超分模型部署 + +| 模型 | 说明 | 模型格式 | 版本 | +|:-----------------------------------------|:----------------------|:-------|:----------------------------------------------------------------------------------| +| [PaddleGAN/BasicVSR](./basicvsr) | BasicVSR 系列模型 | paddle | [develop](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) | +| [PaddleGAN/EDVR](./edvr) | EDVR 系列模型 | paddle | [develop](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) | +| [PaddleGAN/PP-MSVSR](./ppmsvsr) | PP-MSVSR 系列模型 | paddle | [develop](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) | diff --git a/examples/vision/sr/basicvsr/README.md b/examples/vision/sr/basicvsr/README.md new file mode 100644 index 00000000000..a077a3cbf22 --- /dev/null +++ b/examples/vision/sr/basicvsr/README.md @@ -0,0 +1,28 @@ +# BasicVSR模型部署 + +## 模型版本说明 + +- [PaddleGAN develop](https://github.com/PaddlePaddle/PaddleGAN) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [BasicVSR](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +## 导出部署模型 + +在部署前，需要先将训练好的BasicVSR导出成部署模型，导出BasicVSR导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:----------------------------------------------------------------------------|:-------|:----- | :------ | +| [BasicVSR](https://bj.bcebos.com/paddlehub/fastdeploy/BasicVSR_reds_x4.tgz) | 30.1MB | - | + +**注意**：非常不建议在没有独立显卡的设备上运行该模型 + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/sr/basicvsr/cpp/CMakeLists.txt b/examples/vision/sr/basicvsr/cpp/CMakeLists.txt new file mode 100644 index 00000000000..93540a7e83e --- /dev/null +++ b/examples/vision/sr/basicvsr/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/sr/basicvsr/cpp/README.md b/examples/vision/sr/basicvsr/cpp/README.md new file mode 100644 index 00000000000..ff5e17a9905 --- /dev/null +++ b/examples/vision/sr/basicvsr/cpp/README.md @@ -0,0 +1,74 @@ +# BasicVSR C++部署示例 + +本目录下提供`infer.cc`快速完成BasicVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +以Linux上BasicVSR推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +# fastdeploy版本 >= 0.7.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/sr/basicvsr/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 +make -j + +# 下载BasicVSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/BasicVSR_reds_x4.tar +tar -xvf BasicVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 + + +# CPU推理 +./infer_demo BasicVSR_reds_x4 vsr_src.mp4 0 2 +# GPU推理 +./infer_demo BasicVSR_reds_x4 vsr_src.mp4 1 2 +# GPU上TensorRT推理 +./infer_demo BasicVSR_reds_x4 vsr_src.mp4 2 2 +``` + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## BasicVSR C++接口 + +### BasicVSR类 + +```c++ +fastdeploy::vision::sr::BasicVSR( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +BasicVSR模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> BasicVSR::Predict(std::vector& imgs, std::vector& results) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **imgs**: 输入视频帧序列，注意需为HWC，BGR格式 +> > * **results**: 视频超分结果，超分后的视频帧序列 + +- [模型介绍](../../) +- [Python部署](../python) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/basicvsr/cpp/infer.cc b/examples/vision/sr/basicvsr/cpp/infer.cc new file mode 100644 index 00000000000..603179bec57 --- /dev/null +++ b/examples/vision/sr/basicvsr/cpp/infer.cc @@ -0,0 +1,297 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto model = fastdeploy::vision::sr::BasicVSR(model_file, params_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void GpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto model = fastdeploy::vision::sr::BasicVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void TrtInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + // use paddle-TRT + option.EnablePaddleToTrt(); + auto model = fastdeploy::vision::sr::BasicVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + //Note that the resolution between the size and the original input is consistent when the model is exported, + // for example: [1,2,3,180,320], after 4x super separation [1,2,3,720,1080]. + //Therefore, it is very important to derive the model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/video frame number run_option, " + "e.g ./infer_model ./vsr_model_dir ./person.mp4 0 2" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + int frame_num = 2; + if (argc == 5) { + frame_num = std::atoi(argv[4]); + } + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2], frame_num); + } + return 0; +} diff --git a/examples/vision/sr/basicvsr/python/README.md b/examples/vision/sr/basicvsr/python/README.md new file mode 100644 index 00000000000..d5594b70ad2 --- /dev/null +++ b/examples/vision/sr/basicvsr/python/README.md @@ -0,0 +1,61 @@ +# BasicVSR Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成BasicVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/sr/basicvsr/python + +# 下载BasicVSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/BasicVSR_reds_x4.tar +tar -xvf BasicVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 +# CPU推理 +python infer.py --model BasicVSR_reds_x4 --video person.mp4 --frame_num 2 --device cpu +# GPU推理 +python infer.py --model BasicVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model BasicVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu --use_trt True +``` + +## BasicVSR Python接口 + +```python +fd.vision.sr.BasicVSR(model_file, params_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +BasicVSR模型加载和初始化，其中model_file和params_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> BasicVSR.predict(frames) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **frames**(list[np.ndarray]): 输入数据，注意需为HWC，BGR格式, frames为视频帧序列 + +> **返回** list[np.ndarray] 为超分后的视频帧序列 + + +## 其它文档 + +- [BasicVSR 模型介绍](..) +- [BasicVSR C++部署](../cpp) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/basicvsr/python/infer.py b/examples/vision/sr/basicvsr/python/infer.py new file mode 100644 index 00000000000..8b1a9dbb73a --- /dev/null +++ b/examples/vision/sr/basicvsr/python/infer.py @@ -0,0 +1,86 @@ +import cv2 +import os +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--video", type=str, required=True, help="Path of test video file.") + parser.add_argument("--frame_num", type=int, default=2, help="frame num") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + if args.use_trt: + option.use_trt_backend() + option.enable_paddle_to_trt() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.sr.BasicVSR( + model_file, params_file, runtime_option=runtime_option) + +# 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] +capture = cv2.VideoCapture(args.video) +video_out_name = "output.mp4" +video_fps = capture.get(cv2.CAP_PROP_FPS) +video_frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) +# 注意导出模型时尺寸与原始输入的分辨一致比如：[1,2,3,180,320],经过4x超分后[1,2,3,720,1280] +# 所以导出模型相当重要(最关键的是根据netron查看网络输出shape) +out_width = 1280 +out_height = 720 +print(f"fps: {video_fps}\tframe_count: {video_frame_count}") +# Create VideoWriter for output +video_out_dir = "./" +video_out_path = os.path.join(video_out_dir, video_out_name) +fucc = cv2.VideoWriter_fourcc(*"mp4v") +video_out = cv2.VideoWriter(video_out_path, fucc, video_fps, + (out_width, out_height), True) +if not video_out.isOpened(): + print("create video writer failed!") +# Capture all frames and do inference +frame_id = 0 +reach_end = False +while capture.isOpened(): + imgs = [] + for i in range(args.frame_num): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + # cv2.imshow("13", item) + # cv2.waitKey(30) + video_out.write(item) + print("Processing frame: ", frame_id) + frame_id += 1 +print("inference finished, output video saved at: ", video_out_path) +capture.release() +video_out.release() diff --git a/examples/vision/sr/edvr/README.md b/examples/vision/sr/edvr/README.md new file mode 100644 index 00000000000..398715cd5e3 --- /dev/null +++ b/examples/vision/sr/edvr/README.md @@ -0,0 +1,28 @@ +# EDVR模型部署 + +## 模型版本说明 + +- [PaddleGAN develop](https://github.com/PaddlePaddle/PaddleGAN) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [EDVR](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +## 导出部署模型 + +在部署前，需要先将训练好的EDVR导出成部署模型，导出EDVR导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:--------------------------------------------------------------------------------|:-------|:----- | :------ | +| [EDVR](https://bj.bcebos.com/paddlehub/fastdeploy/EDVR_M_wo_tsa_SRx4.tgz) | 14.9MB | - | + +**注意**：非常不建议在没有独立显卡的设备上运行该模型 + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/sr/edvr/cpp/CMakeLists.txt b/examples/vision/sr/edvr/cpp/CMakeLists.txt new file mode 100644 index 00000000000..93540a7e83e --- /dev/null +++ b/examples/vision/sr/edvr/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/sr/edvr/cpp/README.md b/examples/vision/sr/edvr/cpp/README.md new file mode 100644 index 00000000000..c40b8cf83b6 --- /dev/null +++ b/examples/vision/sr/edvr/cpp/README.md @@ -0,0 +1,75 @@ +# EDVR C++部署示例 + +本目录下提供`infer.cc`快速完成EDVR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上EDVR推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +# fastdeploy版本 >= 0.7.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/sr/edvr/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 +make -j + +# 下载EDVR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/EDVR_M_wo_tsa_SRx4.tar +tar -xvf EDVR_M_wo_tsa_SRx4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 + + +# CPU推理 +./infer_demo EDVR_M_wo_tsa_SRx4 vsr_src.mp4 0 2 +# GPU推理 +./infer_demo EDVR_M_wo_tsa_SRx4 vsr_src.mp4 1 2 +# GPU上TensorRT推理 +./infer_demo EDVR_M_wo_tsa_SRx4 vsr_src.mp4 2 2 +``` + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## EDVR C++接口 + +### EDVR类 + +```c++ +fastdeploy::vision::sr::EDVR( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +EDVR模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> EDVR::Predict(std::vector& imgs, std::vector& results) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **imgs**: 输入视频帧序列，注意需为HWC，BGR格式 +> > * **results**: 视频超分结果，超分后的视频帧序列 + +- [模型介绍](../../) +- [Python部署](../python) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/edvr/cpp/infer.cc b/examples/vision/sr/edvr/cpp/infer.cc new file mode 100644 index 00000000000..3e9e81ca947 --- /dev/null +++ b/examples/vision/sr/edvr/cpp/infer.cc @@ -0,0 +1,279 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto model = fastdeploy::vision::sr::EDVR(model_file, params_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + std::vector imgs; + while (capture.read(frame)){ + if (!frame.empty()) + { + if(frame_id < frame_num){ + imgs.push_back(frame); + frame_id ++; + continue; + } + imgs.erase(imgs.begin()); + imgs.push_back(frame); + } + frame_id ++; + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void GpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto model = fastdeploy::vision::sr::EDVR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + std::vector imgs; + while (capture.read(frame)){ + if (!frame.empty()) + { + if(frame_id < frame_num){ + imgs.push_back(frame); + frame_id ++; + continue; + } + imgs.erase(imgs.begin()); + imgs.push_back(frame); + } + frame_id ++; + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void TrtInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + // use paddle-TRT + option.EnablePaddleToTrt(); + auto model = fastdeploy::vision::sr::EDVR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + //Note that the resolution between the size and the original input is consistent when the model is exported, + // for example: [1,2,3,180,320], after 4x super separation [1,2,3,720,1080]. + //Therefore, it is very important to derive the model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + std::vector imgs; + while (capture.read(frame)){ + if (!frame.empty()) + { + if(frame_id < frame_num){ + imgs.push_back(frame); + frame_id ++; + continue; + } + imgs.erase(imgs.begin()); + imgs.push_back(frame); + } + frame_id ++; + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/video frame number run_option, " + "e.g ./infer_model ./vsr_model_dir ./person.mp4 0 2" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + int frame_num = 2; + if (argc == 5) { + frame_num = std::atoi(argv[4]); + } + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2], frame_num); + } + return 0; +} diff --git a/examples/vision/sr/edvr/python/README.md b/examples/vision/sr/edvr/python/README.md new file mode 100644 index 00000000000..83f0a01e3ac --- /dev/null +++ b/examples/vision/sr/edvr/python/README.md @@ -0,0 +1,61 @@ +# EDVR Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成EDVR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/sr/edvr/python + +# 下载VSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/EDVR_M_wo_tsa_SRx4.tar +tar -xvf EDVR_M_wo_tsa_SRx4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 +# CPU推理 +python infer.py --model EDVR_M_wo_tsa_SRx4 --video person.mp4 --frame_num 2 --device cpu +# GPU推理 +python infer.py --model EDVR_M_wo_tsa_SRx4 --video person.mp4 --frame_num 2 --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model EDVR_M_wo_tsa_SRx4 --video person.mp4 --frame_num 2 --device gpu --use_trt True +``` + +## EDVR Python接口 + +```python +fd.vision.sr.EDVR(model_file, params_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +EDVR模型加载和初始化，其中model_file和params_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> EDVR.predict(frames) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **frames**(list[np.ndarray]): 输入数据，注意需为HWC，BGR格式, frames为视频帧序列 + +> **返回** list[np.ndarray] 为超分后的视频帧序列 + + +## 其它文档 + +- [EDVR 模型介绍](..) +- [EDVR C++部署](../cpp) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/edvr/python/infer.py b/examples/vision/sr/edvr/python/infer.py new file mode 100644 index 00000000000..fe30bcf4f0a --- /dev/null +++ b/examples/vision/sr/edvr/python/infer.py @@ -0,0 +1,89 @@ +import cv2 +import os +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--video", type=str, required=True, help="Path of test video file.") + parser.add_argument("--frame_num", type=int, default=2, help="frame num") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + if args.use_trt: + option.use_trt_backend() + option.enable_paddle_to_trt() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.sr.EDVR( + model_file, params_file, runtime_option=runtime_option) + +# 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] +capture = cv2.VideoCapture(args.video) +video_out_name = "output.mp4" +video_fps = capture.get(cv2.CAP_PROP_FPS) +video_frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) +# 注意导出模型时尺寸与原始输入的分辨一致比如：[1,2,3,180,320],经过4x超分后[1,3,720,1280](注意此处与PP-MSVSR不同) +# 所以导出模型相当重要 +out_width = 1280 +out_height = 720 +print(f"fps: {video_fps}\tframe_count: {video_frame_count}") +# Create VideoWriter for output +video_out_dir = "./" +video_out_path = os.path.join(video_out_dir, video_out_name) +fucc = cv2.VideoWriter_fourcc(*"mp4v") +video_out = cv2.VideoWriter(video_out_path, fucc, video_fps, + (out_width, out_height), True) +if not video_out.isOpened(): + print("create video writer failed!") +# Capture all frames and do inference +frame_id = 0 +imgs = [] +while capture.isOpened(): + ret, frame = capture.read() + if frame_id < args.frame_num and frame is not None: + imgs.append(frame) + frame_id += 1 + continue + # 始终保持imgs队列中具有frame_num帧 + imgs.pop(0) + imgs.append(frame) + frame_id += 1 + # 视频读取完毕退出 + if not ret: + break + results = model.predict(imgs) + for item in results: + # cv2.imshow("13", item) + # cv2.waitKey(30) + video_out.write(item) + print("Processing frame: ", frame_id) + frame_id += 1 +print("inference finished, output video saved at: ", video_out_path) +capture.release() +video_out.release() diff --git a/examples/vision/sr/ppmsvsr/README.md b/examples/vision/sr/ppmsvsr/README.md new file mode 100644 index 00000000000..944e6144627 --- /dev/null +++ b/examples/vision/sr/ppmsvsr/README.md @@ -0,0 +1,27 @@ +# PP-MSVSR模型部署 + +## 模型版本说明 + +- [PaddleGAN develop](https://github.com/PaddlePaddle/PaddleGAN) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [PP-MSVSR](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +## 导出部署模型 + +在部署前，需要先将训练好的PP-MSVSR导出成部署模型，导出PP-MSVSR导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md)。 + + +| 模型 | 参数大小 | 精度 | 备注 | +|:----------------------------------------------------------------------------|:------|:----- | :------ | +| [PP-MSVSR](https://bj.bcebos.com/paddlehub/fastdeploy/PP-MSVSR_reds_x4.tgz) | 8.8MB | - | + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt b/examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt new file mode 100644 index 00000000000..93540a7e83e --- /dev/null +++ b/examples/vision/sr/ppmsvsr/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/sr/ppmsvsr/cpp/README.md b/examples/vision/sr/ppmsvsr/cpp/README.md new file mode 100644 index 00000000000..2130ffe7e81 --- /dev/null +++ b/examples/vision/sr/ppmsvsr/cpp/README.md @@ -0,0 +1,75 @@ +# VSR C++部署示例 + +本目录下提供`infer.cc`快速完成PP-MSVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. 根据开发环境，下载预编译部署库和samples代码，参考[FastDeploy预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +以Linux上 PP-MSVSR 推理为例，在本目录执行如下命令即可完成编译测试（如若只需在CPU上部署，可在[Fastdeploy C++预编译库](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md/CPP_prebuilt_libraries.md)下载CPU推理库） + +```bash +#下载SDK，编译模型examples代码（SDK中包含了examples代码） +# fastdeploy版本 >= 0.7.0 +wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.7.0.tgz +tar xvf fastdeploy-linux-x64-gpu-0.7.0.tgz +cd fastdeploy-linux-x64-gpu-0.7.0/examples/vision/sr/ppmsvsr/cpp/ +mkdir build && cd build +cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.7.0 +make -j + +# 下载PP-MSVSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP-MSVSR_reds_x4.tar +tar -xvf PP-MSVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 + + +# CPU推理 +./infer_demo PP-MSVSR_reds_x4 vsr_src.mp4 0 2 +# GPU推理 +./infer_demo PP-MSVSR_reds_x4 vsr_src.mp4 1 2 +# GPU上TensorRT推理 +./infer_demo PP-MSVSR_reds_x4 vsr_src.mp4 2 2 +``` + +以上命令只适用于Linux或MacOS, Windows下SDK的使用方式请参考: +- [如何在Windows中使用FastDeploy C++ SDK](../../../../../docs/cn/faq/use_sdk_on_windows.md) + +## PP-MSVSR C++接口 + +### PPMSVSR类 + +```c++ +fastdeploy::vision::sr::PPMSVSR( + const string& model_file, + const string& params_file = "", + const RuntimeOption& runtime_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE) +``` + +PP-MSVSR模型加载和初始化，其中model_file为导出的Paddle模型格式。 + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +#### Predict函数 + +> ```c++ +> PPMSVSR::Predict(std::vector& imgs, std::vector& results) +> ``` +> +> 模型预测接口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **imgs**: 输入视频帧序列，注意需为HWC，BGR格式 +> > * **results**: 视频超分结果，超分后的视频帧序列 + +- [模型介绍](../../) +- [Python部署](../python) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/ppmsvsr/cpp/infer.cc b/examples/vision/sr/ppmsvsr/cpp/infer.cc new file mode 100644 index 00000000000..2267b83a82d --- /dev/null +++ b/examples/vision/sr/ppmsvsr/cpp/infer.cc @@ -0,0 +1,298 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto model = fastdeploy::vision::sr::PPMSVSR(model_file, params_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void GpuInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + + auto option = fastdeploy::RuntimeOption(); + // use paddle-TRT + option.UseGpu(); + option.UseTrtBackend(); + option.EnablePaddleToTrt(); + auto model = fastdeploy::vision::sr::PPMSVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +void TrtInfer(const std::string& model_dir, + const std::string& video_file, int frame_num) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + auto model = fastdeploy::vision::sr::PPMSVSR( + model_file, params_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + // note: input/output shape is [b, n, c, h, w] (n = frame_nums; b=1(default)) + // b and n is dependent on export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + cv::VideoCapture capture; + // change your save video path + std::string video_out_name = "output.mp4"; + capture.open(video_file); + if (!capture.isOpened()) + { + std::cout<<"can not open video "<(capture.get(cv::CAP_PROP_FPS)); + int video_frame_count = static_cast(capture.get(cv::CAP_PROP_FRAME_COUNT)); + // Set fixed size for output frame, only for msvsr model + //Note that the resolution between the size and the original input is consistent when the model is exported, + // for example: [1,2,3,180,320], after 4x super separation [1,2,3,720,1080]. + //Therefore, it is very important to derive the model + int out_width = 1280; + int out_height = 720; + std::cout << "fps: " << video_fps << "\tframe_count: " << video_frame_count << std::endl; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path("./"); + video_out_path += video_out_name; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path, fcc, video_fps, cv::Size(out_width, out_height), true); + if (!video_out.isOpened()) + { + std::cout << "create video writer failed!" << std::endl; + return; + } + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + bool reach_end = false; + while (capture.isOpened()) + { + std::vector imgs; + for (int i = 0; i < frame_num; i++) + { + capture.read(frame); + if (!frame.empty()) + { + imgs.push_back(frame); + }else{ + reach_end = true; + } + } + if (reach_end) + { + break; + } + std::vector results; + model.Predict(imgs, results); + for (auto &item : results) + { + // cv::imshow("13",item); + // cv::waitKey(30); + video_out.write(item); + std::cout << "Processing frame: "<< frame_id << std::endl; + frame_id += 1; + } + } + std::cout << "inference finished, output video saved at " << video_out_path << std::endl; + capture.release(); + video_out.release(); +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/video frame number run_option, " + "e.g ./infer_model ./vsr_model_dir ./person.mp4 0 2" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + int frame_num = 2; + if (argc == 5) { + frame_num = std::atoi(argv[4]); + } + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2], frame_num); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2], frame_num); + } + return 0; +} diff --git a/examples/vision/sr/ppmsvsr/python/README.md b/examples/vision/sr/ppmsvsr/python/README.md new file mode 100644 index 00000000000..8e4c736ae22 --- /dev/null +++ b/examples/vision/sr/ppmsvsr/python/README.md @@ -0,0 +1,61 @@ +# PP-MSVSR Python部署示例 + +在部署前，需确认以下两个步骤 + +- 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) +- 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) + +本目录下提供`infer.py`快速完成PP-MSVSR在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 + +```bash +#下载部署示例代码 +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy/examples/vision/sr/ppmsvsr/python + +# 下载VSR模型文件和测试视频 +wget https://bj.bcebos.com/paddlehub/fastdeploy/PP-MSVSR_reds_x4.tar +tar -xvf PP-MSVSR_reds_x4.tar +wget https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4 +# CPU推理 +python infer.py --model PP-MSVSR_reds_x4 --video person.mp4 --frame_num 2 --device cpu +# GPU推理 +python infer.py --model PP-MSVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu +# GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） +python infer.py --model PP-MSVSR_reds_x4 --video person.mp4 --frame_num 2 --device gpu --use_trt True +``` + +## VSR Python接口 + +```python +fd.vision.sr.PPMSVSR(model_file, params_file, runtime_option=None, model_format=ModelFormat.PADDLE) +``` + +PP-MSVSR模型加载和初始化，其中model_file和params_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md) + +**参数** + +> * **model_file**(str): 模型文件路径 +> * **params_file**(str): 参数文件路径 +> * **runtime_option**(RuntimeOption): 后端推理配置，默认为None，即采用默认配置 +> * **model_format**(ModelFormat): 模型格式，默认为Paddle格式 + +### predict函数 + +> ```python +> PPMSVSR.predict(frames) +> ``` +> +> 模型预测结口，输入图像直接输出检测结果。 +> +> **参数** +> +> > * **frames**(list[np.ndarray]): 输入数据，注意需为HWC，BGR格式, frames为视频帧序列 + +> **返回** list[np.ndarray] 为超分后的视频帧序列 + + +## 其它文档 + +- [PP-MSVSR 模型介绍](..) +- [PP-MSVSR C++部署](../cpp) +- [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) diff --git a/examples/vision/sr/ppmsvsr/python/infer.py b/examples/vision/sr/ppmsvsr/python/infer.py new file mode 100644 index 00000000000..de7ab851ece --- /dev/null +++ b/examples/vision/sr/ppmsvsr/python/infer.py @@ -0,0 +1,86 @@ +import cv2 +import os +import fastdeploy as fd + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True, help="Path of model.") + parser.add_argument( + "--video", type=str, required=True, help="Path of test video file.") + parser.add_argument("--frame_num", type=int, default=2, help="frame num") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + if args.device.lower() == "gpu": + option.use_gpu() + if args.use_trt: + option.use_trt_backend() + option.enable_paddle_to_trt() + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +model = fd.vision.sr.PPMSVSR( + model_file, params_file, runtime_option=runtime_option) + +# 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] +capture = cv2.VideoCapture(args.video) +video_out_name = "output.mp4" +video_fps = capture.get(cv2.CAP_PROP_FPS) +video_frame_count = capture.get(cv2.CAP_PROP_FRAME_COUNT) +# 注意导出模型时尺寸与原始输入的分辨一致比如：[1,2,3,180,320],经过4x超分后[1,2,3,720,1280] +# 所以导出模型相当重要 +out_width = 1280 +out_height = 720 +print(f"fps: {video_fps}\tframe_count: {video_frame_count}") +# Create VideoWriter for output +video_out_dir = "./" +video_out_path = os.path.join(video_out_dir, video_out_name) +fucc = cv2.VideoWriter_fourcc(*"mp4v") +video_out = cv2.VideoWriter(video_out_path, fucc, video_fps, + (out_width, out_height), True) +if not video_out.isOpened(): + print("create video writer failed!") +# Capture all frames and do inference +frame_id = 0 +reach_end = False +while capture.isOpened(): + imgs = [] + for i in range(args.frame_num): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + # cv2.imshow("13", item) + # cv2.waitKey(30) + video_out.write(item) + print("Processing frame: ", frame_id) + frame_id += 1 +print("inference finished, output video saved at: ", video_out_path) +capture.release() +video_out.release() diff --git a/examples/vision/tracking/pptracking/cpp/infer.cc b/examples/vision/tracking/pptracking/cpp/infer.cc index 58b4d4b6155..0d47e386bd0 100644 --- a/examples/vision/tracking/pptracking/cpp/infer.cc +++ b/examples/vision/tracking/pptracking/cpp/infer.cc @@ -48,7 +48,7 @@ void CpuInfer(const std::string& model_dir, const std::string& video_file) { std::cerr << "Failed to predict." << std::endl; return; } - // such as adding this code can cancel trail datat bind + // such as adding this code can cancel trail data binding // if(count++ == 10) model.UnbindRecorder(); // std::cout << result.Str() << std::endl; cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, 0.0, &recorder); @@ -91,7 +91,7 @@ void GpuInfer(const std::string& model_dir, const std::string& video_file) { std::cerr << "Failed to predict." << std::endl; return; } - // such as adding this code can cancel trail datat bind + // such as adding this code can cancel trail data binding //if(count++ == 10) model.UnbindRecorder(); // std::cout << result.Str() << std::endl; cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, 0.0, &trail_recorder); @@ -135,7 +135,7 @@ void TrtInfer(const std::string& model_dir, const std::string& video_file) { std::cerr << "Failed to predict." << std::endl; return; } - // such as adding this code can cancel trail datat bind + // such as adding this code can cancel trail data binding // if(count++ == 10) model.UnbindRecorder(); // std::cout << result.Str() << std::endl; cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, 0.0, &recorder); diff --git a/examples/vision/tracking/pptracking/python/README.md b/examples/vision/tracking/pptracking/python/README.md index d3b943759d5..48f8300c9ec 100644 --- a/examples/vision/tracking/pptracking/python/README.md +++ b/examples/vision/tracking/pptracking/python/README.md @@ -30,7 +30,7 @@ python infer.py --model fairmot_hrnetv2_w18_dlafpn_30e_576x320 --video person.mp fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=None, model_format=ModelFormat.PADDLE) ``` -PP-Tracking模型加载和初始化，其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) +PP-Tracking模型加载和初始化，其中model_file, params_file以及config_file为训练模型导出的Paddle inference文件，具体请参考其文档说明[模型导出](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/pptracking/cpp/README.md) **参数** @@ -42,7 +42,7 @@ PP-Tracking模型加载和初始化，其中model_file, params_file以及config_ ### predict函数 -> ```python +> ```python > PPTracking.predict(frame) > ``` > diff --git a/examples/vision/tracking/pptracking/python/infer.py b/examples/vision/tracking/pptracking/python/infer.py index 378d89bc145..768509ab82b 100644 --- a/examples/vision/tracking/pptracking/python/infer.py +++ b/examples/vision/tracking/pptracking/python/infer.py @@ -21,8 +21,7 @@ def parse_arguments(): import argparse import ast parser = argparse.ArgumentParser() - parser.add_argument( - "--model", required=True, help="Path of PaddleSeg model.") + parser.add_argument("--model", required=True, help="Path of model.") parser.add_argument( "--video", type=str, required=True, help="Path of test video file.") parser.add_argument( diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index 581d3b91ae5..00a51cefb51 100755 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -55,6 +55,7 @@ #include "fastdeploy/vision/segmentation/ppseg/model.h" #include "fastdeploy/vision/tracking/pptracking/model.h" #include "fastdeploy/vision/headpose/contrib/fsanet.h" +#include "fastdeploy/vision/sr/ppsr/model.h" #endif #include "fastdeploy/vision/visualize/visualize.h" diff --git a/fastdeploy/vision/sr/ppsr/basicvsr.cc b/fastdeploy/vision/sr/ppsr/basicvsr.cc new file mode 100644 index 00000000000..6f1691cc19a --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/basicvsr.cc @@ -0,0 +1,38 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/sr/ppsr/basicvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +BasicVSR::BasicVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + // unsupported ORT backend + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + initialized = Initialize(); +} +} // namespace sr +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/sr/ppsr/basicvsr.h b/fastdeploy/vision/sr/ppsr/basicvsr.h new file mode 100644 index 00000000000..976372121d9 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/basicvsr.h @@ -0,0 +1,41 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +class FASTDEPLOY_DECL BasicVSR : public PPMSVSR{ + public: + /** + * Set path of model file and configuration file, and the configuration of runtime + * @param[in] model_file Path of model file, e.g BasicVSR/model.pdmodel + * @param[in] params_file Path of parameter file, e.g BasicVSR/model.pdiparams + * @param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * @param[in] model_format Model format of the loaded model, default is Paddle format + */ + BasicVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// model name contained BasicVSR + std::string ModelName() const override { return "BasicVSR"; } +}; + +} // namespace sr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/ppsr/edvr.cc b/fastdeploy/vision/sr/ppsr/edvr.cc new file mode 100644 index 00000000000..f9bb4e0cc19 --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/edvr.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/sr/ppsr/edvr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +EDVR::EDVR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + // unsupported ORT backend + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + initialized = Initialize(); +} + +bool EDVR::Postprocess(std::vector& infer_results, std::vector& results){ + // group to image + // output_shape is [b, n, c, h, w] n = frame_nums b=1(default) + // b and n is dependence export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + auto output_shape = infer_results[0].shape; + // EDVR + int h_ = output_shape[2]; + int w_ = output_shape[3]; + int c_ = output_shape[1]; + int frame_num = 1; + float *out_data = static_cast(infer_results[0].Data()); + cv::Mat temp = cv::Mat::zeros(h_, w_, CV_32FC3); // RGB image + int pix_num = h_ * w_; + int frame_pix_num = pix_num * c_; + for (int frame = 0; frame < frame_num; frame++) { + int index = 0; + for (int h = 0; h < h_; ++h) { + for (int w = 0; w < w_; ++w) { + temp.at(h, w) = {out_data[2 * pix_num + index + frame_pix_num * frame], + out_data[pix_num + index + frame_pix_num * frame], + out_data[index + frame_pix_num * frame]}; + index += 1; + } + } + // tmp data type is float[0-1.0],convert to uint type + cv::Mat res = cv::Mat::zeros(temp.size(), CV_8UC3); + temp.convertTo(res, CV_8UC3, 255); + results.push_back(res); + } + return true; +} +} // namespace sr +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/sr/ppsr/edvr.h b/fastdeploy/vision/sr/ppsr/edvr.h new file mode 100644 index 00000000000..1e460c0286a --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/edvr.h @@ -0,0 +1,45 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + + +class FASTDEPLOY_DECL EDVR : public PPMSVSR{ + public: + /** + * Set path of model file and configuration file, and the configuration of runtime + * @param[in] model_file Path of model file, e.g EDVR/model.pdmodel + * @param[in] params_file Path of parameter file, e.g EDVR/model.pdiparams + * @param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * @param[in] model_format Model format of the loaded model, default is Paddle format + */ + EDVR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// model name contained EDVR + std::string ModelName() const override { return "EDVR"; } + + private: + bool Postprocess(std::vector& infer_results, + std::vector& results) override; +}; +} // namespace sr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/ppsr/model.h b/fastdeploy/vision/sr/ppsr/model.h new file mode 100644 index 00000000000..1d53983358c --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/model.h @@ -0,0 +1,18 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" +#include "fastdeploy/vision/sr/ppsr/edvr.h" +#include "fastdeploy/vision/sr/ppsr/basicvsr.h" diff --git a/fastdeploy/vision/sr/ppsr/ppmsvsr.cc b/fastdeploy/vision/sr/ppsr/ppmsvsr.cc new file mode 100644 index 00000000000..888f7609add --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/ppmsvsr.cc @@ -0,0 +1,128 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/sr/ppsr/ppmsvsr.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + +PPMSVSR::PPMSVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + // unsupported ORT backend + valid_cpu_backends = {Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER}; + + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + initialized = Initialize(); +} + +bool PPMSVSR::Initialize(){ + if (!InitRuntime()) { + FDERROR << "Failed to initialize fastdeploy backend." << std::endl; + return false; + } + mean_ = {0., 0., 0.}; + scale_ = {1., 1., 1.}; + return true; +} + +bool PPMSVSR::Preprocess(Mat* mat, std::vector& output) { + + BGR2RGB::Run(mat); + Normalize::Run(mat, mean_, scale_, true); + HWC2CHW::Run(mat); + // Csat float + float* ptr = static_cast(mat->Data()); + size_t size = mat->Width() * mat->Height() * mat->Channels(); + output = std::vector(ptr, ptr + size); + return true; +} + +bool PPMSVSR::Predict(std::vector& imgs, std::vector& results) { + + // Theoretically, the more frame nums there are, the better the result will be, + // but it will lead to a significant increase in memory + int frame_num = imgs.size(); + int rows = imgs[0].rows; + int cols = imgs[0].cols; + int channels = imgs[0].channels(); + std::vector input_tensors; + input_tensors.resize(1); + std::vector all_data_temp; + for (int i = 0; i < frame_num; i++) { + Mat mat(imgs[i]); + std::vector data_temp; + Preprocess(&mat, data_temp); + all_data_temp.insert(all_data_temp.end(), data_temp.begin(), data_temp.end()); + } + // share memory in order to avoid memory copy, data type must be float32 + input_tensors[0].SetExternalData({1 ,frame_num , channels, rows, cols}, FDDataType::FP32, + all_data_temp.data()); + input_tensors[0].shape = {1, frame_num, channels, rows, cols}; + input_tensors[0].name = InputInfoOfRuntime(0).name; + std::vector output_tensors; + if (!Infer(input_tensors, &output_tensors)) { + FDERROR << "Failed to inference." << std::endl; + return false; + } + if (!Postprocess(output_tensors, results)) { + FDERROR << "Failed to post process." << std::endl; + return false; + } + return true; +} + +bool PPMSVSR::Postprocess(std::vector& infer_results, std::vector& results){ + // group to image + // output_shape is [b, n, c, h, w] n = frame_nums b=1(default) + // b and n is dependence export model shape + // see https://github.com/PaddlePaddle/PaddleGAN/blob/develop/docs/zh_CN/tutorials/video_super_resolution.md + auto output_shape = infer_results[0].shape; + // PP-MSVSR + int h_ = output_shape[3]; + int w_ = output_shape[4]; + int c_ = output_shape[2]; + int frame_num = output_shape[1]; + + float *out_data = static_cast(infer_results[0].Data()); + cv::Mat temp = cv::Mat::zeros(h_, w_, CV_32FC3); // RGB image + int pix_num = h_ * w_; + int frame_pix_num = pix_num * c_; + for (int frame = 0; frame < frame_num; frame++) { + int index = 0; + for (int h = 0; h < h_; ++h) { + for (int w = 0; w < w_; ++w) { + temp.at(h, w) = {out_data[2 * pix_num + index + frame_pix_num * frame], + out_data[pix_num + index + frame_pix_num * frame], + out_data[index + frame_pix_num * frame]}; + index += 1; + } + } + // tmp data type is float[0-1.0],convert to uint type + cv::Mat res = cv::Mat::zeros(temp.size(), CV_8UC3); + temp.convertTo(res, CV_8UC3, 255); + results.push_back(res); + } + return true; +} +} // namespace sr +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/sr/ppsr/ppmsvsr.h b/fastdeploy/vision/sr/ppsr/ppmsvsr.h new file mode 100644 index 00000000000..49cdfd03f0d --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/ppmsvsr.h @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/fastdeploy_model.h" + +namespace fastdeploy { +namespace vision { +namespace sr { + + +class FASTDEPLOY_DECL PPMSVSR:public FastDeployModel{ + public: + /** + * Set path of model file and configuration file, and the configuration of runtime + * @param[in] model_file Path of model file, e.g PPMSVSR/model.pdmodel + * @param[in] params_file Path of parameter file, e.g PPMSVSR/model.pdiparams + * @param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * @param[in] model_format Model format of the loaded model, default is Paddle format + */ + PPMSVSR(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + /// model name contained PP-MSVSR。 + std::string ModelName() const override { return "PPMSVSR"; } + /** + * get super resolution frame sequence + * @param[in] imgs origin frame sequences + * @param[in] results super resolution frame sequence + * @return true if the prediction successed, otherwise false + */ + virtual bool Predict(std::vector& imgs, + std::vector& results); + + protected: + PPMSVSR(){}; + + virtual bool Initialize(); + + virtual bool Preprocess(Mat* mat, std::vector& output); + + virtual bool Postprocess(std::vector& infer_results, + std::vector& results); + + std::vector mean_; + std::vector scale_; +}; +} // namespace sr +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/ppsr/ppsr_pybind.cc b/fastdeploy/vision/sr/ppsr/ppsr_pybind.cc new file mode 100644 index 00000000000..5688281c6bf --- /dev/null +++ b/fastdeploy/vision/sr/ppsr/ppsr_pybind.cc @@ -0,0 +1,70 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy{ +void BindPPSR(pybind11::module &m) { + pybind11::class_(m, "PPMSVSR") + .def(pybind11::init()) + .def("predict", [](vision::sr::PPMSVSR& self, std::vector& datas){ + std::vector inputs; + for (auto& data: datas){ + auto mat = PyArrayToCvMat(data); + inputs.push_back(mat); + } + std::vector res; + std::vector res_pyarray; + self.Predict(inputs, res); + for (auto& img: res){ + auto ret = pybind11::array_t({img.rows, img.cols, img.channels()}, img.data); + res_pyarray.push_back(ret); + } + return res_pyarray; + }); + pybind11::class_(m, "EDVR") + .def(pybind11::init()) + .def("predict", [](vision::sr::EDVR& self, std::vector& datas){ + std::vector inputs; + for (auto& data: datas){ + auto mat = PyArrayToCvMat(data); + inputs.push_back(mat); + } + std::vector res; + std::vector res_pyarray; + self.Predict(inputs, res); + for (auto& img: res){ + auto ret = pybind11::array_t({img.rows, img.cols, img.channels()}, img.data); + res_pyarray.push_back(ret); + } + return res_pyarray; + }); + pybind11::class_(m, "BasicVSR") + .def(pybind11::init()) + .def("predict", [](vision::sr::BasicVSR& self, std::vector& datas){ + std::vector inputs; + for (auto& data: datas){ + auto mat = PyArrayToCvMat(data); + inputs.push_back(mat); + } + std::vector res; + std::vector res_pyarray; + self.Predict(inputs, res); + for (auto& img: res){ + auto ret = pybind11::array_t({img.rows, img.cols, img.channels()}, img.data); + res_pyarray.push_back(ret); + } + return res_pyarray; + }); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/sr/sr_pybind.cc b/fastdeploy/vision/sr/sr_pybind.cc new file mode 100644 index 00000000000..03be14df373 --- /dev/null +++ b/fastdeploy/vision/sr/sr_pybind.cc @@ -0,0 +1,25 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { + + void BindPPSR(pybind11::module& m); + + void BindSR(pybind11::module& m) { + auto sr_module = m.def_submodule("sr", "sr(super resolution) submodule"); + BindPPSR(sr_module); + } +} // namespace fastdeploy diff --git a/fastdeploy/vision/vision_pybind.cc b/fastdeploy/vision/vision_pybind.cc index a1fc6cac683..c2cacae5c7f 100755 --- a/fastdeploy/vision/vision_pybind.cc +++ b/fastdeploy/vision/vision_pybind.cc @@ -27,6 +27,7 @@ void BindOcr(pybind11::module& m); void BindTracking(pybind11::module& m); void BindKeyPointDetection(pybind11::module& m); void BindHeadPose(pybind11::module& m); +void BindSR(pybind11::module& m); #ifdef ENABLE_VISION_VISUALIZE void BindVisualize(pybind11::module& m); #endif @@ -142,6 +143,7 @@ void BindVision(pybind11::module& m) { BindTracking(m); BindKeyPointDetection(m); BindHeadPose(m); + BindSR(m); #ifdef ENABLE_VISION_VISUALIZE BindVisualize(m); #endif diff --git a/python/fastdeploy/vision/__init__.py b/python/fastdeploy/vision/__init__.py index 047591ccde2..a5531a8a908 100755 --- a/python/fastdeploy/vision/__init__.py +++ b/python/fastdeploy/vision/__init__.py @@ -24,6 +24,7 @@ from . import faceid from . import ocr from . import headpose +from . import sr from . import evaluation from .utils import fd_result_to_json from .visualize import * diff --git a/python/fastdeploy/vision/sr/__init__.py b/python/fastdeploy/vision/sr/__init__.py new file mode 100644 index 00000000000..d056cce1179 --- /dev/null +++ b/python/fastdeploy/vision/sr/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from .ppsr import PPMSVSR, EDVR, BasicVSR diff --git a/python/fastdeploy/vision/sr/ppsr/__init__.py b/python/fastdeploy/vision/sr/ppsr/__init__.py new file mode 100644 index 00000000000..38e462d067c --- /dev/null +++ b/python/fastdeploy/vision/sr/ppsr/__init__.py @@ -0,0 +1,107 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class PPMSVSR(FastDeployModel): + def __init__(self, + model_file, + params_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a VSR model exported by PaddleGAN. + + :param model_file: (str)Path of model file, e.g PPMSVSR/inference.pdmodel + :param params_file: (str)Path of parameters file, e.g PPMSVSR/inference.pdiparams + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPMSVSR, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "PPMSVSR model only support model format of ModelFormat.Paddle now." + self._model = C.vision.sr.PPMSVSR(model_file, params_file, + self._runtime_option, model_format) + assert self.initialized, "PPMSVSR model initialize failed." + + def predict(self, input_images): + """Predict the super resolution frame sequences for an input frame sequences + + :param input_images: list[numpy.ndarray] The input image data, 3-D array with layout HWC, BGR format + :return: list[numpy.ndarray] + """ + assert input_images is not None, "The input image data is None." + return self._model.predict(input_images) + + +class EDVR(PPMSVSR): + def __init__(self, + model_file, + params_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a EDVR model exported by PaddleGAN. + + :param model_file: (str)Path of model file, e.g EDVR/inference.pdmodel + :param params_file: (str)Path of parameters file, e.g EDVR/inference.pdiparams + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPMSVSR, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "EDVR model only support model format of ModelFormat.Paddle now." + self._model = C.vision.sr.EDVR(model_file, params_file, + self._runtime_option, model_format) + assert self.initialized, "EDVR model initialize failed." + + def predict(self, input_images): + """Predict the super resolution frame sequences for an input frame sequences + + :param input_images: list[numpy.ndarray] The input image data, 3-D array with layout HWC, BGR format + :return: list[numpy.ndarray] + """ + assert input_images is not None, "The input image data is None." + return self._model.predict(input_images) + + +class BasicVSR(PPMSVSR): + def __init__(self, + model_file, + params_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + """Load a EDVR model exported by PaddleGAN. + + :param model_file: (str)Path of model file, e.g BasicVSR/inference.pdmodel + :param params_file: (str)Path of parameters file, e.g BasicVSR/inference.pdiparams + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ + super(PPMSVSR, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "BasicVSR model only support model format of ModelFormat.Paddle now." + self._model = C.vision.sr.BasicVSR(model_file, params_file, + self._runtime_option, model_format) + assert self.initialized, "BasicVSR model initialize failed." + + def predict(self, input_images): + """Predict the super resolution frame sequences for an input frame sequences + + :param input_images: list[numpy.ndarray] The input image data, 3-D array with layout HWC, BGR format + :return: list[numpy.ndarray] + """ + assert input_images is not None, "The input image data is None." + return self._model.predict(input_images) diff --git a/tests/models/test_basicvsr.py b/tests/models/test_basicvsr.py new file mode 100644 index 00000000000..479343444e3 --- /dev/null +++ b/tests/models/test_basicvsr.py @@ -0,0 +1,71 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np +import pickle +import runtime_config as rc + + +def test_basicvsr(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/basicvsr.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/basicvsr/BasicVSR_reds_x4" + # use default backend + runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + model = fd.vision.sr.PPMSVSR( + model_file, params_file, runtime_option=rc.test_option) + # 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] + capture = cv2.VideoCapture("./resources/vsr_src.mp4") + frame_id = 0 + reach_end = False + t = 0 + while capture.isOpened(): + imgs = [] + for i in range(2): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + if t < 10: + ret = pickle.load( + open("./resources/basicvsr/frame_" + str(t) + ".pkl", + "rb")) + mean_diff = np.fabs(ret["mean"] - item.mean()) + std_diff = np.fabs(ret["std"] - item.std()) + shape_diff = max( + np.fabs(np.array(ret["shape"]) - np.array(item.shape))) + thres = 1e-02 + assert mean_diff < thres, "The mean diff is %f, which is bigger than %f" % ( + mean_diff, thres) + assert std_diff < thres, "The std diff is %f, which is bigger than %f" % ( + std_diff, thres) + assert shape_diff <= 0, "The shape diff is %f, which is bigger than %f" % ( + shape_diff, 0) + t = t + 1 + frame_id += 1 + if t >= 10: + break + capture.release() diff --git a/tests/models/test_edvr.py b/tests/models/test_edvr.py new file mode 100644 index 00000000000..a9f9517e7d6 --- /dev/null +++ b/tests/models/test_edvr.py @@ -0,0 +1,76 @@ +test_pptracking.py # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np +import pickle +import runtime_config as rc + + +def test_edvr(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/edvr.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/edvr/EDVR_M_wo_tsa_SRx4" + # use default backend + runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + model = fd.vision.sr.EDVR( + model_file, params_file, runtime_option=rc.test_option) + + # 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] + capture = cv2.VideoCapture("./resources/vsr_src.mp4") + # Create VideoWriter for output + frame_id = 0 + imgs = [] + t = 0 + while capture.isOpened(): + ret, frame = capture.read() + if frame_id < 5 and frame is not None: + imgs.append(frame) + frame_id += 1 + continue + # 始终保持imgs队列中具有frame_num帧 + imgs.pop(0) + imgs.append(frame) + frame_id += 1 + # 视频读取完毕退出 + if not ret: + break + results = model.predict(imgs) + for item in results: + if frame_id <= 4: + continue + if t < 10: + ret = pickle.load( + open("./resources/edvr/frame_" + str(t) + ".pkl", "rb")) + mean_diff = np.fabs(ret["mean"] - item.mean()) + std_diff = np.fabs(ret["std"] - item.std()) + shape_diff = max( + np.fabs(np.array(ret["shape"]) - np.array(item.shape))) + thres = 1e-03 + assert mean_diff < thres, "The mean diff is %f, which is bigger than %f" % ( + mean_diff, thres) + assert std_diff < thres, "The std diff is %f, which is bigger than %f" % ( + std_diff, thres) + assert shape_diff <= 0, "The shape diff is %f, which is bigger than %f" % ( + shape_diff, 0) + t = t + 1 + if t >= 10: + break + capture.release() diff --git a/tests/models/test_ppmsvsr.py b/tests/models/test_ppmsvsr.py new file mode 100644 index 00000000000..09cfe458587 --- /dev/null +++ b/tests/models/test_ppmsvsr.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np +import pickle +import runtime_config as rc + + +def test_ppmsvsr(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/ppmsvsr.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/vsr_src.mp4" + fd.download_and_decompress(model_url, "resources") + fd.download(input_url, "resources") + model_path = "resources/ppmsvsr/MSVSR_reds_x4" + # use default backend + # runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + model = fd.vision.sr.PPMSVSR( + model_file, params_file, runtime_option=rc.test_option) + # 该处应该与你导出模型的第二个维度一致模型输入shape=[b,n,c,h,w] + capture = cv2.VideoCapture("./resources/vsr_src.mp4") + frame_id = 0 + reach_end = False + t = 0 + while capture.isOpened(): + imgs = [] + for i in range(2): + _, frame = capture.read() + if frame is not None: + imgs.append(frame) + else: + reach_end = True + if reach_end: + break + results = model.predict(imgs) + for item in results: + if t < 10: + ret = pickle.load( + open("./resources/ppmsvsr/frame_" + str(t) + ".pkl", "rb")) + mean_diff = np.fabs(ret["mean"] - item.mean()) + std_diff = np.fabs(ret["std"] - item.std()) + shape_diff = max( + np.fabs(np.array(ret["shape"]) - np.array(item.shape))) + thres = 1e-03 + assert mean_diff < thres, "The mean diff is %f, which is bigger than %f" % ( + mean_diff, thres) + assert std_diff < thres, "The std diff is %f, which is bigger than %f" % ( + std_diff, thres) + assert shape_diff <= 0, "The shape diff is %f, which is bigger than %f" % ( + shape_diff, 0) + t = t + 1 + frame_id += 1 + if t >= 10: + break + capture.release() From c3aefe0b27d8b5833b1ea0efb67d3971e135a453 Mon Sep 17 00:00:00 2001 From: WJJ1995 Date: Mon, 21 Nov 2022 11:01:28 +0800 Subject: [PATCH 073/136] [Tools] Add x2paddle tool (#631) * add onnx_ort_runtime demo * rm in requirements * support batch eval * fixed MattingResults bug * move assignment for DetectionResult * integrated x2paddle * add model convert readme * update readme * re-lint Co-authored-by: Jason --- .../download_prebuilt_libraries.md | 8 +- .../download_prebuilt_libraries.md | 8 +- tools/README.md | 47 +++++++++- tools/README_EN.md | 46 ++++++++- tools/common_tools/common_tools.py | 93 +++++++++++++++++-- 5 files changed, 186 insertions(+), 16 deletions(-) mode change 100644 => 100755 tools/README.md mode change 100644 => 100755 tools/README_EN.md mode change 100644 => 100755 tools/common_tools/common_tools.py diff --git a/docs/cn/build_and_install/download_prebuilt_libraries.md b/docs/cn/build_and_install/download_prebuilt_libraries.md index f296363e6b7..105e0f0e713 100755 --- a/docs/cn/build_and_install/download_prebuilt_libraries.md +++ b/docs/cn/build_and_install/download_prebuilt_libraries.md @@ -3,8 +3,10 @@ FastDeploy提供各平台预编译库，供开发者直接下载安装使用。当然FastDeploy编译也非常容易，开发者也可根据自身需求编译FastDeploy。 本文分为两部分： -- [1.GPU部署环境](##GPU部署环境) -- [2.CPU部署环境](##CPU部署环境) +- [1.GPU部署环境](#1) +- [2.CPU部署环境](#2) + +

## GPU部署环境 @@ -49,6 +51,8 @@ Develop版本（Nightly build） | Linux x64 | [fastdeploy-linux-x64-gpu-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-gpu-0.0.0.tgz) | g++ 8.2, CUDA 11.2, cuDNN 8.2编译产出 | | Windows x64 | [fastdeploy-win-x64-gpu-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-gpu-0.0.0.zip) | Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2编译产出 | +

+ ## CPU部署环境 ### 环境要求 diff --git a/docs/en/build_and_install/download_prebuilt_libraries.md b/docs/en/build_and_install/download_prebuilt_libraries.md index 565d39d93d9..f7f161dfe68 100755 --- a/docs/en/build_and_install/download_prebuilt_libraries.md +++ b/docs/en/build_and_install/download_prebuilt_libraries.md @@ -3,8 +3,10 @@ FastDeploy provides pre-built libraries for developers to download and install directly. Meanwhile, FastDeploy also offers easy access to compile so that developers can compile FastDeploy according to their own needs. This article is divided into two parts: -- [1.GPU Deployment Environment](#gpu-deployment-environment) -- [2.CPU Deployment Environment](#cpu-deployment-environment) +- [1.GPU Deployment Environment](#1) +- [2.CPU Deployment Environment](#2) + +

## GPU Deployment Environment @@ -53,6 +55,8 @@ Install the Develop version（Nightly build） | Linux x64 | [fastdeploy-linux-x64-gpu-0.0.0.tgz](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-linux-x64-gpu-0.0.0.tgz) | g++ 8.2, CUDA 11.2, cuDNN 8.2 | | Windows x64 | [fastdeploy-win-x64-gpu-0.0.0.zip](https://fastdeploy.bj.bcebos.com/dev/cpp/fastdeploy-win-x64-gpu-0.0.0.zip) | Visual Studio 16 2019, CUDA 11.2, cuDNN 8.2 | +

+ ## CPU Deployment Environment ### Environment Requirement diff --git a/tools/README.md b/tools/README.md old mode 100644 new mode 100755 index c166b8f9f09..7a1a36853b6 --- a/tools/README.md +++ b/tools/README.md @@ -1,8 +1,14 @@ # FastDeploy 工具包 FastDeploy提供了一系列高效易用的工具优化部署体验, 提升推理性能. -例如, FastDeploy基于PaddleSlim的Auto Compression Toolkit(ACT), 给用户提供了一键模型自动化压缩的工具, 用户可以轻松地通过一行命令对模型进行自动化压缩, 并在FastDeploy上部署压缩后的模型, 提升推理速度. 本文档将以FastDeploy一键模型自动化压缩工具为例, 介绍如何安装此工具, 并提供相应的使用文档. -## FastDeploy一键模型自动化压缩工具 +- [1.自动压缩工具包](#1) +- [2.模型转换工具包](#2) + +

+ +## 一键模型自动化压缩工具 + +FastDeploy基于PaddleSlim的Auto Compression Toolkit(ACT), 给用户提供了一键模型自动化压缩的工具, 用户可以轻松地通过一行命令对模型进行自动化压缩, 并在FastDeploy上部署压缩后的模型, 提升推理速度. 本文档将以FastDeploy一键模型自动化压缩工具为例, 介绍如何安装此工具, 并提供相应的使用文档. ### 环境准备 1.用户参考PaddlePaddle官网, 安装develop版本 @@ -33,3 +39,40 @@ python setup.py install fastdeploy --auto_compress --config_path=./configs/detection/yolov5s_quant.yaml --method='PTQ' --save_dir='./yolov5s_ptq_model/' ``` 详细使用文档请参考[FastDeploy一键模型自动化压缩工具](./auto_compression/README.md) + +

+ +## 模型转换工具 + +FastDeploy 基于 X2Paddle 为用户提供了模型转换的工具, 用户可以轻松地通过一行命令将外部框架模型快速迁移至飞桨框架，目前支持 ONNX、TensorFlow 以及 Caffe，支持大部分主流的CV和NLP的模型转换。 + +### 环境准备 + +1. PaddlePaddle 安装，可参考如下文档快速安装 +``` +https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/develop/install/pip/linux-pip.html +``` + +2. X2Paddle 安装 + +如需使用稳定版本，可通过pip方式安装X2Paddle： +```shell +pip install x2paddle +``` + +如需体验最新功能，可使用源码安装方式： +```shell +git clone https://github.com/PaddlePaddle/X2Paddle.git +cd X2Paddle +python setup.py install +``` + +### 使用方式 + +按照以上步骤成功安装后,即可使用 FastDeploy 一键转换工具, 示例如下: + +```bash +fastdeploy --convert --framework onnx --model yolov5s.onnx --save_dir pd_model +``` + +更多详细内容可参考[X2Paddle](https://github.com/PaddlePaddle/X2Paddle) diff --git a/tools/README_EN.md b/tools/README_EN.md old mode 100644 new mode 100755 index 3a8313fa9f5..26305ac9148 --- a/tools/README_EN.md +++ b/tools/README_EN.md @@ -1,9 +1,14 @@ # FastDeploy Toolkit FastDeploy provides a series of efficient and easy-to-use tools to optimize the deployment experience and improve inference performance. -For example, based on PaddleSlim's Auto Compression Toolkit (ACT), FastDeploy provides users with a one-click model automation compression tool that allows users to easily compress the model with a single command. This document will take FastDeploy's one-click model automation compression tool as an example, introduce how to install the tool, and provide the corresponding documentation for usage. +- [1.Auto Compression Tool](#1) +- [2.Model Conversion Tool](#2) -## FastDeploy One-Click Model Auto Compression Tool +

+ +## One-Click Model Auto Compression Tool + +Based on PaddleSlim's Auto Compression Toolkit (ACT), FastDeploy provides users with a one-click model automation compression tool that allows users to easily compress the model with a single command. This document will take FastDeploy's one-click model automation compression tool as an example, introduce how to install the tool, and provide the corresponding documentation for usage. ### Environmental Preparation 1.Install PaddlePaddle develop version @@ -33,3 +38,40 @@ After the above steps are successfully installed, you can use FastDeploy one-cli fastdeploy --auto_compress --config_path=./configs/detection/yolov5s_quant.yaml --method='PTQ' --save_dir='./yolov5s_ptq_model/' ``` For detailed documentation, please refer to [FastDeploy One-Click Model Auto Compression Tool](./auto_compression/README.md) + +

+ +## Model Conversion Tool + +Based on X2Paddle, FastDeploy provides users with a model conversion tool. Users can easily migrate external framework models to the Paddle framework with one line of commands. Currently, ONNX, TensorFlow and Caffe are supported, and most mainstream CV and NLP model conversions are supported. + +### Environmental Preparation + +1. Install PaddlePaddle, refer to the following documents for quick installation +``` +https://www.paddlepaddle.org.cn/install/quick?docurl=/documentation/docs/zh/develop/install/pip/linux-pip.html +``` + +2. Install X2Paddle + +To use the stable version, install X2Paddle via pip: +```shell +pip install x2paddle +``` + +To experience the latest features, you can use the source installation method: +```shell +git clone https://github.com/PaddlePaddle/X2Paddle.git +cd X2Paddle +python setup.py install +``` + +### How to use + +After successful installation according to the above steps, you can use the FastDeploy one-click conversion tool. The example is as follows: + +```bash +fastdeploy --convert --framework onnx --model yolov5s.onnx --save_dir pd_model +``` + +For more details, please refer to[X2Paddle](https://github.com/PaddlePaddle/X2Paddle) diff --git a/tools/common_tools/common_tools.py b/tools/common_tools/common_tools.py old mode 100644 new mode 100755 index 04fa215f72d..06d704481ae --- a/tools/common_tools/common_tools.py +++ b/tools/common_tools/common_tools.py @@ -1,8 +1,8 @@ import argparse +import ast def argsparser(): - parser = argparse.ArgumentParser(description=__doc__) ## argumentments for auto compression parser.add_argument('--auto_compress', default=False, action='store_true') @@ -10,31 +10,71 @@ def argsparser(): '--config_path', type=str, default=None, - help="path of compression strategy config.", - required=True) + help="path of compression strategy config.") parser.add_argument( '--method', type=str, default=None, - help="choose PTQ or QAT as quantization method", - required=True) + help="choose PTQ or QAT as quantization method") parser.add_argument( '--save_dir', type=str, default='./output', - help="directory to save compressed model.") + help="directory to save model.") parser.add_argument( '--devices', type=str, default='gpu', help="which device used to compress.") - + ## arguments for other x2paddle + parser.add_argument('--convert', default=False, action='store_true') + parser.add_argument( + '--framework', + type=str, + default=None, + help="define which deeplearning framework(tensorflow/caffe/onnx)") + parser.add_argument( + '--model', + type=str, + default=None, + help="define model file path for tensorflow or onnx") + parser.add_argument( + "--prototxt", + "-p", + type=str, + default=None, + help="prototxt file of caffe model") + parser.add_argument( + "--weight", + "-w", + type=str, + default=None, + help="weight file of caffe model") + parser.add_argument( + "--caffe_proto", + "-c", + type=str, + default=None, + help="optional: the .py file compiled by caffe proto file of caffe model" + ) + parser.add_argument( + "--input_shape_dict", + "-isd", + type=str, + default=None, + help="define input shapes, e.g --input_shape_dict=\"{'image':[1, 3, 608, 608]}\" or" \ + "--input_shape_dict=\"{'image':[1, 3, 608, 608], 'im_shape': [1, 2], 'scale_factor': [1, 2]}\"") + parser.add_argument( + "--enable_code_optim", + "-co", + type=ast.literal_eval, + default=False, + help="Turn on code optimization") ## arguments for other tools return parser def main(): - args = argsparser().parse_args() if args.auto_compress == True: try: @@ -45,6 +85,43 @@ def main(): print( "Can not start auto compresssion successfully! Please check if you have installed it!" ) + if args.convert == True: + try: + import platform + import logging + v0, v1, v2 = platform.python_version().split('.') + if not (int(v0) >= 3 and int(v1) >= 5): + logging.info("[ERROR] python>=3.5 is required") + return + import paddle + v0, v1, v2 = paddle.__version__.split('.') + logging.info("paddle.__version__ = {}".format(paddle.__version__)) + if v0 == '0' and v1 == '0' and v2 == '0': + logging.info( + "[WARNING] You are use develop version of paddlepaddle") + elif int(v0) != 2 or int(v1) < 0: + logging.info("[ERROR] paddlepaddle>=2.0.0 is required") + return + from x2paddle.convert import tf2paddle, caffe2paddle, onnx2paddle + if args.framework == "tensorflow": + assert args.model is not None, "--model should be defined while convert tensorflow model" + tf2paddle(args.model, args.save_dir) + elif args.framework == "caffe": + assert args.prototxt is not None and args.weight is not None, "--prototxt and --weight should be defined while convert caffe model" + caffe2paddle(args.prototxt, args.weight, args.save_dir, + args.caffe_proto) + elif args.framework == "onnx": + assert args.model is not None, "--model should be defined while convert onnx model" + onnx2paddle( + args.model, + args.save_dir, + input_shape_dict=args.input_shape_dict) + else: + raise Exception( + "--framework only support tensorflow/caffe/onnx now") + except ImportError: + print( + "Model convert failed! Please check if you have installed it!") if __name__ == '__main__': From b8bbe34e946710113be4d075446f148f43ee3ca0 Mon Sep 17 00:00:00 2001 From: leiqing <54695910+leiqing1@users.noreply.github.com> Date: Mon, 21 Nov 2022 11:23:36 +0800 Subject: [PATCH 074/136] Update README_EN.md --- README_EN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README_EN.md b/README_EN.md index d3f7f26d2e2..b2fc0353cd4 100644 --- a/README_EN.md +++ b/README_EN.md @@ -49,11 +49,11 @@ Including image classification, object detection, image segmentation, face detec - **🖥️ Server-side and Cloud Deployment: Support more CV models, improve deployment performance** - Support [PaddleClas](./examples/vision/classification/paddleclas/serving) model service-based deployment; - Support [Stable Diffusion](./examples/multimodal/stable_diffusion) model deployment; - - Upgrade PaddleClas、PaddleDetection、YOLOv5 deployment code to support predict and batch_predict; + - Upgrade PaddleClas、PaddleDetection、YOLOv5 deployment code to support `predict` and `batch_predict`; - Add the Pad function operator for the FDTensor to support Padding of the input during batch prediction; - Add Python API to_dlpack interface for FDTensor to support copyless transfer of FDTensor between frameworks. - **📱 Mobile and Edge Device Deployment: support new backend，support more CV model** - - Support Paddle Lite TIM-VX for supporting hardware such as Rockchip RV1109,RV1126, RK1808, Amlogic A311D, etc. And provide a seamless deployment experience with other inference engines include Paddle Inference、Paddle Inference TensorRT、Paddle Lite、TensorRT、OpenVINO、ONNX Runtime、RKNPU2; + - Integrate Paddle Lite TIM-VX for supporting hardware such as Rockchip RV1109,RV1126, RK1808, Amlogic A311D, etc. And provide a seamless deployment experience with other inference engines include Paddle Inference、Paddle Inference TensorRT、Paddle Lite、TensorRT、OpenVINO、ONNX Runtime、RKNPU2; - Support Image classification model [ResNet50_vd](./examples/vision/classification/paddleclas/rk1126/cpp) on Rockchip RV1126; - Support Face detection model [SCRFD](./examples/vision/facedet/scrfd/rknpu2) on Rockchip RK3588, RK3568 and other hardware. From 52451eb3f8c7c176581f498bbc2e4c63e6033de1 Mon Sep 17 00:00:00 2001 From: Zheng_Bicheng <58363586+Zheng-Bicheng@users.noreply.github.com> Date: Mon, 21 Nov 2022 13:44:34 +0800 Subject: [PATCH 075/136] [Model] Add Picodet RKNPU2 (#635) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * * 更新picodet cpp代码 * * 更新文档 * 更新picodet cpp example * * 删除无用的debug代码 * 新增python example * * 修改c++代码 * * 修改python代码 * * 修改postprocess代码 * 修复没有scale_factor导致的bug * 修复错误 * 更正代码格式 * 更正代码格式 --- .../paddledetection/rknpu2/README.md | 83 ++++++++++- .../paddledetection/rknpu2/cpp/CMakeLists.txt | 2 +- .../paddledetection/rknpu2/cpp/README.md | 2 +- .../rknpu2/cpp/infer_picodet.cc | 78 ++++------ .../paddledetection/rknpu2/python/README.md | 6 +- .../paddledetection/rknpu2/python/infer.py | 58 ++++---- fastdeploy/vision/detection/ppdet/base.cc | 31 ++-- fastdeploy/vision/detection/ppdet/model.h | 1 + .../vision/detection/ppdet/postprocessor.cc | 135 +++++++++++++++--- .../vision/detection/ppdet/postprocessor.h | 21 ++- .../vision/detection/ppdet/ppdet_pybind.cc | 4 + .../vision/detection/ppdet/preprocessor.cc | 37 +++-- .../vision/detection/ppdet/__init__.py | 7 +- .../vision/detection/rknpu2/__init__.py | 44 ------ python/requirements.txt | 1 + python/setup.py | 15 +- .../RK3568/picodet_s_416_coco_lcnet.yaml | 6 +- .../config/RK3568/picodet_s_416_coco_npu.yaml | 5 - .../RK3588/picodet_s_416_coco_lcnet.yaml | 4 +- .../config/RK3588/picodet_s_416_coco_npu.yaml | 5 - 20 files changed, 345 insertions(+), 200 deletions(-) delete mode 100644 python/fastdeploy/vision/detection/rknpu2/__init__.py delete mode 100644 tools/rknpu2/config/RK3568/picodet_s_416_coco_npu.yaml delete mode 100644 tools/rknpu2/config/RK3588/picodet_s_416_coco_npu.yaml diff --git a/examples/vision/detection/paddledetection/rknpu2/README.md b/examples/vision/detection/paddledetection/rknpu2/README.md index d75e09aa2bc..d5f339db5a7 100644 --- a/examples/vision/detection/paddledetection/rknpu2/README.md +++ b/examples/vision/detection/paddledetection/rknpu2/README.md @@ -13,7 +13,9 @@ RKNPU部署模型前需要将Paddle模型转换成RKNN模型，具体步骤如 ## 模型转换example -下面以Picodet-npu为例子,教大家如何转换PaddleDetection模型到RKNN模型。 +以下步骤均在Ubuntu电脑上完成，请参考配置文档完成转换模型环境配置。下面以Picodet-s为例子,教大家如何转换PaddleDetection模型到RKNN模型。 + +### 导出ONNX模型 ```bash # 下载Paddle静态图模型并解压 wget https://paddledet.bj.bcebos.com/deploy/Inference/picodet_s_416_coco_lcnet.tar @@ -26,12 +28,89 @@ paddle2onnx --model_dir picodet_s_416_coco_lcnet \ --save_file picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx \ --enable_dev_version True +# 固定shape python -m paddle2onnx.optimize --input_model picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx \ --output_model picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx \ --input_shape_dict "{'image':[1,3,416,416]}" +``` + +### 编写模型导出配置文件 +以转化RK3568的RKNN模型为例子，我们需要编辑tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml，来转换ONNX模型到RKNN模型。 + +**修改normalize参数** + +如果你需要在NPU上执行normalize操作，请根据你的模型配置normalize参数，例如: +```yaml +model_path: ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx +output_folder: ./picodet_s_416_coco_lcnet +target_platform: RK3568 +normalize: + mean: [[0.485,0.456,0.406],[0,0,0]] + std: [[0.229,0.224,0.225],[0.003921,0.003921]] +outputs: ['tmp_17','p2o.Concat.9'] +``` + +**修改outputs参数** +由于Paddle2ONNX版本的不同，转换模型的输出节点名称也有所不同，请使用[Netron](https://netron.app)，并找到以下蓝色方框标记的NonMaxSuppression节点，红色方框的节点名称即为目标名称。 + +例如，使用Netron可视化后，得到以下图片: +![](https://user-images.githubusercontent.com/58363586/202728663-4af0b843-d012-4aeb-8a66-626b7b87ca69.png) + +找到蓝色方框标记的NonMaxSuppression节点，可以看到红色方框标记的两个节点名称为tmp_17和p2o.Concat.9,因此需要修改outputs参数，修改后如下: +```yaml +model_path: ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx +output_folder: ./picodet_s_416_coco_lcnet +target_platform: RK3568 +normalize: None +outputs: ['tmp_17','p2o.Concat.9'] +``` + +### 转换模型 +```bash + # ONNX模型转RKNN模型 # 转换模型,模型将生成在picodet_s_320_coco_lcnet_non_postprocess目录下 -python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3588/picodet_s_416_coco_lcnet.yaml +python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml +``` + +### 修改模型运行时的配置文件 + +配置文件中，我们只需要修改**Preprocess**下的**Normalize**和**Permute**. + +**删除Permute** + +RKNPU只支持NHWC的输入格式，因此需要删除Permute操作.删除后，配置文件Precess部分后如下: +```yaml +Preprocess: +- interp: 2 + keep_ratio: false + target_size: + - 416 + - 416 + type: Resize +- is_scale: true + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 + type: NormalizeImage +``` + +**根据模型转换文件决定是否删除Normalize** + +RKNPU支持使用NPU进行Normalize操作，如果你在导出模型时配置了Normalize参数，请删除**Normalize**.删除后配置文件Precess部分如下: +```yaml +Preprocess: +- interp: 2 + keep_ratio: false + target_size: + - 416 + - 416 + type: Resize ``` - [Python部署](./python) diff --git a/examples/vision/detection/paddledetection/rknpu2/cpp/CMakeLists.txt b/examples/vision/detection/paddledetection/rknpu2/cpp/CMakeLists.txt index b4eca78ec54..19624777333 100644 --- a/examples/vision/detection/paddledetection/rknpu2/cpp/CMakeLists.txt +++ b/examples/vision/detection/paddledetection/rknpu2/cpp/CMakeLists.txt @@ -33,5 +33,5 @@ install(DIRECTORY ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/opencv/lib DESTIN file(GLOB PADDLETOONNX_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/paddle2onnx/lib/*) install(PROGRAMS ${PADDLETOONNX_LIBS} DESTINATION lib) -file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/RK3588/lib/*) +file(GLOB RKNPU2_LIBS ${FASTDEPLOY_INSTALL_DIR}/third_libs/install/rknpu2_runtime/${RKNN2_TARGET_SOC}/lib/*) install(PROGRAMS ${RKNPU2_LIBS} DESTINATION lib) diff --git a/examples/vision/detection/paddledetection/rknpu2/cpp/README.md b/examples/vision/detection/paddledetection/rknpu2/cpp/README.md index d0b13197124..c5ebbdeefc3 100644 --- a/examples/vision/detection/paddledetection/rknpu2/cpp/README.md +++ b/examples/vision/detection/paddledetection/rknpu2/cpp/README.md @@ -62,7 +62,7 @@ make install ```bash cd ./build/install -./rknpu_test +./infer_picodet model/picodet_s_416_coco_lcnet images/000000014439.jpg ``` diff --git a/examples/vision/detection/paddledetection/rknpu2/cpp/infer_picodet.cc b/examples/vision/detection/paddledetection/rknpu2/cpp/infer_picodet.cc index 862d28adc64..12f405b524a 100644 --- a/examples/vision/detection/paddledetection/rknpu2/cpp/infer_picodet.cc +++ b/examples/vision/detection/paddledetection/rknpu2/cpp/infer_picodet.cc @@ -14,73 +14,53 @@ #include #include #include "fastdeploy/vision.h" +#include +double __get_us(struct timeval t) { return (t.tv_sec * 1000000 + t.tv_usec); } +void InferPicodet(const std::string& model_dir, const std::string& image_file); -void InferPicodet(const std::string& device = "cpu"); +int main(int argc, char* argv[]) { + if (argc < 3) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/image run_option, " + "e.g ./infer_model ./picodet_model_dir ./test.jpeg" + << std::endl; + return -1; + } -int main() { - InferPicodet("npu"); - return 0; -} + InferPicodet(argv[1], argv[2]); -fastdeploy::RuntimeOption GetOption(const std::string& device) { - auto option = fastdeploy::RuntimeOption(); - if (device == "npu") { - option.UseRKNPU2(); - } else { - option.UseCpu(); - } - return option; + return 0; } -fastdeploy::ModelFormat GetFormat(const std::string& device) { - auto format = fastdeploy::ModelFormat::ONNX; - if (device == "npu") { - format = fastdeploy::ModelFormat::RKNN; - } else { - format = fastdeploy::ModelFormat::ONNX; - } - return format; -} +void InferPicodet(const std::string& model_dir, const std::string& image_file) { + struct timeval start_time, stop_time; + auto model_file = model_dir + "/picodet_s_416_coco_lcnet_rk3568.rknn"; + auto params_file = ""; + auto config_file = model_dir + "/infer_cfg.yml"; -std::string GetModelPath(std::string& model_path, const std::string& device) { - if (device == "npu") { - model_path += "rknn"; - } else { - model_path += "onnx"; - } - return model_path; -} + auto option = fastdeploy::RuntimeOption(); + option.UseRKNPU2(); -void InferPicodet(const std::string &device) { - std::string model_file = "./model/picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet_rk3588."; - std::string params_file; - std::string config_file = "./model/picodet_s_416_coco_lcnet/infer_cfg.yml"; + auto format = fastdeploy::ModelFormat::RKNN; - fastdeploy::RuntimeOption option = GetOption(device); - fastdeploy::ModelFormat format = GetFormat(device); - model_file = GetModelPath(model_file, device); - auto model = fastdeploy::vision::detection::RKPicoDet( + auto model = fastdeploy::vision::detection::PicoDet( model_file, params_file, config_file,option,format); - if (!model.Initialized()) { - std::cerr << "Failed to initialize." << std::endl; - return; - } - auto image_file = "./images/000000014439.jpg"; + model.GetPostprocessor().ApplyDecodeAndNMS(); + auto im = cv::imread(image_file); fastdeploy::vision::DetectionResult res; - clock_t start = clock(); + gettimeofday(&start_time, NULL); if (!model.Predict(&im, &res)) { std::cerr << "Failed to predict." << std::endl; return; } - clock_t end = clock(); - auto dur = static_cast(end - start); - printf("picodet_npu use time:%f\n", (dur / CLOCKS_PER_SEC)); + gettimeofday(&stop_time, NULL); + printf("infer use %f ms\n", (__get_us(stop_time) - __get_us(start_time)) / 1000); std::cout << res.Str() << std::endl; auto vis_im = fastdeploy::vision::VisDetection(im, res,0.5); - cv::imwrite("picodet_npu_result.jpg", vis_im); - std::cout << "Visualized result saved in ./picodet_npu_result.jpg" << std::endl; + cv::imwrite("picodet_result.jpg", vis_im); + std::cout << "Visualized result saved in ./picodet_result.jpg" << std::endl; } \ No newline at end of file diff --git a/examples/vision/detection/paddledetection/rknpu2/python/README.md b/examples/vision/detection/paddledetection/rknpu2/python/README.md index 23b13cd3b93..f191063f083 100644 --- a/examples/vision/detection/paddledetection/rknpu2/python/README.md +++ b/examples/vision/detection/paddledetection/rknpu2/python/README.md @@ -15,11 +15,11 @@ cd FastDeploy/examples/vision/detection/paddledetection/rknpu2/python wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg # copy model -cp -r ./picodet_s_416_coco_npu /path/to/FastDeploy/examples/vision/detection/rknpu2detection/paddledetection/python +cp -r ./picodet_s_416_coco_lcnet /path/to/FastDeploy/examples/vision/detection/rknpu2detection/paddledetection/python # 推理 -python3 infer.py --model_file ./picodet_s_416_coco_npu/picodet_s_416_coco_npu_3588.rknn \ - --config_file ./picodet_s_416_coco_npu/infer_cfg.yml \ +python3 infer.py --model_file ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet_rk3568.rknn \ + --config_file ./picodet_s_416_coco_lcnet/infer_cfg.yml \ --image 000000014439.jpg ``` diff --git a/examples/vision/detection/paddledetection/rknpu2/python/infer.py b/examples/vision/detection/paddledetection/rknpu2/python/infer.py index ae2d8796a61..62372105cad 100644 --- a/examples/vision/detection/paddledetection/rknpu2/python/infer.py +++ b/examples/vision/detection/paddledetection/rknpu2/python/infer.py @@ -28,32 +28,32 @@ def parse_arguments(): return parser.parse_args() -def build_option(args): - option = fd.RuntimeOption() - option.use_rknpu2() - return option - - -args = parse_arguments() - -# 配置runtime，加载模型 -runtime_option = build_option(args) -model_file = args.model_file -params_file = "" -config_file = args.config_file -model = fd.vision.detection.RKPicoDet( - model_file, - params_file, - config_file, - runtime_option=runtime_option, - model_format=fd.ModelFormat.RKNN) - -# 预测图片分割结果 -im = cv2.imread(args.image) -result = model.predict(im.copy()) -print(result) - -# 可视化结果 -vis_im = fd.vision.vis_detection(im, result, score_threshold=0.5) -cv2.imwrite("visualized_result.jpg", vis_im) -print("Visualized result save in ./visualized_result.jpg") +if __name__ == "__main__": + args = parse_arguments() + + model_file = args.model_file + params_file = "" + config_file = args.config_file + + # 配置runtime，加载模型 + runtime_option = fd.RuntimeOption() + runtime_option.use_rknpu2() + + model = fd.vision.detection.PicoDet( + model_file, + params_file, + config_file, + runtime_option=runtime_option, + model_format=fd.ModelFormat.RKNN) + + model.postprocessor.apply_decode_and_nms() + + # 预测图片分割结果 + im = cv2.imread(args.image) + result = model.predict(im.copy()) + print(result) + + # 可视化结果 + vis_im = fd.vision.vis_detection(im, result, score_threshold=0.5) + cv2.imwrite("visualized_result.jpg", vis_im) + print("Visualized result save in ./visualized_result.jpg") diff --git a/fastdeploy/vision/detection/ppdet/base.cc b/fastdeploy/vision/detection/ppdet/base.cc index 1db42f1582d..0d4e0f2904f 100755 --- a/fastdeploy/vision/detection/ppdet/base.cc +++ b/fastdeploy/vision/detection/ppdet/base.cc @@ -6,10 +6,12 @@ namespace fastdeploy { namespace vision { namespace detection { -PPDetBase::PPDetBase(const std::string& model_file, const std::string& params_file, - const std::string& config_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) : preprocessor_(config_file) { +PPDetBase::PPDetBase(const std::string& model_file, + const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) + : preprocessor_(config_file) { runtime_option = custom_option; runtime_option.model_format = model_format; runtime_option.model_file = model_file; @@ -37,7 +39,8 @@ bool PPDetBase::Predict(const cv::Mat& im, DetectionResult* result) { return true; } -bool PPDetBase::BatchPredict(const std::vector& imgs, std::vector* results) { +bool PPDetBase::BatchPredict(const std::vector& imgs, + std::vector* results) { std::vector fd_images = WrapMat(imgs); if (!preprocessor_.Run(&fd_images, &reused_input_tensors_)) { FDERROR << "Failed to preprocess the input image." << std::endl; @@ -46,8 +49,13 @@ bool PPDetBase::BatchPredict(const std::vector& imgs, std::vector(reused_input_tensors_[1].Data())); + } + + // Some models don't need scale_factor and im_shape as input + while (reused_input_tensors_.size() != NumInputsOfRuntime()) { reused_input_tensors_.pop_back(); } @@ -57,12 +65,13 @@ bool PPDetBase::BatchPredict(const std::vector& imgs, std::vector* results) { +bool PaddleDetPostprocessor::ProcessMask( + const FDTensor& tensor, std::vector* results) { auto shape = tensor.Shape(); if (tensor.Dtype() != FDDataType::INT32) { - FDERROR << "The data type of out mask tensor should be INT32, but now it's " << tensor.Dtype() << std::endl; + FDERROR << "The data type of out mask tensor should be INT32, but now it's " + << tensor.Dtype() << std::endl; return false; } int64_t out_mask_h = shape[1]; @@ -46,20 +48,34 @@ bool PaddleDetPostprocessor::ProcessMask(const FDTensor& tensor, std::vector((*results)[i].masks[j].Data()); + int32_t* keep_mask_ptr = + reinterpret_cast((*results)[i].masks[j].Data()); for (int row = y1; row < y2; ++row) { size_t keep_nbytes_in_col = keep_mask_w * sizeof(int32_t); const int32_t* out_row_start_ptr = current_ptr + row * out_mask_w + x1; int32_t* keep_row_start_ptr = keep_mask_ptr + (row - y1) * keep_mask_w; std::memcpy(keep_row_start_ptr, out_row_start_ptr, keep_nbytes_in_col); } - index += 1; + index += 1; } } return true; } -bool PaddleDetPostprocessor::Run(const std::vector& tensors, std::vector* results) { +bool PaddleDetPostprocessor::Run(const std::vector& tensors, + std::vector* results) { + if (DecodeAndNMSApplied()) { + FDASSERT(tensors.size() == 2, + "While postprocessing with ApplyDecodeAndNMS, " + "there should be 2 outputs for this model, but now it's %zu.", + tensors.size()); + FDASSERT(tensors[0].shape.size() == 3, + "While postprocessing with ApplyDecodeAndNMS, " + "the rank of the first outputs should be 3, but now it's %zu", + tensors[0].shape.size()); + return ProcessUnDecodeResults(tensors, results); + } + if (tensors[0].shape[0] == 0) { // No detected boxes return true; @@ -69,13 +85,13 @@ bool PaddleDetPostprocessor::Run(const std::vector& tensors, std::vect std::vector num_boxes(tensors[1].shape[0]); int total_num_boxes = 0; if (tensors[1].dtype == FDDataType::INT32) { - const int32_t* data = static_cast(tensors[1].CpuData()); + const auto* data = static_cast(tensors[1].CpuData()); for (size_t i = 0; i < tensors[1].shape[0]; ++i) { num_boxes[i] = static_cast(data[i]); total_num_boxes += num_boxes[i]; } } else if (tensors[1].dtype == FDDataType::INT64) { - const int64_t* data = static_cast(tensors[1].CpuData()); + const auto* data = static_cast(tensors[1].CpuData()); for (size_t i = 0; i < tensors[1].shape[0]; ++i) { num_boxes[i] = static_cast(data[i]); } @@ -83,33 +99,37 @@ bool PaddleDetPostprocessor::Run(const std::vector& tensors, std::vect // Special case for TensorRT, it has fixed output shape of NMS // So there's invalid boxes in its' output boxes - int num_output_boxes = tensors[0].Shape()[0]; + int num_output_boxes = static_cast(tensors[0].Shape()[0]); bool contain_invalid_boxes = false; if (total_num_boxes != num_output_boxes) { if (num_output_boxes % num_boxes.size() == 0) { contain_invalid_boxes = true; } else { - FDERROR << "Cannot handle the output data for this model, unexpected situation." << std::endl; + FDERROR << "Cannot handle the output data for this model, unexpected " + "situation." + << std::endl; return false; } } // Get boxes for each input image results->resize(num_boxes.size()); - const float* box_data = static_cast(tensors[0].CpuData()); + const auto* box_data = static_cast(tensors[0].CpuData()); int offset = 0; for (size_t i = 0; i < num_boxes.size(); ++i) { const float* ptr = box_data + offset; (*results)[i].Reserve(num_boxes[i]); for (size_t j = 0; j < num_boxes[i]; ++j) { - (*results)[i].label_ids.push_back(static_cast(round(ptr[j * 6]))); - (*results)[i].scores.push_back(ptr[j * 6 + 1]); - (*results)[i].boxes.emplace_back(std::array({ptr[j * 6 + 2], ptr[j * 6 + 3], ptr[j * 6 + 4], ptr[j * 6 + 5]})); + (*results)[i].label_ids.push_back( + static_cast(round(ptr[j * 6]))); + (*results)[i].scores.push_back(ptr[j * 6 + 1]); + (*results)[i].boxes.emplace_back(std::array( + {ptr[j * 6 + 2], ptr[j * 6 + 3], ptr[j * 6 + 4], ptr[j * 6 + 5]})); } if (contain_invalid_boxes) { - offset += (num_output_boxes * 6 / num_boxes.size()); + offset += static_cast(num_output_boxes * 6 / num_boxes.size()); } else { - offset += (num_boxes[i] * 6); + offset += static_cast(num_boxes[i] * 6); } } @@ -119,7 +139,10 @@ bool PaddleDetPostprocessor::Run(const std::vector& tensors, std::vect } if (tensors[2].Shape()[0] != num_output_boxes) { - FDERROR << "The first dimension of output mask tensor:" << tensors[2].Shape()[0] << " is not equal to the first dimension of output boxes tensor:" << num_output_boxes << "." << std::endl; + FDERROR << "The first dimension of output mask tensor:" + << tensors[2].Shape()[0] + << " is not equal to the first dimension of output boxes tensor:" + << num_output_boxes << "." << std::endl; return false; } @@ -127,6 +150,80 @@ bool PaddleDetPostprocessor::Run(const std::vector& tensors, std::vect return ProcessMask(tensors[2], results); } -} // namespace detection -} // namespace vision -} // namespace fastdeploy +void PaddleDetPostprocessor::ApplyDecodeAndNMS() { + apply_decode_and_nms_ = true; +} + +bool PaddleDetPostprocessor::ProcessUnDecodeResults( + const std::vector& tensors, + std::vector* results) { + if (tensors.size() != 2) { + return false; + } + + int boxes_index = 0; + int scores_index = 1; + if (tensors[0].shape[1] == tensors[1].shape[2]) { + boxes_index = 0; + scores_index = 1; + } else if (tensors[0].shape[2] == tensors[1].shape[1]) { + boxes_index = 1; + scores_index = 0; + } else { + FDERROR << "The shape of boxes and scores should be [batch, boxes_num, " + "4], [batch, classes_num, boxes_num]" + << std::endl; + return false; + } + + backend::MultiClassNMS nms; + nms.background_label = -1; + nms.keep_top_k = 100; + nms.nms_eta = 1.0; + nms.nms_threshold = 0.5; + nms.score_threshold = 0.3; + nms.nms_top_k = 1000; + nms.normalized = true; + nms.Compute(static_cast(tensors[boxes_index].Data()), + static_cast(tensors[scores_index].Data()), + tensors[boxes_index].shape, tensors[scores_index].shape); + + auto num_boxes = nms.out_num_rois_data; + auto box_data = static_cast(nms.out_box_data.data()); + // Get boxes for each input image + results->resize(num_boxes.size()); + int offset = 0; + for (size_t i = 0; i < num_boxes.size(); ++i) { + const float* ptr = box_data + offset; + (*results)[i].Reserve(num_boxes[i]); + for (size_t j = 0; j < num_boxes[i]; ++j) { + (*results)[i].label_ids.push_back( + static_cast(round(ptr[j * 6]))); + (*results)[i].scores.push_back(ptr[j * 6 + 1]); + (*results)[i].boxes.emplace_back(std::array( + {ptr[j * 6 + 2] / GetScaleFactor()[1], + ptr[j * 6 + 3] / GetScaleFactor()[0], + ptr[j * 6 + 4] / GetScaleFactor()[1], + ptr[j * 6 + 5] / GetScaleFactor()[0]})); + } + offset += (num_boxes[i] * 6); + } + return true; +} + +std::vector PaddleDetPostprocessor::GetScaleFactor(){ + return scale_factor_; +} + +void PaddleDetPostprocessor::SetScaleFactor(float* scale_factor_value){ + for (int i = 0; i < scale_factor_.size(); ++i) { + scale_factor_[i] = scale_factor_value[i]; + } +} + +bool PaddleDetPostprocessor::DecodeAndNMSApplied() { + return apply_decode_and_nms_; +} +} // namespace detection +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/postprocessor.h b/fastdeploy/vision/detection/ppdet/postprocessor.h index 54be1bfd95f..9a1410dc9e9 100644 --- a/fastdeploy/vision/detection/ppdet/postprocessor.h +++ b/fastdeploy/vision/detection/ppdet/postprocessor.h @@ -32,11 +32,28 @@ class FASTDEPLOY_DECL PaddleDetPostprocessor { * \return true if the postprocess successed, otherwise false */ bool Run(const std::vector& tensors, - std::vector* result); + std::vector* result); + + /// Apply box decoding and nms step for the outputs for the model.This is + /// only available for those model exported without box decoding and nms. + void ApplyDecodeAndNMS(); + + bool DecodeAndNMSApplied(); + + /// Set scale_factor_ value.This is only available for those model exported + /// without box decoding and nms. + void SetScaleFactor(float* scale_factor_value); + private: // Process mask tensor for MaskRCNN bool ProcessMask(const FDTensor& tensor, - std::vector* results); + std::vector* results); + + bool apply_decode_and_nms_ = false; + std::vector scale_factor_{1.0, 1.0}; + std::vector GetScaleFactor(); + bool ProcessUnDecodeResults(const std::vector& tensors, + std::vector* results); }; } // namespace detection diff --git a/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc b/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc index 252097608e1..46cd5e8ead9 100644 --- a/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc +++ b/fastdeploy/vision/detection/ppdet/ppdet_pybind.cc @@ -43,6 +43,10 @@ void BindPPDet(pybind11::module& m) { } return results; }) + .def("apply_decode_and_nms", + [](vision::detection::PaddleDetPostprocessor& self){ + self.ApplyDecodeAndNMS(); + }) .def("run", [](vision::detection::PaddleDetPostprocessor& self, std::vector& input_array) { std::vector results; std::vector inputs; diff --git a/fastdeploy/vision/detection/ppdet/preprocessor.cc b/fastdeploy/vision/detection/ppdet/preprocessor.cc index b1179d0362c..bb38c67ec6a 100644 --- a/fastdeploy/vision/detection/ppdet/preprocessor.cc +++ b/fastdeploy/vision/detection/ppdet/preprocessor.cc @@ -22,11 +22,13 @@ namespace vision { namespace detection { PaddleDetPreprocessor::PaddleDetPreprocessor(const std::string& config_file) { - FDASSERT(BuildPreprocessPipelineFromConfig(config_file), "Failed to create PaddleDetPreprocessor."); + FDASSERT(BuildPreprocessPipelineFromConfig(config_file), + "Failed to create PaddleDetPreprocessor."); initialized_ = true; } -bool PaddleDetPreprocessor::BuildPreprocessPipelineFromConfig(const std::string& config_file) { +bool PaddleDetPreprocessor::BuildPreprocessPipelineFromConfig( + const std::string& config_file) { processors_.clear(); YAML::Node cfg; try { @@ -106,8 +108,6 @@ bool PaddleDetPreprocessor::BuildPreprocessPipelineFromConfig(const std::string& // permute = cast + HWC2CHW processors_.push_back(std::make_shared("float")); processors_.push_back(std::make_shared()); - } else { - processors_.push_back(std::make_shared()); } // Fusion will improve performance @@ -116,13 +116,15 @@ bool PaddleDetPreprocessor::BuildPreprocessPipelineFromConfig(const std::string& return true; } -bool PaddleDetPreprocessor::Run(std::vector* images, std::vector* outputs) { +bool PaddleDetPreprocessor::Run(std::vector* images, + std::vector* outputs) { if (!initialized_) { FDERROR << "The preprocessor is not initialized." << std::endl; return false; } if (images->size() == 0) { - FDERROR << "The size of input images should be greater than 0." << std::endl; + FDERROR << "The size of input images should be greater than 0." + << std::endl; return false; } @@ -140,8 +142,9 @@ bool PaddleDetPreprocessor::Run(std::vector* images, std::vector max_hw({-1, -1}); - float* scale_factor_ptr = reinterpret_cast((*outputs)[1].MutableData()); - float* im_shape_ptr = reinterpret_cast((*outputs)[2].MutableData()); + float* scale_factor_ptr = + reinterpret_cast((*outputs)[1].MutableData()); + float* im_shape_ptr = reinterpret_cast((*outputs)[2].MutableData()); for (size_t i = 0; i < images->size(); ++i) { int origin_w = (*images)[i].Width(); int origin_h = (*images)[i].Height(); @@ -149,7 +152,8 @@ bool PaddleDetPreprocessor::Run(std::vector* images, std::vectorName() << "." << std::endl; + FDERROR << "Failed to processs image:" << i << " in " + << processors_[i]->Name() << "." << std::endl; return false; } if (processors_[j]->Name().find("Resize") != std::string::npos) { @@ -166,15 +170,18 @@ bool PaddleDetPreprocessor::Run(std::vector* images, std::vector im_tensors(images->size()); + std::vector im_tensors(images->size()); for (size_t i = 0; i < images->size(); ++i) { if ((*images)[i].Height() < max_hw[0] || (*images)[i].Width() < max_hw[1]) { // if the size of image less than max_hw, pad to max_hw FDTensor tensor; (*images)[i].ShareWithTensor(&tensor); - function::Pad(tensor, &(im_tensors[i]), {0, 0, max_hw[0] - (*images)[i].Height(), max_hw[1] - (*images)[i].Width()}, 0); + function::Pad(tensor, &(im_tensors[i]), + {0, 0, max_hw[0] - (*images)[i].Height(), + max_hw[1] - (*images)[i].Width()}, + 0); } else { // No need pad (*images)[i].ShareWithTensor(&(im_tensors[i])); @@ -196,6 +203,6 @@ bool PaddleDetPreprocessor::Run(std::vector* images, std::vector=0.0.1 +pyyaml diff --git a/python/setup.py b/python/setup.py index a411a12be26..991d87b0371 100755 --- a/python/setup.py +++ b/python/setup.py @@ -61,7 +61,8 @@ "OFF") setup_configs["ENABLE_PADDLE_BACKEND"] = os.getenv("ENABLE_PADDLE_BACKEND", "OFF") -setup_configs["ENABLE_POROS_BACKEND"] = os.getenv("ENABLE_POROS_BACKEND", "OFF") +setup_configs["ENABLE_POROS_BACKEND"] = os.getenv("ENABLE_POROS_BACKEND", + "OFF") setup_configs["ENABLE_TRT_BACKEND"] = os.getenv("ENABLE_TRT_BACKEND", "OFF") setup_configs["ENABLE_LITE_BACKEND"] = os.getenv("ENABLE_LITE_BACKEND", "OFF") setup_configs["ENABLE_VISION"] = os.getenv("ENABLE_VISION", "OFF") @@ -71,13 +72,15 @@ setup_configs["WITH_IPU"] = os.getenv("WITH_IPU", "OFF") setup_configs["BUILD_ON_JETSON"] = os.getenv("BUILD_ON_JETSON", "OFF") setup_configs["TRT_DIRECTORY"] = os.getenv("TRT_DIRECTORY", "UNDEFINED") -setup_configs["CUDA_DIRECTORY"] = os.getenv("CUDA_DIRECTORY", "/usr/local/cuda") +setup_configs["CUDA_DIRECTORY"] = os.getenv("CUDA_DIRECTORY", + "/usr/local/cuda") setup_configs["LIBRARY_NAME"] = PACKAGE_NAME setup_configs["PY_LIBRARY_NAME"] = PACKAGE_NAME + "_main" setup_configs["OPENCV_DIRECTORY"] = os.getenv("OPENCV_DIRECTORY", "") setup_configs["ORT_DIRECTORY"] = os.getenv("ORT_DIRECTORY", "") - setup_configs["RKNN2_TARGET_SOC"] = os.getenv("RKNN2_TARGET_SOC", "") +if setup_configs["RKNN2_TARGET_SOC"] != "": + REQUIRED_PACKAGES = REQUIRED_PACKAGES.replace("opencv-python", "") if setup_configs["WITH_GPU"] == "ON" or setup_configs[ "BUILD_ON_JETSON"] == "ON": @@ -105,7 +108,8 @@ # Default value is set to TRUE\1 to keep the settings same as the current ones. # However going forward the recomemded way to is to set this to False\0 -USE_MSVC_STATIC_RUNTIME = bool(os.getenv('USE_MSVC_STATIC_RUNTIME', '1') == '1') +USE_MSVC_STATIC_RUNTIME = bool( + os.getenv('USE_MSVC_STATIC_RUNTIME', '1') == '1') ONNX_NAMESPACE = os.getenv('ONNX_NAMESPACE', 'paddle2onnx') ################################################################################ # Version @@ -135,7 +139,8 @@ @contextmanager def cd(path): if not os.path.isabs(path): - raise RuntimeError('Can only cd to absolute path, got: {}'.format(path)) + raise RuntimeError('Can only cd to absolute path, got: {}'.format( + path)) orig_path = os.getcwd() os.chdir(path) try: diff --git a/tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml b/tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml index 98548916340..7bb141eca31 100644 --- a/tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml +++ b/tools/rknpu2/config/RK3568/picodet_s_416_coco_lcnet.yaml @@ -1,7 +1,5 @@ model_path: ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx output_folder: ./picodet_s_416_coco_lcnet target_platform: RK3568 -normalize: - mean: [[0.485,0.456,0.406]] - std: [[0.229,0.224,0.225]] -outputs: ['tmp_16','p2o.Concat.9'] +normalize: None +outputs: ['tmp_17','p2o.Concat.9'] diff --git a/tools/rknpu2/config/RK3568/picodet_s_416_coco_npu.yaml b/tools/rknpu2/config/RK3568/picodet_s_416_coco_npu.yaml deleted file mode 100644 index 723acc8b55c..00000000000 --- a/tools/rknpu2/config/RK3568/picodet_s_416_coco_npu.yaml +++ /dev/null @@ -1,5 +0,0 @@ -model_path: ./picodet_s_416_coco_npu/picodet_s_416_coco_npu.onnx -output_folder: ./picodet_s_416_coco_npu -target_platform: RK3568 -normalize: None -outputs: ['tmp_16','p2o.Concat.17'] diff --git a/tools/rknpu2/config/RK3588/picodet_s_416_coco_lcnet.yaml b/tools/rknpu2/config/RK3588/picodet_s_416_coco_lcnet.yaml index 6110e8c0f97..ba12a4be167 100644 --- a/tools/rknpu2/config/RK3588/picodet_s_416_coco_lcnet.yaml +++ b/tools/rknpu2/config/RK3588/picodet_s_416_coco_lcnet.yaml @@ -1,7 +1,5 @@ model_path: ./picodet_s_416_coco_lcnet/picodet_s_416_coco_lcnet.onnx output_folder: ./picodet_s_416_coco_lcnet target_platform: RK3588 -normalize: - mean: [[0.485,0.456,0.406]] - std: [[0.229,0.224,0.225]] +normalize: None outputs: ['tmp_16','p2o.Concat.9'] diff --git a/tools/rknpu2/config/RK3588/picodet_s_416_coco_npu.yaml b/tools/rknpu2/config/RK3588/picodet_s_416_coco_npu.yaml deleted file mode 100644 index 356fcfad886..00000000000 --- a/tools/rknpu2/config/RK3588/picodet_s_416_coco_npu.yaml +++ /dev/null @@ -1,5 +0,0 @@ -model_path: ./picodet_s_416_coco_npu/picodet_s_416_coco_npu.onnx -output_folder: ./picodet_s_416_coco_npu -target_platform: RK3588 -normalize: None -outputs: ['tmp_16','p2o.Concat.17'] From 852d73cdb9d1310c9ec0165c602229e293fa8916 Mon Sep 17 00:00:00 2001 From: Jack Zhou Date: Mon, 21 Nov 2022 13:45:00 +0800 Subject: [PATCH 076/136] [Other] faster_tokenizer->fast_tokenizer (#636) * faster_tokenizer->fast_tokenizer * ErnieFasterTokenizer->ErnieFastTokenizer * update the fastdeploy_init Co-authored-by: Jason --- CMakeLists.txt | 4 +- FastDeploy.cmake.in | 8 +- cmake/fast_tokenizer.cmake | 108 ++++++++++++++++++ cmake/faster_tokenizer.cmake | 108 ------------------ examples/text/ernie-3.0/cpp/seq_cls_infer.cc | 14 +-- .../text/ernie-3.0/python/requirements.txt | 2 +- .../text/ernie-3.0/python/seq_cls_infer.py | 2 +- fastdeploy/text/uie/model.cc | 64 +++++------ fastdeploy/text/uie/model.h | 12 +- scripts/fastdeploy_init.bat | 10 +- scripts/fastdeploy_init.sh | 5 - serving/Dockerfile | 2 +- serving/Dockerfile_cpu | 2 +- tests/release_task/cpp_run.bat | 4 +- 14 files changed, 170 insertions(+), 175 deletions(-) create mode 100644 cmake/fast_tokenizer.cmake delete mode 100644 cmake/faster_tokenizer.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 387a56f87f2..0b24ea2e729 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -415,14 +415,14 @@ endif() if(ANDROID OR IOS) if(ENABLE_TEXT) set(ENABLE_TEXT OFF CACHE BOOL "Force ENABLE_TEXT OFF" FORCE) - message(STATUS "Found Android or IOS, force ENABLE_TEXT OFF. We do not support faster_tokenizer with Android/IOS now.") + message(STATUS "Found Android or IOS, force ENABLE_TEXT OFF. We do not support fast_tokenizer with Android/IOS now.") endif() endif() if(ENABLE_TEXT) add_definitions(-DENABLE_TEXT) list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_TEXT_SRCS}) - include(${PROJECT_SOURCE_DIR}/cmake/faster_tokenizer.cmake) + include(${PROJECT_SOURCE_DIR}/cmake/fast_tokenizer.cmake) endif() if(ENABLE_PADDLE_FRONTEND) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 92439af613e..5e2eacf3a55 100755 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -213,10 +213,10 @@ if (ENABLE_TEXT) message(FATAL_ERROR "Not support fastdeploy text APIs with Android now!") endif() # Add dependency libs later - find_library(FASTER_TOKENIZER_LIB core_tokenizers ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/faster_tokenizer/lib NO_DEFAULT_PATH) - list(APPEND FASTDEPLOY_LIBS ${FASTER_TOKENIZER_LIB}) - list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/faster_tokenizer/include) - list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/faster_tokenizer/third_party/include) + find_library(FAST_TOKENIZER_LIB core_tokenizers ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/lib NO_DEFAULT_PATH) + list(APPEND FASTDEPLOY_LIBS ${FAST_TOKENIZER_LIB}) + list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/include) + list(APPEND FASTDEPLOY_INCS ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/fast_tokenizer/third_party/include) endif() if(ENABLE_PADDLE_FRONTEND) diff --git a/cmake/fast_tokenizer.cmake b/cmake/fast_tokenizer.cmake new file mode 100644 index 00000000000..9550d28b833 --- /dev/null +++ b/cmake/fast_tokenizer.cmake @@ -0,0 +1,108 @@ + + +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +include(ExternalProject) + +set(FASTTOKENIZER_PROJECT "extern_fast_tokenizer") +set(FASTTOKENIZER_PREFIX_DIR ${THIRD_PARTY_PATH}/fast_tokenizer) +set(FASTTOKENIZER_SOURCE_DIR + ${THIRD_PARTY_PATH}/fast_tokenizer/src/${FASTTOKENIZER_PROJECT}) +set(FASTTOKENIZER_INSTALL_DIR ${THIRD_PARTY_PATH}/install/fast_tokenizer) +set(FASTTOKENIZER_INC_DIR + "${FASTTOKENIZER_INSTALL_DIR}/include" + "${FASTTOKENIZER_INSTALL_DIR}/third_party/include" + CACHE PATH "fast_tokenizer include directory." FORCE) +set(FASTTOKENIZER_LIB_DIR + "${FASTTOKENIZER_INSTALL_DIR}/lib/" + CACHE PATH "fast_tokenizer lib directory." FORCE) +set(FASTTOKENIZER_THIRD_LIB_DIR + "${FASTTOKENIZER_INSTALL_DIR}/third_party/lib/" + CACHE PATH "fast_tokenizer lib directory." FORCE) +set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" + "${FASTTOKENIZER_LIB_DIR}") + +include_directories(${FASTTOKENIZER_INC_DIR}) + +# Set lib path +if(WIN32) +set(FASTTOKENIZER_COMPILE_LIB "${FASTTOKENIZER_LIB_DIR}/core_tokenizers.lib" + CACHE FILEPATH "fast_tokenizer compile library." FORCE) +message("FASTTOKENIZER_COMPILE_LIB = ${FASTTOKENIZER_COMPILE_LIB}") +set(ICUDT_LIB "${FASTTOKENIZER_THIRD_LIB_DIR}/icudt.lib") +set(ICUUC_LIB "${FASTTOKENIZER_THIRD_LIB_DIR}/icuuc.lib") + +elseif(APPLE) +set(FASTTOKENIZER_COMPILE_LIB "${FASTTOKENIZER_LIB_DIR}/libcore_tokenizers.dylib" + CACHE FILEPATH "fast_tokenizer compile library." FORCE) +else() + +set(FASTTOKENIZER_COMPILE_LIB "${FASTTOKENIZER_LIB_DIR}/libcore_tokenizers.so" + CACHE FILEPATH "fast_tokenizer compile library." FORCE) +message("FASTTOKENIZER_COMPILE_LIB = ${FASTTOKENIZER_COMPILE_LIB}") +endif(WIN32) + +set(FASTTOKENIZER_URL_BASE "https://bj.bcebos.com/paddlenlp/fast_tokenizer/") +set(FASTTOKENIZER_VERSION "1.0.0") + +# Set download url +if(WIN32) + set(FASTTOKENIZER_FILE "fast_tokenizer-win-x64-${FASTTOKENIZER_VERSION}.zip") + if(NOT CMAKE_CL_64) + set(FASTTOKENIZER_FILE "fast_tokenizer-win-x86-${FASTTOKENIZER_VERSION}.zip") + endif() +elseif(APPLE) + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64") + set(FASTTOKENIZER_FILE "fast_tokenizer-osx-arm64-${FASTTOKENIZER_VERSION}.tgz") + else() + set(FASTTOKENIZER_FILE "fast_tokenizer-osx-x86_64-${FASTTOKENIZER_VERSION}.tgz") + endif() +else() + if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") + set(FASTTOKENIZER_FILE "fast_tokenizer-linux-aarch64-${FASTTOKENIZER_VERSION}.tgz") + else() + set(FASTTOKENIZER_FILE "fast_tokenizer-linux-x64-${FASTTOKENIZER_VERSION}.tgz") + endif() +endif() +set(FASTTOKENIZER_URL "${FASTTOKENIZER_URL_BASE}${FASTTOKENIZER_FILE}") + +ExternalProject_Add( + ${FASTTOKENIZER_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${FASTTOKENIZER_URL} + PREFIX ${FASTTOKENIZER_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E copy_directory ${FASTTOKENIZER_SOURCE_DIR} ${FASTTOKENIZER_INSTALL_DIR} + BUILD_BYPRODUCTS ${FASTTOKENIZER_COMPILE_LIB}) + +add_library(fast_tokenizer STATIC IMPORTED GLOBAL) +set_property(TARGET fast_tokenizer PROPERTY IMPORTED_LOCATION ${FASTTOKENIZER_COMPILE_LIB}) +add_dependencies(fast_tokenizer ${FASTTOKENIZER_PROJECT}) +list(APPEND DEPEND_LIBS fast_tokenizer) + +if (WIN32) + add_library(icudt STATIC IMPORTED GLOBAL) + set_property(TARGET icudt PROPERTY IMPORTED_LOCATION ${ICUDT_LIB}) + add_dependencies(icudt ${FASTTOKENIZER_PROJECT}) + list(APPEND DEPEND_LIBS icudt) + + add_library(icuuc STATIC IMPORTED GLOBAL) + set_property(TARGET icuuc PROPERTY IMPORTED_LOCATION ${ICUUC_LIB}) + add_dependencies(icuuc ${FASTTOKENIZER_PROJECT}) + list(APPEND DEPEND_LIBS icuuc) +endif() diff --git a/cmake/faster_tokenizer.cmake b/cmake/faster_tokenizer.cmake deleted file mode 100644 index 730aeff66be..00000000000 --- a/cmake/faster_tokenizer.cmake +++ /dev/null @@ -1,108 +0,0 @@ - - -# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -include(ExternalProject) - -set(FASTERTOKENIZER_PROJECT "extern_faster_tokenizer") -set(FASTERTOKENIZER_PREFIX_DIR ${THIRD_PARTY_PATH}/faster_tokenizer) -set(FASTERTOKENIZER_SOURCE_DIR - ${THIRD_PARTY_PATH}/faster_tokenizer/src/${FASTERTOKENIZER_PROJECT}) -set(FASTERTOKENIZER_INSTALL_DIR ${THIRD_PARTY_PATH}/install/faster_tokenizer) -set(FASTERTOKENIZER_INC_DIR - "${FASTERTOKENIZER_INSTALL_DIR}/include" - "${FASTERTOKENIZER_INSTALL_DIR}/third_party/include" - CACHE PATH "faster_tokenizer include directory." FORCE) -set(FASTERTOKENIZER_LIB_DIR - "${FASTERTOKENIZER_INSTALL_DIR}/lib/" - CACHE PATH "faster_tokenizer lib directory." FORCE) -set(FASTERTOKENIZER_THIRD_LIB_DIR - "${FASTERTOKENIZER_INSTALL_DIR}/third_party/lib/" - CACHE PATH "faster_tokenizer lib directory." FORCE) -set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" - "${FASTERTOKENIZER_LIB_DIR}") - -include_directories(${FASTERTOKENIZER_INC_DIR}) - -# Set lib path -if(WIN32) -set(FASTERTOKENIZER_COMPILE_LIB "${FASTERTOKENIZER_LIB_DIR}/core_tokenizers.lib" - CACHE FILEPATH "faster_tokenizer compile library." FORCE) -message("FASTERTOKENIZER_COMPILE_LIB = ${FASTERTOKENIZER_COMPILE_LIB}") -set(ICUDT_LIB "${FASTERTOKENIZER_THIRD_LIB_DIR}/icudt.lib") -set(ICUUC_LIB "${FASTERTOKENIZER_THIRD_LIB_DIR}/icuuc.lib") - -elseif(APPLE) -set(FASTERTOKENIZER_COMPILE_LIB "${FASTERTOKENIZER_LIB_DIR}/libcore_tokenizers.dylib" - CACHE FILEPATH "faster_tokenizer compile library." FORCE) -else() - -set(FASTERTOKENIZER_COMPILE_LIB "${FASTERTOKENIZER_LIB_DIR}/libcore_tokenizers.so" - CACHE FILEPATH "faster_tokenizer compile library." FORCE) -message("FASTERTOKENIZER_COMPILE_LIB = ${FASTERTOKENIZER_COMPILE_LIB}") -endif(WIN32) - -set(FASTERTOKENIZER_URL_BASE "https://bj.bcebos.com/paddlenlp/faster_tokenizer/") -set(FASTERTOKENIZER_VERSION "dev") - -# Set download url -if(WIN32) - set(FASTERTOKENIZER_FILE "faster_tokenizer-win-x64-${FASTERTOKENIZER_VERSION}.zip") - if(NOT CMAKE_CL_64) - set(FASTERTOKENIZER_FILE "faster_tokenizer-win-x86-${FASTERTOKENIZER_VERSION}.zip") - endif() -elseif(APPLE) - if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64") - set(FASTERTOKENIZER_FILE "faster_tokenizer-osx-arm64-${FASTERTOKENIZER_VERSION}.tgz") - else() - set(FASTERTOKENIZER_FILE "faster_tokenizer-osx-x86_64-${FASTERTOKENIZER_VERSION}.tgz") - endif() -else() - if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") - set(FASTERTOKENIZER_FILE "faster_tokenizer-linux-aarch64-${FASTERTOKENIZER_VERSION}.tgz") - else() - set(FASTERTOKENIZER_FILE "faster_tokenizer-linux-x64-${FASTERTOKENIZER_VERSION}.tgz") - endif() -endif() -set(FASTERTOKENIZER_URL "${FASTERTOKENIZER_URL_BASE}${FASTERTOKENIZER_FILE}") - -ExternalProject_Add( - ${FASTERTOKENIZER_PROJECT} - ${EXTERNAL_PROJECT_LOG_ARGS} - URL ${FASTERTOKENIZER_URL} - PREFIX ${FASTERTOKENIZER_PREFIX_DIR} - DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - UPDATE_COMMAND "" - INSTALL_COMMAND - ${CMAKE_COMMAND} -E copy_directory ${FASTERTOKENIZER_SOURCE_DIR} ${FASTERTOKENIZER_INSTALL_DIR} - BUILD_BYPRODUCTS ${FASTERTOKENIZER_COMPILE_LIB}) - -add_library(faster_tokenizer STATIC IMPORTED GLOBAL) -set_property(TARGET faster_tokenizer PROPERTY IMPORTED_LOCATION ${FASTERTOKENIZER_COMPILE_LIB}) -add_dependencies(faster_tokenizer ${FASTERTOKENIZER_PROJECT}) -list(APPEND DEPEND_LIBS faster_tokenizer) - -if (WIN32) - add_library(icudt STATIC IMPORTED GLOBAL) - set_property(TARGET icudt PROPERTY IMPORTED_LOCATION ${ICUDT_LIB}) - add_dependencies(icudt ${FASTERTOKENIZER_PROJECT}) - list(APPEND DEPEND_LIBS icudt) - - add_library(icuuc STATIC IMPORTED GLOBAL) - set_property(TARGET icuuc PROPERTY IMPORTED_LOCATION ${ICUUC_LIB}) - add_dependencies(icuuc ${FASTERTOKENIZER_PROJECT}) - list(APPEND DEPEND_LIBS icuuc) -endif() \ No newline at end of file diff --git a/examples/text/ernie-3.0/cpp/seq_cls_infer.cc b/examples/text/ernie-3.0/cpp/seq_cls_infer.cc index b7ab495bdbe..c0f58da8d2b 100644 --- a/examples/text/ernie-3.0/cpp/seq_cls_infer.cc +++ b/examples/text/ernie-3.0/cpp/seq_cls_infer.cc @@ -18,11 +18,11 @@ #include "fastdeploy/function/softmax.h" #include "fastdeploy/runtime.h" #include "fastdeploy/utils/path.h" -#include "faster_tokenizer/tokenizers/ernie_faster_tokenizer.h" +#include "fast_tokenizer/tokenizers/ernie_fast_tokenizer.h" #include "gflags/gflags.h" using namespace paddlenlp; -using namespace faster_tokenizer::tokenizers_impl; +using namespace fast_tokenizer::tokenizers_impl; #ifdef WIN32 const char sep = '\\'; #else @@ -124,10 +124,10 @@ struct SeqClsResult { struct ErnieForSequenceClassificationPredictor { fastdeploy::Runtime runtime_; - ErnieFasterTokenizer tokenizer_; + ErnieFastTokenizer tokenizer_; ErnieForSequenceClassificationPredictor( const fastdeploy::RuntimeOption& option, - const ErnieFasterTokenizer& tokenizer) + const ErnieFastTokenizer& tokenizer) : tokenizer_(tokenizer) { runtime_.Init(option); } @@ -135,8 +135,8 @@ struct ErnieForSequenceClassificationPredictor { bool Preprocess(const std::vector& texts, const std::vector& texts_pair, std::vector* inputs) { - std::vector encodings; - std::vector text_pair_input; + std::vector encodings; + std::vector text_pair_input; // 1. Tokenize the text or (text, text_pair) if (texts_pair.empty()) { for (int i = 0; i < texts.size(); ++i) { @@ -242,7 +242,7 @@ int main(int argc, char* argv[]) { return -1; } } - ErnieFasterTokenizer tokenizer(vocab_path); + ErnieFastTokenizer tokenizer(vocab_path); ErnieForSequenceClassificationPredictor predictor(option, tokenizer); diff --git a/examples/text/ernie-3.0/python/requirements.txt b/examples/text/ernie-3.0/python/requirements.txt index 29711008e23..27f236104ca 100755 --- a/examples/text/ernie-3.0/python/requirements.txt +++ b/examples/text/ernie-3.0/python/requirements.txt @@ -1,2 +1,2 @@ -faster_tokenizer +fast-tokenizer-python paddlenlp diff --git a/examples/text/ernie-3.0/python/seq_cls_infer.py b/examples/text/ernie-3.0/python/seq_cls_infer.py index de67884a1d3..eb99d1d8438 100644 --- a/examples/text/ernie-3.0/python/seq_cls_infer.py +++ b/examples/text/ernie-3.0/python/seq_cls_infer.py @@ -15,7 +15,7 @@ import distutils.util import numpy as np -import faster_tokenizer +import fast_tokenizer from paddlenlp.transformers import AutoTokenizer import fastdeploy as fd diff --git a/fastdeploy/text/uie/model.cc b/fastdeploy/text/uie/model.cc index c04d4d6d3db..490a4a0fbe0 100644 --- a/fastdeploy/text/uie/model.cc +++ b/fastdeploy/text/uie/model.cc @@ -19,8 +19,8 @@ #include #include -#include "faster_tokenizer/pretokenizers/pretokenizer.h" -#include "faster_tokenizer/utils/utf8.h" +#include "fast_tokenizer/pretokenizers/pretokenizer.h" +#include "fast_tokenizer/utils/utf8.h" namespace fastdeploy { namespace text { @@ -30,9 +30,9 @@ static std::string DBC2SBC(const std::string& content) { size_t content_utf8_len = 0; while (content_utf8_len < content.length()) { uint32_t content_char; - auto content_char_width = faster_tokenizer::utils::UTF8ToUInt32( + auto content_char_width = fast_tokenizer::utils::UTF8ToUInt32( content.data() + content_utf8_len, &content_char); - content_char = faster_tokenizer::utils::UTF8ToUnicode(content_char); + content_char = fast_tokenizer::utils::UTF8ToUnicode(content_char); if (content_char == 0x3000) { content_char = 0x0020; } else { @@ -43,9 +43,9 @@ static std::string DBC2SBC(const std::string& content) { } else { char dst_char[5] = {0}; uint32_t utf8_uint32 = - faster_tokenizer::utils::UnicodeToUTF8(content_char); + fast_tokenizer::utils::UnicodeToUTF8(content_char); uint32_t utf8_char_count = - faster_tokenizer::utils::UnicodeToUTF8Char(utf8_uint32, dst_char); + fast_tokenizer::utils::UnicodeToUTF8Char(utf8_uint32, dst_char); result.append(dst_char, utf8_char_count); } content_utf8_len += content_char_width; @@ -177,8 +177,8 @@ UIEModel::UIEModel(const std::string& model_file, initialized = Initialize(); SetSchema(schema); tokenizer_.EnableTruncMethod( - max_length, 0, faster_tokenizer::core::Direction::RIGHT, - faster_tokenizer::core::TruncStrategy::LONGEST_FIRST); + max_length, 0, fast_tokenizer::core::Direction::RIGHT, + fast_tokenizer::core::TruncStrategy::LONGEST_FIRST); } UIEModel::UIEModel(const std::string& model_file, @@ -198,8 +198,8 @@ UIEModel::UIEModel(const std::string& model_file, initialized = Initialize(); SetSchema(schema); tokenizer_.EnableTruncMethod( - max_length, 0, faster_tokenizer::core::Direction::RIGHT, - faster_tokenizer::core::TruncStrategy::LONGEST_FIRST); + max_length, 0, fast_tokenizer::core::Direction::RIGHT, + fast_tokenizer::core::TruncStrategy::LONGEST_FIRST); } UIEModel::UIEModel(const std::string& model_file, @@ -219,8 +219,8 @@ UIEModel::UIEModel(const std::string& model_file, initialized = Initialize(); SetSchema(schema); tokenizer_.EnableTruncMethod( - max_length, 0, faster_tokenizer::core::Direction::RIGHT, - faster_tokenizer::core::TruncStrategy::LONGEST_FIRST); + max_length, 0, fast_tokenizer::core::Direction::RIGHT, + fast_tokenizer::core::TruncStrategy::LONGEST_FIRST); } bool UIEModel::Initialize() { @@ -253,7 +253,7 @@ void UIEModel::AutoSplitter(const std::vector& texts, size_t cnt_org = 0; size_t cnt_short = 0; for (auto& text : texts) { - auto text_len = faster_tokenizer::utils::GetUnicodeLenFromUTF8( + auto text_len = fast_tokenizer::utils::GetUnicodeLenFromUTF8( text.c_str(), text.length()); if (text_len <= max_length) { short_texts->push_back(text); @@ -264,14 +264,14 @@ void UIEModel::AutoSplitter(const std::vector& texts, } cnt_short += 1; } else { - faster_tokenizer::pretokenizers::CharToBytesOffsetConverter converter( + fast_tokenizer::pretokenizers::CharToBytesOffsetConverter converter( text); for (size_t start = 0; start < text_len; start += max_length) { size_t end = start + max_length; if (end > text_len) { end = text_len; } - faster_tokenizer::core::Offset byte_offset; + fast_tokenizer::core::Offset byte_offset; converter.convert({start, end}, &byte_offset); short_texts->emplace_back(text.data() + byte_offset.first, byte_offset.second - byte_offset.first); @@ -344,12 +344,12 @@ void UIEModel::GetSpan(const std::vector& start_idx_prob, } void UIEModel::GetSpanIdxAndProbs( const SPAN_SET& span_set, - const std::vector& offset_mapping, + const std::vector& offset_mapping, std::vector* span_idxs, std::vector* probs) const { auto first_sep_idx = std::find_if(offset_mapping.begin() + 1, offset_mapping.end(), - [](const faster_tokenizer::core::Offset& offset) { - return offset == faster_tokenizer::core::Offset(0, 0); + [](const fast_tokenizer::core::Offset& offset) { + return offset == fast_tokenizer::core::Offset(0, 0); }); auto prompt_end_token_id = std::distance(offset_mapping.begin(), first_sep_idx) - 1; @@ -384,9 +384,9 @@ void UIEModel::ConvertSpanToUIEResult( std::string span_text; std::vector offset_mapping; if (span_idxs[i][j].is_prompt_) { - faster_tokenizer::pretokenizers::CharToBytesOffsetConverter converter( + fast_tokenizer::pretokenizers::CharToBytesOffsetConverter converter( prompt); - faster_tokenizer::core::Offset byte_offset; + fast_tokenizer::core::Offset byte_offset; converter.convert({start, end}, &byte_offset); span_text = prompt.substr(byte_offset.first, byte_offset.second - byte_offset.first); @@ -394,9 +394,9 @@ void UIEModel::ConvertSpanToUIEResult( start = 0; end = 0; } else { - faster_tokenizer::pretokenizers::CharToBytesOffsetConverter converter( + fast_tokenizer::pretokenizers::CharToBytesOffsetConverter converter( text); - faster_tokenizer::core::Offset byte_offset; + fast_tokenizer::core::Offset byte_offset; converter.convert({start, end}, &byte_offset); span_text = text.substr(byte_offset.first, byte_offset.second - byte_offset.first); @@ -461,14 +461,14 @@ void UIEModel::AutoJoiner(const std::vector& short_texts, for (auto&& result_idx : input_mapping_item) { if (result_idx == 0) { result_list = std::move((*results)[result_idx]); - offset += faster_tokenizer::utils::GetUnicodeLenFromUTF8( + offset += fast_tokenizer::utils::GetUnicodeLenFromUTF8( short_texts[result_idx].c_str(), short_texts[result_idx].size()); } else { for (auto&& curr_result : (*results)[result_idx]) { curr_result.start_ += offset; curr_result.end_ += offset; } - offset += faster_tokenizer::utils::GetUnicodeLenFromUTF8( + offset += fast_tokenizer::utils::GetUnicodeLenFromUTF8( short_texts[result_idx].c_str(), short_texts[result_idx].size()); result_list.insert(result_list.end(), (*results)[result_idx].begin(), (*results)[result_idx].end()); @@ -521,13 +521,13 @@ bool UIEModel::ConstructTextsAndPrompts( auto max_prompt_iter = std::max_element( prompts->begin(), prompts->end(), [](const std::string& lhs, const std::string& rhs) { - auto lhs_ulen = faster_tokenizer::utils::GetUnicodeLenFromUTF8( + auto lhs_ulen = fast_tokenizer::utils::GetUnicodeLenFromUTF8( lhs.c_str(), lhs.length()); - auto rhs_ulen = faster_tokenizer::utils::GetUnicodeLenFromUTF8( + auto rhs_ulen = fast_tokenizer::utils::GetUnicodeLenFromUTF8( rhs.c_str(), rhs.length()); return lhs_ulen < rhs_ulen; }); - auto max_prompt_len = faster_tokenizer::utils::GetUnicodeLenFromUTF8( + auto max_prompt_len = fast_tokenizer::utils::GetUnicodeLenFromUTF8( max_prompt_iter->c_str(), max_prompt_iter->length()); auto max_predict_len = max_length_ - 3 - max_prompt_len; @@ -547,10 +547,10 @@ bool UIEModel::ConstructTextsAndPrompts( void UIEModel::Preprocess( const std::vector& input_texts, const std::vector& prompts, - std::vector* encodings, + std::vector* encodings, std::vector* inputs) { // 1. Tokenize the short texts and short prompts - std::vector text_pair_input; + std::vector text_pair_input; for (int i = 0; i < input_texts.size(); ++i) { text_pair_input.emplace_back( std::pair(prompts[i], input_texts[i])); @@ -596,7 +596,7 @@ void UIEModel::Preprocess( void UIEModel::Postprocess( const std::vector& outputs, - const std::vector& encodings, + const std::vector& encodings, const std::vector& short_input_texts, const std::vector& short_prompts, const std::vector>& input_mapping_with_short_text, @@ -611,7 +611,7 @@ void UIEModel::Postprocess( GetCandidateIdx(end_prob, outputs[1].shape[0], outputs[1].shape[1], &end_candidate_idx_prob, position_prob_); - std::vector> offset_mapping; + std::vector> offset_mapping; for (int i = 0; i < encodings.size(); ++i) { auto&& curr_offsets = encodings[i].GetOffsets(); offset_mapping.push_back(curr_offsets); @@ -739,7 +739,7 @@ void UIEModel::Predict( if (has_prompt) { // 2. Convert texts and prompts to FDTensor std::vector inputs; - std::vector encodings; + std::vector encodings; Preprocess(short_input_texts, short_prompts, &encodings, &inputs); // 3. Infer diff --git a/fastdeploy/text/uie/model.h b/fastdeploy/text/uie/model.h index 48b21f8b0f7..c813369d3f9 100644 --- a/fastdeploy/text/uie/model.h +++ b/fastdeploy/text/uie/model.h @@ -21,7 +21,7 @@ #include #include "fastdeploy/fastdeploy_model.h" #include "fastdeploy/utils/unique_ptr.h" -#include "faster_tokenizer/tokenizers/ernie_faster_tokenizer.h" +#include "fast_tokenizer/tokenizers/ernie_fast_tokenizer.h" using namespace paddlenlp; @@ -133,11 +133,11 @@ struct FASTDEPLOY_DECL UIEModel : public FastDeployModel { std::vector>* input_mapping_with_short_text); void Preprocess(const std::vector& input_texts, const std::vector& prompts, - std::vector* encodings, + std::vector* encodings, std::vector* inputs); void Postprocess( const std::vector& outputs, - const std::vector& encodings, + const std::vector& encodings, const std::vector& short_input_texts, const std::vector& short_prompts, const std::vector>& input_mapping_with_short_text, @@ -167,7 +167,7 @@ struct FASTDEPLOY_DECL UIEModel : public FastDeployModel { }; using SPAN_SET = std::set, IdxProbCmp>; struct SpanIdx { - faster_tokenizer::core::Offset offset_; + fast_tokenizer::core::Offset offset_; bool is_prompt_; }; void SetValidBackend(); @@ -188,7 +188,7 @@ struct FASTDEPLOY_DECL UIEModel : public FastDeployModel { SPAN_SET* span_set) const; void GetSpanIdxAndProbs( const SPAN_SET& span_set, - const std::vector& offset_mapping, + const std::vector& offset_mapping, std::vector* span_idxs, std::vector* probs) const; void ConvertSpanToUIEResult( const std::vector& texts, @@ -200,7 +200,7 @@ struct FASTDEPLOY_DECL UIEModel : public FastDeployModel { size_t max_length_; float position_prob_; SchemaLanguage schema_language_; - faster_tokenizer::tokenizers_impl::ErnieFasterTokenizer tokenizer_; + fast_tokenizer::tokenizers_impl::ErnieFastTokenizer tokenizer_; }; } // namespace text diff --git a/scripts/fastdeploy_init.bat b/scripts/fastdeploy_init.bat index a7b8567b256..aaaf8e75e63 100644 --- a/scripts/fastdeploy_init.bat +++ b/scripts/fastdeploy_init.bat @@ -46,8 +46,8 @@ if "%__script_action_type%" == "show" ( echo !__3rd_lib_file! | findstr "opencv">nul && set __3rd_needed_flag=true echo !__3rd_lib_file! | findstr "opencv">nul && set __api_tag=!__api_tag!::vision if "!__3rd_needed_flag!"=="true" (echo !__3rd_lib_file! | findstr d\.lib>nul && set __3rd_needed_flag=false) - echo !__3rd_lib_file! | findstr "faster_tokenizer">nul && set __3rd_needed_flag=true - echo !__3rd_lib_file! | findstr "faster_tokenizer">nul && set __api_tag=!__api_tag!::text + echo !__3rd_lib_file! | findstr "fast_tokenizer">nul && set __3rd_needed_flag=true + echo !__3rd_lib_file! | findstr "fast_tokenizer">nul && set __api_tag=!__api_tag!::text if "!__3rd_needed_flag!"=="true" (echo [Lib] !__3rd_lib_file! **[NEEDED][!__api_tag!]**) else (echo [Lib] !__3rd_lib_file!) ) @@ -58,8 +58,8 @@ if "%__script_action_type%" == "show" ( set __3rd_include_dir=%%a && set __3rd_needed_flag=false && set __api_tag=fastdeploy echo !__3rd_include_dir! | findstr "opencv">nul && set __3rd_needed_flag=true echo !__3rd_include_dir! | findstr "opencv">nul && set __api_tag=!__api_tag!::vision - echo !__3rd_include_dir! | findstr "faster_tokenizer">nul && set __3rd_needed_flag=true - echo !__3rd_include_dir! | findstr "faster_tokenizer">nul && set __api_tag=!__api_tag!::text + echo !__3rd_include_dir! | findstr "fast_tokenizer">nul && set __3rd_needed_flag=true + echo !__3rd_include_dir! | findstr "fast_tokenizer">nul && set __api_tag=!__api_tag!::text if "!__3rd_needed_flag!"=="true" (echo [Include] !__3rd_include_dir! **[NEEDED][!__api_tag!]**) else (echo [Include] !__3rd_include_dir!) ) @@ -164,4 +164,4 @@ echo --------------------------------------------------------------------------- goto:eof @rem end -@echo on \ No newline at end of file +@echo on diff --git a/scripts/fastdeploy_init.sh b/scripts/fastdeploy_init.sh index 1e5bc0bb355..0f39e884424 100644 --- a/scripts/fastdeploy_init.sh +++ b/scripts/fastdeploy_init.sh @@ -60,9 +60,4 @@ if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/paddlelite ]; th echo "Paddle Lite Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/paddlelite/lib" fi -if [ -d ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer ]; then - export LD_LIBRARY_PATH=${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer/lib:${LD_LIBRARY_PATH} - echo "Faster Tokenizer Lib: ${INSTALLED_PREBUILT_FASTDEPLOY_DIR}/third_libs/install/faster_tokenizer/lib" -fi - cd ${CURRENT_EXE_DIR} diff --git a/serving/Dockerfile b/serving/Dockerfile index cca94ebe931..fc5a4569357 100644 --- a/serving/Dockerfile +++ b/serving/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update \ RUN apt-get update \ && apt-get install -y --no-install-recommends libre2-5 libb64-0d python3 python3-pip libarchive-dev ffmpeg libsm6 libxext6 \ && python3 -m pip install -U pip \ - && python3 -m pip install paddlepaddle-gpu paddlenlp faster_tokenizer + && python3 -m pip install paddlepaddle-gpu paddlenlp fast-tokenizer-python COPY python/dist/*.whl /opt/fastdeploy/ RUN python3 -m pip install /opt/fastdeploy/*.whl \ diff --git a/serving/Dockerfile_cpu b/serving/Dockerfile_cpu index 9e6d8b0ef47..59171e85f06 100644 --- a/serving/Dockerfile_cpu +++ b/serving/Dockerfile_cpu @@ -19,7 +19,7 @@ ENV TZ=Asia/Shanghai \ RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libgomp1 ffmpeg libsm6 libxext6 \ && python3 -m pip install -U pip \ - && python3 -m pip install paddlepaddle paddlenlp faster_tokenizer + && python3 -m pip install paddlepaddle paddlenlp fast-tokenizer-python COPY python/dist/*.whl *.whl /opt/fastdeploy/ RUN python3 -m pip install /opt/fastdeploy/*.whl \ diff --git a/tests/release_task/cpp_run.bat b/tests/release_task/cpp_run.bat index 4d92ea51b73..8584733c3e6 100644 --- a/tests/release_task/cpp_run.bat +++ b/tests/release_task/cpp_run.bat @@ -58,8 +58,8 @@ set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\insta set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mklml\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle2onnx\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\tensorrt\lib;%PATH% -set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\lib;%PATH% -set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\third_party\lib;%PATH% +set PATH=%FASTDEPLOY_HOME%\third_libs\install\fast_tokenizer\lib;%PATH% +set PATH=%FASTDEPLOY_HOME%\third_libs\install\fast_tokenizer\third_party\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\yaml-cpp\lib;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv\build\x64\vc15\bin;%PATH% set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\bin;%PATH% From 58fcded1cb7b3e04f88c3b00e3aa9c1864a86656 Mon Sep 17 00:00:00 2001 From: jiangjiajun Date: Mon, 21 Nov 2022 06:58:54 +0000 Subject: [PATCH 077/136] Remove static libraries while building --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b24ea2e729..203e35d9db6 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -368,6 +368,8 @@ if(ENABLE_TRT_BACKEND) find_package(Python COMPONENTS Interpreter Development REQUIRED) message(STATUS "Copying ${TRT_DIRECTORY}/lib to ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib ...") execute_process(COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/copy_directory.py ${TRT_DIRECTORY}/lib ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib) + file(GLOB_RECURSE TRT_STAIC_LIBS ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib/*.a) + file(REMOVE ${TRT_STAIC_LIBS}) if(UNIX AND (NOT APPLE) AND (NOT ANDROID)) execute_process(COMMAND sh -c "ls *.so*" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib COMMAND sh -c "xargs ${PATCHELF_EXE} --set-rpath '$ORIGIN'" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib From d2b1e1579c07a3411ca85ed19770da456f5b3aaa Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Mon, 21 Nov 2022 15:22:28 +0800 Subject: [PATCH 078/136] [Android] add fastdeploy android sdk download task (#646) * [Bug Fix] fix android app detail page errors * [Android] fix realtime camera mode and shutter * [Bug Fix] fix AllocateSegmentationResultFromJava error * [Bug Fix] fix camera preview size setting problem * [Model] use uint8 buffer instead of fp32 in ppseg postprocess * [Model] revert changes in ppseg * [Model] revert postprocess changes in ppseg * [Android] add fastdeploy android sdk download task --- java/android/app/build.gradle | 77 +++++++++++++++---- .../app/src/main/res/values/strings.xml | 2 +- java/android/fastdeploy/build.gradle | 18 +++-- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/java/android/app/build.gradle b/java/android/app/build.gradle index d106ff18a48..0e2b5f3268b 100644 --- a/java/android/app/build.gradle +++ b/java/android/app/build.gradle @@ -25,33 +25,39 @@ android { } dependencies { - implementation fileTree(include: ['*.jar'], dir: 'libs') + implementation fileTree(include: ['*.aar'], dir: 'libs') implementation 'com.android.support:appcompat-v7:28.0.0' //noinspection GradleDependency implementation 'com.android.support.constraint:constraint-layout:1.1.3' implementation 'com.android.support:design:28.0.0' implementation 'org.jetbrains:annotations:15.0' - implementation project(path: ':fastdeploy') //noinspection GradleDependency testImplementation 'junit:junit:4.12' androidTestImplementation 'com.android.support.test:runner:1.0.2' androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' } -def archives = [ +def FD_MODEL = [ [ - 'src': 'https://bj.bcebos.com/paddlehub/fastdeploy/picodet_s_320_coco_lcnet.tgz', - 'dest' : 'src/main/assets/models' + 'src' : 'https://bj.bcebos.com/paddlehub/fastdeploy/picodet_s_320_coco_lcnet.tgz', + 'dest': 'src/main/assets/models' ], [ - 'src': 'https://bj.bcebos.com/paddlehub/fastdeploy/picodet_l_320_coco_lcnet.tgz', - 'dest' : 'src/main/assets/models' + 'src' : 'https://bj.bcebos.com/paddlehub/fastdeploy/picodet_l_320_coco_lcnet.tgz', + 'dest': 'src/main/assets/models' ] ] -task downloadAndExtractArchives(type: DefaultTask) { +def FD_JAVA_SDK = [ + [ + 'src' : 'https://bj.bcebos.com/fastdeploy/test/fastdeploy-android-sdk-latest-dev.aar', + 'dest': 'libs' + ] +] + +task downloadAndExtractModels(type: DefaultTask) { doFirst { - println "Downloading and extracting archives including models ..." + println "Downloading and extracting fastdeploy models ..." } doLast { // Prepare cache folder for archives @@ -59,24 +65,63 @@ task downloadAndExtractArchives(type: DefaultTask) { if (!file("${cachePath}").exists()) { mkdir "${cachePath}" } - archives.eachWithIndex { archive, index -> + FD_MODEL.eachWithIndex { model, index -> MessageDigest messageDigest = MessageDigest.getInstance('MD5') - messageDigest.update(archive.src.bytes) + messageDigest.update(model.src.bytes) String cacheName = new BigInteger(1, messageDigest.digest()).toString(32) // Download the target archive if not exists - boolean copyFiles = !file("${archive.dest}").exists() + boolean copyFiles = !file("${model.dest}").exists() if (!file("${cachePath}/${cacheName}.tgz").exists()) { - ant.get(src: archive.src, dest: file("${cachePath}/${cacheName}.tgz")) - copyFiles = true // force to copy files from the latest archive files + println "Downloading ${model.src} -> ${cachePath}/${cacheName}.tgz" + ant.get(src: model.src, dest: file("${cachePath}/${cacheName}.tgz")) + copyFiles = true + // force to copy files from the latest archive files } // Extract the target archive if its dest path does not exists if (copyFiles) { + println "Coping ${cachePath}/${cacheName}.aar -> ${model.dest}" copy { from tarTree("${cachePath}/${cacheName}.tgz") - into "${archive.dest}" + into "${model.dest}" + } + } + } + } +} + +task downloadAndExtractSDKs(type: DefaultTask) { + doFirst { + println "Downloading and extracting fastdeploy android java sdk ..." + } + doLast { + // Prepare cache folder for archives + String cachePath = "cache" + if (!file("${cachePath}").exists()) { + mkdir "${cachePath}" + } + FD_JAVA_SDK.eachWithIndex { sdk, index -> + String[] sdkPaths = sdk.src.split("/") + // e.g fastdeploy-android-sdk-latest-dev.aar + String sdkName = sdkPaths[sdkPaths.length - 1] + // Download the target SDK if not exists + boolean copyFiles = !file("${sdk.dest}/${sdkName}").exists() + if (!file("${cachePath}/${sdkName}").exists()) { + println "Downloading ${sdk.src} -> ${cachePath}/${sdkName}" + ant.get(src: sdk.src, dest: file("${cachePath}/${sdkName}")) + copyFiles = true + // force to copy files from the latest downloaded sdk + } + // Copy the target archive if its dest path does not exists + if (copyFiles) { + println "Coping ${cachePath}/${sdkName} -> ${sdk.dest}/${sdkName}" + copy { + from "${cachePath}/${sdkName}" + into "${sdk.dest}" } } } } } -preBuild.dependsOn downloadAndExtractArchives \ No newline at end of file + +preBuild.dependsOn downloadAndExtractSDKs +preBuild.dependsOn downloadAndExtractModels \ No newline at end of file diff --git a/java/android/app/src/main/res/values/strings.xml b/java/android/app/src/main/res/values/strings.xml index 267871056fe..b5c396f5f78 100644 --- a/java/android/app/src/main/res/values/strings.xml +++ b/java/android/app/src/main/res/values/strings.xml @@ -35,7 +35,7 @@ models/scrfd_500m_bnkps_shape320x320_pd - models/portrait_pp_humansegv2_lite_256x144_inference_model + models/human_pp_humansegv1_lite_192x192_inference_model 拍照识别 实时识别 diff --git a/java/android/fastdeploy/build.gradle b/java/android/fastdeploy/build.gradle index b83ccee5480..a7bc3f67e24 100644 --- a/java/android/fastdeploy/build.gradle +++ b/java/android/fastdeploy/build.gradle @@ -44,7 +44,7 @@ dependencies { androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' } -def archives = [ +def FD_CXX_LIB = [ [ 'src' : 'https://bj.bcebos.com/fastdeploy/test/fastdeploy-android-latest-shared-dev.tgz', 'dest': 'libs' @@ -53,7 +53,7 @@ def archives = [ task downloadAndExtractLibs(type: DefaultTask) { doFirst { - println "Downloading and extracting archives including libs ..." + println "Downloading and extracting fastdeploy android c++ lib ..." } doLast { // Prepare cache folder for archives @@ -61,24 +61,26 @@ task downloadAndExtractLibs(type: DefaultTask) { if (!file("${cachePath}").exists()) { mkdir "${cachePath}" } - archives.eachWithIndex { archive, index -> + FD_CXX_LIB.eachWithIndex { lib, index -> MessageDigest messageDigest = MessageDigest.getInstance('MD5') - messageDigest.update(archive.src.bytes) + messageDigest.update(lib.src.bytes) String cacheName = new BigInteger(1, messageDigest.digest()).toString(32) // Download the target archive if not exists - boolean copyFiles = !file("${archive.dest}").exists() + boolean copyFiles = !file("${lib.dest}").exists() if (!file("${cachePath}/${cacheName}.tgz").exists()) { - ant.get(src: archive.src, dest: file("${cachePath}/${cacheName}.tgz")) - copyFiles = true // force to copy files from the latest archive files + ant.get(src: lib.src, dest: file("${cachePath}/${cacheName}.tgz")) + copyFiles = true + // force to copy files from the latest archive files } // Extract the target archive if its dest path does not exists if (copyFiles) { copy { from tarTree("${cachePath}/${cacheName}.tgz") - into "${archive.dest}" + into "${lib.dest}" } } } } } + preBuild.dependsOn downloadAndExtractLibs \ No newline at end of file From 759b799a9022254afb66753174963627f1f2e977 Mon Sep 17 00:00:00 2001 From: yunyaoXYY <109218879+yunyaoXYY@users.noreply.github.com> Date: Mon, 21 Nov 2022 16:44:40 +0800 Subject: [PATCH 079/136] [Bug Fix] Fix PPOCRv2 python example (#645) * Imporve OCR Readme * Improve OCR Readme * Improve OCR Readme * Improve OCR Readme * Improve OCR Readme * Add Initialize function to PP-OCR * Add Initialize function to PP-OCR * Add Initialize function to PP-OCR * Make all the model links come from PaddleOCR * Improve OCR readme * Improve OCR readme * Improve OCR readme * Improve OCR readme * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add Readme for vision results * Add check for label file in postprocess of Rec model * Add check for label file in postprocess of Rec model * Add check for label file in postprocess of Rec model * Add check for label file in postprocess of Rec model * Add check for label file in postprocess of Rec model * Add check for label file in postprocess of Rec model * Add comments to create API docs * Improve OCR comments * Rename OCR and add comments * Make sure previous python example works * Make sure previous python example works * Fix Rec model bug * Fix Rec model bug * Fix rec model bug * Add SetTrtMaxBatchSize function for TensorRT * Add SetTrtMaxBatchSize Pybind * Add set_trt_max_batch_size python function * Set TRT dynamic shape in PPOCR examples * Set TRT dynamic shape in PPOCR examples * Set TRT dynamic shape in PPOCR examples * Fix PPOCRv2 python example Co-authored-by: Jason --- examples/vision/ocr/PP-OCRv2/python/infer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/vision/ocr/PP-OCRv2/python/infer.py b/examples/vision/ocr/PP-OCRv2/python/infer.py index 1bb94eb7e60..f3274c9e46c 100644 --- a/examples/vision/ocr/PP-OCRv2/python/infer.py +++ b/examples/vision/ocr/PP-OCRv2/python/infer.py @@ -96,6 +96,10 @@ def build_option(args): rec_params_file = os.path.join(args.rec_model, "inference.pdiparams") rec_label_file = args.rec_label_file +# 对于三个模型，均采用同样的部署配置 +# 用户也可根据自行需求分别配置 +runtime_option = build_option(args) + det_option = runtime_option cls_option = runtime_option rec_option = runtime_option From b963f7c4d5d7d26dd7bb0c2c4a73ba9844650b8b Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 21 Nov 2022 16:44:52 +0800 Subject: [PATCH 080/136] [Other] Remove static libraries while building (#644) Remove static libraries while building From d2779d13d19fe1d756fd87a9847de9e258db2777 Mon Sep 17 00:00:00 2001 From: Jason Date: Mon, 21 Nov 2022 17:04:38 +0800 Subject: [PATCH 081/136] Delete Doxyfile --- Doxyfile | 2494 ------------------------------------------------------ 1 file changed, 2494 deletions(-) delete mode 100644 Doxyfile diff --git a/Doxyfile b/Doxyfile deleted file mode 100644 index 46806d5418a..00000000000 --- a/Doxyfile +++ /dev/null @@ -1,2494 +0,0 @@ -# Doxyfile 1.8.13 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all text -# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv -# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv -# for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = "FastDeploy" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = "latest" - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = "Fast & Easy to Deploy!" - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = "./docs" - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:\n" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines. - -ALIASES = - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: -# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: -# Fortran. In the later case the parser tries to guess whether the code is fixed -# or free formatted code, this is the default for Fortran type files), VHDL. For -# instance to make doxygen treat .inc files as Fortran files (default is PHP), -# and .f files as C (default is Fortran), use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 0 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = NO - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = NO - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = YES - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = YES - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. - -CASE_SENSE_NAMES = YES - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = NO - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = YES - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = ./fastdeploy/ - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: http://www.gnu.org/software/libiconv) for the list of -# possible encodings. -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. - -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ - *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f \ - *.for \ - *.tcl \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = fastdeploy/README.md - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# function all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see http://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the config file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the -# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the -# cost of reduced performance. This can be particularly helpful with template -# rich C++ code for which doxygen's built-in parser lacks the necessary type -# information. -# Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse-libclang=ON option for CMake. -# The default value is: NO. - -CLANG_ASSISTED_PARSING = NO - -# If clang assisted parsing is enabled you can provide the compiler with command -# line options that you would normally use when invoking the compiler. Note that -# the include paths will already be set by doxygen for the files and directories -# specified with INPUT and INCLUDE_PATH. -# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. - -CLANG_OPTIONS = - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# http://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: http://developer.apple.com/tools/xcode/), introduced with -# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- -# folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# http://www.mathjax.org) which uses client side Javascript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from http://www.mathjax.org before deployment. -# The default value is: http://cdn.mathjax.org/mathjax/latest. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /