From 7af4dba46d12ba2a91a1c002f47a1381aeb8a28d Mon Sep 17 00:00:00 2001 From: DefTruth Date: Fri, 28 Oct 2022 20:46:53 +0800 Subject: [PATCH 1/3] [Android] integrate flycv into Andorid --- CMakeLists.txt | 1 + cmake/flycv.cmake | 89 +++++++++++++++++++++++++++++++----------- cmake/paddlelite.cmake | 12 ++++-- 3 files changed, 76 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc963cd20a0..ccff224ab22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,7 @@ option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF) option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF) option(ENABLE_VISION "Whether to enable vision models usage." OFF) option(ENABLE_TEXT "Whether to enable text models usage." OFF) +option(ENABLE_FLYCV "Whether to enable flycv to boost image preprocess." OFF) option(WITH_TESTING "Whether to compile with unittest." OFF) ############################# Options for Android cross compiling ######################### option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF) diff --git a/cmake/flycv.cmake b/cmake/flycv.cmake index 4b3bbf386e1..32cedb26652 100644 --- a/cmake/flycv.cmake +++ b/cmake/flycv.cmake @@ -21,13 +21,30 @@ set(FLYCV_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flycv) set(FLYCV_INC_DIR "${FLYCV_INSTALL_DIR}/include" CACHE PATH "flycv include directory." FORCE) -set(FLYCV_LIB_DIR - "${FLYCV_INSTALL_DIR}/lib/" - CACHE PATH "flycv lib directory." FORCE) +if(ANDROID) + set(FLYCV_LIB_DIR + "${FLYCV_INSTALL_DIR}/lib/${ANDROID_ABI}" + CACHE PATH "flycv lib directory." FORCE) +else() + set(FLYCV_LIB_DIR + "${FLYCV_INSTALL_DIR}/lib/" + CACHE PATH "flycv lib directory." FORCE) +endif() set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${FLYCV_LIB_DIR}") include_directories(${FLYCV_INC_DIR}) + +# ABI check +if(ANDROID) + if((NOT ANDROID_ABI MATCHES "armeabi-v7a") AND (NOT ANDROID_ABI MATCHES "arm64-v8a")) + message(FATAL_ERROR "FastDeploy with FlyCV only support armeabi-v7a, arm64-v8a now.") + endif() + if(NOT ANDROID_TOOLCHAIN MATCHES "clang") + message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with FlyCV, but found ${ANDROID_TOOLCHAIN}.") + endif() +endif() + if(WIN32) set(FLYCV_COMPILE_LIB "${FLYCV_INSTALL_DIR}/lib/flycv.lib" @@ -35,7 +52,11 @@ if(WIN32) elseif(APPLE) set(FLYCV_COMPILE_LIB "${FLYCV_INSTALL_DIR}/lib/libflycv.dylib" - CACHE FILEPATH "flycv compile library." FORCE) + CACHE FILEPATH "flycv compile library." FORCE) +elseif(ANDROID) + set(FLYCV_COMPILE_LIB + "${FLYCV_INSTALL_DIR}/lib/${ANDROID_ABI}/libflycv.so" + CACHE FILEPATH "flycv compile library." FORCE) else() set(FLYCV_COMPILE_LIB "${FLYCV_INSTALL_DIR}/lib/libflycv_shared.so" @@ -45,41 +66,63 @@ endif(WIN32) set(FLYCV_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/") set(FLYCV_VERSION "1.0.1025") if(WIN32) - message(FATAL_ERROR "FalconCV is not supported on Windows now.") + message(FATAL_ERROR "FlyCV is not supported on Windows now.") set(FLYCV_FILE "flycv-win-x64-${FLYCV_VERSION}.zip") elseif(APPLE) - message(FATAL_ERROR "FalconCV is not supported on Mac OSX now.") + message(FATAL_ERROR "FlyCV is not supported on Mac OSX now.") if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64") set(FLYCV_FILE "flycv-osx-arm64-${FLYCV_VERSION}.tgz") else() set(FLYCV_FILE "flycv-osx-x86_64-${FLYCV_VERSION}.tgz") endif() +elseif(ANDROID) + set(FLYCV_FILE "flycv-android-${FLYCV_VERSION}.tgz") else() if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64") set(FLYCV_FILE "flycv-linux-aarch64-${FLYCV_VERSION}.tgz") else() - message(FATAL_ERROR "FalconCV is not supported on Linux x64 now.") + message(FATAL_ERROR "FlyCV is not supported on Linux x64 now.") set(FLYCV_FILE "flycv-linux-x64-${FLYCV_VERSION}.tgz") endif() endif() set(FLYCV_URL "${FLYCV_URL_BASE}${FLYCV_FILE}") -ExternalProject_Add( - ${FLYCV_PROJECT} - ${EXTERNAL_PROJECT_LOG_ARGS} - URL ${FLYCV_URL} - PREFIX ${FLYCV_PREFIX_DIR} - DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - UPDATE_COMMAND "" - INSTALL_COMMAND - ${CMAKE_COMMAND} -E remove_directory ${FLYCV_INSTALL_DIR} && - ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR} && - ${CMAKE_COMMAND} -E rename ${FLYCV_SOURCE_DIR}/lib/ - ${FLYCV_LIB_DIR} && ${CMAKE_COMMAND} -E copy_directory - ${FLYCV_SOURCE_DIR}/include ${FLYCV_INC_DIR} - BUILD_BYPRODUCTS ${FLYCV_COMPILE_LIB}) +if(ANDROID) + ExternalProject_Add( + ${FLYCV_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${FLYCV_URL} + PREFIX ${FLYCV_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${FLYCV_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR}/lib && + ${CMAKE_COMMAND} -E rename ${FLYCV_SOURCE_DIR}/lib/${ANDROID_ABI} + ${FLYCV_LIB_DIR} && ${CMAKE_COMMAND} -E copy_directory + ${FLYCV_SOURCE_DIR}/include ${FLYCV_INC_DIR} + BUILD_BYPRODUCTS ${FLYCV_COMPILE_LIB}) +else() + ExternalProject_Add( + ${FLYCV_PROJECT} + ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${FLYCV_URL} + PREFIX ${FLYCV_PREFIX_DIR} + DOWNLOAD_NO_PROGRESS 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND + ${CMAKE_COMMAND} -E remove_directory ${FLYCV_INSTALL_DIR} && + ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR} && + ${CMAKE_COMMAND} -E rename ${FLYCV_SOURCE_DIR}/lib/ + ${FLYCV_LIB_DIR} && ${CMAKE_COMMAND} -E copy_directory + ${FLYCV_SOURCE_DIR}/include ${FLYCV_INC_DIR} + BUILD_BYPRODUCTS ${FLYCV_COMPILE_LIB}) +endif() add_library(external_flycv STATIC IMPORTED GLOBAL) set_property(TARGET external_flycv PROPERTY IMPORTED_LOCATION diff --git a/cmake/paddlelite.cmake b/cmake/paddlelite.cmake index a2489c34b32..1f807bfcecf 100644 --- a/cmake/paddlelite.cmake +++ b/cmake/paddlelite.cmake @@ -23,9 +23,15 @@ set(PADDLELITE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${PADDLELITE_FILENAME}) set(PADDLELITE_INC_DIR "${PADDLELITE_INSTALL_DIR}/include" CACHE PATH "paddlelite include directory." FORCE) -set(PADDLELITE_LIB_DIR - "${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI}" - CACHE PATH "paddlelite lib directory." FORCE) +if(ANDROID) + set(PADDLELITE_LIB_DIR + "${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI}" + CACHE PATH "paddlelite lib directory." FORCE) +else() + set(PADDLELITE_LIB_DIR + "${PADDLELITE_INSTALL_DIR}/lib/" + CACHE PATH "paddlelite lib directory." FORCE) +endif() set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLELITE_LIB_DIR}") set(PADDLELITE_URL_PREFIX "https://bj.bcebos.com/fastdeploy/third_libs") From fce44e765e29c5cb594abc7e161b4f58ae1248a7 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Fri, 28 Oct 2022 20:51:00 +0800 Subject: [PATCH 2/3] Falconcv (#27) * [Bug Fix ]Update ppseg visualize python api && demo (#454) Update ppseg visualize python api && demo * [Model] update pptracking letterboxresize and add some comment (#438) * add override mark * delete some * recovery * recovery * add tracking * add tracking py_bind and example * add pptracking * add pptracking * iomanip head file * add opencv_video lib * add python libs package Signed-off-by: ChaoII <849453582@qq.com> * complete comments Signed-off-by: ChaoII <849453582@qq.com> * add jdeTracker_ member variable Signed-off-by: ChaoII <849453582@qq.com> * add 'FASTDEPLOY_DECL' macro Signed-off-by: ChaoII <849453582@qq.com> * remove kwargs params Signed-off-by: ChaoII <849453582@qq.com> * [Doc]update pptracking docs * delete 'ENABLE_PADDLE_FRONTEND' switch * add pptracking unit test * update pptracking unit test Signed-off-by: ChaoII <849453582@qq.com> * modify test video file path and remove trt test * update unit test model url * remove 'FASTDEPLOY_DECL' macro Signed-off-by: ChaoII <849453582@qq.com> * fix build python packages about pptracking on win32 Signed-off-by: ChaoII <849453582@qq.com> * update comment Signed-off-by: ChaoII <849453582@qq.com> * add pptracking model explain Signed-off-by: ChaoII <849453582@qq.com> Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: Jason Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: ChaoII <849453582@qq.com> Co-authored-by: Jason --- .../segmentation/paddleseg/cpp/infer.cc | 6 +- .../segmentation/paddleseg/python/infer.py | 2 +- examples/vision/tracking/pptracking/README.md | 35 +++++++++ .../vision/tracking/pptracking/cpp/README.md | 1 - .../vision/common/processors/letter_box.cc | 71 +++++++++++++++++++ .../processors}/letter_box.h | 37 +++++----- .../vision/common/processors/transform.h | 1 + fastdeploy/vision/common/result.h | 15 +++- .../vision/tracking/pptracking/letter_box.cc | 59 --------------- .../vision/tracking/pptracking/model.cc | 37 ---------- fastdeploy/vision/tracking/pptracking/model.h | 15 +--- .../vision/tracking/pptracking/__init__.py | 13 ++++ .../fastdeploy/vision/visualize/__init__.py | 4 +- 13 files changed, 161 insertions(+), 135 deletions(-) create mode 100644 examples/vision/tracking/pptracking/README.md create mode 100644 fastdeploy/vision/common/processors/letter_box.cc rename fastdeploy/vision/{tracking/pptracking => common/processors}/letter_box.h (52%) delete mode 100644 fastdeploy/vision/tracking/pptracking/letter_box.cc diff --git a/examples/vision/segmentation/paddleseg/cpp/infer.cc b/examples/vision/segmentation/paddleseg/cpp/infer.cc index 37f16f04eb3..d3c46270435 100644 --- a/examples/vision/segmentation/paddleseg/cpp/infer.cc +++ b/examples/vision/segmentation/paddleseg/cpp/infer.cc @@ -43,7 +43,7 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { } std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + auto vis_im = fastdeploy::vision::VisSegmentation(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; } @@ -72,7 +72,7 @@ void GpuInfer(const std::string& model_dir, const std::string& image_file) { } std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + auto vis_im = fastdeploy::vision::VisSegmentation(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; } @@ -102,7 +102,7 @@ void TrtInfer(const std::string& model_dir, const std::string& image_file) { } std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + auto vis_im = fastdeploy::vision::VisSegmentation(im, res, 0.5); cv::imwrite("vis_result.jpg", vis_im); std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; } diff --git a/examples/vision/segmentation/paddleseg/python/infer.py b/examples/vision/segmentation/paddleseg/python/infer.py index dd65ebf2735..866e32bfb8b 100644 --- a/examples/vision/segmentation/paddleseg/python/infer.py +++ b/examples/vision/segmentation/paddleseg/python/infer.py @@ -53,5 +53,5 @@ def build_option(args): print(result) # 可视化结果 -vis_im = fd.vision.visualize.vis_segmentation(im, result) +vis_im = fd.vision.vis_segmentation(im, result, weight=0.5) cv2.imwrite("vis_img.png", vis_im) diff --git a/examples/vision/tracking/pptracking/README.md b/examples/vision/tracking/pptracking/README.md new file mode 100644 index 00000000000..35b9d7173d2 --- /dev/null +++ b/examples/vision/tracking/pptracking/README.md @@ -0,0 +1,35 @@ +# PP-Tracking模型部署 + +## 模型版本说明 + +- [PaddleDetection release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [PP-Tracking系列模型](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/configs/mot) + + +## 导出部署模型 + +在部署前，需要先将训练好的PP-Tracking导出成部署模型，导出PPTracking导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/pptracking/cpp/README.md)。 + + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了PP-Tracking行人跟踪垂类模型，开发者可直接下载使用，更多模型参见[PPTracking](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/pptracking/README_cn.md)。 + +| 模型 | 参数大小 | 精度 | 备注 | +|:-----------------------------------------------------------------------------------------------------|:-------|:----- | :------ | +| [PP-Tracking](https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz) | 51.2MB | - | + +**说明** +- 仅支持JDE模型（JDE，FairMOT，MCFairMOT)； +- 目前暂不支持SDE模型的部署，待PaddleDetection官方更新SED部署代码后，对SDE模型进行支持。 + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/tracking/pptracking/cpp/README.md b/examples/vision/tracking/pptracking/cpp/README.md index e5a67106d36..9adef9dc793 100644 --- a/examples/vision/tracking/pptracking/cpp/README.md +++ b/examples/vision/tracking/pptracking/cpp/README.md @@ -22,7 +22,6 @@ make -j wget https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz tar -xvf fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz wget https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4 -wget https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4 # CPU推理 diff --git a/fastdeploy/vision/common/processors/letter_box.cc b/fastdeploy/vision/common/processors/letter_box.cc new file mode 100644 index 00000000000..423aef1d361 --- /dev/null +++ b/fastdeploy/vision/common/processors/letter_box.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/letter_box.h" + +namespace fastdeploy{ +namespace vision{ +bool LetterBoxResize::ImplByOpenCV(Mat* mat) { + + if (mat->Channels() != color_.size()) { + FDERROR << "Pad: Require input channels equals to size of padding value, " + "but now channels = " + << mat->Channels() + << ", the size of padding values = " << color_.size() << "." + << std::endl; + return false; + } + cv::Mat* im = mat->GetOpenCVMat(); + // generate scale_factor + int origin_w = mat->Width(); + int origin_h = mat->Height(); + int target_h = target_size_[0]; + int target_w = target_size_[1]; + float ratio_h = static_cast(target_h) / static_cast(origin_h); + float ratio_w = static_cast(target_w) / static_cast(origin_w); + float resize_scale = std::min(ratio_h, ratio_w); + // get_resized_shape + int new_shape_w = std::round(im->cols * resize_scale); + int new_shape_h = std::round(im->rows * resize_scale); + // calculate pad + float padw = (target_size_[1] - new_shape_w) / 2.; + float padh = (target_size_[0] - new_shape_h) / 2.; + int top = std::round(padh - 0.1); + int bottom = std::round(padh + 0.1); + int left = std::round(padw - 0.1); + int right = std::round(padw + 0.1); + cv::resize(*im, *im, cv::Size(new_shape_w, new_shape_h), 0, 0, cv::INTER_AREA); + cv::Scalar color; + if (color_.size() == 1) { + color = cv::Scalar(color_[0]); + } else if (color_.size() == 2) { + color = cv::Scalar(color_[0], color_[1]); + } else if (color_.size() == 3) { + color = cv::Scalar(color_[0], color_[1], color_[2]); + } else { + color = cv::Scalar(color_[0], color_[1], color_[2], color_[3]); + } + cv::copyMakeBorder(*im, *im, top, bottom, left, right, cv::BORDER_CONSTANT, color); + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + return true; +} + +bool LetterBoxResize::Run(Mat* mat, const std::vector& target_size, const std::vector& color, ProcLib lib) { + auto l = LetterBoxResize(target_size,color); + return l(mat, lib); +} + +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/letter_box.h b/fastdeploy/vision/common/processors/letter_box.h similarity index 52% rename from fastdeploy/vision/tracking/pptracking/letter_box.h rename to fastdeploy/vision/common/processors/letter_box.h index 17f9f283383..25003b161a1 100644 --- a/fastdeploy/vision/tracking/pptracking/letter_box.h +++ b/fastdeploy/vision/common/processors/letter_box.h @@ -1,4 +1,4 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,23 +14,28 @@ #pragma once -#include "fastdeploy/vision/common/processors/transform.h" -#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/base.h" + namespace fastdeploy { namespace vision { -namespace tracking { -class LetterBoxResize: public Processor{ -public: - LetterBoxResize(const std::vector& target_size, const std::vector& color); - bool ImplByOpenCV(Mat* mat) override; - std::string Name() override { return "LetterBoxResize"; } -private: - std::vector target_size_; - std::vector color_; -}; -} // namespace tracking -} // namespace vision -} // namespace fastdeploy +class LetterBoxResize : public Processor { + public: + LetterBoxResize(const std::vector& target_size, const std::vector& color) { + target_size_ = target_size; + color_ = color; + } + + bool ImplByOpenCV(Mat* mat); + std::string Name() { return "LetterBoxResize"; } + static bool Run(Mat* mat,const std::vector& target_size, const std::vector& color, + ProcLib lib = ProcLib::OPENCV); + + private: + std::vector target_size_; + std::vector color_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/transform.h b/fastdeploy/vision/common/processors/transform.h index c6bd7cc2ce2..8d1b5010d06 100644 --- a/fastdeploy/vision/common/processors/transform.h +++ b/fastdeploy/vision/common/processors/transform.h @@ -32,6 +32,7 @@ #include "fastdeploy/vision/common/processors/stride_pad.h" #include "fastdeploy/vision/common/processors/normalize_and_permute.h" #include "fastdeploy/vision/common/processors/warp_affine.h" +#include "fastdeploy/vision/common/processors/letter_box.h" namespace fastdeploy { namespace vision { diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index c9a8d113a4e..f08c00d82a7 100644 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -155,16 +155,25 @@ struct FASTDEPLOY_DECL OCRResult : public BaseResult { std::string Str(); }; +/*! @brief MOT(Multi-Object Tracking) result structure for all the MOT models + */ struct FASTDEPLOY_DECL MOTResult : public BaseResult { - // left top right bottom + /** \brief All the tracking object boxes for an input image, the size of `boxes` is the number of tracking objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax] + */ std::vector> boxes; + /** \brief All the tracking object ids + */ std::vector ids; + /** \brief The confidence for all the tracking objects + */ std::vector scores; + /** \brief The classify label id for all the tracking object + */ std::vector class_ids; ResultType type = ResultType::MOT; - + /// Clear MOT result void Clear(); - + /// Debug function, convert the result to string to print std::string Str(); }; diff --git a/fastdeploy/vision/tracking/pptracking/letter_box.cc b/fastdeploy/vision/tracking/pptracking/letter_box.cc deleted file mode 100644 index bc8e0cbcb26..00000000000 --- a/fastdeploy/vision/tracking/pptracking/letter_box.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/tracking/pptracking/letter_box.h" - -namespace fastdeploy { -namespace vision { -namespace tracking { - -LetterBoxResize::LetterBoxResize(const std::vector& target_size, const std::vector& color){ - target_size_=target_size; - color_=color; -} -bool LetterBoxResize::ImplByOpenCV(Mat* mat){ - if (mat->Channels() != color_.size()) { - FDERROR << "Pad: Require input channels equals to size of padding value, " - "but now channels = " - << mat->Channels() - << ", the size of padding values = " << color_.size() << "." - << std::endl; - return false; - } - // generate scale_factor - int origin_w = mat->Width(); - int origin_h = mat->Height(); - int target_h = target_size_[0]; - int target_w = target_size_[1]; - float ratio_h = static_cast(target_h) / static_cast(origin_h); - float ratio_w = static_cast(target_w) / static_cast(origin_w); - float resize_scale = std::min(ratio_h, ratio_w); - - int new_shape_w = std::round(mat->Width() * resize_scale); - int new_shape_h = std::round(mat->Height() * resize_scale); - float padw = (target_size_[1] - new_shape_w) / 2.; - float padh = (target_size_[0] - new_shape_h) / 2.; - int top = std::round(padh - 0.1); - int bottom = std::round(padh + 0.1); - int left = std::round(padw - 0.1); - int right = std::round(padw + 0.1); - - Resize::Run(mat,new_shape_w,new_shape_h); - Pad::Run(mat,top,bottom,left,right,color_); - return true; -} - -} // namespace tracking -} // namespace vision -} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index b4915f8a355..064892889b5 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -151,32 +151,7 @@ bool PPTracking::BuildPreprocessPipelineFromConfig(){ return true; } -void PPTracking::GetNmsInfo() { - if (runtime_option.model_format == ModelFormat::PADDLE) { - std::string contents; - if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { - return; - } - auto reader = paddle2onnx::PaddleReader(contents.c_str(), contents.size()); - if (reader.has_nms) { - has_nms_ = true; - background_label = reader.nms_params.background_label; - keep_top_k = reader.nms_params.keep_top_k; - nms_eta = reader.nms_params.nms_eta; - nms_threshold = reader.nms_params.nms_threshold; - score_threshold = reader.nms_params.score_threshold; - nms_top_k = reader.nms_params.nms_top_k; - normalized = reader.nms_params.normalized; - } - } -} - bool PPTracking::Initialize() { - // remove multiclass_nms3 now - // this is a trick operation for ppyoloe while inference on trt - GetNmsInfo(); - runtime_option.remove_multiclass_nms_ = true; - runtime_option.custom_op_info_["multiclass_nms3"] = "MultiClassNMS"; if (!BuildPreprocessPipelineFromConfig()) { FDERROR << "Failed to build preprocess pipeline from configuration file." << std::endl; @@ -186,18 +161,6 @@ bool PPTracking::Initialize() { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; return false; } - - if (has_nms_ && runtime_option.backend == Backend::TRT) { - FDINFO << "Detected operator multiclass_nms3 in your model, will replace " - "it with fastdeploy::backend::MultiClassNMS(background_label=" - << background_label << ", keep_top_k=" << keep_top_k - << ", nms_eta=" << nms_eta << ", nms_threshold=" << nms_threshold - << ", score_threshold=" << score_threshold - << ", nms_top_k=" << nms_top_k << ", normalized=" << normalized - << ")." << std::endl; - has_nms_ = false; - } - // create JDETracker instance std::unique_ptr jdeTracker(new JDETracker); jdeTracker_ = std::move(jdeTracker); diff --git a/fastdeploy/vision/tracking/pptracking/model.h b/fastdeploy/vision/tracking/pptracking/model.h index 040ed383f13..eece2700024 100644 --- a/fastdeploy/vision/tracking/pptracking/model.h +++ b/fastdeploy/vision/tracking/pptracking/model.h @@ -18,7 +18,7 @@ #include "fastdeploy/fastdeploy_model.h" #include "fastdeploy/vision/common/result.h" #include "fastdeploy/vision/tracking/pptracking/tracker.h" -#include "fastdeploy/vision/tracking/pptracking/letter_box.h" +//#include "fastdeploy/vision/tracking/pptracking/letter_box.h" namespace fastdeploy { namespace vision { @@ -56,8 +56,8 @@ class FASTDEPLOY_DECL PPTracking: public FastDeployModel { private: bool BuildPreprocessPipelineFromConfig(); + bool Initialize(); - void GetNmsInfo(); bool Preprocess(Mat* img, std::vector* outputs); @@ -69,19 +69,8 @@ class FASTDEPLOY_DECL PPTracking: public FastDeployModel { float conf_thresh_; float tracked_thresh_; float min_box_area_; - bool is_scale_ = true; std::unique_ptr jdeTracker_; - // configuration for nms - int64_t background_label = -1; - int64_t keep_top_k = 300; - float nms_eta = 1.0; - float nms_threshold = 0.7; - float score_threshold = 0.01; - int64_t nms_top_k = 10000; - bool normalized = true; - bool has_nms_ = true; - }; } // namespace tracking diff --git a/python/fastdeploy/vision/tracking/pptracking/__init__.py b/python/fastdeploy/vision/tracking/pptracking/__init__.py index 9da0d669fa9..89ca2a7b014 100644 --- a/python/fastdeploy/vision/tracking/pptracking/__init__.py +++ b/python/fastdeploy/vision/tracking/pptracking/__init__.py @@ -24,6 +24,14 @@ def __init__(self, config_file, runtime_option=None, model_format=ModelFormat.PADDLE): + """Load a PPTracking model exported by PaddleDetection. + + :param model_file: (str)Path of model file, e.g pptracking/model.pdmodel + :param params_file: (str)Path of parameters file, e.g ppyoloe/model.pdiparams + :param config_file: (str)Path of configuration file for deployment, e.g ppyoloe/infer_cfg.yml + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ super(PPTracking, self).__init__(runtime_option) assert model_format == ModelFormat.PADDLE, "PPTracking model only support model format of ModelFormat.Paddle now." @@ -33,5 +41,10 @@ def __init__(self, assert self.initialized, "PPTracking model initialize failed." def predict(self, input_image): + """Predict the MOT result for an input image + + :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :return: MOTResult + """ assert input_image is not None, "The input image data is None." return self._model.predict(input_image) diff --git a/python/fastdeploy/vision/visualize/__init__.py b/python/fastdeploy/vision/visualize/__init__.py index 37ac727dca2..403ac1e3add 100644 --- a/python/fastdeploy/vision/visualize/__init__.py +++ b/python/fastdeploy/vision/visualize/__init__.py @@ -36,8 +36,8 @@ def vis_face_detection(im_data, face_det_result, line_size=1, font_size=0.5): font_size) -def vis_segmentation(im_data, seg_result): - return C.vision.vis_segmentation(im_data, seg_result) +def vis_segmentation(im_data, seg_result, weight=0.5): + return C.vision.vis_segmentation(im_data, seg_result, weight) def vis_matting_alpha(im_data, From f698c793be8211cd745243b2d09000b48870d087 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Fri, 28 Oct 2022 22:07:26 +0800 Subject: [PATCH 3/3] [cmake] optimize examples CMakeLists.txt --- FastDeploy.cmake.in | 12 +++++++-- cmake/flycv.cmake | 2 +- examples/CMakeLists.txt | 25 ++++++++++++++++--- .../vision/tracking/pptracking/model.cc | 1 - 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in index 3ecb2a3f68f..e1a1cf2f4af 100755 --- a/FastDeploy.cmake.in +++ b/FastDeploy.cmake.in @@ -189,9 +189,17 @@ if(ENABLE_VISION) if(ENABLE_FLYCV) include_directories(${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/include) - find_library(FLYCV_LIB flycv_shared ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/lib NO_DEFAULT_PATH) - list(APPEND FASTDEPLOY_LIBS ${FLYCV_LIB}) + set(FLYCV_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/lib) + if(ANDROID) + add_library(flycv_shared STATIC IMPORTED GLOBAL) + set_property(TARGET flycv_shared PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libflycv_shared.so) + list(APPEND FASTDEPLOY_LIBS flycv_shared) + else() + find_library(FLYCV_LIB flycv_shared ${FLYCV_LIB_DIR} NO_DEFAULT_PATH) + list(APPEND FASTDEPLOY_LIBS ${FLYCV_LIB}) + endif() endif() + endif() if (ENABLE_TEXT) diff --git a/cmake/flycv.cmake b/cmake/flycv.cmake index 32cedb26652..52a77fb806a 100644 --- a/cmake/flycv.cmake +++ b/cmake/flycv.cmake @@ -55,7 +55,7 @@ elseif(APPLE) CACHE FILEPATH "flycv compile library." FORCE) elseif(ANDROID) set(FLYCV_COMPILE_LIB - "${FLYCV_INSTALL_DIR}/lib/${ANDROID_ABI}/libflycv.so" + "${FLYCV_INSTALL_DIR}/lib/${ANDROID_ABI}/libflycv_shared.so" CACHE FILEPATH "flycv compile library." FORCE) else() set(FLYCV_COMPILE_LIB diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 7af04b9443b..6f3c65988b6 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -37,6 +37,12 @@ function(config_fastdeploy_executable_link_flags TARGET_NAME) endif() endfunction() +# Usage: add_fastdeploy_executable_cc_files(xxx_var vision detection) +function(add_fastdeploy_executable_cc_files CC_FILES_VAR FIELD SUB_FIELD) + file(GLOB_RECURSE _EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/${FIELD}/${SUB_FIELD}/*/cpp/*.cc) + set(${CC_FILES_VAR} ${_EXAMPLE_SRCS} PARENT_SCOPE) +endfunction() + set(EXAMPLES_NUM 0) function(add_fastdeploy_executable FIELD CC_FILE) # temp target name/file var in function scope @@ -69,9 +75,22 @@ if(BUILD_EXAMPLES AND ENABLE_VISION) if(EXISTS ${PROJECT_SOURCE_DIR}/examples/vision) message(STATUS "") message(STATUS "*************FastDeploy Vision Examples Summary**********") - file(GLOB_RECURSE ALL_VISION_EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/*/*/cpp/*.cc) - foreach(_CC_FILE ${ALL_VISION_EXAMPLE_SRCS}) - add_fastdeploy_executable(vision ${_CC_FILE}) + set(ALL_VISION_SUD_FIELDS classification + detection + facedet + faceid + keypointdetection + matting + ocr + segmentation) + if(NOT ANDROID) + list(APPEND ALL_VISION_SUD_FIELDS tracking) + endif() + foreach(_SUB_FIELD ${ALL_VISION_SUD_FIELDS}) + add_fastdeploy_executable_cc_files(_SUB_CC_FILES vision ${_SUB_FIELD}) + foreach(_CC_FILE ${_SUB_CC_FILES}) + add_fastdeploy_executable(vision ${_CC_FILE}) + endforeach() endforeach() message(STATUS " [FastDeploy Executable Path] : ${EXECUTABLE_OUTPUT_PATH}") endif() diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index 064892889b5..97d4e1ab948 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -14,7 +14,6 @@ #include "fastdeploy/vision/tracking/pptracking/model.h" #include "yaml-cpp/yaml.h" -#include "paddle2onnx/converter.h" namespace fastdeploy { namespace vision {