From 92eb0ab61c7b1af373dee128d0cb12f93e03bd2b Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Fri, 30 Sep 2022 16:08:55 +0800 Subject: [PATCH 01/25] add override mark --- fastdeploy/backends/ort/ort_backend.h | 10 ++-- fastdeploy/backends/paddle/paddle_backend.h | 10 ++-- fastdeploy/core/fd_type.cc | 51 +++++++++++++++++++++ fastdeploy/core/fd_type.h | 4 ++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/fastdeploy/backends/ort/ort_backend.h b/fastdeploy/backends/ort/ort_backend.h index 10c841ab293..445d555405e 100644 --- a/fastdeploy/backends/ort/ort_backend.h +++ b/fastdeploy/backends/ort/ort_backend.h @@ -66,14 +66,14 @@ class OrtBackend : public BaseBackend { const OrtBackendOption& option = OrtBackendOption(), bool from_memory_buffer = false); - bool Infer(std::vector& inputs, std::vector* outputs); + bool Infer(std::vector& inputs, std::vector* outputs) override; - int NumInputs() const { return inputs_desc_.size(); } + int NumInputs() const override { return inputs_desc_.size(); } - int NumOutputs() const { return outputs_desc_.size(); } + int NumOutputs() const override { return outputs_desc_.size(); } - TensorInfo GetInputInfo(int index); - TensorInfo GetOutputInfo(int index); + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; std::vector GetInputInfos() override; std::vector GetOutputInfos() override; static std::vector custom_operators_; diff --git a/fastdeploy/backends/paddle/paddle_backend.h b/fastdeploy/backends/paddle/paddle_backend.h index ce4b33d7c43..43f5a4a1749 100644 --- a/fastdeploy/backends/paddle/paddle_backend.h +++ b/fastdeploy/backends/paddle/paddle_backend.h @@ -67,14 +67,14 @@ class PaddleBackend : public BaseBackend { const std::string& model_file, const std::string& params_file, const PaddleBackendOption& option = PaddleBackendOption()); - bool Infer(std::vector& inputs, std::vector* outputs); + bool Infer(std::vector& inputs, std::vector* outputs) override; - int NumInputs() const { return inputs_desc_.size(); } + int NumInputs() const override { return inputs_desc_.size(); } - int NumOutputs() const { return outputs_desc_.size(); } + int NumOutputs() const override { return outputs_desc_.size(); } - TensorInfo GetInputInfo(int index); - TensorInfo GetOutputInfo(int index); + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; std::vector GetInputInfos() override; std::vector GetOutputInfos() override; diff --git a/fastdeploy/core/fd_type.cc b/fastdeploy/core/fd_type.cc index 15844802aaa..376f868b303 100644 --- a/fastdeploy/core/fd_type.cc +++ b/fastdeploy/core/fd_type.cc @@ -57,6 +57,22 @@ std::string Str(const Device& d) { return out; } +std::ostream& operator<<(std::ostream& out,const Device& d){ + switch (d) { + case Device::CPU: + out << "Device::CPU"; + break; + case Device::GPU: + out << "Device::GPU"; + break; + default: + out << "Device::UNKOWN"; + } + return out; +} + + + std::string Str(const FDDataType& fdt) { std::string out; switch (fdt) { @@ -93,6 +109,41 @@ std::string Str(const FDDataType& fdt) { return out; } +std::ostream& operator<<(std::ostream& out,const FDDataType& fdt){ + switch (fdt) { + case FDDataType::BOOL: + out << "FDDataType::BOOL"; + break; + case FDDataType::INT16: + out << "FDDataType::INT16"; + break; + case FDDataType::INT32: + out << "FDDataType::INT32"; + break; + case FDDataType::INT64: + out << "FDDataType::INT64"; + break; + case FDDataType::FP32: + out << "FDDataType::FP32"; + break; + case FDDataType::FP64: + out << "FDDataType::FP64"; + break; + case FDDataType::FP16: + out << "FDDataType::FP16"; + break; + case FDDataType::UINT8: + out << "FDDataType::UINT8"; + break; + case FDDataType::INT8: + out << "FDDataType::INT8"; + break; + default: + out << "FDDataType::UNKNOWN"; + } + return out; +} + template const FDDataType TypeToDataType::dtype = UNKNOWN1; diff --git a/fastdeploy/core/fd_type.h b/fastdeploy/core/fd_type.h index 22f33e0e5ad..f2746c51b0b 100644 --- a/fastdeploy/core/fd_type.h +++ b/fastdeploy/core/fd_type.h @@ -51,6 +51,10 @@ enum FASTDEPLOY_DECL FDDataType { INT8 }; +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,const Device& d); + +FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out,const FDDataType& fdt); + FASTDEPLOY_DECL std::string Str(const FDDataType& fdt); FASTDEPLOY_DECL int32_t FDDataTypeSize(const FDDataType& data_dtype); From cff22b888109c17473ce770f32aed2297a6f3cd8 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Sun, 9 Oct 2022 11:41:41 +0800 Subject: [PATCH 02/25] delete some --- fastdeploy/vision/detection/ppdet/jde_track.cpp | 5 +++++ fastdeploy/vision/detection/ppdet/jde_track.h | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 fastdeploy/vision/detection/ppdet/jde_track.cpp create mode 100644 fastdeploy/vision/detection/ppdet/jde_track.h diff --git a/fastdeploy/vision/detection/ppdet/jde_track.cpp b/fastdeploy/vision/detection/ppdet/jde_track.cpp new file mode 100644 index 00000000000..ac5580771f8 --- /dev/null +++ b/fastdeploy/vision/detection/ppdet/jde_track.cpp @@ -0,0 +1,5 @@ +// +// Created by aichao on 2022/10/8. +// + +#include "jde_track.h" diff --git a/fastdeploy/vision/detection/ppdet/jde_track.h b/fastdeploy/vision/detection/ppdet/jde_track.h new file mode 100644 index 00000000000..6509a991ec5 --- /dev/null +++ b/fastdeploy/vision/detection/ppdet/jde_track.h @@ -0,0 +1,16 @@ +// +// Created by aichao on 2022/10/8. +// + +#ifndef FASTDEPLOY_JDE_TRACK_H +#define FASTDEPLOY_JDE_TRACK_H + + + +class jde_track { + +}; + + + +#endif //FASTDEPLOY_JDE_TRACK_H From db3c81e0042cd0a657bd9712e195b5bb5dea7314 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Sun, 9 Oct 2022 14:45:32 +0800 Subject: [PATCH 03/25] recovery --- .../tracking/pptracking/jde_predictor.cc | 235 ++++++++ .../tracking/pptracking/jde_predictor.h | 97 ++++ .../vision/tracking/pptracking/lapjv.cpp | 409 ++++++++++++++ fastdeploy/vision/tracking/pptracking/lapjv.h | 64 +++ .../vision/tracking/pptracking/pipeline.cc | 367 +++++++++++++ .../vision/tracking/pptracking/pipeline.h | 142 +++++ .../vision/tracking/pptracking/predictor.cc | 35 ++ .../vision/tracking/pptracking/predictor.h | 99 ++++ .../tracking/pptracking/sde_predictor.cc | 46 ++ .../tracking/pptracking/sde_predictor.h | 106 ++++ .../vision/tracking/pptracking/tracker.cc | 304 ++++++++++ .../vision/tracking/pptracking/tracker.h | 72 +++ .../vision/tracking/pptracking/trajectory.cc | 517 ++++++++++++++++++ .../vision/tracking/pptracking/trajectory.h | 230 ++++++++ .../tracking/pptracking/utils/postprocess.cc | 207 +++++++ .../tracking/pptracking/utils/postprocess.h | 62 +++ .../pptracking/utils/preprocess_op.cc | 187 +++++++ .../tracking/pptracking/utils/preprocess_op.h | 171 ++++++ .../vision/tracking/pptracking/utils/utils.h | 44 ++ fastdeploy/vision/tracking/tracking_pybind.cc | 3 + 20 files changed, 3397 insertions(+) create mode 100644 fastdeploy/vision/tracking/pptracking/jde_predictor.cc create mode 100644 fastdeploy/vision/tracking/pptracking/jde_predictor.h create mode 100644 fastdeploy/vision/tracking/pptracking/lapjv.cpp create mode 100644 fastdeploy/vision/tracking/pptracking/lapjv.h create mode 100644 fastdeploy/vision/tracking/pptracking/pipeline.cc create mode 100644 fastdeploy/vision/tracking/pptracking/pipeline.h create mode 100644 fastdeploy/vision/tracking/pptracking/predictor.cc create mode 100644 fastdeploy/vision/tracking/pptracking/predictor.h create mode 100644 fastdeploy/vision/tracking/pptracking/sde_predictor.cc create mode 100644 fastdeploy/vision/tracking/pptracking/sde_predictor.h create mode 100644 fastdeploy/vision/tracking/pptracking/tracker.cc create mode 100644 fastdeploy/vision/tracking/pptracking/tracker.h create mode 100644 fastdeploy/vision/tracking/pptracking/trajectory.cc create mode 100644 fastdeploy/vision/tracking/pptracking/trajectory.h create mode 100644 fastdeploy/vision/tracking/pptracking/utils/postprocess.cc create mode 100644 fastdeploy/vision/tracking/pptracking/utils/postprocess.h create mode 100644 fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc create mode 100644 fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h create mode 100644 fastdeploy/vision/tracking/pptracking/utils/utils.h create mode 100644 fastdeploy/vision/tracking/tracking_pybind.cc diff --git a/fastdeploy/vision/tracking/pptracking/jde_predictor.cc b/fastdeploy/vision/tracking/pptracking/jde_predictor.cc new file mode 100644 index 00000000000..0673d8a0bb1 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/jde_predictor.cc @@ -0,0 +1,235 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +// for setprecision +#include +#include +#include "include/jde_predictor.h" + +using namespace paddle_infer; // NOLINT + +namespace PaddleDetection { + +// Load Model and create model predictor +void JDEPredictor::LoadModel(const std::string& model_dir, + const std::string& run_mode) { + paddle_infer::Config config; + std::string prog_file = model_dir + OS_PATH_SEP + "model.pdmodel"; + std::string params_file = model_dir + OS_PATH_SEP + "model.pdiparams"; + config.SetModel(prog_file, params_file); + if (this->device_ == "GPU") { + config.EnableUseGpu(200, this->gpu_id_); + config.SwitchIrOptim(true); + // use tensorrt + if (run_mode != "paddle") { + auto precision = paddle_infer::Config::Precision::kFloat32; + if (run_mode == "trt_fp32") { + precision = paddle_infer::Config::Precision::kFloat32; + } else if (run_mode == "trt_fp16") { + precision = paddle_infer::Config::Precision::kHalf; + } else if (run_mode == "trt_int8") { + precision = paddle_infer::Config::Precision::kInt8; + } else { + printf( + "run_mode should be 'paddle', 'trt_fp32', 'trt_fp16' or " + "'trt_int8'"); + } + // set tensorrt + config.EnableTensorRtEngine(1 << 30, + 1, + this->min_subgraph_size_, + precision, + false, + this->trt_calib_mode_); + } + } else if (this->device_ == "XPU") { + config.EnableXpu(10 * 1024 * 1024); + } else { + config.DisableGpu(); + if (this->use_mkldnn_) { + config.EnableMKLDNN(); + // cache 10 different shapes for mkldnn to avoid memory leak + config.SetMkldnnCacheCapacity(10); + } + config.SetCpuMathLibraryNumThreads(this->cpu_math_library_num_threads_); + } + config.SwitchUseFeedFetchOps(false); + config.SwitchIrOptim(true); + config.DisableGlogInfo(); + // Memory optimization + config.EnableMemoryOptim(); + predictor_ = std::move(CreatePredictor(config)); +} + +void FilterDets(const float conf_thresh, + const cv::Mat dets, + std::vector* index) { + for (int i = 0; i < dets.rows; ++i) { + float score = *dets.ptr(i, 4); + if (score > conf_thresh) { + index->push_back(i); + } + } +} + +void JDEPredictor::Preprocess(const cv::Mat& ori_im) { + // Clone the image : keep the original mat for postprocess + cv::Mat im = ori_im.clone(); + preprocessor_.Run(&im, &inputs_); +} + +void JDEPredictor::Postprocess(const cv::Mat dets, + const cv::Mat emb, + MOTResult* result) { + result->clear(); + std::vector tracks; + std::vector valid; + FilterDets(conf_thresh_, dets, &valid); + cv::Mat new_dets, new_emb; + for (int i = 0; i < valid.size(); ++i) { + new_dets.push_back(dets.row(valid[i])); + new_emb.push_back(emb.row(valid[i])); + } + JDETracker::instance()->update(new_dets, new_emb, &tracks); + if (tracks.size() == 0) { + MOTTrack mot_track; + Rect ret = {*dets.ptr(0, 0), + *dets.ptr(0, 1), + *dets.ptr(0, 2), + *dets.ptr(0, 3)}; + mot_track.ids = 1; + mot_track.score = *dets.ptr(0, 4); + mot_track.rects = ret; + result->push_back(mot_track); + } else { + std::vector::iterator titer; + for (titer = tracks.begin(); titer != tracks.end(); ++titer) { + if (titer->score < threshold_) { + continue; + } else { + float w = titer->ltrb[2] - titer->ltrb[0]; + float h = titer->ltrb[3] - titer->ltrb[1]; + bool vertical = w / h > 1.6; + float area = w * h; + if (area > min_box_area_ && !vertical) { + MOTTrack mot_track; + Rect ret = { + titer->ltrb[0], titer->ltrb[1], titer->ltrb[2], titer->ltrb[3]}; + mot_track.rects = ret; + mot_track.score = titer->score; + mot_track.ids = titer->id; + result->push_back(mot_track); + } + } + } + } +} + +void JDEPredictor::Predict(const std::vector imgs, + const double threshold, + MOTResult* result, + std::vector* times) { + auto preprocess_start = std::chrono::steady_clock::now(); + int batch_size = imgs.size(); + + // in_data_batch + std::vector in_data_all; + std::vector im_shape_all(batch_size * 2); + std::vector scale_factor_all(batch_size * 2); + + // Preprocess image + for (int bs_idx = 0; bs_idx < batch_size; bs_idx++) { + cv::Mat im = imgs.at(bs_idx); + Preprocess(im); + im_shape_all[bs_idx * 2] = inputs_.im_shape_[0]; + im_shape_all[bs_idx * 2 + 1] = inputs_.im_shape_[1]; + + scale_factor_all[bs_idx * 2] = inputs_.scale_factor_[0]; + scale_factor_all[bs_idx * 2 + 1] = inputs_.scale_factor_[1]; + + in_data_all.insert( + in_data_all.end(), inputs_.im_data_.begin(), inputs_.im_data_.end()); + } + + // Prepare input tensor + auto input_names = predictor_->GetInputNames(); + for (const auto& tensor_name : input_names) { + auto in_tensor = predictor_->GetInputHandle(tensor_name); + if (tensor_name == "image") { + int rh = inputs_.in_net_shape_[0]; + int rw = inputs_.in_net_shape_[1]; + in_tensor->Reshape({batch_size, 3, rh, rw}); + in_tensor->CopyFromCpu(in_data_all.data()); + } else if (tensor_name == "im_shape") { + in_tensor->Reshape({batch_size, 2}); + in_tensor->CopyFromCpu(im_shape_all.data()); + } else if (tensor_name == "scale_factor") { + in_tensor->Reshape({batch_size, 2}); + in_tensor->CopyFromCpu(scale_factor_all.data()); + } + } + + auto preprocess_end = std::chrono::steady_clock::now(); + std::vector bbox_shape; + std::vector emb_shape; + + // Run predictor + auto inference_start = std::chrono::steady_clock::now(); + predictor_->Run(); + // Get output tensor + auto output_names = predictor_->GetOutputNames(); + auto bbox_tensor = predictor_->GetOutputHandle(output_names[0]); + bbox_shape = bbox_tensor->shape(); + auto emb_tensor = predictor_->GetOutputHandle(output_names[1]); + emb_shape = emb_tensor->shape(); + // Calculate bbox length + int bbox_size = 1; + for (int j = 0; j < bbox_shape.size(); ++j) { + bbox_size *= bbox_shape[j]; + } + // Calculate emb length + int emb_size = 1; + for (int j = 0; j < emb_shape.size(); ++j) { + emb_size *= emb_shape[j]; + } + + bbox_data_.resize(bbox_size); + bbox_tensor->CopyToCpu(bbox_data_.data()); + + emb_data_.resize(emb_size); + emb_tensor->CopyToCpu(emb_data_.data()); + auto inference_end = std::chrono::steady_clock::now(); + + // Postprocessing result + auto postprocess_start = std::chrono::steady_clock::now(); + result->clear(); + + cv::Mat dets(bbox_shape[0], 6, CV_32FC1, bbox_data_.data()); + cv::Mat emb(bbox_shape[0], emb_shape[1], CV_32FC1, emb_data_.data()); + + Postprocess(dets, emb, result); + + auto postprocess_end = std::chrono::steady_clock::now(); + + std::chrono::duration preprocess_diff = + preprocess_end - preprocess_start; + (*times)[0] += static_cast(preprocess_diff.count() * 1000); + std::chrono::duration inference_diff = inference_end - inference_start; + (*times)[1] += static_cast(inference_diff.count() * 1000); + std::chrono::duration postprocess_diff = + postprocess_end - postprocess_start; + (*times)[2] += static_cast(postprocess_diff.count() * 1000); +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/jde_predictor.h b/fastdeploy/vision/tracking/pptracking/jde_predictor.h new file mode 100644 index 00000000000..32f5921061c --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/jde_predictor.h @@ -0,0 +1,97 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "paddle_inference_api.h" // NOLINT + +#include "include/config_parser.h" +#include "include/preprocess_op.h" +#include "include/utils.h" + +using namespace paddle_infer; // NOLINT + +namespace PaddleDetection { + +class JDEPredictor { + public: + explicit JDEPredictor(const std::string& device = "CPU", + const std::string& model_dir = "", + const double threshold = -1., + const std::string& run_mode = "paddle", + const int gpu_id = 0, + const bool use_mkldnn = false, + const int cpu_threads = 1, + bool trt_calib_mode = false, + const int min_box_area = 200) { + this->device_ = device; + this->gpu_id_ = gpu_id; + this->use_mkldnn_ = use_mkldnn; + this->cpu_math_library_num_threads_ = cpu_threads; + this->trt_calib_mode_ = trt_calib_mode; + this->min_box_area_ = min_box_area; + + config_.load_config(model_dir); + this->min_subgraph_size_ = config_.min_subgraph_size_; + preprocessor_.Init(config_.preprocess_info_); + LoadModel(model_dir, run_mode); + this->conf_thresh_ = config_.conf_thresh_; + } + + // Load Paddle inference model + void LoadModel(const std::string& model_dir, + const std::string& run_mode = "paddle"); + + // Run predictor + void Predict(const std::vector imgs, + const double threshold = 0.5, + MOTResult* result = nullptr, + std::vector* times = nullptr); + + private: + std::string device_ = "CPU"; + float threhold = 0.5; + int gpu_id_ = 0; + bool use_mkldnn_ = false; + int cpu_math_library_num_threads_ = 1; + int min_subgraph_size_ = 3; + bool trt_calib_mode_ = false; + + // Preprocess image and copy data to input buffer + void Preprocess(const cv::Mat& image_mat); + // Postprocess result + void Postprocess(const cv::Mat dets, const cv::Mat emb, MOTResult* result); + + std::shared_ptr predictor_; + Preprocessor preprocessor_; + ImageBlob inputs_; + std::vector bbox_data_; + std::vector emb_data_; + double threshold_; + ConfigPaser config_; + float min_box_area_; + float conf_thresh_; +}; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.cpp b/fastdeploy/vision/tracking/pptracking/lapjv.cpp new file mode 100644 index 00000000000..bb710e740e5 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/lapjv.cpp @@ -0,0 +1,409 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/gatagat/lap/blob/master/lap/lapjv.cpp +// Ths copyright of gatagat/lap is as follows: +// MIT License + +#include +#include +#include + +#include "include/lapjv.h" + +namespace PaddleDetection { + +/** Column-reduction and reduction transfer for a dense cost matrix. + */ +int _ccrrt_dense( + const int n, float *cost[], int *free_rows, int *x, int *y, float *v) { + int n_free_rows; + bool *unique; + + for (int i = 0; i < n; i++) { + x[i] = -1; + v[i] = LARGE; + y[i] = 0; + } + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + const float c = cost[i][j]; + if (c < v[j]) { + v[j] = c; + y[j] = i; + } + } + } + NEW(unique, bool, n); + memset(unique, TRUE, n); + { + int j = n; + do { + j--; + const int i = y[j]; + if (x[i] < 0) { + x[i] = j; + } else { + unique[i] = FALSE; + y[j] = -1; + } + } while (j > 0); + } + n_free_rows = 0; + for (int i = 0; i < n; i++) { + if (x[i] < 0) { + free_rows[n_free_rows++] = i; + } else if (unique[i]) { + const int j = x[i]; + float min = LARGE; + for (int j2 = 0; j2 < n; j2++) { + if (j2 == static_cast(j)) { + continue; + } + const float c = cost[i][j2] - v[j2]; + if (c < min) { + min = c; + } + } + v[j] -= min; + } + } + FREE(unique); + return n_free_rows; +} + +/** Augmenting row reduction for a dense cost matrix. + */ +int _carr_dense(const int n, + float *cost[], + const int n_free_rows, + int *free_rows, + int *x, + int *y, + float *v) { + int current = 0; + int new_free_rows = 0; + int rr_cnt = 0; + while (current < n_free_rows) { + int i0; + int j1, j2; + float v1, v2, v1_new; + bool v1_lowers; + + rr_cnt++; + const int free_i = free_rows[current++]; + j1 = 0; + v1 = cost[free_i][0] - v[0]; + j2 = -1; + v2 = LARGE; + for (int j = 1; j < n; j++) { + const float c = cost[free_i][j] - v[j]; + if (c < v2) { + if (c >= v1) { + v2 = c; + j2 = j; + } else { + v2 = v1; + v1 = c; + j2 = j1; + j1 = j; + } + } + } + i0 = y[j1]; + v1_new = v[j1] - (v2 - v1); + v1_lowers = v1_new < v[j1]; + if (rr_cnt < current * n) { + if (v1_lowers) { + v[j1] = v1_new; + } else if (i0 >= 0 && j2 >= 0) { + j1 = j2; + i0 = y[j2]; + } + if (i0 >= 0) { + if (v1_lowers) { + free_rows[--current] = i0; + } else { + free_rows[new_free_rows++] = i0; + } + } + } else { + if (i0 >= 0) { + free_rows[new_free_rows++] = i0; + } + } + x[free_i] = j1; + y[j1] = free_i; + } + return new_free_rows; +} + +/** Find columns with minimum d[j] and put them on the SCAN list. + */ +int _find_dense(const int n, int lo, float *d, int *cols, int *y) { + int hi = lo + 1; + float mind = d[cols[lo]]; + for (int k = hi; k < n; k++) { + int j = cols[k]; + if (d[j] <= mind) { + if (d[j] < mind) { + hi = lo; + mind = d[j]; + } + cols[k] = cols[hi]; + cols[hi++] = j; + } + } + return hi; +} + +// Scan all columns in TODO starting from arbitrary column in SCAN +// and try to decrease d of the TODO columns using the SCAN column. +int _scan_dense(const int n, + float *cost[], + int *plo, + int *phi, + float *d, + int *cols, + int *pred, + int *y, + float *v) { + int lo = *plo; + int hi = *phi; + float h, cred_ij; + + while (lo != hi) { + int j = cols[lo++]; + const int i = y[j]; + const float mind = d[j]; + h = cost[i][j] - v[j] - mind; + // For all columns in TODO + for (int k = hi; k < n; k++) { + j = cols[k]; + cred_ij = cost[i][j] - v[j] - h; + if (cred_ij < d[j]) { + d[j] = cred_ij; + pred[j] = i; + if (cred_ij == mind) { + if (y[j] < 0) { + return j; + } + cols[k] = cols[hi]; + cols[hi++] = j; + } + } + } + } + *plo = lo; + *phi = hi; + return -1; +} + +/** Single iteration of modified Dijkstra shortest path algorithm as explained + * in the JV paper. + * + * This is a dense matrix version. + * + * \return The closest free column index. + */ +int find_path_dense(const int n, + float *cost[], + const int start_i, + int *y, + float *v, + int *pred) { + int lo = 0, hi = 0; + int final_j = -1; + int n_ready = 0; + int *cols; + float *d; + + NEW(cols, int, n); + NEW(d, float, n); + + for (int i = 0; i < n; i++) { + cols[i] = i; + pred[i] = start_i; + d[i] = cost[start_i][i] - v[i]; + } + while (final_j == -1) { + // No columns left on the SCAN list. + if (lo == hi) { + n_ready = lo; + hi = _find_dense(n, lo, d, cols, y); + for (int k = lo; k < hi; k++) { + const int j = cols[k]; + if (y[j] < 0) { + final_j = j; + } + } + } + if (final_j == -1) { + final_j = _scan_dense(n, cost, &lo, &hi, d, cols, pred, y, v); + } + } + + { + const float mind = d[cols[lo]]; + for (int k = 0; k < n_ready; k++) { + const int j = cols[k]; + v[j] += d[j] - mind; + } + } + + FREE(cols); + FREE(d); + + return final_j; +} + +/** Augment for a dense cost matrix. + */ +int _ca_dense(const int n, + float *cost[], + const int n_free_rows, + int *free_rows, + int *x, + int *y, + float *v) { + int *pred; + + NEW(pred, int, n); + + for (int *pfree_i = free_rows; pfree_i < free_rows + n_free_rows; pfree_i++) { + int i = -1, j; + int k = 0; + + j = find_path_dense(n, cost, *pfree_i, y, v, pred); + while (i != *pfree_i) { + i = pred[j]; + y[j] = i; + SWAP_INDICES(j, x[i]); + k++; + } + } + FREE(pred); + return 0; +} + +/** Solve dense sparse LAP. + */ +int lapjv_internal(const cv::Mat &cost, + const bool extend_cost, + const float cost_limit, + int *x, + int *y) { + int n_rows = cost.rows; + int n_cols = cost.cols; + int n; + if (n_rows == n_cols) { + n = n_rows; + } else if (!extend_cost) { + throw std::invalid_argument( + "Square cost array expected. If cost is intentionally non-square, pass " + "extend_cost=True."); + } + + // Get extend cost + if (extend_cost || cost_limit < LARGE) { + n = n_rows + n_cols; + } + cv::Mat cost_expand(n, n, CV_32F); + float expand_value; + if (cost_limit < LARGE) { + expand_value = cost_limit / 2; + } else { + double max_v; + minMaxLoc(cost, nullptr, &max_v); + expand_value = static_cast(max_v) + 1.; + } + + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + cost_expand.at(i, j) = expand_value; + if (i >= n_rows && j >= n_cols) { + cost_expand.at(i, j) = 0; + } else if (i < n_rows && j < n_cols) { + cost_expand.at(i, j) = cost.at(i, j); + } + } + } + + // Convert Mat to pointer array + float **cost_ptr; + NEW(cost_ptr, float *, n); + for (int i = 0; i < n; ++i) { + NEW(cost_ptr[i], float, n); + } + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + cost_ptr[i][j] = cost_expand.at(i, j); + } + } + + int ret; + int *free_rows; + float *v; + int *x_c; + int *y_c; + + NEW(free_rows, int, n); + NEW(v, float, n); + NEW(x_c, int, n); + NEW(y_c, int, n); + + ret = _ccrrt_dense(n, cost_ptr, free_rows, x_c, y_c, v); + int i = 0; + while (ret > 0 && i < 2) { + ret = _carr_dense(n, cost_ptr, ret, free_rows, x_c, y_c, v); + i++; + } + if (ret > 0) { + ret = _ca_dense(n, cost_ptr, ret, free_rows, x_c, y_c, v); + } + FREE(v); + FREE(free_rows); + for (int i = 0; i < n; ++i) { + FREE(cost_ptr[i]); + } + FREE(cost_ptr); + if (ret != 0) { + if (ret == -1) { + throw "Out of memory."; + } + throw "Unknown error (lapjv_internal)"; + } + // Get output of x, y, opt + for (int i = 0; i < n; ++i) { + if (i < n_rows) { + x[i] = x_c[i]; + if (x[i] >= n_cols) { + x[i] = -1; + } + } + if (i < n_cols) { + y[i] = y_c[i]; + if (y[i] >= n_rows) { + y[i] = -1; + } + } + } + + FREE(x_c); + FREE(y_c); + return ret; +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.h b/fastdeploy/vision/tracking/pptracking/lapjv.h new file mode 100644 index 00000000000..ffaa010c005 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/lapjv.h @@ -0,0 +1,64 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/gatagat/lap/blob/master/lap/lapjv.h +// Ths copyright of gatagat/lap is as follows: +// MIT License + +#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ +#define DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ +#define LARGE 1000000 + +#if !defined TRUE +#define TRUE 1 +#endif +#if !defined FALSE +#define FALSE 0 +#endif + +#define NEW(x, t, n) \ + if ((x = reinterpret_cast(malloc(sizeof(t) * (n)))) == 0) { \ + return -1; \ + } +#define FREE(x) \ + if (x != 0) { \ + free(x); \ + x = 0; \ + } +#define SWAP_INDICES(a, b) \ + { \ + int_t _temp_index = a; \ + a = b; \ + b = _temp_index; \ + } +#include + +namespace PaddleDetection { + +typedef signed int int_t; +typedef unsigned int uint_t; +typedef double cost_t; +typedef char boolean; +typedef enum fp_t { FP_1 = 1, FP_2 = 2, FP_DYNAMIC = 3 } fp_t; + +int lapjv_internal(const cv::Mat &cost, + const bool extend_cost, + const float cost_limit, + int *x, + int *y); + +} // namespace PaddleDetection + +#endif // DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ diff --git a/fastdeploy/vision/tracking/pptracking/pipeline.cc b/fastdeploy/vision/tracking/pptracking/pipeline.cc new file mode 100644 index 00000000000..7c22c630f9f --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/pipeline.cc @@ -0,0 +1,367 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +// for setprecision +#include +#include +#include +#include + +#include "include/pipeline.h" +#include "include/postprocess.h" +#include "include/predictor.h" + +namespace PaddleDetection { + +void Pipeline::SetInput(const std::string& input_video) { + input_.push_back(input_video); +} + +void Pipeline::ClearInput() { + input_.clear(); + stream_.clear(); +} + +void Pipeline::SelectModel(const std::string& scene, + const bool tiny_obj, + const bool is_mtmct, + const std::string track_model_dir, + const std::string det_model_dir, + const std::string reid_model_dir) { + // model_dir has higher priority + if (!track_model_dir.empty()) { + track_model_dir_ = track_model_dir; + return; + } + if (!det_model_dir.empty() && !reid_model_dir.empty()) { + det_model_dir_ = det_model_dir; + reid_model_dir_ = reid_model_dir; + return; + } + + // Single camera model, based on FairMot + if (scene == "pedestrian") { + if (tiny_obj) { + track_model_dir_ = "../pedestrian_track_tiny"; + } else { + track_model_dir_ = "../pedestrian_track"; + } + } else if (scene != "vehicle") { + if (tiny_obj) { + track_model_dir_ = "../vehicle_track_tiny"; + } else { + track_model_dir_ = "../vehicle_track"; + } + } else if (scene == "multiclass") { + if (tiny_obj) { + track_model_dir_ = "../multiclass_track_tiny"; + } else { + track_model_dir_ = "../multiclass_track"; + } + } + + // Multi-camera model, based on PicoDet & LCNet + if (is_mtmct && scene == "pedestrian") { + det_model_dir_ = "../pedestrian_det"; + reid_model_dir_ = "../pedestrian_reid"; + } else if (is_mtmct && scene == "vehicle") { + det_model_dir_ = "../vehicle_det"; + reid_model_dir_ = "../vehicle_reid"; + } else if (is_mtmct && scene == "multiclass") { + throw "Multi-camera tracking is not supported in multiclass scene now."; + } +} + +void Pipeline::InitPredictor() { + if (track_model_dir_.empty() && det_model_dir_.empty()) { + throw "Predictor must receive track_model or det_model!"; + } + + if (!track_model_dir_.empty()) { + jde_sct_ = std::make_shared(device_, + track_model_dir_, + threshold_, + run_mode_, + gpu_id_, + use_mkldnn_, + cpu_threads_, + trt_calib_mode_); + } + if (!det_model_dir_.empty()) { + sde_sct_ = std::make_shared(device_, + det_model_dir_, + reid_model_dir_, + threshold_, + run_mode_, + gpu_id_, + use_mkldnn_, + cpu_threads_, + trt_calib_mode_); + } +} + +void Pipeline::Run() { + if (track_model_dir_.empty() && det_model_dir_.empty()) { + LOG(ERROR) << "Pipeline must use SelectModel before Run"; + return; + } + if (input_.size() == 0) { + LOG(ERROR) << "Pipeline must use SetInput before Run"; + return; + } + + if (!track_model_dir_.empty()) { + // single camera + if (input_.size() > 1) { + throw "Single camera tracking except single video, but received %d", + input_.size(); + } + PredictMOT(input_[0]); + } else { + // multi cameras + if (input_.size() != 2) { + throw "Multi camera tracking except two videos, but received %d", + input_.size(); + } + PredictMTMCT(input_); + } +} + +void Pipeline::PredictMOT(const std::string& video_path) { + // Open video + cv::VideoCapture capture; + capture.open(video_path.c_str()); + if (!capture.isOpened()) { + printf("can not open video : %s\n", video_path.c_str()); + return; + } + + // Get Video info : resolution, fps + int video_width = static_cast(capture.get(CV_CAP_PROP_FRAME_WIDTH)); + int video_height = static_cast(capture.get(CV_CAP_PROP_FRAME_HEIGHT)); + int video_fps = static_cast(capture.get(CV_CAP_PROP_FPS)); + + LOG(INFO) << "----------------------- Input info -----------------------"; + LOG(INFO) << "video_width: " << video_width; + LOG(INFO) << "video_height: " << video_height; + LOG(INFO) << "input fps: " << video_fps; + + // Create VideoWriter for output + cv::VideoWriter video_out; + std::string video_out_path = output_dir_ + OS_PATH_SEP + "mot_output.mp4"; + int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); + video_out.open(video_out_path.c_str(), + fcc, // 0x00000021, + video_fps, + cv::Size(video_width, video_height), + true); + if (!video_out.isOpened()) { + printf("create video writer failed!\n"); + return; + } + + PaddleDetection::MOTResult result; + std::vector det_times(3); + std::set id_set; + std::set interval_id_set; + std::vector in_id_list; + std::vector out_id_list; + std::map> prev_center; + Rect entrance = {0, + static_cast(video_height) / 2, + static_cast(video_width), + static_cast(video_height) / 2}; + double times; + double total_time; + // Capture all frames and do inference + cv::Mat frame; + int frame_id = 0; + + std::vector records; + std::vector flow_records; + records.push_back("result format: frame_id, track_id, x1, y1, w, h\n"); + + LOG(INFO) << "------------------- Predict info ------------------------"; + while (capture.read(frame)) { + if (frame.empty()) { + break; + } + std::vector imgs; + imgs.push_back(frame); + jde_sct_->Predict(imgs, threshold_, &result, &det_times); + frame_id += 1; + total_time = std::accumulate(det_times.begin(), det_times.end(), 0.); + times = total_time / frame_id; + + LOG(INFO) << "frame_id: " << frame_id + << " predict time(s): " << times / 1000; + + cv::Mat out_img = PaddleDetection::VisualizeTrackResult( + frame, result, 1000. / times, frame_id); + + // TODO(qianhui): the entrance line can be set by users + PaddleDetection::FlowStatistic(result, + frame_id, + secs_interval_, + do_entrance_counting_, + video_fps, + entrance, + &id_set, + &interval_id_set, + &in_id_list, + &out_id_list, + &prev_center, + &flow_records); + + if (save_result_) { + PaddleDetection::SaveMOTResult(result, frame_id, &records); + } + + // Draw the entrance line + if (do_entrance_counting_) { + float line_thickness = std::max(1, static_cast(video_width / 500.)); + cv::Point pt1 = cv::Point(entrance.left, entrance.top); + cv::Point pt2 = cv::Point(entrance.right, entrance.bottom); + cv::line(out_img, pt1, pt2, cv::Scalar(0, 255, 255), line_thickness); + } + video_out.write(out_img); + } + capture.release(); + video_out.release(); + PrintBenchmarkLog(det_times, frame_id); + LOG(INFO) << "-------------------- Final Output info -------------------"; + LOG(INFO) << "Total frame: " << frame_id; + LOG(INFO) << "Visualized output saved as " << video_out_path.c_str(); + if (save_result_) { + FILE* fp; + + std::string result_output_path = + output_dir_ + OS_PATH_SEP + "mot_output.txt"; + if ((fp = fopen(result_output_path.c_str(), "w+")) == NULL) { + printf("Open %s error.\n", result_output_path.c_str()); + return; + } + for (int l; l < records.size(); ++l) { + fprintf(fp, records[l].c_str()); + } + + fclose(fp); + LOG(INFO) << "txt result output saved as " << result_output_path.c_str(); + + result_output_path = output_dir_ + OS_PATH_SEP + "flow_statistic.txt"; + if ((fp = fopen(result_output_path.c_str(), "w+")) == NULL) { + printf("Open %s error.\n", result_output_path); + return; + } + for (int l; l < flow_records.size(); ++l) { + fprintf(fp, flow_records[l].c_str()); + } + fclose(fp); + LOG(INFO) << "txt flow statistic saved as " << result_output_path.c_str(); + } +} + +void Pipeline::PredictMTMCT(const std::vector video_path) { + throw "Not Implement!"; +} + +void Pipeline::RunMOTStream(const cv::Mat img, + const int frame_id, + const int video_fps, + const Rect entrance, + cv::Mat out_img, + std::vector* records, + std::set* id_set, + std::set* interval_id_set, + std::vector* in_id_list, + std::vector* out_id_list, + std::map>* prev_center, + std::vector* flow_records) { + PaddleDetection::MOTResult result; + std::vector det_times(3); + double times; + double total_time; + + LOG(INFO) << "------------------- Predict info ------------------------"; + std::vector imgs; + imgs.push_back(img); + jde_sct_->Predict(imgs, threshold_, &result, &det_times); + total_time = std::accumulate(det_times.begin(), det_times.end(), 0.); + times = total_time / frame_id; + + LOG(INFO) << "frame_id: " << frame_id << " predict time(s): " << times / 1000; + + out_img = PaddleDetection::VisualizeTrackResult( + img, result, 1000. / times, frame_id); + + // Count total number + // Count in & out number + PaddleDetection::FlowStatistic(result, + frame_id, + secs_interval_, + do_entrance_counting_, + video_fps, + entrance, + id_set, + interval_id_set, + in_id_list, + out_id_list, + prev_center, + flow_records); + + PrintBenchmarkLog(det_times, frame_id); + if (save_result_) { + PaddleDetection::SaveMOTResult(result, frame_id, records); + } +} + +void Pipeline::RunMTMCTStream(const std::vector imgs, + std::vector* records) { + throw "Not Implement!"; +} + +void Pipeline::PrintBenchmarkLog(const std::vector det_time, + const int img_num) { + LOG(INFO) << "----------------------- Config info -----------------------"; + LOG(INFO) << "runtime_device: " << device_; + LOG(INFO) << "ir_optim: " + << "True"; + LOG(INFO) << "enable_memory_optim: " + << "True"; + int has_trt = run_mode_.find("trt"); + if (has_trt >= 0) { + LOG(INFO) << "enable_tensorrt: " + << "True"; + std::string precision = run_mode_.substr(4, 8); + LOG(INFO) << "precision: " << precision; + } else { + LOG(INFO) << "enable_tensorrt: " + << "False"; + LOG(INFO) << "precision: " + << "fp32"; + } + LOG(INFO) << "enable_mkldnn: " << (use_mkldnn_ ? "True" : "False"); + LOG(INFO) << "cpu_math_library_num_threads: " << cpu_threads_; + LOG(INFO) << "----------------------- Perf info ------------------------"; + LOG(INFO) << "Total number of predicted data: " << img_num + << " and total time spent(s): " + << std::accumulate(det_time.begin(), det_time.end(), 0.) / 1000; + int num = std::max(1, img_num); + LOG(INFO) << "preproce_time(ms): " << det_time[0] / num + << ", inference_time(ms): " << det_time[1] / num + << ", postprocess_time(ms): " << det_time[2] / num; +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/pipeline.h b/fastdeploy/vision/tracking/pptracking/pipeline.h new file mode 100644 index 00000000000..d17b4d35aef --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/pipeline.h @@ -0,0 +1,142 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_ +#define DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#elif LINUX +#include +#include +#endif + +#include "include/jde_predictor.h" +#include "include/sde_predictor.h" + +namespace PaddleDetection { + +class Pipeline { + public: + explicit Pipeline(const std::string& device, + const double threshold, + const std::string& output_dir, + const std::string& run_mode = "paddle", + const int gpu_id = 0, + const bool use_mkldnn = false, + const int cpu_threads = 1, + const bool trt_calib_mode = false, + const bool do_entrance_counting = false, + const bool save_result = false, + const std::string& scene = "pedestrian", + const bool tiny_obj = false, + const bool is_mtmct = false, + const int secs_interval = 10, + const std::string track_model_dir = "", + const std::string det_model_dir = "", + const std::string reid_model_dir = "") { + std::vector input; + this->input_ = input; + this->device_ = device; + this->threshold_ = threshold; + this->output_dir_ = output_dir; + this->run_mode_ = run_mode; + this->gpu_id_ = gpu_id; + this->use_mkldnn_ = use_mkldnn; + this->cpu_threads_ = cpu_threads; + this->trt_calib_mode_ = trt_calib_mode; + this->do_entrance_counting_ = do_entrance_counting; + this->secs_interval_ = secs_interval_; + this->save_result_ = save_result; + SelectModel(scene, + tiny_obj, + is_mtmct, + track_model_dir, + det_model_dir, + reid_model_dir); + InitPredictor(); + } + + // Set input, it must execute before Run() + void SetInput(const std::string& input_video); + void ClearInput(); + + // Run pipeline in video + void Run(); + void PredictMOT(const std::string& video_path); + void PredictMTMCT(const std::vector video_inputs); + + // Run pipeline in stream + void RunMOTStream(const cv::Mat img, + const int frame_id, + const int video_fps, + const Rect entrance, + cv::Mat out_img, + std::vector* records, + std::set* count_set, + std::set* interval_count_set, + std::vector* in_count_list, + std::vector* out_count_list, + std::map>* prev_center, + std::vector* flow_records); + void RunMTMCTStream(const std::vector imgs, + std::vector* records); + + void PrintBenchmarkLog(const std::vector det_time, const int img_num); + + private: + // Select model according to scenes, it must execute before Run() + void SelectModel(const std::string& scene = "pedestrian", + const bool tiny_obj = false, + const bool is_mtmct = false, + const std::string track_model_dir = "", + const std::string det_model_dir = "", + const std::string reid_model_dir = ""); + void InitPredictor(); + + std::shared_ptr jde_sct_; + std::shared_ptr sde_sct_; + + std::vector input_; + std::vector stream_; + std::string device_; + double threshold_; + std::string output_dir_; + std::string track_model_dir_; + std::string det_model_dir_; + std::string reid_model_dir_; + std::string run_mode_ = "paddle"; + int gpu_id_ = 0; + bool use_mkldnn_ = false; + int cpu_threads_ = 1; + bool trt_calib_mode_ = false; + bool do_entrance_counting_ = false; + bool save_result_ = false; + int secs_interval_ = 10; +}; + +} // namespace PaddleDetection + +#endif // DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_ diff --git a/fastdeploy/vision/tracking/pptracking/predictor.cc b/fastdeploy/vision/tracking/pptracking/predictor.cc new file mode 100644 index 00000000000..ea479f3ab04 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/predictor.cc @@ -0,0 +1,35 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +// for setprecision +#include +#include +#include "include/predictor.h" + +using namespace paddle_infer; // NOLINT + +namespace PaddleDetection { + +void Predictor::Predict(const std::vector imgs, + const double threshold, + MOTResult* result, + std::vector* times) { + if (use_jde_) { + jde_sct_->Predict(imgs, threshold, result, times); + } else { + sde_sct_->Predict(imgs, threshold, result, times); + } +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/predictor.h b/fastdeploy/vision/tracking/pptracking/predictor.h new file mode 100644 index 00000000000..cfb6306513a --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/predictor.h @@ -0,0 +1,99 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "paddle_inference_api.h" // NOLINT + +#include "include/config_parser.h" +#include "include/jde_predictor.h" +#include "include/preprocess_op.h" +#include "include/sde_predictor.h" + +using namespace paddle_infer; // NOLINT + +namespace PaddleDetection { + +class Predictor { + public: + explicit Predictor(const std::string& device = "CPU", + const std::string& track_model_dir = "", + const std::string& det_model_dir = "", + const std::string& reid_model_dir = "", + const double threshold = -1., + const std::string& run_mode = "paddle", + const int gpu_id = 0, + const bool use_mkldnn = false, + const int cpu_threads = 1, + bool trt_calib_mode = false, + const int min_box_area = 200) { + if (track_model_dir.empty() && det_model_dir.empty()) { + throw "Predictor must receive track_model or det_model!"; + } + + if (!track_model_dir.empty() && !det_model_dir.empty()) { + throw "Predictor only receive one of track_model or det_model!"; + } + + if (!track_model_dir.empty()) { + jde_sct_ = + std::make_shared(device, + track_model_dir, + threshold, + run_mode, + gpu_id, + use_mkldnn, + cpu_threads, + trt_calib_mode, + min_box_area); + use_jde_ = true; + } + if (!det_model_dir.empty()) { + sde_sct_ = std::make_shared(device, + det_model_dir, + reid_model_dir, + threshold, + run_mode, + gpu_id, + use_mkldnn, + cpu_threads, + trt_calib_mode, + min_box_area); + use_jde_ = false; + } + } + + // Run predictor + void Predict(const std::vector imgs, + const double threshold = 0.5, + MOTResult* result = nullptr, + std::vector* times = nullptr); + + private: + std::shared_ptr jde_sct_; + std::shared_ptr sde_sct_; + bool use_jde_ = true; +}; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/sde_predictor.cc b/fastdeploy/vision/tracking/pptracking/sde_predictor.cc new file mode 100644 index 00000000000..e469e8ddc5a --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/sde_predictor.cc @@ -0,0 +1,46 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +// for setprecision +#include +#include +#include "include/sde_predictor.h" + +using namespace paddle_infer; // NOLINT + +namespace PaddleDetection { + +// Load Model and create model predictor +void SDEPredictor::LoadModel(const std::string& det_model_dir, + const std::string& reid_model_dir, + const std::string& run_mode) { + throw "Not Implement"; +} + +void SDEPredictor::Preprocess(const cv::Mat& ori_im) { throw "Not Implement"; } + +void SDEPredictor::Postprocess(const cv::Mat dets, + const cv::Mat emb, + MOTResult* result) { + throw "Not Implement"; +} + +void SDEPredictor::Predict(const std::vector imgs, + const double threshold, + MOTResult* result, + std::vector* times) { + throw "Not Implement"; +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/sde_predictor.h b/fastdeploy/vision/tracking/pptracking/sde_predictor.h new file mode 100644 index 00000000000..3919eb105d1 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/sde_predictor.h @@ -0,0 +1,106 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "paddle_inference_api.h" // NOLINT + +#include "include/config_parser.h" +#include "include/preprocess_op.h" +#include "include/utils.h" + +using namespace paddle_infer; // NOLINT + +namespace PaddleDetection { + +class SDEPredictor { + public: + explicit SDEPredictor(const std::string& device, + const std::string& det_model_dir = "", + const std::string& reid_model_dir = "", + const double threshold = -1., + const std::string& run_mode = "paddle", + const int gpu_id = 0, + const bool use_mkldnn = false, + const int cpu_threads = 1, + bool trt_calib_mode = false, + const int min_box_area = 200) { + this->device_ = device; + this->gpu_id_ = gpu_id; + this->use_mkldnn_ = use_mkldnn; + this->cpu_math_library_num_threads_ = cpu_threads; + this->trt_calib_mode_ = trt_calib_mode; + this->min_box_area_ = min_box_area; + + det_config_.load_config(det_model_dir); + this->min_subgraph_size_ = det_config_.min_subgraph_size_; + det_preprocessor_.Init(det_config_.preprocess_info_); + + reid_config_.load_config(reid_model_dir); + reid_preprocessor_.Init(reid_config_.preprocess_info_); + + LoadModel(det_model_dir, reid_model_dir, run_mode); + this->conf_thresh_ = det_config_.conf_thresh_; + } + + // Load Paddle inference model + void LoadModel(const std::string& det_model_dir, + const std::string& reid_model_dir, + const std::string& run_mode = "paddle"); + + // Run predictor + void Predict(const std::vector imgs, + const double threshold = 0.5, + MOTResult* result = nullptr, + std::vector* times = nullptr); + + private: + std::string device_ = "CPU"; + float threhold = 0.5; + int gpu_id_ = 0; + bool use_mkldnn_ = false; + int cpu_math_library_num_threads_ = 1; + int min_subgraph_size_ = 3; + bool trt_calib_mode_ = false; + + // Preprocess image and copy data to input buffer + void Preprocess(const cv::Mat& image_mat); + // Postprocess result + void Postprocess(const cv::Mat dets, const cv::Mat emb, MOTResult* result); + + std::shared_ptr det_predictor_; + std::shared_ptr reid_predictor_; + Preprocessor det_preprocessor_; + Preprocessor reid_preprocessor_; + ImageBlob inputs_; + std::vector bbox_data_; + std::vector emb_data_; + double threshold_; + ConfigPaser det_config_; + ConfigPaser reid_config_; + float min_box_area_ = 200; + float conf_thresh_; +}; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc new file mode 100644 index 00000000000..9540e39f670 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/tracker.cc @@ -0,0 +1,304 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.cpp +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#include +#include +#include +#include + +#include "include/lapjv.h" +#include "include/tracker.h" + +#define mat2vec4f(m) \ + cv::Vec4f(*m.ptr(0, 0), \ + *m.ptr(0, 1), \ + *m.ptr(0, 2), \ + *m.ptr(0, 3)) + +namespace PaddleDetection { + +static std::map chi2inv95 = {{1, 3.841459f}, + {2, 5.991465f}, + {3, 7.814728f}, + {4, 9.487729f}, + {5, 11.070498f}, + {6, 12.591587f}, + {7, 14.067140f}, + {8, 15.507313f}, + {9, 16.918978f}}; + +JDETracker *JDETracker::me = new JDETracker; + +JDETracker *JDETracker::instance(void) { return me; } + +JDETracker::JDETracker(void) + : timestamp(0), max_lost_time(30), lambda(0.98f), det_thresh(0.3f) {} + +bool JDETracker::update(const cv::Mat &dets, + const cv::Mat &emb, + std::vector *tracks) { + ++timestamp; + TrajectoryPool candidates(dets.rows); + for (int i = 0; i < dets.rows; ++i) { + float score = *dets.ptr(i, 1); + const cv::Mat <rb_ = dets(cv::Rect(2, i, 4, 1)); + cv::Vec4f ltrb = mat2vec4f(ltrb_); + const cv::Mat &embedding = emb(cv::Rect(0, i, emb.cols, 1)); + candidates[i] = Trajectory(ltrb, score, embedding); + } + + TrajectoryPtrPool tracked_trajectories; + TrajectoryPtrPool unconfirmed_trajectories; + for (size_t i = 0; i < this->tracked_trajectories.size(); ++i) { + if (this->tracked_trajectories[i].is_activated) + tracked_trajectories.push_back(&this->tracked_trajectories[i]); + else + unconfirmed_trajectories.push_back(&this->tracked_trajectories[i]); + } + + TrajectoryPtrPool trajectory_pool = + tracked_trajectories + &(this->lost_trajectories); + + for (size_t i = 0; i < trajectory_pool.size(); ++i) + trajectory_pool[i]->predict(); + + Match matches; + std::vector mismatch_row; + std::vector mismatch_col; + + cv::Mat cost = motion_distance(trajectory_pool, candidates); + linear_assignment(cost, 0.7f, &matches, &mismatch_row, &mismatch_col); + + MatchIterator miter; + TrajectoryPtrPool activated_trajectories; + TrajectoryPtrPool retrieved_trajectories; + + for (miter = matches.begin(); miter != matches.end(); miter++) { + Trajectory *pt = trajectory_pool[miter->first]; + Trajectory &ct = candidates[miter->second]; + if (pt->state == Tracked) { + pt->update(&ct, timestamp); + activated_trajectories.push_back(pt); + } else { + pt->reactivate(&ct, timestamp); + retrieved_trajectories.push_back(pt); + } + } + + TrajectoryPtrPool next_candidates(mismatch_col.size()); + for (size_t i = 0; i < mismatch_col.size(); ++i) + next_candidates[i] = &candidates[mismatch_col[i]]; + + TrajectoryPtrPool next_trajectory_pool; + for (size_t i = 0; i < mismatch_row.size(); ++i) { + int j = mismatch_row[i]; + if (trajectory_pool[j]->state == Tracked) + next_trajectory_pool.push_back(trajectory_pool[j]); + } + + cost = iou_distance(next_trajectory_pool, next_candidates); + linear_assignment(cost, 0.5f, &matches, &mismatch_row, &mismatch_col); + + for (miter = matches.begin(); miter != matches.end(); miter++) { + Trajectory *pt = next_trajectory_pool[miter->first]; + Trajectory *ct = next_candidates[miter->second]; + if (pt->state == Tracked) { + pt->update(ct, timestamp); + activated_trajectories.push_back(pt); + } else { + pt->reactivate(ct, timestamp); + retrieved_trajectories.push_back(pt); + } + } + + TrajectoryPtrPool lost_trajectories; + for (size_t i = 0; i < mismatch_row.size(); ++i) { + Trajectory *pt = next_trajectory_pool[mismatch_row[i]]; + if (pt->state != Lost) { + pt->mark_lost(); + lost_trajectories.push_back(pt); + } + } + + TrajectoryPtrPool nnext_candidates(mismatch_col.size()); + for (size_t i = 0; i < mismatch_col.size(); ++i) + nnext_candidates[i] = next_candidates[mismatch_col[i]]; + cost = iou_distance(unconfirmed_trajectories, nnext_candidates); + linear_assignment(cost, 0.7f, &matches, &mismatch_row, &mismatch_col); + + for (miter = matches.begin(); miter != matches.end(); miter++) { + unconfirmed_trajectories[miter->first]->update( + nnext_candidates[miter->second], timestamp); + activated_trajectories.push_back(unconfirmed_trajectories[miter->first]); + } + + TrajectoryPtrPool removed_trajectories; + + for (size_t i = 0; i < mismatch_row.size(); ++i) { + unconfirmed_trajectories[mismatch_row[i]]->mark_removed(); + removed_trajectories.push_back(unconfirmed_trajectories[mismatch_row[i]]); + } + + for (size_t i = 0; i < mismatch_col.size(); ++i) { + if (nnext_candidates[mismatch_col[i]]->score < det_thresh) continue; + nnext_candidates[mismatch_col[i]]->activate(timestamp); + activated_trajectories.push_back(nnext_candidates[mismatch_col[i]]); + } + + for (size_t i = 0; i < this->lost_trajectories.size(); ++i) { + Trajectory < = this->lost_trajectories[i]; + if (timestamp - lt.timestamp > max_lost_time) { + lt.mark_removed(); + removed_trajectories.push_back(<); + } + } + + TrajectoryPoolIterator piter; + for (piter = this->tracked_trajectories.begin(); + piter != this->tracked_trajectories.end();) { + if (piter->state != Tracked) + piter = this->tracked_trajectories.erase(piter); + else + ++piter; + } + + this->tracked_trajectories += activated_trajectories; + this->tracked_trajectories += retrieved_trajectories; + + this->lost_trajectories -= this->tracked_trajectories; + this->lost_trajectories += lost_trajectories; + this->lost_trajectories -= this->removed_trajectories; + this->removed_trajectories += removed_trajectories; + remove_duplicate_trajectory(&this->tracked_trajectories, + &this->lost_trajectories); + + tracks->clear(); + for (size_t i = 0; i < this->tracked_trajectories.size(); ++i) { + if (this->tracked_trajectories[i].is_activated) { + Track track = {this->tracked_trajectories[i].id, + this->tracked_trajectories[i].score, + this->tracked_trajectories[i].ltrb}; + tracks->push_back(track); + } + } + return 0; +} + +cv::Mat JDETracker::motion_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b) { + if (0 == a.size() || 0 == b.size()) + return cv::Mat(a.size(), b.size(), CV_32F); + + cv::Mat edists = embedding_distance(a, b); + cv::Mat mdists = mahalanobis_distance(a, b); + cv::Mat fdists = lambda * edists + (1 - lambda) * mdists; + + const float gate_thresh = chi2inv95[4]; + for (int i = 0; i < fdists.rows; ++i) { + for (int j = 0; j < fdists.cols; ++j) { + if (*mdists.ptr(i, j) > gate_thresh) + *fdists.ptr(i, j) = FLT_MAX; + } + } + + return fdists; +} + +void JDETracker::linear_assignment(const cv::Mat &cost, + float cost_limit, + Match *matches, + std::vector *mismatch_row, + std::vector *mismatch_col) { + matches->clear(); + mismatch_row->clear(); + mismatch_col->clear(); + if (cost.empty()) { + for (int i = 0; i < cost.rows; ++i) mismatch_row->push_back(i); + for (int i = 0; i < cost.cols; ++i) mismatch_col->push_back(i); + return; + } + + float opt = 0; + cv::Mat x(cost.rows, 1, CV_32S); + cv::Mat y(cost.cols, 1, CV_32S); + + lapjv_internal(cost, + true, + cost_limit, + reinterpret_cast(x.data), + reinterpret_cast(y.data)); + + for (int i = 0; i < x.rows; ++i) { + int j = *x.ptr(i); + if (j >= 0) + matches->insert({i, j}); + else + mismatch_row->push_back(i); + } + + for (int i = 0; i < y.rows; ++i) { + int j = *y.ptr(i); + if (j < 0) mismatch_col->push_back(i); + } + + return; +} + +void JDETracker::remove_duplicate_trajectory(TrajectoryPool *a, + TrajectoryPool *b, + float iou_thresh) { + if (a->size() == 0 || b->size() == 0) return; + + cv::Mat dist = iou_distance(*a, *b); + cv::Mat mask = dist < iou_thresh; + std::vector idx; + cv::findNonZero(mask, idx); + + std::vector da; + std::vector db; + for (size_t i = 0; i < idx.size(); ++i) { + int ta = (*a)[idx[i].y].timestamp - (*a)[idx[i].y].starttime; + int tb = (*b)[idx[i].x].timestamp - (*b)[idx[i].x].starttime; + if (ta > tb) + db.push_back(idx[i].x); + else + da.push_back(idx[i].y); + } + + int id = 0; + TrajectoryPoolIterator piter; + for (piter = a->begin(); piter != a->end();) { + std::vector::iterator iter = find(da.begin(), da.end(), id++); + if (iter != da.end()) + piter = a->erase(piter); + else + ++piter; + } + + id = 0; + for (piter = b->begin(); piter != b->end();) { + std::vector::iterator iter = find(db.begin(), db.end(), id++); + if (iter != db.end()) + piter = b->erase(piter); + else + ++piter; + } +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h new file mode 100644 index 00000000000..244530f140a --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -0,0 +1,72 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.h +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#pragma once + +#include +#include + +#include +#include +#include +#include "include/trajectory.h" + +namespace PaddleDetection { + +typedef std::map Match; +typedef std::map::iterator MatchIterator; + +struct Track { + int id; + float score; + cv::Vec4f ltrb; +}; + +class JDETracker { + public: + static JDETracker *instance(void); + virtual bool update(const cv::Mat &dets, + const cv::Mat &emb, + std::vector *tracks); + + private: + JDETracker(void); + virtual ~JDETracker(void) {} + cv::Mat motion_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b); + void linear_assignment(const cv::Mat &cost, + float cost_limit, + Match *matches, + std::vector *mismatch_row, + std::vector *mismatch_col); + void remove_duplicate_trajectory(TrajectoryPool *a, + TrajectoryPool *b, + float iou_thresh = 0.15f); + + private: + static JDETracker *me; + int timestamp; + TrajectoryPool tracked_trajectories; + TrajectoryPool lost_trajectories; + TrajectoryPool removed_trajectories; + int max_lost_time; + float lambda; + float det_thresh; +}; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.cc b/fastdeploy/vision/tracking/pptracking/trajectory.cc new file mode 100644 index 00000000000..0ff2e1a5fc7 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/trajectory.cc @@ -0,0 +1,517 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.cpp +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#include "include/trajectory.h" +#include + +namespace PaddleDetection { + +void TKalmanFilter::init(const cv::Mat &measurement) { + measurement.copyTo(statePost(cv::Rect(0, 0, 1, 4))); + statePost(cv::Rect(0, 4, 1, 4)).setTo(0); + statePost.copyTo(statePre); + + float varpos = 2 * std_weight_position * (*measurement.ptr(3)); + varpos *= varpos; + float varvel = 10 * std_weight_velocity * (*measurement.ptr(3)); + varvel *= varvel; + + errorCovPost.setTo(0); + *errorCovPost.ptr(0, 0) = varpos; + *errorCovPost.ptr(1, 1) = varpos; + *errorCovPost.ptr(2, 2) = 1e-4f; + *errorCovPost.ptr(3, 3) = varpos; + *errorCovPost.ptr(4, 4) = varvel; + *errorCovPost.ptr(5, 5) = varvel; + *errorCovPost.ptr(6, 6) = 1e-10f; + *errorCovPost.ptr(7, 7) = varvel; + errorCovPost.copyTo(errorCovPre); +} + +const cv::Mat &TKalmanFilter::predict() { + float varpos = std_weight_position * (*statePre.ptr(3)); + varpos *= varpos; + float varvel = std_weight_velocity * (*statePre.ptr(3)); + varvel *= varvel; + + processNoiseCov.setTo(0); + *processNoiseCov.ptr(0, 0) = varpos; + *processNoiseCov.ptr(1, 1) = varpos; + *processNoiseCov.ptr(2, 2) = 1e-4f; + *processNoiseCov.ptr(3, 3) = varpos; + *processNoiseCov.ptr(4, 4) = varvel; + *processNoiseCov.ptr(5, 5) = varvel; + *processNoiseCov.ptr(6, 6) = 1e-10f; + *processNoiseCov.ptr(7, 7) = varvel; + + return cv::KalmanFilter::predict(); +} + +const cv::Mat &TKalmanFilter::correct(const cv::Mat &measurement) { + float varpos = std_weight_position * (*measurement.ptr(3)); + varpos *= varpos; + + measurementNoiseCov.setTo(0); + *measurementNoiseCov.ptr(0, 0) = varpos; + *measurementNoiseCov.ptr(1, 1) = varpos; + *measurementNoiseCov.ptr(2, 2) = 1e-2f; + *measurementNoiseCov.ptr(3, 3) = varpos; + + return cv::KalmanFilter::correct(measurement); +} + +void TKalmanFilter::project(cv::Mat *mean, cv::Mat *covariance) const { + float varpos = std_weight_position * (*statePost.ptr(3)); + varpos *= varpos; + + cv::Mat measurementNoiseCov_ = cv::Mat::eye(4, 4, CV_32F); + *measurementNoiseCov_.ptr(0, 0) = varpos; + *measurementNoiseCov_.ptr(1, 1) = varpos; + *measurementNoiseCov_.ptr(2, 2) = 1e-2f; + *measurementNoiseCov_.ptr(3, 3) = varpos; + + *mean = measurementMatrix * statePost; + cv::Mat temp = measurementMatrix * errorCovPost; + gemm(temp, + measurementMatrix, + 1, + measurementNoiseCov_, + 1, + *covariance, + cv::GEMM_2_T); +} + +int Trajectory::count = 0; + +const cv::Mat &Trajectory::predict(void) { + if (state != Tracked) *cv::KalmanFilter::statePost.ptr(7) = 0; + return TKalmanFilter::predict(); +} + +void Trajectory::update(Trajectory *traj, + int timestamp_, + bool update_embedding_) { + timestamp = timestamp_; + ++length; + ltrb = traj->ltrb; + xyah = traj->xyah; + TKalmanFilter::correct(cv::Mat(traj->xyah)); + state = Tracked; + is_activated = true; + score = traj->score; + if (update_embedding_) update_embedding(traj->current_embedding); +} + +void Trajectory::activate(int timestamp_) { + id = next_id(); + TKalmanFilter::init(cv::Mat(xyah)); + length = 0; + state = Tracked; + if (timestamp_ == 1) { + is_activated = true; + } + timestamp = timestamp_; + starttime = timestamp_; +} + +void Trajectory::reactivate(Trajectory *traj, int timestamp_, bool newid) { + TKalmanFilter::correct(cv::Mat(traj->xyah)); + update_embedding(traj->current_embedding); + length = 0; + state = Tracked; + is_activated = true; + timestamp = timestamp_; + if (newid) id = next_id(); +} + +void Trajectory::update_embedding(const cv::Mat &embedding) { + current_embedding = embedding / cv::norm(embedding); + if (smooth_embedding.empty()) { + smooth_embedding = current_embedding; + } else { + smooth_embedding = eta * smooth_embedding + (1 - eta) * current_embedding; + } + smooth_embedding = smooth_embedding / cv::norm(smooth_embedding); +} + +TrajectoryPool operator+(const TrajectoryPool &a, const TrajectoryPool &b) { + TrajectoryPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; + + for (size_t i = 0; i < b.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i].id); + if (iter == ids.end()) { + sum.push_back(b[i]); + ids.push_back(b[i].id); + } + } + + return sum; +} + +TrajectoryPool operator+(const TrajectoryPool &a, const TrajectoryPtrPool &b) { + TrajectoryPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; + + for (size_t i = 0; i < b.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); + if (iter == ids.end()) { + sum.push_back(*b[i]); + ids.push_back(b[i]->id); + } + } + + return sum; +} + +TrajectoryPool &operator+=(TrajectoryPool &a, // NOLINT + const TrajectoryPtrPool &b) { + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; + + for (size_t i = 0; i < b.size(); ++i) { + if (b[i]->smooth_embedding.empty()) continue; + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); + if (iter == ids.end()) { + a.push_back(*b[i]); + ids.push_back(b[i]->id); + } + } + + return a; +} + +TrajectoryPool operator-(const TrajectoryPool &a, const TrajectoryPool &b) { + TrajectoryPool dif; + std::vector ids(b.size()); + for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i].id; + + for (size_t i = 0; i < a.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), a[i].id); + if (iter == ids.end()) dif.push_back(a[i]); + } + + return dif; +} + +TrajectoryPool &operator-=(TrajectoryPool &a, // NOLINT + const TrajectoryPool &b) { + std::vector ids(b.size()); + for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i].id; + + TrajectoryPoolIterator piter; + for (piter = a.begin(); piter != a.end();) { + std::vector::iterator iter = find(ids.begin(), ids.end(), piter->id); + if (iter == ids.end()) + ++piter; + else + piter = a.erase(piter); + } + + return a; +} + +TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + TrajectoryPtrPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i]->id; + + for (size_t i = 0; i < b.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); + if (iter == ids.end()) { + sum.push_back(b[i]); + ids.push_back(b[i]->id); + } + } + + return sum; +} + +TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, TrajectoryPool *b) { + TrajectoryPtrPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i]->id; + + for (size_t i = 0; i < b->size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), (*b)[i].id); + if (iter == ids.end()) { + sum.push_back(&(*b)[i]); + ids.push_back((*b)[i].id); + } + } + + return sum; +} + +TrajectoryPtrPool operator-(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + TrajectoryPtrPool dif; + std::vector ids(b.size()); + for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i]->id; + + for (size_t i = 0; i < a.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), a[i]->id); + if (iter == ids.end()) dif.push_back(a[i]); + } + + return dif; +} + +cv::Mat embedding_distance(const TrajectoryPool &a, const TrajectoryPool &b) { + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + cv::Mat u = a[i].smooth_embedding; + cv::Mat v = b[j].smooth_embedding; + double uv = u.dot(v); + double uu = u.dot(u); + double vv = v.dot(v); + double dist = std::abs(1. - uv / std::sqrt(uu * vv)); + // double dist = cv::norm(a[i].smooth_embedding, b[j].smooth_embedding, + // cv::NORM_L2); + distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); + } + } + return dists; +} + +cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + // double dist = cv::norm(a[i]->smooth_embedding, b[j]->smooth_embedding, + // cv::NORM_L2); + // distsi[j] = static_cast(dist); + cv::Mat u = a[i]->smooth_embedding; + cv::Mat v = b[j]->smooth_embedding; + double uv = u.dot(v); + double uu = u.dot(u); + double vv = v.dot(v); + double dist = std::abs(1. - uv / std::sqrt(uu * vv)); + distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); + } + } + + return dists; +} + +cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b) { + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + // double dist = cv::norm(a[i]->smooth_embedding, b[j].smooth_embedding, + // cv::NORM_L2); + // distsi[j] = static_cast(dist); + cv::Mat u = a[i]->smooth_embedding; + cv::Mat v = b[j].smooth_embedding; + double uv = u.dot(v); + double uu = u.dot(u); + double vv = v.dot(v); + double dist = std::abs(1. - uv / std::sqrt(uu * vv)); + distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); + } + } + + return dists; +} + +cv::Mat mahalanobis_distance(const TrajectoryPool &a, const TrajectoryPool &b) { + std::vector means(a.size()); + std::vector icovariances(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + cv::Mat covariance; + a[i].project(&means[i], &covariance); + cv::invert(covariance, icovariances[i]); + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Mat x(b[j].xyah); + float dist = + static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); + distsi[j] = dist * dist; + } + } + + return dists; +} + +cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + std::vector means(a.size()); + std::vector icovariances(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + cv::Mat covariance; + a[i]->project(&means[i], &covariance); + cv::invert(covariance, icovariances[i]); + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Mat x(b[j]->xyah); + float dist = + static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); + distsi[j] = dist * dist; + } + } + + return dists; +} + +cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b) { + std::vector means(a.size()); + std::vector icovariances(a.size()); + + for (size_t i = 0; i < a.size(); ++i) { + cv::Mat covariance; + a[i]->project(&means[i], &covariance); + cv::invert(covariance, icovariances[i]); + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Mat x(b[j].xyah); + float dist = + static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); + distsi[j] = dist * dist; + } + } + + return dists; +} + +static inline float calc_inter_area(const cv::Vec4f &a, const cv::Vec4f &b) { + if (a[2] < b[0] || a[0] > b[2] || a[3] < b[1] || a[1] > b[3]) return 0.f; + + float w = std::min(a[2], b[2]) - std::max(a[0], b[0]); + float h = std::min(a[3], b[3]) - std::max(a[1], b[1]); + return w * h; +} + +cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b) { + std::vector areaa(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + float w = a[i].ltrb[2] - a[i].ltrb[0]; + float h = a[i].ltrb[3] - a[i].ltrb[1]; + areaa[i] = w * h; + } + + std::vector areab(b.size()); + for (size_t j = 0; j < b.size(); ++j) { + float w = b[j].ltrb[2] - b[j].ltrb[0]; + float h = b[j].ltrb[3] - b[j].ltrb[1]; + areab[j] = w * h; + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + const cv::Vec4f &boxa = a[i].ltrb; + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Vec4f &boxb = b[j].ltrb; + float inters = calc_inter_area(boxa, boxb); + distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); + } + } + + return dists; +} + +cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPtrPool &b) { + std::vector areaa(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + float w = a[i]->ltrb[2] - a[i]->ltrb[0]; + float h = a[i]->ltrb[3] - a[i]->ltrb[1]; + areaa[i] = w * h; + } + + std::vector areab(b.size()); + for (size_t j = 0; j < b.size(); ++j) { + float w = b[j]->ltrb[2] - b[j]->ltrb[0]; + float h = b[j]->ltrb[3] - b[j]->ltrb[1]; + areab[j] = w * h; + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + const cv::Vec4f &boxa = a[i]->ltrb; + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Vec4f &boxb = b[j]->ltrb; + float inters = calc_inter_area(boxa, boxb); + distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); + } + } + + return dists; +} + +cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b) { + std::vector areaa(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + float w = a[i]->ltrb[2] - a[i]->ltrb[0]; + float h = a[i]->ltrb[3] - a[i]->ltrb[1]; + areaa[i] = w * h; + } + + std::vector areab(b.size()); + for (size_t j = 0; j < b.size(); ++j) { + float w = b[j].ltrb[2] - b[j].ltrb[0]; + float h = b[j].ltrb[3] - b[j].ltrb[1]; + areab[j] = w * h; + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + const cv::Vec4f &boxa = a[i]->ltrb; + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Vec4f &boxb = b[j].ltrb; + float inters = calc_inter_area(boxa, boxb); + distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); + } + } + + return dists; +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h new file mode 100644 index 00000000000..c21e8cac368 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -0,0 +1,230 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.h +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#pragma once + +#include + +#include +#include +#include +#include "opencv2/video/tracking.hpp" + +namespace PaddleDetection { + +typedef enum { New = 0, Tracked = 1, Lost = 2, Removed = 3 } TrajectoryState; + +class Trajectory; +typedef std::vector TrajectoryPool; +typedef std::vector::iterator TrajectoryPoolIterator; +typedef std::vector TrajectoryPtrPool; +typedef std::vector::iterator TrajectoryPtrPoolIterator; + +class TKalmanFilter : public cv::KalmanFilter { + public: + TKalmanFilter(void); + virtual ~TKalmanFilter(void) {} + virtual void init(const cv::Mat &measurement); + virtual const cv::Mat &predict(); + virtual const cv::Mat &correct(const cv::Mat &measurement); + virtual void project(cv::Mat *mean, cv::Mat *covariance) const; + + private: + float std_weight_position; + float std_weight_velocity; +}; + +inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) { + cv::KalmanFilter::transitionMatrix = cv::Mat::eye(8, 8, CV_32F); + for (int i = 0; i < 4; ++i) + cv::KalmanFilter::transitionMatrix.at(i, i + 4) = 1; + cv::KalmanFilter::measurementMatrix = cv::Mat::eye(4, 8, CV_32F); + std_weight_position = 1 / 20.f; + std_weight_velocity = 1 / 160.f; +} + +class Trajectory : public TKalmanFilter { + public: + Trajectory(); + Trajectory(const cv::Vec4f <rb, float score, const cv::Mat &embedding); + Trajectory(const Trajectory &other); + Trajectory &operator=(const Trajectory &rhs); + virtual ~Trajectory(void) {} + + static int next_id(); + virtual const cv::Mat &predict(void); + virtual void update(Trajectory *traj, + int timestamp, + bool update_embedding = true); + virtual void activate(int timestamp); + virtual void reactivate(Trajectory *traj, int timestamp, bool newid = false); + virtual void mark_lost(void); + virtual void mark_removed(void); + + friend TrajectoryPool operator+(const TrajectoryPool &a, + const TrajectoryPool &b); + friend TrajectoryPool operator+(const TrajectoryPool &a, + const TrajectoryPtrPool &b); + friend TrajectoryPool &operator+=(TrajectoryPool &a, // NOLINT + const TrajectoryPtrPool &b); + friend TrajectoryPool operator-(const TrajectoryPool &a, + const TrajectoryPool &b); + friend TrajectoryPool &operator-=(TrajectoryPool &a, // NOLINT + const TrajectoryPool &b); + friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, + TrajectoryPool *b); + friend TrajectoryPtrPool operator-(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + + friend cv::Mat embedding_distance(const TrajectoryPool &a, + const TrajectoryPool &b); + friend cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b); + + friend cv::Mat mahalanobis_distance(const TrajectoryPool &a, + const TrajectoryPool &b); + friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b); + + friend cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b); + friend cv::Mat iou_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend cv::Mat iou_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b); + + private: + void update_embedding(const cv::Mat &embedding); + + public: + TrajectoryState state; + cv::Vec4f ltrb; + cv::Mat smooth_embedding; + int id; + bool is_activated; + int timestamp; + int starttime; + float score; + + private: + static int count; + cv::Vec4f xyah; + cv::Mat current_embedding; + float eta; + int length; +}; + +inline cv::Vec4f ltrb2xyah(const cv::Vec4f <rb) { + cv::Vec4f xyah; + xyah[0] = (ltrb[0] + ltrb[2]) * 0.5f; + xyah[1] = (ltrb[1] + ltrb[3]) * 0.5f; + xyah[3] = ltrb[3] - ltrb[1]; + xyah[2] = (ltrb[2] - ltrb[0]) / xyah[3]; + return xyah; +} + +inline Trajectory::Trajectory() + : state(New), + ltrb(cv::Vec4f()), + smooth_embedding(cv::Mat()), + id(0), + is_activated(false), + timestamp(0), + starttime(0), + score(0), + eta(0.9), + length(0) {} + +inline Trajectory::Trajectory(const cv::Vec4f <rb_, + float score_, + const cv::Mat &embedding) + : state(New), + ltrb(ltrb_), + smooth_embedding(cv::Mat()), + id(0), + is_activated(false), + timestamp(0), + starttime(0), + score(score_), + eta(0.9), + length(0) { + xyah = ltrb2xyah(ltrb); + update_embedding(embedding); +} + +inline Trajectory::Trajectory(const Trajectory &other) + : state(other.state), + ltrb(other.ltrb), + id(other.id), + is_activated(other.is_activated), + timestamp(other.timestamp), + starttime(other.starttime), + xyah(other.xyah), + score(other.score), + eta(other.eta), + length(other.length) { + other.smooth_embedding.copyTo(smooth_embedding); + other.current_embedding.copyTo(current_embedding); + // copy state in KalmanFilter + + other.statePre.copyTo(cv::KalmanFilter::statePre); + other.statePost.copyTo(cv::KalmanFilter::statePost); + other.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre); + other.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost); +} + +inline Trajectory &Trajectory::operator=(const Trajectory &rhs) { + this->state = rhs.state; + this->ltrb = rhs.ltrb; + rhs.smooth_embedding.copyTo(this->smooth_embedding); + this->id = rhs.id; + this->is_activated = rhs.is_activated; + this->timestamp = rhs.timestamp; + this->starttime = rhs.starttime; + this->xyah = rhs.xyah; + this->score = rhs.score; + rhs.current_embedding.copyTo(this->current_embedding); + this->eta = rhs.eta; + this->length = rhs.length; + + // copy state in KalmanFilter + + rhs.statePre.copyTo(cv::KalmanFilter::statePre); + rhs.statePost.copyTo(cv::KalmanFilter::statePost); + rhs.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre); + rhs.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost); + + return *this; +} + +inline int Trajectory::next_id() { + ++count; + return count; +} + +inline void Trajectory::mark_lost(void) { state = Lost; } + +inline void Trajectory::mark_removed(void) { state = Removed; } + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/postprocess.cc b/fastdeploy/vision/tracking/pptracking/utils/postprocess.cc new file mode 100644 index 00000000000..47ea5a7cc48 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/utils/postprocess.cc @@ -0,0 +1,207 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +// for setprecision +#include +#include +#include +#include "include/postprocess.h" + +namespace PaddleDetection { + +cv::Scalar GetColor(int idx) { + idx = idx * 3; + cv::Scalar color = + cv::Scalar((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255); + return color; +} + +cv::Mat VisualizeTrackResult(const cv::Mat& img, + const MOTResult& results, + const float fps, + const int frame_id) { + cv::Mat vis_img = img.clone(); + int im_h = img.rows; + int im_w = img.cols; + float text_scale = std::max(1, static_cast(im_w / 1600.)); + float text_thickness = 2.; + float line_thickness = std::max(1, static_cast(im_w / 500.)); + + std::ostringstream oss; + oss << std::setiosflags(std::ios::fixed) << std::setprecision(4); + oss << "frame: " << frame_id << " "; + oss << "fps: " << fps << " "; + oss << "num: " << results.size(); + std::string text = oss.str(); + + cv::Point origin; + origin.x = 0; + origin.y = static_cast(15 * text_scale); + cv::putText(vis_img, + text, + origin, + cv::FONT_HERSHEY_PLAIN, + text_scale, + (0, 0, 255), + 2); + + for (int i = 0; i < results.size(); ++i) { + const int obj_id = results[i].ids; + const float score = results[i].score; + + cv::Scalar color = GetColor(obj_id); + + cv::Point pt1 = cv::Point(results[i].rects.left, results[i].rects.top); + cv::Point pt2 = cv::Point(results[i].rects.right, results[i].rects.bottom); + cv::Point id_pt = + cv::Point(results[i].rects.left, results[i].rects.top + 10); + cv::Point score_pt = + cv::Point(results[i].rects.left, results[i].rects.top - 10); + cv::rectangle(vis_img, pt1, pt2, color, line_thickness); + + std::ostringstream idoss; + idoss << std::setiosflags(std::ios::fixed) << std::setprecision(4); + idoss << obj_id; + std::string id_text = idoss.str(); + + cv::putText(vis_img, + id_text, + id_pt, + cv::FONT_HERSHEY_PLAIN, + text_scale, + cv::Scalar(0, 255, 255), + text_thickness); + + std::ostringstream soss; + soss << std::setiosflags(std::ios::fixed) << std::setprecision(2); + soss << score; + std::string score_text = soss.str(); + + cv::putText(vis_img, + score_text, + score_pt, + cv::FONT_HERSHEY_PLAIN, + text_scale, + cv::Scalar(0, 255, 255), + text_thickness); + } + return vis_img; +} + +void FlowStatistic(const MOTResult& results, + const int frame_id, + const int secs_interval, + const bool do_entrance_counting, + const int video_fps, + const Rect entrance, + std::set* id_set, + std::set* interval_id_set, + std::vector* in_id_list, + std::vector* out_id_list, + std::map>* prev_center, + std::vector* records) { + if (frame_id == 0) interval_id_set->clear(); + + if (do_entrance_counting) { + // Count in and out number: + // Use horizontal center line as the entrance just for simplification. + // If a person located in the above the horizontal center line + // at the previous frame and is in the below the line at the current frame, + // the in number is increased by one. + // If a person was in the below the horizontal center line + // at the previous frame and locates in the below the line at the current + // frame, + // the out number is increased by one. + // TODO(qianhui): if the entrance is not the horizontal center line, + // the counting method should be optimized. + + float entrance_y = entrance.top; + for (const auto& result : results) { + float center_x = (result.rects.left + result.rects.right) / 2; + float center_y = (result.rects.top + result.rects.bottom) / 2; + int ids = result.ids; + std::map>::iterator iter; + iter = prev_center->find(ids); + if (iter != prev_center->end()) { + if (iter->second[1] <= entrance_y && center_y > entrance_y) { + in_id_list->push_back(ids); + } + if (iter->second[1] >= entrance_y && center_y < entrance_y) { + out_id_list->push_back(ids); + } + (*prev_center)[ids][0] = center_x; + (*prev_center)[ids][1] = center_y; + } else { + prev_center->insert( + std::pair>(ids, {center_x, center_y})); + } + } + } + + // Count totol number, number at a manual-setting interval + for (const auto& result : results) { + id_set->insert(result.ids); + interval_id_set->insert(result.ids); + } + + std::ostringstream os; + os << "Frame id: " << frame_id << ", Total count: " << id_set->size(); + if (do_entrance_counting) { + os << ", In count: " << in_id_list->size() + << ", Out count: " << out_id_list->size(); + } + + // Reset counting at the interval beginning + int curr_interval_count = -1; + if (frame_id % video_fps == 0 && frame_id / video_fps % secs_interval == 0) { + curr_interval_count = interval_id_set->size(); + os << ", Count during " << secs_interval + << " secs: " << curr_interval_count; + interval_id_set->clear(); + } + os << "\n"; + std::string record = os.str(); + records->push_back(record); + LOG(INFO) << record; +} + +void SaveMOTResult(const MOTResult& results, + const int frame_id, + std::vector* records) { + // result format: frame_id, track_id, x1, y1, w, h + std::string record; + for (int i = 0; i < results.size(); ++i) { + MOTTrack mot_track = results[i]; + int ids = mot_track.ids; + float score = mot_track.score; + Rect rects = mot_track.rects; + float x1 = rects.left; + float y1 = rects.top; + float x2 = rects.right; + float y2 = rects.bottom; + float w = x2 - x1; + float h = y2 - y1; + if (w == 0 || h == 0) { + continue; + } + std::ostringstream os; + os << frame_id << " " << ids << " " << x1 << " " << y1 << " " << w << " " + << h << "\n"; + record = os.str(); + records->push_back(record); + } +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/postprocess.h b/fastdeploy/vision/tracking/pptracking/utils/postprocess.h new file mode 100644 index 00000000000..41b10960351 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/utils/postprocess.h @@ -0,0 +1,62 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "include/utils.h" + +namespace PaddleDetection { + +// Generate visualization color +cv::Scalar GetColor(int idx); + +// Visualize Tracking Results +cv::Mat VisualizeTrackResult(const cv::Mat& img, + const MOTResult& results, + const float fps, + const int frame_id); + +// Pedestrian/Vehicle Counting +void FlowStatistic(const MOTResult& results, + const int frame_id, + const int secs_interval, + const bool do_entrance_counting, + const int video_fps, + const Rect entrance, + std::set* id_set, + std::set* interval_id_set, + std::vector* in_id_list, + std::vector* out_id_list, + std::map>* prev_center, + std::vector* records); + +// Save Tracking Results +void SaveMOTResult(const MOTResult& results, + const int frame_id, + std::vector* records); + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc b/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc new file mode 100644 index 00000000000..3158ad67b60 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc @@ -0,0 +1,187 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "include/preprocess_op.h" + +namespace PaddleDetection { + +void InitInfo::Run(cv::Mat* im, ImageBlob* data) { + data->im_shape_ = {static_cast(im->rows), + static_cast(im->cols)}; + data->scale_factor_ = {1., 1.}; + data->in_net_shape_ = {static_cast(im->rows), + static_cast(im->cols)}; +} + +void NormalizeImage::Run(cv::Mat* im, ImageBlob* data) { + double e = 1.0; + if (is_scale_) { + e /= 255.0; + } + (*im).convertTo(*im, CV_32FC3, e); + for (int h = 0; h < im->rows; h++) { + for (int w = 0; w < im->cols; w++) { + im->at(h, w)[0] = + (im->at(h, w)[0] - mean_[0]) / scale_[0]; + im->at(h, w)[1] = + (im->at(h, w)[1] - mean_[1]) / scale_[1]; + im->at(h, w)[2] = + (im->at(h, w)[2] - mean_[2]) / scale_[2]; + } + } +} + +void Permute::Run(cv::Mat* im, ImageBlob* data) { + (*im).convertTo(*im, CV_32FC3); + int rh = im->rows; + int rw = im->cols; + int rc = im->channels(); + (data->im_data_).resize(rc * rh * rw); + float* base = (data->im_data_).data(); + for (int i = 0; i < rc; ++i) { + cv::extractChannel(*im, cv::Mat(rh, rw, CV_32FC1, base + i * rh * rw), i); + } +} + +void Resize::Run(cv::Mat* im, ImageBlob* data) { + auto resize_scale = GenerateScale(*im); + data->im_shape_ = {static_cast(im->cols * resize_scale.first), + static_cast(im->rows * resize_scale.second)}; + data->in_net_shape_ = {static_cast(im->cols * resize_scale.first), + static_cast(im->rows * resize_scale.second)}; + cv::resize( + *im, *im, cv::Size(), resize_scale.first, resize_scale.second, interp_); + data->im_shape_ = { + static_cast(im->rows), static_cast(im->cols), + }; + data->scale_factor_ = { + resize_scale.second, resize_scale.first, + }; +} + +std::pair Resize::GenerateScale(const cv::Mat& im) { + std::pair resize_scale; + int origin_w = im.cols; + int origin_h = im.rows; + + if (keep_ratio_) { + int im_size_max = std::max(origin_w, origin_h); + int im_size_min = std::min(origin_w, origin_h); + int target_size_max = + *std::max_element(target_size_.begin(), target_size_.end()); + int target_size_min = + *std::min_element(target_size_.begin(), target_size_.end()); + float scale_min = + static_cast(target_size_min) / static_cast(im_size_min); + float scale_max = + static_cast(target_size_max) / static_cast(im_size_max); + float scale_ratio = std::min(scale_min, scale_max); + resize_scale = {scale_ratio, scale_ratio}; + } else { + resize_scale.first = + static_cast(target_size_[1]) / static_cast(origin_w); + resize_scale.second = + static_cast(target_size_[0]) / static_cast(origin_h); + } + return resize_scale; +} + +void LetterBoxResize::Run(cv::Mat* im, ImageBlob* data) { + float resize_scale = GenerateScale(*im); + int new_shape_w = std::round(im->cols * resize_scale); + int new_shape_h = std::round(im->rows * resize_scale); + data->im_shape_ = {static_cast(new_shape_h), + static_cast(new_shape_w)}; + float padw = (target_size_[1] - new_shape_w) / 2.; + float padh = (target_size_[0] - new_shape_h) / 2.; + + int top = std::round(padh - 0.1); + int bottom = std::round(padh + 0.1); + int left = std::round(padw - 0.1); + int right = std::round(padw + 0.1); + + cv::resize( + *im, *im, cv::Size(new_shape_w, new_shape_h), 0, 0, cv::INTER_AREA); + + data->in_net_shape_ = { + static_cast(im->rows), static_cast(im->cols), + }; + cv::copyMakeBorder(*im, + *im, + top, + bottom, + left, + right, + cv::BORDER_CONSTANT, + cv::Scalar(127.5)); + + data->in_net_shape_ = { + static_cast(im->rows), static_cast(im->cols), + }; + + data->scale_factor_ = { + resize_scale, resize_scale, + }; +} + +float LetterBoxResize::GenerateScale(const cv::Mat& im) { + int origin_w = im.cols; + int origin_h = im.rows; + + int target_h = target_size_[0]; + int target_w = target_size_[1]; + + float ratio_h = static_cast(target_h) / static_cast(origin_h); + float ratio_w = static_cast(target_w) / static_cast(origin_w); + float resize_scale = std::min(ratio_h, ratio_w); + return resize_scale; +} + +void PadStride::Run(cv::Mat* im, ImageBlob* data) { + if (stride_ <= 0) { + return; + } + int rc = im->channels(); + int rh = im->rows; + int rw = im->cols; + int nh = (rh / stride_) * stride_ + (rh % stride_ != 0) * stride_; + int nw = (rw / stride_) * stride_ + (rw % stride_ != 0) * stride_; + cv::copyMakeBorder( + *im, *im, 0, nh - rh, 0, nw - rw, cv::BORDER_CONSTANT, cv::Scalar(0)); + data->in_net_shape_ = { + static_cast(im->rows), static_cast(im->cols), + }; +} + +// Preprocessor op running order +const std::vector Preprocessor::RUN_ORDER = {"InitInfo", + "Resize", + "LetterBoxResize", + "NormalizeImage", + "PadStride", + "Permute"}; + +void Preprocessor::Run(cv::Mat* im, ImageBlob* data) { + for (const auto& name : RUN_ORDER) { + if (ops_.find(name) != ops_.end()) { + ops_[name]->Run(im, data); + } + } +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h b/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h new file mode 100644 index 00000000000..b45388c91cd --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h @@ -0,0 +1,171 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace PaddleDetection { + +// Object for storing all preprocessed data +class ImageBlob { + public: + // image width and height + std::vector im_shape_; + // Buffer for image data after preprocessing + std::vector im_data_; + // in net data shape(after pad) + std::vector in_net_shape_; + // Evaluation image width and height + // std::vector eval_im_size_f_; + // Scale factor for image size to origin image size + std::vector scale_factor_; +}; + +// Abstraction of preprocessing opration class +class PreprocessOp { + public: + virtual void Init(const YAML::Node& item) = 0; + virtual void Run(cv::Mat* im, ImageBlob* data) = 0; +}; + +class InitInfo : public PreprocessOp { + public: + virtual void Init(const YAML::Node& item) {} + virtual void Run(cv::Mat* im, ImageBlob* data); +}; + +class NormalizeImage : public PreprocessOp { + public: + virtual void Init(const YAML::Node& item) { + mean_ = item["mean"].as>(); + scale_ = item["std"].as>(); + is_scale_ = item["is_scale"].as(); + } + + virtual void Run(cv::Mat* im, ImageBlob* data); + + private: + // CHW or HWC + std::vector mean_; + std::vector scale_; + bool is_scale_; +}; + +class Permute : public PreprocessOp { + public: + virtual void Init(const YAML::Node& item) {} + virtual void Run(cv::Mat* im, ImageBlob* data); +}; + +class Resize : public PreprocessOp { + public: + virtual void Init(const YAML::Node& item) { + interp_ = item["interp"].as(); + keep_ratio_ = item["keep_ratio"].as(); + target_size_ = item["target_size"].as>(); + } + + // Compute best resize scale for x-dimension, y-dimension + std::pair GenerateScale(const cv::Mat& im); + + virtual void Run(cv::Mat* im, ImageBlob* data); + + private: + int interp_; + bool keep_ratio_; + std::vector target_size_; + std::vector in_net_shape_; +}; + +class LetterBoxResize : public PreprocessOp { + public: + virtual void Init(const YAML::Node& item) { + target_size_ = item["target_size"].as>(); + } + + float GenerateScale(const cv::Mat& im); + + virtual void Run(cv::Mat* im, ImageBlob* data); + + private: + std::vector target_size_; + std::vector in_net_shape_; +}; +// Models with FPN need input shape % stride == 0 +class PadStride : public PreprocessOp { + public: + virtual void Init(const YAML::Node& item) { + stride_ = item["stride"].as(); + } + + virtual void Run(cv::Mat* im, ImageBlob* data); + + private: + int stride_; +}; + +class Preprocessor { + public: + void Init(const YAML::Node& config_node) { + // initialize image info at first + ops_["InitInfo"] = std::make_shared(); + for (const auto& item : config_node) { + auto op_name = item["type"].as(); + + ops_[op_name] = CreateOp(op_name); + ops_[op_name]->Init(item); + } + } + + std::shared_ptr CreateOp(const std::string& name) { + if (name == "Resize") { + return std::make_shared(); + } else if (name == "LetterBoxResize") { + return std::make_shared(); + } else if (name == "Permute") { + return std::make_shared(); + } else if (name == "NormalizeImage") { + return std::make_shared(); + } else if (name == "PadStride") { + // use PadStride instead of PadBatch + return std::make_shared(); + } + std::cerr << "can not find function of OP: " << name + << " and return: nullptr" << std::endl; + return nullptr; + } + + void Run(cv::Mat* im, ImageBlob* data); + + public: + static const std::vector RUN_ORDER; + + private: + std::unordered_map> ops_; +}; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/utils.h b/fastdeploy/vision/tracking/pptracking/utils/utils.h new file mode 100644 index 00000000000..9d94492a430 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/utils/utils.h @@ -0,0 +1,44 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "include/tracker.h" + +namespace PaddleDetection { + +struct Rect { + float left; + float top; + float right; + float bottom; +}; + +struct MOTTrack { + int ids; + float score; + Rect rects; + int class_id = -1; +}; + +typedef std::vector MOTResult; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/tracking_pybind.cc b/fastdeploy/vision/tracking/tracking_pybind.cc new file mode 100644 index 00000000000..cfec96db820 --- /dev/null +++ b/fastdeploy/vision/tracking/tracking_pybind.cc @@ -0,0 +1,3 @@ +// +// Created by aichao on 2022/10/9. +// From 2910b3ff22112545a3d796e23bcb80261b3566e7 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Tue, 11 Oct 2022 10:05:41 +0800 Subject: [PATCH 04/25] recovery --- .../tracking/pptracking/jde_predictor.cc | 235 -------- .../tracking/pptracking/jde_predictor.h | 97 ---- .../vision/tracking/pptracking/lapjv.cpp | 409 -------------- fastdeploy/vision/tracking/pptracking/lapjv.h | 64 --- .../vision/tracking/pptracking/pipeline.cc | 367 ------------- .../vision/tracking/pptracking/pipeline.h | 142 ----- .../vision/tracking/pptracking/predictor.cc | 35 -- .../vision/tracking/pptracking/predictor.h | 99 ---- .../tracking/pptracking/sde_predictor.cc | 46 -- .../tracking/pptracking/sde_predictor.h | 106 ---- .../vision/tracking/pptracking/tracker.cc | 304 ---------- .../vision/tracking/pptracking/tracker.h | 72 --- .../vision/tracking/pptracking/trajectory.cc | 517 ------------------ .../vision/tracking/pptracking/trajectory.h | 230 -------- .../tracking/pptracking/utils/postprocess.cc | 207 ------- .../tracking/pptracking/utils/postprocess.h | 62 --- .../pptracking/utils/preprocess_op.cc | 187 ------- .../tracking/pptracking/utils/preprocess_op.h | 171 ------ .../vision/tracking/pptracking/utils/utils.h | 44 -- fastdeploy/vision/tracking/tracking_pybind.cc | 3 - 20 files changed, 3397 deletions(-) delete mode 100644 fastdeploy/vision/tracking/pptracking/jde_predictor.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/jde_predictor.h delete mode 100644 fastdeploy/vision/tracking/pptracking/lapjv.cpp delete mode 100644 fastdeploy/vision/tracking/pptracking/lapjv.h delete mode 100644 fastdeploy/vision/tracking/pptracking/pipeline.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/pipeline.h delete mode 100644 fastdeploy/vision/tracking/pptracking/predictor.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/predictor.h delete mode 100644 fastdeploy/vision/tracking/pptracking/sde_predictor.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/sde_predictor.h delete mode 100644 fastdeploy/vision/tracking/pptracking/tracker.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/tracker.h delete mode 100644 fastdeploy/vision/tracking/pptracking/trajectory.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/trajectory.h delete mode 100644 fastdeploy/vision/tracking/pptracking/utils/postprocess.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/utils/postprocess.h delete mode 100644 fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h delete mode 100644 fastdeploy/vision/tracking/pptracking/utils/utils.h delete mode 100644 fastdeploy/vision/tracking/tracking_pybind.cc diff --git a/fastdeploy/vision/tracking/pptracking/jde_predictor.cc b/fastdeploy/vision/tracking/pptracking/jde_predictor.cc deleted file mode 100644 index 0673d8a0bb1..00000000000 --- a/fastdeploy/vision/tracking/pptracking/jde_predictor.cc +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include -// for setprecision -#include -#include -#include "include/jde_predictor.h" - -using namespace paddle_infer; // NOLINT - -namespace PaddleDetection { - -// Load Model and create model predictor -void JDEPredictor::LoadModel(const std::string& model_dir, - const std::string& run_mode) { - paddle_infer::Config config; - std::string prog_file = model_dir + OS_PATH_SEP + "model.pdmodel"; - std::string params_file = model_dir + OS_PATH_SEP + "model.pdiparams"; - config.SetModel(prog_file, params_file); - if (this->device_ == "GPU") { - config.EnableUseGpu(200, this->gpu_id_); - config.SwitchIrOptim(true); - // use tensorrt - if (run_mode != "paddle") { - auto precision = paddle_infer::Config::Precision::kFloat32; - if (run_mode == "trt_fp32") { - precision = paddle_infer::Config::Precision::kFloat32; - } else if (run_mode == "trt_fp16") { - precision = paddle_infer::Config::Precision::kHalf; - } else if (run_mode == "trt_int8") { - precision = paddle_infer::Config::Precision::kInt8; - } else { - printf( - "run_mode should be 'paddle', 'trt_fp32', 'trt_fp16' or " - "'trt_int8'"); - } - // set tensorrt - config.EnableTensorRtEngine(1 << 30, - 1, - this->min_subgraph_size_, - precision, - false, - this->trt_calib_mode_); - } - } else if (this->device_ == "XPU") { - config.EnableXpu(10 * 1024 * 1024); - } else { - config.DisableGpu(); - if (this->use_mkldnn_) { - config.EnableMKLDNN(); - // cache 10 different shapes for mkldnn to avoid memory leak - config.SetMkldnnCacheCapacity(10); - } - config.SetCpuMathLibraryNumThreads(this->cpu_math_library_num_threads_); - } - config.SwitchUseFeedFetchOps(false); - config.SwitchIrOptim(true); - config.DisableGlogInfo(); - // Memory optimization - config.EnableMemoryOptim(); - predictor_ = std::move(CreatePredictor(config)); -} - -void FilterDets(const float conf_thresh, - const cv::Mat dets, - std::vector* index) { - for (int i = 0; i < dets.rows; ++i) { - float score = *dets.ptr(i, 4); - if (score > conf_thresh) { - index->push_back(i); - } - } -} - -void JDEPredictor::Preprocess(const cv::Mat& ori_im) { - // Clone the image : keep the original mat for postprocess - cv::Mat im = ori_im.clone(); - preprocessor_.Run(&im, &inputs_); -} - -void JDEPredictor::Postprocess(const cv::Mat dets, - const cv::Mat emb, - MOTResult* result) { - result->clear(); - std::vector tracks; - std::vector valid; - FilterDets(conf_thresh_, dets, &valid); - cv::Mat new_dets, new_emb; - for (int i = 0; i < valid.size(); ++i) { - new_dets.push_back(dets.row(valid[i])); - new_emb.push_back(emb.row(valid[i])); - } - JDETracker::instance()->update(new_dets, new_emb, &tracks); - if (tracks.size() == 0) { - MOTTrack mot_track; - Rect ret = {*dets.ptr(0, 0), - *dets.ptr(0, 1), - *dets.ptr(0, 2), - *dets.ptr(0, 3)}; - mot_track.ids = 1; - mot_track.score = *dets.ptr(0, 4); - mot_track.rects = ret; - result->push_back(mot_track); - } else { - std::vector::iterator titer; - for (titer = tracks.begin(); titer != tracks.end(); ++titer) { - if (titer->score < threshold_) { - continue; - } else { - float w = titer->ltrb[2] - titer->ltrb[0]; - float h = titer->ltrb[3] - titer->ltrb[1]; - bool vertical = w / h > 1.6; - float area = w * h; - if (area > min_box_area_ && !vertical) { - MOTTrack mot_track; - Rect ret = { - titer->ltrb[0], titer->ltrb[1], titer->ltrb[2], titer->ltrb[3]}; - mot_track.rects = ret; - mot_track.score = titer->score; - mot_track.ids = titer->id; - result->push_back(mot_track); - } - } - } - } -} - -void JDEPredictor::Predict(const std::vector imgs, - const double threshold, - MOTResult* result, - std::vector* times) { - auto preprocess_start = std::chrono::steady_clock::now(); - int batch_size = imgs.size(); - - // in_data_batch - std::vector in_data_all; - std::vector im_shape_all(batch_size * 2); - std::vector scale_factor_all(batch_size * 2); - - // Preprocess image - for (int bs_idx = 0; bs_idx < batch_size; bs_idx++) { - cv::Mat im = imgs.at(bs_idx); - Preprocess(im); - im_shape_all[bs_idx * 2] = inputs_.im_shape_[0]; - im_shape_all[bs_idx * 2 + 1] = inputs_.im_shape_[1]; - - scale_factor_all[bs_idx * 2] = inputs_.scale_factor_[0]; - scale_factor_all[bs_idx * 2 + 1] = inputs_.scale_factor_[1]; - - in_data_all.insert( - in_data_all.end(), inputs_.im_data_.begin(), inputs_.im_data_.end()); - } - - // Prepare input tensor - auto input_names = predictor_->GetInputNames(); - for (const auto& tensor_name : input_names) { - auto in_tensor = predictor_->GetInputHandle(tensor_name); - if (tensor_name == "image") { - int rh = inputs_.in_net_shape_[0]; - int rw = inputs_.in_net_shape_[1]; - in_tensor->Reshape({batch_size, 3, rh, rw}); - in_tensor->CopyFromCpu(in_data_all.data()); - } else if (tensor_name == "im_shape") { - in_tensor->Reshape({batch_size, 2}); - in_tensor->CopyFromCpu(im_shape_all.data()); - } else if (tensor_name == "scale_factor") { - in_tensor->Reshape({batch_size, 2}); - in_tensor->CopyFromCpu(scale_factor_all.data()); - } - } - - auto preprocess_end = std::chrono::steady_clock::now(); - std::vector bbox_shape; - std::vector emb_shape; - - // Run predictor - auto inference_start = std::chrono::steady_clock::now(); - predictor_->Run(); - // Get output tensor - auto output_names = predictor_->GetOutputNames(); - auto bbox_tensor = predictor_->GetOutputHandle(output_names[0]); - bbox_shape = bbox_tensor->shape(); - auto emb_tensor = predictor_->GetOutputHandle(output_names[1]); - emb_shape = emb_tensor->shape(); - // Calculate bbox length - int bbox_size = 1; - for (int j = 0; j < bbox_shape.size(); ++j) { - bbox_size *= bbox_shape[j]; - } - // Calculate emb length - int emb_size = 1; - for (int j = 0; j < emb_shape.size(); ++j) { - emb_size *= emb_shape[j]; - } - - bbox_data_.resize(bbox_size); - bbox_tensor->CopyToCpu(bbox_data_.data()); - - emb_data_.resize(emb_size); - emb_tensor->CopyToCpu(emb_data_.data()); - auto inference_end = std::chrono::steady_clock::now(); - - // Postprocessing result - auto postprocess_start = std::chrono::steady_clock::now(); - result->clear(); - - cv::Mat dets(bbox_shape[0], 6, CV_32FC1, bbox_data_.data()); - cv::Mat emb(bbox_shape[0], emb_shape[1], CV_32FC1, emb_data_.data()); - - Postprocess(dets, emb, result); - - auto postprocess_end = std::chrono::steady_clock::now(); - - std::chrono::duration preprocess_diff = - preprocess_end - preprocess_start; - (*times)[0] += static_cast(preprocess_diff.count() * 1000); - std::chrono::duration inference_diff = inference_end - inference_start; - (*times)[1] += static_cast(inference_diff.count() * 1000); - std::chrono::duration postprocess_diff = - postprocess_end - postprocess_start; - (*times)[2] += static_cast(postprocess_diff.count() * 1000); -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/jde_predictor.h b/fastdeploy/vision/tracking/pptracking/jde_predictor.h deleted file mode 100644 index 32f5921061c..00000000000 --- a/fastdeploy/vision/tracking/pptracking/jde_predictor.h +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "paddle_inference_api.h" // NOLINT - -#include "include/config_parser.h" -#include "include/preprocess_op.h" -#include "include/utils.h" - -using namespace paddle_infer; // NOLINT - -namespace PaddleDetection { - -class JDEPredictor { - public: - explicit JDEPredictor(const std::string& device = "CPU", - const std::string& model_dir = "", - const double threshold = -1., - const std::string& run_mode = "paddle", - const int gpu_id = 0, - const bool use_mkldnn = false, - const int cpu_threads = 1, - bool trt_calib_mode = false, - const int min_box_area = 200) { - this->device_ = device; - this->gpu_id_ = gpu_id; - this->use_mkldnn_ = use_mkldnn; - this->cpu_math_library_num_threads_ = cpu_threads; - this->trt_calib_mode_ = trt_calib_mode; - this->min_box_area_ = min_box_area; - - config_.load_config(model_dir); - this->min_subgraph_size_ = config_.min_subgraph_size_; - preprocessor_.Init(config_.preprocess_info_); - LoadModel(model_dir, run_mode); - this->conf_thresh_ = config_.conf_thresh_; - } - - // Load Paddle inference model - void LoadModel(const std::string& model_dir, - const std::string& run_mode = "paddle"); - - // Run predictor - void Predict(const std::vector imgs, - const double threshold = 0.5, - MOTResult* result = nullptr, - std::vector* times = nullptr); - - private: - std::string device_ = "CPU"; - float threhold = 0.5; - int gpu_id_ = 0; - bool use_mkldnn_ = false; - int cpu_math_library_num_threads_ = 1; - int min_subgraph_size_ = 3; - bool trt_calib_mode_ = false; - - // Preprocess image and copy data to input buffer - void Preprocess(const cv::Mat& image_mat); - // Postprocess result - void Postprocess(const cv::Mat dets, const cv::Mat emb, MOTResult* result); - - std::shared_ptr predictor_; - Preprocessor preprocessor_; - ImageBlob inputs_; - std::vector bbox_data_; - std::vector emb_data_; - double threshold_; - ConfigPaser config_; - float min_box_area_; - float conf_thresh_; -}; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.cpp b/fastdeploy/vision/tracking/pptracking/lapjv.cpp deleted file mode 100644 index bb710e740e5..00000000000 --- a/fastdeploy/vision/tracking/pptracking/lapjv.cpp +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code is based on: -// https://github.com/gatagat/lap/blob/master/lap/lapjv.cpp -// Ths copyright of gatagat/lap is as follows: -// MIT License - -#include -#include -#include - -#include "include/lapjv.h" - -namespace PaddleDetection { - -/** Column-reduction and reduction transfer for a dense cost matrix. - */ -int _ccrrt_dense( - const int n, float *cost[], int *free_rows, int *x, int *y, float *v) { - int n_free_rows; - bool *unique; - - for (int i = 0; i < n; i++) { - x[i] = -1; - v[i] = LARGE; - y[i] = 0; - } - for (int i = 0; i < n; i++) { - for (int j = 0; j < n; j++) { - const float c = cost[i][j]; - if (c < v[j]) { - v[j] = c; - y[j] = i; - } - } - } - NEW(unique, bool, n); - memset(unique, TRUE, n); - { - int j = n; - do { - j--; - const int i = y[j]; - if (x[i] < 0) { - x[i] = j; - } else { - unique[i] = FALSE; - y[j] = -1; - } - } while (j > 0); - } - n_free_rows = 0; - for (int i = 0; i < n; i++) { - if (x[i] < 0) { - free_rows[n_free_rows++] = i; - } else if (unique[i]) { - const int j = x[i]; - float min = LARGE; - for (int j2 = 0; j2 < n; j2++) { - if (j2 == static_cast(j)) { - continue; - } - const float c = cost[i][j2] - v[j2]; - if (c < min) { - min = c; - } - } - v[j] -= min; - } - } - FREE(unique); - return n_free_rows; -} - -/** Augmenting row reduction for a dense cost matrix. - */ -int _carr_dense(const int n, - float *cost[], - const int n_free_rows, - int *free_rows, - int *x, - int *y, - float *v) { - int current = 0; - int new_free_rows = 0; - int rr_cnt = 0; - while (current < n_free_rows) { - int i0; - int j1, j2; - float v1, v2, v1_new; - bool v1_lowers; - - rr_cnt++; - const int free_i = free_rows[current++]; - j1 = 0; - v1 = cost[free_i][0] - v[0]; - j2 = -1; - v2 = LARGE; - for (int j = 1; j < n; j++) { - const float c = cost[free_i][j] - v[j]; - if (c < v2) { - if (c >= v1) { - v2 = c; - j2 = j; - } else { - v2 = v1; - v1 = c; - j2 = j1; - j1 = j; - } - } - } - i0 = y[j1]; - v1_new = v[j1] - (v2 - v1); - v1_lowers = v1_new < v[j1]; - if (rr_cnt < current * n) { - if (v1_lowers) { - v[j1] = v1_new; - } else if (i0 >= 0 && j2 >= 0) { - j1 = j2; - i0 = y[j2]; - } - if (i0 >= 0) { - if (v1_lowers) { - free_rows[--current] = i0; - } else { - free_rows[new_free_rows++] = i0; - } - } - } else { - if (i0 >= 0) { - free_rows[new_free_rows++] = i0; - } - } - x[free_i] = j1; - y[j1] = free_i; - } - return new_free_rows; -} - -/** Find columns with minimum d[j] and put them on the SCAN list. - */ -int _find_dense(const int n, int lo, float *d, int *cols, int *y) { - int hi = lo + 1; - float mind = d[cols[lo]]; - for (int k = hi; k < n; k++) { - int j = cols[k]; - if (d[j] <= mind) { - if (d[j] < mind) { - hi = lo; - mind = d[j]; - } - cols[k] = cols[hi]; - cols[hi++] = j; - } - } - return hi; -} - -// Scan all columns in TODO starting from arbitrary column in SCAN -// and try to decrease d of the TODO columns using the SCAN column. -int _scan_dense(const int n, - float *cost[], - int *plo, - int *phi, - float *d, - int *cols, - int *pred, - int *y, - float *v) { - int lo = *plo; - int hi = *phi; - float h, cred_ij; - - while (lo != hi) { - int j = cols[lo++]; - const int i = y[j]; - const float mind = d[j]; - h = cost[i][j] - v[j] - mind; - // For all columns in TODO - for (int k = hi; k < n; k++) { - j = cols[k]; - cred_ij = cost[i][j] - v[j] - h; - if (cred_ij < d[j]) { - d[j] = cred_ij; - pred[j] = i; - if (cred_ij == mind) { - if (y[j] < 0) { - return j; - } - cols[k] = cols[hi]; - cols[hi++] = j; - } - } - } - } - *plo = lo; - *phi = hi; - return -1; -} - -/** Single iteration of modified Dijkstra shortest path algorithm as explained - * in the JV paper. - * - * This is a dense matrix version. - * - * \return The closest free column index. - */ -int find_path_dense(const int n, - float *cost[], - const int start_i, - int *y, - float *v, - int *pred) { - int lo = 0, hi = 0; - int final_j = -1; - int n_ready = 0; - int *cols; - float *d; - - NEW(cols, int, n); - NEW(d, float, n); - - for (int i = 0; i < n; i++) { - cols[i] = i; - pred[i] = start_i; - d[i] = cost[start_i][i] - v[i]; - } - while (final_j == -1) { - // No columns left on the SCAN list. - if (lo == hi) { - n_ready = lo; - hi = _find_dense(n, lo, d, cols, y); - for (int k = lo; k < hi; k++) { - const int j = cols[k]; - if (y[j] < 0) { - final_j = j; - } - } - } - if (final_j == -1) { - final_j = _scan_dense(n, cost, &lo, &hi, d, cols, pred, y, v); - } - } - - { - const float mind = d[cols[lo]]; - for (int k = 0; k < n_ready; k++) { - const int j = cols[k]; - v[j] += d[j] - mind; - } - } - - FREE(cols); - FREE(d); - - return final_j; -} - -/** Augment for a dense cost matrix. - */ -int _ca_dense(const int n, - float *cost[], - const int n_free_rows, - int *free_rows, - int *x, - int *y, - float *v) { - int *pred; - - NEW(pred, int, n); - - for (int *pfree_i = free_rows; pfree_i < free_rows + n_free_rows; pfree_i++) { - int i = -1, j; - int k = 0; - - j = find_path_dense(n, cost, *pfree_i, y, v, pred); - while (i != *pfree_i) { - i = pred[j]; - y[j] = i; - SWAP_INDICES(j, x[i]); - k++; - } - } - FREE(pred); - return 0; -} - -/** Solve dense sparse LAP. - */ -int lapjv_internal(const cv::Mat &cost, - const bool extend_cost, - const float cost_limit, - int *x, - int *y) { - int n_rows = cost.rows; - int n_cols = cost.cols; - int n; - if (n_rows == n_cols) { - n = n_rows; - } else if (!extend_cost) { - throw std::invalid_argument( - "Square cost array expected. If cost is intentionally non-square, pass " - "extend_cost=True."); - } - - // Get extend cost - if (extend_cost || cost_limit < LARGE) { - n = n_rows + n_cols; - } - cv::Mat cost_expand(n, n, CV_32F); - float expand_value; - if (cost_limit < LARGE) { - expand_value = cost_limit / 2; - } else { - double max_v; - minMaxLoc(cost, nullptr, &max_v); - expand_value = static_cast(max_v) + 1.; - } - - for (int i = 0; i < n; ++i) { - for (int j = 0; j < n; ++j) { - cost_expand.at(i, j) = expand_value; - if (i >= n_rows && j >= n_cols) { - cost_expand.at(i, j) = 0; - } else if (i < n_rows && j < n_cols) { - cost_expand.at(i, j) = cost.at(i, j); - } - } - } - - // Convert Mat to pointer array - float **cost_ptr; - NEW(cost_ptr, float *, n); - for (int i = 0; i < n; ++i) { - NEW(cost_ptr[i], float, n); - } - for (int i = 0; i < n; ++i) { - for (int j = 0; j < n; ++j) { - cost_ptr[i][j] = cost_expand.at(i, j); - } - } - - int ret; - int *free_rows; - float *v; - int *x_c; - int *y_c; - - NEW(free_rows, int, n); - NEW(v, float, n); - NEW(x_c, int, n); - NEW(y_c, int, n); - - ret = _ccrrt_dense(n, cost_ptr, free_rows, x_c, y_c, v); - int i = 0; - while (ret > 0 && i < 2) { - ret = _carr_dense(n, cost_ptr, ret, free_rows, x_c, y_c, v); - i++; - } - if (ret > 0) { - ret = _ca_dense(n, cost_ptr, ret, free_rows, x_c, y_c, v); - } - FREE(v); - FREE(free_rows); - for (int i = 0; i < n; ++i) { - FREE(cost_ptr[i]); - } - FREE(cost_ptr); - if (ret != 0) { - if (ret == -1) { - throw "Out of memory."; - } - throw "Unknown error (lapjv_internal)"; - } - // Get output of x, y, opt - for (int i = 0; i < n; ++i) { - if (i < n_rows) { - x[i] = x_c[i]; - if (x[i] >= n_cols) { - x[i] = -1; - } - } - if (i < n_cols) { - y[i] = y_c[i]; - if (y[i] >= n_rows) { - y[i] = -1; - } - } - } - - FREE(x_c); - FREE(y_c); - return ret; -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.h b/fastdeploy/vision/tracking/pptracking/lapjv.h deleted file mode 100644 index ffaa010c005..00000000000 --- a/fastdeploy/vision/tracking/pptracking/lapjv.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code is based on: -// https://github.com/gatagat/lap/blob/master/lap/lapjv.h -// Ths copyright of gatagat/lap is as follows: -// MIT License - -#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ -#define DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ -#define LARGE 1000000 - -#if !defined TRUE -#define TRUE 1 -#endif -#if !defined FALSE -#define FALSE 0 -#endif - -#define NEW(x, t, n) \ - if ((x = reinterpret_cast(malloc(sizeof(t) * (n)))) == 0) { \ - return -1; \ - } -#define FREE(x) \ - if (x != 0) { \ - free(x); \ - x = 0; \ - } -#define SWAP_INDICES(a, b) \ - { \ - int_t _temp_index = a; \ - a = b; \ - b = _temp_index; \ - } -#include - -namespace PaddleDetection { - -typedef signed int int_t; -typedef unsigned int uint_t; -typedef double cost_t; -typedef char boolean; -typedef enum fp_t { FP_1 = 1, FP_2 = 2, FP_DYNAMIC = 3 } fp_t; - -int lapjv_internal(const cv::Mat &cost, - const bool extend_cost, - const float cost_limit, - int *x, - int *y); - -} // namespace PaddleDetection - -#endif // DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ diff --git a/fastdeploy/vision/tracking/pptracking/pipeline.cc b/fastdeploy/vision/tracking/pptracking/pipeline.cc deleted file mode 100644 index 7c22c630f9f..00000000000 --- a/fastdeploy/vision/tracking/pptracking/pipeline.cc +++ /dev/null @@ -1,367 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -// for setprecision -#include -#include -#include -#include - -#include "include/pipeline.h" -#include "include/postprocess.h" -#include "include/predictor.h" - -namespace PaddleDetection { - -void Pipeline::SetInput(const std::string& input_video) { - input_.push_back(input_video); -} - -void Pipeline::ClearInput() { - input_.clear(); - stream_.clear(); -} - -void Pipeline::SelectModel(const std::string& scene, - const bool tiny_obj, - const bool is_mtmct, - const std::string track_model_dir, - const std::string det_model_dir, - const std::string reid_model_dir) { - // model_dir has higher priority - if (!track_model_dir.empty()) { - track_model_dir_ = track_model_dir; - return; - } - if (!det_model_dir.empty() && !reid_model_dir.empty()) { - det_model_dir_ = det_model_dir; - reid_model_dir_ = reid_model_dir; - return; - } - - // Single camera model, based on FairMot - if (scene == "pedestrian") { - if (tiny_obj) { - track_model_dir_ = "../pedestrian_track_tiny"; - } else { - track_model_dir_ = "../pedestrian_track"; - } - } else if (scene != "vehicle") { - if (tiny_obj) { - track_model_dir_ = "../vehicle_track_tiny"; - } else { - track_model_dir_ = "../vehicle_track"; - } - } else if (scene == "multiclass") { - if (tiny_obj) { - track_model_dir_ = "../multiclass_track_tiny"; - } else { - track_model_dir_ = "../multiclass_track"; - } - } - - // Multi-camera model, based on PicoDet & LCNet - if (is_mtmct && scene == "pedestrian") { - det_model_dir_ = "../pedestrian_det"; - reid_model_dir_ = "../pedestrian_reid"; - } else if (is_mtmct && scene == "vehicle") { - det_model_dir_ = "../vehicle_det"; - reid_model_dir_ = "../vehicle_reid"; - } else if (is_mtmct && scene == "multiclass") { - throw "Multi-camera tracking is not supported in multiclass scene now."; - } -} - -void Pipeline::InitPredictor() { - if (track_model_dir_.empty() && det_model_dir_.empty()) { - throw "Predictor must receive track_model or det_model!"; - } - - if (!track_model_dir_.empty()) { - jde_sct_ = std::make_shared(device_, - track_model_dir_, - threshold_, - run_mode_, - gpu_id_, - use_mkldnn_, - cpu_threads_, - trt_calib_mode_); - } - if (!det_model_dir_.empty()) { - sde_sct_ = std::make_shared(device_, - det_model_dir_, - reid_model_dir_, - threshold_, - run_mode_, - gpu_id_, - use_mkldnn_, - cpu_threads_, - trt_calib_mode_); - } -} - -void Pipeline::Run() { - if (track_model_dir_.empty() && det_model_dir_.empty()) { - LOG(ERROR) << "Pipeline must use SelectModel before Run"; - return; - } - if (input_.size() == 0) { - LOG(ERROR) << "Pipeline must use SetInput before Run"; - return; - } - - if (!track_model_dir_.empty()) { - // single camera - if (input_.size() > 1) { - throw "Single camera tracking except single video, but received %d", - input_.size(); - } - PredictMOT(input_[0]); - } else { - // multi cameras - if (input_.size() != 2) { - throw "Multi camera tracking except two videos, but received %d", - input_.size(); - } - PredictMTMCT(input_); - } -} - -void Pipeline::PredictMOT(const std::string& video_path) { - // Open video - cv::VideoCapture capture; - capture.open(video_path.c_str()); - if (!capture.isOpened()) { - printf("can not open video : %s\n", video_path.c_str()); - return; - } - - // Get Video info : resolution, fps - int video_width = static_cast(capture.get(CV_CAP_PROP_FRAME_WIDTH)); - int video_height = static_cast(capture.get(CV_CAP_PROP_FRAME_HEIGHT)); - int video_fps = static_cast(capture.get(CV_CAP_PROP_FPS)); - - LOG(INFO) << "----------------------- Input info -----------------------"; - LOG(INFO) << "video_width: " << video_width; - LOG(INFO) << "video_height: " << video_height; - LOG(INFO) << "input fps: " << video_fps; - - // Create VideoWriter for output - cv::VideoWriter video_out; - std::string video_out_path = output_dir_ + OS_PATH_SEP + "mot_output.mp4"; - int fcc = cv::VideoWriter::fourcc('m', 'p', '4', 'v'); - video_out.open(video_out_path.c_str(), - fcc, // 0x00000021, - video_fps, - cv::Size(video_width, video_height), - true); - if (!video_out.isOpened()) { - printf("create video writer failed!\n"); - return; - } - - PaddleDetection::MOTResult result; - std::vector det_times(3); - std::set id_set; - std::set interval_id_set; - std::vector in_id_list; - std::vector out_id_list; - std::map> prev_center; - Rect entrance = {0, - static_cast(video_height) / 2, - static_cast(video_width), - static_cast(video_height) / 2}; - double times; - double total_time; - // Capture all frames and do inference - cv::Mat frame; - int frame_id = 0; - - std::vector records; - std::vector flow_records; - records.push_back("result format: frame_id, track_id, x1, y1, w, h\n"); - - LOG(INFO) << "------------------- Predict info ------------------------"; - while (capture.read(frame)) { - if (frame.empty()) { - break; - } - std::vector imgs; - imgs.push_back(frame); - jde_sct_->Predict(imgs, threshold_, &result, &det_times); - frame_id += 1; - total_time = std::accumulate(det_times.begin(), det_times.end(), 0.); - times = total_time / frame_id; - - LOG(INFO) << "frame_id: " << frame_id - << " predict time(s): " << times / 1000; - - cv::Mat out_img = PaddleDetection::VisualizeTrackResult( - frame, result, 1000. / times, frame_id); - - // TODO(qianhui): the entrance line can be set by users - PaddleDetection::FlowStatistic(result, - frame_id, - secs_interval_, - do_entrance_counting_, - video_fps, - entrance, - &id_set, - &interval_id_set, - &in_id_list, - &out_id_list, - &prev_center, - &flow_records); - - if (save_result_) { - PaddleDetection::SaveMOTResult(result, frame_id, &records); - } - - // Draw the entrance line - if (do_entrance_counting_) { - float line_thickness = std::max(1, static_cast(video_width / 500.)); - cv::Point pt1 = cv::Point(entrance.left, entrance.top); - cv::Point pt2 = cv::Point(entrance.right, entrance.bottom); - cv::line(out_img, pt1, pt2, cv::Scalar(0, 255, 255), line_thickness); - } - video_out.write(out_img); - } - capture.release(); - video_out.release(); - PrintBenchmarkLog(det_times, frame_id); - LOG(INFO) << "-------------------- Final Output info -------------------"; - LOG(INFO) << "Total frame: " << frame_id; - LOG(INFO) << "Visualized output saved as " << video_out_path.c_str(); - if (save_result_) { - FILE* fp; - - std::string result_output_path = - output_dir_ + OS_PATH_SEP + "mot_output.txt"; - if ((fp = fopen(result_output_path.c_str(), "w+")) == NULL) { - printf("Open %s error.\n", result_output_path.c_str()); - return; - } - for (int l; l < records.size(); ++l) { - fprintf(fp, records[l].c_str()); - } - - fclose(fp); - LOG(INFO) << "txt result output saved as " << result_output_path.c_str(); - - result_output_path = output_dir_ + OS_PATH_SEP + "flow_statistic.txt"; - if ((fp = fopen(result_output_path.c_str(), "w+")) == NULL) { - printf("Open %s error.\n", result_output_path); - return; - } - for (int l; l < flow_records.size(); ++l) { - fprintf(fp, flow_records[l].c_str()); - } - fclose(fp); - LOG(INFO) << "txt flow statistic saved as " << result_output_path.c_str(); - } -} - -void Pipeline::PredictMTMCT(const std::vector video_path) { - throw "Not Implement!"; -} - -void Pipeline::RunMOTStream(const cv::Mat img, - const int frame_id, - const int video_fps, - const Rect entrance, - cv::Mat out_img, - std::vector* records, - std::set* id_set, - std::set* interval_id_set, - std::vector* in_id_list, - std::vector* out_id_list, - std::map>* prev_center, - std::vector* flow_records) { - PaddleDetection::MOTResult result; - std::vector det_times(3); - double times; - double total_time; - - LOG(INFO) << "------------------- Predict info ------------------------"; - std::vector imgs; - imgs.push_back(img); - jde_sct_->Predict(imgs, threshold_, &result, &det_times); - total_time = std::accumulate(det_times.begin(), det_times.end(), 0.); - times = total_time / frame_id; - - LOG(INFO) << "frame_id: " << frame_id << " predict time(s): " << times / 1000; - - out_img = PaddleDetection::VisualizeTrackResult( - img, result, 1000. / times, frame_id); - - // Count total number - // Count in & out number - PaddleDetection::FlowStatistic(result, - frame_id, - secs_interval_, - do_entrance_counting_, - video_fps, - entrance, - id_set, - interval_id_set, - in_id_list, - out_id_list, - prev_center, - flow_records); - - PrintBenchmarkLog(det_times, frame_id); - if (save_result_) { - PaddleDetection::SaveMOTResult(result, frame_id, records); - } -} - -void Pipeline::RunMTMCTStream(const std::vector imgs, - std::vector* records) { - throw "Not Implement!"; -} - -void Pipeline::PrintBenchmarkLog(const std::vector det_time, - const int img_num) { - LOG(INFO) << "----------------------- Config info -----------------------"; - LOG(INFO) << "runtime_device: " << device_; - LOG(INFO) << "ir_optim: " - << "True"; - LOG(INFO) << "enable_memory_optim: " - << "True"; - int has_trt = run_mode_.find("trt"); - if (has_trt >= 0) { - LOG(INFO) << "enable_tensorrt: " - << "True"; - std::string precision = run_mode_.substr(4, 8); - LOG(INFO) << "precision: " << precision; - } else { - LOG(INFO) << "enable_tensorrt: " - << "False"; - LOG(INFO) << "precision: " - << "fp32"; - } - LOG(INFO) << "enable_mkldnn: " << (use_mkldnn_ ? "True" : "False"); - LOG(INFO) << "cpu_math_library_num_threads: " << cpu_threads_; - LOG(INFO) << "----------------------- Perf info ------------------------"; - LOG(INFO) << "Total number of predicted data: " << img_num - << " and total time spent(s): " - << std::accumulate(det_time.begin(), det_time.end(), 0.) / 1000; - int num = std::max(1, img_num); - LOG(INFO) << "preproce_time(ms): " << det_time[0] / num - << ", inference_time(ms): " << det_time[1] / num - << ", postprocess_time(ms): " << det_time[2] / num; -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/pipeline.h b/fastdeploy/vision/tracking/pptracking/pipeline.h deleted file mode 100644 index d17b4d35aef..00000000000 --- a/fastdeploy/vision/tracking/pptracking/pipeline.h +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_ -#define DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#elif LINUX -#include -#include -#endif - -#include "include/jde_predictor.h" -#include "include/sde_predictor.h" - -namespace PaddleDetection { - -class Pipeline { - public: - explicit Pipeline(const std::string& device, - const double threshold, - const std::string& output_dir, - const std::string& run_mode = "paddle", - const int gpu_id = 0, - const bool use_mkldnn = false, - const int cpu_threads = 1, - const bool trt_calib_mode = false, - const bool do_entrance_counting = false, - const bool save_result = false, - const std::string& scene = "pedestrian", - const bool tiny_obj = false, - const bool is_mtmct = false, - const int secs_interval = 10, - const std::string track_model_dir = "", - const std::string det_model_dir = "", - const std::string reid_model_dir = "") { - std::vector input; - this->input_ = input; - this->device_ = device; - this->threshold_ = threshold; - this->output_dir_ = output_dir; - this->run_mode_ = run_mode; - this->gpu_id_ = gpu_id; - this->use_mkldnn_ = use_mkldnn; - this->cpu_threads_ = cpu_threads; - this->trt_calib_mode_ = trt_calib_mode; - this->do_entrance_counting_ = do_entrance_counting; - this->secs_interval_ = secs_interval_; - this->save_result_ = save_result; - SelectModel(scene, - tiny_obj, - is_mtmct, - track_model_dir, - det_model_dir, - reid_model_dir); - InitPredictor(); - } - - // Set input, it must execute before Run() - void SetInput(const std::string& input_video); - void ClearInput(); - - // Run pipeline in video - void Run(); - void PredictMOT(const std::string& video_path); - void PredictMTMCT(const std::vector video_inputs); - - // Run pipeline in stream - void RunMOTStream(const cv::Mat img, - const int frame_id, - const int video_fps, - const Rect entrance, - cv::Mat out_img, - std::vector* records, - std::set* count_set, - std::set* interval_count_set, - std::vector* in_count_list, - std::vector* out_count_list, - std::map>* prev_center, - std::vector* flow_records); - void RunMTMCTStream(const std::vector imgs, - std::vector* records); - - void PrintBenchmarkLog(const std::vector det_time, const int img_num); - - private: - // Select model according to scenes, it must execute before Run() - void SelectModel(const std::string& scene = "pedestrian", - const bool tiny_obj = false, - const bool is_mtmct = false, - const std::string track_model_dir = "", - const std::string det_model_dir = "", - const std::string reid_model_dir = ""); - void InitPredictor(); - - std::shared_ptr jde_sct_; - std::shared_ptr sde_sct_; - - std::vector input_; - std::vector stream_; - std::string device_; - double threshold_; - std::string output_dir_; - std::string track_model_dir_; - std::string det_model_dir_; - std::string reid_model_dir_; - std::string run_mode_ = "paddle"; - int gpu_id_ = 0; - bool use_mkldnn_ = false; - int cpu_threads_ = 1; - bool trt_calib_mode_ = false; - bool do_entrance_counting_ = false; - bool save_result_ = false; - int secs_interval_ = 10; -}; - -} // namespace PaddleDetection - -#endif // DEPLOY_PPTRACKING_CPP_INCLUDE_PIPELINE_H_ diff --git a/fastdeploy/vision/tracking/pptracking/predictor.cc b/fastdeploy/vision/tracking/pptracking/predictor.cc deleted file mode 100644 index ea479f3ab04..00000000000 --- a/fastdeploy/vision/tracking/pptracking/predictor.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include -// for setprecision -#include -#include -#include "include/predictor.h" - -using namespace paddle_infer; // NOLINT - -namespace PaddleDetection { - -void Predictor::Predict(const std::vector imgs, - const double threshold, - MOTResult* result, - std::vector* times) { - if (use_jde_) { - jde_sct_->Predict(imgs, threshold, result, times); - } else { - sde_sct_->Predict(imgs, threshold, result, times); - } -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/predictor.h b/fastdeploy/vision/tracking/pptracking/predictor.h deleted file mode 100644 index cfb6306513a..00000000000 --- a/fastdeploy/vision/tracking/pptracking/predictor.h +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "paddle_inference_api.h" // NOLINT - -#include "include/config_parser.h" -#include "include/jde_predictor.h" -#include "include/preprocess_op.h" -#include "include/sde_predictor.h" - -using namespace paddle_infer; // NOLINT - -namespace PaddleDetection { - -class Predictor { - public: - explicit Predictor(const std::string& device = "CPU", - const std::string& track_model_dir = "", - const std::string& det_model_dir = "", - const std::string& reid_model_dir = "", - const double threshold = -1., - const std::string& run_mode = "paddle", - const int gpu_id = 0, - const bool use_mkldnn = false, - const int cpu_threads = 1, - bool trt_calib_mode = false, - const int min_box_area = 200) { - if (track_model_dir.empty() && det_model_dir.empty()) { - throw "Predictor must receive track_model or det_model!"; - } - - if (!track_model_dir.empty() && !det_model_dir.empty()) { - throw "Predictor only receive one of track_model or det_model!"; - } - - if (!track_model_dir.empty()) { - jde_sct_ = - std::make_shared(device, - track_model_dir, - threshold, - run_mode, - gpu_id, - use_mkldnn, - cpu_threads, - trt_calib_mode, - min_box_area); - use_jde_ = true; - } - if (!det_model_dir.empty()) { - sde_sct_ = std::make_shared(device, - det_model_dir, - reid_model_dir, - threshold, - run_mode, - gpu_id, - use_mkldnn, - cpu_threads, - trt_calib_mode, - min_box_area); - use_jde_ = false; - } - } - - // Run predictor - void Predict(const std::vector imgs, - const double threshold = 0.5, - MOTResult* result = nullptr, - std::vector* times = nullptr); - - private: - std::shared_ptr jde_sct_; - std::shared_ptr sde_sct_; - bool use_jde_ = true; -}; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/sde_predictor.cc b/fastdeploy/vision/tracking/pptracking/sde_predictor.cc deleted file mode 100644 index e469e8ddc5a..00000000000 --- a/fastdeploy/vision/tracking/pptracking/sde_predictor.cc +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -#include -// for setprecision -#include -#include -#include "include/sde_predictor.h" - -using namespace paddle_infer; // NOLINT - -namespace PaddleDetection { - -// Load Model and create model predictor -void SDEPredictor::LoadModel(const std::string& det_model_dir, - const std::string& reid_model_dir, - const std::string& run_mode) { - throw "Not Implement"; -} - -void SDEPredictor::Preprocess(const cv::Mat& ori_im) { throw "Not Implement"; } - -void SDEPredictor::Postprocess(const cv::Mat dets, - const cv::Mat emb, - MOTResult* result) { - throw "Not Implement"; -} - -void SDEPredictor::Predict(const std::vector imgs, - const double threshold, - MOTResult* result, - std::vector* times) { - throw "Not Implement"; -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/sde_predictor.h b/fastdeploy/vision/tracking/pptracking/sde_predictor.h deleted file mode 100644 index 3919eb105d1..00000000000 --- a/fastdeploy/vision/tracking/pptracking/sde_predictor.h +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "paddle_inference_api.h" // NOLINT - -#include "include/config_parser.h" -#include "include/preprocess_op.h" -#include "include/utils.h" - -using namespace paddle_infer; // NOLINT - -namespace PaddleDetection { - -class SDEPredictor { - public: - explicit SDEPredictor(const std::string& device, - const std::string& det_model_dir = "", - const std::string& reid_model_dir = "", - const double threshold = -1., - const std::string& run_mode = "paddle", - const int gpu_id = 0, - const bool use_mkldnn = false, - const int cpu_threads = 1, - bool trt_calib_mode = false, - const int min_box_area = 200) { - this->device_ = device; - this->gpu_id_ = gpu_id; - this->use_mkldnn_ = use_mkldnn; - this->cpu_math_library_num_threads_ = cpu_threads; - this->trt_calib_mode_ = trt_calib_mode; - this->min_box_area_ = min_box_area; - - det_config_.load_config(det_model_dir); - this->min_subgraph_size_ = det_config_.min_subgraph_size_; - det_preprocessor_.Init(det_config_.preprocess_info_); - - reid_config_.load_config(reid_model_dir); - reid_preprocessor_.Init(reid_config_.preprocess_info_); - - LoadModel(det_model_dir, reid_model_dir, run_mode); - this->conf_thresh_ = det_config_.conf_thresh_; - } - - // Load Paddle inference model - void LoadModel(const std::string& det_model_dir, - const std::string& reid_model_dir, - const std::string& run_mode = "paddle"); - - // Run predictor - void Predict(const std::vector imgs, - const double threshold = 0.5, - MOTResult* result = nullptr, - std::vector* times = nullptr); - - private: - std::string device_ = "CPU"; - float threhold = 0.5; - int gpu_id_ = 0; - bool use_mkldnn_ = false; - int cpu_math_library_num_threads_ = 1; - int min_subgraph_size_ = 3; - bool trt_calib_mode_ = false; - - // Preprocess image and copy data to input buffer - void Preprocess(const cv::Mat& image_mat); - // Postprocess result - void Postprocess(const cv::Mat dets, const cv::Mat emb, MOTResult* result); - - std::shared_ptr det_predictor_; - std::shared_ptr reid_predictor_; - Preprocessor det_preprocessor_; - Preprocessor reid_preprocessor_; - ImageBlob inputs_; - std::vector bbox_data_; - std::vector emb_data_; - double threshold_; - ConfigPaser det_config_; - ConfigPaser reid_config_; - float min_box_area_ = 200; - float conf_thresh_; -}; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc deleted file mode 100644 index 9540e39f670..00000000000 --- a/fastdeploy/vision/tracking/pptracking/tracker.cc +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code is based on: -// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.cpp -// Ths copyright of CnybTseng/JDE is as follows: -// MIT License - -#include -#include -#include -#include - -#include "include/lapjv.h" -#include "include/tracker.h" - -#define mat2vec4f(m) \ - cv::Vec4f(*m.ptr(0, 0), \ - *m.ptr(0, 1), \ - *m.ptr(0, 2), \ - *m.ptr(0, 3)) - -namespace PaddleDetection { - -static std::map chi2inv95 = {{1, 3.841459f}, - {2, 5.991465f}, - {3, 7.814728f}, - {4, 9.487729f}, - {5, 11.070498f}, - {6, 12.591587f}, - {7, 14.067140f}, - {8, 15.507313f}, - {9, 16.918978f}}; - -JDETracker *JDETracker::me = new JDETracker; - -JDETracker *JDETracker::instance(void) { return me; } - -JDETracker::JDETracker(void) - : timestamp(0), max_lost_time(30), lambda(0.98f), det_thresh(0.3f) {} - -bool JDETracker::update(const cv::Mat &dets, - const cv::Mat &emb, - std::vector *tracks) { - ++timestamp; - TrajectoryPool candidates(dets.rows); - for (int i = 0; i < dets.rows; ++i) { - float score = *dets.ptr(i, 1); - const cv::Mat <rb_ = dets(cv::Rect(2, i, 4, 1)); - cv::Vec4f ltrb = mat2vec4f(ltrb_); - const cv::Mat &embedding = emb(cv::Rect(0, i, emb.cols, 1)); - candidates[i] = Trajectory(ltrb, score, embedding); - } - - TrajectoryPtrPool tracked_trajectories; - TrajectoryPtrPool unconfirmed_trajectories; - for (size_t i = 0; i < this->tracked_trajectories.size(); ++i) { - if (this->tracked_trajectories[i].is_activated) - tracked_trajectories.push_back(&this->tracked_trajectories[i]); - else - unconfirmed_trajectories.push_back(&this->tracked_trajectories[i]); - } - - TrajectoryPtrPool trajectory_pool = - tracked_trajectories + &(this->lost_trajectories); - - for (size_t i = 0; i < trajectory_pool.size(); ++i) - trajectory_pool[i]->predict(); - - Match matches; - std::vector mismatch_row; - std::vector mismatch_col; - - cv::Mat cost = motion_distance(trajectory_pool, candidates); - linear_assignment(cost, 0.7f, &matches, &mismatch_row, &mismatch_col); - - MatchIterator miter; - TrajectoryPtrPool activated_trajectories; - TrajectoryPtrPool retrieved_trajectories; - - for (miter = matches.begin(); miter != matches.end(); miter++) { - Trajectory *pt = trajectory_pool[miter->first]; - Trajectory &ct = candidates[miter->second]; - if (pt->state == Tracked) { - pt->update(&ct, timestamp); - activated_trajectories.push_back(pt); - } else { - pt->reactivate(&ct, timestamp); - retrieved_trajectories.push_back(pt); - } - } - - TrajectoryPtrPool next_candidates(mismatch_col.size()); - for (size_t i = 0; i < mismatch_col.size(); ++i) - next_candidates[i] = &candidates[mismatch_col[i]]; - - TrajectoryPtrPool next_trajectory_pool; - for (size_t i = 0; i < mismatch_row.size(); ++i) { - int j = mismatch_row[i]; - if (trajectory_pool[j]->state == Tracked) - next_trajectory_pool.push_back(trajectory_pool[j]); - } - - cost = iou_distance(next_trajectory_pool, next_candidates); - linear_assignment(cost, 0.5f, &matches, &mismatch_row, &mismatch_col); - - for (miter = matches.begin(); miter != matches.end(); miter++) { - Trajectory *pt = next_trajectory_pool[miter->first]; - Trajectory *ct = next_candidates[miter->second]; - if (pt->state == Tracked) { - pt->update(ct, timestamp); - activated_trajectories.push_back(pt); - } else { - pt->reactivate(ct, timestamp); - retrieved_trajectories.push_back(pt); - } - } - - TrajectoryPtrPool lost_trajectories; - for (size_t i = 0; i < mismatch_row.size(); ++i) { - Trajectory *pt = next_trajectory_pool[mismatch_row[i]]; - if (pt->state != Lost) { - pt->mark_lost(); - lost_trajectories.push_back(pt); - } - } - - TrajectoryPtrPool nnext_candidates(mismatch_col.size()); - for (size_t i = 0; i < mismatch_col.size(); ++i) - nnext_candidates[i] = next_candidates[mismatch_col[i]]; - cost = iou_distance(unconfirmed_trajectories, nnext_candidates); - linear_assignment(cost, 0.7f, &matches, &mismatch_row, &mismatch_col); - - for (miter = matches.begin(); miter != matches.end(); miter++) { - unconfirmed_trajectories[miter->first]->update( - nnext_candidates[miter->second], timestamp); - activated_trajectories.push_back(unconfirmed_trajectories[miter->first]); - } - - TrajectoryPtrPool removed_trajectories; - - for (size_t i = 0; i < mismatch_row.size(); ++i) { - unconfirmed_trajectories[mismatch_row[i]]->mark_removed(); - removed_trajectories.push_back(unconfirmed_trajectories[mismatch_row[i]]); - } - - for (size_t i = 0; i < mismatch_col.size(); ++i) { - if (nnext_candidates[mismatch_col[i]]->score < det_thresh) continue; - nnext_candidates[mismatch_col[i]]->activate(timestamp); - activated_trajectories.push_back(nnext_candidates[mismatch_col[i]]); - } - - for (size_t i = 0; i < this->lost_trajectories.size(); ++i) { - Trajectory < = this->lost_trajectories[i]; - if (timestamp - lt.timestamp > max_lost_time) { - lt.mark_removed(); - removed_trajectories.push_back(<); - } - } - - TrajectoryPoolIterator piter; - for (piter = this->tracked_trajectories.begin(); - piter != this->tracked_trajectories.end();) { - if (piter->state != Tracked) - piter = this->tracked_trajectories.erase(piter); - else - ++piter; - } - - this->tracked_trajectories += activated_trajectories; - this->tracked_trajectories += retrieved_trajectories; - - this->lost_trajectories -= this->tracked_trajectories; - this->lost_trajectories += lost_trajectories; - this->lost_trajectories -= this->removed_trajectories; - this->removed_trajectories += removed_trajectories; - remove_duplicate_trajectory(&this->tracked_trajectories, - &this->lost_trajectories); - - tracks->clear(); - for (size_t i = 0; i < this->tracked_trajectories.size(); ++i) { - if (this->tracked_trajectories[i].is_activated) { - Track track = {this->tracked_trajectories[i].id, - this->tracked_trajectories[i].score, - this->tracked_trajectories[i].ltrb}; - tracks->push_back(track); - } - } - return 0; -} - -cv::Mat JDETracker::motion_distance(const TrajectoryPtrPool &a, - const TrajectoryPool &b) { - if (0 == a.size() || 0 == b.size()) - return cv::Mat(a.size(), b.size(), CV_32F); - - cv::Mat edists = embedding_distance(a, b); - cv::Mat mdists = mahalanobis_distance(a, b); - cv::Mat fdists = lambda * edists + (1 - lambda) * mdists; - - const float gate_thresh = chi2inv95[4]; - for (int i = 0; i < fdists.rows; ++i) { - for (int j = 0; j < fdists.cols; ++j) { - if (*mdists.ptr(i, j) > gate_thresh) - *fdists.ptr(i, j) = FLT_MAX; - } - } - - return fdists; -} - -void JDETracker::linear_assignment(const cv::Mat &cost, - float cost_limit, - Match *matches, - std::vector *mismatch_row, - std::vector *mismatch_col) { - matches->clear(); - mismatch_row->clear(); - mismatch_col->clear(); - if (cost.empty()) { - for (int i = 0; i < cost.rows; ++i) mismatch_row->push_back(i); - for (int i = 0; i < cost.cols; ++i) mismatch_col->push_back(i); - return; - } - - float opt = 0; - cv::Mat x(cost.rows, 1, CV_32S); - cv::Mat y(cost.cols, 1, CV_32S); - - lapjv_internal(cost, - true, - cost_limit, - reinterpret_cast(x.data), - reinterpret_cast(y.data)); - - for (int i = 0; i < x.rows; ++i) { - int j = *x.ptr(i); - if (j >= 0) - matches->insert({i, j}); - else - mismatch_row->push_back(i); - } - - for (int i = 0; i < y.rows; ++i) { - int j = *y.ptr(i); - if (j < 0) mismatch_col->push_back(i); - } - - return; -} - -void JDETracker::remove_duplicate_trajectory(TrajectoryPool *a, - TrajectoryPool *b, - float iou_thresh) { - if (a->size() == 0 || b->size() == 0) return; - - cv::Mat dist = iou_distance(*a, *b); - cv::Mat mask = dist < iou_thresh; - std::vector idx; - cv::findNonZero(mask, idx); - - std::vector da; - std::vector db; - for (size_t i = 0; i < idx.size(); ++i) { - int ta = (*a)[idx[i].y].timestamp - (*a)[idx[i].y].starttime; - int tb = (*b)[idx[i].x].timestamp - (*b)[idx[i].x].starttime; - if (ta > tb) - db.push_back(idx[i].x); - else - da.push_back(idx[i].y); - } - - int id = 0; - TrajectoryPoolIterator piter; - for (piter = a->begin(); piter != a->end();) { - std::vector::iterator iter = find(da.begin(), da.end(), id++); - if (iter != da.end()) - piter = a->erase(piter); - else - ++piter; - } - - id = 0; - for (piter = b->begin(); piter != b->end();) { - std::vector::iterator iter = find(db.begin(), db.end(), id++); - if (iter != db.end()) - piter = b->erase(piter); - else - ++piter; - } -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h deleted file mode 100644 index 244530f140a..00000000000 --- a/fastdeploy/vision/tracking/pptracking/tracker.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code is based on: -// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.h -// Ths copyright of CnybTseng/JDE is as follows: -// MIT License - -#pragma once - -#include -#include - -#include -#include -#include -#include "include/trajectory.h" - -namespace PaddleDetection { - -typedef std::map Match; -typedef std::map::iterator MatchIterator; - -struct Track { - int id; - float score; - cv::Vec4f ltrb; -}; - -class JDETracker { - public: - static JDETracker *instance(void); - virtual bool update(const cv::Mat &dets, - const cv::Mat &emb, - std::vector *tracks); - - private: - JDETracker(void); - virtual ~JDETracker(void) {} - cv::Mat motion_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b); - void linear_assignment(const cv::Mat &cost, - float cost_limit, - Match *matches, - std::vector *mismatch_row, - std::vector *mismatch_col); - void remove_duplicate_trajectory(TrajectoryPool *a, - TrajectoryPool *b, - float iou_thresh = 0.15f); - - private: - static JDETracker *me; - int timestamp; - TrajectoryPool tracked_trajectories; - TrajectoryPool lost_trajectories; - TrajectoryPool removed_trajectories; - int max_lost_time; - float lambda; - float det_thresh; -}; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.cc b/fastdeploy/vision/tracking/pptracking/trajectory.cc deleted file mode 100644 index 0ff2e1a5fc7..00000000000 --- a/fastdeploy/vision/tracking/pptracking/trajectory.cc +++ /dev/null @@ -1,517 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code is based on: -// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.cpp -// Ths copyright of CnybTseng/JDE is as follows: -// MIT License - -#include "include/trajectory.h" -#include - -namespace PaddleDetection { - -void TKalmanFilter::init(const cv::Mat &measurement) { - measurement.copyTo(statePost(cv::Rect(0, 0, 1, 4))); - statePost(cv::Rect(0, 4, 1, 4)).setTo(0); - statePost.copyTo(statePre); - - float varpos = 2 * std_weight_position * (*measurement.ptr(3)); - varpos *= varpos; - float varvel = 10 * std_weight_velocity * (*measurement.ptr(3)); - varvel *= varvel; - - errorCovPost.setTo(0); - *errorCovPost.ptr(0, 0) = varpos; - *errorCovPost.ptr(1, 1) = varpos; - *errorCovPost.ptr(2, 2) = 1e-4f; - *errorCovPost.ptr(3, 3) = varpos; - *errorCovPost.ptr(4, 4) = varvel; - *errorCovPost.ptr(5, 5) = varvel; - *errorCovPost.ptr(6, 6) = 1e-10f; - *errorCovPost.ptr(7, 7) = varvel; - errorCovPost.copyTo(errorCovPre); -} - -const cv::Mat &TKalmanFilter::predict() { - float varpos = std_weight_position * (*statePre.ptr(3)); - varpos *= varpos; - float varvel = std_weight_velocity * (*statePre.ptr(3)); - varvel *= varvel; - - processNoiseCov.setTo(0); - *processNoiseCov.ptr(0, 0) = varpos; - *processNoiseCov.ptr(1, 1) = varpos; - *processNoiseCov.ptr(2, 2) = 1e-4f; - *processNoiseCov.ptr(3, 3) = varpos; - *processNoiseCov.ptr(4, 4) = varvel; - *processNoiseCov.ptr(5, 5) = varvel; - *processNoiseCov.ptr(6, 6) = 1e-10f; - *processNoiseCov.ptr(7, 7) = varvel; - - return cv::KalmanFilter::predict(); -} - -const cv::Mat &TKalmanFilter::correct(const cv::Mat &measurement) { - float varpos = std_weight_position * (*measurement.ptr(3)); - varpos *= varpos; - - measurementNoiseCov.setTo(0); - *measurementNoiseCov.ptr(0, 0) = varpos; - *measurementNoiseCov.ptr(1, 1) = varpos; - *measurementNoiseCov.ptr(2, 2) = 1e-2f; - *measurementNoiseCov.ptr(3, 3) = varpos; - - return cv::KalmanFilter::correct(measurement); -} - -void TKalmanFilter::project(cv::Mat *mean, cv::Mat *covariance) const { - float varpos = std_weight_position * (*statePost.ptr(3)); - varpos *= varpos; - - cv::Mat measurementNoiseCov_ = cv::Mat::eye(4, 4, CV_32F); - *measurementNoiseCov_.ptr(0, 0) = varpos; - *measurementNoiseCov_.ptr(1, 1) = varpos; - *measurementNoiseCov_.ptr(2, 2) = 1e-2f; - *measurementNoiseCov_.ptr(3, 3) = varpos; - - *mean = measurementMatrix * statePost; - cv::Mat temp = measurementMatrix * errorCovPost; - gemm(temp, - measurementMatrix, - 1, - measurementNoiseCov_, - 1, - *covariance, - cv::GEMM_2_T); -} - -int Trajectory::count = 0; - -const cv::Mat &Trajectory::predict(void) { - if (state != Tracked) *cv::KalmanFilter::statePost.ptr(7) = 0; - return TKalmanFilter::predict(); -} - -void Trajectory::update(Trajectory *traj, - int timestamp_, - bool update_embedding_) { - timestamp = timestamp_; - ++length; - ltrb = traj->ltrb; - xyah = traj->xyah; - TKalmanFilter::correct(cv::Mat(traj->xyah)); - state = Tracked; - is_activated = true; - score = traj->score; - if (update_embedding_) update_embedding(traj->current_embedding); -} - -void Trajectory::activate(int timestamp_) { - id = next_id(); - TKalmanFilter::init(cv::Mat(xyah)); - length = 0; - state = Tracked; - if (timestamp_ == 1) { - is_activated = true; - } - timestamp = timestamp_; - starttime = timestamp_; -} - -void Trajectory::reactivate(Trajectory *traj, int timestamp_, bool newid) { - TKalmanFilter::correct(cv::Mat(traj->xyah)); - update_embedding(traj->current_embedding); - length = 0; - state = Tracked; - is_activated = true; - timestamp = timestamp_; - if (newid) id = next_id(); -} - -void Trajectory::update_embedding(const cv::Mat &embedding) { - current_embedding = embedding / cv::norm(embedding); - if (smooth_embedding.empty()) { - smooth_embedding = current_embedding; - } else { - smooth_embedding = eta * smooth_embedding + (1 - eta) * current_embedding; - } - smooth_embedding = smooth_embedding / cv::norm(smooth_embedding); -} - -TrajectoryPool operator+(const TrajectoryPool &a, const TrajectoryPool &b) { - TrajectoryPool sum; - sum.insert(sum.end(), a.begin(), a.end()); - - std::vector ids(a.size()); - for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; - - for (size_t i = 0; i < b.size(); ++i) { - std::vector::iterator iter = find(ids.begin(), ids.end(), b[i].id); - if (iter == ids.end()) { - sum.push_back(b[i]); - ids.push_back(b[i].id); - } - } - - return sum; -} - -TrajectoryPool operator+(const TrajectoryPool &a, const TrajectoryPtrPool &b) { - TrajectoryPool sum; - sum.insert(sum.end(), a.begin(), a.end()); - - std::vector ids(a.size()); - for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; - - for (size_t i = 0; i < b.size(); ++i) { - std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); - if (iter == ids.end()) { - sum.push_back(*b[i]); - ids.push_back(b[i]->id); - } - } - - return sum; -} - -TrajectoryPool &operator+=(TrajectoryPool &a, // NOLINT - const TrajectoryPtrPool &b) { - std::vector ids(a.size()); - for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; - - for (size_t i = 0; i < b.size(); ++i) { - if (b[i]->smooth_embedding.empty()) continue; - std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); - if (iter == ids.end()) { - a.push_back(*b[i]); - ids.push_back(b[i]->id); - } - } - - return a; -} - -TrajectoryPool operator-(const TrajectoryPool &a, const TrajectoryPool &b) { - TrajectoryPool dif; - std::vector ids(b.size()); - for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i].id; - - for (size_t i = 0; i < a.size(); ++i) { - std::vector::iterator iter = find(ids.begin(), ids.end(), a[i].id); - if (iter == ids.end()) dif.push_back(a[i]); - } - - return dif; -} - -TrajectoryPool &operator-=(TrajectoryPool &a, // NOLINT - const TrajectoryPool &b) { - std::vector ids(b.size()); - for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i].id; - - TrajectoryPoolIterator piter; - for (piter = a.begin(); piter != a.end();) { - std::vector::iterator iter = find(ids.begin(), ids.end(), piter->id); - if (iter == ids.end()) - ++piter; - else - piter = a.erase(piter); - } - - return a; -} - -TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b) { - TrajectoryPtrPool sum; - sum.insert(sum.end(), a.begin(), a.end()); - - std::vector ids(a.size()); - for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i]->id; - - for (size_t i = 0; i < b.size(); ++i) { - std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); - if (iter == ids.end()) { - sum.push_back(b[i]); - ids.push_back(b[i]->id); - } - } - - return sum; -} - -TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, TrajectoryPool *b) { - TrajectoryPtrPool sum; - sum.insert(sum.end(), a.begin(), a.end()); - - std::vector ids(a.size()); - for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i]->id; - - for (size_t i = 0; i < b->size(); ++i) { - std::vector::iterator iter = find(ids.begin(), ids.end(), (*b)[i].id); - if (iter == ids.end()) { - sum.push_back(&(*b)[i]); - ids.push_back((*b)[i].id); - } - } - - return sum; -} - -TrajectoryPtrPool operator-(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b) { - TrajectoryPtrPool dif; - std::vector ids(b.size()); - for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i]->id; - - for (size_t i = 0; i < a.size(); ++i) { - std::vector::iterator iter = find(ids.begin(), ids.end(), a[i]->id); - if (iter == ids.end()) dif.push_back(a[i]); - } - - return dif; -} - -cv::Mat embedding_distance(const TrajectoryPool &a, const TrajectoryPool &b) { - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - cv::Mat u = a[i].smooth_embedding; - cv::Mat v = b[j].smooth_embedding; - double uv = u.dot(v); - double uu = u.dot(u); - double vv = v.dot(v); - double dist = std::abs(1. - uv / std::sqrt(uu * vv)); - // double dist = cv::norm(a[i].smooth_embedding, b[j].smooth_embedding, - // cv::NORM_L2); - distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); - } - } - return dists; -} - -cv::Mat embedding_distance(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b) { - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - // double dist = cv::norm(a[i]->smooth_embedding, b[j]->smooth_embedding, - // cv::NORM_L2); - // distsi[j] = static_cast(dist); - cv::Mat u = a[i]->smooth_embedding; - cv::Mat v = b[j]->smooth_embedding; - double uv = u.dot(v); - double uu = u.dot(u); - double vv = v.dot(v); - double dist = std::abs(1. - uv / std::sqrt(uu * vv)); - distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); - } - } - - return dists; -} - -cv::Mat embedding_distance(const TrajectoryPtrPool &a, - const TrajectoryPool &b) { - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - // double dist = cv::norm(a[i]->smooth_embedding, b[j].smooth_embedding, - // cv::NORM_L2); - // distsi[j] = static_cast(dist); - cv::Mat u = a[i]->smooth_embedding; - cv::Mat v = b[j].smooth_embedding; - double uv = u.dot(v); - double uu = u.dot(u); - double vv = v.dot(v); - double dist = std::abs(1. - uv / std::sqrt(uu * vv)); - distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); - } - } - - return dists; -} - -cv::Mat mahalanobis_distance(const TrajectoryPool &a, const TrajectoryPool &b) { - std::vector means(a.size()); - std::vector icovariances(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - cv::Mat covariance; - a[i].project(&means[i], &covariance); - cv::invert(covariance, icovariances[i]); - } - - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - const cv::Mat x(b[j].xyah); - float dist = - static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); - distsi[j] = dist * dist; - } - } - - return dists; -} - -cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b) { - std::vector means(a.size()); - std::vector icovariances(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - cv::Mat covariance; - a[i]->project(&means[i], &covariance); - cv::invert(covariance, icovariances[i]); - } - - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - const cv::Mat x(b[j]->xyah); - float dist = - static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); - distsi[j] = dist * dist; - } - } - - return dists; -} - -cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, - const TrajectoryPool &b) { - std::vector means(a.size()); - std::vector icovariances(a.size()); - - for (size_t i = 0; i < a.size(); ++i) { - cv::Mat covariance; - a[i]->project(&means[i], &covariance); - cv::invert(covariance, icovariances[i]); - } - - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - const cv::Mat x(b[j].xyah); - float dist = - static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); - distsi[j] = dist * dist; - } - } - - return dists; -} - -static inline float calc_inter_area(const cv::Vec4f &a, const cv::Vec4f &b) { - if (a[2] < b[0] || a[0] > b[2] || a[3] < b[1] || a[1] > b[3]) return 0.f; - - float w = std::min(a[2], b[2]) - std::max(a[0], b[0]); - float h = std::min(a[3], b[3]) - std::max(a[1], b[1]); - return w * h; -} - -cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b) { - std::vector areaa(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - float w = a[i].ltrb[2] - a[i].ltrb[0]; - float h = a[i].ltrb[3] - a[i].ltrb[1]; - areaa[i] = w * h; - } - - std::vector areab(b.size()); - for (size_t j = 0; j < b.size(); ++j) { - float w = b[j].ltrb[2] - b[j].ltrb[0]; - float h = b[j].ltrb[3] - b[j].ltrb[1]; - areab[j] = w * h; - } - - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - const cv::Vec4f &boxa = a[i].ltrb; - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - const cv::Vec4f &boxb = b[j].ltrb; - float inters = calc_inter_area(boxa, boxb); - distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); - } - } - - return dists; -} - -cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPtrPool &b) { - std::vector areaa(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - float w = a[i]->ltrb[2] - a[i]->ltrb[0]; - float h = a[i]->ltrb[3] - a[i]->ltrb[1]; - areaa[i] = w * h; - } - - std::vector areab(b.size()); - for (size_t j = 0; j < b.size(); ++j) { - float w = b[j]->ltrb[2] - b[j]->ltrb[0]; - float h = b[j]->ltrb[3] - b[j]->ltrb[1]; - areab[j] = w * h; - } - - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - const cv::Vec4f &boxa = a[i]->ltrb; - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - const cv::Vec4f &boxb = b[j]->ltrb; - float inters = calc_inter_area(boxa, boxb); - distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); - } - } - - return dists; -} - -cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b) { - std::vector areaa(a.size()); - for (size_t i = 0; i < a.size(); ++i) { - float w = a[i]->ltrb[2] - a[i]->ltrb[0]; - float h = a[i]->ltrb[3] - a[i]->ltrb[1]; - areaa[i] = w * h; - } - - std::vector areab(b.size()); - for (size_t j = 0; j < b.size(); ++j) { - float w = b[j].ltrb[2] - b[j].ltrb[0]; - float h = b[j].ltrb[3] - b[j].ltrb[1]; - areab[j] = w * h; - } - - cv::Mat dists(a.size(), b.size(), CV_32F); - for (size_t i = 0; i < a.size(); ++i) { - const cv::Vec4f &boxa = a[i]->ltrb; - float *distsi = dists.ptr(i); - for (size_t j = 0; j < b.size(); ++j) { - const cv::Vec4f &boxb = b[j].ltrb; - float inters = calc_inter_area(boxa, boxb); - distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); - } - } - - return dists; -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h deleted file mode 100644 index c21e8cac368..00000000000 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The code is based on: -// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.h -// Ths copyright of CnybTseng/JDE is as follows: -// MIT License - -#pragma once - -#include - -#include -#include -#include -#include "opencv2/video/tracking.hpp" - -namespace PaddleDetection { - -typedef enum { New = 0, Tracked = 1, Lost = 2, Removed = 3 } TrajectoryState; - -class Trajectory; -typedef std::vector TrajectoryPool; -typedef std::vector::iterator TrajectoryPoolIterator; -typedef std::vector TrajectoryPtrPool; -typedef std::vector::iterator TrajectoryPtrPoolIterator; - -class TKalmanFilter : public cv::KalmanFilter { - public: - TKalmanFilter(void); - virtual ~TKalmanFilter(void) {} - virtual void init(const cv::Mat &measurement); - virtual const cv::Mat &predict(); - virtual const cv::Mat &correct(const cv::Mat &measurement); - virtual void project(cv::Mat *mean, cv::Mat *covariance) const; - - private: - float std_weight_position; - float std_weight_velocity; -}; - -inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) { - cv::KalmanFilter::transitionMatrix = cv::Mat::eye(8, 8, CV_32F); - for (int i = 0; i < 4; ++i) - cv::KalmanFilter::transitionMatrix.at(i, i + 4) = 1; - cv::KalmanFilter::measurementMatrix = cv::Mat::eye(4, 8, CV_32F); - std_weight_position = 1 / 20.f; - std_weight_velocity = 1 / 160.f; -} - -class Trajectory : public TKalmanFilter { - public: - Trajectory(); - Trajectory(const cv::Vec4f <rb, float score, const cv::Mat &embedding); - Trajectory(const Trajectory &other); - Trajectory &operator=(const Trajectory &rhs); - virtual ~Trajectory(void) {} - - static int next_id(); - virtual const cv::Mat &predict(void); - virtual void update(Trajectory *traj, - int timestamp, - bool update_embedding = true); - virtual void activate(int timestamp); - virtual void reactivate(Trajectory *traj, int timestamp, bool newid = false); - virtual void mark_lost(void); - virtual void mark_removed(void); - - friend TrajectoryPool operator+(const TrajectoryPool &a, - const TrajectoryPool &b); - friend TrajectoryPool operator+(const TrajectoryPool &a, - const TrajectoryPtrPool &b); - friend TrajectoryPool &operator+=(TrajectoryPool &a, // NOLINT - const TrajectoryPtrPool &b); - friend TrajectoryPool operator-(const TrajectoryPool &a, - const TrajectoryPool &b); - friend TrajectoryPool &operator-=(TrajectoryPool &a, // NOLINT - const TrajectoryPool &b); - friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b); - friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, - TrajectoryPool *b); - friend TrajectoryPtrPool operator-(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b); - - friend cv::Mat embedding_distance(const TrajectoryPool &a, - const TrajectoryPool &b); - friend cv::Mat embedding_distance(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b); - friend cv::Mat embedding_distance(const TrajectoryPtrPool &a, - const TrajectoryPool &b); - - friend cv::Mat mahalanobis_distance(const TrajectoryPool &a, - const TrajectoryPool &b); - friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b); - friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, - const TrajectoryPool &b); - - friend cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b); - friend cv::Mat iou_distance(const TrajectoryPtrPool &a, - const TrajectoryPtrPool &b); - friend cv::Mat iou_distance(const TrajectoryPtrPool &a, - const TrajectoryPool &b); - - private: - void update_embedding(const cv::Mat &embedding); - - public: - TrajectoryState state; - cv::Vec4f ltrb; - cv::Mat smooth_embedding; - int id; - bool is_activated; - int timestamp; - int starttime; - float score; - - private: - static int count; - cv::Vec4f xyah; - cv::Mat current_embedding; - float eta; - int length; -}; - -inline cv::Vec4f ltrb2xyah(const cv::Vec4f <rb) { - cv::Vec4f xyah; - xyah[0] = (ltrb[0] + ltrb[2]) * 0.5f; - xyah[1] = (ltrb[1] + ltrb[3]) * 0.5f; - xyah[3] = ltrb[3] - ltrb[1]; - xyah[2] = (ltrb[2] - ltrb[0]) / xyah[3]; - return xyah; -} - -inline Trajectory::Trajectory() - : state(New), - ltrb(cv::Vec4f()), - smooth_embedding(cv::Mat()), - id(0), - is_activated(false), - timestamp(0), - starttime(0), - score(0), - eta(0.9), - length(0) {} - -inline Trajectory::Trajectory(const cv::Vec4f <rb_, - float score_, - const cv::Mat &embedding) - : state(New), - ltrb(ltrb_), - smooth_embedding(cv::Mat()), - id(0), - is_activated(false), - timestamp(0), - starttime(0), - score(score_), - eta(0.9), - length(0) { - xyah = ltrb2xyah(ltrb); - update_embedding(embedding); -} - -inline Trajectory::Trajectory(const Trajectory &other) - : state(other.state), - ltrb(other.ltrb), - id(other.id), - is_activated(other.is_activated), - timestamp(other.timestamp), - starttime(other.starttime), - xyah(other.xyah), - score(other.score), - eta(other.eta), - length(other.length) { - other.smooth_embedding.copyTo(smooth_embedding); - other.current_embedding.copyTo(current_embedding); - // copy state in KalmanFilter - - other.statePre.copyTo(cv::KalmanFilter::statePre); - other.statePost.copyTo(cv::KalmanFilter::statePost); - other.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre); - other.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost); -} - -inline Trajectory &Trajectory::operator=(const Trajectory &rhs) { - this->state = rhs.state; - this->ltrb = rhs.ltrb; - rhs.smooth_embedding.copyTo(this->smooth_embedding); - this->id = rhs.id; - this->is_activated = rhs.is_activated; - this->timestamp = rhs.timestamp; - this->starttime = rhs.starttime; - this->xyah = rhs.xyah; - this->score = rhs.score; - rhs.current_embedding.copyTo(this->current_embedding); - this->eta = rhs.eta; - this->length = rhs.length; - - // copy state in KalmanFilter - - rhs.statePre.copyTo(cv::KalmanFilter::statePre); - rhs.statePost.copyTo(cv::KalmanFilter::statePost); - rhs.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre); - rhs.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost); - - return *this; -} - -inline int Trajectory::next_id() { - ++count; - return count; -} - -inline void Trajectory::mark_lost(void) { state = Lost; } - -inline void Trajectory::mark_removed(void) { state = Removed; } - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/postprocess.cc b/fastdeploy/vision/tracking/pptracking/utils/postprocess.cc deleted file mode 100644 index 47ea5a7cc48..00000000000 --- a/fastdeploy/vision/tracking/pptracking/utils/postprocess.cc +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -// for setprecision -#include -#include -#include -#include "include/postprocess.h" - -namespace PaddleDetection { - -cv::Scalar GetColor(int idx) { - idx = idx * 3; - cv::Scalar color = - cv::Scalar((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255); - return color; -} - -cv::Mat VisualizeTrackResult(const cv::Mat& img, - const MOTResult& results, - const float fps, - const int frame_id) { - cv::Mat vis_img = img.clone(); - int im_h = img.rows; - int im_w = img.cols; - float text_scale = std::max(1, static_cast(im_w / 1600.)); - float text_thickness = 2.; - float line_thickness = std::max(1, static_cast(im_w / 500.)); - - std::ostringstream oss; - oss << std::setiosflags(std::ios::fixed) << std::setprecision(4); - oss << "frame: " << frame_id << " "; - oss << "fps: " << fps << " "; - oss << "num: " << results.size(); - std::string text = oss.str(); - - cv::Point origin; - origin.x = 0; - origin.y = static_cast(15 * text_scale); - cv::putText(vis_img, - text, - origin, - cv::FONT_HERSHEY_PLAIN, - text_scale, - (0, 0, 255), - 2); - - for (int i = 0; i < results.size(); ++i) { - const int obj_id = results[i].ids; - const float score = results[i].score; - - cv::Scalar color = GetColor(obj_id); - - cv::Point pt1 = cv::Point(results[i].rects.left, results[i].rects.top); - cv::Point pt2 = cv::Point(results[i].rects.right, results[i].rects.bottom); - cv::Point id_pt = - cv::Point(results[i].rects.left, results[i].rects.top + 10); - cv::Point score_pt = - cv::Point(results[i].rects.left, results[i].rects.top - 10); - cv::rectangle(vis_img, pt1, pt2, color, line_thickness); - - std::ostringstream idoss; - idoss << std::setiosflags(std::ios::fixed) << std::setprecision(4); - idoss << obj_id; - std::string id_text = idoss.str(); - - cv::putText(vis_img, - id_text, - id_pt, - cv::FONT_HERSHEY_PLAIN, - text_scale, - cv::Scalar(0, 255, 255), - text_thickness); - - std::ostringstream soss; - soss << std::setiosflags(std::ios::fixed) << std::setprecision(2); - soss << score; - std::string score_text = soss.str(); - - cv::putText(vis_img, - score_text, - score_pt, - cv::FONT_HERSHEY_PLAIN, - text_scale, - cv::Scalar(0, 255, 255), - text_thickness); - } - return vis_img; -} - -void FlowStatistic(const MOTResult& results, - const int frame_id, - const int secs_interval, - const bool do_entrance_counting, - const int video_fps, - const Rect entrance, - std::set* id_set, - std::set* interval_id_set, - std::vector* in_id_list, - std::vector* out_id_list, - std::map>* prev_center, - std::vector* records) { - if (frame_id == 0) interval_id_set->clear(); - - if (do_entrance_counting) { - // Count in and out number: - // Use horizontal center line as the entrance just for simplification. - // If a person located in the above the horizontal center line - // at the previous frame and is in the below the line at the current frame, - // the in number is increased by one. - // If a person was in the below the horizontal center line - // at the previous frame and locates in the below the line at the current - // frame, - // the out number is increased by one. - // TODO(qianhui): if the entrance is not the horizontal center line, - // the counting method should be optimized. - - float entrance_y = entrance.top; - for (const auto& result : results) { - float center_x = (result.rects.left + result.rects.right) / 2; - float center_y = (result.rects.top + result.rects.bottom) / 2; - int ids = result.ids; - std::map>::iterator iter; - iter = prev_center->find(ids); - if (iter != prev_center->end()) { - if (iter->second[1] <= entrance_y && center_y > entrance_y) { - in_id_list->push_back(ids); - } - if (iter->second[1] >= entrance_y && center_y < entrance_y) { - out_id_list->push_back(ids); - } - (*prev_center)[ids][0] = center_x; - (*prev_center)[ids][1] = center_y; - } else { - prev_center->insert( - std::pair>(ids, {center_x, center_y})); - } - } - } - - // Count totol number, number at a manual-setting interval - for (const auto& result : results) { - id_set->insert(result.ids); - interval_id_set->insert(result.ids); - } - - std::ostringstream os; - os << "Frame id: " << frame_id << ", Total count: " << id_set->size(); - if (do_entrance_counting) { - os << ", In count: " << in_id_list->size() - << ", Out count: " << out_id_list->size(); - } - - // Reset counting at the interval beginning - int curr_interval_count = -1; - if (frame_id % video_fps == 0 && frame_id / video_fps % secs_interval == 0) { - curr_interval_count = interval_id_set->size(); - os << ", Count during " << secs_interval - << " secs: " << curr_interval_count; - interval_id_set->clear(); - } - os << "\n"; - std::string record = os.str(); - records->push_back(record); - LOG(INFO) << record; -} - -void SaveMOTResult(const MOTResult& results, - const int frame_id, - std::vector* records) { - // result format: frame_id, track_id, x1, y1, w, h - std::string record; - for (int i = 0; i < results.size(); ++i) { - MOTTrack mot_track = results[i]; - int ids = mot_track.ids; - float score = mot_track.score; - Rect rects = mot_track.rects; - float x1 = rects.left; - float y1 = rects.top; - float x2 = rects.right; - float y2 = rects.bottom; - float w = x2 - x1; - float h = y2 - y1; - if (w == 0 || h == 0) { - continue; - } - std::ostringstream os; - os << frame_id << " " << ids << " " << x1 << " " << y1 << " " << w << " " - << h << "\n"; - record = os.str(); - records->push_back(record); - } -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/postprocess.h b/fastdeploy/vision/tracking/pptracking/utils/postprocess.h deleted file mode 100644 index 41b10960351..00000000000 --- a/fastdeploy/vision/tracking/pptracking/utils/postprocess.h +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "include/utils.h" - -namespace PaddleDetection { - -// Generate visualization color -cv::Scalar GetColor(int idx); - -// Visualize Tracking Results -cv::Mat VisualizeTrackResult(const cv::Mat& img, - const MOTResult& results, - const float fps, - const int frame_id); - -// Pedestrian/Vehicle Counting -void FlowStatistic(const MOTResult& results, - const int frame_id, - const int secs_interval, - const bool do_entrance_counting, - const int video_fps, - const Rect entrance, - std::set* id_set, - std::set* interval_id_set, - std::vector* in_id_list, - std::vector* out_id_list, - std::map>* prev_center, - std::vector* records); - -// Save Tracking Results -void SaveMOTResult(const MOTResult& results, - const int frame_id, - std::vector* records); - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc b/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc deleted file mode 100644 index 3158ad67b60..00000000000 --- a/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.cc +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#include "include/preprocess_op.h" - -namespace PaddleDetection { - -void InitInfo::Run(cv::Mat* im, ImageBlob* data) { - data->im_shape_ = {static_cast(im->rows), - static_cast(im->cols)}; - data->scale_factor_ = {1., 1.}; - data->in_net_shape_ = {static_cast(im->rows), - static_cast(im->cols)}; -} - -void NormalizeImage::Run(cv::Mat* im, ImageBlob* data) { - double e = 1.0; - if (is_scale_) { - e /= 255.0; - } - (*im).convertTo(*im, CV_32FC3, e); - for (int h = 0; h < im->rows; h++) { - for (int w = 0; w < im->cols; w++) { - im->at(h, w)[0] = - (im->at(h, w)[0] - mean_[0]) / scale_[0]; - im->at(h, w)[1] = - (im->at(h, w)[1] - mean_[1]) / scale_[1]; - im->at(h, w)[2] = - (im->at(h, w)[2] - mean_[2]) / scale_[2]; - } - } -} - -void Permute::Run(cv::Mat* im, ImageBlob* data) { - (*im).convertTo(*im, CV_32FC3); - int rh = im->rows; - int rw = im->cols; - int rc = im->channels(); - (data->im_data_).resize(rc * rh * rw); - float* base = (data->im_data_).data(); - for (int i = 0; i < rc; ++i) { - cv::extractChannel(*im, cv::Mat(rh, rw, CV_32FC1, base + i * rh * rw), i); - } -} - -void Resize::Run(cv::Mat* im, ImageBlob* data) { - auto resize_scale = GenerateScale(*im); - data->im_shape_ = {static_cast(im->cols * resize_scale.first), - static_cast(im->rows * resize_scale.second)}; - data->in_net_shape_ = {static_cast(im->cols * resize_scale.first), - static_cast(im->rows * resize_scale.second)}; - cv::resize( - *im, *im, cv::Size(), resize_scale.first, resize_scale.second, interp_); - data->im_shape_ = { - static_cast(im->rows), static_cast(im->cols), - }; - data->scale_factor_ = { - resize_scale.second, resize_scale.first, - }; -} - -std::pair Resize::GenerateScale(const cv::Mat& im) { - std::pair resize_scale; - int origin_w = im.cols; - int origin_h = im.rows; - - if (keep_ratio_) { - int im_size_max = std::max(origin_w, origin_h); - int im_size_min = std::min(origin_w, origin_h); - int target_size_max = - *std::max_element(target_size_.begin(), target_size_.end()); - int target_size_min = - *std::min_element(target_size_.begin(), target_size_.end()); - float scale_min = - static_cast(target_size_min) / static_cast(im_size_min); - float scale_max = - static_cast(target_size_max) / static_cast(im_size_max); - float scale_ratio = std::min(scale_min, scale_max); - resize_scale = {scale_ratio, scale_ratio}; - } else { - resize_scale.first = - static_cast(target_size_[1]) / static_cast(origin_w); - resize_scale.second = - static_cast(target_size_[0]) / static_cast(origin_h); - } - return resize_scale; -} - -void LetterBoxResize::Run(cv::Mat* im, ImageBlob* data) { - float resize_scale = GenerateScale(*im); - int new_shape_w = std::round(im->cols * resize_scale); - int new_shape_h = std::round(im->rows * resize_scale); - data->im_shape_ = {static_cast(new_shape_h), - static_cast(new_shape_w)}; - float padw = (target_size_[1] - new_shape_w) / 2.; - float padh = (target_size_[0] - new_shape_h) / 2.; - - int top = std::round(padh - 0.1); - int bottom = std::round(padh + 0.1); - int left = std::round(padw - 0.1); - int right = std::round(padw + 0.1); - - cv::resize( - *im, *im, cv::Size(new_shape_w, new_shape_h), 0, 0, cv::INTER_AREA); - - data->in_net_shape_ = { - static_cast(im->rows), static_cast(im->cols), - }; - cv::copyMakeBorder(*im, - *im, - top, - bottom, - left, - right, - cv::BORDER_CONSTANT, - cv::Scalar(127.5)); - - data->in_net_shape_ = { - static_cast(im->rows), static_cast(im->cols), - }; - - data->scale_factor_ = { - resize_scale, resize_scale, - }; -} - -float LetterBoxResize::GenerateScale(const cv::Mat& im) { - int origin_w = im.cols; - int origin_h = im.rows; - - int target_h = target_size_[0]; - int target_w = target_size_[1]; - - float ratio_h = static_cast(target_h) / static_cast(origin_h); - float ratio_w = static_cast(target_w) / static_cast(origin_w); - float resize_scale = std::min(ratio_h, ratio_w); - return resize_scale; -} - -void PadStride::Run(cv::Mat* im, ImageBlob* data) { - if (stride_ <= 0) { - return; - } - int rc = im->channels(); - int rh = im->rows; - int rw = im->cols; - int nh = (rh / stride_) * stride_ + (rh % stride_ != 0) * stride_; - int nw = (rw / stride_) * stride_ + (rw % stride_ != 0) * stride_; - cv::copyMakeBorder( - *im, *im, 0, nh - rh, 0, nw - rw, cv::BORDER_CONSTANT, cv::Scalar(0)); - data->in_net_shape_ = { - static_cast(im->rows), static_cast(im->cols), - }; -} - -// Preprocessor op running order -const std::vector Preprocessor::RUN_ORDER = {"InitInfo", - "Resize", - "LetterBoxResize", - "NormalizeImage", - "PadStride", - "Permute"}; - -void Preprocessor::Run(cv::Mat* im, ImageBlob* data) { - for (const auto& name : RUN_ORDER) { - if (ops_.find(name) != ops_.end()) { - ops_[name]->Run(im, data); - } - } -} - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h b/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h deleted file mode 100644 index b45388c91cd..00000000000 --- a/fastdeploy/vision/tracking/pptracking/utils/preprocess_op.h +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace PaddleDetection { - -// Object for storing all preprocessed data -class ImageBlob { - public: - // image width and height - std::vector im_shape_; - // Buffer for image data after preprocessing - std::vector im_data_; - // in net data shape(after pad) - std::vector in_net_shape_; - // Evaluation image width and height - // std::vector eval_im_size_f_; - // Scale factor for image size to origin image size - std::vector scale_factor_; -}; - -// Abstraction of preprocessing opration class -class PreprocessOp { - public: - virtual void Init(const YAML::Node& item) = 0; - virtual void Run(cv::Mat* im, ImageBlob* data) = 0; -}; - -class InitInfo : public PreprocessOp { - public: - virtual void Init(const YAML::Node& item) {} - virtual void Run(cv::Mat* im, ImageBlob* data); -}; - -class NormalizeImage : public PreprocessOp { - public: - virtual void Init(const YAML::Node& item) { - mean_ = item["mean"].as>(); - scale_ = item["std"].as>(); - is_scale_ = item["is_scale"].as(); - } - - virtual void Run(cv::Mat* im, ImageBlob* data); - - private: - // CHW or HWC - std::vector mean_; - std::vector scale_; - bool is_scale_; -}; - -class Permute : public PreprocessOp { - public: - virtual void Init(const YAML::Node& item) {} - virtual void Run(cv::Mat* im, ImageBlob* data); -}; - -class Resize : public PreprocessOp { - public: - virtual void Init(const YAML::Node& item) { - interp_ = item["interp"].as(); - keep_ratio_ = item["keep_ratio"].as(); - target_size_ = item["target_size"].as>(); - } - - // Compute best resize scale for x-dimension, y-dimension - std::pair GenerateScale(const cv::Mat& im); - - virtual void Run(cv::Mat* im, ImageBlob* data); - - private: - int interp_; - bool keep_ratio_; - std::vector target_size_; - std::vector in_net_shape_; -}; - -class LetterBoxResize : public PreprocessOp { - public: - virtual void Init(const YAML::Node& item) { - target_size_ = item["target_size"].as>(); - } - - float GenerateScale(const cv::Mat& im); - - virtual void Run(cv::Mat* im, ImageBlob* data); - - private: - std::vector target_size_; - std::vector in_net_shape_; -}; -// Models with FPN need input shape % stride == 0 -class PadStride : public PreprocessOp { - public: - virtual void Init(const YAML::Node& item) { - stride_ = item["stride"].as(); - } - - virtual void Run(cv::Mat* im, ImageBlob* data); - - private: - int stride_; -}; - -class Preprocessor { - public: - void Init(const YAML::Node& config_node) { - // initialize image info at first - ops_["InitInfo"] = std::make_shared(); - for (const auto& item : config_node) { - auto op_name = item["type"].as(); - - ops_[op_name] = CreateOp(op_name); - ops_[op_name]->Init(item); - } - } - - std::shared_ptr CreateOp(const std::string& name) { - if (name == "Resize") { - return std::make_shared(); - } else if (name == "LetterBoxResize") { - return std::make_shared(); - } else if (name == "Permute") { - return std::make_shared(); - } else if (name == "NormalizeImage") { - return std::make_shared(); - } else if (name == "PadStride") { - // use PadStride instead of PadBatch - return std::make_shared(); - } - std::cerr << "can not find function of OP: " << name - << " and return: nullptr" << std::endl; - return nullptr; - } - - void Run(cv::Mat* im, ImageBlob* data); - - public: - static const std::vector RUN_ORDER; - - private: - std::unordered_map> ops_; -}; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils/utils.h b/fastdeploy/vision/tracking/pptracking/utils/utils.h deleted file mode 100644 index 9d94492a430..00000000000 --- a/fastdeploy/vision/tracking/pptracking/utils/utils.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "include/tracker.h" - -namespace PaddleDetection { - -struct Rect { - float left; - float top; - float right; - float bottom; -}; - -struct MOTTrack { - int ids; - float score; - Rect rects; - int class_id = -1; -}; - -typedef std::vector MOTResult; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/tracking_pybind.cc b/fastdeploy/vision/tracking/tracking_pybind.cc deleted file mode 100644 index cfec96db820..00000000000 --- a/fastdeploy/vision/tracking/tracking_pybind.cc +++ /dev/null @@ -1,3 +0,0 @@ -// -// Created by aichao on 2022/10/9. -// From c54af78912cde4a1e3ac33decee388a7bc60a723 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Tue, 11 Oct 2022 18:12:14 +0800 Subject: [PATCH 05/25] add tracking --- .../vision/tracking/pptracking/lapjv.cc | 413 ++++++++++++++ fastdeploy/vision/tracking/pptracking/lapjv.h | 64 +++ .../vision/tracking/pptracking/model.cc | 203 +++++++ fastdeploy/vision/tracking/pptracking/model.h | 56 ++ .../vision/tracking/pptracking/tracker.cc | 304 ++++++++++ .../vision/tracking/pptracking/tracker.h | 72 +++ .../vision/tracking/pptracking/trajectory.cc | 517 ++++++++++++++++++ .../vision/tracking/pptracking/trajectory.h | 230 ++++++++ fastdeploy/vision/tracking/pptracking/utils.h | 45 ++ fastdeploy/vision/visualize/mot.cc | 3 + 10 files changed, 1907 insertions(+) create mode 100644 fastdeploy/vision/tracking/pptracking/lapjv.cc create mode 100644 fastdeploy/vision/tracking/pptracking/lapjv.h create mode 100644 fastdeploy/vision/tracking/pptracking/model.cc create mode 100644 fastdeploy/vision/tracking/pptracking/model.h create mode 100644 fastdeploy/vision/tracking/pptracking/tracker.cc create mode 100644 fastdeploy/vision/tracking/pptracking/tracker.h create mode 100644 fastdeploy/vision/tracking/pptracking/trajectory.cc create mode 100644 fastdeploy/vision/tracking/pptracking/trajectory.h create mode 100644 fastdeploy/vision/tracking/pptracking/utils.h create mode 100644 fastdeploy/vision/visualize/mot.cc diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.cc b/fastdeploy/vision/tracking/pptracking/lapjv.cc new file mode 100644 index 00000000000..c943d391aa6 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/lapjv.cc @@ -0,0 +1,413 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/gatagat/lap/blob/master/lap/lapjv.cpp +// Ths copyright of gatagat/lap is as follows: +// MIT License + +#include +#include +#include + +#include "include/lapjv.h" + +namespace fastdeploy { +namespace vision { +namespace tracking { + +/** Column-reduction and reduction transfer for a dense cost matrix. + */ +int _ccrrt_dense( + const int n, float *cost[], int *free_rows, int *x, int *y, float *v) { + int n_free_rows; + bool *unique; + + for (int i = 0; i < n; i++) { + x[i] = -1; + v[i] = LARGE; + y[i] = 0; + } + for (int i = 0; i < n; i++) { + for (int j = 0; j < n; j++) { + const float c = cost[i][j]; + if (c < v[j]) { + v[j] = c; + y[j] = i; + } + } + } + NEW(unique, bool, n); + memset(unique, TRUE, n); + { + int j = n; + do { + j--; + const int i = y[j]; + if (x[i] < 0) { + x[i] = j; + } else { + unique[i] = FALSE; + y[j] = -1; + } + } while (j > 0); + } + n_free_rows = 0; + for (int i = 0; i < n; i++) { + if (x[i] < 0) { + free_rows[n_free_rows++] = i; + } else if (unique[i]) { + const int j = x[i]; + float min = LARGE; + for (int j2 = 0; j2 < n; j2++) { + if (j2 == static_cast(j)) { + continue; + } + const float c = cost[i][j2] - v[j2]; + if (c < min) { + min = c; + } + } + v[j] -= min; + } + } + FREE(unique); + return n_free_rows; +} + +/** Augmenting row reduction for a dense cost matrix. + */ +int _carr_dense(const int n, + float *cost[], + const int n_free_rows, + int *free_rows, + int *x, + int *y, + float *v) { + int current = 0; + int new_free_rows = 0; + int rr_cnt = 0; + while (current < n_free_rows) { + int i0; + int j1, j2; + float v1, v2, v1_new; + bool v1_lowers; + + rr_cnt++; + const int free_i = free_rows[current++]; + j1 = 0; + v1 = cost[free_i][0] - v[0]; + j2 = -1; + v2 = LARGE; + for (int j = 1; j < n; j++) { + const float c = cost[free_i][j] - v[j]; + if (c < v2) { + if (c >= v1) { + v2 = c; + j2 = j; + } else { + v2 = v1; + v1 = c; + j2 = j1; + j1 = j; + } + } + } + i0 = y[j1]; + v1_new = v[j1] - (v2 - v1); + v1_lowers = v1_new < v[j1]; + if (rr_cnt < current * n) { + if (v1_lowers) { + v[j1] = v1_new; + } else if (i0 >= 0 && j2 >= 0) { + j1 = j2; + i0 = y[j2]; + } + if (i0 >= 0) { + if (v1_lowers) { + free_rows[--current] = i0; + } else { + free_rows[new_free_rows++] = i0; + } + } + } else { + if (i0 >= 0) { + free_rows[new_free_rows++] = i0; + } + } + x[free_i] = j1; + y[j1] = free_i; + } + return new_free_rows; +} + +/** Find columns with minimum d[j] and put them on the SCAN list. + */ +int _find_dense(const int n, int lo, float *d, int *cols, int *y) { + int hi = lo + 1; + float mind = d[cols[lo]]; + for (int k = hi; k < n; k++) { + int j = cols[k]; + if (d[j] <= mind) { + if (d[j] < mind) { + hi = lo; + mind = d[j]; + } + cols[k] = cols[hi]; + cols[hi++] = j; + } + } + return hi; +} + +// Scan all columns in TODO starting from arbitrary column in SCAN +// and try to decrease d of the TODO columns using the SCAN column. +int _scan_dense(const int n, + float *cost[], + int *plo, + int *phi, + float *d, + int *cols, + int *pred, + int *y, + float *v) { + int lo = *plo; + int hi = *phi; + float h, cred_ij; + + while (lo != hi) { + int j = cols[lo++]; + const int i = y[j]; + const float mind = d[j]; + h = cost[i][j] - v[j] - mind; + // For all columns in TODO + for (int k = hi; k < n; k++) { + j = cols[k]; + cred_ij = cost[i][j] - v[j] - h; + if (cred_ij < d[j]) { + d[j] = cred_ij; + pred[j] = i; + if (cred_ij == mind) { + if (y[j] < 0) { + return j; + } + cols[k] = cols[hi]; + cols[hi++] = j; + } + } + } + } + *plo = lo; + *phi = hi; + return -1; +} + +/** Single iteration of modified Dijkstra shortest path algorithm as explained + * in the JV paper. + * + * This is a dense matrix version. + * + * \return The closest free column index. + */ +int find_path_dense(const int n, + float *cost[], + const int start_i, + int *y, + float *v, + int *pred) { + int lo = 0, hi = 0; + int final_j = -1; + int n_ready = 0; + int *cols; + float *d; + + NEW(cols, int, n); + NEW(d, float, n); + + for (int i = 0; i < n; i++) { + cols[i] = i; + pred[i] = start_i; + d[i] = cost[start_i][i] - v[i]; + } + while (final_j == -1) { + // No columns left on the SCAN list. + if (lo == hi) { + n_ready = lo; + hi = _find_dense(n, lo, d, cols, y); + for (int k = lo; k < hi; k++) { + const int j = cols[k]; + if (y[j] < 0) { + final_j = j; + } + } + } + if (final_j == -1) { + final_j = _scan_dense(n, cost, &lo, &hi, d, cols, pred, y, v); + } + } + + { + const float mind = d[cols[lo]]; + for (int k = 0; k < n_ready; k++) { + const int j = cols[k]; + v[j] += d[j] - mind; + } + } + + FREE(cols); + FREE(d); + + return final_j; +} + +/** Augment for a dense cost matrix. + */ +int _ca_dense(const int n, + float *cost[], + const int n_free_rows, + int *free_rows, + int *x, + int *y, + float *v) { + int *pred; + + NEW(pred, int, n); + + for (int *pfree_i = free_rows; pfree_i < free_rows + n_free_rows; pfree_i++) { + int i = -1, j; + int k = 0; + + j = find_path_dense(n, cost, *pfree_i, y, v, pred); + while (i != *pfree_i) { + i = pred[j]; + y[j] = i; + SWAP_INDICES(j, x[i]); + k++; + } + } + FREE(pred); + return 0; +} + +/** Solve dense sparse LAP. + */ +int lapjv_internal(const cv::Mat &cost, + const bool extend_cost, + const float cost_limit, + int *x, + int *y) { + int n_rows = cost.rows; + int n_cols = cost.cols; + int n; + if (n_rows == n_cols) { + n = n_rows; + } else if (!extend_cost) { + throw std::invalid_argument( + "Square cost array expected. If cost is intentionally non-square, pass " + "extend_cost=True."); + } + + // Get extend cost + if (extend_cost || cost_limit < LARGE) { + n = n_rows + n_cols; + } + cv::Mat cost_expand(n, n, CV_32F); + float expand_value; + if (cost_limit < LARGE) { + expand_value = cost_limit / 2; + } else { + double max_v; + minMaxLoc(cost, nullptr, &max_v); + expand_value = static_cast(max_v) + 1.; + } + + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + cost_expand.at(i, j) = expand_value; + if (i >= n_rows && j >= n_cols) { + cost_expand.at(i, j) = 0; + } else if (i < n_rows && j < n_cols) { + cost_expand.at(i, j) = cost.at(i, j); + } + } + } + + // Convert Mat to pointer array + float **cost_ptr; + NEW(cost_ptr, float *, n); + for (int i = 0; i < n; ++i) { + NEW(cost_ptr[i], float, n); + } + for (int i = 0; i < n; ++i) { + for (int j = 0; j < n; ++j) { + cost_ptr[i][j] = cost_expand.at(i, j); + } + } + + int ret; + int *free_rows; + float *v; + int *x_c; + int *y_c; + + NEW(free_rows, int, n); + NEW(v, float, n); + NEW(x_c, int, n); + NEW(y_c, int, n); + + ret = _ccrrt_dense(n, cost_ptr, free_rows, x_c, y_c, v); + int i = 0; + while (ret > 0 && i < 2) { + ret = _carr_dense(n, cost_ptr, ret, free_rows, x_c, y_c, v); + i++; + } + if (ret > 0) { + ret = _ca_dense(n, cost_ptr, ret, free_rows, x_c, y_c, v); + } + FREE(v); + FREE(free_rows); + for (int i = 0; i < n; ++i) { + FREE(cost_ptr[i]); + } + FREE(cost_ptr); + if (ret != 0) { + if (ret == -1) { + throw "Out of memory."; + } + throw "Unknown error (lapjv_internal)"; + } + // Get output of x, y, opt + for (int i = 0; i < n; ++i) { + if (i < n_rows) { + x[i] = x_c[i]; + if (x[i] >= n_cols) { + x[i] = -1; + } + } + if (i < n_cols) { + y[i] = y_c[i]; + if (y[i] >= n_rows) { + y[i] = -1; + } + } + } + + FREE(x_c); + FREE(y_c); + return ret; +} + +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.h b/fastdeploy/vision/tracking/pptracking/lapjv.h new file mode 100644 index 00000000000..ffaa010c005 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/lapjv.h @@ -0,0 +1,64 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/gatagat/lap/blob/master/lap/lapjv.h +// Ths copyright of gatagat/lap is as follows: +// MIT License + +#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ +#define DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ +#define LARGE 1000000 + +#if !defined TRUE +#define TRUE 1 +#endif +#if !defined FALSE +#define FALSE 0 +#endif + +#define NEW(x, t, n) \ + if ((x = reinterpret_cast(malloc(sizeof(t) * (n)))) == 0) { \ + return -1; \ + } +#define FREE(x) \ + if (x != 0) { \ + free(x); \ + x = 0; \ + } +#define SWAP_INDICES(a, b) \ + { \ + int_t _temp_index = a; \ + a = b; \ + b = _temp_index; \ + } +#include + +namespace PaddleDetection { + +typedef signed int int_t; +typedef unsigned int uint_t; +typedef double cost_t; +typedef char boolean; +typedef enum fp_t { FP_1 = 1, FP_2 = 2, FP_DYNAMIC = 3 } fp_t; + +int lapjv_internal(const cv::Mat &cost, + const bool extend_cost, + const float cost_limit, + int *x, + int *y); + +} // namespace PaddleDetection + +#endif // DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc new file mode 100644 index 00000000000..13c12a4e338 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -0,0 +1,203 @@ +// +// Created by aichao on 2022/10/10. +// + +#include "fastdeploy/vision/tracking/pptracking/predictor.h" +namespace fastdeploy { +namespace vision { +namespace tracking { + +PPTracking::PPTracking(const std::string& model_file, const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + if (model_format == ModelFormat::ONNX) + { + valid_cpu_backends = {Backend::ORT,Backend::OPENVINO}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; + } else { + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + } + runtime_option = custom_option; + runtime_option.model_format = model_format; + runtime_option.model_file = model_file; + runtime_option.params_file = params_file; + + std::cout<<"--------"< input_tensors; + + if (!Preprocess(&mat, &input_tensors)) { + FDERROR << "Failed to preprocess input image." << std::endl; + return false; + } + float* tmp = static_cast(input_tensors[1].Data()); + + std::vector output_tensors; + if (!Infer(input_tensors, &output_tensors)) { + FDERROR << "Failed to inference." << std::endl; + return false; + } + + if (!Postprocess(output_tensors, result)) { + FDERROR << "Failed to post process." << std::endl; + return false; + } + + return true; + + +} + +void PPTracking::LetterBoxResize(Mat* im){ + + // generate scale_factor + int origin_w = im->Width(); + int origin_h = im->Height(); + int target_h = target_size_[0]; + int target_w = target_size_[1]; + float ratio_h = static_cast(target_h) / static_cast(origin_h); + float ratio_w = static_cast(target_w) / static_cast(origin_w); + float resize_scale = std::min(ratio_h, ratio_w); + + int new_shape_w = std::round(im->Width() * resize_scale); + int new_shape_h = std::round(im->Height() * resize_scale); + float padw = (target_size_[1] - new_shape_w) / 2.; + float padh = (target_size_[0] - new_shape_h) / 2.; + int top = std::round(padh - 0.1); + int bottom = std::round(padh + 0.1); + int left = std::round(padw - 0.1); + int right = std::round(padw + 0.1); + + Resize::Run(im,new_shape_w,new_shape_h); + std::vector color{127.5,127.5,127.5}; + Pad::Run(im,top,bottom,left,right,color); +} + +bool PPTracking::Preprocess(Mat* mat, std::vector* outputs) { + + int origin_w = mat->Width(); + int origin_h = mat->Height(); + LetterBoxResize(mat); + Normalize::Run(mat,mean_,scale_,is_scale_); + HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + + outputs->resize(3); + // image_shape + (*outputs)[0].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(0).name); + float* shape = static_cast((*outputs)[0].MutableData()); + shape[0] = mat->Height(); + shape[1] = mat->Width(); + // image + (*outputs)[1].name = InputInfoOfRuntime(1).name; + mat->ShareWithTensor(&((*outputs)[1])); + (*outputs)[1].ExpandDim(0); + // scale + (*outputs)[2].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(2).name); + float* scale = static_cast((*outputs)[2].MutableData()); + scale[0] = mat->Height() * 1.0 / origin_h; + scale[1] = mat->Width() * 1.0 / origin_w; + return true; +} + + +void FilterDets(const float conf_thresh,const cv::Mat dets,std::vector* index) { + for (int i = 0; i < dets.rows; ++i) { + float score = *dets.ptr(i, 4); + if (score > conf_thresh) { + index->push_back(i); + } + } +} + +bool PPTracking::Postprocess(std::vector& infer_result, MOTResult *result){ + + + auto bbox_shape = infer_result[0].shape; + auto bbox_data = static_cast(infer_result[0].Data()); + + auto emb_shape = infer_result[1].shape; + auto emb_data = static_cast(infer_result[1].Data()); + + cv::Mat dets(bbox_shape[0], 6, CV_32FC1, bbox_data); + cv::Mat emb(bbox_shape[0], emb_shape[1], CV_32FC1, emb_data); + + + result->clear(); + std::vector tracks; + std::vector valid; + FilterDets(conf_thresh_, dets, &valid); + cv::Mat new_dets, new_emb; + for (int i = 0; i < valid.size(); ++i) { + new_dets.push_back(dets.row(valid[i])); + new_emb.push_back(emb.row(valid[i])); + } + JDETracker::instance()->update(new_dets, new_emb, &tracks); + if (tracks.size() == 0) { + MOTTrack mot_track; + Rect ret = {*dets.ptr(0, 0), + *dets.ptr(0, 1), + *dets.ptr(0, 2), + *dets.ptr(0, 3)}; + mot_track.ids = 1; + mot_track.score = *dets.ptr(0, 4); + mot_track.rects = ret; + result->push_back(mot_track); + } else { + std::vector::iterator titer; + for (titer = tracks.begin(); titer != tracks.end(); ++titer) { + if (titer->score < threshold_) { + continue; + } else { + float w = titer->ltrb[2] - titer->ltrb[0]; + float h = titer->ltrb[3] - titer->ltrb[1]; + bool vertical = w / h > 1.6; + float area = w * h; + if (area > min_box_area_ && !vertical) { + MOTTrack mot_track; + Rect ret = { + titer->ltrb[0], titer->ltrb[1], titer->ltrb[2], titer->ltrb[3]}; + mot_track.rects = ret; + mot_track.score = titer->score; + mot_track.ids = titer->id; + result->push_back(mot_track); + } + } + } + } + return true; +} + +cv::Mat PPTracking::Visualize(const cv::Mat& img, + const MOTResult& results, + const float fps, + const int frame_id){ + +return VisualizeTrackResult(img,results,fps,frame_id); + +} + +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/model.h b/fastdeploy/vision/tracking/pptracking/model.h new file mode 100644 index 00000000000..7b11705ee0f --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/model.h @@ -0,0 +1,56 @@ +// +// Created by aichao on 2022/10/10. +// + +#pragma once +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/tracking/pptracking/utils.h" +#include "fastdeploy/vision/tracking/pptracking/postprocess.h" + +namespace fastdeploy { +namespace vision { +namespace tracking { + +class FASTDEPLOY_DECL PPTracking: public FastDeployModel { + + public: + PPTracking(const std::string& model_file, const std::string& params_file = "", + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); + + std::string ModelName() const { return "pptracking/jde"; } + + virtual bool Predict(cv::Mat* img, MOTResult* result); + static cv::Mat Visualize(const cv::Mat& img, + const MOTResult& results, + float fps, + int frame_id); + + private: + + + std::vector mean_; + std::vector scale_; + bool is_scale_ = true; + std::vector target_size_; + float conf_thresh_; + float threshold_; + float min_box_area_; + + bool Initialize(); + + + void LetterBoxResize(Mat* im); + + bool Preprocess(Mat* img, std::vector* outputs); + + bool Postprocess(std::vector& infer_result, MOTResult *result); + + +}; + +} // namespace tracking +} // namespace vision +} // namespace fastdeploy + diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc new file mode 100644 index 00000000000..9540e39f670 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/tracker.cc @@ -0,0 +1,304 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.cpp +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#include +#include +#include +#include + +#include "include/lapjv.h" +#include "include/tracker.h" + +#define mat2vec4f(m) \ + cv::Vec4f(*m.ptr(0, 0), \ + *m.ptr(0, 1), \ + *m.ptr(0, 2), \ + *m.ptr(0, 3)) + +namespace PaddleDetection { + +static std::map chi2inv95 = {{1, 3.841459f}, + {2, 5.991465f}, + {3, 7.814728f}, + {4, 9.487729f}, + {5, 11.070498f}, + {6, 12.591587f}, + {7, 14.067140f}, + {8, 15.507313f}, + {9, 16.918978f}}; + +JDETracker *JDETracker::me = new JDETracker; + +JDETracker *JDETracker::instance(void) { return me; } + +JDETracker::JDETracker(void) + : timestamp(0), max_lost_time(30), lambda(0.98f), det_thresh(0.3f) {} + +bool JDETracker::update(const cv::Mat &dets, + const cv::Mat &emb, + std::vector *tracks) { + ++timestamp; + TrajectoryPool candidates(dets.rows); + for (int i = 0; i < dets.rows; ++i) { + float score = *dets.ptr(i, 1); + const cv::Mat <rb_ = dets(cv::Rect(2, i, 4, 1)); + cv::Vec4f ltrb = mat2vec4f(ltrb_); + const cv::Mat &embedding = emb(cv::Rect(0, i, emb.cols, 1)); + candidates[i] = Trajectory(ltrb, score, embedding); + } + + TrajectoryPtrPool tracked_trajectories; + TrajectoryPtrPool unconfirmed_trajectories; + for (size_t i = 0; i < this->tracked_trajectories.size(); ++i) { + if (this->tracked_trajectories[i].is_activated) + tracked_trajectories.push_back(&this->tracked_trajectories[i]); + else + unconfirmed_trajectories.push_back(&this->tracked_trajectories[i]); + } + + TrajectoryPtrPool trajectory_pool = + tracked_trajectories + &(this->lost_trajectories); + + for (size_t i = 0; i < trajectory_pool.size(); ++i) + trajectory_pool[i]->predict(); + + Match matches; + std::vector mismatch_row; + std::vector mismatch_col; + + cv::Mat cost = motion_distance(trajectory_pool, candidates); + linear_assignment(cost, 0.7f, &matches, &mismatch_row, &mismatch_col); + + MatchIterator miter; + TrajectoryPtrPool activated_trajectories; + TrajectoryPtrPool retrieved_trajectories; + + for (miter = matches.begin(); miter != matches.end(); miter++) { + Trajectory *pt = trajectory_pool[miter->first]; + Trajectory &ct = candidates[miter->second]; + if (pt->state == Tracked) { + pt->update(&ct, timestamp); + activated_trajectories.push_back(pt); + } else { + pt->reactivate(&ct, timestamp); + retrieved_trajectories.push_back(pt); + } + } + + TrajectoryPtrPool next_candidates(mismatch_col.size()); + for (size_t i = 0; i < mismatch_col.size(); ++i) + next_candidates[i] = &candidates[mismatch_col[i]]; + + TrajectoryPtrPool next_trajectory_pool; + for (size_t i = 0; i < mismatch_row.size(); ++i) { + int j = mismatch_row[i]; + if (trajectory_pool[j]->state == Tracked) + next_trajectory_pool.push_back(trajectory_pool[j]); + } + + cost = iou_distance(next_trajectory_pool, next_candidates); + linear_assignment(cost, 0.5f, &matches, &mismatch_row, &mismatch_col); + + for (miter = matches.begin(); miter != matches.end(); miter++) { + Trajectory *pt = next_trajectory_pool[miter->first]; + Trajectory *ct = next_candidates[miter->second]; + if (pt->state == Tracked) { + pt->update(ct, timestamp); + activated_trajectories.push_back(pt); + } else { + pt->reactivate(ct, timestamp); + retrieved_trajectories.push_back(pt); + } + } + + TrajectoryPtrPool lost_trajectories; + for (size_t i = 0; i < mismatch_row.size(); ++i) { + Trajectory *pt = next_trajectory_pool[mismatch_row[i]]; + if (pt->state != Lost) { + pt->mark_lost(); + lost_trajectories.push_back(pt); + } + } + + TrajectoryPtrPool nnext_candidates(mismatch_col.size()); + for (size_t i = 0; i < mismatch_col.size(); ++i) + nnext_candidates[i] = next_candidates[mismatch_col[i]]; + cost = iou_distance(unconfirmed_trajectories, nnext_candidates); + linear_assignment(cost, 0.7f, &matches, &mismatch_row, &mismatch_col); + + for (miter = matches.begin(); miter != matches.end(); miter++) { + unconfirmed_trajectories[miter->first]->update( + nnext_candidates[miter->second], timestamp); + activated_trajectories.push_back(unconfirmed_trajectories[miter->first]); + } + + TrajectoryPtrPool removed_trajectories; + + for (size_t i = 0; i < mismatch_row.size(); ++i) { + unconfirmed_trajectories[mismatch_row[i]]->mark_removed(); + removed_trajectories.push_back(unconfirmed_trajectories[mismatch_row[i]]); + } + + for (size_t i = 0; i < mismatch_col.size(); ++i) { + if (nnext_candidates[mismatch_col[i]]->score < det_thresh) continue; + nnext_candidates[mismatch_col[i]]->activate(timestamp); + activated_trajectories.push_back(nnext_candidates[mismatch_col[i]]); + } + + for (size_t i = 0; i < this->lost_trajectories.size(); ++i) { + Trajectory < = this->lost_trajectories[i]; + if (timestamp - lt.timestamp > max_lost_time) { + lt.mark_removed(); + removed_trajectories.push_back(<); + } + } + + TrajectoryPoolIterator piter; + for (piter = this->tracked_trajectories.begin(); + piter != this->tracked_trajectories.end();) { + if (piter->state != Tracked) + piter = this->tracked_trajectories.erase(piter); + else + ++piter; + } + + this->tracked_trajectories += activated_trajectories; + this->tracked_trajectories += retrieved_trajectories; + + this->lost_trajectories -= this->tracked_trajectories; + this->lost_trajectories += lost_trajectories; + this->lost_trajectories -= this->removed_trajectories; + this->removed_trajectories += removed_trajectories; + remove_duplicate_trajectory(&this->tracked_trajectories, + &this->lost_trajectories); + + tracks->clear(); + for (size_t i = 0; i < this->tracked_trajectories.size(); ++i) { + if (this->tracked_trajectories[i].is_activated) { + Track track = {this->tracked_trajectories[i].id, + this->tracked_trajectories[i].score, + this->tracked_trajectories[i].ltrb}; + tracks->push_back(track); + } + } + return 0; +} + +cv::Mat JDETracker::motion_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b) { + if (0 == a.size() || 0 == b.size()) + return cv::Mat(a.size(), b.size(), CV_32F); + + cv::Mat edists = embedding_distance(a, b); + cv::Mat mdists = mahalanobis_distance(a, b); + cv::Mat fdists = lambda * edists + (1 - lambda) * mdists; + + const float gate_thresh = chi2inv95[4]; + for (int i = 0; i < fdists.rows; ++i) { + for (int j = 0; j < fdists.cols; ++j) { + if (*mdists.ptr(i, j) > gate_thresh) + *fdists.ptr(i, j) = FLT_MAX; + } + } + + return fdists; +} + +void JDETracker::linear_assignment(const cv::Mat &cost, + float cost_limit, + Match *matches, + std::vector *mismatch_row, + std::vector *mismatch_col) { + matches->clear(); + mismatch_row->clear(); + mismatch_col->clear(); + if (cost.empty()) { + for (int i = 0; i < cost.rows; ++i) mismatch_row->push_back(i); + for (int i = 0; i < cost.cols; ++i) mismatch_col->push_back(i); + return; + } + + float opt = 0; + cv::Mat x(cost.rows, 1, CV_32S); + cv::Mat y(cost.cols, 1, CV_32S); + + lapjv_internal(cost, + true, + cost_limit, + reinterpret_cast(x.data), + reinterpret_cast(y.data)); + + for (int i = 0; i < x.rows; ++i) { + int j = *x.ptr(i); + if (j >= 0) + matches->insert({i, j}); + else + mismatch_row->push_back(i); + } + + for (int i = 0; i < y.rows; ++i) { + int j = *y.ptr(i); + if (j < 0) mismatch_col->push_back(i); + } + + return; +} + +void JDETracker::remove_duplicate_trajectory(TrajectoryPool *a, + TrajectoryPool *b, + float iou_thresh) { + if (a->size() == 0 || b->size() == 0) return; + + cv::Mat dist = iou_distance(*a, *b); + cv::Mat mask = dist < iou_thresh; + std::vector idx; + cv::findNonZero(mask, idx); + + std::vector da; + std::vector db; + for (size_t i = 0; i < idx.size(); ++i) { + int ta = (*a)[idx[i].y].timestamp - (*a)[idx[i].y].starttime; + int tb = (*b)[idx[i].x].timestamp - (*b)[idx[i].x].starttime; + if (ta > tb) + db.push_back(idx[i].x); + else + da.push_back(idx[i].y); + } + + int id = 0; + TrajectoryPoolIterator piter; + for (piter = a->begin(); piter != a->end();) { + std::vector::iterator iter = find(da.begin(), da.end(), id++); + if (iter != da.end()) + piter = a->erase(piter); + else + ++piter; + } + + id = 0; + for (piter = b->begin(); piter != b->end();) { + std::vector::iterator iter = find(db.begin(), db.end(), id++); + if (iter != db.end()) + piter = b->erase(piter); + else + ++piter; + } +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h new file mode 100644 index 00000000000..244530f140a --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -0,0 +1,72 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/jdetracker.h +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#pragma once + +#include +#include + +#include +#include +#include +#include "include/trajectory.h" + +namespace PaddleDetection { + +typedef std::map Match; +typedef std::map::iterator MatchIterator; + +struct Track { + int id; + float score; + cv::Vec4f ltrb; +}; + +class JDETracker { + public: + static JDETracker *instance(void); + virtual bool update(const cv::Mat &dets, + const cv::Mat &emb, + std::vector *tracks); + + private: + JDETracker(void); + virtual ~JDETracker(void) {} + cv::Mat motion_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b); + void linear_assignment(const cv::Mat &cost, + float cost_limit, + Match *matches, + std::vector *mismatch_row, + std::vector *mismatch_col); + void remove_duplicate_trajectory(TrajectoryPool *a, + TrajectoryPool *b, + float iou_thresh = 0.15f); + + private: + static JDETracker *me; + int timestamp; + TrajectoryPool tracked_trajectories; + TrajectoryPool lost_trajectories; + TrajectoryPool removed_trajectories; + int max_lost_time; + float lambda; + float det_thresh; +}; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.cc b/fastdeploy/vision/tracking/pptracking/trajectory.cc new file mode 100644 index 00000000000..0ff2e1a5fc7 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/trajectory.cc @@ -0,0 +1,517 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.cpp +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#include "include/trajectory.h" +#include + +namespace PaddleDetection { + +void TKalmanFilter::init(const cv::Mat &measurement) { + measurement.copyTo(statePost(cv::Rect(0, 0, 1, 4))); + statePost(cv::Rect(0, 4, 1, 4)).setTo(0); + statePost.copyTo(statePre); + + float varpos = 2 * std_weight_position * (*measurement.ptr(3)); + varpos *= varpos; + float varvel = 10 * std_weight_velocity * (*measurement.ptr(3)); + varvel *= varvel; + + errorCovPost.setTo(0); + *errorCovPost.ptr(0, 0) = varpos; + *errorCovPost.ptr(1, 1) = varpos; + *errorCovPost.ptr(2, 2) = 1e-4f; + *errorCovPost.ptr(3, 3) = varpos; + *errorCovPost.ptr(4, 4) = varvel; + *errorCovPost.ptr(5, 5) = varvel; + *errorCovPost.ptr(6, 6) = 1e-10f; + *errorCovPost.ptr(7, 7) = varvel; + errorCovPost.copyTo(errorCovPre); +} + +const cv::Mat &TKalmanFilter::predict() { + float varpos = std_weight_position * (*statePre.ptr(3)); + varpos *= varpos; + float varvel = std_weight_velocity * (*statePre.ptr(3)); + varvel *= varvel; + + processNoiseCov.setTo(0); + *processNoiseCov.ptr(0, 0) = varpos; + *processNoiseCov.ptr(1, 1) = varpos; + *processNoiseCov.ptr(2, 2) = 1e-4f; + *processNoiseCov.ptr(3, 3) = varpos; + *processNoiseCov.ptr(4, 4) = varvel; + *processNoiseCov.ptr(5, 5) = varvel; + *processNoiseCov.ptr(6, 6) = 1e-10f; + *processNoiseCov.ptr(7, 7) = varvel; + + return cv::KalmanFilter::predict(); +} + +const cv::Mat &TKalmanFilter::correct(const cv::Mat &measurement) { + float varpos = std_weight_position * (*measurement.ptr(3)); + varpos *= varpos; + + measurementNoiseCov.setTo(0); + *measurementNoiseCov.ptr(0, 0) = varpos; + *measurementNoiseCov.ptr(1, 1) = varpos; + *measurementNoiseCov.ptr(2, 2) = 1e-2f; + *measurementNoiseCov.ptr(3, 3) = varpos; + + return cv::KalmanFilter::correct(measurement); +} + +void TKalmanFilter::project(cv::Mat *mean, cv::Mat *covariance) const { + float varpos = std_weight_position * (*statePost.ptr(3)); + varpos *= varpos; + + cv::Mat measurementNoiseCov_ = cv::Mat::eye(4, 4, CV_32F); + *measurementNoiseCov_.ptr(0, 0) = varpos; + *measurementNoiseCov_.ptr(1, 1) = varpos; + *measurementNoiseCov_.ptr(2, 2) = 1e-2f; + *measurementNoiseCov_.ptr(3, 3) = varpos; + + *mean = measurementMatrix * statePost; + cv::Mat temp = measurementMatrix * errorCovPost; + gemm(temp, + measurementMatrix, + 1, + measurementNoiseCov_, + 1, + *covariance, + cv::GEMM_2_T); +} + +int Trajectory::count = 0; + +const cv::Mat &Trajectory::predict(void) { + if (state != Tracked) *cv::KalmanFilter::statePost.ptr(7) = 0; + return TKalmanFilter::predict(); +} + +void Trajectory::update(Trajectory *traj, + int timestamp_, + bool update_embedding_) { + timestamp = timestamp_; + ++length; + ltrb = traj->ltrb; + xyah = traj->xyah; + TKalmanFilter::correct(cv::Mat(traj->xyah)); + state = Tracked; + is_activated = true; + score = traj->score; + if (update_embedding_) update_embedding(traj->current_embedding); +} + +void Trajectory::activate(int timestamp_) { + id = next_id(); + TKalmanFilter::init(cv::Mat(xyah)); + length = 0; + state = Tracked; + if (timestamp_ == 1) { + is_activated = true; + } + timestamp = timestamp_; + starttime = timestamp_; +} + +void Trajectory::reactivate(Trajectory *traj, int timestamp_, bool newid) { + TKalmanFilter::correct(cv::Mat(traj->xyah)); + update_embedding(traj->current_embedding); + length = 0; + state = Tracked; + is_activated = true; + timestamp = timestamp_; + if (newid) id = next_id(); +} + +void Trajectory::update_embedding(const cv::Mat &embedding) { + current_embedding = embedding / cv::norm(embedding); + if (smooth_embedding.empty()) { + smooth_embedding = current_embedding; + } else { + smooth_embedding = eta * smooth_embedding + (1 - eta) * current_embedding; + } + smooth_embedding = smooth_embedding / cv::norm(smooth_embedding); +} + +TrajectoryPool operator+(const TrajectoryPool &a, const TrajectoryPool &b) { + TrajectoryPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; + + for (size_t i = 0; i < b.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i].id); + if (iter == ids.end()) { + sum.push_back(b[i]); + ids.push_back(b[i].id); + } + } + + return sum; +} + +TrajectoryPool operator+(const TrajectoryPool &a, const TrajectoryPtrPool &b) { + TrajectoryPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; + + for (size_t i = 0; i < b.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); + if (iter == ids.end()) { + sum.push_back(*b[i]); + ids.push_back(b[i]->id); + } + } + + return sum; +} + +TrajectoryPool &operator+=(TrajectoryPool &a, // NOLINT + const TrajectoryPtrPool &b) { + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i].id; + + for (size_t i = 0; i < b.size(); ++i) { + if (b[i]->smooth_embedding.empty()) continue; + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); + if (iter == ids.end()) { + a.push_back(*b[i]); + ids.push_back(b[i]->id); + } + } + + return a; +} + +TrajectoryPool operator-(const TrajectoryPool &a, const TrajectoryPool &b) { + TrajectoryPool dif; + std::vector ids(b.size()); + for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i].id; + + for (size_t i = 0; i < a.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), a[i].id); + if (iter == ids.end()) dif.push_back(a[i]); + } + + return dif; +} + +TrajectoryPool &operator-=(TrajectoryPool &a, // NOLINT + const TrajectoryPool &b) { + std::vector ids(b.size()); + for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i].id; + + TrajectoryPoolIterator piter; + for (piter = a.begin(); piter != a.end();) { + std::vector::iterator iter = find(ids.begin(), ids.end(), piter->id); + if (iter == ids.end()) + ++piter; + else + piter = a.erase(piter); + } + + return a; +} + +TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + TrajectoryPtrPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i]->id; + + for (size_t i = 0; i < b.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), b[i]->id); + if (iter == ids.end()) { + sum.push_back(b[i]); + ids.push_back(b[i]->id); + } + } + + return sum; +} + +TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, TrajectoryPool *b) { + TrajectoryPtrPool sum; + sum.insert(sum.end(), a.begin(), a.end()); + + std::vector ids(a.size()); + for (size_t i = 0; i < a.size(); ++i) ids[i] = a[i]->id; + + for (size_t i = 0; i < b->size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), (*b)[i].id); + if (iter == ids.end()) { + sum.push_back(&(*b)[i]); + ids.push_back((*b)[i].id); + } + } + + return sum; +} + +TrajectoryPtrPool operator-(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + TrajectoryPtrPool dif; + std::vector ids(b.size()); + for (size_t i = 0; i < b.size(); ++i) ids[i] = b[i]->id; + + for (size_t i = 0; i < a.size(); ++i) { + std::vector::iterator iter = find(ids.begin(), ids.end(), a[i]->id); + if (iter == ids.end()) dif.push_back(a[i]); + } + + return dif; +} + +cv::Mat embedding_distance(const TrajectoryPool &a, const TrajectoryPool &b) { + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + cv::Mat u = a[i].smooth_embedding; + cv::Mat v = b[j].smooth_embedding; + double uv = u.dot(v); + double uu = u.dot(u); + double vv = v.dot(v); + double dist = std::abs(1. - uv / std::sqrt(uu * vv)); + // double dist = cv::norm(a[i].smooth_embedding, b[j].smooth_embedding, + // cv::NORM_L2); + distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); + } + } + return dists; +} + +cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + // double dist = cv::norm(a[i]->smooth_embedding, b[j]->smooth_embedding, + // cv::NORM_L2); + // distsi[j] = static_cast(dist); + cv::Mat u = a[i]->smooth_embedding; + cv::Mat v = b[j]->smooth_embedding; + double uv = u.dot(v); + double uu = u.dot(u); + double vv = v.dot(v); + double dist = std::abs(1. - uv / std::sqrt(uu * vv)); + distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); + } + } + + return dists; +} + +cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b) { + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + // double dist = cv::norm(a[i]->smooth_embedding, b[j].smooth_embedding, + // cv::NORM_L2); + // distsi[j] = static_cast(dist); + cv::Mat u = a[i]->smooth_embedding; + cv::Mat v = b[j].smooth_embedding; + double uv = u.dot(v); + double uu = u.dot(u); + double vv = v.dot(v); + double dist = std::abs(1. - uv / std::sqrt(uu * vv)); + distsi[j] = static_cast(std::max(std::min(dist, 2.), 0.)); + } + } + + return dists; +} + +cv::Mat mahalanobis_distance(const TrajectoryPool &a, const TrajectoryPool &b) { + std::vector means(a.size()); + std::vector icovariances(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + cv::Mat covariance; + a[i].project(&means[i], &covariance); + cv::invert(covariance, icovariances[i]); + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Mat x(b[j].xyah); + float dist = + static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); + distsi[j] = dist * dist; + } + } + + return dists; +} + +cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b) { + std::vector means(a.size()); + std::vector icovariances(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + cv::Mat covariance; + a[i]->project(&means[i], &covariance); + cv::invert(covariance, icovariances[i]); + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Mat x(b[j]->xyah); + float dist = + static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); + distsi[j] = dist * dist; + } + } + + return dists; +} + +cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b) { + std::vector means(a.size()); + std::vector icovariances(a.size()); + + for (size_t i = 0; i < a.size(); ++i) { + cv::Mat covariance; + a[i]->project(&means[i], &covariance); + cv::invert(covariance, icovariances[i]); + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Mat x(b[j].xyah); + float dist = + static_cast(cv::Mahalanobis(x, means[i], icovariances[i])); + distsi[j] = dist * dist; + } + } + + return dists; +} + +static inline float calc_inter_area(const cv::Vec4f &a, const cv::Vec4f &b) { + if (a[2] < b[0] || a[0] > b[2] || a[3] < b[1] || a[1] > b[3]) return 0.f; + + float w = std::min(a[2], b[2]) - std::max(a[0], b[0]); + float h = std::min(a[3], b[3]) - std::max(a[1], b[1]); + return w * h; +} + +cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b) { + std::vector areaa(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + float w = a[i].ltrb[2] - a[i].ltrb[0]; + float h = a[i].ltrb[3] - a[i].ltrb[1]; + areaa[i] = w * h; + } + + std::vector areab(b.size()); + for (size_t j = 0; j < b.size(); ++j) { + float w = b[j].ltrb[2] - b[j].ltrb[0]; + float h = b[j].ltrb[3] - b[j].ltrb[1]; + areab[j] = w * h; + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + const cv::Vec4f &boxa = a[i].ltrb; + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Vec4f &boxb = b[j].ltrb; + float inters = calc_inter_area(boxa, boxb); + distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); + } + } + + return dists; +} + +cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPtrPool &b) { + std::vector areaa(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + float w = a[i]->ltrb[2] - a[i]->ltrb[0]; + float h = a[i]->ltrb[3] - a[i]->ltrb[1]; + areaa[i] = w * h; + } + + std::vector areab(b.size()); + for (size_t j = 0; j < b.size(); ++j) { + float w = b[j]->ltrb[2] - b[j]->ltrb[0]; + float h = b[j]->ltrb[3] - b[j]->ltrb[1]; + areab[j] = w * h; + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + const cv::Vec4f &boxa = a[i]->ltrb; + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Vec4f &boxb = b[j]->ltrb; + float inters = calc_inter_area(boxa, boxb); + distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); + } + } + + return dists; +} + +cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b) { + std::vector areaa(a.size()); + for (size_t i = 0; i < a.size(); ++i) { + float w = a[i]->ltrb[2] - a[i]->ltrb[0]; + float h = a[i]->ltrb[3] - a[i]->ltrb[1]; + areaa[i] = w * h; + } + + std::vector areab(b.size()); + for (size_t j = 0; j < b.size(); ++j) { + float w = b[j].ltrb[2] - b[j].ltrb[0]; + float h = b[j].ltrb[3] - b[j].ltrb[1]; + areab[j] = w * h; + } + + cv::Mat dists(a.size(), b.size(), CV_32F); + for (size_t i = 0; i < a.size(); ++i) { + const cv::Vec4f &boxa = a[i]->ltrb; + float *distsi = dists.ptr(i); + for (size_t j = 0; j < b.size(); ++j) { + const cv::Vec4f &boxb = b[j].ltrb; + float inters = calc_inter_area(boxa, boxb); + distsi[j] = 1.f - inters / (areaa[i] + areab[j] - inters); + } + } + + return dists; +} + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h new file mode 100644 index 00000000000..c21e8cac368 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -0,0 +1,230 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code is based on: +// https://github.com/CnybTseng/JDE/blob/master/platforms/common/trajectory.h +// Ths copyright of CnybTseng/JDE is as follows: +// MIT License + +#pragma once + +#include + +#include +#include +#include +#include "opencv2/video/tracking.hpp" + +namespace PaddleDetection { + +typedef enum { New = 0, Tracked = 1, Lost = 2, Removed = 3 } TrajectoryState; + +class Trajectory; +typedef std::vector TrajectoryPool; +typedef std::vector::iterator TrajectoryPoolIterator; +typedef std::vector TrajectoryPtrPool; +typedef std::vector::iterator TrajectoryPtrPoolIterator; + +class TKalmanFilter : public cv::KalmanFilter { + public: + TKalmanFilter(void); + virtual ~TKalmanFilter(void) {} + virtual void init(const cv::Mat &measurement); + virtual const cv::Mat &predict(); + virtual const cv::Mat &correct(const cv::Mat &measurement); + virtual void project(cv::Mat *mean, cv::Mat *covariance) const; + + private: + float std_weight_position; + float std_weight_velocity; +}; + +inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) { + cv::KalmanFilter::transitionMatrix = cv::Mat::eye(8, 8, CV_32F); + for (int i = 0; i < 4; ++i) + cv::KalmanFilter::transitionMatrix.at(i, i + 4) = 1; + cv::KalmanFilter::measurementMatrix = cv::Mat::eye(4, 8, CV_32F); + std_weight_position = 1 / 20.f; + std_weight_velocity = 1 / 160.f; +} + +class Trajectory : public TKalmanFilter { + public: + Trajectory(); + Trajectory(const cv::Vec4f <rb, float score, const cv::Mat &embedding); + Trajectory(const Trajectory &other); + Trajectory &operator=(const Trajectory &rhs); + virtual ~Trajectory(void) {} + + static int next_id(); + virtual const cv::Mat &predict(void); + virtual void update(Trajectory *traj, + int timestamp, + bool update_embedding = true); + virtual void activate(int timestamp); + virtual void reactivate(Trajectory *traj, int timestamp, bool newid = false); + virtual void mark_lost(void); + virtual void mark_removed(void); + + friend TrajectoryPool operator+(const TrajectoryPool &a, + const TrajectoryPool &b); + friend TrajectoryPool operator+(const TrajectoryPool &a, + const TrajectoryPtrPool &b); + friend TrajectoryPool &operator+=(TrajectoryPool &a, // NOLINT + const TrajectoryPtrPool &b); + friend TrajectoryPool operator-(const TrajectoryPool &a, + const TrajectoryPool &b); + friend TrajectoryPool &operator-=(TrajectoryPool &a, // NOLINT + const TrajectoryPool &b); + friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend TrajectoryPtrPool operator+(const TrajectoryPtrPool &a, + TrajectoryPool *b); + friend TrajectoryPtrPool operator-(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + + friend cv::Mat embedding_distance(const TrajectoryPool &a, + const TrajectoryPool &b); + friend cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend cv::Mat embedding_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b); + + friend cv::Mat mahalanobis_distance(const TrajectoryPool &a, + const TrajectoryPool &b); + friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend cv::Mat mahalanobis_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b); + + friend cv::Mat iou_distance(const TrajectoryPool &a, const TrajectoryPool &b); + friend cv::Mat iou_distance(const TrajectoryPtrPool &a, + const TrajectoryPtrPool &b); + friend cv::Mat iou_distance(const TrajectoryPtrPool &a, + const TrajectoryPool &b); + + private: + void update_embedding(const cv::Mat &embedding); + + public: + TrajectoryState state; + cv::Vec4f ltrb; + cv::Mat smooth_embedding; + int id; + bool is_activated; + int timestamp; + int starttime; + float score; + + private: + static int count; + cv::Vec4f xyah; + cv::Mat current_embedding; + float eta; + int length; +}; + +inline cv::Vec4f ltrb2xyah(const cv::Vec4f <rb) { + cv::Vec4f xyah; + xyah[0] = (ltrb[0] + ltrb[2]) * 0.5f; + xyah[1] = (ltrb[1] + ltrb[3]) * 0.5f; + xyah[3] = ltrb[3] - ltrb[1]; + xyah[2] = (ltrb[2] - ltrb[0]) / xyah[3]; + return xyah; +} + +inline Trajectory::Trajectory() + : state(New), + ltrb(cv::Vec4f()), + smooth_embedding(cv::Mat()), + id(0), + is_activated(false), + timestamp(0), + starttime(0), + score(0), + eta(0.9), + length(0) {} + +inline Trajectory::Trajectory(const cv::Vec4f <rb_, + float score_, + const cv::Mat &embedding) + : state(New), + ltrb(ltrb_), + smooth_embedding(cv::Mat()), + id(0), + is_activated(false), + timestamp(0), + starttime(0), + score(score_), + eta(0.9), + length(0) { + xyah = ltrb2xyah(ltrb); + update_embedding(embedding); +} + +inline Trajectory::Trajectory(const Trajectory &other) + : state(other.state), + ltrb(other.ltrb), + id(other.id), + is_activated(other.is_activated), + timestamp(other.timestamp), + starttime(other.starttime), + xyah(other.xyah), + score(other.score), + eta(other.eta), + length(other.length) { + other.smooth_embedding.copyTo(smooth_embedding); + other.current_embedding.copyTo(current_embedding); + // copy state in KalmanFilter + + other.statePre.copyTo(cv::KalmanFilter::statePre); + other.statePost.copyTo(cv::KalmanFilter::statePost); + other.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre); + other.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost); +} + +inline Trajectory &Trajectory::operator=(const Trajectory &rhs) { + this->state = rhs.state; + this->ltrb = rhs.ltrb; + rhs.smooth_embedding.copyTo(this->smooth_embedding); + this->id = rhs.id; + this->is_activated = rhs.is_activated; + this->timestamp = rhs.timestamp; + this->starttime = rhs.starttime; + this->xyah = rhs.xyah; + this->score = rhs.score; + rhs.current_embedding.copyTo(this->current_embedding); + this->eta = rhs.eta; + this->length = rhs.length; + + // copy state in KalmanFilter + + rhs.statePre.copyTo(cv::KalmanFilter::statePre); + rhs.statePost.copyTo(cv::KalmanFilter::statePost); + rhs.errorCovPre.copyTo(cv::KalmanFilter::errorCovPre); + rhs.errorCovPost.copyTo(cv::KalmanFilter::errorCovPost); + + return *this; +} + +inline int Trajectory::next_id() { + ++count; + return count; +} + +inline void Trajectory::mark_lost(void) { state = Lost; } + +inline void Trajectory::mark_removed(void) { state = Removed; } + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/pptracking/utils.h b/fastdeploy/vision/tracking/pptracking/utils.h new file mode 100644 index 00000000000..1d95b5765a9 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/utils.h @@ -0,0 +1,45 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "include/tracker.h" + + +namespace PaddleDetection { + +struct Rect { + float left; + float top; + float right; + float bottom; +}; + +struct MOTTrack { + int ids; + float score; + Rect rects; + int class_id = -1; +}; + +typedef std::vector MOTResult; + +} // namespace PaddleDetection diff --git a/fastdeploy/vision/visualize/mot.cc b/fastdeploy/vision/visualize/mot.cc new file mode 100644 index 00000000000..7ecd4e14f7e --- /dev/null +++ b/fastdeploy/vision/visualize/mot.cc @@ -0,0 +1,3 @@ +// +// Created by aichao on 2022/10/11. +// From f8f6998be73baea10fea4dbeaa6f00c12713987b Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Wed, 12 Oct 2022 14:53:20 +0800 Subject: [PATCH 06/25] add tracking py_bind and example --- .../tracking/pptracking/cpp/CMakeLists.txt | 14 ++ .../vision/tracking/pptracking/cpp/infer.cc | 128 ++++++++++++++++++ .../tracking/pptracking/python/infer.py | 57 ++++++++ .../tracking/pptracking/pptracking_pybind.cc | 60 ++++++++ fastdeploy/vision/tracking/pptracking/utils.h | 45 ------ fastdeploy/vision/tracking/tracking_pybind.cc | 34 +++++ python/fastdeploy/vision/tracking/__init__.py | 0 .../vision/tracking/pptracking/__init__.py | 0 8 files changed, 293 insertions(+), 45 deletions(-) create mode 100644 examples/vision/tracking/pptracking/cpp/CMakeLists.txt create mode 100644 examples/vision/tracking/pptracking/cpp/infer.cc create mode 100644 examples/vision/tracking/pptracking/python/infer.py create mode 100644 fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc delete mode 100644 fastdeploy/vision/tracking/pptracking/utils.h create mode 100644 fastdeploy/vision/tracking/tracking_pybind.cc create mode 100644 python/fastdeploy/vision/tracking/__init__.py create mode 100644 python/fastdeploy/vision/tracking/pptracking/__init__.py diff --git a/examples/vision/tracking/pptracking/cpp/CMakeLists.txt b/examples/vision/tracking/pptracking/cpp/CMakeLists.txt new file mode 100644 index 00000000000..93540a7e83e --- /dev/null +++ b/examples/vision/tracking/pptracking/cpp/CMakeLists.txt @@ -0,0 +1,14 @@ +PROJECT(infer_demo C CXX) +CMAKE_MINIMUM_REQUIRED (VERSION 3.10) + +# 指定下载解压后的fastdeploy库路径 +option(FASTDEPLOY_INSTALL_DIR "Path of downloaded fastdeploy sdk.") + +include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake) + +# 添加FastDeploy依赖头文件 +include_directories(${FASTDEPLOY_INCS}) + +add_executable(infer_demo ${PROJECT_SOURCE_DIR}/infer.cc) +# 添加FastDeploy库依赖 +target_link_libraries(infer_demo ${FASTDEPLOY_LIBS}) diff --git a/examples/vision/tracking/pptracking/cpp/infer.cc b/examples/vision/tracking/pptracking/cpp/infer.cc new file mode 100644 index 00000000000..b953db63ef2 --- /dev/null +++ b/examples/vision/tracking/pptracking/cpp/infer.cc @@ -0,0 +1,128 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision.h" + +#ifdef WIN32 +const char sep = '\\'; +#else +const char sep = '/'; +#endif + +void CpuInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto config_file = model_dir + sep + "deploy.yaml"; + auto model = fastdeploy::vision::segmentation::PaddleSegModel( + model_file, params_file, config_file); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + + fastdeploy::vision::SegmentationResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} + +void GpuInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto config_file = model_dir + sep + "deploy.yaml"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + auto model = fastdeploy::vision::segmentation::PaddleSegModel( + model_file, params_file, config_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + + fastdeploy::vision::SegmentationResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} + +void TrtInfer(const std::string& model_dir, const std::string& image_file) { + auto model_file = model_dir + sep + "model.pdmodel"; + auto params_file = model_dir + sep + "model.pdiparams"; + auto config_file = model_dir + sep + "deploy.yaml"; + + auto option = fastdeploy::RuntimeOption(); + option.UseGpu(); + option.UseTrtBackend(); + auto model = fastdeploy::vision::segmentation::PaddleSegModel( + model_file, params_file, config_file, option); + + if (!model.Initialized()) { + std::cerr << "Failed to initialize." << std::endl; + return; + } + + auto im = cv::imread(image_file); + + fastdeploy::vision::SegmentationResult res; + if (!model.Predict(&im, &res)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + + std::cout << res.Str() << std::endl; + auto vis_im = fastdeploy::vision::VisSegmentation(im, res); + cv::imwrite("vis_result.jpg", vis_im); + std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; +} + +int main(int argc, char* argv[]) { + if (argc < 4) { + std::cout + << "Usage: infer_demo path/to/model_dir path/to/image run_option, " + "e.g ./infer_model ./ppseg_model_dir ./test.jpeg 0" + << std::endl; + std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " + "with gpu; 2: run with gpu and use tensorrt backend." + << std::endl; + return -1; + } + + if (std::atoi(argv[3]) == 0) { + CpuInfer(argv[1], argv[2]); + } else if (std::atoi(argv[3]) == 1) { + GpuInfer(argv[1], argv[2]); + } else if (std::atoi(argv[3]) == 2) { + TrtInfer(argv[1], argv[2]); + } + return 0; +} diff --git a/examples/vision/tracking/pptracking/python/infer.py b/examples/vision/tracking/pptracking/python/infer.py new file mode 100644 index 00000000000..dd65ebf2735 --- /dev/null +++ b/examples/vision/tracking/pptracking/python/infer.py @@ -0,0 +1,57 @@ +import fastdeploy as fd +import cv2 +import os + + +def parse_arguments(): + import argparse + import ast + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", required=True, help="Path of PaddleSeg model.") + parser.add_argument( + "--image", type=str, required=True, help="Path of test image file.") + parser.add_argument( + "--device", + type=str, + default='cpu', + help="Type of inference device, support 'cpu' or 'gpu'.") + parser.add_argument( + "--use_trt", + type=ast.literal_eval, + default=False, + help="Wether to use tensorrt.") + return parser.parse_args() + + +def build_option(args): + option = fd.RuntimeOption() + + if args.device.lower() == "gpu": + option.use_gpu() + + if args.use_trt: + option.use_trt_backend() + option.set_trt_input_shape("x", [1, 3, 256, 256], [1, 3, 1024, 1024], + [1, 3, 2048, 2048]) + return option + + +args = parse_arguments() + +# 配置runtime，加载模型 +runtime_option = build_option(args) +model_file = os.path.join(args.model, "model.pdmodel") +params_file = os.path.join(args.model, "model.pdiparams") +config_file = os.path.join(args.model, "deploy.yaml") +model = fd.vision.segmentation.PaddleSegModel( + model_file, params_file, config_file, runtime_option=runtime_option) + +# 预测图片分割结果 +im = cv2.imread(args.image) +result = model.predict(im.copy()) +print(result) + +# 可视化结果 +vis_im = fd.vision.visualize.vis_segmentation(im, result) +cv2.imwrite("vis_img.png", vis_im) diff --git a/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc b/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc new file mode 100644 index 00000000000..2e4e3e85ba2 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc @@ -0,0 +1,60 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindPPOCRModel(pybind11::module& m) { + // DBDetector + pybind11::class_(m, "DBDetector") + .def(pybind11::init()) + .def(pybind11::init<>()) + + .def_readwrite("max_side_len", &vision::ocr::DBDetector::max_side_len) + .def_readwrite("det_db_thresh", &vision::ocr::DBDetector::det_db_thresh) + .def_readwrite("det_db_box_thresh", + &vision::ocr::DBDetector::det_db_box_thresh) + .def_readwrite("det_db_unclip_ratio", + &vision::ocr::DBDetector::det_db_unclip_ratio) + .def_readwrite("det_db_score_mode", + &vision::ocr::DBDetector::det_db_score_mode) + .def_readwrite("use_dilation", &vision::ocr::DBDetector::use_dilation) + .def_readwrite("mean", &vision::ocr::DBDetector::mean) + .def_readwrite("scale", &vision::ocr::DBDetector::scale) + .def_readwrite("is_scale", &vision::ocr::DBDetector::is_scale); + + // Classifier + pybind11::class_(m, "Classifier") + .def(pybind11::init()) + .def(pybind11::init<>()) + + .def_readwrite("cls_thresh", &vision::ocr::Classifier::cls_thresh) + .def_readwrite("cls_image_shape", + &vision::ocr::Classifier::cls_image_shape) + .def_readwrite("cls_batch_num", &vision::ocr::Classifier::cls_batch_num); + + // Recognizer + pybind11::class_(m, "Recognizer") + + .def(pybind11::init()) + .def(pybind11::init<>()) + + .def_readwrite("rec_img_h", &vision::ocr::Recognizer::rec_img_h) + .def_readwrite("rec_img_w", &vision::ocr::Recognizer::rec_img_w) + .def_readwrite("rec_batch_num", &vision::ocr::Recognizer::rec_batch_num); +} +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/utils.h b/fastdeploy/vision/tracking/pptracking/utils.h deleted file mode 100644 index 1d95b5765a9..00000000000 --- a/fastdeploy/vision/tracking/pptracking/utils.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "include/tracker.h" - - -namespace PaddleDetection { - -struct Rect { - float left; - float top; - float right; - float bottom; -}; - -struct MOTTrack { - int ids; - float score; - Rect rects; - int class_id = -1; -}; - -typedef std::vector MOTResult; - -} // namespace PaddleDetection diff --git a/fastdeploy/vision/tracking/tracking_pybind.cc b/fastdeploy/vision/tracking/tracking_pybind.cc new file mode 100644 index 00000000000..768ce9d9528 --- /dev/null +++ b/fastdeploy/vision/tracking/tracking_pybind.cc @@ -0,0 +1,34 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "fastdeploy/pybind/main.h" + +namespace fastdeploy { +void BindPPTracking(pybind11::module &m) { + pybind11::class_( + m, "PPTracking") + .def(pybind11::init()) + .def("predict", + [](vision::tracking::PPTracking &self, + pybind11::array &data) { + auto mat = PyArrayToCvMat(data); + vision::MOTResult *res = new vision::MOTResult(); + self.Predict(&mat, res); + return res; + }) + .def("__repr__",[](vision::tracking::PPTracking &self){ + return self.ModelName(); + }) +} +} // namespace fastdeploy diff --git a/python/fastdeploy/vision/tracking/__init__.py b/python/fastdeploy/vision/tracking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/python/fastdeploy/vision/tracking/pptracking/__init__.py b/python/fastdeploy/vision/tracking/pptracking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 9f79627f0e19b2477fb123481fc2aee0ac95974b Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Wed, 12 Oct 2022 17:23:52 +0800 Subject: [PATCH 07/25] add pptracking --- python/fastdeploy/libs/__init__.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 python/fastdeploy/libs/__init__.py diff --git a/python/fastdeploy/libs/__init__.py b/python/fastdeploy/libs/__init__.py deleted file mode 100644 index 8b137891791..00000000000 --- a/python/fastdeploy/libs/__init__.py +++ /dev/null @@ -1 +0,0 @@ - From 6b9a5c4c47edc7829b0dab5b1f7fbebba66bc8a4 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Wed, 12 Oct 2022 17:25:13 +0800 Subject: [PATCH 08/25] add pptracking --- benchmark/run_benchmark_yolo.sh | 0 .../vision/tracking/pptracking/cpp/infer.cc | 118 ++++-- .../tracking/pptracking/python/infer.py | 44 +- fastdeploy/vision.h | 1 + fastdeploy/vision/common/result.cc | 20 + fastdeploy/vision/common/result.h | 17 + .../vision/detection/ppdet/jde_track.cpp | 5 - fastdeploy/vision/detection/ppdet/jde_track.h | 16 - .../vision/tracking/pptracking/lapjv.cc | 2 +- fastdeploy/vision/tracking/pptracking/lapjv.h | 12 +- .../vision/tracking/pptracking/model.cc | 401 ++++++++++++------ fastdeploy/vision/tracking/pptracking/model.h | 108 ++++- .../tracking/pptracking/pptracking_pybind.cc | 55 +-- .../vision/tracking/pptracking/tracker.cc | 12 +- .../vision/tracking/pptracking/tracker.h | 10 +- .../vision/tracking/pptracking/trajectory.cc | 10 +- .../vision/tracking/pptracking/trajectory.h | 8 +- fastdeploy/vision/tracking/tracking_pybind.cc | 26 +- fastdeploy/vision/vision_pybind.cc | 13 + fastdeploy/vision/visualize/mot.cc | 98 ++++- fastdeploy/vision/visualize/ocr.cc | 1 - fastdeploy/vision/visualize/visualize.h | 3 + .../vision/visualize/visualize_pybind.cc | 16 + python/fastdeploy/vision/__init__.py | 2 +- python/fastdeploy/vision/tracking/__init__.py | 16 + .../vision/tracking/pptracking/__init__.py | 37 ++ .../fastdeploy/vision/visualize/__init__.py | 4 + 27 files changed, 744 insertions(+), 311 deletions(-) mode change 100644 => 100755 benchmark/run_benchmark_yolo.sh delete mode 100644 fastdeploy/vision/detection/ppdet/jde_track.cpp delete mode 100644 fastdeploy/vision/detection/ppdet/jde_track.h diff --git a/benchmark/run_benchmark_yolo.sh b/benchmark/run_benchmark_yolo.sh old mode 100644 new mode 100755 diff --git a/examples/vision/tracking/pptracking/cpp/infer.cc b/examples/vision/tracking/pptracking/cpp/infer.cc index b953db63ef2..709159eb423 100644 --- a/examples/vision/tracking/pptracking/cpp/infer.cc +++ b/examples/vision/tracking/pptracking/cpp/infer.cc @@ -20,11 +20,11 @@ const char sep = '\\'; const char sep = '/'; #endif -void CpuInfer(const std::string& model_dir, const std::string& image_file) { +void CpuInfer(const std::string& model_dir, const std::string& video_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; - auto config_file = model_dir + sep + "deploy.yaml"; - auto model = fastdeploy::vision::segmentation::PaddleSegModel( + auto config_file = model_dir + sep + "infer_cfg.yml"; + auto model = fastdeploy::vision::tracking::PPTracking( model_file, params_file, config_file); if (!model.Initialized()) { @@ -32,28 +32,38 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file) { return; } - auto im = cv::imread(image_file); - - fastdeploy::vision::SegmentationResult res; - if (!model.Predict(&im, &res)) { - std::cerr << "Failed to predict." << std::endl; - return; + fastdeploy::vision::MOTResult result; + cv::Mat frame; + int frame_id=0; + cv::VideoCapture capture(video_file); + // according to the time of prediction to calculate fps + float fps= 0.0f; + while (capture.read(frame)) { + if (frame.empty()) { + break; + } + if (!model.Predict(&frame, &result)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + // std::cout << result.Str() << std::endl; + cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, fps , frame_id); + cv::imshow("mot",out_img); + cv::waitKey(30); + frame_id++; } - - std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::VisSegmentation(im, res); - cv::imwrite("vis_result.jpg", vis_im); - std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + capture.release(); + cv::destroyAllWindows(); } -void GpuInfer(const std::string& model_dir, const std::string& image_file) { +void GpuInfer(const std::string& model_dir, const std::string& video_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; - auto config_file = model_dir + sep + "deploy.yaml"; + auto config_file = model_dir + sep + "infer_cfg.yml"; auto option = fastdeploy::RuntimeOption(); option.UseGpu(); - auto model = fastdeploy::vision::segmentation::PaddleSegModel( + auto model = fastdeploy::vision::tracking::PPTracking( model_file, params_file, config_file, option); if (!model.Initialized()) { @@ -61,29 +71,39 @@ void GpuInfer(const std::string& model_dir, const std::string& image_file) { return; } - auto im = cv::imread(image_file); - - fastdeploy::vision::SegmentationResult res; - if (!model.Predict(&im, &res)) { - std::cerr << "Failed to predict." << std::endl; - return; + fastdeploy::vision::MOTResult result; + cv::Mat frame; + int frame_id=0; + cv::VideoCapture capture(video_file); + // according to the time of prediction to calculate fps + float fps= 0.0f; + while (capture.read(frame)) { + if (frame.empty()) { + break; + } + if (!model.Predict(&frame, &result)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + // std::cout << result.Str() << std::endl; + cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, fps , frame_id); + cv::imshow("mot",out_img); + cv::waitKey(30); + frame_id++; } - - std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::VisSegmentation(im, res); - cv::imwrite("vis_result.jpg", vis_im); - std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + capture.release(); + cv::destroyAllWindows(); } -void TrtInfer(const std::string& model_dir, const std::string& image_file) { +void TrtInfer(const std::string& model_dir, const std::string& video_file) { auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; - auto config_file = model_dir + sep + "deploy.yaml"; + auto config_file = model_dir + sep + "infer_cfg.yml"; auto option = fastdeploy::RuntimeOption(); option.UseGpu(); option.UseTrtBackend(); - auto model = fastdeploy::vision::segmentation::PaddleSegModel( + auto model = fastdeploy::vision::tracking::PPTracking( model_file, params_file, config_file, option); if (!model.Initialized()) { @@ -91,25 +111,35 @@ void TrtInfer(const std::string& model_dir, const std::string& image_file) { return; } - auto im = cv::imread(image_file); - - fastdeploy::vision::SegmentationResult res; - if (!model.Predict(&im, &res)) { - std::cerr << "Failed to predict." << std::endl; - return; + fastdeploy::vision::MOTResult result; + cv::Mat frame; + int frame_id=0; + cv::VideoCapture capture(video_file); + // according to the time of prediction to calculate fps + float fps= 0.0f; + while (capture.read(frame)) { + if (frame.empty()) { + break; + } + if (!model.Predict(&frame, &result)) { + std::cerr << "Failed to predict." << std::endl; + return; + } + // std::cout << result.Str() << std::endl; + cv::Mat out_img = fastdeploy::vision::VisMOT(frame, result, fps , frame_id); + cv::imshow("mot",out_img); + cv::waitKey(30); + frame_id++; } - - std::cout << res.Str() << std::endl; - auto vis_im = fastdeploy::vision::VisSegmentation(im, res); - cv::imwrite("vis_result.jpg", vis_im); - std::cout << "Visualized result saved in ./vis_result.jpg" << std::endl; + capture.release(); + cv::destroyAllWindows(); } int main(int argc, char* argv[]) { if (argc < 4) { std::cout - << "Usage: infer_demo path/to/model_dir path/to/image run_option, " - "e.g ./infer_model ./ppseg_model_dir ./test.jpeg 0" + << "Usage: infer_demo path/to/model_dir path/to/video run_option, " + "e.g ./infer_model ./pptracking_model_dir ./person.mp4 0" << std::endl; std::cout << "The data type of run_option is int, 0: run with cpu; 1: run " "with gpu; 2: run with gpu and use tensorrt backend." diff --git a/examples/vision/tracking/pptracking/python/infer.py b/examples/vision/tracking/pptracking/python/infer.py index dd65ebf2735..6ad91737d9f 100644 --- a/examples/vision/tracking/pptracking/python/infer.py +++ b/examples/vision/tracking/pptracking/python/infer.py @@ -1,5 +1,20 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import fastdeploy as fd import cv2 +import time import os @@ -10,7 +25,7 @@ def parse_arguments(): parser.add_argument( "--model", required=True, help="Path of PaddleSeg model.") parser.add_argument( - "--image", type=str, required=True, help="Path of test image file.") + "--video", type=str, required=True, help="Path of test video file.") parser.add_argument( "--device", type=str, @@ -43,15 +58,24 @@ def build_option(args): runtime_option = build_option(args) model_file = os.path.join(args.model, "model.pdmodel") params_file = os.path.join(args.model, "model.pdiparams") -config_file = os.path.join(args.model, "deploy.yaml") -model = fd.vision.segmentation.PaddleSegModel( +config_file = os.path.join(args.model, "infer_cfg.yml") +model = fd.vision.tracking.PPTracking( model_file, params_file, config_file, runtime_option=runtime_option) # 预测图片分割结果 -im = cv2.imread(args.image) -result = model.predict(im.copy()) -print(result) - -# 可视化结果 -vis_im = fd.vision.visualize.vis_segmentation(im, result) -cv2.imwrite("vis_img.png", vis_im) +cap = cv2.VideoCapture(args.video) +frame_id = 0 +while True: + start_time = time.time() + frame_id = frame_id+1 + _, frame = cap.read() + if frame is None: + break + result = model.predict(frame) + end_time = time.time() + fps = 1.0/(end_time-start_time) + img = fd.vision.vis_mot(frame, result, fps, frame_id) + cv2.imshow("video", img) + cv2.waitKey(30) +cap.release() +cv2.destroyAllWindows() diff --git a/fastdeploy/vision.h b/fastdeploy/vision.h index e7590f82859..3c232be46fa 100644 --- a/fastdeploy/vision.h +++ b/fastdeploy/vision.h @@ -45,6 +45,7 @@ #include "fastdeploy/vision/ocr/ppocr/ppocr_system_v3.h" #include "fastdeploy/vision/ocr/ppocr/recognizer.h" #include "fastdeploy/vision/segmentation/ppseg/model.h" +#include "fastdeploy/vision/tracking/pptracking/model.h" #endif #include "fastdeploy/vision/visualize/visualize.h" diff --git a/fastdeploy/vision/common/result.cc b/fastdeploy/vision/common/result.cc index d4f586d6aa2..6a32104e4f2 100644 --- a/fastdeploy/vision/common/result.cc +++ b/fastdeploy/vision/common/result.cc @@ -125,6 +125,26 @@ void OCRResult::Clear() { cls_labels.clear(); } +void MOTResult::Clear(){ + boxes.clear(); + ids.clear(); + scores.clear(); + class_ids.clear(); +} + +std::string MOTResult::Str(){ + std::string out; + out = "MOTResult:\nall boxes counts: "+std::to_string(boxes.size())+"\n"; + out += "[xmin\tymin\txmax\tymax\tid\tscore]\n"; + for (size_t i = 0; i < boxes.size(); ++i) { + out = out + "["+ std::to_string(boxes[i][0]) + "\t" + + std::to_string(boxes[i][1]) + "\t" + std::to_string(boxes[i][2]) + + "\t" + std::to_string(boxes[i][3]) + "\t" + + std::to_string(ids[i]) + "\t" + std::to_string(scores[i]) + "]\n"; + } + return out; +} + FaceDetectionResult::FaceDetectionResult(const FaceDetectionResult& res) { boxes.assign(res.boxes.begin(), res.boxes.end()); landmarks.assign(res.landmarks.begin(), res.landmarks.end()); diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index b3dc922cb17..02e9b12dfe8 100644 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -26,6 +26,7 @@ enum FASTDEPLOY_DECL ResultType { DETECTION, SEGMENTATION, OCR, + MOT, FACE_DETECTION, FACE_RECOGNITION, MATTING, @@ -130,6 +131,21 @@ struct FASTDEPLOY_DECL OCRResult : public BaseResult { std::string Str(); }; +struct FASTDEPLOY_DECL MOTResult : public BaseResult { + // left top right bottom + std::vector> boxes; + std::vector ids; + std::vector scores; + std::vector class_ids; + ResultType type = ResultType::MOT; + + void Clear(); + + std::string Str(); +}; + + + struct FASTDEPLOY_DECL FaceDetectionResult : public BaseResult { // box: xmin, ymin, xmax, ymax std::vector> boxes; @@ -215,5 +231,6 @@ struct FASTDEPLOY_DECL MattingResult : public BaseResult { std::string Str(); }; + } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/detection/ppdet/jde_track.cpp b/fastdeploy/vision/detection/ppdet/jde_track.cpp deleted file mode 100644 index ac5580771f8..00000000000 --- a/fastdeploy/vision/detection/ppdet/jde_track.cpp +++ /dev/null @@ -1,5 +0,0 @@ -// -// Created by aichao on 2022/10/8. -// - -#include "jde_track.h" diff --git a/fastdeploy/vision/detection/ppdet/jde_track.h b/fastdeploy/vision/detection/ppdet/jde_track.h deleted file mode 100644 index 6509a991ec5..00000000000 --- a/fastdeploy/vision/detection/ppdet/jde_track.h +++ /dev/null @@ -1,16 +0,0 @@ -// -// Created by aichao on 2022/10/8. -// - -#ifndef FASTDEPLOY_JDE_TRACK_H -#define FASTDEPLOY_JDE_TRACK_H - - - -class jde_track { - -}; - - - -#endif //FASTDEPLOY_JDE_TRACK_H diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.cc b/fastdeploy/vision/tracking/pptracking/lapjv.cc index c943d391aa6..292f2a996a6 100644 --- a/fastdeploy/vision/tracking/pptracking/lapjv.cc +++ b/fastdeploy/vision/tracking/pptracking/lapjv.cc @@ -21,7 +21,7 @@ #include #include -#include "include/lapjv.h" +#include "fastdeploy/vision/tracking/pptracking/lapjv.h" namespace fastdeploy { namespace vision { diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.h b/fastdeploy/vision/tracking/pptracking/lapjv.h index ffaa010c005..38bd37bcb4b 100644 --- a/fastdeploy/vision/tracking/pptracking/lapjv.h +++ b/fastdeploy/vision/tracking/pptracking/lapjv.h @@ -17,8 +17,7 @@ // Ths copyright of gatagat/lap is as follows: // MIT License -#ifndef DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ -#define DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ +#pragma once #define LARGE 1000000 #if !defined TRUE @@ -45,7 +44,9 @@ } #include -namespace PaddleDetection { +namespace fastdeploy { +namespace vision { +namespace tracking { typedef signed int int_t; typedef unsigned int uint_t; @@ -59,6 +60,7 @@ int lapjv_internal(const cv::Mat &cost, int *x, int *y); -} // namespace PaddleDetection +} // namespace tracking +} // namespace vision +} // namespace fastdeploy -#endif // DEPLOY_PPTRACKING_CPP_INCLUDE_LAPJV_H_ diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index 13c12a4e338..2cb133c191d 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -1,45 +1,210 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // -// Created by aichao on 2022/10/10. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // - -#include "fastdeploy/vision/tracking/pptracking/predictor.h" +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/tracking/pptracking/model.h" +#include "yaml-cpp/yaml.h" +#ifdef ENABLE_PADDLE_FRONTEND +#include "paddle2onnx/converter.h" +#endif namespace fastdeploy { namespace vision { namespace tracking { -PPTracking::PPTracking(const std::string& model_file, const std::string& params_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format){ - if (model_format == ModelFormat::ONNX) - { - valid_cpu_backends = {Backend::ORT,Backend::OPENVINO}; - valid_gpu_backends = {Backend::ORT, Backend::TRT}; - } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::OPENVINO}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; - } +PPTracking::PPTracking(const std::string& model_file, + const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format){ + config_file_=config_file; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_gpu_backends = {Backend::PDINFER, Backend::ORT}; + runtime_option = custom_option; runtime_option.model_format = model_format; runtime_option.model_file = model_file; runtime_option.params_file = params_file; - std::cout<<"--------"<(); + } else { + FDERROR << "Please set draw_threshold." << std::endl; + return false; + } +// Get config for tracker + if (cfg["tracker"].IsDefined()) { + if (cfg["tracker"]["conf_thres"].IsDefined()) { + conf_thresh_ = cfg["tracker"]["conf_thres"].as(); + } + else { + std::cerr << "Please set conf_thres in tracker." << std::endl; + return false; + } + if (cfg["tracker"]["min_box_area"].IsDefined()) { + min_box_area_ = cfg["tracker"]["min_box_area"].as(); + } + if (cfg["tracker"]["tracked_thresh"].IsDefined()) { + tracked_thresh_ = cfg["tracker"]["tracked_thresh"].as(); + } + } + + + + processors_.push_back(std::make_shared()); + for (const auto& op : cfg["Preprocess"]) { + std::string op_name = op["type"].as(); + if (op_name == "Resize") { + bool keep_ratio = op["keep_ratio"].as(); + auto target_size = op["target_size"].as>(); + int interp = op["interp"].as(); + FDASSERT(target_size.size() == 2, + "Require size of target_size be 2, but now it's %lu.", + target_size.size()); + if (!keep_ratio) { + int width = target_size[1]; + int height = target_size[0]; + processors_.push_back( + std::make_shared(width, height, -1.0, -1.0, interp, false)); + } else { + int min_target_size = std::min(target_size[0], target_size[1]); + int max_target_size = std::max(target_size[0], target_size[1]); + std::vector max_size; + if (max_target_size > 0) { + max_size.push_back(max_target_size); + max_size.push_back(max_target_size); + } + processors_.push_back(std::make_shared( + min_target_size, interp, true, max_size)); + } + + } + else if(op_name == "LetterBoxResize"){ + auto target_size = op["target_size"].as>(); + FDASSERT(target_size.size() == 2,"Require size of target_size be 2, but now it's %lu.", + target_size.size()); + std::vector color{127.0f,127.0f,127.0f}; + if (op["fill_value"].IsDefined()){ + color =op["fill_value"].as>(); + } + processors_.push_back(std::make_shared(target_size, color)); + } + else if (op_name == "NormalizeImage") { + auto mean = op["mean"].as>(); + auto std = op["std"].as>(); + bool is_scale = true; + if (op["is_scale"]) { + is_scale = op["is_scale"].as(); + } + std::string norm_type = "mean_std"; + if (op["norm_type"]) { + norm_type = op["norm_type"].as(); + } + if (norm_type != "mean_std") { + std::fill(mean.begin(), mean.end(), 0.0); + std::fill(std.begin(), std.end(), 1.0); + } + processors_.push_back(std::make_shared(mean, std, is_scale)); + } + else if (op_name == "Permute") { + // Do nothing, do permute as the last operation + continue; + // processors_.push_back(std::make_shared()); + } else if (op_name == "Pad") { + auto size = op["size"].as>(); + auto value = op["fill_value"].as>(); + processors_.push_back(std::make_shared("float")); + processors_.push_back( + std::make_shared(size[1], size[0], value)); + } else if (op_name == "PadStride") { + auto stride = op["stride"].as(); + processors_.push_back( + std::make_shared(stride, std::vector(3, 0))); + } else { + FDERROR << "Unexcepted preprocess operator: " << op_name << "." + << std::endl; + return false; + } + } + processors_.push_back(std::make_shared()); + return true; +} + +void PPTracking::GetNmsInfo() { +#ifdef ENABLE_PADDLE_FRONTEND + if (runtime_option.model_format == ModelFormat::PADDLE) { + std::string contents; + if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { + return; + } + auto reader = paddle2onnx::PaddleReader(contents.c_str(), contents.size()); + if (reader.has_nms) { + has_nms_ = true; + background_label = reader.nms_params.background_label; + keep_top_k = reader.nms_params.keep_top_k; + nms_eta = reader.nms_params.nms_eta; + nms_threshold = reader.nms_params.nms_threshold; + score_threshold = reader.nms_params.score_threshold; + nms_top_k = reader.nms_params.nms_top_k; + normalized = reader.nms_params.normalized; + } + } +#endif +} + bool PPTracking::Initialize() { - target_size_ ={320,576}; - mean_={0.0f,0.0f,0.0f}; - scale_={1.0f,1.0f,1.0f}; - conf_thresh_=0.4; - threshold_=0.5; - min_box_area_=200; +#ifdef ENABLE_PADDLE_FRONTEND + // remove multiclass_nms3 now + // this is a trick operation for ppyoloe while inference on trt + GetNmsInfo(); + runtime_option.remove_multiclass_nms_ = true; + runtime_option.custom_op_info_["multiclass_nms3"] = "MultiClassNMS"; +#endif + if (!BuildPreprocessPipelineFromConfig()) { + FDERROR << "Failed to build preprocess pipeline from configuration file." + << std::endl; + return false; + } if (!InitRuntime()) { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; return false; } + + if (has_nms_ && runtime_option.backend == Backend::TRT) { + FDINFO << "Detected operator multiclass_nms3 in your model, will replace " + "it with fastdeploy::backend::MultiClassNMS(background_label=" + << background_label << ", keep_top_k=" << keep_top_k + << ", nms_eta=" << nms_eta << ", nms_threshold=" << nms_threshold + << ", score_threshold=" << score_threshold + << ", nms_top_k=" << nms_top_k << ", normalized=" << normalized + << ")." << std::endl; + has_nms_ = false; + } return true; } @@ -51,8 +216,6 @@ bool PPTracking::Predict(cv::Mat *img, MOTResult *result) { FDERROR << "Failed to preprocess input image." << std::endl; return false; } - float* tmp = static_cast(input_tensors[1].Data()); - std::vector output_tensors; if (!Infer(input_tensors, &output_tensors)) { FDERROR << "Failed to inference." << std::endl; @@ -63,139 +226,109 @@ bool PPTracking::Predict(cv::Mat *img, MOTResult *result) { FDERROR << "Failed to post process." << std::endl; return false; } - return true; - - } -void PPTracking::LetterBoxResize(Mat* im){ - - // generate scale_factor - int origin_w = im->Width(); - int origin_h = im->Height(); - int target_h = target_size_[0]; - int target_w = target_size_[1]; - float ratio_h = static_cast(target_h) / static_cast(origin_h); - float ratio_w = static_cast(target_w) / static_cast(origin_w); - float resize_scale = std::min(ratio_h, ratio_w); - - int new_shape_w = std::round(im->Width() * resize_scale); - int new_shape_h = std::round(im->Height() * resize_scale); - float padw = (target_size_[1] - new_shape_w) / 2.; - float padh = (target_size_[0] - new_shape_h) / 2.; - int top = std::round(padh - 0.1); - int bottom = std::round(padh + 0.1); - int left = std::round(padw - 0.1); - int right = std::round(padw + 0.1); - - Resize::Run(im,new_shape_w,new_shape_h); - std::vector color{127.5,127.5,127.5}; - Pad::Run(im,top,bottom,left,right,color); -} bool PPTracking::Preprocess(Mat* mat, std::vector* outputs) { - int origin_w = mat->Width(); - int origin_h = mat->Height(); - LetterBoxResize(mat); - Normalize::Run(mat,mean_,scale_,is_scale_); - HWC2CHW::Run(mat); - Cast::Run(mat, "float"); - - outputs->resize(3); - // image_shape - (*outputs)[0].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(0).name); - float* shape = static_cast((*outputs)[0].MutableData()); - shape[0] = mat->Height(); - shape[1] = mat->Width(); - // image - (*outputs)[1].name = InputInfoOfRuntime(1).name; - mat->ShareWithTensor(&((*outputs)[1])); - (*outputs)[1].ExpandDim(0); - // scale - (*outputs)[2].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(2).name); - float* scale = static_cast((*outputs)[2].MutableData()); - scale[0] = mat->Height() * 1.0 / origin_h; - scale[1] = mat->Width() * 1.0 / origin_w; - return true; + int origin_w = mat->Width(); + int origin_h = mat->Height(); + + for (size_t i = 0; i < processors_.size(); ++i) { + if (!(*(processors_[i].get()))(mat)) { + FDERROR << "Failed to process image data in " << processors_[i]->Name() + << "." << std::endl; + return false; + } + } + +// LetterBoxResize(mat); +// Normalize::Run(mat,mean_,scale_,is_scale_); +// HWC2CHW::Run(mat); + Cast::Run(mat, "float"); + + outputs->resize(3); + // image_shape + (*outputs)[0].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(0).name); + float* shape = static_cast((*outputs)[0].MutableData()); + shape[0] = mat->Height(); + shape[1] = mat->Width(); + // image + (*outputs)[1].name = InputInfoOfRuntime(1).name; + mat->ShareWithTensor(&((*outputs)[1])); + (*outputs)[1].ExpandDim(0); + // scale + (*outputs)[2].Allocate({1, 2}, FDDataType::FP32, InputInfoOfRuntime(2).name); + float* scale = static_cast((*outputs)[2].MutableData()); + scale[0] = mat->Height() * 1.0 / origin_h; + scale[1] = mat->Width() * 1.0 / origin_w; + return true; } -void FilterDets(const float conf_thresh,const cv::Mat dets,std::vector* index) { +void FilterDets(const float conf_thresh,const cv::Mat& dets,std::vector* index) { for (int i = 0; i < dets.rows; ++i) { float score = *dets.ptr(i, 4); - if (score > conf_thresh) { - index->push_back(i); - } + if (score > conf_thresh) { + index->push_back(i); + } } } bool PPTracking::Postprocess(std::vector& infer_result, MOTResult *result){ - auto bbox_shape = infer_result[0].shape; - auto bbox_data = static_cast(infer_result[0].Data()); + auto bbox_shape = infer_result[0].shape; + auto bbox_data = static_cast(infer_result[0].Data()); - auto emb_shape = infer_result[1].shape; - auto emb_data = static_cast(infer_result[1].Data()); + auto emb_shape = infer_result[1].shape; + auto emb_data = static_cast(infer_result[1].Data()); - cv::Mat dets(bbox_shape[0], 6, CV_32FC1, bbox_data); - cv::Mat emb(bbox_shape[0], emb_shape[1], CV_32FC1, emb_data); + cv::Mat dets(bbox_shape[0], 6, CV_32FC1, bbox_data); + cv::Mat emb(bbox_shape[0], emb_shape[1], CV_32FC1, emb_data); - result->clear(); - std::vector tracks; - std::vector valid; - FilterDets(conf_thresh_, dets, &valid); - cv::Mat new_dets, new_emb; - for (int i = 0; i < valid.size(); ++i) { - new_dets.push_back(dets.row(valid[i])); - new_emb.push_back(emb.row(valid[i])); - } - JDETracker::instance()->update(new_dets, new_emb, &tracks); - if (tracks.size() == 0) { - MOTTrack mot_track; - Rect ret = {*dets.ptr(0, 0), - *dets.ptr(0, 1), - *dets.ptr(0, 2), - *dets.ptr(0, 3)}; - mot_track.ids = 1; - mot_track.score = *dets.ptr(0, 4); - mot_track.rects = ret; - result->push_back(mot_track); - } else { - std::vector::iterator titer; - for (titer = tracks.begin(); titer != tracks.end(); ++titer) { - if (titer->score < threshold_) { - continue; - } else { - float w = titer->ltrb[2] - titer->ltrb[0]; - float h = titer->ltrb[3] - titer->ltrb[1]; - bool vertical = w / h > 1.6; - float area = w * h; - if (area > min_box_area_ && !vertical) { - MOTTrack mot_track; - Rect ret = { - titer->ltrb[0], titer->ltrb[1], titer->ltrb[2], titer->ltrb[3]}; - mot_track.rects = ret; - mot_track.score = titer->score; - mot_track.ids = titer->id; - result->push_back(mot_track); - } - } + result->Clear(); + std::vector tracks; + std::vector valid; + FilterDets(conf_thresh_, dets, &valid); + cv::Mat new_dets, new_emb; + for (int i = 0; i < valid.size(); ++i) { + new_dets.push_back(dets.row(valid[i])); + new_emb.push_back(emb.row(valid[i])); + } + JDETracker::instance()->update(new_dets, new_emb, &tracks); + if (tracks.size() == 0) { + std::array box={int(*dets.ptr(0, 0)), + int(*dets.ptr(0, 1)), + int(*dets.ptr(0, 2)), + int(*dets.ptr(0, 3))}; + result->boxes.push_back(box); + result->ids.push_back(1); + result->scores.push_back(*dets.ptr(0, 4)); + + } else { + std::vector::iterator titer; + for (titer = tracks.begin(); titer != tracks.end(); ++titer) { + if (titer->score < tracked_thresh_) { + continue; + } else { + float w = titer->ltrb[2] - titer->ltrb[0]; + float h = titer->ltrb[3] - titer->ltrb[1]; + bool vertical = w / h > 1.6; + float area = w * h; + if (area > min_box_area_ && !vertical) { + std::array box = { + int(titer->ltrb[0]), int(titer->ltrb[1]), int(titer->ltrb[2]), int(titer->ltrb[3])}; + result->boxes.push_back(box); + result->ids.push_back(titer->id); + result->scores.push_back(titer->score); } + } } - return true; -} - -cv::Mat PPTracking::Visualize(const cv::Mat& img, - const MOTResult& results, - const float fps, - const int frame_id){ - -return VisualizeTrackResult(img,results,fps,frame_id); - + } + return true; } } // namespace tracking diff --git a/fastdeploy/vision/tracking/pptracking/model.h b/fastdeploy/vision/tracking/pptracking/model.h index 7b11705ee0f..18641cf6fef 100644 --- a/fastdeploy/vision/tracking/pptracking/model.h +++ b/fastdeploy/vision/tracking/pptracking/model.h @@ -1,52 +1,116 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // -// Created by aichao on 2022/10/10. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #pragma once #include "fastdeploy/vision/common/processors/transform.h" #include "fastdeploy/fastdeploy_model.h" -#include "fastdeploy/vision/tracking/pptracking/utils.h" -#include "fastdeploy/vision/tracking/pptracking/postprocess.h" +#include "fastdeploy/vision/common/result.h" +#include "fastdeploy/vision/tracking/pptracking/tracker.h" namespace fastdeploy { namespace vision { namespace tracking { +class LetterBoxResize: public Processor{ +public: + LetterBoxResize(const std::vector& target_size, const std::vector& color){ + target_size_=target_size; + color_=color; + } + bool ImplByOpenCV(Mat* mat) override{ + + if (mat->Channels() != color_.size()) { + FDERROR << "Pad: Require input channels equals to size of padding value, " + "but now channels = " + << mat->Channels() + << ", the size of padding values = " << color_.size() << "." + << std::endl; + return false; + } + // generate scale_factor + int origin_w = mat->Width(); + int origin_h = mat->Height(); + int target_h = target_size_[0]; + int target_w = target_size_[1]; + float ratio_h = static_cast(target_h) / static_cast(origin_h); + float ratio_w = static_cast(target_w) / static_cast(origin_w); + float resize_scale = std::min(ratio_h, ratio_w); + + int new_shape_w = std::round(mat->Width() * resize_scale); + int new_shape_h = std::round(mat->Height() * resize_scale); + float padw = (target_size_[1] - new_shape_w) / 2.; + float padh = (target_size_[0] - new_shape_h) / 2.; + int top = std::round(padh - 0.1); + int bottom = std::round(padh + 0.1); + int left = std::round(padw - 0.1); + int right = std::round(padw + 0.1); + + Resize::Run(mat,new_shape_w,new_shape_h); + Pad::Run(mat,top,bottom,left,right,color_); + return true; + } + std::string Name() override { return "LetterBoxResize"; } + +private: + std::vector target_size_; + std::vector color_; +}; + class FASTDEPLOY_DECL PPTracking: public FastDeployModel { - public: - PPTracking(const std::string& model_file, const std::string& params_file = "", - const RuntimeOption& custom_option = RuntimeOption(), - const ModelFormat& model_format = ModelFormat::PADDLE); +public: + PPTracking(const std::string& model_file, + const std::string& params_file, + const std::string& config_file, + const RuntimeOption& custom_option = RuntimeOption(), + const ModelFormat& model_format = ModelFormat::PADDLE); - std::string ModelName() const { return "pptracking/jde"; } + std::string ModelName() const override { return "pptracking"; } virtual bool Predict(cv::Mat* img, MOTResult* result); - static cv::Mat Visualize(const cv::Mat& img, - const MOTResult& results, - float fps, - int frame_id); - private: - std::vector mean_; - std::vector scale_; - bool is_scale_ = true; - std::vector target_size_; - float conf_thresh_; - float threshold_; - float min_box_area_; +private: + bool BuildPreprocessPipelineFromConfig(); bool Initialize(); + void GetNmsInfo(); - - void LetterBoxResize(Mat* im); +// void LetterBoxResize(Mat* im); bool Preprocess(Mat* img, std::vector* outputs); bool Postprocess(std::vector& infer_result, MOTResult *result); + std::vector> processors_; + std::string config_file_; + float draw_threshold_; + float conf_thresh_; + float tracked_thresh_; + float min_box_area_; + bool is_scale_ = true; + + // configuration for nms + int64_t background_label = -1; + int64_t keep_top_k = 300; + float nms_eta = 1.0; + float nms_threshold = 0.7; + float score_threshold = 0.01; + int64_t nms_top_k = 10000; + bool normalized = true; + bool has_nms_ = true; }; diff --git a/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc b/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc index 2e4e3e85ba2..d56437ad504 100644 --- a/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc +++ b/fastdeploy/vision/tracking/pptracking/pptracking_pybind.cc @@ -11,50 +11,21 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -#include #include "fastdeploy/pybind/main.h" namespace fastdeploy { -void BindPPOCRModel(pybind11::module& m) { - // DBDetector - pybind11::class_(m, "DBDetector") - .def(pybind11::init()) - .def(pybind11::init<>()) - - .def_readwrite("max_side_len", &vision::ocr::DBDetector::max_side_len) - .def_readwrite("det_db_thresh", &vision::ocr::DBDetector::det_db_thresh) - .def_readwrite("det_db_box_thresh", - &vision::ocr::DBDetector::det_db_box_thresh) - .def_readwrite("det_db_unclip_ratio", - &vision::ocr::DBDetector::det_db_unclip_ratio) - .def_readwrite("det_db_score_mode", - &vision::ocr::DBDetector::det_db_score_mode) - .def_readwrite("use_dilation", &vision::ocr::DBDetector::use_dilation) - .def_readwrite("mean", &vision::ocr::DBDetector::mean) - .def_readwrite("scale", &vision::ocr::DBDetector::scale) - .def_readwrite("is_scale", &vision::ocr::DBDetector::is_scale); - - // Classifier - pybind11::class_(m, "Classifier") - .def(pybind11::init()) - .def(pybind11::init<>()) - - .def_readwrite("cls_thresh", &vision::ocr::Classifier::cls_thresh) - .def_readwrite("cls_image_shape", - &vision::ocr::Classifier::cls_image_shape) - .def_readwrite("cls_batch_num", &vision::ocr::Classifier::cls_batch_num); - - // Recognizer - pybind11::class_(m, "Recognizer") - - .def(pybind11::init()) - .def(pybind11::init<>()) - - .def_readwrite("rec_img_h", &vision::ocr::Recognizer::rec_img_h) - .def_readwrite("rec_img_w", &vision::ocr::Recognizer::rec_img_w) - .def_readwrite("rec_batch_num", &vision::ocr::Recognizer::rec_batch_num); +void BindPPTracking(pybind11::module &m) { + pybind11::class_( + m, "PPTracking") + .def(pybind11::init()) + .def("predict", + [](vision::tracking::PPTracking &self, + pybind11::array &data) { + auto mat = PyArrayToCvMat(data); + vision::MOTResult *res = new vision::MOTResult(); + self.Predict(&mat, res); + return res; + }); } } // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc index 9540e39f670..f7940d2a80e 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.cc +++ b/fastdeploy/vision/tracking/pptracking/tracker.cc @@ -22,8 +22,8 @@ #include #include -#include "include/lapjv.h" -#include "include/tracker.h" +#include "fastdeploy/vision/tracking/pptracking/lapjv.h" +#include "fastdeploy/vision/tracking/pptracking/tracker.h" #define mat2vec4f(m) \ cv::Vec4f(*m.ptr(0, 0), \ @@ -31,7 +31,9 @@ *m.ptr(0, 2), \ *m.ptr(0, 3)) -namespace PaddleDetection { +namespace fastdeploy { +namespace vision { +namespace tracking { static std::map chi2inv95 = {{1, 3.841459f}, {2, 5.991465f}, @@ -301,4 +303,6 @@ void JDETracker::remove_duplicate_trajectory(TrajectoryPool *a, } } -} // namespace PaddleDetection +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h index 244530f140a..349a83ef965 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.h +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -25,9 +25,11 @@ #include #include #include -#include "include/trajectory.h" +#include "fastdeploy/vision/tracking/pptracking/trajectory.h" -namespace PaddleDetection { +namespace fastdeploy { +namespace vision { +namespace tracking { typedef std::map Match; typedef std::map::iterator MatchIterator; @@ -69,4 +71,6 @@ class JDETracker { float det_thresh; }; -} // namespace PaddleDetection +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.cc b/fastdeploy/vision/tracking/pptracking/trajectory.cc index 0ff2e1a5fc7..4c8c6e076a3 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.cc +++ b/fastdeploy/vision/tracking/pptracking/trajectory.cc @@ -17,10 +17,12 @@ // Ths copyright of CnybTseng/JDE is as follows: // MIT License -#include "include/trajectory.h" +#include "fastdeploy/vision/tracking/pptracking/trajectory.h" #include -namespace PaddleDetection { +namespace fastdeploy { +namespace vision { +namespace tracking { void TKalmanFilter::init(const cv::Mat &measurement) { measurement.copyTo(statePost(cv::Rect(0, 0, 1, 4))); @@ -514,4 +516,6 @@ cv::Mat iou_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b) { return dists; } -} // namespace PaddleDetection +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h index c21e8cac368..e22d8323767 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -26,7 +26,9 @@ #include #include "opencv2/video/tracking.hpp" -namespace PaddleDetection { +namespace fastdeploy { +namespace vision { +namespace tracking { typedef enum { New = 0, Tracked = 1, Lost = 2, Removed = 3 } TrajectoryState; @@ -227,4 +229,6 @@ inline void Trajectory::mark_lost(void) { state = Lost; } inline void Trajectory::mark_removed(void) { state = Removed; } -} // namespace PaddleDetection +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/tracking_pybind.cc b/fastdeploy/vision/tracking/tracking_pybind.cc index 768ce9d9528..7341c85f27b 100644 --- a/fastdeploy/vision/tracking/tracking_pybind.cc +++ b/fastdeploy/vision/tracking/tracking_pybind.cc @@ -11,24 +11,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include "fastdeploy/pybind/main.h" namespace fastdeploy { -void BindPPTracking(pybind11::module &m) { - pybind11::class_( - m, "PPTracking") - .def(pybind11::init()) - .def("predict", - [](vision::tracking::PPTracking &self, - pybind11::array &data) { - auto mat = PyArrayToCvMat(data); - vision::MOTResult *res = new vision::MOTResult(); - self.Predict(&mat, res); - return res; - }) - .def("__repr__",[](vision::tracking::PPTracking &self){ - return self.ModelName(); - }) -} + + void BindPPTracking(pybind11::module& m); + + void BindTracking(pybind11::module& m) { + auto tracking_module = + m.def_submodule("tracking", "object tracking models."); + BindPPTracking(tracking_module); + } } // namespace fastdeploy diff --git a/fastdeploy/vision/vision_pybind.cc b/fastdeploy/vision/vision_pybind.cc index dbade6c82c0..20b28cfac93 100644 --- a/fastdeploy/vision/vision_pybind.cc +++ b/fastdeploy/vision/vision_pybind.cc @@ -23,6 +23,7 @@ void BindMatting(pybind11::module& m); void BindFaceDet(pybind11::module& m); void BindFaceId(pybind11::module& m); void BindOcr(pybind11::module& m); +void BindTracking(pybind11::module& m); #ifdef ENABLE_VISION_VISUALIZE void BindVisualize(pybind11::module& m); #endif @@ -62,6 +63,15 @@ void BindVision(pybind11::module& m) { .def("__repr__", &vision::OCRResult::Str) .def("__str__", &vision::OCRResult::Str); + pybind11::class_(m, "MOTResult") + .def(pybind11::init()) + .def_readwrite("boxes", &vision::MOTResult::boxes) + .def_readwrite("ids", &vision::MOTResult::ids) + .def_readwrite("scores", &vision::MOTResult::scores) + .def_readwrite("class_ids", &vision::MOTResult::class_ids) + .def("__repr__", &vision::MOTResult::Str) + .def("__str__", &vision::MOTResult::Str); + pybind11::class_(m, "FaceDetectionResult") .def(pybind11::init()) .def_readwrite("boxes", &vision::FaceDetectionResult::boxes) @@ -102,6 +112,9 @@ void BindVision(pybind11::module& m) { BindFaceId(m); BindMatting(m); BindOcr(m); + BindTracking(m); + + #ifdef ENABLE_VISION_VISUALIZE BindVisualize(m); #endif diff --git a/fastdeploy/vision/visualize/mot.cc b/fastdeploy/vision/visualize/mot.cc index 7ecd4e14f7e..ff1eac2e9b8 100644 --- a/fastdeploy/vision/visualize/mot.cc +++ b/fastdeploy/vision/visualize/mot.cc @@ -1,3 +1,99 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // -// Created by aichao on 2022/10/11. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef ENABLE_VISION_VISUALIZE +#include "fastdeploy/vision/visualize/visualize.h" + +namespace fastdeploy { +namespace vision { + +cv::Scalar GetMOTBoxColor(int idx) { + idx = idx * 3; + cv::Scalar color = cv::Scalar((37 * idx) % 255, (17 * idx) % 255, (29 * idx) % 255); + return color; +} + + +cv::Mat VisMOT(const cv::Mat &img, const MOTResult &results, float fps, int frame_id) { + + cv::Mat vis_img = img.clone(); + int im_h = img.rows; + int im_w = img.cols; + float text_scale = std::max(1, static_cast(im_w / 1600.)); + float text_thickness = 2.; + float line_thickness = std::max(1, static_cast(im_w / 500.)); + + std::ostringstream oss; + oss << std::setiosflags(std::ios::fixed) << std::setprecision(4); + oss << "frame: " << frame_id << " "; + oss << "fps: " << fps << " "; + oss << "num: " << results.boxes.size(); + std::string text = oss.str(); + + cv::Point origin; + origin.x = 0; + origin.y = static_cast(15 * text_scale); + cv::putText(vis_img, + text, + origin, + cv::FONT_HERSHEY_PLAIN, + text_scale, + cv::Scalar(0, 0, 255), + text_thickness); + + for (int i = 0; i < results.boxes.size(); ++i) { + const int obj_id = results.ids[i]; + const float score = results.scores[i]; + + cv::Scalar color = GetMOTBoxColor(obj_id); + + cv::Point pt1 = cv::Point(results.boxes[i][0], results.boxes[i][1]); + cv::Point pt2 = cv::Point(results.boxes[i][2], results.boxes[i][3]); + cv::Point id_pt = + cv::Point(results.boxes[i][0], results.boxes[i][1] + 10); + cv::Point score_pt = + cv::Point(results.boxes[i][0], results.boxes[i][1] - 10); + cv::rectangle(vis_img, pt1, pt2, color, line_thickness); + + std::ostringstream idoss; + idoss << std::setiosflags(std::ios::fixed) << std::setprecision(4); + idoss << obj_id; + std::string id_text = idoss.str(); + + cv::putText(vis_img, + id_text, + id_pt, + cv::FONT_HERSHEY_PLAIN, + text_scale, + cv::Scalar(0, 255, 255), + text_thickness); + + std::ostringstream soss; + soss << std::setiosflags(std::ios::fixed) << std::setprecision(2); + soss << score; + std::string score_text = soss.str(); + + cv::putText(vis_img, + score_text, + score_pt, + cv::FONT_HERSHEY_PLAIN, + text_scale, + cv::Scalar(0, 255, 255), + text_thickness); + } + return vis_img; +} +}// namespace vision +} //namespace fastdepoly +#endif diff --git a/fastdeploy/vision/visualize/ocr.cc b/fastdeploy/vision/visualize/ocr.cc index 5a47b49096e..ac8f3631246 100644 --- a/fastdeploy/vision/visualize/ocr.cc +++ b/fastdeploy/vision/visualize/ocr.cc @@ -15,7 +15,6 @@ #ifdef ENABLE_VISION_VISUALIZE #include "fastdeploy/vision/visualize/visualize.h" -#include "opencv2/imgproc/imgproc.hpp" namespace fastdeploy { namespace vision { diff --git a/fastdeploy/vision/visualize/visualize.h b/fastdeploy/vision/visualize/visualize.h index 2af747676a2..ef456b941f4 100644 --- a/fastdeploy/vision/visualize/visualize.h +++ b/fastdeploy/vision/visualize/visualize.h @@ -65,6 +65,9 @@ FASTDEPLOY_DECL cv::Mat VisMatting(const cv::Mat& im, const MattingResult& result, bool remove_small_connected_area = false); FASTDEPLOY_DECL cv::Mat VisOcr(const cv::Mat& im, const OCRResult& ocr_result); + +FASTDEPLOY_DECL cv::Mat VisMOT(const cv::Mat& img,const MOTResult& results, float fps=0.0, int frame_id=0); + FASTDEPLOY_DECL cv::Mat SwapBackground( const cv::Mat& im, const cv::Mat& background, const MattingResult& result, bool remove_small_connected_area = false); diff --git a/fastdeploy/vision/visualize/visualize_pybind.cc b/fastdeploy/vision/visualize/visualize_pybind.cc index 7b429a62949..3da7a2e33e4 100644 --- a/fastdeploy/vision/visualize/visualize_pybind.cc +++ b/fastdeploy/vision/visualize/visualize_pybind.cc @@ -75,6 +75,14 @@ void BindVisualize(pybind11::module& m) { vision::Mat(vis_im).ShareWithTensor(&out); return TensorToPyArray(out); }) + .def("vis_mot", + [](pybind11::array& im_data, vision::MOTResult& result,float fps, int frame_id) { + auto im = PyArrayToCvMat(im_data); + auto vis_im = vision::VisMOT(im, result,fps,frame_id); + FDTensor out; + vision::Mat(vis_im).ShareWithTensor(&out); + return TensorToPyArray(out); + }) .def("vis_matting", [](pybind11::array& im_data, vision::MattingResult& result, bool remove_small_connected_area) { @@ -155,6 +163,14 @@ void BindVisualize(pybind11::module& m) { vision::Mat(vis_im).ShareWithTensor(&out); return TensorToPyArray(out); }) + .def_static("vis_mot", + [](pybind11::array& im_data, vision::MOTResult& result,float fps, int frame_id) { + auto im = PyArrayToCvMat(im_data); + auto vis_im = vision::VisMOT(im, result,fps,frame_id); + FDTensor out; + vision::Mat(vis_im).ShareWithTensor(&out); + return TensorToPyArray(out); + }) .def_static("vis_matting_alpha", [](pybind11::array& im_data, vision::MattingResult& result, bool remove_small_connected_area) { diff --git a/python/fastdeploy/vision/__init__.py b/python/fastdeploy/vision/__init__.py index 86f4f978f55..ae82a1cbdf2 100644 --- a/python/fastdeploy/vision/__init__.py +++ b/python/fastdeploy/vision/__init__.py @@ -16,7 +16,7 @@ from . import detection from . import classification from . import segmentation - +from . import tracking from . import matting from . import facedet from . import faceid diff --git a/python/fastdeploy/vision/tracking/__init__.py b/python/fastdeploy/vision/tracking/__init__.py index e69de29bb2d..946dfd97168 100644 --- a/python/fastdeploy/vision/tracking/__init__.py +++ b/python/fastdeploy/vision/tracking/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import + +from .pptracking import PPTracking diff --git a/python/fastdeploy/vision/tracking/pptracking/__init__.py b/python/fastdeploy/vision/tracking/pptracking/__init__.py index e69de29bb2d..9da0d669fa9 100644 --- a/python/fastdeploy/vision/tracking/pptracking/__init__.py +++ b/python/fastdeploy/vision/tracking/pptracking/__init__.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from .... import FastDeployModel, ModelFormat +from .... import c_lib_wrap as C + + +class PPTracking(FastDeployModel): + def __init__(self, + model_file, + params_file, + config_file, + runtime_option=None, + model_format=ModelFormat.PADDLE): + super(PPTracking, self).__init__(runtime_option) + + assert model_format == ModelFormat.PADDLE, "PPTracking model only support model format of ModelFormat.Paddle now." + self._model = C.vision.tracking.PPTracking( + model_file, params_file, config_file, self._runtime_option, + model_format) + assert self.initialized, "PPTracking model initialize failed." + + def predict(self, input_image): + assert input_image is not None, "The input image data is None." + return self._model.predict(input_image) diff --git a/python/fastdeploy/vision/visualize/__init__.py b/python/fastdeploy/vision/visualize/__init__.py index 60912d66e17..5b078ec8fa5 100644 --- a/python/fastdeploy/vision/visualize/__init__.py +++ b/python/fastdeploy/vision/visualize/__init__.py @@ -93,3 +93,7 @@ def swap_background(im_data, def vis_ppocr(im_data, det_result): return C.vision.vis_ppocr(im_data, det_result) + + +def vis_mot(im_data, mot_result, fps, frame_id): + return C.vision.vis_mot(im_data, mot_result, fps=0.0, frame_id=0) From d800402a2576007ee5ec79955ce248c37624f509 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Thu, 13 Oct 2022 08:46:33 +0800 Subject: [PATCH 09/25] iomanip head file --- fastdeploy/vision/visualize/mot.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/fastdeploy/vision/visualize/mot.cc b/fastdeploy/vision/visualize/mot.cc index ff1eac2e9b8..9877b2d4e08 100644 --- a/fastdeploy/vision/visualize/mot.cc +++ b/fastdeploy/vision/visualize/mot.cc @@ -14,6 +14,7 @@ #ifdef ENABLE_VISION_VISUALIZE #include "fastdeploy/vision/visualize/visualize.h" +#include namespace fastdeploy { namespace vision { From c92e5d52141e522e81d8841fbef550fdec55d2e0 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Thu, 13 Oct 2022 09:45:56 +0800 Subject: [PATCH 10/25] add opencv_video lib --- cmake/opencv.cmake | 2 +- python/scripts/process_libraries.py.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/opencv.cmake b/cmake/opencv.cmake index 7557e682cbc..4b2f092932a 100644 --- a/cmake/opencv.cmake +++ b/cmake/opencv.cmake @@ -186,6 +186,6 @@ else() endif() find_package(OpenCV REQUIRED PATHS ${OpenCV_DIR}) include_directories(${OpenCV_INCLUDE_DIRS}) - list(APPEND DEPEND_LIBS opencv_core opencv_highgui opencv_imgproc opencv_imgcodecs) + list(APPEND DEPEND_LIBS opencv_core opencv_video opencv_highgui opencv_imgproc opencv_imgcodecs) endif() endif() diff --git a/python/scripts/process_libraries.py.in b/python/scripts/process_libraries.py.in index 66581e4f0dd..395c0297968 100644 --- a/python/scripts/process_libraries.py.in +++ b/python/scripts/process_libraries.py.in @@ -46,7 +46,7 @@ def process_on_linux(current_dir): if len(items) != 4: os.remove(os.path.join(root, f)) continue - if items[0].strip() not in ["libopencv_highgui", "libopencv_videoio", "libopencv_imgcodecs", "libopencv_imgproc", "libopencv_core"]: + if items[0].strip() not in ["libopencv_highgui", "libopencv_video", "libopencv_videoio", "libopencv_imgcodecs", "libopencv_imgproc", "libopencv_core"]: os.remove(os.path.join(root, f)) all_libs_paths = [third_libs_path] + user_specified_dirs From a0b7f67e8d1f78d0647d15e33c815c8444d06aaf Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Thu, 13 Oct 2022 10:44:16 +0800 Subject: [PATCH 11/25] add python libs package Signed-off-by: ChaoII <849453582@qq.com> --- python/fastdeploy/libs/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 python/fastdeploy/libs/__init__.py diff --git a/python/fastdeploy/libs/__init__.py b/python/fastdeploy/libs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From a4dedb88c1a9817d41536ec7730bc291e1282850 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Thu, 13 Oct 2022 16:41:54 +0800 Subject: [PATCH 12/25] complete comments Signed-off-by: ChaoII <849453582@qq.com> --- .../vision/tracking/pptracking/lapjv.cc | 2 +- fastdeploy/vision/tracking/pptracking/lapjv.h | 2 +- .../vision/tracking/pptracking/letter_box.cc | 59 +++++++++++++++++ .../vision/tracking/pptracking/letter_box.h | 36 ++++++++++ .../vision/tracking/pptracking/model.cc | 6 +- fastdeploy/vision/tracking/pptracking/model.h | 65 +++++-------------- .../vision/tracking/pptracking/tracker.cc | 2 +- .../vision/tracking/pptracking/tracker.h | 2 +- .../vision/tracking/pptracking/trajectory.cc | 2 +- .../vision/tracking/pptracking/trajectory.h | 2 +- 10 files changed, 119 insertions(+), 59 deletions(-) create mode 100644 fastdeploy/vision/tracking/pptracking/letter_box.cc create mode 100644 fastdeploy/vision/tracking/pptracking/letter_box.h diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.cc b/fastdeploy/vision/tracking/pptracking/lapjv.cc index 292f2a996a6..32546b11c63 100644 --- a/fastdeploy/vision/tracking/pptracking/lapjv.cc +++ b/fastdeploy/vision/tracking/pptracking/lapjv.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fastdeploy/vision/tracking/pptracking/lapjv.h b/fastdeploy/vision/tracking/pptracking/lapjv.h index 38bd37bcb4b..b8e122204a4 100644 --- a/fastdeploy/vision/tracking/pptracking/lapjv.h +++ b/fastdeploy/vision/tracking/pptracking/lapjv.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fastdeploy/vision/tracking/pptracking/letter_box.cc b/fastdeploy/vision/tracking/pptracking/letter_box.cc new file mode 100644 index 00000000000..bc8e0cbcb26 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/letter_box.cc @@ -0,0 +1,59 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/tracking/pptracking/letter_box.h" + +namespace fastdeploy { +namespace vision { +namespace tracking { + +LetterBoxResize::LetterBoxResize(const std::vector& target_size, const std::vector& color){ + target_size_=target_size; + color_=color; +} +bool LetterBoxResize::ImplByOpenCV(Mat* mat){ + if (mat->Channels() != color_.size()) { + FDERROR << "Pad: Require input channels equals to size of padding value, " + "but now channels = " + << mat->Channels() + << ", the size of padding values = " << color_.size() << "." + << std::endl; + return false; + } + // generate scale_factor + int origin_w = mat->Width(); + int origin_h = mat->Height(); + int target_h = target_size_[0]; + int target_w = target_size_[1]; + float ratio_h = static_cast(target_h) / static_cast(origin_h); + float ratio_w = static_cast(target_w) / static_cast(origin_w); + float resize_scale = std::min(ratio_h, ratio_w); + + int new_shape_w = std::round(mat->Width() * resize_scale); + int new_shape_h = std::round(mat->Height() * resize_scale); + float padw = (target_size_[1] - new_shape_w) / 2.; + float padh = (target_size_[0] - new_shape_h) / 2.; + int top = std::round(padh - 0.1); + int bottom = std::round(padh + 0.1); + int left = std::round(padw - 0.1); + int right = std::round(padw + 0.1); + + Resize::Run(mat,new_shape_w,new_shape_h); + Pad::Run(mat,top,bottom,left,right,color_); + return true; +} + +} // namespace tracking +} // namespace vision +} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/tracking/pptracking/letter_box.h b/fastdeploy/vision/tracking/pptracking/letter_box.h new file mode 100644 index 00000000000..17f9f283383 --- /dev/null +++ b/fastdeploy/vision/tracking/pptracking/letter_box.h @@ -0,0 +1,36 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/transform.h" +#include "fastdeploy/fastdeploy_model.h" +namespace fastdeploy { +namespace vision { +namespace tracking { +class LetterBoxResize: public Processor{ +public: + LetterBoxResize(const std::vector& target_size, const std::vector& color); + bool ImplByOpenCV(Mat* mat) override; + std::string Name() override { return "LetterBoxResize"; } +private: + std::vector target_size_; + std::vector color_; +}; + +} // namespace tracking +} // namespace vision +} // namespace fastdeploy + + diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index 2cb133c191d..e204869d54d 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -56,7 +56,7 @@ bool PPTracking::BuildPreprocessPipelineFromConfig(){ FDERROR << "Please set draw_threshold." << std::endl; return false; } -// Get config for tracker + // Get config for tracker if (cfg["tracker"].IsDefined()) { if (cfg["tracker"]["conf_thres"].IsDefined()) { conf_thresh_ = cfg["tracker"]["conf_thres"].as(); @@ -73,8 +73,6 @@ bool PPTracking::BuildPreprocessPipelineFromConfig(){ } } - - processors_.push_back(std::make_shared()); for (const auto& op : cfg["Preprocess"]) { std::string op_name = op["type"].as(); @@ -277,8 +275,6 @@ void FilterDets(const float conf_thresh,const cv::Mat& dets,std::vector* in } bool PPTracking::Postprocess(std::vector& infer_result, MOTResult *result){ - - auto bbox_shape = infer_result[0].shape; auto bbox_data = static_cast(infer_result[0].Data()); diff --git a/fastdeploy/vision/tracking/pptracking/model.h b/fastdeploy/vision/tracking/pptracking/model.h index 18641cf6fef..a9a396e00fc 100644 --- a/fastdeploy/vision/tracking/pptracking/model.h +++ b/fastdeploy/vision/tracking/pptracking/model.h @@ -13,83 +13,52 @@ // limitations under the License. #pragma once + #include "fastdeploy/vision/common/processors/transform.h" #include "fastdeploy/fastdeploy_model.h" #include "fastdeploy/vision/common/result.h" #include "fastdeploy/vision/tracking/pptracking/tracker.h" +#include "fastdeploy/vision/tracking/pptracking/letter_box.h" namespace fastdeploy { namespace vision { namespace tracking { -class LetterBoxResize: public Processor{ -public: - LetterBoxResize(const std::vector& target_size, const std::vector& color){ - target_size_=target_size; - color_=color; - } - bool ImplByOpenCV(Mat* mat) override{ - - if (mat->Channels() != color_.size()) { - FDERROR << "Pad: Require input channels equals to size of padding value, " - "but now channels = " - << mat->Channels() - << ", the size of padding values = " << color_.size() << "." - << std::endl; - return false; - } - // generate scale_factor - int origin_w = mat->Width(); - int origin_h = mat->Height(); - int target_h = target_size_[0]; - int target_w = target_size_[1]; - float ratio_h = static_cast(target_h) / static_cast(origin_h); - float ratio_w = static_cast(target_w) / static_cast(origin_w); - float resize_scale = std::min(ratio_h, ratio_w); - - int new_shape_w = std::round(mat->Width() * resize_scale); - int new_shape_h = std::round(mat->Height() * resize_scale); - float padw = (target_size_[1] - new_shape_w) / 2.; - float padh = (target_size_[0] - new_shape_h) / 2.; - int top = std::round(padh - 0.1); - int bottom = std::round(padh + 0.1); - int left = std::round(padw - 0.1); - int right = std::round(padw + 0.1); - - Resize::Run(mat,new_shape_w,new_shape_h); - Pad::Run(mat,top,bottom,left,right,color_); - return true; - } - std::string Name() override { return "LetterBoxResize"; } - -private: - std::vector target_size_; - std::vector color_; -}; - class FASTDEPLOY_DECL PPTracking: public FastDeployModel { public: + /** \brief Set path of model file and configuration file, and the configuration of runtime + * + * \param[in] model_file Path of model file, e.g pptracking/model.pdmodel + * \param[in] params_file Path of parameter file, e.g pptracking/model.pdiparams, if the model format is ONNX, this parameter will be ignored + * \param[in] config_file Path of configuration file for deployment, e.g pptracking/infer_cfg.yml + * \param[in] custom_option RuntimeOption for inference, the default will use cpu, and choose the backend defined in `valid_cpu_backends` + * \param[in] model_format Model format of the loaded model, default is Paddle format + */ PPTracking(const std::string& model_file, const std::string& params_file, const std::string& config_file, const RuntimeOption& custom_option = RuntimeOption(), const ModelFormat& model_format = ModelFormat::PADDLE); + /// Get model's name std::string ModelName() const override { return "pptracking"; } + /** \brief Predict the detection result for an input image(consecutive) + * + * \param[in] im The input image data which is consecutive frame, comes from imread() or videoCapture.read() + * \param[in] result The output tracking result will be writen to this structure + * \return true if the prediction successed, otherwise false + */ virtual bool Predict(cv::Mat* img, MOTResult* result); - private: bool BuildPreprocessPipelineFromConfig(); bool Initialize(); void GetNmsInfo(); -// void LetterBoxResize(Mat* im); - bool Preprocess(Mat* img, std::vector* outputs); bool Postprocess(std::vector& infer_result, MOTResult *result); diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc index f7940d2a80e..e4bdc62819b 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.cc +++ b/fastdeploy/vision/tracking/pptracking/tracker.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h index 349a83ef965..23db00b5813 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.h +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.cc b/fastdeploy/vision/tracking/pptracking/trajectory.cc index 4c8c6e076a3..d40b2df7708 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.cc +++ b/fastdeploy/vision/tracking/pptracking/trajectory.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h index e22d8323767..f722d828fe9 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 95523291c99d9a14f6c72093eefa41625d330d40 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Mon, 17 Oct 2022 16:11:12 +0800 Subject: [PATCH 13/25] add jdeTracker_ member variable Signed-off-by: ChaoII <849453582@qq.com> --- fastdeploy/vision/facedet/contrib/scrfd.cc | 2 +- fastdeploy/vision/tracking/pptracking/model.cc | 7 ++++++- fastdeploy/vision/tracking/pptracking/model.h | 1 + fastdeploy/vision/tracking/pptracking/tracker.cc | 5 +---- fastdeploy/vision/tracking/pptracking/tracker.h | 9 +++++---- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/fastdeploy/vision/facedet/contrib/scrfd.cc b/fastdeploy/vision/facedet/contrib/scrfd.cc index d87d66068c4..7d69744104d 100644 --- a/fastdeploy/vision/facedet/contrib/scrfd.cc +++ b/fastdeploy/vision/facedet/contrib/scrfd.cc @@ -63,7 +63,7 @@ SCRFD::SCRFD(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { if (model_format == ModelFormat::ONNX) { - valid_cpu_backends = {Backend::ORT}; + valid_cpu_backends = {Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index e204869d54d..6a7e906a3fa 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -203,6 +203,11 @@ bool PPTracking::Initialize() { << ")." << std::endl; has_nms_ = false; } + + // create JDETracker instance + std::unique_ptr jdeTracker(new JDETracker); + jdeTracker_ = std::move(jdeTracker); + return true; } @@ -294,7 +299,7 @@ bool PPTracking::Postprocess(std::vector& infer_result, MOTResult *res new_dets.push_back(dets.row(valid[i])); new_emb.push_back(emb.row(valid[i])); } - JDETracker::instance()->update(new_dets, new_emb, &tracks); + jdeTracker_->update(new_dets, new_emb, &tracks); if (tracks.size() == 0) { std::array box={int(*dets.ptr(0, 0)), int(*dets.ptr(0, 1)), diff --git a/fastdeploy/vision/tracking/pptracking/model.h b/fastdeploy/vision/tracking/pptracking/model.h index a9a396e00fc..040ed383f13 100644 --- a/fastdeploy/vision/tracking/pptracking/model.h +++ b/fastdeploy/vision/tracking/pptracking/model.h @@ -70,6 +70,7 @@ class FASTDEPLOY_DECL PPTracking: public FastDeployModel { float tracked_thresh_; float min_box_area_; bool is_scale_ = true; + std::unique_ptr jdeTracker_; // configuration for nms int64_t background_label = -1; diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc index e4bdc62819b..9c1df243527 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.cc +++ b/fastdeploy/vision/tracking/pptracking/tracker.cc @@ -45,11 +45,8 @@ static std::map chi2inv95 = {{1, 3.841459f}, {8, 15.507313f}, {9, 16.918978f}}; -JDETracker *JDETracker::me = new JDETracker; -JDETracker *JDETracker::instance(void) { return me; } - -JDETracker::JDETracker(void) +JDETracker::JDETracker() : timestamp(0), max_lost_time(30), lambda(0.98f), det_thresh(0.3f) {} bool JDETracker::update(const cv::Mat &dets, diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h index 23db00b5813..1256d06201d 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.h +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -42,14 +42,15 @@ struct Track { class JDETracker { public: - static JDETracker *instance(void); + + JDETracker(); + virtual bool update(const cv::Mat &dets, const cv::Mat &emb, std::vector *tracks); - + virtual ~JDETracker() {} private: - JDETracker(void); - virtual ~JDETracker(void) {} + cv::Mat motion_distance(const TrajectoryPtrPool &a, const TrajectoryPool &b); void linear_assignment(const cv::Mat &cost, float cost_limit, From 0a180214cef3c9fe01340d3155f09957663b6c7c Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Tue, 18 Oct 2022 08:43:31 +0800 Subject: [PATCH 14/25] add 'FASTDEPLOY_DECL' macro Signed-off-by: ChaoII <849453582@qq.com> --- .gitignore | 1 + fastdeploy/vision/tracking/pptracking/tracker.h | 4 ++-- fastdeploy/vision/tracking/pptracking/trajectory.h | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 5df9ed5dd7e..38274eead1e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ fpython/astdeploy/libs/third_libs fastdeploy/core/config.h fastdeploy/pybind/main.cc python/fastdeploy/libs/lib* +python/fastdeploy/libs/third_libs __pycache__ build_fd_android.sh python/scripts/process_libraries.py diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h index 1256d06201d..29652154e21 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.h +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -25,6 +25,7 @@ #include #include #include +#include "fastdeploy/fastdeploy_model.h" #include "fastdeploy/vision/tracking/pptracking/trajectory.h" namespace fastdeploy { @@ -40,7 +41,7 @@ struct Track { cv::Vec4f ltrb; }; -class JDETracker { +class FASTDEPLOY_DECL JDETracker { public: JDETracker(); @@ -62,7 +63,6 @@ class JDETracker { float iou_thresh = 0.15f); private: - static JDETracker *me; int timestamp; TrajectoryPool tracked_trajectories; TrajectoryPool lost_trajectories; diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h index f722d828fe9..a4799b7dea1 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -20,7 +20,7 @@ #pragma once #include - +#include "fastdeploy/fastdeploy_model.h" #include #include #include @@ -38,7 +38,7 @@ typedef std::vector::iterator TrajectoryPoolIterator; typedef std::vector TrajectoryPtrPool; typedef std::vector::iterator TrajectoryPtrPoolIterator; -class TKalmanFilter : public cv::KalmanFilter { +class FASTDEPLOY_DECL TKalmanFilter : public cv::KalmanFilter { public: TKalmanFilter(void); virtual ~TKalmanFilter(void) {} @@ -61,7 +61,7 @@ inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) { std_weight_velocity = 1 / 160.f; } -class Trajectory : public TKalmanFilter { +class FASTDEPLOY_DECL Trajectory : public TKalmanFilter { public: Trajectory(); Trajectory(const cv::Vec4f <rb, float score, const cv::Mat &embedding); From e86411c377a12248d3f8d85d63e8024c365779f6 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Tue, 18 Oct 2022 11:22:52 +0800 Subject: [PATCH 15/25] remove kwargs params Signed-off-by: ChaoII <849453582@qq.com> --- python/fastdeploy/vision/visualize/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fastdeploy/vision/visualize/__init__.py b/python/fastdeploy/vision/visualize/__init__.py index 5b078ec8fa5..4ba69d304eb 100644 --- a/python/fastdeploy/vision/visualize/__init__.py +++ b/python/fastdeploy/vision/visualize/__init__.py @@ -96,4 +96,4 @@ def vis_ppocr(im_data, det_result): def vis_mot(im_data, mot_result, fps, frame_id): - return C.vision.vis_mot(im_data, mot_result, fps=0.0, frame_id=0) + return C.vision.vis_mot(im_data, mot_result, fps, frame_id) From 9a3f1a4e031681f4c2f73d81fb9cc48990b60642 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Fri, 21 Oct 2022 16:37:04 +0800 Subject: [PATCH 16/25] [Doc]update pptracking docs --- examples/vision/tracking/pptracking/cpp/README.md | 4 ++-- examples/vision/tracking/pptracking/python/README.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/vision/tracking/pptracking/cpp/README.md b/examples/vision/tracking/pptracking/cpp/README.md index 7118c6a3c0b..da0deb3f3c7 100644 --- a/examples/vision/tracking/pptracking/cpp/README.md +++ b/examples/vision/tracking/pptracking/cpp/README.md @@ -1,4 +1,4 @@ -# PP-Matting C++部署示例 +# PP-Tracking C++部署示例 本目录下提供`infer.cc`快速完成PP-Tracking在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。 @@ -13,7 +13,7 @@ #下载SDK，编译模型examples代码（SDK中包含了examples代码） wget https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-x64-gpu-0.3.0.tgz tar xvf fastdeploy-linux-x64-gpu-0.3.0.tgz -cd fastdeploy-linux-x64-gpu-0.3.0/examples/vision/matting/ppmatting/cpp/ +cd fastdeploy-linux-x64-gpu-0.3.0/examples/vision/tracking/pptracking/cpp/ mkdir build && cd build cmake .. -DFASTDEPLOY_INSTALL_DIR=${PWD}/../../../../../../../fastdeploy-linux-x64-gpu-0.3.0 make -j diff --git a/examples/vision/tracking/pptracking/python/README.md b/examples/vision/tracking/pptracking/python/README.md index f52b90c7ba7..d3b943759d5 100644 --- a/examples/vision/tracking/pptracking/python/README.md +++ b/examples/vision/tracking/pptracking/python/README.md @@ -5,7 +5,7 @@ - 1. 软硬件环境满足要求，参考[FastDeploy环境要求](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) - 2. FastDeploy Python whl包安装，参考[FastDeploy Python安装](../../../../../docs/cn/build_and_install/download_prebuilt_libraries.md) -本目录下提供`infer.py`快速完成PP-Matting在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 +本目录下提供`infer.py`快速完成PP-Tracking在CPU/GPU，以及GPU上通过TensorRT加速部署的示例。执行如下脚本即可完成 ```bash #下载部署示例代码 @@ -64,7 +64,7 @@ PP-Tracking模型加载和初始化，其中model_file, params_file以及config_ ## 其它文档 -- [PP-Matting 模型介绍](..) -- [PP-Matting C++部署](../cpp) +- [PP-Tracking 模型介绍](..) +- [PP-Tracking C++部署](../cpp) - [模型预测结果说明](../../../../../docs/api/vision_results/) - [如何切换模型推理后端引擎](../../../../../docs/cn/faq/how_to_change_backend.md) From df0d5510464f8b0b5bbba2ad72270cb20aa705f8 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Fri, 21 Oct 2022 16:44:26 +0800 Subject: [PATCH 17/25] delete 'ENABLE_PADDLE_FRONTEND' switch --- fastdeploy/vision/tracking/pptracking/model.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index 6a7e906a3fa..b4915f8a355 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -14,9 +14,8 @@ #include "fastdeploy/vision/tracking/pptracking/model.h" #include "yaml-cpp/yaml.h" -#ifdef ENABLE_PADDLE_FRONTEND #include "paddle2onnx/converter.h" -#endif + namespace fastdeploy { namespace vision { namespace tracking { @@ -153,7 +152,6 @@ bool PPTracking::BuildPreprocessPipelineFromConfig(){ } void PPTracking::GetNmsInfo() { -#ifdef ENABLE_PADDLE_FRONTEND if (runtime_option.model_format == ModelFormat::PADDLE) { std::string contents; if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { @@ -171,18 +169,14 @@ void PPTracking::GetNmsInfo() { normalized = reader.nms_params.normalized; } } -#endif } bool PPTracking::Initialize() { - -#ifdef ENABLE_PADDLE_FRONTEND // remove multiclass_nms3 now // this is a trick operation for ppyoloe while inference on trt GetNmsInfo(); runtime_option.remove_multiclass_nms_ = true; runtime_option.custom_op_info_["multiclass_nms3"] = "MultiClassNMS"; -#endif if (!BuildPreprocessPipelineFromConfig()) { FDERROR << "Failed to build preprocess pipeline from configuration file." << std::endl; From 23f329dac1da6b91acb02e4f1da60a3aeaff4c44 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Fri, 21 Oct 2022 18:05:52 +0800 Subject: [PATCH 18/25] add pptracking unit test --- tests/eval_example/test_pptracking.py | 85 +++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/eval_example/test_pptracking.py diff --git a/tests/eval_example/test_pptracking.py b/tests/eval_example/test_pptracking.py new file mode 100644 index 00000000000..ccbe345c1d8 --- /dev/null +++ b/tests/eval_example/test_pptracking.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import fastdeploy as fd +import cv2 +import os +import numpy as np + + +def test_pptracking_cpu(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4" + fd.download_and_decompress(model_url, ".") + fd.download(input_url, ".") + model_path = "pptracking/fairmot_hrnetv2_w18_dlafpn_30e_576x320" + # use default backend + runtime_option = fd.RuntimeOption() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + model = fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=runtime_option) + cap = cv2.VideoCapture(input_url) + frame_id = 0 + while True: + _, frame = cap.read() + if frame is None: + break + result = model.predict(frame) + # compare diff + expect_scores = np.load("pptracking/tracking_result" + str(frame_id) + ".npy") + result_scores = np.array(result.scores) + diff = np.fabs(expect_scores - result_scores) + thres = 1e-05 + assert diff.max() < thres, "The label diff is %f, which is bigger than %f" % (diff.max(), thres) + frame_id = frame_id + 1 + cv2.waitKey(30) + if frame_id >= 10: + cap.release() + cv2.destroyAllWindows() + break + + +def test_pptracking_gpu_trt(): + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz" + input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4" + fd.download_and_decompress(model_url, ".") + fd.download(input_url, ".") + model_path = "pptracking/fairmot_hrnetv2_w18_dlafpn_30e_576x320" + runtime_option = fd.RuntimeOption() + runtime_option.use_gpu() + runtime_option.use_trt_backend() + model_file = os.path.join(model_path, "model.pdmodel") + params_file = os.path.join(model_path, "model.pdiparams") + config_file = os.path.join(model_path, "infer_cfg.yml") + model = fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=runtime_option) + cap = cv2.VideoCapture(input_url) + frame_id = 0 + while True: + _, frame = cap.read() + if frame is None: + break + result = model.predict(frame) + # compare diff + expect_scores = np.load("pptracking/tracking_result" + str(frame_id) + ".npy") + result_scores = np.array(result.scores) + diff = np.fabs(expect_scores - result_scores) + thres = 1e-05 + assert diff.max() < thres, "The label diff is %f, which is bigger than %f" % (diff.max(), thres) + frame_id = frame_id + 1 + cv2.waitKey(30) + if frame_id >= 10: + cap.release() + cv2.destroyAllWindows() + break From a5f554425c32e97ca841cc73ac9d8e9e3de9c242 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Sat, 22 Oct 2022 11:52:36 +0800 Subject: [PATCH 19/25] update pptracking unit test Signed-off-by: ChaoII <849453582@qq.com> --- tests/eval_example/test_pptracking.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/eval_example/test_pptracking.py b/tests/eval_example/test_pptracking.py index ccbe345c1d8..13d3a98ecd1 100644 --- a/tests/eval_example/test_pptracking.py +++ b/tests/eval_example/test_pptracking.py @@ -16,6 +16,7 @@ import cv2 import os import numpy as np +import pickle def test_pptracking_cpu(): @@ -38,11 +39,12 @@ def test_pptracking_cpu(): break result = model.predict(frame) # compare diff - expect_scores = np.load("pptracking/tracking_result" + str(frame_id) + ".npy") - result_scores = np.array(result.scores) - diff = np.fabs(expect_scores - result_scores) + expect = pickle.load(open("pptracking/frame" + str(frame_id) + ".pkl", "rb")) + diff_boxes = np.fabs(np.array(expect["boxes"]) - np.array(result.boxes)) + diff_scores = np.fabs(np.array(expect["scores"]) - np.array(result.scores)) + diff = max(diff_boxes.max(), diff_scores.max()) thres = 1e-05 - assert diff.max() < thres, "The label diff is %f, which is bigger than %f" % (diff.max(), thres) + assert diff < thres, "The label diff is %f, which is bigger than %f" % (diff, thres) frame_id = frame_id + 1 cv2.waitKey(30) if frame_id >= 10: @@ -72,11 +74,12 @@ def test_pptracking_gpu_trt(): break result = model.predict(frame) # compare diff - expect_scores = np.load("pptracking/tracking_result" + str(frame_id) + ".npy") - result_scores = np.array(result.scores) - diff = np.fabs(expect_scores - result_scores) + expect = pickle.load(open("pptracking/frame" + str(frame_id) + ".pkl", "rb")) + diff_boxes = np.fabs(np.array(expect["boxes"]) - np.array(result.boxes)) + diff_scores = np.fabs(np.array(expect["scores"]) - np.array(result.scores)) + diff = max(diff_boxes.max(), diff_scores.max()) thres = 1e-05 - assert diff.max() < thres, "The label diff is %f, which is bigger than %f" % (diff.max(), thres) + assert diff < thres, "The label diff is %f, which is bigger than %f" % (diff, thres) frame_id = frame_id + 1 cv2.waitKey(30) if frame_id >= 10: From 20edce2971853de851bbe12e05f060cc5884b3a7 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Mon, 24 Oct 2022 10:55:33 +0800 Subject: [PATCH 20/25] modify test video file path and remove trt test --- tests/eval_example/test_pptracking.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/eval_example/test_pptracking.py b/tests/eval_example/test_pptracking.py index 13d3a98ecd1..34f1a005261 100644 --- a/tests/eval_example/test_pptracking.py +++ b/tests/eval_example/test_pptracking.py @@ -31,7 +31,7 @@ def test_pptracking_cpu(): params_file = os.path.join(model_path, "model.pdiparams") config_file = os.path.join(model_path, "infer_cfg.yml") model = fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=runtime_option) - cap = cv2.VideoCapture(input_url) + cap = cv2.VideoCapture("./person.mp4") frame_id = 0 while True: _, frame = cap.read() @@ -53,7 +53,7 @@ def test_pptracking_cpu(): break -def test_pptracking_gpu_trt(): +def test_pptracking_gpu(): model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4" fd.download_and_decompress(model_url, ".") @@ -61,12 +61,13 @@ def test_pptracking_gpu_trt(): model_path = "pptracking/fairmot_hrnetv2_w18_dlafpn_30e_576x320" runtime_option = fd.RuntimeOption() runtime_option.use_gpu() - runtime_option.use_trt_backend() + # Not supported trt backend, up to now + # runtime_option.use_trt_backend() model_file = os.path.join(model_path, "model.pdmodel") params_file = os.path.join(model_path, "model.pdiparams") config_file = os.path.join(model_path, "infer_cfg.yml") model = fd.vision.tracking.PPTracking(model_file, params_file, config_file, runtime_option=runtime_option) - cap = cv2.VideoCapture(input_url) + cap = cv2.VideoCapture("./person.mp4") frame_id = 0 while True: _, frame = cap.read() From 09d9a7f1d933d42408082f5305bb6be899fa6ce5 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Mon, 24 Oct 2022 11:52:00 +0800 Subject: [PATCH 21/25] update unit test model url --- tests/eval_example/test_pptracking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/eval_example/test_pptracking.py b/tests/eval_example/test_pptracking.py index 34f1a005261..ee1cb9bc512 100644 --- a/tests/eval_example/test_pptracking.py +++ b/tests/eval_example/test_pptracking.py @@ -20,7 +20,7 @@ def test_pptracking_cpu(): - model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz" + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/pptracking.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4" fd.download_and_decompress(model_url, ".") fd.download(input_url, ".") @@ -54,7 +54,7 @@ def test_pptracking_cpu(): def test_pptracking_gpu(): - model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz" + model_url = "https://bj.bcebos.com/paddlehub/fastdeploy/pptracking.tgz" input_url = "https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4" fd.download_and_decompress(model_url, ".") fd.download(input_url, ".") From b23bb449ce6dd15690a5d2dd9f758b76edc7791e Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Tue, 25 Oct 2022 10:23:53 +0800 Subject: [PATCH 22/25] remove 'FASTDEPLOY_DECL' macro Signed-off-by: ChaoII <849453582@qq.com> --- .gitignore | 1 + fastdeploy/vision/tracking/pptracking/trajectory.h | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 38274eead1e..05b0ef1aba2 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ fastdeploy/version.py fastdeploy/core/config.h python/fastdeploy/c_lib_wrap.py python/fastdeploy/LICENSE* +python/build_cpu.sh python/fastdeploy/ThirdPartyNotices* *.so* fpython/astdeploy/libs/third_libs diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h index a4799b7dea1..0ab5179ac81 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -20,7 +20,6 @@ #pragma once #include -#include "fastdeploy/fastdeploy_model.h" #include #include #include @@ -38,7 +37,7 @@ typedef std::vector::iterator TrajectoryPoolIterator; typedef std::vector TrajectoryPtrPool; typedef std::vector::iterator TrajectoryPtrPoolIterator; -class FASTDEPLOY_DECL TKalmanFilter : public cv::KalmanFilter { +class TKalmanFilter : public cv::KalmanFilter { public: TKalmanFilter(void); virtual ~TKalmanFilter(void) {} @@ -61,7 +60,7 @@ inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) { std_weight_velocity = 1 / 160.f; } -class FASTDEPLOY_DECL Trajectory : public TKalmanFilter { +class Trajectory : public TKalmanFilter { public: Trajectory(); Trajectory(const cv::Vec4f <rb, float score, const cv::Mat &embedding); From dd993330c2cb2060fc018022054e230d270b57d6 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Tue, 25 Oct 2022 17:43:38 +0800 Subject: [PATCH 23/25] fix build python packages about pptracking on win32 Signed-off-by: ChaoII <849453582@qq.com> --- .../vision/tracking/pptracking/tracker.cc | 6 +++--- .../vision/tracking/pptracking/tracker.h | 1 + .../vision/tracking/pptracking/trajectory.cc | 10 ++++------ .../vision/tracking/pptracking/trajectory.h | 19 ++++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/fastdeploy/vision/tracking/pptracking/tracker.cc b/fastdeploy/vision/tracking/pptracking/tracker.cc index 9c1df243527..0026d889c3c 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.cc +++ b/fastdeploy/vision/tracking/pptracking/tracker.cc @@ -95,7 +95,7 @@ bool JDETracker::update(const cv::Mat &dets, pt->update(&ct, timestamp); activated_trajectories.push_back(pt); } else { - pt->reactivate(&ct, timestamp); + pt->reactivate(&ct, count,timestamp); retrieved_trajectories.push_back(pt); } } @@ -121,7 +121,7 @@ bool JDETracker::update(const cv::Mat &dets, pt->update(ct, timestamp); activated_trajectories.push_back(pt); } else { - pt->reactivate(ct, timestamp); + pt->reactivate(ct,count, timestamp); retrieved_trajectories.push_back(pt); } } @@ -156,7 +156,7 @@ bool JDETracker::update(const cv::Mat &dets, for (size_t i = 0; i < mismatch_col.size(); ++i) { if (nnext_candidates[mismatch_col[i]]->score < det_thresh) continue; - nnext_candidates[mismatch_col[i]]->activate(timestamp); + nnext_candidates[mismatch_col[i]]->activate(count, timestamp); activated_trajectories.push_back(nnext_candidates[mismatch_col[i]]); } diff --git a/fastdeploy/vision/tracking/pptracking/tracker.h b/fastdeploy/vision/tracking/pptracking/tracker.h index 29652154e21..92344450e2d 100644 --- a/fastdeploy/vision/tracking/pptracking/tracker.h +++ b/fastdeploy/vision/tracking/pptracking/tracker.h @@ -70,6 +70,7 @@ class FASTDEPLOY_DECL JDETracker { int max_lost_time; float lambda; float det_thresh; + int count = 0; }; } // namespace tracking diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.cc b/fastdeploy/vision/tracking/pptracking/trajectory.cc index d40b2df7708..fd54b4c2e22 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.cc +++ b/fastdeploy/vision/tracking/pptracking/trajectory.cc @@ -99,8 +99,6 @@ void TKalmanFilter::project(cv::Mat *mean, cv::Mat *covariance) const { cv::GEMM_2_T); } -int Trajectory::count = 0; - const cv::Mat &Trajectory::predict(void) { if (state != Tracked) *cv::KalmanFilter::statePost.ptr(7) = 0; return TKalmanFilter::predict(); @@ -120,8 +118,8 @@ void Trajectory::update(Trajectory *traj, if (update_embedding_) update_embedding(traj->current_embedding); } -void Trajectory::activate(int timestamp_) { - id = next_id(); +void Trajectory::activate(int &cnt,int timestamp_) { + id = next_id(cnt); TKalmanFilter::init(cv::Mat(xyah)); length = 0; state = Tracked; @@ -132,14 +130,14 @@ void Trajectory::activate(int timestamp_) { starttime = timestamp_; } -void Trajectory::reactivate(Trajectory *traj, int timestamp_, bool newid) { +void Trajectory::reactivate(Trajectory *traj,int &cnt, int timestamp_, bool newid) { TKalmanFilter::correct(cv::Mat(traj->xyah)); update_embedding(traj->current_embedding); length = 0; state = Tracked; is_activated = true; timestamp = timestamp_; - if (newid) id = next_id(); + if (newid) id = next_id(cnt); } void Trajectory::update_embedding(const cv::Mat &embedding) { diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h index 0ab5179ac81..a869f840991 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -20,6 +20,7 @@ #pragma once #include +#include "fastdeploy/fastdeploy_model.h" #include #include #include @@ -37,7 +38,7 @@ typedef std::vector::iterator TrajectoryPoolIterator; typedef std::vector TrajectoryPtrPool; typedef std::vector::iterator TrajectoryPtrPoolIterator; -class TKalmanFilter : public cv::KalmanFilter { +class FASTDEPLOY_DECL TKalmanFilter : public cv::KalmanFilter { public: TKalmanFilter(void); virtual ~TKalmanFilter(void) {} @@ -60,7 +61,7 @@ inline TKalmanFilter::TKalmanFilter(void) : cv::KalmanFilter(8, 4) { std_weight_velocity = 1 / 160.f; } -class Trajectory : public TKalmanFilter { +class FASTDEPLOY_DECL Trajectory : public TKalmanFilter { public: Trajectory(); Trajectory(const cv::Vec4f <rb, float score, const cv::Mat &embedding); @@ -68,13 +69,13 @@ class Trajectory : public TKalmanFilter { Trajectory &operator=(const Trajectory &rhs); virtual ~Trajectory(void) {} - static int next_id(); + int next_id(int &nt); virtual const cv::Mat &predict(void); virtual void update(Trajectory *traj, int timestamp, bool update_embedding = true); - virtual void activate(int timestamp); - virtual void reactivate(Trajectory *traj, int timestamp, bool newid = false); + virtual void activate(int& cnt, int timestamp); + virtual void reactivate(Trajectory *traj, int & cnt,int timestamp, bool newid = false); virtual void mark_lost(void); virtual void mark_removed(void); @@ -129,7 +130,7 @@ class Trajectory : public TKalmanFilter { float score; private: - static int count; +// int count=0; cv::Vec4f xyah; cv::Mat current_embedding; float eta; @@ -219,9 +220,9 @@ inline Trajectory &Trajectory::operator=(const Trajectory &rhs) { return *this; } -inline int Trajectory::next_id() { - ++count; - return count; +inline int Trajectory::next_id(int &cnt) { + ++cnt; + return cnt; } inline void Trajectory::mark_lost(void) { state = Lost; } From e1b5cf4d13acc52ced73d51eca446527f50cf918 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Wed, 26 Oct 2022 16:02:02 +0800 Subject: [PATCH 24/25] update comment Signed-off-by: ChaoII <849453582@qq.com> --- .../vision/common/processors/letter_box.cc | 71 +++++++++++++++++++ .../processors}/letter_box.h | 37 +++++----- .../vision/common/processors/transform.h | 1 + fastdeploy/vision/common/result.h | 15 +++- .../vision/tracking/pptracking/letter_box.cc | 59 --------------- .../vision/tracking/pptracking/model.cc | 37 ---------- fastdeploy/vision/tracking/pptracking/model.h | 15 +--- .../vision/tracking/pptracking/__init__.py | 13 ++++ 8 files changed, 120 insertions(+), 128 deletions(-) create mode 100644 fastdeploy/vision/common/processors/letter_box.cc rename fastdeploy/vision/{tracking/pptracking => common/processors}/letter_box.h (52%) delete mode 100644 fastdeploy/vision/tracking/pptracking/letter_box.cc diff --git a/fastdeploy/vision/common/processors/letter_box.cc b/fastdeploy/vision/common/processors/letter_box.cc new file mode 100644 index 00000000000..423aef1d361 --- /dev/null +++ b/fastdeploy/vision/common/processors/letter_box.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/letter_box.h" + +namespace fastdeploy{ +namespace vision{ +bool LetterBoxResize::ImplByOpenCV(Mat* mat) { + + if (mat->Channels() != color_.size()) { + FDERROR << "Pad: Require input channels equals to size of padding value, " + "but now channels = " + << mat->Channels() + << ", the size of padding values = " << color_.size() << "." + << std::endl; + return false; + } + cv::Mat* im = mat->GetOpenCVMat(); + // generate scale_factor + int origin_w = mat->Width(); + int origin_h = mat->Height(); + int target_h = target_size_[0]; + int target_w = target_size_[1]; + float ratio_h = static_cast(target_h) / static_cast(origin_h); + float ratio_w = static_cast(target_w) / static_cast(origin_w); + float resize_scale = std::min(ratio_h, ratio_w); + // get_resized_shape + int new_shape_w = std::round(im->cols * resize_scale); + int new_shape_h = std::round(im->rows * resize_scale); + // calculate pad + float padw = (target_size_[1] - new_shape_w) / 2.; + float padh = (target_size_[0] - new_shape_h) / 2.; + int top = std::round(padh - 0.1); + int bottom = std::round(padh + 0.1); + int left = std::round(padw - 0.1); + int right = std::round(padw + 0.1); + cv::resize(*im, *im, cv::Size(new_shape_w, new_shape_h), 0, 0, cv::INTER_AREA); + cv::Scalar color; + if (color_.size() == 1) { + color = cv::Scalar(color_[0]); + } else if (color_.size() == 2) { + color = cv::Scalar(color_[0], color_[1]); + } else if (color_.size() == 3) { + color = cv::Scalar(color_[0], color_[1], color_[2]); + } else { + color = cv::Scalar(color_[0], color_[1], color_[2], color_[3]); + } + cv::copyMakeBorder(*im, *im, top, bottom, left, right, cv::BORDER_CONSTANT, color); + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + return true; +} + +bool LetterBoxResize::Run(Mat* mat, const std::vector& target_size, const std::vector& color, ProcLib lib) { + auto l = LetterBoxResize(target_size,color); + return l(mat, lib); +} + +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/tracking/pptracking/letter_box.h b/fastdeploy/vision/common/processors/letter_box.h similarity index 52% rename from fastdeploy/vision/tracking/pptracking/letter_box.h rename to fastdeploy/vision/common/processors/letter_box.h index 17f9f283383..25003b161a1 100644 --- a/fastdeploy/vision/tracking/pptracking/letter_box.h +++ b/fastdeploy/vision/common/processors/letter_box.h @@ -1,4 +1,4 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,23 +14,28 @@ #pragma once -#include "fastdeploy/vision/common/processors/transform.h" -#include "fastdeploy/fastdeploy_model.h" +#include "fastdeploy/vision/common/processors/base.h" + namespace fastdeploy { namespace vision { -namespace tracking { -class LetterBoxResize: public Processor{ -public: - LetterBoxResize(const std::vector& target_size, const std::vector& color); - bool ImplByOpenCV(Mat* mat) override; - std::string Name() override { return "LetterBoxResize"; } -private: - std::vector target_size_; - std::vector color_; -}; -} // namespace tracking -} // namespace vision -} // namespace fastdeploy +class LetterBoxResize : public Processor { + public: + LetterBoxResize(const std::vector& target_size, const std::vector& color) { + target_size_ = target_size; + color_ = color; + } + + bool ImplByOpenCV(Mat* mat); + std::string Name() { return "LetterBoxResize"; } + static bool Run(Mat* mat,const std::vector& target_size, const std::vector& color, + ProcLib lib = ProcLib::OPENCV); + + private: + std::vector target_size_; + std::vector color_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/transform.h b/fastdeploy/vision/common/processors/transform.h index b16b95b3819..6e99f2e3539 100644 --- a/fastdeploy/vision/common/processors/transform.h +++ b/fastdeploy/vision/common/processors/transform.h @@ -31,3 +31,4 @@ #include "fastdeploy/vision/common/processors/resize_by_short.h" #include "fastdeploy/vision/common/processors/stride_pad.h" #include "fastdeploy/vision/common/processors/warp_affine.h" +#include "fastdeploy/vision/common/processors/letter_box.h" diff --git a/fastdeploy/vision/common/result.h b/fastdeploy/vision/common/result.h index c9a8d113a4e..f08c00d82a7 100644 --- a/fastdeploy/vision/common/result.h +++ b/fastdeploy/vision/common/result.h @@ -155,16 +155,25 @@ struct FASTDEPLOY_DECL OCRResult : public BaseResult { std::string Str(); }; +/*! @brief MOT(Multi-Object Tracking) result structure for all the MOT models + */ struct FASTDEPLOY_DECL MOTResult : public BaseResult { - // left top right bottom + /** \brief All the tracking object boxes for an input image, the size of `boxes` is the number of tracking objects, and the element of `boxes` is a array of 4 float values, means [xmin, ymin, xmax, ymax] + */ std::vector> boxes; + /** \brief All the tracking object ids + */ std::vector ids; + /** \brief The confidence for all the tracking objects + */ std::vector scores; + /** \brief The classify label id for all the tracking object + */ std::vector class_ids; ResultType type = ResultType::MOT; - + /// Clear MOT result void Clear(); - + /// Debug function, convert the result to string to print std::string Str(); }; diff --git a/fastdeploy/vision/tracking/pptracking/letter_box.cc b/fastdeploy/vision/tracking/pptracking/letter_box.cc deleted file mode 100644 index bc8e0cbcb26..00000000000 --- a/fastdeploy/vision/tracking/pptracking/letter_box.cc +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "fastdeploy/vision/tracking/pptracking/letter_box.h" - -namespace fastdeploy { -namespace vision { -namespace tracking { - -LetterBoxResize::LetterBoxResize(const std::vector& target_size, const std::vector& color){ - target_size_=target_size; - color_=color; -} -bool LetterBoxResize::ImplByOpenCV(Mat* mat){ - if (mat->Channels() != color_.size()) { - FDERROR << "Pad: Require input channels equals to size of padding value, " - "but now channels = " - << mat->Channels() - << ", the size of padding values = " << color_.size() << "." - << std::endl; - return false; - } - // generate scale_factor - int origin_w = mat->Width(); - int origin_h = mat->Height(); - int target_h = target_size_[0]; - int target_w = target_size_[1]; - float ratio_h = static_cast(target_h) / static_cast(origin_h); - float ratio_w = static_cast(target_w) / static_cast(origin_w); - float resize_scale = std::min(ratio_h, ratio_w); - - int new_shape_w = std::round(mat->Width() * resize_scale); - int new_shape_h = std::round(mat->Height() * resize_scale); - float padw = (target_size_[1] - new_shape_w) / 2.; - float padh = (target_size_[0] - new_shape_h) / 2.; - int top = std::round(padh - 0.1); - int bottom = std::round(padh + 0.1); - int left = std::round(padw - 0.1); - int right = std::round(padw + 0.1); - - Resize::Run(mat,new_shape_w,new_shape_h); - Pad::Run(mat,top,bottom,left,right,color_); - return true; -} - -} // namespace tracking -} // namespace vision -} // namespace fastdeploy \ No newline at end of file diff --git a/fastdeploy/vision/tracking/pptracking/model.cc b/fastdeploy/vision/tracking/pptracking/model.cc index b4915f8a355..064892889b5 100644 --- a/fastdeploy/vision/tracking/pptracking/model.cc +++ b/fastdeploy/vision/tracking/pptracking/model.cc @@ -151,32 +151,7 @@ bool PPTracking::BuildPreprocessPipelineFromConfig(){ return true; } -void PPTracking::GetNmsInfo() { - if (runtime_option.model_format == ModelFormat::PADDLE) { - std::string contents; - if (!ReadBinaryFromFile(runtime_option.model_file, &contents)) { - return; - } - auto reader = paddle2onnx::PaddleReader(contents.c_str(), contents.size()); - if (reader.has_nms) { - has_nms_ = true; - background_label = reader.nms_params.background_label; - keep_top_k = reader.nms_params.keep_top_k; - nms_eta = reader.nms_params.nms_eta; - nms_threshold = reader.nms_params.nms_threshold; - score_threshold = reader.nms_params.score_threshold; - nms_top_k = reader.nms_params.nms_top_k; - normalized = reader.nms_params.normalized; - } - } -} - bool PPTracking::Initialize() { - // remove multiclass_nms3 now - // this is a trick operation for ppyoloe while inference on trt - GetNmsInfo(); - runtime_option.remove_multiclass_nms_ = true; - runtime_option.custom_op_info_["multiclass_nms3"] = "MultiClassNMS"; if (!BuildPreprocessPipelineFromConfig()) { FDERROR << "Failed to build preprocess pipeline from configuration file." << std::endl; @@ -186,18 +161,6 @@ bool PPTracking::Initialize() { FDERROR << "Failed to initialize fastdeploy backend." << std::endl; return false; } - - if (has_nms_ && runtime_option.backend == Backend::TRT) { - FDINFO << "Detected operator multiclass_nms3 in your model, will replace " - "it with fastdeploy::backend::MultiClassNMS(background_label=" - << background_label << ", keep_top_k=" << keep_top_k - << ", nms_eta=" << nms_eta << ", nms_threshold=" << nms_threshold - << ", score_threshold=" << score_threshold - << ", nms_top_k=" << nms_top_k << ", normalized=" << normalized - << ")." << std::endl; - has_nms_ = false; - } - // create JDETracker instance std::unique_ptr jdeTracker(new JDETracker); jdeTracker_ = std::move(jdeTracker); diff --git a/fastdeploy/vision/tracking/pptracking/model.h b/fastdeploy/vision/tracking/pptracking/model.h index 040ed383f13..eece2700024 100644 --- a/fastdeploy/vision/tracking/pptracking/model.h +++ b/fastdeploy/vision/tracking/pptracking/model.h @@ -18,7 +18,7 @@ #include "fastdeploy/fastdeploy_model.h" #include "fastdeploy/vision/common/result.h" #include "fastdeploy/vision/tracking/pptracking/tracker.h" -#include "fastdeploy/vision/tracking/pptracking/letter_box.h" +//#include "fastdeploy/vision/tracking/pptracking/letter_box.h" namespace fastdeploy { namespace vision { @@ -56,8 +56,8 @@ class FASTDEPLOY_DECL PPTracking: public FastDeployModel { private: bool BuildPreprocessPipelineFromConfig(); + bool Initialize(); - void GetNmsInfo(); bool Preprocess(Mat* img, std::vector* outputs); @@ -69,19 +69,8 @@ class FASTDEPLOY_DECL PPTracking: public FastDeployModel { float conf_thresh_; float tracked_thresh_; float min_box_area_; - bool is_scale_ = true; std::unique_ptr jdeTracker_; - // configuration for nms - int64_t background_label = -1; - int64_t keep_top_k = 300; - float nms_eta = 1.0; - float nms_threshold = 0.7; - float score_threshold = 0.01; - int64_t nms_top_k = 10000; - bool normalized = true; - bool has_nms_ = true; - }; } // namespace tracking diff --git a/python/fastdeploy/vision/tracking/pptracking/__init__.py b/python/fastdeploy/vision/tracking/pptracking/__init__.py index 9da0d669fa9..89ca2a7b014 100644 --- a/python/fastdeploy/vision/tracking/pptracking/__init__.py +++ b/python/fastdeploy/vision/tracking/pptracking/__init__.py @@ -24,6 +24,14 @@ def __init__(self, config_file, runtime_option=None, model_format=ModelFormat.PADDLE): + """Load a PPTracking model exported by PaddleDetection. + + :param model_file: (str)Path of model file, e.g pptracking/model.pdmodel + :param params_file: (str)Path of parameters file, e.g ppyoloe/model.pdiparams + :param config_file: (str)Path of configuration file for deployment, e.g ppyoloe/infer_cfg.yml + :param runtime_option: (fastdeploy.RuntimeOption)RuntimeOption for inference this model, if it's None, will use the default backend on CPU + :param model_format: (fastdeploy.ModelForamt)Model format of the loaded model + """ super(PPTracking, self).__init__(runtime_option) assert model_format == ModelFormat.PADDLE, "PPTracking model only support model format of ModelFormat.Paddle now." @@ -33,5 +41,10 @@ def __init__(self, assert self.initialized, "PPTracking model initialize failed." def predict(self, input_image): + """Predict the MOT result for an input image + + :param input_image: (numpy.ndarray)The input image data, 3-D array with layout HWC, BGR format + :return: MOTResult + """ assert input_image is not None, "The input image data is None." return self._model.predict(input_image) From 60ca6d49826a673d198cd2ef8ee062e672577878 Mon Sep 17 00:00:00 2001 From: ChaoII <849453582@qq.com> Date: Wed, 26 Oct 2022 17:29:12 +0800 Subject: [PATCH 25/25] add pptracking model explain Signed-off-by: ChaoII <849453582@qq.com> --- examples/vision/tracking/pptracking/README.md | 35 +++++++++++++++++++ .../vision/tracking/pptracking/cpp/README.md | 1 - 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 examples/vision/tracking/pptracking/README.md diff --git a/examples/vision/tracking/pptracking/README.md b/examples/vision/tracking/pptracking/README.md new file mode 100644 index 00000000000..35b9d7173d2 --- /dev/null +++ b/examples/vision/tracking/pptracking/README.md @@ -0,0 +1,35 @@ +# PP-Tracking模型部署 + +## 模型版本说明 + +- [PaddleDetection release/2.5](https://github.com/PaddlePaddle/PaddleDetection/tree/release/2.5) + +## 支持模型列表 + +目前FastDeploy支持如下模型的部署 + +- [PP-Tracking系列模型](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/configs/mot) + + +## 导出部署模型 + +在部署前，需要先将训练好的PP-Tracking导出成部署模型，导出PPTracking导出模型步骤，参考文档[导出模型](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/pptracking/cpp/README.md)。 + + +## 下载预训练模型 + +为了方便开发者的测试，下面提供了PP-Tracking行人跟踪垂类模型，开发者可直接下载使用，更多模型参见[PPTracking](https://github.com/PaddlePaddle/PaddleDetection/blob/release/2.5/deploy/pptracking/README_cn.md)。 + +| 模型 | 参数大小 | 精度 | 备注 | +|:-----------------------------------------------------------------------------------------------------|:-------|:----- | :------ | +| [PP-Tracking](https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz) | 51.2MB | - | + +**说明** +- 仅支持JDE模型（JDE，FairMOT，MCFairMOT)； +- 目前暂不支持SDE模型的部署，待PaddleDetection官方更新SED部署代码后，对SDE模型进行支持。 + + +## 详细部署文档 + +- [Python部署](python) +- [C++部署](cpp) diff --git a/examples/vision/tracking/pptracking/cpp/README.md b/examples/vision/tracking/pptracking/cpp/README.md index da0deb3f3c7..6c28dd1c3ed 100644 --- a/examples/vision/tracking/pptracking/cpp/README.md +++ b/examples/vision/tracking/pptracking/cpp/README.md @@ -22,7 +22,6 @@ make -j wget https://bj.bcebos.com/paddlehub/fastdeploy/fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz tar -xvf fairmot_hrnetv2_w18_dlafpn_30e_576x320.tgz wget https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4 -wget https://bj.bcebos.com/paddlehub/fastdeploy/person.mp4 # CPU推理