diff --git a/README.md b/README.md index 9ab49f7..7f00fe3 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ or | +-- camera_6.mp4 ``` -In case of mp4 files, df3d will first expand them into images using ffmpeg. Please check the sample data for a real exampe: https://github.com/NeLy-EPFL/DeepFly3D/tree/master/sample/test +In case of mp4 files, df3d will first expand them into images using ffmpeg. Please check the sample data for a real example: https://github.com/NeLy-EPFL/DeepFly3D/tree/master/sample/test # Basic Usage @@ -153,7 +153,7 @@ df3d-cli /your/image/path This command assumes your cameras are numbered in the default order:

- +

in which case your data will look like this if cameras 0, 1, 2 are shown left-to-right in the top row and cameras 4, 5, 6 are show left-to-right in the bottom row: @@ -172,8 +172,9 @@ then your order is 6 5 4 3 2 1 0, so you'd need to run `df3d-cli /your/image/pat ``` usage: df3d-cli [-h] [-v] [-vv] [-d] [--output-folder OUTPUT_FOLDER] [-r] [-f] [-o] [-n NUM_IMAGES_MAX] - [--order [CAMERA_IDS [CAMERA_IDS ...]]] [--video-2d] - [--video-3d] [--skip-pose-estimation] + [--order [CAMERA_IDS [CAMERA_IDS ...]]] [--skip-pose-estimation] + [--video-2d] [--video-3d] [--output-fps OUTPUT_FPS] + [--batch-size BATCH_SIZE] [--pin-memory-disabled] INPUT DeepFly3D pose estimation @@ -197,13 +198,24 @@ optional arguments: results -n NUM_IMAGES_MAX, --num-images-max NUM_IMAGES_MAX Maximal number of images to process. - --order [CAMERA_IDS [CAMERA_IDS ...]], --camera-ids [CAMERA_IDS [CAMERA_IDS ...]] + --order [CAMERA_IDS [CAMERA_IDS ...]] Ordering of the cameras provided as an ordered list of ids. Example: 0 1 4 3 2 5 6. + --skip-pose-estimation + Skip 2D and 3D pose estimation. Use in combination with --video-2d + or --video-3d to generate videos without rerunning pose estimation. --video-2d Generate pose2d videos --video-3d Generate pose3d videos - --skip-pose-estimation - Skip 2D and 3D pose estimation + --output-fps OUTPUT_FPS + FPS for output videos. If not specified, uses the FPS from + the input videos. + --batch-size BATCH_SIZE + Batch size for inference - how many images are processed + through the model at once + --pin-memory-disabled + Whether to disable `pin_memory` in the dataloader. + Keeping pin memory enabled usually speeds up processing, + but sometimes leads to memory leaks. ``` Therefore, you can create advanced queries in df3d-cli, for example: @@ -218,6 +230,7 @@ df3d-cli -f /path/to/text.txt \ # process each line from the text file --skip-pose-estimation \ # will not run 2d pose estimation, instead will do calibration, triangulation and will save results --video-2d \ # will make 2d video for each folder --video-3d \ # will make 3d video for each folder + --output-fps 15.0 \ # set output video FPS to 15 (instead of using the input video FPS) ``` To test df3d-cli, you run it on a folder for only 100 images, make videos, and print agressivelly for debugging: @@ -299,6 +312,8 @@ Using the flag --video-2d with df3d-cli will create the following video: Using the flag --video-3d with df3d-cli will create the following video: ![Alt text](./images/out3d.gif?raw=true "Title") +When generating videos with `--video-2d` or `--video-3d`, you can control the output video frame rate using the `--output-fps` flag. If not specified, the output video framerate will be set to equal the input videos' framerate. + # Output df3d-cli saves results under df3d_result.pk file. You can read it using, diff --git a/df3d/cli.py b/df3d/cli.py index 76e4f78..6c55a76 100644 --- a/df3d/cli.py +++ b/df3d/cli.py @@ -148,6 +148,13 @@ def parse_cli_args(): help="Whether to disable `pin_memory` in the dataloader. Keeping this enabled usually speeds up the processing, but sometimes leads to memory leaks. See https://github.com/NeLy-EPFL/DeepFly2D/issues/6", action="store_true", ) + parser.add_argument( + "--output-fps", + help="FPS for output videos. If not specified, uses the FPS from the input " + "videos. If specified, overrides the input video FPS.", + type=float, + default=None, + ) args = parser.parse_args() args.input_folder = Path(args.input_folder).expanduser().resolve() if args.output_folder is None: @@ -295,9 +302,12 @@ def run(args): core.calibrate_calc(0, core.max_img_id) core.save() + # Use output_fps if specified, otherwise use core.fps which comes from the input videos + fps = args.output_fps if args.output_fps is not None else core.fps + if args.video_2d: video.make_pose2d_video( - core.plot_2d, core.num_images, core.input_folder, core.output_folder + core.plot_2d, core.num_images, core.input_folder, core.output_folder, fps=fps ) if args.video_3d: @@ -307,6 +317,7 @@ def run(args): core.num_images, core.input_folder, core.output_folder, + fps=fps, ) if args.delete_images: diff --git a/df3d/core.py b/df3d/core.py index 449d12f..6038536 100644 --- a/df3d/core.py +++ b/df3d/core.py @@ -76,6 +76,7 @@ def __init__( self.output_folder = output_folder self.expand_videos() # turn .mp4 into .jpg + self.fps = self.get_fps() self.num_images_max = num_images_max if num_images_max is not None else 0 self.max_img_id = get_max_img_id(self.input_folder) if self.num_images_max > 0: @@ -412,6 +413,36 @@ def setup_camera_ordering(self, camera_ordering) -> np.ndarray: # self.cidread2cid, self.cid2cidread = read_camera_order(self.output_folder) return np.array(camera_ordering) + def get_fps(self): + rates = [] + for vid in glob.glob(os.path.join(self.input_folder, "camera_?.mp4")): + cmd = ["ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=avg_frame_rate", "-of", + "default=noprint_wrappers=1:nokey=1", vid] + try: + rates.append(subprocess.check_output(cmd, text=True)) + except: + logger.warning(f"Command failed: {' '.join(cmd)}") + break + if len(rates) == 0: + return None + if any(rate != rates[0] for rate in rates): + logger.warning("Framerates of input videos differ from one another," + f" using the first one: {rates}") + rate = rates[0] + try: + return float(rate) + except ValueError: + pass + try: + numerator, denominator = map(int, rate.split('/')) + return numerator / denominator if denominator != 0 else None + except ValueError: + pass + logger.warning(f'Could not parse framerate from string "{rate}" returned' + ' by ffprobe command, so setting fps to None.') + return None + def expand_videos(self): """expands video camera_x.mp4 into set of images camera_x_img_y.jpg""" for vid in glob.glob(os.path.join(self.input_folder, "camera_?.mp4")): diff --git a/df3d/video.py b/df3d/video.py index be74c78..5a2d4a4 100644 --- a/df3d/video.py +++ b/df3d/video.py @@ -15,9 +15,11 @@ img3d_aspect = (2, 2) # this is the aspect ration for one image on the 3d video's grid img2d_aspect = (2, 1) # this is the aspect ration for one image on the 3d video's grid video_width = 5000 # total width of the 2d and 3d videos +default_fps = 30 -def make_pose2d_video(plot_2d, num_images, input_folder, output_folder): +def make_pose2d_video(plot_2d, num_images, input_folder, + output_folder, fps=default_fps): """Creates pose2d estimation videos and writes it to output_folder. Parameters: @@ -43,10 +45,11 @@ def stack(img_id): video_name = 'video_pose2d_' + input_folder.replace('/', '_') + '.mp4' video_path = os.path.join(input_folder, output_folder, video_name) - _make_video(video_path, generator) + _make_video(video_path, generator, fps=fps) -def make_pose3d_video(points3d, plot_2d, num_images, input_folder, output_folder): +def make_pose3d_video(points3d, plot_2d, num_images, input_folder, + output_folder, fps=default_fps): """Creates pose3d estimation videos and writes it to output_folder. Parameters: @@ -72,16 +75,18 @@ def stack(img_id): generator = imgs_generator() video_name = 'video_pose3d_' + input_folder.replace('/', '_') + '.mp4' video_path = os.path.join(input_folder, output_folder, video_name) - _make_video(video_path, generator) + _make_video(video_path, generator, fps=fps) -def _make_video(video_path, imgs): +def _make_video(video_path, imgs, fps=default_fps): """Code used to generate a video using cv2. Parameters: video_path: a path ending with .mp4, for instance: "/results/pose2d.mp4" imgs: an iterable or generator with the images to turn into a video """ + if fps is None: + fps = default_fps first_frame = next(imgs) imgs = itertools.chain([first_frame], imgs) @@ -89,7 +94,6 @@ def _make_video(video_path, imgs): shape = int(first_frame.shape[1]), int(first_frame.shape[0]) logger.debug('Saving video to: ' + video_path) fourcc = cv2.VideoWriter_fourcc(*'mp4v') - fps = 30 output_shape = _resize(current_shape=shape, new_width=video_width) logger.debug('Video size is: {}'.format(output_shape)) video_writer = cv2.VideoWriter(video_path, fourcc, fps, output_shape) diff --git a/tests/data/reference_df3d/video_pose2d.mp4 b/tests/data/reference_df3d/video_pose2d.mp4 index 970ab02..f35429a 100644 Binary files a/tests/data/reference_df3d/video_pose2d.mp4 and b/tests/data/reference_df3d/video_pose2d.mp4 differ diff --git a/tests/data/reference_df3d/video_pose3d.mp4 b/tests/data/reference_df3d/video_pose3d.mp4 index 42ddcd4..a8b1d57 100644 Binary files a/tests/data/reference_df3d/video_pose3d.mp4 and b/tests/data/reference_df3d/video_pose3d.mp4 differ diff --git a/tests/run_df3d_on_sample_data.sh b/tests/run_df3d_on_sample_data.sh index b33041d..0d20f01 100755 --- a/tests/run_df3d_on_sample_data.sh +++ b/tests/run_df3d_on_sample_data.sh @@ -2,4 +2,4 @@ # You can run this to confirm that your DeepFly3D installation is working correctly. # If successful, you will get df3d results files in the directory `data/reference_df3d/` -df3d-cli data/reference/ -v --video-2d --video-3d +df3d-cli data/reference/ -v --video-2d --video-3d --output-fps 5 diff --git a/tests/test_df3d.py b/tests/test_df3d.py index 2bc1813..49bd50b 100644 --- a/tests/test_df3d.py +++ b/tests/test_df3d.py @@ -22,6 +22,7 @@ TEST_DATA_LOCATION_RESULT_FILE_3D = f"{TEST_DATA_LOCATION_RESULT}/df3d_result_3d.pkl" TEST_DATA_LOCATION_REFERENCE_VIDEO_2D = f"{TEST_DATA_LOCATION_RESULT}/video_pose2d.mp4" TEST_DATA_LOCATION_REFERENCE_VIDEO_3D = f"{TEST_DATA_LOCATION_RESULT}/video_pose3d.mp4" +TEST_DATA_VIDEO_FRAMERATE = 5 TEST_DATA_LOCATION_WORKING = f"{TEST_DATA_LOCATION}/working" TEST_DATA_LOCATION_WORKING_RESULT = f"{TEST_DATA_LOCATION_WORKING}_df3d" @@ -254,7 +255,8 @@ def test_video_2d(self): ) df3d.video.make_pose2d_video( - core.plot_2d, core.num_images, core.input_folder, core.output_folder + core.plot_2d, core.num_images, core.input_folder, + core.output_folder, fps=TEST_DATA_VIDEO_FRAMERATE ) video_name = "video_pose2d_" + core.input_folder.replace("/", "_") + ".mp4" @@ -298,6 +300,7 @@ def test_video_3d(self): core.num_images, core.input_folder, core.output_folder, + fps=TEST_DATA_VIDEO_FRAMERATE, ) video_name = "video_pose3d_" + core.input_folder.replace("/", "_") + ".mp4"