Pull true fps from input videos and use it when making new videos

README.md

@@ -141,7 +141,7 @@ or
 |   +-- camera_6.mp4
 ```
 
-In case of mp4 files, df3d will first expand them into images using ffmpeg. Please check the sample data for a real exampe: https://github.com/NeLy-EPFL/DeepFly3D/tree/master/sample/test
+In case of mp4 files, df3d will first expand them into images using ffmpeg. Please check the sample data for a real example: https://github.com/NeLy-EPFL/DeepFly3D/tree/master/sample/test
 
 # Basic Usage
 
@@ -153,7 +153,7 @@ df3d-cli /your/image/path
 This command assumes your cameras are numbered in the default order:
 
 <p align="center">
-  <img src="https://github.com/NeLy-EPFL/DeepFly3D/blob/master/images/camera_order.png">
+  <img src="./images/camera_order.png">
 </p>
 
 in which case your data will look like this if cameras 0, 1, 2 are shown left-to-right in the top row and cameras 4, 5, 6 are show left-to-right in the bottom row:
@@ -172,8 +172,9 @@ then your order is 6 5 4 3 2 1 0, so you'd need to run `df3d-cli /your/image/pat
 ```
 usage: df3d-cli [-h] [-v] [-vv] [-d] [--output-folder OUTPUT_FOLDER] [-r] [-f]
                 [-o] [-n NUM_IMAGES_MAX]
-                [--order [CAMERA_IDS [CAMERA_IDS ...]]] [--video-2d]
-                [--video-3d] [--skip-pose-estimation]
+                [--order [CAMERA_IDS [CAMERA_IDS ...]]] [--skip-pose-estimation]
+                [--video-2d] [--video-3d] [--output-fps OUTPUT_FPS]
+                [--batch-size BATCH_SIZE] [--pin-memory-disabled]
                 INPUT
 
 DeepFly3D pose estimation
@@ -197,13 +198,24 @@ optional arguments:
                         results
   -n NUM_IMAGES_MAX, --num-images-max NUM_IMAGES_MAX
                         Maximal number of images to process.
-  --order [CAMERA_IDS [CAMERA_IDS ...]], --camera-ids [CAMERA_IDS [CAMERA_IDS ...]]
+  --order [CAMERA_IDS [CAMERA_IDS ...]]
                         Ordering of the cameras provided as an ordered list of
                         ids. Example: 0 1 4 3 2 5 6.
+  --skip-pose-estimation
+                        Skip 2D and 3D pose estimation. Use in combination with --video-2d
+                        or --video-3d to generate videos without rerunning pose estimation.
   --video-2d            Generate pose2d videos
   --video-3d            Generate pose3d videos
-  --skip-pose-estimation
-                        Skip 2D and 3D pose estimation
+  --output-fps OUTPUT_FPS
+                        FPS for output videos. If not specified, uses the FPS from
+                        the input videos.
+  --batch-size BATCH_SIZE
+                        Batch size for inference - how many images are processed
+                        through the model at once
+  --pin-memory-disabled
+                        Whether to disable `pin_memory` in the dataloader.
+                        Keeping pin memory enabled usually speeds up processing,
+                        but sometimes leads to memory leaks.
 ```
 
 Therefore, you can create advanced queries in df3d-cli, for example:
@@ -218,6 +230,7 @@ df3d-cli -f /path/to/text.txt    \  # process each line from the text file
          --skip-pose-estimation  \  # will not run 2d pose estimation, instead will do calibration, triangulation and will save results
          --video-2d              \  # will make 2d video for each folder 
          --video-3d              \  # will make 3d video for each folder
+         --output-fps 15.0       \  # set output video FPS to 15 (instead of using the input video FPS)
 ```
 
 To test df3d-cli, you run it on a folder for only 100 images, make videos, and print agressivelly for debugging:
@@ -299,6 +312,8 @@ Using the flag --video-2d with df3d-cli will create the following video:
 Using the flag --video-3d with df3d-cli will create the following video:
 ![Alt text](./images/out3d.gif?raw=true "Title")
 
+When generating videos with `--video-2d` or `--video-3d`, you can control the output video frame rate using the `--output-fps` flag. If not specified, the output video framerate will be set to equal the input videos' framerate.
+
 # Output
 
 df3d-cli saves results under df3d_result.pk file. You can read it using,

df3d/cli.py

@@ -148,6 +148,13 @@ def parse_cli_args():
         help="Whether to disable `pin_memory` in the dataloader. Keeping this enabled usually speeds up the processing, but sometimes leads to memory leaks. See https://github.com/NeLy-EPFL/DeepFly2D/issues/6",
         action="store_true",
     )
+    parser.add_argument(
+        "--output-fps",
+        help="FPS for output videos. If not specified, uses the FPS from the input "
+             "videos. If specified, overrides the input video FPS.",
+        type=float,
+        default=None,
+    )
     args = parser.parse_args()
     args.input_folder = Path(args.input_folder).expanduser().resolve()
     if args.output_folder is None:
@@ -295,9 +302,12 @@ def run(args):
         core.calibrate_calc(0, core.max_img_id)
         core.save()
 
+    # Use output_fps if specified, otherwise use core.fps which comes from the input videos
+    fps = args.output_fps if args.output_fps is not None else core.fps
+
     if args.video_2d:
         video.make_pose2d_video(
-            core.plot_2d, core.num_images, core.input_folder, core.output_folder
+            core.plot_2d, core.num_images, core.input_folder, core.output_folder, fps=fps
         )
 
     if args.video_3d:
@@ -307,6 +317,7 @@ def run(args):
             core.num_images,
             core.input_folder,
             core.output_folder,
+            fps=fps,
         )
 
     if args.delete_images:

df3d/core.py

@@ -76,6 +76,7 @@ def __init__(
             self.output_folder = output_folder
 
         self.expand_videos()  # turn .mp4 into .jpg
+        self.fps = self.get_fps()
         self.num_images_max = num_images_max if num_images_max is not None else 0
         self.max_img_id = get_max_img_id(self.input_folder)
         if self.num_images_max > 0:
@@ -412,6 +413,36 @@ def setup_camera_ordering(self, camera_ordering) -> np.ndarray:
         # self.cidread2cid, self.cid2cidread = read_camera_order(self.output_folder)
         return np.array(camera_ordering)
 
+    def get_fps(self):
+        rates = []
+        for vid in glob.glob(os.path.join(self.input_folder, "camera_?.mp4")):
+            cmd = ["ffprobe", "-v", "error", "-select_streams", "v:0",
+                   "-show_entries", "stream=avg_frame_rate", "-of",
+                   "default=noprint_wrappers=1:nokey=1", vid]
+            try:
+                rates.append(subprocess.check_output(cmd, text=True))
+            except:
+                logger.warning(f"Command failed: {' '.join(cmd)}")
+                break
+        if len(rates) == 0:
+            return None
+        if any(rate != rates[0] for rate in rates):
+            logger.warning("Framerates of input videos differ from one another,"
+                           f" using the first one: {rates}")
+        rate = rates[0]
+        try:
+            return float(rate)
+        except ValueError:
+            pass
+        try:
+            numerator, denominator = map(int, rate.split('/'))
+            return numerator / denominator if denominator != 0 else None
+        except ValueError:
+            pass
+        logger.warning(f'Could not parse framerate from string "{rate}" returned'
+                       ' by ffprobe command, so setting fps to None.')
+        return None
+
     def expand_videos(self):
         """expands video camera_x.mp4 into set of images camera_x_img_y.jpg"""
         for vid in glob.glob(os.path.join(self.input_folder, "camera_?.mp4")):

df3d/video.py

@@ -15,9 +15,11 @@
 img3d_aspect = (2, 2)  # this is the aspect ration for one image on the 3d video's grid
 img2d_aspect = (2, 1)  # this is the aspect ration for one image on the 3d video's grid
 video_width = 5000  # total width of the 2d and 3d videos
+default_fps = 30
 
 
-def make_pose2d_video(plot_2d, num_images, input_folder, output_folder):
+def make_pose2d_video(plot_2d, num_images, input_folder,
+                      output_folder, fps=default_fps):
     """Creates pose2d estimation videos and writes it to output_folder.
 
     Parameters:
@@ -43,10 +45,11 @@ def stack(img_id):
 
     video_name = 'video_pose2d_' + input_folder.replace('/', '_') + '.mp4'
     video_path = os.path.join(input_folder, output_folder, video_name)
-    _make_video(video_path, generator)
+    _make_video(video_path, generator, fps=fps)
 
 
-def make_pose3d_video(points3d, plot_2d, num_images, input_folder, output_folder):
+def make_pose3d_video(points3d, plot_2d, num_images, input_folder,
+                      output_folder, fps=default_fps):
     """Creates pose3d estimation videos and writes it to output_folder.
 
     Parameters:
@@ -72,24 +75,25 @@ def stack(img_id):
     generator = imgs_generator()
     video_name = 'video_pose3d_' + input_folder.replace('/', '_') + '.mp4'
     video_path = os.path.join(input_folder, output_folder, video_name)
-    _make_video(video_path, generator)
+    _make_video(video_path, generator, fps=fps)
 
 
-def _make_video(video_path, imgs):
+def _make_video(video_path, imgs, fps=default_fps):
     """Code used to generate a video using cv2.
 
     Parameters:
     video_path: a path ending with .mp4, for instance: "/results/pose2d.mp4"
     imgs: an iterable or generator with the images to turn into a video
     """
+    if fps is None:
+        fps = default_fps
 
     first_frame = next(imgs)
     imgs = itertools.chain([first_frame], imgs)
 
     shape = int(first_frame.shape[1]), int(first_frame.shape[0])
     logger.debug('Saving video to: ' + video_path)
     fourcc = cv2.VideoWriter_fourcc(*'mp4v')
-    fps = 30
     output_shape = _resize(current_shape=shape, new_width=video_width)
     logger.debug('Video size is: {}'.format(output_shape))
     video_writer = cv2.VideoWriter(video_path, fourcc, fps, output_shape)

tests/run_df3d_on_sample_data.sh

@@ -2,4 +2,4 @@
 # You can run this to confirm that your DeepFly3D installation is working correctly.
 # If successful, you will get df3d results files in the directory `data/reference_df3d/`
 
-df3d-cli data/reference/ -v --video-2d --video-3d
+df3d-cli data/reference/ -v --video-2d --video-3d --output-fps 5

tests/test_df3d.py

@@ -22,6 +22,7 @@
 TEST_DATA_LOCATION_RESULT_FILE_3D = f"{TEST_DATA_LOCATION_RESULT}/df3d_result_3d.pkl"
 TEST_DATA_LOCATION_REFERENCE_VIDEO_2D = f"{TEST_DATA_LOCATION_RESULT}/video_pose2d.mp4"
 TEST_DATA_LOCATION_REFERENCE_VIDEO_3D = f"{TEST_DATA_LOCATION_RESULT}/video_pose3d.mp4"
+TEST_DATA_VIDEO_FRAMERATE = 5
 TEST_DATA_LOCATION_WORKING = f"{TEST_DATA_LOCATION}/working"
 TEST_DATA_LOCATION_WORKING_RESULT = f"{TEST_DATA_LOCATION_WORKING}_df3d"
 
@@ -254,7 +255,8 @@ def test_video_2d(self):
         )
 
         df3d.video.make_pose2d_video(
-            core.plot_2d, core.num_images, core.input_folder, core.output_folder
+            core.plot_2d, core.num_images, core.input_folder,
+            core.output_folder, fps=TEST_DATA_VIDEO_FRAMERATE
         )
 
         video_name = "video_pose2d_" + core.input_folder.replace("/", "_") + ".mp4"
@@ -298,6 +300,7 @@ def test_video_3d(self):
             core.num_images,
             core.input_folder,
             core.output_folder,
+            fps=TEST_DATA_VIDEO_FRAMERATE,
         )
 
         video_name = "video_pose3d_" + core.input_folder.replace("/", "_") + ".mp4"