Feature Request: Support adding Frame side data.

[anthornt]

Overview

I would like to be able to add side data to Frames. Specifically closed caption data (Type.A53_CC).

Existing FFmpeg API

av_frame_new_side_data()

Expected PyAV API

I would like to add side data to selected frames, for example by adding a new API Frame.side_data.add().

Example (modified examples/numpy/generate_video.py):

stream = container.add_stream("libx264", rate=fps)
...
for frame_i in range(total_frames):
    ...
    frame = av.VideoFrame.from_ndarray(img, format="rgb24")

    if (frame_i % fps) == 0:
        # Add closed caption
        data = bytearray()
        ...
        frame.side_data.add(data, Type.A53_CC)

Investigation

I tried the attached patches to generate video with 608 style closed captions by modifying the generate video example.

I didn't manage to get the Frame.side_data updated correctly.

20230817.0.patch

[anthornt]

This patchset looks better:
20230821.0.patch

diff --git a/av/sidedata/sidedata.pyx b/av/sidedata/sidedata.pyx
index ec7de59..3dd0917 100644
--- a/av/sidedata/sidedata.pyx
+++ b/av/sidedata/sidedata.pyx
@@ -4,6 +4,9 @@ from collections.abc import Mapping
 
 from av.sidedata.motionvectors import MotionVectors
 
+from libc.string cimport memcpy
+
+from av.bytesource cimport ByteSource, bytesource
 
 cdef object _cinit_bypass_sentinel = object()
 
@@ -99,6 +102,17 @@ cdef class _SideDataContainer(object):
         type_ = Type.get(key)
         return self._by_type[type_]
 
+    def add(self, buffer, type):
+        # Add Frame side data
+        cdef ByteSource source = bytesource(buffer)
+        i = self.frame.ptr.nb_side_data
+        ptr = lib.av_frame_new_side_data(self.frame.ptr, type, source.length)
+        memcpy(ptr.data, source.ptr, source.length)
+
+        # Update side_data
+        cdef SideData data = wrap_side_data(self.frame, i)
+        self._by_index.append(data)
+        self._by_type[data.type] = data
 
 class SideDataContainer(_SideDataContainer, Mapping):
     pass
diff --git a/examples/numpy/generate_video.py b/examples/numpy/generate_video.py
index 80145d5..08c55e9 100644
--- a/examples/numpy/generate_video.py
+++ b/examples/numpy/generate_video.py
@@ -1,15 +1,40 @@
 import numpy as np
 
 import av
+from av.sidedata.sidedata import Type
 
+class CC608Data():
+    cc_type_byte = 0xfc # Field 1
+    odd_parity = bytearray([0] * 128)
+    for idx in range(128):
+        if idx.bit_count() & 1:
+            odd_parity[idx] = idx
+        else:
+            odd_parity[idx] = idx | 0x80
 
-duration = 4
+    def __init__(self):
+        self.ba = bytearray()
+
+    def appendPair(self, cc_data_1, cc_data_2):
+        self.ba.append(self.cc_type_byte)
+        self.ba.append(self.odd_parity[cc_data_1])
+        self.ba.append(self.odd_parity[cc_data_2])
+
+    def appendString(self, string):
+        bytes = bytearray(string.encode('ascii'))
+        if len(bytes) & 1:
+            bytes.append(0)
+        for i in range(0, len(bytes), 2):
+            self.appendPair(bytes[i], bytes[i+1])
+
+
+duration = 20
 fps = 24
 total_frames = duration * fps
 
 container = av.open("test.mp4", mode="w")
 
-stream = container.add_stream("mpeg4", rate=fps)
+stream = container.add_stream("libx264", rate=fps)
 stream.width = 480
 stream.height = 320
 stream.pix_fmt = "yuv420p"
@@ -24,6 +49,21 @@ for frame_i in range(total_frames):
     img = np.round(255 * img).astype(np.uint8)
 
     frame = av.VideoFrame.from_ndarray(img, format="rgb24")
+
+    if (frame_i % fps) == 0:
+        # Add pop-on closed caption
+        data = CC608Data()
+        data.appendPair(0x14, 0x20) # RCL
+        data.appendPair(0x14, 0x2e) # ENM
+        data.appendPair(0x14, 0x60) # Row 15, White
+        data.appendString("CC1: 00:%02d:%02d" % ((frame_i/(60*fps))%60, (frame_i/fps)%60))
+        data.appendPair(0x14, 0x2c) # EDM
+        data.appendPair(0x14, 0x2f) # EOC
+        frame.side_data.add(data.ba, Type.A53_CC)
+
+        for data in frame.side_data:
+            print("Frame %d added %s" % (frame_i, data))
+
     for packet in stream.encode(frame):
         container.mux(packet)
 
diff --git a/include/libavutil/frame.pxd b/include/libavutil/frame.pxd
index aa8dc3a..7dd1d8e 100644
--- a/include/libavutil/frame.pxd
+++ b/include/libavutil/frame.pxd
@@ -12,3 +12,4 @@ cdef extern from "libavutil/frame.h" nogil:
     cdef int av_frame_copy(AVFrame *dst, const AVFrame *src)
     cdef int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
     cdef AVFrameSideData* av_frame_get_side_data(AVFrame *frame, AVFrameSideDataType type)
+    cdef AVFrameSideData* av_frame_new_side_data(AVFrame *frame, AVFrameSideDataType type, int size)

[github-actions]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[WyattBlue]

I think given #1249, we're going to do a one at a time approach for side data. For closed caption data specifically I don't understand why you can't use a subtitle stream.