From 1b5962a82d2a93ffaf73df1748d5d7d83750f08a Mon Sep 17 00:00:00 2001 From: Sebastien Ehrhardt Date: Tue, 19 Dec 2023 11:55:41 +0000 Subject: [PATCH 1/8] Update stream.pyx Add skeleton moving on to test now import c++ objects remove some stuff works final touches, need to implement others add side data tests --- av/stream.pxd | 3 +++ av/stream.pyx | 24 +++++++++++++++++++ include/libavcodec/avcodec.pxd | 40 ++++++++++++++++++++++++++++++++ include/libavformat/avformat.pxd | 4 ++++ include/libavutil/avutil.pxd | 5 +++- tests/test_streams.py | 13 +++++++++++ 6 files changed, 88 insertions(+), 1 deletion(-) diff --git a/av/stream.pxd b/av/stream.pxd index c847f641e..41770a81a 100644 --- a/av/stream.pxd +++ b/av/stream.pxd @@ -1,3 +1,4 @@ +from libc.stdint cimport int64_t, int32_t cimport libav as lib from av.codec.context cimport CodecContext @@ -12,6 +13,8 @@ cdef class Stream: # Stream attributes. cdef readonly Container container cdef readonly dict metadata + cdef readonly int nb_side_data + cdef readonly dict side_data # CodecContext attributes. cdef readonly CodecContext codec_context diff --git a/av/stream.pyx b/av/stream.pyx index f9b6d7ec5..55b738bf8 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -1,7 +1,9 @@ import warnings +from libc.stdint cimport int64_t, uint8_t, int32_t cimport libav as lib +from av.enum cimport define_enum from av.error cimport err_check from av.packet cimport Packet from av.utils cimport ( @@ -16,6 +18,12 @@ from av.deprecation import AVDeprecationWarning cdef object _cinit_bypass_sentinel = object() +SideData = define_enum('SideData', __name__, ( + ('DISPLAYMATRIX', lib.AV_PKT_DATA_DISPLAYMATRIX , + """Display Matrix"""), + # If necessary more can be added from + # https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html#ga9a80bfcacc586b483a973272800edb97 +)) cdef Stream wrap_stream(Container container, lib.AVStream *c_stream, CodecContext codec_context): """Build an av.Stream for an existing AVStream. @@ -79,6 +87,16 @@ cdef class Stream: if self.codec_context: self.codec_context.stream_index = stream.index + self.nb_side_data = stream.nb_side_data + self.side_data = {} + + if self.nb_side_data: + for i in range(self.nb_side_data): + # Get side_data that we know how to get + if SideData.get(stream.side_data[i].type) == 'DISPLAYMATRIX': + # Use dumpsidedata maybe here I guess : https://www.ffmpeg.org/doxygen/trunk/dump_8c_source.html#l00430 + self.side_data['DISPLAYMATRIX'] = lib.av_display_rotation_get(stream.side_data[i].data) + self.metadata = avdict_to_dict( stream.metadata, encoding=self.container.metadata_encoding, @@ -103,6 +121,12 @@ cdef class Stream: AVDeprecationWarning ) + + if name == 'side_data': + return self.side_data + elif name == 'nb_side_data': + return self.nb_side_data + # Convenience getter for codec context properties. if self.codec_context is not None: return getattr(self.codec_context, name) diff --git a/include/libavcodec/avcodec.pxd b/include/libavcodec/avcodec.pxd index af7fe6150..87686c68d 100644 --- a/include/libavcodec/avcodec.pxd +++ b/include/libavcodec/avcodec.pxd @@ -258,6 +258,46 @@ cdef extern from "libavcodec/avcodec.h" nogil: cdef int AV_NUM_DATA_POINTERS + cdef enum AVPacketSideDataType: + AV_PKT_DATA_PALETTE + AV_PKT_DATA_NEW_EXTRADATA + AV_PKT_DATA_PARAM_CHANGE + AV_PKT_DATA_H263_MB_INFO + AV_PKT_DATA_REPLAYGAIN + AV_PKT_DATA_DISPLAYMATRIX + AV_PKT_DATA_STEREO3D + AV_PKT_DATA_AUDIO_SERVICE_TYPE + AV_PKT_DATA_QUALITY_STATS + AV_PKT_DATA_FALLBACK_TRACK + AV_PKT_DATA_CPB_PROPERTIES + AV_PKT_DATA_SKIP_SAMPLES + AV_PKT_DATA_JP_DUALMONO + AV_PKT_DATA_STRINGS_METADATA + AV_PKT_DATA_SUBTITLE_POSITION + AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL + AV_PKT_DATA_WEBVTT_IDENTIFIER + AV_PKT_DATA_WEBVTT_SETTINGS + AV_PKT_DATA_METADATA_UPDATE + AV_PKT_DATA_MPEGTS_STREAM_ID + AV_PKT_DATA_MASTERING_DISPLAY_METADATA + AV_PKT_DATA_SPHERICAL + AV_PKT_DATA_CONTENT_LIGHT_LEVEL + AV_PKT_DATA_A53_CC + AV_PKT_DATA_ENCRYPTION_INIT_INFO + AV_PKT_DATA_ENCRYPTION_INFO + AV_PKT_DATA_AFD + AV_PKT_DATA_PRFT + AV_PKT_DATA_ICC_PROFILE + AV_PKT_DATA_DOVI_CONF + AV_PKT_DATA_S12M_TIMECODE + AV_PKT_DATA_DYNAMIC_HDR10_PLUS + AV_PKT_DATA_NB + + cdef struct AVPacketSideData: + uint8_t *data; + size_t size; + AVPacketSideDataType type; + cdef enum AVFrameSideDataType: AV_FRAME_DATA_PANSCAN AV_FRAME_DATA_A53_CC diff --git a/include/libavformat/avformat.pxd b/include/libavformat/avformat.pxd index a4d5991d8..224e76b4e 100644 --- a/include/libavformat/avformat.pxd +++ b/include/libavformat/avformat.pxd @@ -47,6 +47,10 @@ cdef extern from "libavformat/avformat.h" nogil: AVRational r_frame_rate AVRational sample_aspect_ratio + int nb_side_data + AVPacketSideData *side_data + + # http://ffmpeg.org/doxygen/trunk/structAVIOContext.html cdef struct AVIOContext: unsigned char* buffer diff --git a/include/libavutil/avutil.pxd b/include/libavutil/avutil.pxd index 09210bc0c..08f973cac 100644 --- a/include/libavutil/avutil.pxd +++ b/include/libavutil/avutil.pxd @@ -1,9 +1,12 @@ -from libc.stdint cimport int64_t, uint8_t, uint64_t +from libc.stdint cimport int64_t, uint8_t, uint64_t, int32_t cdef extern from "libavutil/mathematics.h" nogil: pass +cdef extern from "libavutil/display.h" nogil: + cdef double av_display_rotation_get(const int32_t matrix[9]) + cdef extern from "libavutil/rational.h" nogil: cdef int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max) diff --git a/tests/test_streams.py b/tests/test_streams.py index b2871d43d..a4e3eae5e 100644 --- a/tests/test_streams.py +++ b/tests/test_streams.py @@ -24,3 +24,16 @@ def test_selection(self): self.assertEqual([video], container.streams.get(video=(0,))) # TODO: Find something in the fate suite with video, audio, and subtitles. + + def test_noside_data(self): + container = av.open(fate_suite("h264/interlaced_crop.mp4")) + video = container.streams.video[0] + + self.assertEqual(video.nb_side_data, 0) + + def test_side_data(self): + container = av.open(fate_suite("mov/displaymatrix.mov")) + video = container.streams.video[0] + + self.assertEqual(video.nb_side_data, 1) + self.assertEqual(video.side_data["DISPLAYMATRIX"], -90.0) From 5bdfb85c12cf791bf48e6809c3764d5b46fc85ec Mon Sep 17 00:00:00 2001 From: Sebastien Ehrhardt Date: Tue, 19 Dec 2023 12:02:57 +0000 Subject: [PATCH 2/8] clean --- av/stream.pxd | 1 - av/stream.pyx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/av/stream.pxd b/av/stream.pxd index 41770a81a..d39585167 100644 --- a/av/stream.pxd +++ b/av/stream.pxd @@ -1,4 +1,3 @@ -from libc.stdint cimport int64_t, int32_t cimport libav as lib from av.codec.context cimport CodecContext diff --git a/av/stream.pyx b/av/stream.pyx index 55b738bf8..392f3f6b3 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -1,6 +1,6 @@ import warnings -from libc.stdint cimport int64_t, uint8_t, int32_t +from libc.stdint cimport int32_t cimport libav as lib from av.enum cimport define_enum From ba7bee1c383ec5fd2d4411f463338b1c2c59c763 Mon Sep 17 00:00:00 2001 From: Sebastien Ehrhardt Date: Tue, 19 Dec 2023 13:14:15 +0000 Subject: [PATCH 3/8] comments --- av/stream.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/av/stream.pyx b/av/stream.pyx index 392f3f6b3..9cc4454ce 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -91,10 +91,9 @@ cdef class Stream: self.side_data = {} if self.nb_side_data: + # Loop over side data to fill up the side_data attribute for i in range(self.nb_side_data): - # Get side_data that we know how to get if SideData.get(stream.side_data[i].type) == 'DISPLAYMATRIX': - # Use dumpsidedata maybe here I guess : https://www.ffmpeg.org/doxygen/trunk/dump_8c_source.html#l00430 self.side_data['DISPLAYMATRIX'] = lib.av_display_rotation_get(stream.side_data[i].data) self.metadata = avdict_to_dict( From c4395879bd6b655b8bbcaf1fbb1bcc651d7a6965 Mon Sep 17 00:00:00 2001 From: Sebastien Ehrhardt Date: Tue, 19 Dec 2023 13:41:10 +0000 Subject: [PATCH 4/8] cosmetic --- av/stream.pxd | 1 + av/stream.pyx | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/av/stream.pxd b/av/stream.pxd index d39585167..8ebda5704 100644 --- a/av/stream.pxd +++ b/av/stream.pxd @@ -21,6 +21,7 @@ cdef class Stream: # Private API. cdef _init(self, Container, lib.AVStream*, CodecContext) cdef _finalize_for_output(self) + cdef _get_side_data(self, lib.AVStream *stream) cdef _set_time_base(self, value) cdef _set_id(self, value) diff --git a/av/stream.pyx b/av/stream.pyx index 9cc4454ce..f44cc31fd 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -87,14 +87,7 @@ cdef class Stream: if self.codec_context: self.codec_context.stream_index = stream.index - self.nb_side_data = stream.nb_side_data - self.side_data = {} - - if self.nb_side_data: - # Loop over side data to fill up the side_data attribute - for i in range(self.nb_side_data): - if SideData.get(stream.side_data[i].type) == 'DISPLAYMATRIX': - self.side_data['DISPLAYMATRIX'] = lib.av_display_rotation_get(stream.side_data[i].data) + self.nb_side_data, self.side_data = self._get_side_data(stream) self.metadata = avdict_to_dict( stream.metadata, @@ -190,6 +183,23 @@ cdef class Stream: return self.codec_context.decode(packet) + cdef _get_side_data(self, lib.AVStream *stream): + """ + Get DISPLAYMATRIX SideDate from a lib.AVStream object. + + :return: ``tuple`` of (number_of_side_date, side_data dict) . + """ + nb_side_data = stream.nb_side_data + side_data = {} + + if nb_side_data: + # Loop over side data to fill up the side_data attribute + for i in range(nb_side_data): + if SideData.get(stream.side_data[i].type) == 'DISPLAYMATRIX': + side_data['DISPLAYMATRIX'] = lib.av_display_rotation_get(stream.side_data[i].data) + + return nb_side_data, side_data + property id: """ The format-specific ID of this stream. From c566ec9ef8e806c6fdd228631efed75e72b14347 Mon Sep 17 00:00:00 2001 From: Sebastien Ehrhardt Date: Tue, 19 Dec 2023 14:06:17 +0000 Subject: [PATCH 5/8] typo --- av/stream.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/av/stream.pyx b/av/stream.pyx index f44cc31fd..9fe7fa3fe 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -187,7 +187,7 @@ cdef class Stream: """ Get DISPLAYMATRIX SideDate from a lib.AVStream object. - :return: ``tuple`` of (number_of_side_date, side_data dict) . + :return: ``tuple`` of (number_of_side_data, side_data dict) . """ nb_side_data = stream.nb_side_data side_data = {} From c64e84b38b0ef12151d089b36166162552758608 Mon Sep 17 00:00:00 2001 From: WyattBlue Date: Tue, 19 Dec 2023 19:39:15 -0500 Subject: [PATCH 6/8] Fix formatting --- av/stream.pyx | 25 ++++++------- include/libavcodec/avcodec.pxd | 66 +++++++++++++++++----------------- 2 files changed, 44 insertions(+), 47 deletions(-) diff --git a/av/stream.pyx b/av/stream.pyx index 9fe7fa3fe..a724029b7 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -18,11 +18,11 @@ from av.deprecation import AVDeprecationWarning cdef object _cinit_bypass_sentinel = object() -SideData = define_enum('SideData', __name__, ( - ('DISPLAYMATRIX', lib.AV_PKT_DATA_DISPLAYMATRIX , - """Display Matrix"""), - # If necessary more can be added from - # https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html#ga9a80bfcacc586b483a973272800edb97 + +# If necessary more can be added from +# https://ffmpeg.org/doxygen/trunk/group__lavc__packet.html#ga9a80bfcacc586b483a973272800edb97 +SideData = define_enum("SideData", __name__, ( + ("DISPLAYMATRIX", lib.AV_PKT_DATA_DISPLAYMATRIX, "Display Matrix"), )) cdef Stream wrap_stream(Container container, lib.AVStream *c_stream, CodecContext codec_context): @@ -113,10 +113,9 @@ cdef class Stream: AVDeprecationWarning ) - - if name == 'side_data': + if name == "side_data": return self.side_data - elif name == 'nb_side_data': + elif name == "nb_side_data": return self.nb_side_data # Convenience getter for codec context properties. @@ -184,19 +183,17 @@ cdef class Stream: return self.codec_context.decode(packet) cdef _get_side_data(self, lib.AVStream *stream): - """ - Get DISPLAYMATRIX SideDate from a lib.AVStream object. + # Get DISPLAYMATRIX SideDate from a lib.AVStream object. + # Returns: tuple[number_of_side_data, dict] - :return: ``tuple`` of (number_of_side_data, side_data dict) . - """ nb_side_data = stream.nb_side_data side_data = {} if nb_side_data: # Loop over side data to fill up the side_data attribute for i in range(nb_side_data): - if SideData.get(stream.side_data[i].type) == 'DISPLAYMATRIX': - side_data['DISPLAYMATRIX'] = lib.av_display_rotation_get(stream.side_data[i].data) + if SideData.get(stream.side_data[i].type) == "DISPLAYMATRIX": + side_data["DISPLAYMATRIX"] = lib.av_display_rotation_get(stream.side_data[i].data) return nb_side_data, side_data diff --git a/include/libavcodec/avcodec.pxd b/include/libavcodec/avcodec.pxd index 87686c68d..1daeac15b 100644 --- a/include/libavcodec/avcodec.pxd +++ b/include/libavcodec/avcodec.pxd @@ -259,39 +259,39 @@ cdef extern from "libavcodec/avcodec.h" nogil: cdef int AV_NUM_DATA_POINTERS cdef enum AVPacketSideDataType: - AV_PKT_DATA_PALETTE - AV_PKT_DATA_NEW_EXTRADATA - AV_PKT_DATA_PARAM_CHANGE - AV_PKT_DATA_H263_MB_INFO - AV_PKT_DATA_REPLAYGAIN - AV_PKT_DATA_DISPLAYMATRIX - AV_PKT_DATA_STEREO3D - AV_PKT_DATA_AUDIO_SERVICE_TYPE - AV_PKT_DATA_QUALITY_STATS - AV_PKT_DATA_FALLBACK_TRACK - AV_PKT_DATA_CPB_PROPERTIES - AV_PKT_DATA_SKIP_SAMPLES - AV_PKT_DATA_JP_DUALMONO - AV_PKT_DATA_STRINGS_METADATA - AV_PKT_DATA_SUBTITLE_POSITION - AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL - AV_PKT_DATA_WEBVTT_IDENTIFIER - AV_PKT_DATA_WEBVTT_SETTINGS - AV_PKT_DATA_METADATA_UPDATE - AV_PKT_DATA_MPEGTS_STREAM_ID - AV_PKT_DATA_MASTERING_DISPLAY_METADATA - AV_PKT_DATA_SPHERICAL - AV_PKT_DATA_CONTENT_LIGHT_LEVEL - AV_PKT_DATA_A53_CC - AV_PKT_DATA_ENCRYPTION_INIT_INFO - AV_PKT_DATA_ENCRYPTION_INFO - AV_PKT_DATA_AFD - AV_PKT_DATA_PRFT - AV_PKT_DATA_ICC_PROFILE - AV_PKT_DATA_DOVI_CONF - AV_PKT_DATA_S12M_TIMECODE - AV_PKT_DATA_DYNAMIC_HDR10_PLUS - AV_PKT_DATA_NB + AV_PKT_DATA_PALETTE + AV_PKT_DATA_NEW_EXTRADATA + AV_PKT_DATA_PARAM_CHANGE + AV_PKT_DATA_H263_MB_INFO + AV_PKT_DATA_REPLAYGAIN + AV_PKT_DATA_DISPLAYMATRIX + AV_PKT_DATA_STEREO3D + AV_PKT_DATA_AUDIO_SERVICE_TYPE + AV_PKT_DATA_QUALITY_STATS + AV_PKT_DATA_FALLBACK_TRACK + AV_PKT_DATA_CPB_PROPERTIES + AV_PKT_DATA_SKIP_SAMPLES + AV_PKT_DATA_JP_DUALMONO + AV_PKT_DATA_STRINGS_METADATA + AV_PKT_DATA_SUBTITLE_POSITION + AV_PKT_DATA_MATROSKA_BLOCKADDITIONAL + AV_PKT_DATA_WEBVTT_IDENTIFIER + AV_PKT_DATA_WEBVTT_SETTINGS + AV_PKT_DATA_METADATA_UPDATE + AV_PKT_DATA_MPEGTS_STREAM_ID + AV_PKT_DATA_MASTERING_DISPLAY_METADATA + AV_PKT_DATA_SPHERICAL + AV_PKT_DATA_CONTENT_LIGHT_LEVEL + AV_PKT_DATA_A53_CC + AV_PKT_DATA_ENCRYPTION_INIT_INFO + AV_PKT_DATA_ENCRYPTION_INFO + AV_PKT_DATA_AFD + AV_PKT_DATA_PRFT + AV_PKT_DATA_ICC_PROFILE + AV_PKT_DATA_DOVI_CONF + AV_PKT_DATA_S12M_TIMECODE + AV_PKT_DATA_DYNAMIC_HDR10_PLUS + AV_PKT_DATA_NB cdef struct AVPacketSideData: uint8_t *data; From b2a108bdb41293980347bc01a0f9216a7a6e8943 Mon Sep 17 00:00:00 2001 From: WyattBlue Date: Tue, 19 Dec 2023 19:49:31 -0500 Subject: [PATCH 7/8] Don't need if statement --- av/stream.pyx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/av/stream.pyx b/av/stream.pyx index a724029b7..08a6bd97b 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -184,16 +184,14 @@ cdef class Stream: cdef _get_side_data(self, lib.AVStream *stream): # Get DISPLAYMATRIX SideDate from a lib.AVStream object. - # Returns: tuple[number_of_side_data, dict] + # Returns: tuple[int, dict[str, Any]] nb_side_data = stream.nb_side_data side_data = {} - if nb_side_data: - # Loop over side data to fill up the side_data attribute - for i in range(nb_side_data): - if SideData.get(stream.side_data[i].type) == "DISPLAYMATRIX": - side_data["DISPLAYMATRIX"] = lib.av_display_rotation_get(stream.side_data[i].data) + for i in range(nb_side_data): + if SideData.get(stream.side_data[i].type) == "DISPLAYMATRIX": + side_data["DISPLAYMATRIX"] = lib.av_display_rotation_get(stream.side_data[i].data) return nb_side_data, side_data From db724c94d6d09ded6712e1b541ffd1abd59ee120 Mon Sep 17 00:00:00 2001 From: WyattBlue Date: Tue, 19 Dec 2023 21:21:38 -0500 Subject: [PATCH 8/8] Remove indirection --- av/stream.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/av/stream.pyx b/av/stream.pyx index 08a6bd97b..f65cee609 100644 --- a/av/stream.pyx +++ b/av/stream.pyx @@ -190,7 +190,8 @@ cdef class Stream: side_data = {} for i in range(nb_side_data): - if SideData.get(stream.side_data[i].type) == "DISPLAYMATRIX": + # Based on: https://www.ffmpeg.org/doxygen/trunk/dump_8c_source.html#l00430 + if stream.side_data[i].type == lib.AV_PKT_DATA_DISPLAYMATRIX: side_data["DISPLAYMATRIX"] = lib.av_display_rotation_get(stream.side_data[i].data) return nb_side_data, side_data