From 368aee9d252ca17130787bed7e520b33021ae42b Mon Sep 17 00:00:00 2001 From: Romain Beauxis Date: Thu, 15 Oct 2026 15:38:48 +0200 Subject: [PATCH] avcodec/bsf: add the side_data bitstream filter Add a filter that removes side data of the given types from packets and from the stream, or keeps only the packets carrying side data of any of the given types, as the select mode of the sidedata frame filter does for frames, but with a list of types. The types are named by their handle, as option constants joined with '+'. --- Changelog | 1 + doc/bitstream_filters.texi | 24 ++++ libavcodec/bitstream_filters.c | 1 + libavcodec/bsf/Makefile | 1 + libavcodec/bsf/side_data.c | 111 ++++++++++++++++++ tests/fate/ffmpeg.mak | 19 +++ .../ref/fate/ffmpeg-side-data-bsf-keep-stream | 34 ++++++ tests/ref/fate/ffmpeg-side-data-bsf-remove | 8 ++ .../fate/ffmpeg-side-data-bsf-remove-stream | 25 ++++ tests/ref/fate/ffmpeg-side-data-bsf-select | 8 ++ .../ref/fate/ffmpeg-side-data-bsf-select-none | 6 + 11 files changed, 238 insertions(+) create mode 100644 libavcodec/bsf/side_data.c create mode 100644 tests/ref/fate/ffmpeg-side-data-bsf-keep-stream create mode 100644 tests/ref/fate/ffmpeg-side-data-bsf-remove create mode 100644 tests/ref/fate/ffmpeg-side-data-bsf-remove-stream create mode 100644 tests/ref/fate/ffmpeg-side-data-bsf-select create mode 100644 tests/ref/fate/ffmpeg-side-data-bsf-select-none diff --git a/Changelog b/Changelog index dfc35389ee..518a469de1 100644 --- a/Changelog +++ b/Changelog @@ -24,6 +24,7 @@ version 9.1: - Infinity IMM6 video decoder - Remove the asf_o demuxer - D3D12 DirectML zero-copy inference for the ONNX Runtime DNN backend +- side_data bitstream filter version 9.0: diff --git a/doc/bitstream_filters.texi b/doc/bitstream_filters.texi index cc1cd453a7..1e6438d46b 100644 --- a/doc/bitstream_filters.texi +++ b/doc/bitstream_filters.texi @@ -1059,6 +1059,30 @@ ffmpeg -i INPUT -c:a copy -bsf:a setts=pts=DTS out.mkv Log basic packet information. Mainly useful for testing, debugging, and development. +@section side_data +Remove side data from packets and from the stream, or select packets by their +side data. + +It accepts the following parameters: +@table @option +@item remove +Set the side data types to remove, joined with @samp{+}, from the packets and +from the side data of the stream. Each type is given by its handle; +@command{ffmpeg -h bsf=side_data} lists them. + +@item select +Set the side data types of the packets to keep, joined with @samp{+}. A packet +is kept if it carries any of them, and dropped otherwise. The selection looks +at the side data before @option{remove} applies. Dropping a packet breaks the +decoding of the packets that depend on it, so this is meant for analysis or for +streams where each packet decodes on its own. +@end table + +For example, to drop the display matrix and the replay gain of a stream: +@example +ffmpeg -i INPUT -c copy -bsf:v side_data=remove=displaymatrix+replaygain output.mkv +@end example + @section smpte436m_to_eia608 Convert from a @code{SMPTE_436M_ANC} data stream to a @code{EIA_608} stream, diff --git a/libavcodec/bitstream_filters.c b/libavcodec/bitstream_filters.c index ed57db5281..4a569b9de4 100644 --- a/libavcodec/bitstream_filters.c +++ b/libavcodec/bitstream_filters.c @@ -66,6 +66,7 @@ extern const FFBitStreamFilter ff_pgs_frame_merge_bsf; extern const FFBitStreamFilter ff_prores_metadata_bsf; extern const FFBitStreamFilter ff_remove_extradata_bsf; extern const FFBitStreamFilter ff_setts_bsf; +extern const FFBitStreamFilter ff_side_data_bsf; extern const FFBitStreamFilter ff_showinfo_bsf; extern const FFBitStreamFilter ff_smpte436m_to_eia608_bsf; extern const FFBitStreamFilter ff_text2movsub_bsf; diff --git a/libavcodec/bsf/Makefile b/libavcodec/bsf/Makefile index c0badf5ed9..ca110ab723 100644 --- a/libavcodec/bsf/Makefile +++ b/libavcodec/bsf/Makefile @@ -48,6 +48,7 @@ OBJS-$(CONFIG_PRORES_METADATA_BSF) += bsf/prores_metadata.o OBJS-$(CONFIG_REMOVE_EXTRADATA_BSF) += bsf/remove_extradata.o OBJS-$(CONFIG_SETTS_BSF) += bsf/setts.o OBJS-$(CONFIG_SHOWINFO_BSF) += bsf/showinfo.o +OBJS-$(CONFIG_SIDE_DATA_BSF) += bsf/side_data.o OBJS-$(CONFIG_SMPTE436M_TO_EIA608_BSF) += bsf/smpte436m_to_eia608.o OBJS-$(CONFIG_TEXT2MOVSUB_BSF) += bsf/movsub.o OBJS-$(CONFIG_TRACE_HEADERS_BSF) += bsf/trace_headers.o diff --git a/libavcodec/bsf/side_data.c b/libavcodec/bsf/side_data.c new file mode 100644 index 0000000000..f9eac5cc23 --- /dev/null +++ b/libavcodec/bsf/side_data.c @@ -0,0 +1,111 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Side data bitstream filter -- remove side data from packets and from the + * stream, or select packets by their side data. + */ + +#include "libavcodec/bsf.h" +#include "libavcodec/bsf_internal.h" +#include "libavcodec/packet_internal.h" + +#include "libavutil/opt.h" + +typedef struct SideDataContext { + const AVClass *class; + + int *remove; + unsigned nb_remove; + int *select; + unsigned nb_select; +} SideDataContext; + +static int side_data_init(AVBSFContext *ctx) +{ + SideDataContext *s = ctx->priv_data; + + for (unsigned i = 0; i < s->nb_remove; i++) + av_packet_side_data_remove(ctx->par_out->coded_side_data, + &ctx->par_out->nb_coded_side_data, + s->remove[i]); + + return 0; +} + +static int is_selected(const SideDataContext *s, const AVPacket *pkt) +{ + if (!s->nb_select) + return 1; + + for (unsigned i = 0; i < s->nb_select; i++) + if (av_packet_side_data_get(pkt->side_data, pkt->side_data_elems, + s->select[i])) + return 1; + + return 0; +} + +static int side_data_filter(AVBSFContext *ctx, AVPacket *pkt) +{ + SideDataContext *s = ctx->priv_data; + int ret = ff_bsf_get_packet_ref(ctx, pkt); + + if (ret < 0) + return ret; + + if (!is_selected(s, pkt)) { + av_packet_unref(pkt); + return AVERROR(EAGAIN); + } + + for (unsigned i = 0; i < s->nb_remove; i++) + av_packet_side_data_remove(pkt->side_data, &pkt->side_data_elems, + s->remove[i]); + + return 0; +} + +static const AVOptionArrayDef types_def = { .sep = '+' }; + +#define OFFSET(x) offsetof(SideDataContext, x) +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_SUBTITLE_PARAM | AV_OPT_FLAG_BSF_PARAM) +#define TYPE_CONST(type, handle) \ + { handle, NULL, 0, AV_OPT_TYPE_CONST, { .i64 = AV_PKT_DATA_ ## type }, .flags = FLAGS, .unit = "type" }, +static const AVOption options[] = { + { "remove", "side data types to remove, joined with '+'", OFFSET(remove), AV_OPT_TYPE_INT | AV_OPT_TYPE_FLAG_ARRAY, { .arr = &types_def }, 0, AV_PKT_DATA_NB - 1, FLAGS, .unit = "type" }, + { "select", "side data types of the packets to keep, joined with '+'", OFFSET(select), AV_OPT_TYPE_INT | AV_OPT_TYPE_FLAG_ARRAY, { .arr = &types_def }, 0, AV_PKT_DATA_NB - 1, FLAGS, .unit = "type" }, + FF_PACKET_SIDE_DATA_HANDLES(TYPE_CONST) + { NULL }, +}; + +static const AVClass side_data_class = { + .class_name = "side_data bsf", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; + +const FFBitStreamFilter ff_side_data_bsf = { + .p.name = "side_data", + .p.priv_class = &side_data_class, + .priv_data_size = sizeof(SideDataContext), + .init = side_data_init, + .filter = side_data_filter, +}; diff --git a/tests/fate/ffmpeg.mak b/tests/fate/ffmpeg.mak index 9da5ed9d61..aa682ba202 100644 --- a/tests/fate/ffmpeg.mak +++ b/tests/fate/ffmpeg.mak @@ -300,6 +300,25 @@ fate-ffmpeg-trim-bsf-priming-duration: CMD = framecrc -copyts -i $(TARGET_SAMPLE FATE_SAMPLES_FFMPEG-$(call DEMMUX, MOV, FRAMECRC, TRIM_BSF PIPE_PROTOCOL) += fate-ffmpeg-trim-bsf-priming-untrimmed fate-ffmpeg-trim-bsf-priming-untrimmed: CMD = framecrc -copyts -i $(TARGET_SAMPLES)/audiomatch/tones_afconvert_16000_mono_aac_lc.m4a -c:a copy -bsf:a trim=start=1:start_type=pkt_index:trim_packets=0 -frames:a 3 +FATE_SAMPLES_FFMPEG-$(call DEMMUX, MOV, FRAMECRC, SIDE_DATA_BSF PIPE_PROTOCOL) += fate-ffmpeg-side-data-bsf-remove +fate-ffmpeg-side-data-bsf-remove: CMD = framecrc -i $(TARGET_SAMPLES)/audiomatch/tones_afconvert_16000_mono_aac_lc.m4a -c:a copy -bsf:a side_data=remove=skip_samples -frames:a 2 + +# Only the first and the last packets carry skip samples: the priming and the +# padding. +FATE_SAMPLES_FFMPEG-$(call DEMMUX, MOV, FRAMECRC, SIDE_DATA_BSF PIPE_PROTOCOL) += fate-ffmpeg-side-data-bsf-select +fate-ffmpeg-side-data-bsf-select: CMD = framecrc -i $(TARGET_SAMPLES)/audiomatch/tones_afconvert_16000_mono_aac_lc.m4a -c:a copy -bsf:a side_data=select=skip_samples -frames:a 2 + +FATE_SAMPLES_FFMPEG-$(call DEMMUX, MOV, FRAMECRC, SIDE_DATA_BSF PIPE_PROTOCOL) += fate-ffmpeg-side-data-bsf-select-none +fate-ffmpeg-side-data-bsf-select-none: CMD = framecrc -i $(TARGET_SAMPLES)/audiomatch/tones_afconvert_16000_mono_aac_lc.m4a -c:a copy -bsf:a side_data=select=palette -frames:a 2 + +# The display matrix is stream side data, which ffprobe shows on the output. +# Removing another type keeps it. +FATE_SAMPLES_FFMPEG_FFPROBE-$(call REMUX, MOV, SIDE_DATA_BSF) += fate-ffmpeg-side-data-bsf-keep-stream +fate-ffmpeg-side-data-bsf-keep-stream: CMD = transcode mov $(TARGET_SAMPLES)/filter/sample-in-issue-505.mov mov "-c copy -an -t 0.5 -bsf:v side_data=remove=palette" "-c copy" "-select_streams v -show_entries stream_side_data_list" + +FATE_SAMPLES_FFMPEG_FFPROBE-$(call REMUX, MOV, SIDE_DATA_BSF) += fate-ffmpeg-side-data-bsf-remove-stream +fate-ffmpeg-side-data-bsf-remove-stream: CMD = transcode mov $(TARGET_SAMPLES)/filter/sample-in-issue-505.mov mov "-c copy -an -t 0.5 -bsf:v side_data=remove=displaymatrix" "-c copy" "-select_streams v -show_entries stream_side_data_list" + # The composed skips cover 800 + 300 of the 1024 samples, leaving nothing. FATE_SAMPLES_FFMPEG-$(call DEMMUX, AAC, FRAMECRC, AAC_PARSER TRIM_BSF PIPE_PROTOCOL) += fate-ffmpeg-trim-bsf-skip-samples-empty fate-ffmpeg-trim-bsf-skip-samples-empty: CMD = framecrc -i $(TARGET_SAMPLES)/audiomatch/tones_afconvert_16000_mono_aac_lc.adts -c:a copy -bsf:a trim=start=3217536,trim=end=3083472 diff --git a/tests/ref/fate/ffmpeg-side-data-bsf-keep-stream b/tests/ref/fate/ffmpeg-side-data-bsf-keep-stream new file mode 100644 index 0000000000..b22eb1e90a --- /dev/null +++ b/tests/ref/fate/ffmpeg-side-data-bsf-keep-stream @@ -0,0 +1,34 @@ +5e6354ce54a9c6e031beee981a3befbd *tests/data/fate/ffmpeg-side-data-bsf-keep-stream.mov +61851 tests/data/fate/ffmpeg-side-data-bsf-keep-stream.mov +#extradata 0: 29, 0x757e07f6 +#tb 0: 1/19200 +#media_type 0: video +#codec_id 0: h264 +#dimensions 0: 480x272 +#sar 0: 0/1 +0, 0, 0, 640, 21606, 0x5ac94251 +0, 640, 640, 640, 3931, 0x753d20e1, F=0x0 +0, 1280, 1280, 640, 3086, 0xe338e2f2, F=0x0 +0, 1920, 1920, 640, 1770, 0xcfaf2034, F=0x0 +0, 2560, 2560, 640, 3240, 0xaa21fc9b, F=0x0 +0, 3200, 3200, 640, 1283, 0x63254a14, F=0x0 +0, 3840, 3840, 640, 2747, 0x1eaf0ce0, F=0x0 +0, 4480, 4480, 640, 2765, 0x12700d6d, F=0x0 +0, 5120, 5120, 640, 3109, 0xd340baa5, F=0x0 +0, 5760, 5760, 640, 3435, 0xb2cc6f4b, F=0x0 +0, 6400, 6400, 640, 3255, 0xd79c0f3e, F=0x0 +0, 7040, 7040, 640, 2968, 0xf29089f2, F=0x0 +0, 7680, 7680, 640, 2825, 0x270546c9, F=0x0 +0, 8320, 8320, 640, 2543, 0x6a95941c, F=0x0 +0, 8960, 8960, 640, 2355, 0x476128dc, F=0x0 +[STREAM] +[SIDE_DATA] +side_data_type=Display Matrix +displaymatrix= +00000000: 0 65536 0 +00000001: -65536 0 0 +00000002: 17825792 0 1073741824 + +rotation=-90 +[/SIDE_DATA] +[/STREAM] diff --git a/tests/ref/fate/ffmpeg-side-data-bsf-remove b/tests/ref/fate/ffmpeg-side-data-bsf-remove new file mode 100644 index 0000000000..1f802bbd45 --- /dev/null +++ b/tests/ref/fate/ffmpeg-side-data-bsf-remove @@ -0,0 +1,8 @@ +#extradata 0: 2, 0x0030001c +#tb 0: 1/16000 +#media_type 0: audio +#codec_id 0: aac +#sample_rate 0: 16000 +#channel_layout_name 0: mono +0, -2112, -2112, 1024, 4, 0x02f70117 +0, -1088, -1088, 1024, 163, 0xd5f85007 diff --git a/tests/ref/fate/ffmpeg-side-data-bsf-remove-stream b/tests/ref/fate/ffmpeg-side-data-bsf-remove-stream new file mode 100644 index 0000000000..9d9ad4b2f5 --- /dev/null +++ b/tests/ref/fate/ffmpeg-side-data-bsf-remove-stream @@ -0,0 +1,25 @@ +7c5b0bf7edaabe9d5267a69d2261cf00 *tests/data/fate/ffmpeg-side-data-bsf-remove-stream.mov +61851 tests/data/fate/ffmpeg-side-data-bsf-remove-stream.mov +#extradata 0: 29, 0x757e07f6 +#tb 0: 1/19200 +#media_type 0: video +#codec_id 0: h264 +#dimensions 0: 480x272 +#sar 0: 0/1 +0, 0, 0, 640, 21606, 0x5ac94251 +0, 640, 640, 640, 3931, 0x753d20e1, F=0x0 +0, 1280, 1280, 640, 3086, 0xe338e2f2, F=0x0 +0, 1920, 1920, 640, 1770, 0xcfaf2034, F=0x0 +0, 2560, 2560, 640, 3240, 0xaa21fc9b, F=0x0 +0, 3200, 3200, 640, 1283, 0x63254a14, F=0x0 +0, 3840, 3840, 640, 2747, 0x1eaf0ce0, F=0x0 +0, 4480, 4480, 640, 2765, 0x12700d6d, F=0x0 +0, 5120, 5120, 640, 3109, 0xd340baa5, F=0x0 +0, 5760, 5760, 640, 3435, 0xb2cc6f4b, F=0x0 +0, 6400, 6400, 640, 3255, 0xd79c0f3e, F=0x0 +0, 7040, 7040, 640, 2968, 0xf29089f2, F=0x0 +0, 7680, 7680, 640, 2825, 0x270546c9, F=0x0 +0, 8320, 8320, 640, 2543, 0x6a95941c, F=0x0 +0, 8960, 8960, 640, 2355, 0x476128dc, F=0x0 +[STREAM] +[/STREAM] diff --git a/tests/ref/fate/ffmpeg-side-data-bsf-select b/tests/ref/fate/ffmpeg-side-data-bsf-select new file mode 100644 index 0000000000..f62264ed76 --- /dev/null +++ b/tests/ref/fate/ffmpeg-side-data-bsf-select @@ -0,0 +1,8 @@ +#extradata 0: 2, 0x0030001c +#tb 0: 1/16000 +#media_type 0: audio +#codec_id 0: aac +#sample_rate 0: 16000 +#channel_layout_name 0: mono +0, -2112, -2112, 1024, 4, 0x02f70117, S=1, Skip Samples, 10, 0x02c80048 +0, 31680, 31680, 1024, 137, 0x90c639e0, S=1, Skip Samples, 10, 0x048a00c2 diff --git a/tests/ref/fate/ffmpeg-side-data-bsf-select-none b/tests/ref/fate/ffmpeg-side-data-bsf-select-none new file mode 100644 index 0000000000..94f3df85b4 --- /dev/null +++ b/tests/ref/fate/ffmpeg-side-data-bsf-select-none @@ -0,0 +1,6 @@ +#extradata 0: 2, 0x0030001c +#tb 0: 1/16000 +#media_type 0: audio +#codec_id 0: aac +#sample_rate 0: 16000 +#channel_layout_name 0: mono