FFmpeg  4.0
vf_mix.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Paul B Mahol
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/avstring.h"
22 #include "libavutil/imgutils.h"
23 #include "libavutil/intreadwrite.h"
24 #include "libavutil/opt.h"
25 #include "libavutil/pixdesc.h"
26 
27 #include "avfilter.h"
28 #include "formats.h"
29 #include "internal.h"
30 #include "framesync.h"
31 #include "video.h"
32 
33 typedef struct MixContext {
34  const AVClass *class;
36  char *weights_str;
37  int nb_inputs;
38  int duration;
39  float *weights;
40  float wfactor;
41 
42  int depth;
43  int nb_planes;
44  int linesize[4];
45  int height[4];
46 
49 } MixContext;
50 
52 {
54  int fmt, ret;
55 
56  for (fmt = 0; av_pix_fmt_desc_get(fmt); fmt++) {
58  if (!(desc->flags & AV_PIX_FMT_FLAG_PAL ||
61  (ret = ff_add_format(&pix_fmts, fmt)) < 0)
62  return ret;
63  }
64 
65  return ff_set_common_formats(ctx, pix_fmts);
66 }
67 
69 {
70  MixContext *s = ctx->priv;
71  char *p, *arg, *saveptr = NULL;
72  int i, ret;
73 
74  s->frames = av_calloc(s->nb_inputs, sizeof(*s->frames));
75  if (!s->frames)
76  return AVERROR(ENOMEM);
77 
78  s->weights = av_calloc(s->nb_inputs, sizeof(*s->weights));
79  if (!s->weights)
80  return AVERROR(ENOMEM);
81 
82  for (i = 0; i < s->nb_inputs; i++) {
83  AVFilterPad pad = { 0 };
84 
86  pad.name = av_asprintf("input%d", i);
87  if (!pad.name)
88  return AVERROR(ENOMEM);
89 
90  if ((ret = ff_insert_inpad(ctx, i, &pad)) < 0) {
91  av_freep(&pad.name);
92  return ret;
93  }
94  }
95 
96  p = s->weights_str;
97  for (i = 0; i < s->nb_inputs; i++) {
98  if (!(arg = av_strtok(p, " ", &saveptr)))
99  break;
100 
101  p = NULL;
102  sscanf(arg, "%f", &s->weights[i]);
103  s->wfactor += s->weights[i];
104  }
105  s->wfactor = 1 / s->wfactor;
106 
107  return 0;
108 }
109 
111 {
112  AVFilterContext *ctx = fs->parent;
113  AVFilterLink *outlink = ctx->outputs[0];
114  MixContext *s = fs->opaque;
115  AVFrame **in = s->frames;
116  AVFrame *out;
117  int i, p, ret, x, y;
118 
119  for (i = 0; i < s->nb_inputs; i++) {
120  if ((ret = ff_framesync_get_frame(&s->fs, i, &in[i], 0)) < 0)
121  return ret;
122  }
123 
124  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
125  if (!out)
126  return AVERROR(ENOMEM);
127  out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base);
128 
129  if (s->depth <= 8) {
130  for (p = 0; p < s->nb_planes; p++) {
131  uint8_t *dst = out->data[p];
132 
133  for (y = 0; y < s->height[p]; y++) {
134  for (x = 0; x < s->linesize[p]; x++) {
135  int val = 0;
136 
137  for (i = 0; i < s->nb_inputs; i++) {
138  uint8_t src = in[i]->data[p][y * s->linesize[p] + x];
139 
140  val += src * s->weights[i];
141  }
142 
143  dst[x] = val * s->wfactor;
144  }
145 
146  dst += out->linesize[p];
147  }
148  }
149  } else {
150  for (p = 0; p < s->nb_planes; p++) {
151  uint16_t *dst = (uint16_t *)out->data[p];
152 
153  for (y = 0; y < s->height[p]; y++) {
154  for (x = 0; x < s->linesize[p]; x++) {
155  int val = 0;
156 
157  for (i = 0; i < s->nb_inputs; i++) {
158  uint16_t src = AV_RN16(in[i]->data[p] + y * s->linesize[p] + x * 2);
159 
160  val += src * s->weights[i];
161  }
162 
163  dst[x] = val * s->wfactor;
164  }
165 
166  dst += out->linesize[p] / 2;
167  }
168  }
169  }
170 
171  return ff_filter_frame(outlink, out);
172 }
173 
174 static int config_output(AVFilterLink *outlink)
175 {
176  AVFilterContext *ctx = outlink->src;
177  MixContext *s = ctx->priv;
178  AVRational time_base = ctx->inputs[0]->time_base;
179  AVRational frame_rate = ctx->inputs[0]->frame_rate;
180  AVFilterLink *inlink = ctx->inputs[0];
181  int height = ctx->inputs[0]->h;
182  int width = ctx->inputs[0]->w;
183  FFFrameSyncIn *in;
184  int i, ret;
185 
186  for (i = 1; i < s->nb_inputs; i++) {
187  if (ctx->inputs[i]->h != height || ctx->inputs[i]->w != width) {
188  av_log(ctx, AV_LOG_ERROR, "Input %d size (%dx%d) does not match input %d size (%dx%d).\n", i, ctx->inputs[i]->w, ctx->inputs[i]->h, 0, width, height);
189  return AVERROR(EINVAL);
190  }
191  }
192 
193  s->desc = av_pix_fmt_desc_get(outlink->format);
194  if (!s->desc)
195  return AVERROR_BUG;
197  s->depth = s->desc->comp[0].depth;
198 
199  outlink->w = width;
200  outlink->h = height;
201  outlink->time_base = time_base;
202  outlink->frame_rate = frame_rate;
203 
204  if ((ret = ff_framesync_init(&s->fs, ctx, s->nb_inputs)) < 0)
205  return ret;
206 
207  in = s->fs.in;
208  s->fs.opaque = s;
210 
211  if ((ret = av_image_fill_linesizes(s->linesize, inlink->format, inlink->w)) < 0)
212  return ret;
213 
214  s->height[1] = s->height[2] = AV_CEIL_RSHIFT(inlink->h, s->desc->log2_chroma_h);
215  s->height[0] = s->height[3] = inlink->h;
216 
217  for (i = 0; i < s->nb_inputs; i++) {
218  AVFilterLink *inlink = ctx->inputs[i];
219 
220  in[i].time_base = inlink->time_base;
221  in[i].sync = 1;
222  in[i].before = EXT_STOP;
223  in[i].after = (s->duration == 1 || (s->duration == 2 && i == 0)) ? EXT_STOP : EXT_INFINITY;
224  }
225 
226  return ff_framesync_configure(&s->fs);
227 }
228 
230 {
231  MixContext *s = ctx->priv;
232  int i;
233 
234  ff_framesync_uninit(&s->fs);
235  av_freep(&s->frames);
236  av_freep(&s->weights);
237 
238  for (i = 0; i < ctx->nb_inputs; i++)
239  av_freep(&ctx->input_pads[i].name);
240 }
241 
243 {
244  MixContext *s = ctx->priv;
245  return ff_framesync_activate(&s->fs);
246 }
247 
248 #define OFFSET(x) offsetof(MixContext, x)
249 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM
250 
251 static const AVOption mix_options[] = {
252  { "inputs", "set number of inputs", OFFSET(nb_inputs), AV_OPT_TYPE_INT, {.i64=2}, 2, INT_MAX, .flags = FLAGS },
253  { "weights", "set weight for each input", OFFSET(weights_str), AV_OPT_TYPE_STRING, {.str="1 1"}, 0, 0, .flags = FLAGS },
254  { "duration", "how to determine end of stream", OFFSET(duration), AV_OPT_TYPE_INT, {.i64=0}, 0, 2, .flags = FLAGS, "duration" },
255  { "longest", "Duration of longest input", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, FLAGS, "duration" },
256  { "shortest", "Duration of shortest input", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, FLAGS, "duration" },
257  { "first", "Duration of first input", 0, AV_OPT_TYPE_CONST, {.i64=2}, 0, 0, FLAGS, "duration" },
258  { NULL },
259 };
260 
261 static const AVFilterPad outputs[] = {
262  {
263  .name = "default",
264  .type = AVMEDIA_TYPE_VIDEO,
265  .config_props = config_output,
266  },
267  { NULL }
268 };
269 
271 
273  .name = "mix",
274  .description = NULL_IF_CONFIG_SMALL("Mix video inputs."),
275  .priv_size = sizeof(MixContext),
276  .priv_class = &mix_class,
278  .outputs = outputs,
279  .init = init,
280  .uninit = uninit,
281  .activate = activate,
283 };
#define AV_PIX_FMT_FLAG_PAL
Pixel format has a palette in data[1], values are indexes in this palette.
Definition: pixdesc.h:132
int nb_planes
Definition: vf_mix.c:43
#define NULL
Definition: coverity.c:32
const char const char void * val
Definition: avisynth_c.h:771
const char * s
Definition: avisynth_c.h:768
int depth
Definition: vf_mix.c:42
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2363
This structure describes decoded (raw) audio or video data.
Definition: frame.h:218
AVOption.
Definition: opt.h:246
const char * fmt
Definition: avisynth_c.h:769
misc image utilities
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2403
Main libavfilter public API header.
#define AVFILTER_FLAG_DYNAMIC_INPUTS
The number of the filter inputs is not determined just by AVFilter.inputs.
Definition: avfilter.h:105
const AVPixFmtDescriptor * desc
Definition: vf_mix.c:35
enum AVMediaType type
AVFilterPad type.
Definition: internal.h:65
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:117
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:99
#define src
Definition: vp8dsp.c:254
#define FLAGS
Definition: vf_mix.c:249
int64_t pts
Timestamp of the current event.
Definition: framesync.h:167
static av_cold void uninit(AVFilterContext *ctx)
Definition: vf_mix.c:229
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:86
void * av_calloc(size_t nmemb, size_t size)
Non-inlined equivalent of av_mallocz_array().
Definition: mem.c:244
const char * name
Pad name.
Definition: internal.h:60
static int activate(AVFilterContext *ctx)
Definition: vf_mix.c:242
AVFilterContext * parent
Parent filter context.
Definition: framesync.h:152
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:346
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1080
AVComponentDescriptor comp[4]
Parameters that describe how pixels are packed.
Definition: pixdesc.h:117
uint8_t
#define av_cold
Definition: attributes.h:82
AVOptions.
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:311
FFFrameSyncIn * in
Pointer to array of inputs.
Definition: framesync.h:203
const char data[16]
Definition: mxf.c:90
static int flags
Definition: log.c:55
FFFrameSync fs
Definition: vf_mix.c:48
int duration
Definition: vf_mix.c:38
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:91
Input stream structure.
Definition: framesync.h:81
#define av_log(a,...)
A filter pad used for either input or output.
Definition: internal.h:54
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
AVFilterPad * input_pads
array of input pads
Definition: avfilter.h:345
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:568
uint8_t log2_chroma_h
Amount to shift the luma height right to find the chroma height.
Definition: pixdesc.h:101
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:293
Frame sync structure.
Definition: framesync.h:146
#define AVERROR(e)
Definition: error.h:43
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
static int config_output(AVFilterLink *outlink)
Definition: vf_mix.c:174
void * priv
private data for use by the filter
Definition: avfilter.h:353
const char * arg
Definition: jacosubdec.c:66
uint16_t width
Definition: gdv.c:47
#define AV_PIX_FMT_FLAG_HWACCEL
Pixel format is an HW accelerated format.
Definition: pixdesc.h:140
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:96
int ff_add_format(AVFilterFormats **avff, int64_t fmt)
Add fmt to the list of media formats contained in *avff.
Definition: formats.c:337
int ff_framesync_activate(FFFrameSync *fs)
Examine the frames in the filter&#39;s input and try to produce output.
Definition: framesync.c:344
int(* on_event)(struct FFFrameSync *fs)
Callback called when a frame event is ready.
Definition: framesync.h:172
int linesize[4]
Definition: vf_mix.c:44
char * av_asprintf(const char *fmt,...)
Definition: avstring.c:113
uint64_t flags
Combination of AV_PIX_FMT_FLAG_...
Definition: pixdesc.h:106
int height[4]
Definition: vf_mix.c:45
unsigned nb_inputs
number of input pads
Definition: avfilter.h:347
static int process_frame(FFFrameSync *fs)
Definition: vf_mix.c:110
AVFormatContext * ctx
Definition: movenc.c:48
AVRational time_base
Time base for the output events.
Definition: framesync.h:162
void * opaque
Opaque pointer, not used by the API.
Definition: framesync.h:177
AVFILTER_DEFINE_CLASS(mix)
static int mix(int c0, int c1)
Definition: 4xm.c:707
AVFilter ff_vf_mix
Definition: vf_mix.c:272
Extend the frame to infinity.
Definition: framesync.h:75
static int query_formats(AVFilterContext *ctx)
Definition: vf_mix.c:51
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:249
int ff_framesync_init(FFFrameSync *fs, AVFilterContext *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:77
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
#define AV_RN16(p)
Definition: intreadwrite.h:360
char * weights_str
string for custom weights for every input
Definition: af_amix.c:165
#define AVERROR_BUG
Internal bug, also see AVERROR_BUG2.
Definition: error.h:50
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31)))) #define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac) { } void ff_audio_convert_free(AudioConvert **ac) { if(! *ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);} AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map) { AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method !=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2) { ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc) { av_free(ac);return NULL;} return ac;} in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar) { ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar ? ac->channels :1;} else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;} int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in) { int use_generic=1;int len=in->nb_samples;int p;if(ac->dc) { av_log(ac->avr, AV_LOG_TRACE, "%d samples - audio_convert: %s to %s (dithered)\", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events...
Definition: framesync.h:139
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:144
int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
Fill plane linesizes for an image with pixel format pix_fmt and width width.
Definition: imgutils.c:89
Rational number (pair of numerator and denominator).
Definition: rational.h:58
const char * name
Filter name.
Definition: avfilter.h:148
#define OFFSET(x)
Definition: vf_mix.c:248
#define AV_PIX_FMT_FLAG_BITSTREAM
All values of a component are bit-wise packed end to end.
Definition: pixdesc.h:136
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:350
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:266
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:232
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok()...
Definition: avstring.c:184
float * weights
custom weights for every input
Definition: af_amix.c:173
Completely stop all streams with this one.
Definition: framesync.h:65
static av_cold int init(AVFilterContext *ctx)
Definition: vf_mix.c:68
float wfactor
Definition: vf_mix.c:40
A list of supported formats for one end of a filter link.
Definition: formats.h:64
int nb_inputs
number of inputs
Definition: af_amix.c:161
AVFrame ** frames
Definition: vf_mix.c:47
An instance of a filter.
Definition: avfilter.h:338
FILE * out
Definition: movenc.c:54
#define av_freep(p)
static const AVOption mix_options[]
Definition: vf_mix.c:251
static const AVFilterPad outputs[]
Definition: vf_mix.c:261
internal API functions
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:256
int depth
Number of bits in the component.
Definition: pixdesc.h:58
for(j=16;j >0;--j)
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58
static int ff_insert_inpad(AVFilterContext *f, unsigned index, AVFilterPad *p)
Insert a new input pad for the filter.
Definition: internal.h:277