FFmpeg  4.0
vf_framepack.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2013 Vittorio Giovara
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Generate a frame packed video, by combining two views in a single surface.
24  */
25 
26 #include <string.h>
27 
28 #include "libavutil/common.h"
29 #include "libavutil/imgutils.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/pixdesc.h"
32 #include "libavutil/rational.h"
33 #include "libavutil/stereo3d.h"
34 
35 #include "avfilter.h"
36 #include "formats.h"
37 #include "internal.h"
38 #include "video.h"
39 
40 #define LEFT 0
41 #define RIGHT 1
42 
43 typedef struct FramepackContext {
44  const AVClass *class;
45 
46  const AVPixFmtDescriptor *pix_desc; ///< agreed pixel format
47 
48  enum AVStereo3DType format; ///< frame pack type output
49 
50  AVFrame *input_views[2]; ///< input frames
51 
52  int64_t double_pts; ///< new pts for frameseq mode
54 
55 static const enum AVPixelFormat formats_supported[] = {
60 };
61 
63 {
64  // this will ensure that formats are the same on all pads
66  if (!fmts_list)
67  return AVERROR(ENOMEM);
68  return ff_set_common_formats(ctx, fmts_list);
69 }
70 
72 {
73  FramepackContext *s = ctx->priv;
74 
75  // clean any leftover frame
78 }
79 
80 static int config_output(AVFilterLink *outlink)
81 {
82  AVFilterContext *ctx = outlink->src;
83  FramepackContext *s = outlink->src->priv;
84 
85  int width = ctx->inputs[LEFT]->w;
86  int height = ctx->inputs[LEFT]->h;
87  AVRational time_base = ctx->inputs[LEFT]->time_base;
88  AVRational frame_rate = ctx->inputs[LEFT]->frame_rate;
89 
90  // check size and fps match on the other input
91  if (width != ctx->inputs[RIGHT]->w ||
92  height != ctx->inputs[RIGHT]->h) {
93  av_log(ctx, AV_LOG_ERROR,
94  "Left and right sizes differ (%dx%d vs %dx%d).\n",
95  width, height,
96  ctx->inputs[RIGHT]->w, ctx->inputs[RIGHT]->h);
97  return AVERROR_INVALIDDATA;
98  } else if (av_cmp_q(time_base, ctx->inputs[RIGHT]->time_base) != 0) {
99  av_log(ctx, AV_LOG_ERROR,
100  "Left and right time bases differ (%d/%d vs %d/%d).\n",
101  time_base.num, time_base.den,
102  ctx->inputs[RIGHT]->time_base.num,
103  ctx->inputs[RIGHT]->time_base.den);
104  return AVERROR_INVALIDDATA;
105  } else if (av_cmp_q(frame_rate, ctx->inputs[RIGHT]->frame_rate) != 0) {
106  av_log(ctx, AV_LOG_ERROR,
107  "Left and right framerates differ (%d/%d vs %d/%d).\n",
108  frame_rate.num, frame_rate.den,
109  ctx->inputs[RIGHT]->frame_rate.num,
110  ctx->inputs[RIGHT]->frame_rate.den);
111  return AVERROR_INVALIDDATA;
112  }
113 
114  s->pix_desc = av_pix_fmt_desc_get(outlink->format);
115  if (!s->pix_desc)
116  return AVERROR_BUG;
117 
118  // modify output properties as needed
119  switch (s->format) {
121  time_base.den *= 2;
122  frame_rate.num *= 2;
123 
125  break;
126  case AV_STEREO3D_COLUMNS:
128  width *= 2;
129  break;
130  case AV_STEREO3D_LINES:
132  height *= 2;
133  break;
134  default:
135  av_log(ctx, AV_LOG_ERROR, "Unknown packing mode.");
136  return AVERROR_INVALIDDATA;
137  }
138 
139  outlink->w = width;
140  outlink->h = height;
141  outlink->time_base = time_base;
142  outlink->frame_rate = frame_rate;
143 
144  return 0;
145 }
146 
147 static void horizontal_frame_pack(AVFilterLink *outlink,
148  AVFrame *out,
149  int interleaved)
150 {
151  AVFilterContext *ctx = outlink->src;
152  FramepackContext *s = ctx->priv;
153  int i, plane;
154 
155  if (interleaved) {
156  const uint8_t *leftp = s->input_views[LEFT]->data[0];
157  const uint8_t *rightp = s->input_views[RIGHT]->data[0];
158  uint8_t *dstp = out->data[0];
159  int length = out->width / 2;
160  int lines = out->height;
161 
162  for (plane = 0; plane < s->pix_desc->nb_components; plane++) {
163  if (plane == 1 || plane == 2) {
164  length = AV_CEIL_RSHIFT(out->width / 2, s->pix_desc->log2_chroma_w);
165  lines = AV_CEIL_RSHIFT(out->height, s->pix_desc->log2_chroma_h);
166  }
167  for (i = 0; i < lines; i++) {
168  int j;
169  leftp = s->input_views[LEFT]->data[plane] +
170  s->input_views[LEFT]->linesize[plane] * i;
171  rightp = s->input_views[RIGHT]->data[plane] +
172  s->input_views[RIGHT]->linesize[plane] * i;
173  dstp = out->data[plane] + out->linesize[plane] * i;
174  for (j = 0; j < length; j++) {
175  // interpolate chroma as necessary
176  if ((s->pix_desc->log2_chroma_w ||
177  s->pix_desc->log2_chroma_h) &&
178  (plane == 1 || plane == 2)) {
179  *dstp++ = (*leftp + *rightp) / 2;
180  *dstp++ = (*leftp + *rightp) / 2;
181  } else {
182  *dstp++ = *leftp;
183  *dstp++ = *rightp;
184  }
185  leftp += 1;
186  rightp += 1;
187  }
188  }
189  }
190  } else {
191  for (i = 0; i < 2; i++) {
192  const uint8_t *src[4];
193  uint8_t *dst[4];
194  int sub_w = s->input_views[i]->width >> s->pix_desc->log2_chroma_w;
195 
196  src[0] = s->input_views[i]->data[0];
197  src[1] = s->input_views[i]->data[1];
198  src[2] = s->input_views[i]->data[2];
199 
200  dst[0] = out->data[0] + i * s->input_views[i]->width;
201  dst[1] = out->data[1] + i * sub_w;
202  dst[2] = out->data[2] + i * sub_w;
203 
204  av_image_copy(dst, out->linesize, src, s->input_views[i]->linesize,
205  s->input_views[i]->format,
206  s->input_views[i]->width,
207  s->input_views[i]->height);
208  }
209  }
210 }
211 
212 static void vertical_frame_pack(AVFilterLink *outlink,
213  AVFrame *out,
214  int interleaved)
215 {
216  AVFilterContext *ctx = outlink->src;
217  FramepackContext *s = ctx->priv;
218  int i;
219 
220  for (i = 0; i < 2; i++) {
221  const uint8_t *src[4];
222  uint8_t *dst[4];
223  int linesizes[4];
224  int sub_h = s->input_views[i]->height >> s->pix_desc->log2_chroma_h;
225 
226  src[0] = s->input_views[i]->data[0];
227  src[1] = s->input_views[i]->data[1];
228  src[2] = s->input_views[i]->data[2];
229 
230  dst[0] = out->data[0] + i * out->linesize[0] *
231  (interleaved + s->input_views[i]->height * (1 - interleaved));
232  dst[1] = out->data[1] + i * out->linesize[1] *
233  (interleaved + sub_h * (1 - interleaved));
234  dst[2] = out->data[2] + i * out->linesize[2] *
235  (interleaved + sub_h * (1 - interleaved));
236 
237  linesizes[0] = out->linesize[0] +
238  interleaved * out->linesize[0];
239  linesizes[1] = out->linesize[1] +
240  interleaved * out->linesize[1];
241  linesizes[2] = out->linesize[2] +
242  interleaved * out->linesize[2];
243 
244  av_image_copy(dst, linesizes, src, s->input_views[i]->linesize,
245  s->input_views[i]->format,
246  s->input_views[i]->width,
247  s->input_views[i]->height);
248  }
249 }
250 
252  AVFrame *dst)
253 {
254  AVFilterContext *ctx = outlink->src;
255  FramepackContext *s = ctx->priv;
256  switch (s->format) {
258  horizontal_frame_pack(outlink, dst, 0);
259  break;
260  case AV_STEREO3D_COLUMNS:
261  horizontal_frame_pack(outlink, dst, 1);
262  break;
264  vertical_frame_pack(outlink, dst, 0);
265  break;
266  case AV_STEREO3D_LINES:
267  vertical_frame_pack(outlink, dst, 1);
268  break;
269  }
270 }
271 
272 static int try_push_frame(AVFilterContext *ctx);
273 
275 {
276  FramepackContext *s = inlink->dst->priv;
277  s->input_views[LEFT] = frame;
278  return try_push_frame(inlink->dst);
279 }
280 
282 {
283  FramepackContext *s = inlink->dst->priv;
284  s->input_views[RIGHT] = frame;
285  return try_push_frame(inlink->dst);
286 }
287 
288 static int request_frame(AVFilterLink *outlink)
289 {
290  AVFilterContext *ctx = outlink->src;
291  FramepackContext *s = ctx->priv;
292  int ret, i;
293 
294  /* get a frame on the either input, stop as soon as a video ends */
295  for (i = 0; i < 2; i++) {
296  if (!s->input_views[i]) {
297  ret = ff_request_frame(ctx->inputs[i]);
298  if (ret < 0)
299  return ret;
300  }
301  }
302  return 0;
303 }
304 
306 {
307  FramepackContext *s = ctx->priv;
308  AVFilterLink *outlink = ctx->outputs[0];
309  AVStereo3D *stereo;
310  int ret, i;
311 
312  if (!(s->input_views[0] && s->input_views[1]))
313  return 0;
314  if (s->format == AV_STEREO3D_FRAMESEQUENCE) {
315  if (s->double_pts == AV_NOPTS_VALUE)
316  s->double_pts = s->input_views[LEFT]->pts;
317 
318  for (i = 0; i < 2; i++) {
319  // set correct timestamps
320  s->input_views[i]->pts = s->double_pts++;
321 
322  // set stereo3d side data
324  if (!stereo)
325  return AVERROR(ENOMEM);
326  stereo->type = s->format;
327  stereo->view = i == LEFT ? AV_STEREO3D_VIEW_LEFT
329 
330  // filter the frame and immediately relinquish its pointer
331  ret = ff_filter_frame(outlink, s->input_views[i]);
332  s->input_views[i] = NULL;
333  if (ret < 0)
334  return ret;
335  }
336  return ret;
337  } else {
338  AVFrame *dst = ff_get_video_buffer(outlink, outlink->w, outlink->h);
339  if (!dst)
340  return AVERROR(ENOMEM);
341 
342  spatial_frame_pack(outlink, dst);
343 
344  // get any property from the original frame
345  ret = av_frame_copy_props(dst, s->input_views[LEFT]);
346  if (ret < 0) {
347  av_frame_free(&dst);
348  return ret;
349  }
350 
351  for (i = 0; i < 2; i++)
352  av_frame_free(&s->input_views[i]);
353 
354  // set stereo3d side data
355  stereo = av_stereo3d_create_side_data(dst);
356  if (!stereo) {
357  av_frame_free(&dst);
358  return AVERROR(ENOMEM);
359  }
360  stereo->type = s->format;
361 
362  return ff_filter_frame(outlink, dst);
363  }
364 }
365 
366 #define OFFSET(x) offsetof(FramepackContext, x)
367 #define V AV_OPT_FLAG_VIDEO_PARAM
368 static const AVOption framepack_options[] = {
369  { "format", "Frame pack output format", OFFSET(format), AV_OPT_TYPE_INT,
370  { .i64 = AV_STEREO3D_SIDEBYSIDE }, 0, INT_MAX, .flags = V, .unit = "format" },
371  { "sbs", "Views are packed next to each other", 0, AV_OPT_TYPE_CONST,
372  { .i64 = AV_STEREO3D_SIDEBYSIDE }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
373  { "tab", "Views are packed on top of each other", 0, AV_OPT_TYPE_CONST,
374  { .i64 = AV_STEREO3D_TOPBOTTOM }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
375  { "frameseq", "Views are one after the other", 0, AV_OPT_TYPE_CONST,
376  { .i64 = AV_STEREO3D_FRAMESEQUENCE }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
377  { "lines", "Views are interleaved by lines", 0, AV_OPT_TYPE_CONST,
378  { .i64 = AV_STEREO3D_LINES }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
379  { "columns", "Views are interleaved by columns", 0, AV_OPT_TYPE_CONST,
380  { .i64 = AV_STEREO3D_COLUMNS }, INT_MIN, INT_MAX, .flags = V, .unit = "format" },
381  { NULL },
382 };
383 
384 AVFILTER_DEFINE_CLASS(framepack);
385 
386 static const AVFilterPad framepack_inputs[] = {
387  {
388  .name = "left",
389  .type = AVMEDIA_TYPE_VIDEO,
390  .filter_frame = filter_frame_left,
391  .needs_fifo = 1,
392  },
393  {
394  .name = "right",
395  .type = AVMEDIA_TYPE_VIDEO,
396  .filter_frame = filter_frame_right,
397  .needs_fifo = 1,
398  },
399  { NULL }
400 };
401 
402 static const AVFilterPad framepack_outputs[] = {
403  {
404  .name = "packed",
405  .type = AVMEDIA_TYPE_VIDEO,
406  .config_props = config_output,
407  .request_frame = request_frame,
408  },
409  { NULL }
410 };
411 
413  .name = "framepack",
414  .description = NULL_IF_CONFIG_SMALL("Generate a frame packed stereoscopic video."),
415  .priv_size = sizeof(FramepackContext),
416  .priv_class = &framepack_class,
418  .inputs = framepack_inputs,
419  .outputs = framepack_outputs,
421 };
int plane
Definition: avisynth_c.h:422
#define NULL
Definition: coverity.c:32
static int filter_frame_right(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_framepack.c:281
const char * s
Definition: avisynth_c.h:768
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2363
This structure describes decoded (raw) audio or video data.
Definition: frame.h:218
AVOption.
Definition: opt.h:246
enum AVStereo3DView view
Determines which views are packed.
Definition: stereo3d.h:190
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:67
misc image utilities
Main libavfilter public API header.
static int try_push_frame(AVFilterContext *ctx)
Definition: vf_framepack.c:305
int num
Numerator.
Definition: rational.h:59
Frame contains only the right view.
Definition: stereo3d.h:161
static enum AVPixelFormat formats_supported[]
Definition: vf_framepack.c:55
Views are next to each other.
Definition: stereo3d.h:67
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:99
#define src
Definition: vp8dsp.c:254
static void vertical_frame_pack(AVFilterLink *outlink, AVFrame *out, int interleaved)
Definition: vf_framepack.c:212
#define RIGHT
Definition: vf_framepack.c:41
uint8_t log2_chroma_w
Amount to shift the luma width right to find the chroma width.
Definition: pixdesc.h:92
Frame contains only the left view.
Definition: stereo3d.h:156
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:283
const char * name
Pad name.
Definition: internal.h:60
#define OFFSET(x)
Definition: vf_framepack.c:366
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:346
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1080
planar YUV 4:2:0, 20bpp, (1 Cr & Cb sample per 2x2 Y & A samples)
Definition: pixfmt.h:97
uint8_t
#define av_cold
Definition: attributes.h:82
AVFILTER_DEFINE_CLASS(framepack)
static av_cold int uninit(AVCodecContext *avctx)
Definition: crystalhd.c:279
AVOptions.
Stereo 3D type: this structure describes how two videos are packed within a single video surface...
Definition: stereo3d.h:176
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:311
static int query_formats(AVFilterContext *ctx)
Definition: vf_framepack.c:62
static AVFrame * frame
#define height
planar YUV 4:4:0 full scale (JPEG), deprecated in favor of AV_PIX_FMT_YUV440P and setting color_range...
Definition: pixfmt.h:96
planar YUV 4:2:2, 16bpp, full scale (JPEG), deprecated in favor of AV_PIX_FMT_YUV422P and setting col...
Definition: pixfmt.h:75
static void horizontal_frame_pack(AVFilterLink *outlink, AVFrame *out, int interleaved)
Definition: vf_framepack.c:147
#define av_log(a,...)
A filter pad used for either input or output.
Definition: internal.h:54
int width
Definition: frame.h:276
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:568
uint8_t log2_chroma_h
Amount to shift the luma height right to find the chroma height.
Definition: pixdesc.h:101
#define V
Definition: vf_framepack.c:367
AVFilter ff_vf_framepack
Definition: vf_framepack.c:412
BYTE * dstp
Definition: avisynth_c.h:813
#define AVERROR(e)
Definition: error.h:43
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:202
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
Views are packed per line, as if interlaced.
Definition: stereo3d.h:129
void * priv
private data for use by the filter
Definition: avfilter.h:353
uint16_t width
Definition: gdv.c:47
int64_t double_pts
new pts for frameseq mode
Definition: vf_framepack.c:52
static int filter_frame_left(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_framepack.c:274
void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4], const uint8_t *src_data[4], const int src_linesizes[4], enum AVPixelFormat pix_fmt, int width, int height)
Copy image in src_data to dst_data.
Definition: imgutils.c:387
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:66
AVStereo3D * av_stereo3d_create_side_data(AVFrame *frame)
Allocate a complete AVFrameSideData and add it to the frame.
Definition: stereo3d.c:33
Views are alternated temporally.
Definition: stereo3d.h:92
uint8_t nb_components
The number of components each pixel has, (1-4)
Definition: pixdesc.h:83
AVStereo3DType
List of possible 3D Types.
Definition: stereo3d.h:51
planar YUV 4:2:0, 12bpp, full scale (JPEG), deprecated in favor of AV_PIX_FMT_YUV420P and setting col...
Definition: pixfmt.h:74
AVFrame * input_views[2]
input frames
Definition: vf_framepack.c:50
AVFormatContext * ctx
Definition: movenc.c:48
static const AVFilterPad inputs[]
Definition: af_acontrast.c:193
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:291
enum AVStereo3DType type
How views are packed within the video.
Definition: stereo3d.h:180
Views are on top of each other.
Definition: stereo3d.h:79
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:249
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
#define LEFT
Definition: vf_framepack.c:40
enum AVStereo3DType format
frame pack type output
Definition: vf_framepack.c:48
#define AVERROR_BUG
Internal bug, also see AVERROR_BUG2.
Definition: error.h:50
planar YUV 4:1:0, 9bpp, (1 Cr & Cb sample per 4x4 Y samples)
Definition: pixfmt.h:68
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:144
Rational number (pair of numerator and denominator).
Definition: rational.h:58
static int request_frame(AVFilterLink *outlink)
Definition: vf_framepack.c:288
static const AVFilterPad framepack_inputs[]
Definition: vf_framepack.c:386
const char * name
Filter name.
Definition: avfilter.h:148
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:350
const AVPixFmtDescriptor * pix_desc
agreed pixel format
Definition: vf_framepack.c:46
static int config_output(AVFilterLink *outlink)
Definition: vf_framepack.c:80
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:232
static av_cold void framepack_uninit(AVFilterContext *ctx)
Definition: vf_framepack.c:71
static int av_cmp_q(AVRational a, AVRational b)
Compare two rationals.
Definition: rational.h:89
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:62
common internal and external API header
Utilties for rational number calculation.
planar YUV 4:4:4, 24bpp, full scale (JPEG), deprecated in favor of AV_PIX_FMT_YUV444P and setting col...
Definition: pixfmt.h:76
Stereoscopic video.
Views are packed per column.
Definition: stereo3d.h:141
int den
Denominator.
Definition: rational.h:60
static const AVOption framepack_options[]
Definition: vf_framepack.c:368
A list of supported formats for one end of a filter link.
Definition: formats.h:64
An instance of a filter.
Definition: avfilter.h:338
int height
Definition: frame.h:276
FILE * out
Definition: movenc.c:54
static const AVFilterPad framepack_outputs[]
Definition: vf_framepack.c:402
#define av_always_inline
Definition: attributes.h:39
static av_always_inline void spatial_frame_pack(AVFilterLink *outlink, AVFrame *dst)
Definition: vf_framepack.c:251
int ff_request_frame(AVFilterLink *link)
Request an input frame from the filter at the other end of the link.
Definition: avfilter.c:407
const char int length
Definition: avisynth_c.h:768
internal API functions
AVPixelFormat
Pixel format.
Definition: pixfmt.h:60
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:652
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:248
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58