FFmpeg  4.0
android_camera.c
Go to the documentation of this file.
1 /*
2  * Android camera input device
3  *
4  * Copyright (C) 2017 Felix Matouschek
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22 
23 #include <errno.h>
24 #include <pthread.h>
25 #include <stdatomic.h>
26 #include <stdbool.h>
27 #include <stdint.h>
28 
29 #include <camera/NdkCameraDevice.h>
30 #include <camera/NdkCameraManager.h>
31 #include <media/NdkImage.h>
32 #include <media/NdkImageReader.h>
33 
34 #include "libavformat/avformat.h"
35 #include "libavformat/internal.h"
36 #include "libavutil/avstring.h"
37 #include "libavutil/display.h"
38 #include "libavutil/imgutils.h"
39 #include "libavutil/log.h"
40 #include "libavutil/opt.h"
41 #include "libavutil/parseutils.h"
42 #include "libavutil/pixfmt.h"
44 #include "libavutil/time.h"
45 
46 #include "version.h"
47 
48 /* This image format is available on all Android devices
49  * supporting the Camera2 API */
50 #define IMAGE_FORMAT_ANDROID AIMAGE_FORMAT_YUV_420_888
51 
52 #define MAX_BUF_COUNT 2
53 #define VIDEO_STREAM_INDEX 0
54 #define VIDEO_TIMEBASE_ANDROID 1000000000
55 
56 #define RETURN_CASE(x) case x: return AV_STRINGIFY(x);
57 #define RETURN_DEFAULT(x) default: return AV_STRINGIFY(x);
58 
59 typedef struct AndroidCameraCtx {
60  const AVClass *class;
61 
67 
70  int width;
71  int height;
74 
75  ACameraManager *camera_mgr;
76  char *camera_id;
77  ACameraMetadata *camera_metadata;
78  ACameraDevice *camera_dev;
79  ACameraDevice_StateCallbacks camera_state_callbacks;
80  AImageReader *image_reader;
81  AImageReader_ImageListener image_listener;
82  ANativeWindow *image_reader_window;
83  ACaptureSessionOutputContainer *capture_session_output_container;
84  ACaptureSessionOutput *capture_session_output;
85  ACameraOutputTarget *camera_output_target;
86  ACaptureRequest *capture_request;
87  ACameraCaptureSession_stateCallbacks capture_session_state_callbacks;
88  ACameraCaptureSession *capture_session;
89 
94 
95 static const char *camera_status_string(camera_status_t val)
96 {
97  switch(val) {
98  RETURN_CASE(ACAMERA_OK)
99  RETURN_CASE(ACAMERA_ERROR_UNKNOWN)
100  RETURN_CASE(ACAMERA_ERROR_INVALID_PARAMETER)
101  RETURN_CASE(ACAMERA_ERROR_CAMERA_DISCONNECTED)
102  RETURN_CASE(ACAMERA_ERROR_NOT_ENOUGH_MEMORY)
103  RETURN_CASE(ACAMERA_ERROR_METADATA_NOT_FOUND)
104  RETURN_CASE(ACAMERA_ERROR_CAMERA_DEVICE)
105  RETURN_CASE(ACAMERA_ERROR_CAMERA_SERVICE)
106  RETURN_CASE(ACAMERA_ERROR_SESSION_CLOSED)
107  RETURN_CASE(ACAMERA_ERROR_INVALID_OPERATION)
108  RETURN_CASE(ACAMERA_ERROR_STREAM_CONFIGURE_FAIL)
109  RETURN_CASE(ACAMERA_ERROR_CAMERA_IN_USE)
110  RETURN_CASE(ACAMERA_ERROR_MAX_CAMERA_IN_USE)
111  RETURN_CASE(ACAMERA_ERROR_CAMERA_DISABLED)
112  RETURN_CASE(ACAMERA_ERROR_PERMISSION_DENIED)
113  RETURN_DEFAULT(ACAMERA_ERROR_UNKNOWN)
114  }
115 }
116 
117 static const char *media_status_string(media_status_t val)
118 {
119  switch(val) {
120  RETURN_CASE(AMEDIA_OK)
121  RETURN_CASE(AMEDIA_ERROR_UNKNOWN)
122  RETURN_CASE(AMEDIA_ERROR_MALFORMED)
123  RETURN_CASE(AMEDIA_ERROR_UNSUPPORTED)
124  RETURN_CASE(AMEDIA_ERROR_INVALID_OBJECT)
125  RETURN_CASE(AMEDIA_ERROR_INVALID_PARAMETER)
126  RETURN_CASE(AMEDIA_ERROR_INVALID_OPERATION)
127  RETURN_CASE(AMEDIA_DRM_NOT_PROVISIONED)
128  RETURN_CASE(AMEDIA_DRM_RESOURCE_BUSY)
129  RETURN_CASE(AMEDIA_DRM_DEVICE_REVOKED)
130  RETURN_CASE(AMEDIA_DRM_SHORT_BUFFER)
131  RETURN_CASE(AMEDIA_DRM_SESSION_NOT_OPENED)
132  RETURN_CASE(AMEDIA_DRM_TAMPER_DETECTED)
133  RETURN_CASE(AMEDIA_DRM_VERIFY_FAILED)
134  RETURN_CASE(AMEDIA_DRM_NEED_KEY)
135  RETURN_CASE(AMEDIA_DRM_LICENSE_EXPIRED)
136  RETURN_CASE(AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE)
137  RETURN_CASE(AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED)
138  RETURN_CASE(AMEDIA_IMGREADER_CANNOT_LOCK_IMAGE)
139  RETURN_CASE(AMEDIA_IMGREADER_CANNOT_UNLOCK_IMAGE)
140  RETURN_CASE(AMEDIA_IMGREADER_IMAGE_NOT_LOCKED)
141  RETURN_DEFAULT(AMEDIA_ERROR_UNKNOWN)
142  }
143 }
144 
145 static const char *error_state_callback_string(int val)
146 {
147  switch(val) {
148  RETURN_CASE(ERROR_CAMERA_IN_USE)
149  RETURN_CASE(ERROR_MAX_CAMERAS_IN_USE)
150  RETURN_CASE(ERROR_CAMERA_DISABLED)
151  RETURN_CASE(ERROR_CAMERA_DEVICE)
152  RETURN_CASE(ERROR_CAMERA_SERVICE)
153  default:
154  return "ERROR_CAMERA_UNKNOWN";
155  }
156 }
157 
158 static void camera_dev_disconnected(void *context, ACameraDevice *device)
159 {
160  AVFormatContext *avctx = context;
161  AndroidCameraCtx *ctx = avctx->priv_data;
162  atomic_store(&ctx->exit, 1);
163  av_log(avctx, AV_LOG_ERROR, "Camera with id %s disconnected.\n",
164  ACameraDevice_getId(device));
165 }
166 
167 static void camera_dev_error(void *context, ACameraDevice *device, int error)
168 {
169  AVFormatContext *avctx = context;
170  AndroidCameraCtx *ctx = avctx->priv_data;
171  atomic_store(&ctx->exit, 1);
172  av_log(avctx, AV_LOG_ERROR, "Error %s on camera with id %s.\n",
173  error_state_callback_string(error), ACameraDevice_getId(device));
174 }
175 
176 static int open_camera(AVFormatContext *avctx)
177 {
178  AndroidCameraCtx *ctx = avctx->priv_data;
179  camera_status_t ret;
180  ACameraIdList *camera_ids;
181 
182  ret = ACameraManager_getCameraIdList(ctx->camera_mgr, &camera_ids);
183  if (ret != ACAMERA_OK) {
184  av_log(avctx, AV_LOG_ERROR, "Failed to get camera id list, error: %s.\n",
185  camera_status_string(ret));
186  return AVERROR_EXTERNAL;
187  }
188 
189  if (ctx->camera_index < camera_ids->numCameras) {
190  ctx->camera_id = av_strdup(camera_ids->cameraIds[ctx->camera_index]);
191  if (!ctx->camera_id) {
192  av_log(avctx, AV_LOG_ERROR, "Failed to allocate memory for camera_id.\n");
193  return AVERROR(ENOMEM);
194  }
195  } else {
196  av_log(avctx, AV_LOG_ERROR, "No camera with index %d available.\n",
197  ctx->camera_index);
198  return AVERROR(ENXIO);
199  }
200 
201  ACameraManager_deleteCameraIdList(camera_ids);
202 
203  ret = ACameraManager_getCameraCharacteristics(ctx->camera_mgr,
204  ctx->camera_id, &ctx->camera_metadata);
205  if (ret != ACAMERA_OK) {
206  av_log(avctx, AV_LOG_ERROR, "Failed to get metadata for camera with id %s, error: %s.\n",
207  ctx->camera_id, camera_status_string(ret));
208  return AVERROR_EXTERNAL;
209  }
210 
211  ctx->camera_state_callbacks.context = avctx;
212  ctx->camera_state_callbacks.onDisconnected = camera_dev_disconnected;
214 
215  ret = ACameraManager_openCamera(ctx->camera_mgr, ctx->camera_id,
216  &ctx->camera_state_callbacks, &ctx->camera_dev);
217  if (ret != ACAMERA_OK) {
218  av_log(avctx, AV_LOG_ERROR, "Failed to open camera with id %s, error: %s.\n",
219  ctx->camera_id, camera_status_string(ret));
220  return AVERROR_EXTERNAL;
221  }
222 
223  return 0;
224 }
225 
227 {
228  AndroidCameraCtx *ctx = avctx->priv_data;
229  ACameraMetadata_const_entry lens_facing;
230  ACameraMetadata_const_entry sensor_orientation;
231 
232  ACameraMetadata_getConstEntry(ctx->camera_metadata,
233  ACAMERA_LENS_FACING, &lens_facing);
234  ACameraMetadata_getConstEntry(ctx->camera_metadata,
235  ACAMERA_SENSOR_ORIENTATION, &sensor_orientation);
236 
237  ctx->lens_facing = lens_facing.data.u8[0];
238  ctx->sensor_orientation = sensor_orientation.data.i32[0];
239 }
240 
242 {
243  AndroidCameraCtx *ctx = avctx->priv_data;
244  ACameraMetadata_const_entry available_configs;
245  int found = 0;
246 
247  ACameraMetadata_getConstEntry(ctx->camera_metadata,
248  ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
249  &available_configs);
250 
251  for (int i = 0; i < available_configs.count; i++) {
252  int32_t input = available_configs.data.i32[i * 4 + 3];
253  int32_t format = available_configs.data.i32[i * 4 + 0];
254 
255  if (input) {
256  continue;
257  }
258 
259  if (format == IMAGE_FORMAT_ANDROID) {
260  int32_t width = available_configs.data.i32[i * 4 + 1];
261  int32_t height = available_configs.data.i32[i * 4 + 2];
262 
263  //Same ratio
264  if ((ctx->requested_width == width && ctx->requested_height == height) ||
265  (ctx->requested_width == height && ctx->requested_height == width)) {
266  ctx->width = width;
267  ctx->height = height;
268  found = 1;
269  break;
270  }
271  }
272  }
273 
274  if (!found || ctx->width == 0 || ctx->height == 0) {
275  ctx->width = available_configs.data.i32[1];
276  ctx->height = available_configs.data.i32[2];
277 
278  av_log(avctx, AV_LOG_WARNING,
279  "Requested video_size %dx%d not available, falling back to %dx%d\n",
280  ctx->requested_width, ctx->requested_height, ctx->width, ctx->height);
281  }
282 
283  return;
284 }
285 
286 static void match_framerate(AVFormatContext *avctx)
287 {
288  AndroidCameraCtx *ctx = avctx->priv_data;
289  ACameraMetadata_const_entry available_framerates;
290  int found = 0;
291  int current_best_match = -1;
292  int requested_framerate = av_q2d(ctx->framerate);
293 
294  ACameraMetadata_getConstEntry(ctx->camera_metadata,
295  ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES,
296  &available_framerates);
297 
298  for (int i = 0; i < available_framerates.count; i++) {
299  int32_t min = available_framerates.data.i32[i * 2 + 0];
300  int32_t max = available_framerates.data.i32[i * 2 + 1];
301 
302  if (requested_framerate == max) {
303  if (min == max) {
304  ctx->framerate_range[0] = min;
305  ctx->framerate_range[1] = max;
306  found = 1;
307  break;
308  } else if (current_best_match >= 0) {
309  int32_t current_best_match_min = available_framerates.data.i32[current_best_match * 2 + 0];
310  if (min > current_best_match_min) {
311  current_best_match = i;
312  }
313  } else {
314  current_best_match = i;
315  }
316  }
317  }
318 
319  if (!found) {
320  if (current_best_match >= 0) {
321  ctx->framerate_range[0] = available_framerates.data.i32[current_best_match * 2 + 0];
322  ctx->framerate_range[1] = available_framerates.data.i32[current_best_match * 2 + 1];
323 
324  } else {
325  ctx->framerate_range[0] = available_framerates.data.i32[0];
326  ctx->framerate_range[1] = available_framerates.data.i32[1];
327  }
328 
329  av_log(avctx, AV_LOG_WARNING,
330  "Requested framerate %d not available, falling back to min: %d and max: %d fps\n",
331  requested_framerate, ctx->framerate_range[0], ctx->framerate_range[1]);
332  }
333 
334  return;
335 }
336 
337 static int get_image_format(AVFormatContext *avctx, AImage *image)
338 {
339  AndroidCameraCtx *ctx = avctx->priv_data;
340  int32_t image_pixelstrides[2];
341  uint8_t *image_plane_data[2];
342  int plane_data_length[2];
343 
344  for (int i = 0; i < 2; i++) {
345  AImage_getPlanePixelStride(image, i + 1, &image_pixelstrides[i]);
346  AImage_getPlaneData(image, i + 1, &image_plane_data[i], &plane_data_length[i]);
347  }
348 
349  if (image_pixelstrides[0] != image_pixelstrides[1]) {
350  av_log(avctx, AV_LOG_ERROR,
351  "Pixel strides of U and V plane should have been the same.\n");
352  return AVERROR_EXTERNAL;
353  }
354 
355  switch (image_pixelstrides[0]) {
356  case 1:
358  break;
359  case 2:
360  if (image_plane_data[0] < image_plane_data[1]) {
362  } else {
364  }
365  break;
366  default:
367  av_log(avctx, AV_LOG_ERROR,
368  "Unknown pixel stride %d of U and V plane, cannot determine camera image format.\n",
369  image_pixelstrides[0]);
370  return AVERROR(ENOSYS);
371  }
372 
373  return 0;
374 }
375 
376 static void image_available(void *context, AImageReader *reader)
377 {
378  AVFormatContext *avctx = context;
379  AndroidCameraCtx *ctx = avctx->priv_data;
380  media_status_t media_status;
381  int ret = 0;
382 
383  AImage *image;
384  int64_t image_timestamp;
385  int32_t image_linestrides[4];
386  uint8_t *image_plane_data[4];
387  int plane_data_length[4];
388 
389  AVPacket pkt;
390  int pkt_buffer_size = 0;
391 
392  media_status = AImageReader_acquireLatestImage(reader, &image);
393  if (media_status != AMEDIA_OK) {
394  if (media_status == AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE) {
395  av_log(avctx, AV_LOG_WARNING,
396  "An image reader frame was discarded");
397  } else {
398  av_log(avctx, AV_LOG_ERROR,
399  "Failed to acquire latest image from image reader, error: %s.\n",
400  media_status_string(media_status));
401  ret = AVERROR_EXTERNAL;
402  }
403  goto error;
404  }
405 
406  // Silently drop frames when exit is set
407  if (atomic_load(&ctx->exit)) {
408  goto error;
409  }
410 
411  // Determine actual image format
412  if (!atomic_load(&ctx->got_image_format)) {
413  ret = get_image_format(avctx, image);
414  if (ret < 0) {
415  av_log(avctx, AV_LOG_ERROR,
416  "Could not get image format of camera.\n");
417  goto error;
418  } else {
419  atomic_store(&ctx->got_image_format, 1);
420  }
421  }
422 
423  pkt_buffer_size = av_image_get_buffer_size(ctx->image_format, ctx->width, ctx->height, 32);
424  AImage_getTimestamp(image, &image_timestamp);
425 
426  AImage_getPlaneRowStride(image, 0, &image_linestrides[0]);
427  AImage_getPlaneData(image, 0, &image_plane_data[0], &plane_data_length[0]);
428 
429  switch (ctx->image_format) {
430  case AV_PIX_FMT_YUV420P:
431  AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
432  AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
433  AImage_getPlaneRowStride(image, 2, &image_linestrides[2]);
434  AImage_getPlaneData(image, 2, &image_plane_data[2], &plane_data_length[2]);
435  break;
436  case AV_PIX_FMT_NV12:
437  AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
438  AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
439  break;
440  case AV_PIX_FMT_NV21:
441  AImage_getPlaneRowStride(image, 2, &image_linestrides[1]);
442  AImage_getPlaneData(image, 2, &image_plane_data[1], &plane_data_length[1]);
443  break;
444  default:
445  av_log(avctx, AV_LOG_ERROR, "Unsupported camera image format.\n");
446  ret = AVERROR(ENOSYS);
447  goto error;
448  }
449 
450  ret = av_new_packet(&pkt, pkt_buffer_size);
451  if (ret < 0) {
452  av_log(avctx, AV_LOG_ERROR,
453  "Failed to create new av packet, error: %s.\n", av_err2str(ret));
454  goto error;
455  }
456 
458  pkt.pts = image_timestamp;
459  av_image_copy_to_buffer(pkt.data, pkt_buffer_size,
460  (const uint8_t * const *) image_plane_data,
461  image_linestrides, ctx->image_format,
462  ctx->width, ctx->height, 32);
463 
465 
466 error:
467  if (ret < 0) {
468  if (ret != AVERROR(EAGAIN)) {
469  av_log(avctx, AV_LOG_ERROR,
470  "Error while processing new image, error: %s.\n", av_err2str(ret));
472  atomic_store(&ctx->exit, 1);
473  } else {
474  av_log(avctx, AV_LOG_WARNING,
475  "Input queue was full, dropping frame, consider raising the input_queue_size option (current value: %d)\n",
476  ctx->input_queue_size);
477  }
478  if (pkt_buffer_size) {
479  av_packet_unref(&pkt);
480  }
481  }
482 
483  AImage_delete(image);
484 
485  return;
486 }
487 
489 {
490  AndroidCameraCtx *ctx = avctx->priv_data;
491  media_status_t ret;
492 
493  ret = AImageReader_new(ctx->width, ctx->height, IMAGE_FORMAT_ANDROID,
494  MAX_BUF_COUNT, &ctx->image_reader);
495  if (ret != AMEDIA_OK) {
496  av_log(avctx, AV_LOG_ERROR,
497  "Failed to create image reader, error: %s.\n", media_status_string(ret));
498  return AVERROR_EXTERNAL;
499  }
500 
501  ctx->image_listener.context = avctx;
502  ctx->image_listener.onImageAvailable = image_available;
503 
504  ret = AImageReader_setImageListener(ctx->image_reader, &ctx->image_listener);
505  if (ret != AMEDIA_OK) {
506  av_log(avctx, AV_LOG_ERROR,
507  "Failed to set image listener on image reader, error: %s.\n",
508  media_status_string(ret));
509  return AVERROR_EXTERNAL;
510  }
511 
512  ret = AImageReader_getWindow(ctx->image_reader, &ctx->image_reader_window);
513  if (ret != AMEDIA_OK) {
514  av_log(avctx, AV_LOG_ERROR,
515  "Could not get image reader window, error: %s.\n",
516  media_status_string(ret));
517  return AVERROR_EXTERNAL;
518  }
519 
520  return 0;
521 }
522 
523 static void capture_session_closed(void *context, ACameraCaptureSession *session)
524 {
525  av_log(context, AV_LOG_INFO, "Android camera capture session was closed.\n");
526 }
527 
528 static void capture_session_ready(void *context, ACameraCaptureSession *session)
529 {
530  av_log(context, AV_LOG_INFO, "Android camera capture session is ready.\n");
531 }
532 
533 static void capture_session_active(void *context, ACameraCaptureSession *session)
534 {
535  av_log(context, AV_LOG_INFO, "Android camera capture session is active.\n");
536 }
537 
539 {
540  AndroidCameraCtx *ctx = avctx->priv_data;
541  camera_status_t ret;
542 
543  ret = ACaptureSessionOutputContainer_create(&ctx->capture_session_output_container);
544  if (ret != ACAMERA_OK) {
545  av_log(avctx, AV_LOG_ERROR,
546  "Failed to create capture session output container, error: %s.\n",
547  camera_status_string(ret));
548  return AVERROR_EXTERNAL;
549  }
550 
551  ANativeWindow_acquire(ctx->image_reader_window);
552 
553  ret = ACaptureSessionOutput_create(ctx->image_reader_window, &ctx->capture_session_output);
554  if (ret != ACAMERA_OK) {
555  av_log(avctx, AV_LOG_ERROR,
556  "Failed to create capture session container, error: %s.\n",
557  camera_status_string(ret));
558  return AVERROR_EXTERNAL;
559  }
560 
561  ret = ACaptureSessionOutputContainer_add(ctx->capture_session_output_container,
563  if (ret != ACAMERA_OK) {
564  av_log(avctx, AV_LOG_ERROR,
565  "Failed to add output to output container, error: %s.\n",
566  camera_status_string(ret));
567  return AVERROR_EXTERNAL;
568  }
569 
570  ret = ACameraOutputTarget_create(ctx->image_reader_window, &ctx->camera_output_target);
571  if (ret != ACAMERA_OK) {
572  av_log(avctx, AV_LOG_ERROR,
573  "Failed to create camera output target, error: %s.\n",
574  camera_status_string(ret));
575  return AVERROR_EXTERNAL;
576  }
577 
578  ret = ACameraDevice_createCaptureRequest(ctx->camera_dev, TEMPLATE_RECORD, &ctx->capture_request);
579  if (ret != ACAMERA_OK) {
580  av_log(avctx, AV_LOG_ERROR,
581  "Failed to create capture request, error: %s.\n",
582  camera_status_string(ret));
583  return AVERROR_EXTERNAL;
584  }
585 
586  ret = ACaptureRequest_setEntry_i32(ctx->capture_request, ACAMERA_CONTROL_AE_TARGET_FPS_RANGE,
587  2, ctx->framerate_range);
588  if (ret != ACAMERA_OK) {
589  av_log(avctx, AV_LOG_ERROR,
590  "Failed to set target fps range in capture request, error: %s.\n",
591  camera_status_string(ret));
592  return AVERROR_EXTERNAL;
593  }
594 
595  ret = ACaptureRequest_addTarget(ctx->capture_request, ctx->camera_output_target);
596  if (ret != ACAMERA_OK) {
597  av_log(avctx, AV_LOG_ERROR,
598  "Failed to add capture request capture request, error: %s.\n",
599  camera_status_string(ret));
600  return AVERROR_EXTERNAL;
601  }
602 
603  ctx->capture_session_state_callbacks.context = avctx;
607 
608  ret = ACameraDevice_createCaptureSession(ctx->camera_dev, ctx->capture_session_output_container,
610  if (ret != ACAMERA_OK) {
611  av_log(avctx, AV_LOG_ERROR,
612  "Failed to create capture session, error: %s.\n",
613  camera_status_string(ret));
614  return AVERROR_EXTERNAL;
615  }
616 
617  ret = ACameraCaptureSession_setRepeatingRequest(ctx->capture_session, NULL, 1, &ctx->capture_request, NULL);
618  if (ret != ACAMERA_OK) {
619  av_log(avctx, AV_LOG_ERROR,
620  "Failed to set repeating request on capture session, error: %s.\n",
621  camera_status_string(ret));
622  return AVERROR_EXTERNAL;
623  }
624 
625  return 0;
626 }
627 
629 {
630  AndroidCameraCtx *ctx = avctx->priv_data;
631 
632  while (!atomic_load(&ctx->got_image_format) && !atomic_load(&ctx->exit)) {
633  //Wait until first frame arrived and actual image format was determined
634  usleep(1000);
635  }
636 
637  return atomic_load(&ctx->got_image_format);
638 }
639 
641 {
642  AndroidCameraCtx *ctx = avctx->priv_data;
643  uint8_t *side_data;
644  int32_t display_matrix[9];
645 
646  av_display_rotation_set(display_matrix, ctx->sensor_orientation);
647 
648  if (ctx->lens_facing == ACAMERA_LENS_FACING_FRONT) {
649  av_display_matrix_flip(display_matrix, 1, 0);
650  }
651 
652  side_data = av_stream_new_side_data(st,
653  AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
654 
655  if (!side_data) {
656  return AVERROR(ENOMEM);
657  }
658 
659  memcpy(side_data, display_matrix, sizeof(display_matrix));
660 
661  return 0;
662 }
663 
665 {
666  AndroidCameraCtx *ctx = avctx->priv_data;
667  AVStream *st;
668  AVCodecParameters *codecpar;
669 
670  st = avformat_new_stream(avctx, NULL);
671  if (!st) {
672  return AVERROR(ENOMEM);
673  }
674 
675  st->id = VIDEO_STREAM_INDEX;
676  st->avg_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
677  st->r_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
678 
679  if (!wait_for_image_format(avctx)) {
680  return AVERROR_EXTERNAL;
681  }
682 
683  codecpar = st->codecpar;
684  codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
685  codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
686  codecpar->format = ctx->image_format;
687  codecpar->width = ctx->width;
688  codecpar->height = ctx->height;
689 
691 
692  return add_display_matrix(avctx, st);
693 }
694 
696 {
697  AndroidCameraCtx *ctx = avctx->priv_data;
698 
699  atomic_store(&ctx->exit, 1);
700 
701  if (ctx->capture_session) {
702  ACameraCaptureSession_stopRepeating(ctx->capture_session);
703  // Following warning is emitted, after capture session closed callback is received:
704  // ACameraCaptureSession: Device is closed but session 0 is not notified
705  // Seems to be a bug in Android, we can ignore this
706  ACameraCaptureSession_close(ctx->capture_session);
707  ctx->capture_session = NULL;
708  }
709 
710  if (ctx->capture_request) {
711  ACaptureRequest_removeTarget(ctx->capture_request, ctx->camera_output_target);
712  ACaptureRequest_free(ctx->capture_request);
713  ctx->capture_request = NULL;
714  }
715 
716  if (ctx->camera_output_target) {
717  ACameraOutputTarget_free(ctx->camera_output_target);
718  ctx->camera_output_target = NULL;
719  }
720 
721  if (ctx->capture_session_output) {
722  ACaptureSessionOutputContainer_remove(ctx->capture_session_output_container,
724  ACaptureSessionOutput_free(ctx->capture_session_output);
726  }
727 
728  if (ctx->image_reader_window) {
729  ANativeWindow_release(ctx->image_reader_window);
730  ctx->image_reader_window = NULL;
731  }
732 
734  ACaptureSessionOutputContainer_free(ctx->capture_session_output_container);
736  }
737 
738  if (ctx->camera_dev) {
739  ACameraDevice_close(ctx->camera_dev);
740  ctx->camera_dev = NULL;
741  }
742 
743  if (ctx->image_reader) {
744  AImageReader_delete(ctx->image_reader);
745  ctx->image_reader = NULL;
746  }
747 
748  if (ctx->camera_metadata) {
749  ACameraMetadata_free(ctx->camera_metadata);
750  ctx->camera_metadata = NULL;
751  }
752 
753  av_freep(&ctx->camera_id);
754 
755  if (ctx->camera_mgr) {
756  ACameraManager_delete(ctx->camera_mgr);
757  ctx->camera_mgr = NULL;
758  }
759 
760  if (ctx->input_queue) {
761  AVPacket pkt;
764  av_packet_unref(&pkt);
765  }
767  }
768 
769  return 0;
770 }
771 
773 {
774  AndroidCameraCtx *ctx = avctx->priv_data;
775  int ret;
776 
777  atomic_init(&ctx->got_image_format, 0);
778  atomic_init(&ctx->exit, 0);
779 
781  if (ret < 0) {
782  av_log(avctx, AV_LOG_ERROR,
783  "Failed to allocate input queue, error: %s.\n", av_err2str(ret));
784  goto error;
785  }
786 
787  ctx->camera_mgr = ACameraManager_create();
788  if (!ctx->camera_mgr) {
789  av_log(avctx, AV_LOG_ERROR, "Failed to create Android camera manager.\n");
790  ret = AVERROR_EXTERNAL;
791  goto error;
792  }
793 
794  ret = open_camera(avctx);
795  if (ret < 0) {
796  av_log(avctx, AV_LOG_ERROR, "Failed to open camera.\n");
797  goto error;
798  }
799 
800  get_sensor_orientation(avctx);
801  match_video_size(avctx);
802  match_framerate(avctx);
803 
804  ret = create_image_reader(avctx);
805  if (ret < 0) {
806  goto error;
807  }
808 
809  ret = create_capture_session(avctx);
810  if (ret < 0) {
811  goto error;
812  }
813 
814  ret = add_video_stream(avctx);
815 
816 error:
817  if (ret < 0) {
819  av_log(avctx, AV_LOG_ERROR, "Failed to open android_camera.\n");
820  }
821 
822  return ret;
823 }
824 
826 {
827  AndroidCameraCtx *ctx = avctx->priv_data;
828  int ret;
829 
830  if (!atomic_load(&ctx->exit)) {
833  } else {
834  ret = AVERROR_EOF;
835  }
836 
837  if (ret < 0) {
838  return ret;
839  } else {
840  return pkt->size;
841  }
842 }
843 
844 #define OFFSET(x) offsetof(AndroidCameraCtx, x)
845 #define DEC AV_OPT_FLAG_DECODING_PARAM
846 static const AVOption options[] = {
847  { "video_size", "set video size given as a string such as 640x480 or hd720", OFFSET(requested_width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, DEC },
848  { "framerate", "set video frame rate", OFFSET(framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "30"}, 0, INT_MAX, DEC },
849  { "camera_index", "set index of camera to use", OFFSET(camera_index), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, DEC },
850  { "input_queue_size", "set maximum number of frames to buffer", OFFSET(input_queue_size), AV_OPT_TYPE_INT, {.i64 = 5}, 0, INT_MAX, DEC },
851  { NULL },
852 };
853 
855  .class_name = "android_camera indev",
856  .item_name = av_default_item_name,
857  .option = options,
858  .version = LIBAVUTIL_VERSION_INT,
860 };
861 
863  .name = "android_camera",
864  .long_name = NULL_IF_CONFIG_SMALL("Android camera input device"),
865  .priv_data_size = sizeof(AndroidCameraCtx),
869  .flags = AVFMT_NOFILE,
870  .priv_class = &android_camera_class,
871 };
#define NULL
Definition: coverity.c:32
const char const char void * val
Definition: avisynth_c.h:771
static void capture_session_ready(void *context, ACameraCaptureSession *session)
static const char * format[]
Definition: af_aiir.c:311
#define DEC
static void capture_session_active(void *context, ACameraCaptureSession *session)
#define atomic_store(object, desired)
Definition: stdatomic.h:85
AVOption.
Definition: opt.h:246
void av_thread_message_queue_set_err_recv(AVThreadMessageQueue *mq, int err)
Set the receiving error code.
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
Definition: imgutils.c:453
ACaptureSessionOutput * capture_session_output
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
void avpriv_set_pts_info(AVStream *s, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)
Set the time base and wrapping info for a given stream.
Definition: utils.c:4811
#define IMAGE_FORMAT_ANDROID
enum AVCodecID codec_id
Specific type of the encoded data (the codec used).
Definition: avcodec.h:3884
int size
Definition: avcodec.h:1431
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:191
static const char * media_status_string(media_status_t val)
#define VIDEO_STREAM_INDEX
static AVPacket pkt
ANativeWindow * image_reader_window
intptr_t atomic_int
Definition: stdatomic.h:55
static int wait_for_image_format(AVFormatContext *avctx)
This struct describes the properties of an encoded stream.
Definition: avcodec.h:3876
void av_display_matrix_flip(int32_t matrix[9], int hflip, int vflip)
Flip the input matrix horizontally and/or vertically.
Definition: display.c:65
void av_display_rotation_set(int32_t matrix[9], double angle)
Initialize a transformation matrix describing a pure counterclockwise rotation by the specified angle...
Definition: display.c:50
Format I/O context.
Definition: avformat.h:1342
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
#define AVFMT_FLAG_NONBLOCK
Do not block when reading packets from input.
Definition: avformat.h:1476
static int add_video_stream(AVFormatContext *avctx)
uint8_t
int av_thread_message_queue_recv(AVThreadMessageQueue *mq, void *msg, unsigned flags)
Receive a message from the queue.
int width
Video only.
Definition: avcodec.h:3950
static const AVClass android_camera_class
AVOptions.
static int android_camera_read_close(AVFormatContext *avctx)
ACaptureSessionOutputContainer * capture_session_output_container
This side data contains a 3x3 transformation matrix describing an affine transformation that needs to...
Definition: avcodec.h:1211
int av_thread_message_queue_send(AVThreadMessageQueue *mq, void *msg, unsigned flags)
Send a message on the queue.
int id
Format-specific stream ID.
Definition: avformat.h:880
static const char * camera_status_string(camera_status_t val)
#define VIDEO_TIMEBASE_ANDROID
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
Definition: utils.c:4441
#define MAX_BUF_COUNT
ACaptureRequest * capture_request
int flags
Flags modifying the (de)muxer behaviour.
Definition: avformat.h:1473
uint8_t * data
Definition: avcodec.h:1430
static double av_q2d(AVRational a)
Convert an AVRational to a double.
Definition: rational.h:104
static int flags
Definition: log.c:55
#define AVERROR_EOF
End of file.
Definition: error.h:55
int32_t framerate_range[2]
static av_cold int read_close(AVFormatContext *ctx)
Definition: libcdio.c:145
#define RETURN_DEFAULT(x)
static void camera_dev_error(void *context, ACameraDevice *device, int error)
#define av_log(a,...)
static int android_camera_read_header(AVFormatContext *avctx)
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:86
AVThreadMessageQueue * input_queue
int av_image_get_buffer_size(enum AVPixelFormat pix_fmt, int width, int height, int align)
Return the size in bytes of the amount of data required to store an image with the given parameters...
Definition: imgutils.c:431
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
#define atomic_load(object)
Definition: stdatomic.h:93
static void capture_session_closed(void *context, ACameraCaptureSession *session)
#define AVERROR(e)
Definition: error.h:43
AVRational framerate
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
Display matrix.
enum AVMediaType codec_type
General type of the encoded data.
Definition: avcodec.h:3880
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:85
AVInputFormat ff_android_camera_demuxer
AVRational avg_frame_rate
Average framerate.
Definition: avformat.h:946
static int create_image_reader(AVFormatContext *avctx)
ACameraCaptureSession * capture_session
as above, but U and V bytes are swapped
Definition: pixfmt.h:86
ACameraDevice_StateCallbacks camera_state_callbacks
static void get_sensor_orientation(AVFormatContext *avctx)
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:119
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
ACameraManager * camera_mgr
int32_t sensor_orientation
uint8_t * av_stream_new_side_data(AVStream *stream, enum AVPacketSideDataType type, int size)
Allocate new information from stream.
Definition: utils.c:5456
static void error(const char *err)
static int read_header(FFV1Context *f)
Definition: ffv1dec.c:530
#define RETURN_CASE(x)
#define OFFSET(x)
Stream structure.
Definition: avformat.h:873
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
Definition: avio_reading.c:42
ACameraMetadata * camera_metadata
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:251
static int open_camera(AVFormatContext *avctx)
void av_thread_message_queue_set_err_send(AVThreadMessageQueue *mq, int err)
Set the sending error code.
static void match_video_size(AVFormatContext *avctx)
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:592
Perform non-blocking operation.
Definition: threadmessage.h:31
static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
Describe the class of an AVClass context structure.
Definition: log.h:67
static const AVOption options[]
static int create_capture_session(AVFormatContext *avctx)
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int av_thread_message_queue_alloc(AVThreadMessageQueue **mq, unsigned nelem, unsigned elsize)
Allocate a new message queue.
Definition: threadmessage.c:40
offset must point to AVRational
Definition: opt.h:236
ACameraOutputTarget * camera_output_target
AImageReader_ImageListener image_listener
offset must point to two consecutive integers
Definition: opt.h:233
misc parsing utilities
Libavdevice version macros.
void av_thread_message_queue_free(AVThreadMessageQueue **mq)
Free a message queue.
Definition: threadmessage.c:91
ACameraCaptureSession_stateCallbacks capture_session_state_callbacks
static void match_framerate(AVFormatContext *avctx)
static const char * error_state_callback_string(int val)
static int android_camera_read_packet(AVFormatContext *avctx, AVPacket *pkt)
Main libavformat public API header.
ACameraDevice * camera_dev
static void image_available(void *context, AImageReader *reader)
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:62
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:465
static void camera_dev_disconnected(void *context, ACameraDevice *device)
pixel format definitions
void * priv_data
Format private data.
Definition: avformat.h:1370
atomic_int got_image_format
#define atomic_init(obj, value)
Definition: stdatomic.h:33
#define av_freep(p)
const char * name
A comma separated list of short names for the format.
Definition: avformat.h:647
AImageReader * image_reader
AVCodecParameters * codecpar
Codec parameters associated with this stream.
Definition: avformat.h:1020
int stream_index
Definition: avcodec.h:1432
AVRational r_frame_rate
Real base framerate of the stream.
Definition: avformat.h:997
float min
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
This structure stores compressed data.
Definition: avcodec.h:1407
static int get_image_format(AVFormatContext *avctx, AImage *image)
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1423