lavd/avfoundation: Fix skewed video output
[ffmpeg.git] / libavdevice / avfoundation.m
index 895bd29..98552ac 100644 (file)
 
 #include "libavutil/pixdesc.h"
 #include "libavutil/opt.h"
+#include "libavutil/avstring.h"
 #include "libavformat/internal.h"
 #include "libavutil/internal.h"
+#include "libavutil/parseutils.h"
 #include "libavutil/time.h"
+#include "libavutil/imgutils.h"
 #include "avdevice.h"
 
-static const int avf_time_base = 100;
+static const int avf_time_base = 1000000;
 
 static const AVRational avf_time_base_q = {
     .num = 1,
@@ -70,7 +73,7 @@ static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
     { AV_PIX_FMT_YUV420P,      kCVPixelFormatType_420YpCbCr8Planar },
     { AV_PIX_FMT_NV12,         kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
     { AV_PIX_FMT_YUYV422,      kCVPixelFormatType_422YpCbCr8_yuvs },
-#if __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
     { AV_PIX_FMT_GRAY8,        kCVPixelFormatType_OneComponent8 },
 #endif
     { AV_PIX_FMT_NONE, 0 }
@@ -80,20 +83,50 @@ typedef struct
 {
     AVClass*        class;
 
-    float           frame_rate;
     int             frames_captured;
+    int             audio_frames_captured;
     int64_t         first_pts;
+    int64_t         first_audio_pts;
     pthread_mutex_t frame_lock;
     pthread_cond_t  frame_wait_cond;
     id              avf_delegate;
+    id              avf_audio_delegate;
+
+    AVRational      framerate;
+    int             width, height;
+
+    int             capture_cursor;
+    int             capture_mouse_clicks;
 
     int             list_devices;
     int             video_device_index;
+    int             video_stream_index;
+    int             audio_device_index;
+    int             audio_stream_index;
+
+    char            *video_filename;
+    char            *audio_filename;
+
+    int             num_video_devices;
+
+    int             audio_channels;
+    int             audio_bits_per_sample;
+    int             audio_float;
+    int             audio_be;
+    int             audio_signed_integer;
+    int             audio_packed;
+    int             audio_non_interleaved;
+
+    int32_t         *audio_buffer;
+    int             audio_buffer_size;
+
     enum AVPixelFormat pixel_format;
 
     AVCaptureSession         *capture_session;
     AVCaptureVideoDataOutput *video_output;
+    AVCaptureAudioDataOutput *audio_output;
     CMSampleBufferRef         current_frame;
+    CMSampleBufferRef         current_audio_frame;
 } AVFContext;
 
 static void lock_frames(AVFContext* ctx)
@@ -152,17 +185,69 @@ static void unlock_frames(AVFContext* ctx)
 
 @end
 
+/** AudioReciever class - delegate for AVCaptureSession
+ */
+@interface AVFAudioReceiver : NSObject
+{
+    AVFContext* _context;
+}
+
+- (id)initWithContext:(AVFContext*)context;
+
+- (void)  captureOutput:(AVCaptureOutput *)captureOutput
+  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
+         fromConnection:(AVCaptureConnection *)connection;
+
+@end
+
+@implementation AVFAudioReceiver
+
+- (id)initWithContext:(AVFContext*)context
+{
+    if (self = [super init]) {
+        _context = context;
+    }
+    return self;
+}
+
+- (void)  captureOutput:(AVCaptureOutput *)captureOutput
+  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
+         fromConnection:(AVCaptureConnection *)connection
+{
+    lock_frames(_context);
+
+    if (_context->current_audio_frame != nil) {
+        CFRelease(_context->current_audio_frame);
+    }
+
+    _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
+
+    pthread_cond_signal(&_context->frame_wait_cond);
+
+    unlock_frames(_context);
+
+    ++_context->audio_frames_captured;
+}
+
+@end
+
 static void destroy_context(AVFContext* ctx)
 {
     [ctx->capture_session stopRunning];
 
     [ctx->capture_session release];
     [ctx->video_output    release];
+    [ctx->audio_output    release];
     [ctx->avf_delegate    release];
+    [ctx->avf_audio_delegate release];
 
     ctx->capture_session = NULL;
     ctx->video_output    = NULL;
+    ctx->audio_output    = NULL;
     ctx->avf_delegate    = NULL;
+    ctx->avf_audio_delegate = NULL;
+
+    av_freep(&ctx->audio_buffer);
 
     pthread_mutex_destroy(&ctx->frame_lock);
     pthread_cond_destroy(&ctx->frame_wait_cond);
@@ -172,20 +257,138 @@ static void destroy_context(AVFContext* ctx)
     }
 }
 
+static void parse_device_name(AVFormatContext *s)
+{
+    AVFContext *ctx = (AVFContext*)s->priv_data;
+    char *tmp = av_strdup(s->url);
+    char *save;
+
+    if (tmp[0] != ':') {
+        ctx->video_filename = av_strtok(tmp,  ":", &save);
+        ctx->audio_filename = av_strtok(NULL, ":", &save);
+    } else {
+        ctx->audio_filename = av_strtok(tmp,  ":", &save);
+    }
+}
+
+/**
+ * Configure the video device.
+ *
+ * Configure the video device using a run-time approach to access properties
+ * since formats, activeFormat are available since  iOS >= 7.0 or OSX >= 10.7
+ * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
+ *
+ * The NSUndefinedKeyException must be handled by the caller of this function.
+ *
+ */
+static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
+{
+    AVFContext *ctx = (AVFContext*)s->priv_data;
+
+    double framerate = av_q2d(ctx->framerate);
+    NSObject *range = nil;
+    NSObject *format = nil;
+    NSObject *selected_range = nil;
+    NSObject *selected_format = nil;
+
+    for (format in [video_device valueForKey:@"formats"]) {
+        CMFormatDescriptionRef formatDescription;
+        CMVideoDimensions dimensions;
+
+        formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
+        dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
+
+        if ((ctx->width == 0 && ctx->height == 0) ||
+            (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
+
+            selected_format = format;
+
+            for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
+                double max_framerate;
+
+                [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
+                if (fabs (framerate - max_framerate) < 0.01) {
+                    selected_range = range;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (!selected_format) {
+        av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device\n",
+            ctx->width, ctx->height);
+        goto unsupported_format;
+    }
+
+    if (!selected_range) {
+        av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device\n",
+            framerate);
+        goto unsupported_format;
+    }
+
+    if ([video_device lockForConfiguration:NULL] == YES) {
+        NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
+
+        [video_device setValue:selected_format forKey:@"activeFormat"];
+        [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
+        [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
+    } else {
+        av_log(s, AV_LOG_ERROR, "Could not lock device for configuration");
+        return AVERROR(EINVAL);
+    }
+
+    return 0;
+
+unsupported_format:
+
+    av_log(s, AV_LOG_ERROR, "Supported modes:\n");
+    for (format in [video_device valueForKey:@"formats"]) {
+        CMFormatDescriptionRef formatDescription;
+        CMVideoDimensions dimensions;
+
+        formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
+        dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
+
+        for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
+            double min_framerate;
+            double max_framerate;
+
+            [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
+            [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
+            av_log(s, AV_LOG_ERROR, "  %dx%d@[%f %f]fps\n",
+                dimensions.width, dimensions.height,
+                min_framerate, max_framerate);
+        }
+    }
+    return AVERROR(EINVAL);
+}
+
 static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
 {
     AVFContext *ctx = (AVFContext*)s->priv_data;
+    int ret;
     NSError *error  = nil;
-    AVCaptureDeviceInput* capture_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
+    AVCaptureInput* capture_input = nil;
+    struct AVFPixelFormatSpec pxl_fmt_spec;
+    NSNumber *pixel_format;
+    NSDictionary *capture_dict;
+    dispatch_queue_t queue;
+
+    if (ctx->video_device_index < ctx->num_video_devices) {
+        capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
+    } else {
+        capture_input = (AVCaptureInput*) video_device;
+    }
 
-    if (!capture_dev_input) {
+    if (!capture_input) {
         av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
                [[error localizedDescription] UTF8String]);
         return 1;
     }
 
-    if ([ctx->capture_session canAddInput:capture_dev_input]) {
-        [ctx->capture_session addInput:capture_dev_input];
+    if ([ctx->capture_session canAddInput:capture_input]) {
+        [ctx->capture_session addInput:capture_input];
     } else {
         av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
         return 1;
@@ -199,8 +402,19 @@ static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
         return 1;
     }
 
+    // Configure device framerate and video size
+    @try {
+        if ((ret = configure_video_device(s, video_device)) < 0) {
+            return ret;
+        }
+    } @catch (NSException *exception) {
+        if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
+          av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
+          return AVERROR_EXTERNAL;
+        }
+    }
+
     // select pixel format
-    struct AVFPixelFormatSpec pxl_fmt_spec;
     pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
 
     for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
@@ -255,8 +469,8 @@ static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
     }
 
     ctx->pixel_format          = pxl_fmt_spec.ff_id;
-    NSNumber     *pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
-    NSDictionary *capture_dict = [NSDictionary dictionaryWithObject:pixel_format
+    pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
+    capture_dict = [NSDictionary dictionaryWithObject:pixel_format
                                                forKey:(id)kCVPixelBufferPixelFormatTypeKey];
 
     [ctx->video_output setVideoSettings:capture_dict];
@@ -264,7 +478,7 @@ static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
 
     ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];
 
-    dispatch_queue_t queue = dispatch_queue_create("avf_queue", NULL);
+    queue = dispatch_queue_create("avf_queue", NULL);
     [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
     dispatch_release(queue);
 
@@ -278,9 +492,60 @@ static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
     return 0;
 }
 
+static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
+{
+    AVFContext *ctx = (AVFContext*)s->priv_data;
+    NSError *error  = nil;
+    AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
+    dispatch_queue_t queue;
+
+    if (!audio_dev_input) {
+        av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
+               [[error localizedDescription] UTF8String]);
+        return 1;
+    }
+
+    if ([ctx->capture_session canAddInput:audio_dev_input]) {
+        [ctx->capture_session addInput:audio_dev_input];
+    } else {
+        av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
+        return 1;
+    }
+
+    // Attaching output
+    ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];
+
+    if (!ctx->audio_output) {
+        av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
+        return 1;
+    }
+
+    ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];
+
+    queue = dispatch_queue_create("avf_audio_queue", NULL);
+    [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
+    dispatch_release(queue);
+
+    if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
+        [ctx->capture_session addOutput:ctx->audio_output];
+    } else {
+        av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
+        return 1;
+    }
+
+    return 0;
+}
+
 static int get_video_config(AVFormatContext *s)
 {
     AVFContext *ctx = (AVFContext*)s->priv_data;
+    CVImageBufferRef image_buffer;
+    CGSize image_buffer_size;
+    AVStream* stream = avformat_new_stream(s, NULL);
+
+    if (!stream) {
+        return 1;
+    }
 
     // Take stream info from the first frame.
     while (ctx->frames_captured < 1) {
@@ -289,25 +554,106 @@ static int get_video_config(AVFormatContext *s)
 
     lock_frames(ctx);
 
+    ctx->video_stream_index = stream->index;
+
+    avpriv_set_pts_info(stream, 64, 1, avf_time_base);
+
+    image_buffer      = CMSampleBufferGetImageBuffer(ctx->current_frame);
+    image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
+
+    stream->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
+    stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+    stream->codecpar->width      = (int)image_buffer_size.width;
+    stream->codecpar->height     = (int)image_buffer_size.height;
+    stream->codecpar->format     = ctx->pixel_format;
+
+    CFRelease(ctx->current_frame);
+    ctx->current_frame = nil;
+
+    unlock_frames(ctx);
+
+    return 0;
+}
+
+static int get_audio_config(AVFormatContext *s)
+{
+    AVFContext *ctx = (AVFContext*)s->priv_data;
+    CMFormatDescriptionRef format_desc;
     AVStream* stream = avformat_new_stream(s, NULL);
 
     if (!stream) {
         return 1;
     }
 
+    // Take stream info from the first frame.
+    while (ctx->audio_frames_captured < 1) {
+        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
+    }
+
+    lock_frames(ctx);
+
+    ctx->audio_stream_index = stream->index;
+
     avpriv_set_pts_info(stream, 64, 1, avf_time_base);
 
-    CVImageBufferRef image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
-    CGSize image_buffer_size      = CVImageBufferGetEncodedSize(image_buffer);
+    format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
+    const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
 
-    stream->codec->codec_id   = AV_CODEC_ID_RAWVIDEO;
-    stream->codec->codec_type = AVMEDIA_TYPE_VIDEO;
-    stream->codec->width      = (int)image_buffer_size.width;
-    stream->codec->height     = (int)image_buffer_size.height;
-    stream->codec->pix_fmt    = ctx->pixel_format;
+    if (!basic_desc) {
+        av_log(s, AV_LOG_ERROR, "audio format not available\n");
+        return 1;
+    }
 
-    CFRelease(ctx->current_frame);
-    ctx->current_frame = nil;
+    stream->codecpar->codec_type     = AVMEDIA_TYPE_AUDIO;
+    stream->codecpar->sample_rate    = basic_desc->mSampleRate;
+    stream->codecpar->channels       = basic_desc->mChannelsPerFrame;
+    stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
+
+    ctx->audio_channels        = basic_desc->mChannelsPerFrame;
+    ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
+    ctx->audio_float           = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
+    ctx->audio_be              = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
+    ctx->audio_signed_integer  = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
+    ctx->audio_packed          = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
+    ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
+
+    if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
+        ctx->audio_float &&
+        ctx->audio_bits_per_sample == 32 &&
+        ctx->audio_packed) {
+        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
+    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
+        ctx->audio_signed_integer &&
+        ctx->audio_bits_per_sample == 16 &&
+        ctx->audio_packed) {
+        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
+    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
+        ctx->audio_signed_integer &&
+        ctx->audio_bits_per_sample == 24 &&
+        ctx->audio_packed) {
+        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
+    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
+        ctx->audio_signed_integer &&
+        ctx->audio_bits_per_sample == 32 &&
+        ctx->audio_packed) {
+        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
+    } else {
+        av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
+        return 1;
+    }
+
+    if (ctx->audio_non_interleaved) {
+        CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
+        ctx->audio_buffer_size        = CMBlockBufferGetDataLength(block_buffer);
+        ctx->audio_buffer             = av_malloc(ctx->audio_buffer_size);
+        if (!ctx->audio_buffer) {
+            av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
+            return 1;
+        }
+    }
+
+    CFRelease(ctx->current_audio_frame);
+    ctx->current_audio_frame = nil;
 
     unlock_frames(ctx);
 
@@ -317,83 +663,224 @@ static int get_video_config(AVFormatContext *s)
 static int avf_read_header(AVFormatContext *s)
 {
     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
+    int capture_screen      = 0;
+    uint32_t num_screens    = 0;
     AVFContext *ctx         = (AVFContext*)s->priv_data;
+    AVCaptureDevice *video_device = nil;
+    AVCaptureDevice *audio_device = nil;
+    // Find capture device
+    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+    ctx->num_video_devices = [devices count];
+
     ctx->first_pts          = av_gettime();
+    ctx->first_audio_pts    = av_gettime();
 
     pthread_mutex_init(&ctx->frame_lock, NULL);
     pthread_cond_init(&ctx->frame_wait_cond, NULL);
 
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
+    CGGetActiveDisplayList(0, NULL, &num_screens);
+#endif
+
     // List devices if requested
     if (ctx->list_devices) {
+        int index = 0;
         av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
-        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+        for (AVCaptureDevice *device in devices) {
+            const char *name = [[device localizedName] UTF8String];
+            index            = [devices indexOfObject:device];
+            av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
+            index++;
+        }
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
+        if (num_screens > 0) {
+            CGDirectDisplayID screens[num_screens];
+            CGGetActiveDisplayList(num_screens, screens, &num_screens);
+            for (int i = 0; i < num_screens; i++) {
+                av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", index + i, i);
+            }
+        }
+#endif
+
+        av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
+        devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
         for (AVCaptureDevice *device in devices) {
             const char *name = [[device localizedName] UTF8String];
             int index  = [devices indexOfObject:device];
             av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
         }
-        goto fail;
+         goto fail;
     }
 
-    // Find capture device
-    AVCaptureDevice *video_device = nil;
+    // parse input filename for video and audio device
+    parse_device_name(s);
 
     // check for device index given in filename
-    if (ctx->video_device_index == -1) {
-        sscanf(s->filename, "%d", &ctx->video_device_index);
+    if (ctx->video_device_index == -1 && ctx->video_filename) {
+        sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
+    }
+    if (ctx->audio_device_index == -1 && ctx->audio_filename) {
+        sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
     }
 
     if (ctx->video_device_index >= 0) {
-        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
+        if (ctx->video_device_index < ctx->num_video_devices) {
+            video_device = [devices objectAtIndex:ctx->video_device_index];
+        } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
+            CGDirectDisplayID screens[num_screens];
+            CGGetActiveDisplayList(num_screens, screens, &num_screens);
+            AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
+
+            if (ctx->framerate.num > 0) {
+                capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
+            }
+
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
+            if (ctx->capture_cursor) {
+                capture_screen_input.capturesCursor = YES;
+            } else {
+                capture_screen_input.capturesCursor = NO;
+            }
+#endif
 
-        if (ctx->video_device_index >= [devices count]) {
+            if (ctx->capture_mouse_clicks) {
+                capture_screen_input.capturesMouseClicks = YES;
+            } else {
+                capture_screen_input.capturesMouseClicks = NO;
+            }
+
+            video_device = (AVCaptureDevice*) capture_screen_input;
+            capture_screen = 1;
+#endif
+         } else {
             av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
             goto fail;
         }
-
-        video_device = [devices objectAtIndex:ctx->video_device_index];
-    } else if (strncmp(s->filename, "",        1) &&
-               strncmp(s->filename, "default", 7)) {
-        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
-
+    } else if (ctx->video_filename &&
+               strncmp(ctx->video_filename, "none", 4)) {
+        if (!strncmp(ctx->video_filename, "default", 7)) {
+            video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
+        } else {
+        // looking for video inputs
         for (AVCaptureDevice *device in devices) {
-            if (!strncmp(s->filename, [[device localizedName] UTF8String], strlen(s->filename))) {
+            if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
                 video_device = device;
                 break;
             }
         }
 
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
+        // looking for screen inputs
+        if (!video_device) {
+            int idx;
+            if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
+                CGDirectDisplayID screens[num_screens];
+                CGGetActiveDisplayList(num_screens, screens, &num_screens);
+                AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
+                video_device = (AVCaptureDevice*) capture_screen_input;
+                ctx->video_device_index = ctx->num_video_devices + idx;
+                capture_screen = 1;
+
+                if (ctx->framerate.num > 0) {
+                    capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
+                }
+
+#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
+                if (ctx->capture_cursor) {
+                    capture_screen_input.capturesCursor = YES;
+                } else {
+                    capture_screen_input.capturesCursor = NO;
+                }
+#endif
+
+                if (ctx->capture_mouse_clicks) {
+                    capture_screen_input.capturesMouseClicks = YES;
+                } else {
+                    capture_screen_input.capturesMouseClicks = NO;
+                }
+            }
+        }
+#endif
+        }
+
         if (!video_device) {
             av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
             goto fail;
         }
-    } else {
-        video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
     }
 
-    // Video capture device not found, looking for AVMediaTypeVideo
-    if (!video_device) {
-        video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
+    // get audio device
+    if (ctx->audio_device_index >= 0) {
+        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
 
-        if (!video_device) {
-            av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
+        if (ctx->audio_device_index >= [devices count]) {
+            av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
             goto fail;
         }
+
+        audio_device = [devices objectAtIndex:ctx->audio_device_index];
+    } else if (ctx->audio_filename &&
+               strncmp(ctx->audio_filename, "none", 4)) {
+        if (!strncmp(ctx->audio_filename, "default", 7)) {
+            audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
+        } else {
+        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
+
+        for (AVCaptureDevice *device in devices) {
+            if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
+                audio_device = device;
+                break;
+            }
+        }
+        }
+
+        if (!audio_device) {
+            av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
+             goto fail;
+        }
     }
 
-    NSString* dev_display_name = [video_device localizedName];
-    av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [dev_display_name UTF8String]);
+    // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
+    if (!video_device && !audio_device) {
+        av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
+        goto fail;
+    }
+
+    if (video_device) {
+        if (ctx->video_device_index < ctx->num_video_devices) {
+            av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
+        } else {
+            av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
+        }
+    }
+    if (audio_device) {
+        av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
+    }
 
     // Initialize capture session
     ctx->capture_session = [[AVCaptureSession alloc] init];
 
-    if (add_video_device(s, video_device)) {
+    if (video_device && add_video_device(s, video_device)) {
         goto fail;
     }
+    if (audio_device && add_audio_device(s, audio_device)) {
+    }
 
     [ctx->capture_session startRunning];
 
-    if (get_video_config(s)) {
+    /* Unlock device configuration only after the session is started so it
+     * does not reset the capture formats */
+    if (!capture_screen) {
+        [video_device unlockForConfiguration];
+    }
+
+    if (video_device && get_video_config(s)) {
+        goto fail;
+    }
+
+    // set audio stream
+    if (audio_device && get_audio_config(s)) {
         goto fail;
     }
 
@@ -406,34 +893,151 @@ fail:
     return AVERROR(EIO);
 }
 
+static int copy_cvpixelbuffer(AVFormatContext *s,
+                               CVPixelBufferRef image_buffer,
+                               AVPacket *pkt)
+{
+    AVFContext *ctx = s->priv_data;
+    int src_linesize[4];
+    const uint8_t *src_data[4];
+    int width  = CVPixelBufferGetWidth(image_buffer);
+    int height = CVPixelBufferGetHeight(image_buffer);
+    int status;
+
+    memset(src_linesize, 0, sizeof(src_linesize));
+    memset(src_data, 0, sizeof(src_data));
+
+    status = CVPixelBufferLockBaseAddress(image_buffer, 0);
+    if (status != kCVReturnSuccess) {
+        av_log(s, AV_LOG_ERROR, "Could not lock base address: %d\n", status);
+        return AVERROR_EXTERNAL;
+    }
+
+    if (CVPixelBufferIsPlanar(image_buffer)) {
+        size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);
+        int i;
+        for(i = 0; i < plane_count; i++){
+            src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);
+            src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);
+        }
+    } else {
+        src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);
+        src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);
+    }
+
+    status = av_image_copy_to_buffer(pkt->data, pkt->size,
+                                     src_data, src_linesize,
+                                     ctx->pixel_format, width, height, 1);
+
+
+
+    CVPixelBufferUnlockBaseAddress(image_buffer, 0);
+
+    return status;
+}
+
 static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
     AVFContext* ctx = (AVFContext*)s->priv_data;
 
     do {
+        CVImageBufferRef image_buffer;
         lock_frames(ctx);
 
-        CVImageBufferRef image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
+        image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
 
         if (ctx->current_frame != nil) {
+            int status;
             if (av_new_packet(pkt, (int)CVPixelBufferGetDataSize(image_buffer)) < 0) {
                 return AVERROR(EIO);
             }
 
-            pkt->pts = pkt->dts = av_rescale_q(av_gettime() - ctx->first_pts,
-                                               AV_TIME_BASE_Q,
-                                               avf_time_base_q);
-            pkt->stream_index  = 0;
-            pkt->flags        |= AV_PKT_FLAG_KEY;
+            CMItemCount count;
+            CMSampleTimingInfo timing_info;
 
-            CVPixelBufferLockBaseAddress(image_buffer, 0);
+            if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
+                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
+                pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
+            }
 
-            void* data = CVPixelBufferGetBaseAddress(image_buffer);
-            memcpy(pkt->data, data, pkt->size);
+            pkt->stream_index  = ctx->video_stream_index;
+            pkt->flags        |= AV_PKT_FLAG_KEY;
 
-            CVPixelBufferUnlockBaseAddress(image_buffer, 0);
+            status = copy_cvpixelbuffer(s, image_buffer, pkt);
             CFRelease(ctx->current_frame);
             ctx->current_frame = nil;
+
+            if (status < 0)
+                return status;
+        } else if (ctx->current_audio_frame != nil) {
+            CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
+            int block_buffer_size         = CMBlockBufferGetDataLength(block_buffer);
+
+            if (!block_buffer || !block_buffer_size) {
+                return AVERROR(EIO);
+            }
+
+            if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
+                return AVERROR_BUFFER_TOO_SMALL;
+            }
+
+            if (av_new_packet(pkt, block_buffer_size) < 0) {
+                return AVERROR(EIO);
+            }
+
+            CMItemCount count;
+            CMSampleTimingInfo timing_info;
+
+            if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
+                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
+                pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
+            }
+
+            pkt->stream_index  = ctx->audio_stream_index;
+            pkt->flags        |= AV_PKT_FLAG_KEY;
+
+            if (ctx->audio_non_interleaved) {
+                int sample, c, shift, num_samples;
+
+                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
+                if (ret != kCMBlockBufferNoErr) {
+                    return AVERROR(EIO);
+                }
+
+                num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
+
+                // transform decoded frame into output format
+                #define INTERLEAVE_OUTPUT(bps)                                         \
+                {                                                                      \
+                    int##bps##_t **src;                                                \
+                    int##bps##_t *dest;                                                \
+                    src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*));      \
+                    if (!src) return AVERROR(EIO);                                     \
+                    for (c = 0; c < ctx->audio_channels; c++) {                        \
+                        src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
+                    }                                                                  \
+                    dest  = (int##bps##_t*)pkt->data;                                  \
+                    shift = bps - ctx->audio_bits_per_sample;                          \
+                    for (sample = 0; sample < num_samples; sample++)                   \
+                        for (c = 0; c < ctx->audio_channels; c++)                      \
+                            *dest++ = src[c][sample] << shift;                         \
+                    av_freep(&src);                                                    \
+                }
+
+                if (ctx->audio_bits_per_sample <= 16) {
+                    INTERLEAVE_OUTPUT(16)
+                } else {
+                    INTERLEAVE_OUTPUT(32)
+                }
+            } else {
+                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
+                if (ret != kCMBlockBufferNoErr) {
+                    return AVERROR(EIO);
+                }
+            }
+
+            CFRelease(ctx->current_audio_frame);
+            ctx->current_audio_frame = nil;
         } else {
             pkt->data = NULL;
             pthread_cond_wait(&ctx->frame_wait_cond, &ctx->frame_lock);
@@ -453,12 +1057,17 @@ static int avf_close(AVFormatContext *s)
 }
 
 static const AVOption options[] = {
-    { "frame_rate", "set frame rate", offsetof(AVFContext, frame_rate), AV_OPT_TYPE_FLOAT, { .dbl = 30.0 }, 0.1, 30.0, AV_OPT_TYPE_VIDEO_RATE, NULL },
     { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM, "list_devices" },
     { "true", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "list_devices" },
     { "false", "", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "list_devices" },
     { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
+    { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
     { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
+    { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
+    { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
+    { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
+    { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
+
     { NULL },
 };