/***************************************************************************** * avcapture.m: AVFoundation (Mac OS X) based video capture module ***************************************************************************** * Copyright © 2008-2013 VLC authors and VideoLAN * * Authors: Michael Feurstein * **************************************************************************** * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation; either version 2.1 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. *****************************************************************************/ /***************************************************************************** * Preamble *****************************************************************************/ #define OS_OBJECT_USE_OBJC 0 #ifdef HAVE_CONFIG_H # include "config.h" #endif #include #include #include #include #include #include #include #import #import #import #ifndef MAC_OS_X_VERSION_10_14 @interface AVCaptureDevice (AVCaptureDeviceAuthorizationSince10_14) + (void)requestAccessForMediaType:(AVMediaType)mediaType completionHandler:(void (^)(BOOL granted))handler API_AVAILABLE(macos(10.14), ios(7.0)); @end #endif /***************************************************************************** * Local prototypes *****************************************************************************/ static int Open(vlc_object_t *p_this); static void Close(vlc_object_t *p_this); static int Demux(demux_t *p_demux); static int Control(demux_t *, int, va_list); /***************************************************************************** * Module descriptor *****************************************************************************/ vlc_module_begin () set_shortname(N_("AVFoundation Video Capture")) set_description(N_("AVFoundation video capture module.")) set_category(CAT_INPUT) set_subcategory(SUBCAT_INPUT_ACCESS) add_shortcut("avcapture") set_capability("access_demux", 10) set_callbacks(Open, Close) vlc_module_end () static vlc_tick_t vlc_CMTime_to_mtime(CMTime timestamp) { CMTime scaled = CMTimeConvertScale( timestamp, CLOCK_FREQ, kCMTimeRoundingMethod_Default); return 1 + scaled.value; } /***************************************************************************** * AVFoundation Bridge *****************************************************************************/ @interface VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput { demux_t *p_avcapture; CVImageBufferRef currentImageBuffer; vlc_tick_t currentPts; vlc_tick_t previousPts; size_t bytesPerRow; long timeScale; BOOL videoDimensionsReady; } @property (readwrite) CMVideoDimensions videoDimensions; - (id)initWithDemux:(demux_t *)p_demux; - (int)width; - (int)height; - (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer; - (vlc_tick_t)currentPts; - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection; - (vlc_tick_t)copyCurrentFrameToBuffer:(void *)buffer; @end @implementation VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput - (id)initWithDemux:(demux_t *)p_demux { if (self = [super init]) { p_avcapture = p_demux; currentImageBuffer = nil; currentPts = 0; previousPts = 0; bytesPerRow = 0; timeScale = 0; videoDimensionsReady = NO; } return self; } - (void)dealloc { @synchronized (self) { CVBufferRelease(currentImageBuffer); currentImageBuffer = nil; bytesPerRow = 0; videoDimensionsReady = NO; } } - (long)timeScale { return timeScale; } - (int)width { return self.videoDimensions.width; } - (int)height { return self.videoDimensions.height; } - (size_t)bytesPerRow { return bytesPerRow; } - (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer { if (!videoDimensionsReady) { CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer); self.videoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription); bytesPerRow = CVPixelBufferGetBytesPerRow(CMSampleBufferGetImageBuffer(sampleBuffer)); videoDimensionsReady = YES; msg_Dbg(p_avcapture, "Dimensions obtained height:%i width:%i bytesPerRow:%lu", [self height], [self width], bytesPerRow); } } -(vlc_tick_t)currentPts { vlc_tick_t pts; @synchronized (self) { if ( !currentImageBuffer || currentPts == previousPts ) return 0; pts = previousPts = currentPts; } return pts; } - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { @autoreleasepool { CVImageBufferRef imageBufferToRelease; CMTime presentationtimestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); CVImageBufferRef videoFrame = CMSampleBufferGetImageBuffer(sampleBuffer); CVBufferRetain(videoFrame); [self getVideoDimensions:sampleBuffer]; @synchronized (self) { imageBufferToRelease = currentImageBuffer; currentImageBuffer = videoFrame; currentPts = vlc_CMTime_to_mtime(presentationtimestamp); timeScale = (long)presentationtimestamp.timescale; } CVBufferRelease(imageBufferToRelease); } } - (vlc_tick_t)copyCurrentFrameToBuffer:(void *)buffer { CVImageBufferRef imageBuffer; vlc_tick_t pts; void *pixels; if ( !currentImageBuffer || currentPts == previousPts ) return 0; @synchronized (self) { imageBuffer = CVBufferRetain(currentImageBuffer); if (imageBuffer) { pts = previousPts = currentPts; CVPixelBufferLockBaseAddress(imageBuffer, 0); pixels = CVPixelBufferGetBaseAddress(imageBuffer); if (pixels) { memcpy(buffer, pixels, CVPixelBufferGetHeight(imageBuffer) * CVPixelBufferGetBytesPerRow(imageBuffer)); } CVPixelBufferUnlockBaseAddress(imageBuffer, 0); } } CVBufferRelease(imageBuffer); if (pixels) return currentPts; else return 0; } @end /***************************************************************************** * Struct *****************************************************************************/ @interface VLCAVCaptureDemux : NSObject { demux_t *_demux; AVCaptureSession *_session; AVCaptureDevice *_device; VLCAVDecompressedVideoOutput *_output; es_out_id_t *_es_video; es_format_t _fmt; int _height, _width; } - (VLCAVCaptureDemux*)init:(demux_t *)demux; - (int)demux; - (vlc_tick_t)pts; - (void)dealloc; @end /***************************************************************************** * Open: *****************************************************************************/ static int Open(vlc_object_t *p_this) { demux_t *p_demux = (demux_t*)p_this; /* Only when selected */ if ( *p_demux->psz_access == '\0' ) return VLC_EGENERIC; if (p_demux->out == NULL) return VLC_EGENERIC; @autoreleasepool { VLCAVCaptureDemux *demux = [[VLCAVCaptureDemux alloc] init:p_demux]; if (demux == nil) return VLC_EGENERIC; p_demux->p_sys = (__bridge_retained void*)demux; } p_demux->pf_demux = Demux; p_demux->pf_control = Control; p_demux->info.i_update = 0; p_demux->info.i_title = 0; p_demux->info.i_seekpoint = 0; return VLC_SUCCESS; } /***************************************************************************** * Close: *****************************************************************************/ static void Close(vlc_object_t *p_this) { demux_t *p_demux = (demux_t*)p_this; VLCAVCaptureDemux *demux = (__bridge_transfer VLCAVCaptureDemux*)p_demux->p_sys; /* Signal ARC we won't use those references anymore. */ p_demux->p_sys = nil; demux = nil; } /***************************************************************************** * Demux: *****************************************************************************/ static int Demux(demux_t *p_demux) { VLCAVCaptureDemux *demux = (__bridge VLCAVCaptureDemux *)p_demux->p_sys; return [demux demux]; } /***************************************************************************** * Control: *****************************************************************************/ static int Control(demux_t *p_demux, int i_query, va_list args) { VLCAVCaptureDemux *demux = (__bridge VLCAVCaptureDemux *)p_demux->p_sys; bool *pb; switch( i_query ) { /* Special for access_demux */ case DEMUX_CAN_PAUSE: case DEMUX_CAN_SEEK: case DEMUX_SET_PAUSE_STATE: case DEMUX_CAN_CONTROL_PACE: pb = va_arg(args, bool *); *pb = false; return VLC_SUCCESS; case DEMUX_GET_PTS_DELAY: *va_arg(args, int64_t *) = INT64_C(1000) * var_InheritInteger(p_demux, "live-caching"); return VLC_SUCCESS; case DEMUX_GET_TIME: *va_arg(args, vlc_tick_t *) = [demux pts]; return VLC_SUCCESS; default: return VLC_EGENERIC; } return VLC_EGENERIC; } @implementation VLCAVCaptureDemux - (VLCAVCaptureDemux *)init:(demux_t *)p_demux { NSString *avf_currdevice_uid; NSArray *myVideoDevices; NSError *o_returnedError; AVCaptureDeviceInput *input = nil; int i, i_width, i_height, deviceCount, ivideo; char *psz_uid = NULL; _demux = p_demux; if (_demux->psz_location && *_demux->psz_location) psz_uid = strdup(_demux->psz_location); msg_Dbg(_demux, "avcapture uid = %s", psz_uid); avf_currdevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid]; myVideoDevices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo] arrayByAddingObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]]; if ( [myVideoDevices count] == 0 ) { vlc_dialog_display_error(_demux, _("No video devices found"), _("Your Mac does not seem to be equipped with a suitable video input device. " "Please check your connectors and drivers.")); msg_Err(_demux, "Can't find any suitable video device"); return nil; } deviceCount = [myVideoDevices count]; for ( ivideo = 0; ivideo < deviceCount; ivideo++ ) { AVCaptureDevice *avf_device; avf_device = [myVideoDevices objectAtIndex:ivideo]; msg_Dbg(_demux, "avcapture %i/%i %s %s", ivideo, deviceCount, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]); if ([[[avf_device uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:avf_currdevice_uid]) { break; } } if ( ivideo < [myVideoDevices count] ) { _device = [myVideoDevices objectAtIndex:ivideo]; } else { msg_Dbg(_demux, "Cannot find designated device as %s, falling back to default.", [avf_currdevice_uid UTF8String]); _device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; } if ( !_device ) { vlc_dialog_display_error(_demux, _("No video devices found"), _("Your Mac does not seem to be equipped with a suitable input device. " "Please check your connectors and drivers.")); msg_Err(_demux, "Can't find any suitable video device"); return nil; } AVCaptureDevice *device = _device; if ( [device isInUseByAnotherApplication] == YES ) { msg_Err(_demux, "default capture device is exclusively in use by another application"); return nil; } if (@available(macOS 10.14, *)) { msg_Dbg(_demux, "Check user consent for access to the video device"); dispatch_semaphore_t sema = dispatch_semaphore_create(0); __block bool accessGranted = NO; [AVCaptureDevice requestAccessForMediaType: AVMediaTypeVideo completionHandler:^(BOOL granted) { accessGranted = granted; dispatch_semaphore_signal(sema); } ]; dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER); dispatch_release(sema); if (!accessGranted) { msg_Err(_demux, "Can't use the video device as access has not been granted by the user"); vlc_dialog_display_error(_demux, _("Problem accessing a system resource"), _("Please open \"System Preferences\" -> \"Security & Privacy\" " "and allow VLC to access your camera.")); return nil; } } input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&o_returnedError]; if ( !input ) { msg_Err(_demux, "can't create a valid capture input facility: %s (%ld)",[[o_returnedError localizedDescription] UTF8String], [o_returnedError code]); return nil; } int chroma = VLC_CODEC_RGB32; memset(&_fmt, 0, sizeof(es_format_t)); es_format_Init(&_fmt, VIDEO_ES, chroma); _session = [[AVCaptureSession alloc] init]; [_session addInput:input]; _output = [[VLCAVDecompressedVideoOutput alloc] initWithDemux:_demux]; [_session addOutput:_output]; dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL); [_output setSampleBufferDelegate:_output queue:queue]; dispatch_release(queue); [_output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]]; [_session startRunning]; input = nil; msg_Dbg(_demux, "AVCapture: Video device ready!"); return self; } - (int)demux { block_t *p_block; @autoreleasepool { @synchronized(_output) { p_block = block_Alloc([_output width] * [_output bytesPerRow]); if ( !p_block ) { msg_Err(_demux, "cannot get block"); return 0; } p_block->i_pts = [_output copyCurrentFrameToBuffer: p_block->p_buffer]; if ( !p_block->i_pts ) { /* Nothing to display yet, just forget */ block_Release(p_block); msleep(10000); return 1; } else if ( !_es_video ) { _fmt.video.i_frame_rate = 1; _fmt.video.i_frame_rate_base = [_output timeScale]; msg_Dbg(_demux, "using frame rate base: %i", _fmt.video.i_frame_rate_base); _width = _fmt.video.i_width = _fmt.video.i_visible_width = [_output width]; _height = _fmt.video.i_height = _fmt.video.i_visible_height = [_output height]; _fmt.video.i_chroma = _fmt.i_codec; _es_video = es_out_Add(_demux->out, &_fmt); video_format_Print(&_demux->obj, "added new video es", &_fmt.video); } } es_out_SetPCR(_demux->out, p_block->i_pts); es_out_Send(_demux->out, _es_video, p_block); } return 1; } - (vlc_tick_t)pts { return [_output currentPts]; } - (void)dealloc { // Perform this on main thread, as the framework itself will sometimes try to synchronously // work on main thread. And this will create a dead lock. [_session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO]; } @end