1/*
2 * Copyright (C) 2013 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#import "config.h"
27
28#if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
29
30#import "AVCaptureDeviceManager.h"
31
32#import "AVAudioCaptureSource.h"
33#import "AVMediaCaptureSource.h"
34#import "AVVideoCaptureSource.h"
35#import "Logging.h"
36#import "MediaConstraints.h"
37#import "MediaStreamSource.h"
38#import "MediaStreamSourceStates.h"
39#import "SoftLinking.h"
40#import "UUID.h"
41#import <AVFoundation/AVFoundation.h>
42#import <objc/runtime.h>
43#import <wtf/MainThread.h>
44#import <wtf/NeverDestroyed.h>
45
46typedef AVCaptureDevice AVCaptureDeviceType;
47typedef AVCaptureSession AVCaptureSessionType;
48
49SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation)
50
51SOFT_LINK_CLASS(AVFoundation, AVCaptureDevice)
52SOFT_LINK_CLASS(AVFoundation, AVCaptureSession)
53
54#define AVCaptureDevice getAVCaptureDeviceClass()
55#define AVCaptureSession getAVCaptureSessionClass()
56
57SOFT_LINK_POINTER(AVFoundation, AVMediaTypeAudio, NSString *)
58SOFT_LINK_POINTER(AVFoundation, AVMediaTypeMuxed, NSString *)
59SOFT_LINK_POINTER(AVFoundation, AVMediaTypeVideo, NSString *)
60SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPreset1280x720, NSString *)
61SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPreset640x480, NSString *)
62SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPreset352x288, NSString *)
63SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPresetLow, NSString *)
64SOFT_LINK_POINTER(AVFoundation, AVCaptureDeviceWasConnectedNotification, NSString *)
65SOFT_LINK_POINTER(AVFoundation, AVCaptureDeviceWasDisconnectedNotification, NSString *)
66
67#define AVMediaTypeAudio getAVMediaTypeAudio()
68#define AVMediaTypeMuxed getAVMediaTypeMuxed()
69#define AVMediaTypeVideo getAVMediaTypeVideo()
70#define AVCaptureSessionPreset1280x720 getAVCaptureSessionPreset1280x720()
71#define AVCaptureSessionPreset640x480 getAVCaptureSessionPreset640x480()
72#define AVCaptureSessionPreset352x288 getAVCaptureSessionPreset352x288()
73#define AVCaptureSessionPresetLow getAVCaptureSessionPresetLow()
74#define AVCaptureDeviceWasConnectedNotification getAVCaptureDeviceWasConnectedNotification()
75#define AVCaptureDeviceWasDisconnectedNotification getAVCaptureDeviceWasDisconnectedNotification()
76
77using namespace WebCore;
78
79@interface WebCoreAVCaptureDeviceManagerObserver : NSObject
80{
81    AVCaptureDeviceManager* m_callback;
82}
83
84-(id)initWithCallback:(AVCaptureDeviceManager*)callback;
85-(void)disconnect;
86-(void)deviceDisconnected:(NSNotification *)notification;
87-(void)deviceConnected:(NSNotification *)notification;
88@end
89
90namespace WebCore {
91
92static void refreshCaptureDeviceList();
93
94class CaptureDevice {
95public:
96    CaptureDevice()
97        :m_enabled(false)
98    {
99    }
100
101    String m_captureDeviceID;
102
103    String m_audioSourceId;
104    RefPtr<AVMediaCaptureSource> m_audioSource;
105
106    String m_videoSourceId;
107    RefPtr<AVMediaCaptureSource> m_videoSource;
108
109    bool m_enabled;
110};
111
112static Vector<CaptureDevice>& captureDeviceList()
113{
114    DEPRECATED_DEFINE_STATIC_LOCAL(Vector<CaptureDevice>, captureDeviceList, ());
115    static bool firstTime = true;
116
117    if (firstTime && !captureDeviceList.size()) {
118        firstTime = false;
119        refreshCaptureDeviceList();
120        AVCaptureDeviceManager::shared().registerForDeviceNotifications();
121    }
122
123    return captureDeviceList;
124}
125
126static bool captureDeviceFromDeviceID(const String& captureDeviceID, CaptureDevice& source)
127{
128    Vector<CaptureDevice>& devices = captureDeviceList();
129
130    size_t count = devices.size();
131    for (size_t i = 0; i < count; ++i) {
132        if (devices[i].m_captureDeviceID == captureDeviceID) {
133            source = devices[i];
134            return true;
135        }
136    }
137
138    return false;
139}
140
141static void refreshCaptureDeviceList()
142{
143    Vector<CaptureDevice>& devices = captureDeviceList();
144
145    for (AVCaptureDeviceType *device in [AVCaptureDevice devices]) {
146        CaptureDevice source;
147
148        if (!captureDeviceFromDeviceID(device.uniqueID, source)) {
149            // An AVCaptureDevice has a unique ID, but we can't use it for the source ID because:
150            // 1. if it provides both audio and video we will need to create two sources for it
151            // 2. the unique ID persists on one system across device connections, disconnections,
152            //    application restarts, and reboots, so it could be used to figerprint a user.
153            source.m_captureDeviceID = device.uniqueID;
154            source.m_enabled = true;
155            if ([device hasMediaType:AVMediaTypeAudio] || [device hasMediaType:AVMediaTypeMuxed])
156                source.m_audioSourceId = createCanonicalUUIDString();
157
158            if ([device hasMediaType:AVMediaTypeVideo] || [device hasMediaType:AVMediaTypeMuxed])
159                source.m_videoSourceId = createCanonicalUUIDString();
160
161            devices.append(source);
162        } else if (!source.m_enabled) {
163            source.m_enabled = true;
164            if (source.m_audioSource)
165                source.m_audioSource->setEnabled(true);
166            if (source.m_videoSource)
167                source.m_videoSource->setEnabled(true);
168        }
169    }
170}
171
172bool AVCaptureDeviceManager::isAvailable()
173{
174    return AVFoundationLibrary();
175}
176
177AVCaptureDeviceManager& AVCaptureDeviceManager::shared()
178{
179    DEPRECATED_DEFINE_STATIC_LOCAL(AVCaptureDeviceManager, manager, ());
180    return manager;
181}
182
183AVCaptureDeviceManager::AVCaptureDeviceManager()
184    : m_objcObserver(adoptNS([[WebCoreAVCaptureDeviceManagerObserver alloc] initWithCallback:this]))
185{
186}
187
188AVCaptureDeviceManager::~AVCaptureDeviceManager()
189{
190    [[NSNotificationCenter defaultCenter] removeObserver:m_objcObserver.get()];
191    [m_objcObserver disconnect];
192}
193
194String AVCaptureDeviceManager::bestSessionPresetForVideoSize(AVCaptureSessionType *captureSession, int width, int height)
195{
196    ASSERT(width >= 0);
197    ASSERT(height >= 0);
198
199    if (width > 1280 || height > 720)
200        // FIXME: this restriction could be adjusted with the videoMaxScaleAndCropFactor property.
201        return emptyString();
202
203    if (width > 640 || height > 480) {
204        if (![captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720])
205            emptyString();
206        return AVCaptureSessionPreset1280x720;
207    }
208
209    if (width > 352 || height > 288) {
210        if (![captureSession canSetSessionPreset:AVCaptureSessionPreset640x480])
211            emptyString();
212        return AVCaptureSessionPreset640x480;
213    }
214
215    if ([captureSession canSetSessionPreset:AVCaptureSessionPreset352x288])
216        return AVCaptureSessionPreset352x288;
217
218    if ([captureSession canSetSessionPreset:AVCaptureSessionPresetLow])
219        return AVCaptureSessionPresetLow;
220
221    return emptyString();
222}
223
224bool AVCaptureDeviceManager::deviceSupportsFacingMode(AVCaptureDeviceType *device, MediaStreamSourceStates::VideoFacingMode facingMode)
225{
226    if (![device hasMediaType:AVMediaTypeVideo])
227        return false;
228
229    switch (facingMode) {
230    case MediaStreamSourceStates::User:
231        if ([device position] == AVCaptureDevicePositionFront)
232            return true;
233        break;
234    case MediaStreamSourceStates::Environment:
235        if ([device position] == AVCaptureDevicePositionBack)
236            return true;
237        break;
238    case MediaStreamSourceStates::Left:
239    case MediaStreamSourceStates::Right:
240    case MediaStreamSourceStates::Unknown:
241        return false;
242    }
243
244    return false;
245}
246
247CaptureDevice* AVCaptureDeviceManager::bestDeviceForFacingMode(MediaStreamSourceStates::VideoFacingMode facingMode)
248{
249    Vector<CaptureDevice>& devices = captureDeviceList();
250
251    size_t count = devices.size();
252    for (size_t i = 0; i < count; ++i) {
253        AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID];
254        ASSERT(device);
255
256        if (device && deviceSupportsFacingMode(device, facingMode))
257            return &devices[i];
258    }
259
260    return 0;
261}
262
263bool AVCaptureDeviceManager::sessionSupportsConstraint(AVCaptureSessionType *session, MediaStreamSource::Type type, const String& name, const String& value)
264{
265    size_t constraint = validConstraintNames().find(name);
266    if (constraint == notFound)
267        return false;
268
269    switch (constraint) {
270    case Width:
271        if (type == MediaStreamSource::Audio)
272            return false;
273
274        return !bestSessionPresetForVideoSize(session, value.toInt(), 0).isEmpty();
275    case Height:
276        if (type == MediaStreamSource::Audio)
277            return false;
278
279        return !bestSessionPresetForVideoSize(session, 0, value.toInt()).isEmpty();
280    case FrameRate: {
281        if (type == MediaStreamSource::Audio)
282            return false;
283
284        // It would make sense to use [AVCaptureConnection videoMinFrameDuration] and
285        // [AVCaptureConnection videoMaxFrameDuration], but they only work with a "live" AVCaptureConnection.
286        float rate = value.toFloat();
287        return rate > 0 && rate <= 60;
288    }
289    case Gain: {
290        if (type != MediaStreamSource::Audio)
291            return false;
292
293        float level = value.toFloat();
294        return level > 0 && level <= 1;
295    }
296    case FacingMode: {
297        if (type == MediaStreamSource::Audio)
298            return false;
299
300        size_t facingMode =  validFacingModes().find(value);
301        if (facingMode != notFound)
302            return false;
303        return bestDeviceForFacingMode(static_cast<MediaStreamSourceStates::VideoFacingMode>(facingMode));
304    }
305    }
306
307    return false;
308}
309
310bool AVCaptureDeviceManager::isValidConstraint(MediaStreamSource::Type type, const String& name)
311{
312    size_t constraint = validConstraintNames().find(name);
313    if (constraint == notFound)
314        return false;
315
316    if (constraint == Gain)
317        return type == MediaStreamSource::Audio;
318
319    return true;
320}
321
322Vector<RefPtr<TrackSourceInfo>> AVCaptureDeviceManager::getSourcesInfo(const String& requestOrigin)
323{
324    UNUSED_PARAM(requestOrigin);
325    Vector<RefPtr<TrackSourceInfo>> sourcesInfo;
326
327    if (!isAvailable())
328        return sourcesInfo;
329
330    Vector<CaptureDevice>& devices = captureDeviceList();
331    size_t count = devices.size();
332    for (size_t i = 0; i < count; ++i) {
333        AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID];
334        ASSERT(device);
335
336        if (!devices[i].m_enabled)
337            continue;
338
339        if (devices[i].m_videoSource)
340            sourcesInfo.append(TrackSourceInfo::create(devices[i].m_videoSourceId, TrackSourceInfo::Video, device.localizedName));
341        if (devices[i].m_audioSource)
342            sourcesInfo.append(TrackSourceInfo::create(devices[i].m_audioSourceId, TrackSourceInfo::Audio, device.localizedName));
343    }
344
345    LOG(Media, "AVCaptureDeviceManager::getSourcesInfo(%p), found %d active devices", this, sourcesInfo.size());
346
347    return sourcesInfo;
348}
349
350bool AVCaptureDeviceManager::verifyConstraintsForMediaType(MediaStreamSource::Type type, MediaConstraints* constraints, String& invalidConstraint)
351{
352    if (!isAvailable())
353        return false;
354
355    if (!constraints)
356        return true;
357
358    Vector<MediaConstraint> mandatoryConstraints;
359    constraints->getMandatoryConstraints(mandatoryConstraints);
360    if (mandatoryConstraints.size()) {
361        RetainPtr<AVCaptureSessionType> session = adoptNS([[AVCaptureSession alloc] init]);
362        for (size_t i = 0; i < mandatoryConstraints.size(); ++i) {
363            const MediaConstraint& constraint = mandatoryConstraints[i];
364            if (!sessionSupportsConstraint(session.get(), type, constraint.m_name, constraint.m_value)) {
365                invalidConstraint = constraint.m_name;
366                return false;
367            }
368        }
369    }
370
371    Vector<MediaConstraint> optionalConstraints;
372    constraints->getOptionalConstraints(optionalConstraints);
373    if (!optionalConstraints.size())
374        return true;
375
376    for (size_t i = 0; i < optionalConstraints.size(); ++i) {
377        const MediaConstraint& constraint = optionalConstraints[i];
378        if (!isValidConstraint(type, constraint.m_name)) {
379            invalidConstraint = constraint.m_name;
380            return false;
381        }
382    }
383
384    return true;
385}
386
387RefPtr<MediaStreamSource> AVCaptureDeviceManager::bestSourceForTypeAndConstraints(MediaStreamSource::Type type, PassRefPtr<MediaConstraints> constraints)
388{
389    if (!isAvailable())
390        return 0;
391
392    Vector<CaptureDevice>& devices = captureDeviceList();
393    size_t count = devices.size();
394    for (size_t i = 0; i < count; ++i) {
395        if (!devices[i].m_enabled)
396            continue;
397
398        // FIXME: consider the constraints when choosing among multiple devices. For now just select the first available
399        // device of the appropriate type.
400        if (type == MediaStreamSource::Audio && !devices[i].m_audioSourceId.isEmpty()) {
401            if (!devices[i].m_audioSource) {
402                AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID];
403                ASSERT(device);
404                devices[i].m_audioSource = AVAudioCaptureSource::create(device, devices[i].m_audioSourceId, constraints);
405            }
406            devices[i].m_audioSource->setReadyState(MediaStreamSource::Live);
407            return devices[i].m_audioSource;
408        }
409
410        if (type == MediaStreamSource::Video && !devices[i].m_videoSourceId.isEmpty()) {
411            if (!devices[i].m_videoSource) {
412                AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID];
413                ASSERT(device);
414                devices[i].m_videoSource = AVVideoCaptureSource::create(device, devices[i].m_videoSourceId, constraints);
415            }
416            devices[i].m_videoSource->setReadyState(MediaStreamSource::Live);
417            return devices[i].m_videoSource;
418        }
419    }
420
421    return 0;
422}
423
424void AVCaptureDeviceManager::registerForDeviceNotifications()
425{
426    [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceConnected:) name:AVCaptureDeviceWasConnectedNotification object:nil];
427    [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceDisconnected:) name:AVCaptureDeviceWasDisconnectedNotification object:nil];
428}
429
430void AVCaptureDeviceManager::deviceConnected()
431{
432    refreshCaptureDeviceList();
433}
434
435void AVCaptureDeviceManager::deviceDisconnected(AVCaptureDeviceType* device)
436{
437    Vector<CaptureDevice>& devices = captureDeviceList();
438
439    size_t count = devices.size();
440    if (!count)
441        return;
442
443    String deviceID = device.uniqueID;
444    for (size_t i = 0; i < count; ++i) {
445        if (devices[i].m_captureDeviceID == deviceID) {
446            LOG(Media, "AVCaptureDeviceManager::deviceDisconnected(%p), device %d disabled", this, i);
447            devices[i].m_enabled = false;
448            if (devices[i].m_audioSource)
449                devices[i].m_audioSource->setEnabled(false);
450            if (devices[i].m_videoSource)
451                devices[i].m_videoSource->setEnabled(false);
452        }
453    }
454}
455
456const Vector<AtomicString>& AVCaptureDeviceManager::validConstraintNames()
457{
458    DEPRECATED_DEFINE_STATIC_LOCAL(Vector<AtomicString>, constraints, ());
459    static NeverDestroyed<AtomicString> heightConstraint("height", AtomicString::ConstructFromLiteral);
460    static NeverDestroyed<AtomicString> widthConstraint("width", AtomicString::ConstructFromLiteral);
461    static NeverDestroyed<AtomicString> frameRateConstraint("frameRate", AtomicString::ConstructFromLiteral);
462    static NeverDestroyed<AtomicString> facingModeConstraint("facingMode", AtomicString::ConstructFromLiteral);
463    static NeverDestroyed<AtomicString> gainConstraint("gain", AtomicString::ConstructFromLiteral);
464
465    if (!constraints.size()) {
466        constraints.insert(Width, widthConstraint);
467        constraints.insert(Height, heightConstraint);
468        constraints.insert(FrameRate, frameRateConstraint);
469        constraints.insert(FacingMode, facingModeConstraint);
470        constraints.insert(Gain, gainConstraint);
471    }
472
473    return constraints;
474}
475
476const Vector<AtomicString>& AVCaptureDeviceManager::validFacingModes()
477{
478    DEPRECATED_DEFINE_STATIC_LOCAL(Vector<AtomicString>, modes, ());
479
480    if (!modes.size()) {
481        modes.insert(MediaStreamSourceStates::User, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::User));
482        modes.insert(MediaStreamSourceStates::Environment, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::Environment));
483        modes.insert(MediaStreamSourceStates::Left, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::Left));
484        modes.insert(MediaStreamSourceStates::Right, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::Right));
485    }
486
487    return modes;
488}
489
490} // namespace WebCore
491
492@implementation WebCoreAVCaptureDeviceManagerObserver
493
494- (id)initWithCallback:(AVCaptureDeviceManager*)callback
495{
496    self = [super init];
497    if (!self)
498        return nil;
499    m_callback = callback;
500    return self;
501}
502
503- (void)disconnect
504{
505    [NSObject cancelPreviousPerformRequestsWithTarget:self];
506    m_callback = 0;
507}
508
509- (void)deviceDisconnected:(NSNotification *)notification
510{
511    if (!m_callback)
512        return;
513
514    dispatch_async(dispatch_get_main_queue(), ^{
515        if (m_callback) {
516            AVCaptureDeviceType *device = [notification object];
517            m_callback->deviceDisconnected(device);
518        }
519    });
520}
521
522- (void)deviceConnected:(NSNotification *)unusedNotification
523{
524    UNUSED_PARAM(unusedNotification);
525    if (!m_callback)
526        return;
527
528    dispatch_async(dispatch_get_main_queue(), ^{
529        if (m_callback)
530            m_callback->deviceConnected();
531    });
532}
533
534@end
535
536#endif // ENABLE(MEDIA_STREAM)
537