1/* 2 * Copyright (C) 2013 Apple Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY 14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR 17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26#import "config.h" 27 28#if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION) 29 30#import "AVCaptureDeviceManager.h" 31 32#import "AVAudioCaptureSource.h" 33#import "AVMediaCaptureSource.h" 34#import "AVVideoCaptureSource.h" 35#import "Logging.h" 36#import "MediaConstraints.h" 37#import "MediaStreamSource.h" 38#import "MediaStreamSourceStates.h" 39#import "SoftLinking.h" 40#import "UUID.h" 41#import <AVFoundation/AVFoundation.h> 42#import <objc/runtime.h> 43#import <wtf/MainThread.h> 44#import <wtf/NeverDestroyed.h> 45 46typedef AVCaptureDevice AVCaptureDeviceType; 47typedef AVCaptureSession AVCaptureSessionType; 48 49SOFT_LINK_FRAMEWORK_OPTIONAL(AVFoundation) 50 51SOFT_LINK_CLASS(AVFoundation, AVCaptureDevice) 52SOFT_LINK_CLASS(AVFoundation, AVCaptureSession) 53 54#define AVCaptureDevice getAVCaptureDeviceClass() 55#define AVCaptureSession getAVCaptureSessionClass() 56 57SOFT_LINK_POINTER(AVFoundation, AVMediaTypeAudio, NSString *) 58SOFT_LINK_POINTER(AVFoundation, AVMediaTypeMuxed, NSString *) 59SOFT_LINK_POINTER(AVFoundation, AVMediaTypeVideo, NSString *) 60SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPreset1280x720, NSString *) 61SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPreset640x480, NSString *) 62SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPreset352x288, NSString *) 63SOFT_LINK_POINTER(AVFoundation, AVCaptureSessionPresetLow, NSString *) 64SOFT_LINK_POINTER(AVFoundation, AVCaptureDeviceWasConnectedNotification, NSString *) 65SOFT_LINK_POINTER(AVFoundation, AVCaptureDeviceWasDisconnectedNotification, NSString *) 66 67#define AVMediaTypeAudio getAVMediaTypeAudio() 68#define AVMediaTypeMuxed getAVMediaTypeMuxed() 69#define AVMediaTypeVideo getAVMediaTypeVideo() 70#define AVCaptureSessionPreset1280x720 getAVCaptureSessionPreset1280x720() 71#define AVCaptureSessionPreset640x480 getAVCaptureSessionPreset640x480() 72#define AVCaptureSessionPreset352x288 getAVCaptureSessionPreset352x288() 73#define AVCaptureSessionPresetLow getAVCaptureSessionPresetLow() 74#define AVCaptureDeviceWasConnectedNotification getAVCaptureDeviceWasConnectedNotification() 75#define AVCaptureDeviceWasDisconnectedNotification getAVCaptureDeviceWasDisconnectedNotification() 76 77using namespace WebCore; 78 79@interface WebCoreAVCaptureDeviceManagerObserver : NSObject 80{ 81 AVCaptureDeviceManager* m_callback; 82} 83 84-(id)initWithCallback:(AVCaptureDeviceManager*)callback; 85-(void)disconnect; 86-(void)deviceDisconnected:(NSNotification *)notification; 87-(void)deviceConnected:(NSNotification *)notification; 88@end 89 90namespace WebCore { 91 92static void refreshCaptureDeviceList(); 93 94class CaptureDevice { 95public: 96 CaptureDevice() 97 :m_enabled(false) 98 { 99 } 100 101 String m_captureDeviceID; 102 103 String m_audioSourceId; 104 RefPtr<AVMediaCaptureSource> m_audioSource; 105 106 String m_videoSourceId; 107 RefPtr<AVMediaCaptureSource> m_videoSource; 108 109 bool m_enabled; 110}; 111 112static Vector<CaptureDevice>& captureDeviceList() 113{ 114 DEPRECATED_DEFINE_STATIC_LOCAL(Vector<CaptureDevice>, captureDeviceList, ()); 115 static bool firstTime = true; 116 117 if (firstTime && !captureDeviceList.size()) { 118 firstTime = false; 119 refreshCaptureDeviceList(); 120 AVCaptureDeviceManager::shared().registerForDeviceNotifications(); 121 } 122 123 return captureDeviceList; 124} 125 126static bool captureDeviceFromDeviceID(const String& captureDeviceID, CaptureDevice& source) 127{ 128 Vector<CaptureDevice>& devices = captureDeviceList(); 129 130 size_t count = devices.size(); 131 for (size_t i = 0; i < count; ++i) { 132 if (devices[i].m_captureDeviceID == captureDeviceID) { 133 source = devices[i]; 134 return true; 135 } 136 } 137 138 return false; 139} 140 141static void refreshCaptureDeviceList() 142{ 143 Vector<CaptureDevice>& devices = captureDeviceList(); 144 145 for (AVCaptureDeviceType *device in [AVCaptureDevice devices]) { 146 CaptureDevice source; 147 148 if (!captureDeviceFromDeviceID(device.uniqueID, source)) { 149 // An AVCaptureDevice has a unique ID, but we can't use it for the source ID because: 150 // 1. if it provides both audio and video we will need to create two sources for it 151 // 2. the unique ID persists on one system across device connections, disconnections, 152 // application restarts, and reboots, so it could be used to figerprint a user. 153 source.m_captureDeviceID = device.uniqueID; 154 source.m_enabled = true; 155 if ([device hasMediaType:AVMediaTypeAudio] || [device hasMediaType:AVMediaTypeMuxed]) 156 source.m_audioSourceId = createCanonicalUUIDString(); 157 158 if ([device hasMediaType:AVMediaTypeVideo] || [device hasMediaType:AVMediaTypeMuxed]) 159 source.m_videoSourceId = createCanonicalUUIDString(); 160 161 devices.append(source); 162 } else if (!source.m_enabled) { 163 source.m_enabled = true; 164 if (source.m_audioSource) 165 source.m_audioSource->setEnabled(true); 166 if (source.m_videoSource) 167 source.m_videoSource->setEnabled(true); 168 } 169 } 170} 171 172bool AVCaptureDeviceManager::isAvailable() 173{ 174 return AVFoundationLibrary(); 175} 176 177AVCaptureDeviceManager& AVCaptureDeviceManager::shared() 178{ 179 DEPRECATED_DEFINE_STATIC_LOCAL(AVCaptureDeviceManager, manager, ()); 180 return manager; 181} 182 183AVCaptureDeviceManager::AVCaptureDeviceManager() 184 : m_objcObserver(adoptNS([[WebCoreAVCaptureDeviceManagerObserver alloc] initWithCallback:this])) 185{ 186} 187 188AVCaptureDeviceManager::~AVCaptureDeviceManager() 189{ 190 [[NSNotificationCenter defaultCenter] removeObserver:m_objcObserver.get()]; 191 [m_objcObserver disconnect]; 192} 193 194String AVCaptureDeviceManager::bestSessionPresetForVideoSize(AVCaptureSessionType *captureSession, int width, int height) 195{ 196 ASSERT(width >= 0); 197 ASSERT(height >= 0); 198 199 if (width > 1280 || height > 720) 200 // FIXME: this restriction could be adjusted with the videoMaxScaleAndCropFactor property. 201 return emptyString(); 202 203 if (width > 640 || height > 480) { 204 if (![captureSession canSetSessionPreset:AVCaptureSessionPreset1280x720]) 205 emptyString(); 206 return AVCaptureSessionPreset1280x720; 207 } 208 209 if (width > 352 || height > 288) { 210 if (![captureSession canSetSessionPreset:AVCaptureSessionPreset640x480]) 211 emptyString(); 212 return AVCaptureSessionPreset640x480; 213 } 214 215 if ([captureSession canSetSessionPreset:AVCaptureSessionPreset352x288]) 216 return AVCaptureSessionPreset352x288; 217 218 if ([captureSession canSetSessionPreset:AVCaptureSessionPresetLow]) 219 return AVCaptureSessionPresetLow; 220 221 return emptyString(); 222} 223 224bool AVCaptureDeviceManager::deviceSupportsFacingMode(AVCaptureDeviceType *device, MediaStreamSourceStates::VideoFacingMode facingMode) 225{ 226 if (![device hasMediaType:AVMediaTypeVideo]) 227 return false; 228 229 switch (facingMode) { 230 case MediaStreamSourceStates::User: 231 if ([device position] == AVCaptureDevicePositionFront) 232 return true; 233 break; 234 case MediaStreamSourceStates::Environment: 235 if ([device position] == AVCaptureDevicePositionBack) 236 return true; 237 break; 238 case MediaStreamSourceStates::Left: 239 case MediaStreamSourceStates::Right: 240 case MediaStreamSourceStates::Unknown: 241 return false; 242 } 243 244 return false; 245} 246 247CaptureDevice* AVCaptureDeviceManager::bestDeviceForFacingMode(MediaStreamSourceStates::VideoFacingMode facingMode) 248{ 249 Vector<CaptureDevice>& devices = captureDeviceList(); 250 251 size_t count = devices.size(); 252 for (size_t i = 0; i < count; ++i) { 253 AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID]; 254 ASSERT(device); 255 256 if (device && deviceSupportsFacingMode(device, facingMode)) 257 return &devices[i]; 258 } 259 260 return 0; 261} 262 263bool AVCaptureDeviceManager::sessionSupportsConstraint(AVCaptureSessionType *session, MediaStreamSource::Type type, const String& name, const String& value) 264{ 265 size_t constraint = validConstraintNames().find(name); 266 if (constraint == notFound) 267 return false; 268 269 switch (constraint) { 270 case Width: 271 if (type == MediaStreamSource::Audio) 272 return false; 273 274 return !bestSessionPresetForVideoSize(session, value.toInt(), 0).isEmpty(); 275 case Height: 276 if (type == MediaStreamSource::Audio) 277 return false; 278 279 return !bestSessionPresetForVideoSize(session, 0, value.toInt()).isEmpty(); 280 case FrameRate: { 281 if (type == MediaStreamSource::Audio) 282 return false; 283 284 // It would make sense to use [AVCaptureConnection videoMinFrameDuration] and 285 // [AVCaptureConnection videoMaxFrameDuration], but they only work with a "live" AVCaptureConnection. 286 float rate = value.toFloat(); 287 return rate > 0 && rate <= 60; 288 } 289 case Gain: { 290 if (type != MediaStreamSource::Audio) 291 return false; 292 293 float level = value.toFloat(); 294 return level > 0 && level <= 1; 295 } 296 case FacingMode: { 297 if (type == MediaStreamSource::Audio) 298 return false; 299 300 size_t facingMode = validFacingModes().find(value); 301 if (facingMode != notFound) 302 return false; 303 return bestDeviceForFacingMode(static_cast<MediaStreamSourceStates::VideoFacingMode>(facingMode)); 304 } 305 } 306 307 return false; 308} 309 310bool AVCaptureDeviceManager::isValidConstraint(MediaStreamSource::Type type, const String& name) 311{ 312 size_t constraint = validConstraintNames().find(name); 313 if (constraint == notFound) 314 return false; 315 316 if (constraint == Gain) 317 return type == MediaStreamSource::Audio; 318 319 return true; 320} 321 322Vector<RefPtr<TrackSourceInfo>> AVCaptureDeviceManager::getSourcesInfo(const String& requestOrigin) 323{ 324 UNUSED_PARAM(requestOrigin); 325 Vector<RefPtr<TrackSourceInfo>> sourcesInfo; 326 327 if (!isAvailable()) 328 return sourcesInfo; 329 330 Vector<CaptureDevice>& devices = captureDeviceList(); 331 size_t count = devices.size(); 332 for (size_t i = 0; i < count; ++i) { 333 AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID]; 334 ASSERT(device); 335 336 if (!devices[i].m_enabled) 337 continue; 338 339 if (devices[i].m_videoSource) 340 sourcesInfo.append(TrackSourceInfo::create(devices[i].m_videoSourceId, TrackSourceInfo::Video, device.localizedName)); 341 if (devices[i].m_audioSource) 342 sourcesInfo.append(TrackSourceInfo::create(devices[i].m_audioSourceId, TrackSourceInfo::Audio, device.localizedName)); 343 } 344 345 LOG(Media, "AVCaptureDeviceManager::getSourcesInfo(%p), found %d active devices", this, sourcesInfo.size()); 346 347 return sourcesInfo; 348} 349 350bool AVCaptureDeviceManager::verifyConstraintsForMediaType(MediaStreamSource::Type type, MediaConstraints* constraints, String& invalidConstraint) 351{ 352 if (!isAvailable()) 353 return false; 354 355 if (!constraints) 356 return true; 357 358 Vector<MediaConstraint> mandatoryConstraints; 359 constraints->getMandatoryConstraints(mandatoryConstraints); 360 if (mandatoryConstraints.size()) { 361 RetainPtr<AVCaptureSessionType> session = adoptNS([[AVCaptureSession alloc] init]); 362 for (size_t i = 0; i < mandatoryConstraints.size(); ++i) { 363 const MediaConstraint& constraint = mandatoryConstraints[i]; 364 if (!sessionSupportsConstraint(session.get(), type, constraint.m_name, constraint.m_value)) { 365 invalidConstraint = constraint.m_name; 366 return false; 367 } 368 } 369 } 370 371 Vector<MediaConstraint> optionalConstraints; 372 constraints->getOptionalConstraints(optionalConstraints); 373 if (!optionalConstraints.size()) 374 return true; 375 376 for (size_t i = 0; i < optionalConstraints.size(); ++i) { 377 const MediaConstraint& constraint = optionalConstraints[i]; 378 if (!isValidConstraint(type, constraint.m_name)) { 379 invalidConstraint = constraint.m_name; 380 return false; 381 } 382 } 383 384 return true; 385} 386 387RefPtr<MediaStreamSource> AVCaptureDeviceManager::bestSourceForTypeAndConstraints(MediaStreamSource::Type type, PassRefPtr<MediaConstraints> constraints) 388{ 389 if (!isAvailable()) 390 return 0; 391 392 Vector<CaptureDevice>& devices = captureDeviceList(); 393 size_t count = devices.size(); 394 for (size_t i = 0; i < count; ++i) { 395 if (!devices[i].m_enabled) 396 continue; 397 398 // FIXME: consider the constraints when choosing among multiple devices. For now just select the first available 399 // device of the appropriate type. 400 if (type == MediaStreamSource::Audio && !devices[i].m_audioSourceId.isEmpty()) { 401 if (!devices[i].m_audioSource) { 402 AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID]; 403 ASSERT(device); 404 devices[i].m_audioSource = AVAudioCaptureSource::create(device, devices[i].m_audioSourceId, constraints); 405 } 406 devices[i].m_audioSource->setReadyState(MediaStreamSource::Live); 407 return devices[i].m_audioSource; 408 } 409 410 if (type == MediaStreamSource::Video && !devices[i].m_videoSourceId.isEmpty()) { 411 if (!devices[i].m_videoSource) { 412 AVCaptureDeviceType *device = [AVCaptureDevice deviceWithUniqueID:devices[i].m_captureDeviceID]; 413 ASSERT(device); 414 devices[i].m_videoSource = AVVideoCaptureSource::create(device, devices[i].m_videoSourceId, constraints); 415 } 416 devices[i].m_videoSource->setReadyState(MediaStreamSource::Live); 417 return devices[i].m_videoSource; 418 } 419 } 420 421 return 0; 422} 423 424void AVCaptureDeviceManager::registerForDeviceNotifications() 425{ 426 [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceConnected:) name:AVCaptureDeviceWasConnectedNotification object:nil]; 427 [[NSNotificationCenter defaultCenter] addObserver:m_objcObserver.get() selector:@selector(deviceDisconnected:) name:AVCaptureDeviceWasDisconnectedNotification object:nil]; 428} 429 430void AVCaptureDeviceManager::deviceConnected() 431{ 432 refreshCaptureDeviceList(); 433} 434 435void AVCaptureDeviceManager::deviceDisconnected(AVCaptureDeviceType* device) 436{ 437 Vector<CaptureDevice>& devices = captureDeviceList(); 438 439 size_t count = devices.size(); 440 if (!count) 441 return; 442 443 String deviceID = device.uniqueID; 444 for (size_t i = 0; i < count; ++i) { 445 if (devices[i].m_captureDeviceID == deviceID) { 446 LOG(Media, "AVCaptureDeviceManager::deviceDisconnected(%p), device %d disabled", this, i); 447 devices[i].m_enabled = false; 448 if (devices[i].m_audioSource) 449 devices[i].m_audioSource->setEnabled(false); 450 if (devices[i].m_videoSource) 451 devices[i].m_videoSource->setEnabled(false); 452 } 453 } 454} 455 456const Vector<AtomicString>& AVCaptureDeviceManager::validConstraintNames() 457{ 458 DEPRECATED_DEFINE_STATIC_LOCAL(Vector<AtomicString>, constraints, ()); 459 static NeverDestroyed<AtomicString> heightConstraint("height", AtomicString::ConstructFromLiteral); 460 static NeverDestroyed<AtomicString> widthConstraint("width", AtomicString::ConstructFromLiteral); 461 static NeverDestroyed<AtomicString> frameRateConstraint("frameRate", AtomicString::ConstructFromLiteral); 462 static NeverDestroyed<AtomicString> facingModeConstraint("facingMode", AtomicString::ConstructFromLiteral); 463 static NeverDestroyed<AtomicString> gainConstraint("gain", AtomicString::ConstructFromLiteral); 464 465 if (!constraints.size()) { 466 constraints.insert(Width, widthConstraint); 467 constraints.insert(Height, heightConstraint); 468 constraints.insert(FrameRate, frameRateConstraint); 469 constraints.insert(FacingMode, facingModeConstraint); 470 constraints.insert(Gain, gainConstraint); 471 } 472 473 return constraints; 474} 475 476const Vector<AtomicString>& AVCaptureDeviceManager::validFacingModes() 477{ 478 DEPRECATED_DEFINE_STATIC_LOCAL(Vector<AtomicString>, modes, ()); 479 480 if (!modes.size()) { 481 modes.insert(MediaStreamSourceStates::User, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::User)); 482 modes.insert(MediaStreamSourceStates::Environment, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::Environment)); 483 modes.insert(MediaStreamSourceStates::Left, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::Left)); 484 modes.insert(MediaStreamSourceStates::Right, MediaStreamSourceStates::facingMode(MediaStreamSourceStates::Right)); 485 } 486 487 return modes; 488} 489 490} // namespace WebCore 491 492@implementation WebCoreAVCaptureDeviceManagerObserver 493 494- (id)initWithCallback:(AVCaptureDeviceManager*)callback 495{ 496 self = [super init]; 497 if (!self) 498 return nil; 499 m_callback = callback; 500 return self; 501} 502 503- (void)disconnect 504{ 505 [NSObject cancelPreviousPerformRequestsWithTarget:self]; 506 m_callback = 0; 507} 508 509- (void)deviceDisconnected:(NSNotification *)notification 510{ 511 if (!m_callback) 512 return; 513 514 dispatch_async(dispatch_get_main_queue(), ^{ 515 if (m_callback) { 516 AVCaptureDeviceType *device = [notification object]; 517 m_callback->deviceDisconnected(device); 518 } 519 }); 520} 521 522- (void)deviceConnected:(NSNotification *)unusedNotification 523{ 524 UNUSED_PARAM(unusedNotification); 525 if (!m_callback) 526 return; 527 528 dispatch_async(dispatch_get_main_queue(), ^{ 529 if (m_callback) 530 m_callback->deviceConnected(); 531 }); 532} 533 534@end 535 536#endif // ENABLE(MEDIA_STREAM) 537