1/* 2 * Copyright (C) 2012-2014 Apple Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY 14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR 17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26#include "config.h" 27 28#if ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS)) 29 30#include "InbandTextTrackPrivateAVF.h" 31 32#include "ISOVTTCue.h" 33#include "InbandTextTrackPrivateClient.h" 34#include "Logging.h" 35#include "SoftLinking.h" 36#include <CoreMedia/CoreMedia.h> 37#include <runtime/ArrayBuffer.h> 38#include <runtime/DataView.h> 39#include <runtime/Int8Array.h> 40#include <wtf/MediaTime.h> 41#include <wtf/NeverDestroyed.h> 42#include <wtf/PassOwnPtr.h> 43#include <wtf/text/CString.h> 44#include <wtf/text/StringBuilder.h> 45#include <wtf/text/WTFString.h> 46#include <wtf/unicode/CharacterNames.h> 47 48#if !PLATFORM(WIN) 49#include "MediaTimeMac.h" 50#endif 51 52#if !PLATFORM(WIN) 53#define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_FRAMEWORK_OPTIONAL(Lib) 54#define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_POINTER_OPTIONAL(Lib, Name, Type) 55#else 56#ifdef DEBUG_ALL 57#define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib) 58#else 59#define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib) 60#endif 61 62#define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type) 63#endif 64 65SOFT_LINK_AVF_FRAMEWORK(CoreMedia) 66 67#if !PLATFORM(WIN) 68SOFT_LINK(CoreMedia, CMSampleBufferGetDataBuffer, CMBlockBufferRef, (CMSampleBufferRef sbuf), (sbuf)) 69SOFT_LINK(CoreMedia, CMBlockBufferCopyDataBytes, OSStatus, (CMBlockBufferRef theSourceBuffer, size_t offsetToData, size_t dataLength, void* destination), (theSourceBuffer, offsetToData, dataLength, destination)) 70SOFT_LINK(CoreMedia, CMBlockBufferGetDataLength, size_t, (CMBlockBufferRef theBuffer), (theBuffer)) 71SOFT_LINK(CoreMedia, CMSampleBufferGetSampleTimingInfo, OSStatus, (CMSampleBufferRef sbuf, CMItemIndex sampleIndex, CMSampleTimingInfo* timingInfoOut), (sbuf, sampleIndex, timingInfoOut)) 72SOFT_LINK(CoreMedia, CMFormatDescriptionGetExtensions, CFDictionaryRef, (CMFormatDescriptionRef desc), (desc)) 73SOFT_LINK(CoreMedia, CMSampleBufferGetFormatDescription, CMFormatDescriptionRef, (CMSampleBufferRef sbuf), (sbuf)) 74#endif 75 76SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_Alignment, CFStringRef) 77SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Start, CFStringRef) 78SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Middle, CFStringRef) 79SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_End, CFStringRef) 80SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BoldStyle, CFStringRef) 81SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ItalicStyle, CFStringRef) 82SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_UnderlineStyle, CFStringRef) 83SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, CFStringRef) 84SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_WritingDirectionSizePercentage, CFStringRef) 85SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, CFStringRef) 86SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_VerticalLayout, CFStringRef) 87SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_LeftToRight, CFStringRef) 88SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_RightToLeft, CFStringRef) 89SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, CFStringRef) 90SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_RelativeFontSize, CFStringRef) 91SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_FontFamilyName, CFStringRef) 92SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ForegroundColorARGB, CFStringRef) 93SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BackgroundColorARGB, CFStringRef) 94SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, CFStringRef) 95SOFT_LINK_AVF_POINTER(CoreMedia, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, CFStringRef) 96 97#define kCMTextMarkupAttribute_Alignment getkCMTextMarkupAttribute_Alignment() 98#define kCMTextMarkupAlignmentType_Start getkCMTextMarkupAlignmentType_Start() 99#define kCMTextMarkupAlignmentType_Middle getkCMTextMarkupAlignmentType_Middle() 100#define kCMTextMarkupAlignmentType_End getkCMTextMarkupAlignmentType_End() 101#define kCMTextMarkupAttribute_BoldStyle getkCMTextMarkupAttribute_BoldStyle() 102#define kCMTextMarkupAttribute_ItalicStyle getkCMTextMarkupAttribute_ItalicStyle() 103#define kCMTextMarkupAttribute_UnderlineStyle getkCMTextMarkupAttribute_UnderlineStyle() 104#define kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection() 105#define kCMTextMarkupAttribute_WritingDirectionSizePercentage getkCMTextMarkupAttribute_WritingDirectionSizePercentage() 106#define kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection() 107#define kCMTextMarkupAttribute_VerticalLayout getkCMTextMarkupAttribute_VerticalLayout() 108#define kCMTextVerticalLayout_LeftToRight getkCMTextVerticalLayout_LeftToRight() 109#define kCMTextVerticalLayout_RightToLeft getkCMTextVerticalLayout_RightToLeft() 110#define kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight getkCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight() 111#define kCMTextMarkupAttribute_RelativeFontSize getkCMTextMarkupAttribute_RelativeFontSize() 112#define kCMTextMarkupAttribute_FontFamilyName getkCMTextMarkupAttribute_FontFamilyName() 113#define kCMTextMarkupAttribute_ForegroundColorARGB getkCMTextMarkupAttribute_ForegroundColorARGB() 114#define kCMTextMarkupAttribute_BackgroundColorARGB getkCMTextMarkupAttribute_BackgroundColorARGB() 115#define kCMTextMarkupAttribute_CharacterBackgroundColorARGB getkCMTextMarkupAttribute_CharacterBackgroundColorARGB() 116#define kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms getkCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms() 117 118namespace JSC { 119class ArrayBuffer; 120} 121 122namespace WebCore { 123 124AVFInbandTrackParent::~AVFInbandTrackParent() 125{ 126} 127 128InbandTextTrackPrivateAVF::InbandTextTrackPrivateAVF(AVFInbandTrackParent* owner, CueFormat format) 129 : InbandTextTrackPrivate(format) 130 , m_owner(owner) 131 , m_pendingCueStatus(None) 132 , m_index(0) 133 , m_hasBeenReported(false) 134 , m_seeking(false) 135 , m_haveReportedVTTHeader(false) 136{ 137} 138 139InbandTextTrackPrivateAVF::~InbandTextTrackPrivateAVF() 140{ 141 disconnect(); 142} 143 144static bool makeRGBA32FromARGBCFArray(CFArrayRef colorArray, RGBA32& color) 145{ 146 if (CFArrayGetCount(colorArray) < 4) 147 return false; 148 149 float componentArray[4]; 150 for (int i = 0; i < 4; i++) { 151 CFNumberRef value = static_cast<CFNumberRef>(CFArrayGetValueAtIndex(colorArray, i)); 152 if (CFGetTypeID(value) != CFNumberGetTypeID()) 153 return false; 154 155 float component; 156 CFNumberGetValue(value, kCFNumberFloatType, &component); 157 componentArray[i] = component; 158 } 159 160 color = makeRGBA32FromFloats(componentArray[1], componentArray[2], componentArray[3], componentArray[0]); 161 return true; 162} 163 164void InbandTextTrackPrivateAVF::processCueAttributes(CFAttributedStringRef attributedString, GenericCueData& cueData) 165{ 166 // Some of the attributes we translate into per-cue WebVTT settings are are repeated on each part of an attributed string so only 167 // process the first instance of each. 168 enum AttributeFlags { 169 Line = 1 << 0, 170 Position = 1 << 1, 171 Size = 1 << 2, 172 Vertical = 1 << 3, 173 Align = 1 << 4, 174 FontName = 1 << 5 175 }; 176 unsigned processed = 0; 177 178 StringBuilder content; 179 String attributedStringValue = CFAttributedStringGetString(attributedString); 180 CFIndex length = attributedStringValue.length(); 181 if (!length) 182 return; 183 184 CFRange effectiveRange = CFRangeMake(0, 0); 185 while ((effectiveRange.location + effectiveRange.length) < length) { 186 187 CFDictionaryRef attributes = CFAttributedStringGetAttributes(attributedString, effectiveRange.location + effectiveRange.length, &effectiveRange); 188 if (!attributes) 189 continue; 190 191 StringBuilder tagStart; 192 CFStringRef valueString; 193 String tagEnd; 194 CFIndex attributeCount = CFDictionaryGetCount(attributes); 195 Vector<const void*> keys(attributeCount); 196 Vector<const void*> values(attributeCount); 197 CFDictionaryGetKeysAndValues(attributes, keys.data(), values.data()); 198 199 for (CFIndex i = 0; i < attributeCount; ++i) { 200 CFStringRef key = static_cast<CFStringRef>(keys[i]); 201 CFTypeRef value = values[i]; 202 if (CFGetTypeID(key) != CFStringGetTypeID() || !CFStringGetLength(key)) 203 continue; 204 205 if (CFStringCompare(key, kCMTextMarkupAttribute_Alignment, 0) == kCFCompareEqualTo) { 206 valueString = static_cast<CFStringRef>(value); 207 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString)) 208 continue; 209 if (processed & Align) 210 continue; 211 processed |= Align; 212 213 if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Start, 0) == kCFCompareEqualTo) 214 cueData.setAlign(GenericCueData::Start); 215 else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Middle, 0) == kCFCompareEqualTo) 216 cueData.setAlign(GenericCueData::Middle); 217 else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_End, 0) == kCFCompareEqualTo) 218 cueData.setAlign(GenericCueData::End); 219 else 220 ASSERT_NOT_REACHED(); 221 222 continue; 223 } 224 225 if (CFStringCompare(key, kCMTextMarkupAttribute_BoldStyle, 0) == kCFCompareEqualTo) { 226 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue) 227 continue; 228 229 tagStart.append("<b>"); 230 tagEnd = "</b>" + tagEnd; 231 continue; 232 } 233 234 if (CFStringCompare(key, kCMTextMarkupAttribute_ItalicStyle, 0) == kCFCompareEqualTo) { 235 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue) 236 continue; 237 238 tagStart.append("<i>"); 239 tagEnd = "</i>" + tagEnd; 240 continue; 241 } 242 243 if (CFStringCompare(key, kCMTextMarkupAttribute_UnderlineStyle, 0) == kCFCompareEqualTo) { 244 if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue) 245 continue; 246 247 tagStart.append("<u>"); 248 tagEnd = "</u>" + tagEnd; 249 continue; 250 } 251 252 if (CFStringCompare(key, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) { 253 if (CFGetTypeID(value) != CFNumberGetTypeID()) 254 continue; 255 if (processed & Line) 256 continue; 257 processed |= Line; 258 259 CFNumberRef valueNumber = static_cast<CFNumberRef>(value); 260 double line; 261 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &line); 262 cueData.setLine(line); 263 continue; 264 } 265 266 if (CFStringCompare(key, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) { 267 if (CFGetTypeID(value) != CFNumberGetTypeID()) 268 continue; 269 if (processed & Position) 270 continue; 271 processed |= Position; 272 273 CFNumberRef valueNumber = static_cast<CFNumberRef>(value); 274 double position; 275 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &position); 276 cueData.setPosition(position); 277 continue; 278 } 279 280 if (CFStringCompare(key, kCMTextMarkupAttribute_WritingDirectionSizePercentage, 0) == kCFCompareEqualTo) { 281 if (CFGetTypeID(value) != CFNumberGetTypeID()) 282 continue; 283 if (processed & Size) 284 continue; 285 processed |= Size; 286 287 CFNumberRef valueNumber = static_cast<CFNumberRef>(value); 288 double size; 289 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &size); 290 cueData.setSize(size); 291 continue; 292 } 293 294 if (CFStringCompare(key, kCMTextMarkupAttribute_VerticalLayout, 0) == kCFCompareEqualTo) { 295 valueString = static_cast<CFStringRef>(value); 296 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString)) 297 continue; 298 299 if (CFStringCompare(valueString, kCMTextVerticalLayout_LeftToRight, 0) == kCFCompareEqualTo) 300 tagStart.append(leftToRightMark); 301 else if (CFStringCompare(valueString, kCMTextVerticalLayout_RightToLeft, 0) == kCFCompareEqualTo) 302 tagStart.append(rightToLeftMark); 303 continue; 304 } 305 306 if (CFStringCompare(key, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, 0) == kCFCompareEqualTo) { 307 if (CFGetTypeID(value) != CFNumberGetTypeID()) 308 continue; 309 310 CFNumberRef valueNumber = static_cast<CFNumberRef>(value); 311 double baseFontSize; 312 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &baseFontSize); 313 cueData.setBaseFontSize(baseFontSize); 314 continue; 315 } 316 317 if (CFStringCompare(key, kCMTextMarkupAttribute_RelativeFontSize, 0) == kCFCompareEqualTo) { 318 if (CFGetTypeID(value) != CFNumberGetTypeID()) 319 continue; 320 321 CFNumberRef valueNumber = static_cast<CFNumberRef>(value); 322 double relativeFontSize; 323 CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &relativeFontSize); 324 cueData.setRelativeFontSize(relativeFontSize); 325 continue; 326 } 327 328 if (CFStringCompare(key, kCMTextMarkupAttribute_FontFamilyName, 0) == kCFCompareEqualTo) { 329 valueString = static_cast<CFStringRef>(value); 330 if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString)) 331 continue; 332 if (processed & FontName) 333 continue; 334 processed |= FontName; 335 336 cueData.setFontName(valueString); 337 continue; 338 } 339 340 if (CFStringCompare(key, kCMTextMarkupAttribute_ForegroundColorARGB, 0) == kCFCompareEqualTo) { 341 CFArrayRef arrayValue = static_cast<CFArrayRef>(value); 342 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID()) 343 continue; 344 345 RGBA32 color; 346 if (!makeRGBA32FromARGBCFArray(arrayValue, color)) 347 continue; 348 cueData.setForegroundColor(color); 349 } 350 351 if (CFStringCompare(key, kCMTextMarkupAttribute_BackgroundColorARGB, 0) == kCFCompareEqualTo) { 352 CFArrayRef arrayValue = static_cast<CFArrayRef>(value); 353 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID()) 354 continue; 355 356 RGBA32 color; 357 if (!makeRGBA32FromARGBCFArray(arrayValue, color)) 358 continue; 359 cueData.setBackgroundColor(color); 360 } 361 362 if (CFStringCompare(key, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, 0) == kCFCompareEqualTo) { 363 CFArrayRef arrayValue = static_cast<CFArrayRef>(value); 364 if (CFGetTypeID(arrayValue) != CFArrayGetTypeID()) 365 continue; 366 367 RGBA32 color; 368 if (!makeRGBA32FromARGBCFArray(arrayValue, color)) 369 continue; 370 cueData.setHighlightColor(color); 371 } 372 } 373 374 content.append(tagStart); 375 content.append(attributedStringValue.substring(effectiveRange.location, effectiveRange.length)); 376 content.append(tagEnd); 377 } 378 379 if (content.length()) 380 cueData.setContent(content.toString()); 381} 382 383void InbandTextTrackPrivateAVF::processCue(CFArrayRef attributedStrings, CFArrayRef nativeSamples, double time) 384{ 385 if (!client()) 386 return; 387 388 processAttributedStrings(attributedStrings, time); 389 processNativeSamples(nativeSamples, time); 390} 391 392void InbandTextTrackPrivateAVF::processAttributedStrings(CFArrayRef attributedStrings, double time) 393{ 394 LOG(Media, "InbandTextTrackPrivateAVF::processAttributedStrings - %li attributed strings at time %.2f\n", attributedStrings ? CFArrayGetCount(attributedStrings) : 0, time); 395 396 Vector<RefPtr<GenericCueData>> arrivingCues; 397 if (attributedStrings) { 398 CFIndex count = CFArrayGetCount(attributedStrings); 399 for (CFIndex i = 0; i < count; i++) { 400 CFAttributedStringRef attributedString = static_cast<CFAttributedStringRef>(CFArrayGetValueAtIndex(attributedStrings, i)); 401 402 if (!attributedString || !CFAttributedStringGetLength(attributedString)) 403 continue; 404 405 RefPtr<GenericCueData> cueData = GenericCueData::create(); 406 processCueAttributes(attributedString, *cueData.get()); 407 if (!cueData->content().length()) 408 continue; 409 410 arrivingCues.append(cueData); 411 412 cueData->setStartTime(time); 413 cueData->setEndTime(std::numeric_limits<double>::infinity()); 414 415 // AVFoundation cue "position" is to the center of the text so adjust relative to the edge because we will use it to 416 // set CSS "left". 417 if (cueData->position() >= 0 && cueData->size() > 0) 418 cueData->setPosition(cueData->position() - cueData->size() / 2); 419 420 LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - considering cue (\"%s\") for time = %.2f, position = %.2f, line = %.2f", this, cueData->content().utf8().data(), cueData->startTime(), cueData->position(), cueData->line()); 421 422 cueData->setStatus(GenericCueData::Partial); 423 } 424 } 425 426 if (m_pendingCueStatus != None) { 427 // Cues do not have an explicit duration, they are displayed until the next "cue" (which might be empty) is emitted. 428 m_currentCueEndTime = time; 429 430 if (m_currentCueEndTime >= m_currentCueStartTime) { 431 for (auto& cueData : m_cues) { 432 // See if one of the newly-arrived cues is an extension of this cue. 433 Vector<RefPtr<GenericCueData>> nonExtensionCues; 434 for (auto& arrivingCue : arrivingCues) { 435 if (!arrivingCue->doesExtendCueData(*cueData)) 436 nonExtensionCues.append(arrivingCue); 437 else 438 LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - found an extension cue (\"%s\") for time = %.2f, end = %.2f, position = %.2f, line = %.2f", this, arrivingCue->content().utf8().data(), arrivingCue->startTime(), arrivingCue->endTime(), arrivingCue->position(), arrivingCue->line()); 439 } 440 441 bool currentCueIsExtended = (arrivingCues.size() != nonExtensionCues.size()); 442 443 arrivingCues = nonExtensionCues; 444 445 if (currentCueIsExtended) 446 continue; 447 448 if (m_pendingCueStatus == Valid) { 449 cueData->setEndTime(m_currentCueEndTime); 450 cueData->setStatus(GenericCueData::Complete); 451 452 LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - updating cue \"%s\": start=%.2f, end=%.2f", this, cueData->content().utf8().data(), cueData->startTime(), m_currentCueEndTime); 453 client()->updateGenericCue(this, cueData.get()); 454 } else { 455 // We have to assume that the implicit duration is invalid for cues delivered during a seek because the AVF decode pipeline may not 456 // see every cue, so DO NOT update cue duration while seeking. 457 LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - ignoring cue delivered during seek: start=%.2f, end=%.2f, content=\"%s\"", this, cueData->startTime(), m_currentCueEndTime, cueData->content().utf8().data()); 458 } 459 } 460 } else 461 LOG(Media, "InbandTextTrackPrivateAVF::processCue negative length cue(s) ignored: start=%.2f, end=%.2f\n", m_currentCueStartTime, m_currentCueEndTime); 462 463 removeCompletedCues(); 464 } 465 466 if (arrivingCues.isEmpty()) 467 return; 468 469 m_currentCueStartTime = time; 470 471 for (auto& cueData : arrivingCues) { 472 473 m_cues.append(cueData); 474 475 LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - adding cue \"%s\" for time = %.2f, end = %.2f, position = %.2f, line = %.2f", this, cueData->content().utf8().data(), cueData->startTime(), cueData->endTime(), cueData->position(), cueData->line()); 476 477 client()->addGenericCue(this, cueData.release()); 478 } 479 480 m_pendingCueStatus = seeking() ? DeliveredDuringSeek : Valid; 481} 482 483void InbandTextTrackPrivateAVF::beginSeeking() 484{ 485 // Forget any partially accumulated cue data as the seek could be to a time outside of the cue's 486 // range, which will mean that the next cue delivered will result in the current cue getting the 487 // incorrect duration. 488 resetCueValues(); 489 m_seeking = true; 490} 491 492void InbandTextTrackPrivateAVF::disconnect() 493{ 494 m_owner = 0; 495 m_index = 0; 496} 497 498void InbandTextTrackPrivateAVF::removeCompletedCues() 499{ 500 if (client()) { 501 long currentCue = m_cues.size() - 1; 502 for (; currentCue >= 0; --currentCue) { 503 if (m_cues[currentCue]->status() != GenericCueData::Complete) 504 continue; 505 506 LOG(Media, "InbandTextTrackPrivateAVF::removeCompletedCues(%p) - removing cue \"%s\": start=%.2f, end=%.2f", this, m_cues[currentCue]->content().utf8().data(), m_cues[currentCue]->startTime(), m_cues[currentCue]->endTime()); 507 508 m_cues.remove(currentCue); 509 } 510 } 511 512 if (m_cues.isEmpty()) 513 m_pendingCueStatus = None; 514 515 m_currentCueStartTime = 0; 516 m_currentCueEndTime = 0; 517} 518 519void InbandTextTrackPrivateAVF::resetCueValues() 520{ 521 if (m_currentCueEndTime && m_cues.size()) 522 LOG(Media, "InbandTextTrackPrivateAVF::resetCueValues flushing data for cues: start=%.2f\n", m_currentCueStartTime); 523 524 if (client()) { 525 for (size_t i = 0; i < m_cues.size(); i++) 526 client()->removeGenericCue(this, m_cues[i].get()); 527 } 528 529 m_cues.resize(0); 530 m_pendingCueStatus = None; 531 m_currentCueStartTime = 0; 532 m_currentCueEndTime = 0; 533} 534 535void InbandTextTrackPrivateAVF::setMode(InbandTextTrackPrivate::Mode newMode) 536{ 537 if (!m_owner) 538 return; 539 540 InbandTextTrackPrivate::Mode oldMode = mode(); 541 InbandTextTrackPrivate::setMode(newMode); 542 543 if (oldMode == newMode) 544 return; 545 546 m_owner->trackModeChanged(); 547} 548 549void InbandTextTrackPrivateAVF::processNativeSamples(CFArrayRef nativeSamples, double presentationTime) 550{ 551#if PLATFORM(WIN) 552 UNUSED_PARAM(nativeSamples); 553 UNUSED_PARAM(presentationTime); 554 ASSERT_NOT_REACHED(); 555#else 556 if (!nativeSamples) 557 return; 558 559 CFIndex count = CFArrayGetCount(nativeSamples); 560 if (!count) 561 return; 562 563 LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples - %li sample buffers at time %.2f\n", count, presentationTime); 564 565 for (CFIndex i = 0; i < count; i++) { 566 567 CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)CFArrayGetValueAtIndex(nativeSamples, i); 568 if (!sampleBuffer) 569 continue; 570 571 CMSampleTimingInfo timingInfo; 572 OSStatus status = CMSampleBufferGetSampleTimingInfo(sampleBuffer, i, &timingInfo); 573 if (status) { 574 LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - CMSampleBufferGetSampleTimingInfo returned error %x for sample %li", this, static_cast<int>(status), i); 575 continue; 576 } 577 578 CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer); 579 size_t bufferLength = CMBlockBufferGetDataLength(blockBuffer); 580 if (bufferLength < ISOBox::boxHeaderSize()) { 581 LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: CMSampleBuffer size length unexpectedly small (%zu)!!", this, bufferLength); 582 continue; 583 } 584 585 m_sampleInputBuffer.resize(m_sampleInputBuffer.size() + bufferLength); 586 CMBlockBufferCopyDataBytes(blockBuffer, 0, bufferLength, m_sampleInputBuffer.data() + m_sampleInputBuffer.size() - bufferLength); 587 588 RefPtr<ArrayBuffer> buffer = ArrayBuffer::create(m_sampleInputBuffer.data(), m_sampleInputBuffer.size()); 589 590 String type = ISOBox::peekType(buffer.get()); 591 size_t boxLength = ISOBox::peekLength(buffer.get()); 592 if (boxLength > buffer->byteLength()) { 593 LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: chunk '%s' size (%zu) larger than buffer length (%u)!!", this, type.utf8().data(), boxLength, buffer->byteLength()); 594 continue; 595 } 596 597 LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - chunk type = '%s', size = %zu", this, type.utf8().data(), boxLength); 598 599 if (type == ISOWebVTTCue::boxType()) { 600#if !PLATFORM(WIN) 601 ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), toMediaTime(timingInfo.duration), buffer.get()); 602#else 603 ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), MediaTime::createWithDouble(CMTimeGetSeconds(timingInfo.duration)), buffer.get()); 604#endif 605 LOG(Media, " sample presentation time = %.2f, duration = %.2f", cueData.presentationTime().toDouble(), cueData.duration().toDouble()); 606 LOG(Media, " id = \"%s\", settings = \"%s\", cue text = \"%s\"", cueData.id().utf8().data(), cueData.settings().utf8().data(), cueData.cueText().utf8().data()); 607 LOG(Media, " sourceID = \"%s\", originalStartTime = \"%s\"", cueData.sourceID().utf8().data(), cueData.originalStartTime().utf8().data()); 608 609 client()->parseWebVTTCueData(this, cueData); 610 } 611 612 do { 613 if (m_haveReportedVTTHeader) 614 break; 615 616 CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer); 617 if (!formatDescription) 618 break; 619 620 CFDictionaryRef extensions = CMFormatDescriptionGetExtensions(formatDescription); 621 if (!extensions) 622 break; 623 624 CFDictionaryRef sampleDescriptionExtensions = static_cast<CFDictionaryRef>(CFDictionaryGetValue(extensions, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms)); 625 if (!sampleDescriptionExtensions) 626 break; 627 628 CFDataRef webvttHeaderData = static_cast<CFDataRef>(CFDictionaryGetValue(sampleDescriptionExtensions, CFSTR("vttC"))); 629 if (!webvttHeaderData) 630 break; 631 632 unsigned length = CFDataGetLength(webvttHeaderData); 633 if (!length) 634 break; 635 636 // A WebVTT header is terminated by "One or more WebVTT line terminators" so append two line feeds to make sure the parser 637 // reccognized this string as a full header. 638 StringBuilder header; 639 header.append(reinterpret_cast<const unsigned char*>(CFDataGetBytePtr(webvttHeaderData)), length); 640 header.append("\n\n"); 641 642 LOG(Media, " vtt header = \n%s", header.toString().utf8().data()); 643 client()->parseWebVTTFileHeader(this, header.toString()); 644 m_haveReportedVTTHeader = true; 645 } while (0); 646 647 m_sampleInputBuffer.remove(0, boxLength); 648 } 649#endif 650} 651 652} // namespace WebCore 653 654#endif // ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS)) 655