1/*
2 * Copyright (C) 2012-2014 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27
28#if ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS))
29
30#include "InbandTextTrackPrivateAVF.h"
31
32#include "ISOVTTCue.h"
33#include "InbandTextTrackPrivateClient.h"
34#include "Logging.h"
35#include "SoftLinking.h"
36#include <CoreMedia/CoreMedia.h>
37#include <runtime/ArrayBuffer.h>
38#include <runtime/DataView.h>
39#include <runtime/Int8Array.h>
40#include <wtf/MediaTime.h>
41#include <wtf/NeverDestroyed.h>
42#include <wtf/PassOwnPtr.h>
43#include <wtf/text/CString.h>
44#include <wtf/text/StringBuilder.h>
45#include <wtf/text/WTFString.h>
46#include <wtf/unicode/CharacterNames.h>
47
48#if !PLATFORM(WIN)
49#include "MediaTimeMac.h"
50#endif
51
52#if !PLATFORM(WIN)
53#define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_FRAMEWORK_OPTIONAL(Lib)
54#define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_POINTER_OPTIONAL(Lib, Name, Type)
55#else
56#ifdef DEBUG_ALL
57#define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_DEBUG_LIBRARY(Lib)
58#else
59#define SOFT_LINK_AVF_FRAMEWORK(Lib) SOFT_LINK_LIBRARY(Lib)
60#endif
61
62#define SOFT_LINK_AVF_POINTER(Lib, Name, Type) SOFT_LINK_VARIABLE_DLL_IMPORT_OPTIONAL(Lib, Name, Type)
63#endif
64
65SOFT_LINK_AVF_FRAMEWORK(CoreMedia)
66
67#if !PLATFORM(WIN)
68SOFT_LINK(CoreMedia, CMSampleBufferGetDataBuffer, CMBlockBufferRef, (CMSampleBufferRef sbuf), (sbuf))
69SOFT_LINK(CoreMedia, CMBlockBufferCopyDataBytes, OSStatus, (CMBlockBufferRef theSourceBuffer, size_t offsetToData, size_t dataLength, void* destination), (theSourceBuffer, offsetToData, dataLength, destination))
70SOFT_LINK(CoreMedia, CMBlockBufferGetDataLength, size_t, (CMBlockBufferRef theBuffer), (theBuffer))
71SOFT_LINK(CoreMedia, CMSampleBufferGetSampleTimingInfo, OSStatus, (CMSampleBufferRef sbuf, CMItemIndex sampleIndex, CMSampleTimingInfo* timingInfoOut), (sbuf, sampleIndex, timingInfoOut))
72SOFT_LINK(CoreMedia, CMFormatDescriptionGetExtensions, CFDictionaryRef, (CMFormatDescriptionRef desc), (desc))
73SOFT_LINK(CoreMedia, CMSampleBufferGetFormatDescription, CMFormatDescriptionRef, (CMSampleBufferRef sbuf), (sbuf))
74#endif
75
76SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_Alignment, CFStringRef)
77SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Start, CFStringRef)
78SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_Middle, CFStringRef)
79SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAlignmentType_End, CFStringRef)
80SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BoldStyle, CFStringRef)
81SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ItalicStyle, CFStringRef)
82SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_UnderlineStyle, CFStringRef)
83SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, CFStringRef)
84SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_WritingDirectionSizePercentage, CFStringRef)
85SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, CFStringRef)
86SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_VerticalLayout, CFStringRef)
87SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_LeftToRight, CFStringRef)
88SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextVerticalLayout_RightToLeft, CFStringRef)
89SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, CFStringRef)
90SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_RelativeFontSize, CFStringRef)
91SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_FontFamilyName, CFStringRef)
92SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_ForegroundColorARGB, CFStringRef)
93SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_BackgroundColorARGB, CFStringRef)
94SOFT_LINK_AVF_POINTER(CoreMedia, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, CFStringRef)
95SOFT_LINK_AVF_POINTER(CoreMedia, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms, CFStringRef)
96
97#define kCMTextMarkupAttribute_Alignment getkCMTextMarkupAttribute_Alignment()
98#define kCMTextMarkupAlignmentType_Start getkCMTextMarkupAlignmentType_Start()
99#define kCMTextMarkupAlignmentType_Middle getkCMTextMarkupAlignmentType_Middle()
100#define kCMTextMarkupAlignmentType_End getkCMTextMarkupAlignmentType_End()
101#define kCMTextMarkupAttribute_BoldStyle getkCMTextMarkupAttribute_BoldStyle()
102#define kCMTextMarkupAttribute_ItalicStyle getkCMTextMarkupAttribute_ItalicStyle()
103#define kCMTextMarkupAttribute_UnderlineStyle getkCMTextMarkupAttribute_UnderlineStyle()
104#define kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection()
105#define kCMTextMarkupAttribute_WritingDirectionSizePercentage getkCMTextMarkupAttribute_WritingDirectionSizePercentage()
106#define kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection getkCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection()
107#define kCMTextMarkupAttribute_VerticalLayout getkCMTextMarkupAttribute_VerticalLayout()
108#define kCMTextVerticalLayout_LeftToRight getkCMTextVerticalLayout_LeftToRight()
109#define kCMTextVerticalLayout_RightToLeft getkCMTextVerticalLayout_RightToLeft()
110#define kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight getkCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight()
111#define kCMTextMarkupAttribute_RelativeFontSize getkCMTextMarkupAttribute_RelativeFontSize()
112#define kCMTextMarkupAttribute_FontFamilyName getkCMTextMarkupAttribute_FontFamilyName()
113#define kCMTextMarkupAttribute_ForegroundColorARGB getkCMTextMarkupAttribute_ForegroundColorARGB()
114#define kCMTextMarkupAttribute_BackgroundColorARGB getkCMTextMarkupAttribute_BackgroundColorARGB()
115#define kCMTextMarkupAttribute_CharacterBackgroundColorARGB getkCMTextMarkupAttribute_CharacterBackgroundColorARGB()
116#define kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms getkCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms()
117
118namespace JSC {
119class ArrayBuffer;
120}
121
122namespace WebCore {
123
124AVFInbandTrackParent::~AVFInbandTrackParent()
125{
126}
127
128InbandTextTrackPrivateAVF::InbandTextTrackPrivateAVF(AVFInbandTrackParent* owner, CueFormat format)
129    : InbandTextTrackPrivate(format)
130    , m_owner(owner)
131    , m_pendingCueStatus(None)
132    , m_index(0)
133    , m_hasBeenReported(false)
134    , m_seeking(false)
135    , m_haveReportedVTTHeader(false)
136{
137}
138
139InbandTextTrackPrivateAVF::~InbandTextTrackPrivateAVF()
140{
141    disconnect();
142}
143
144static bool makeRGBA32FromARGBCFArray(CFArrayRef colorArray, RGBA32& color)
145{
146    if (CFArrayGetCount(colorArray) < 4)
147        return false;
148
149    float componentArray[4];
150    for (int i = 0; i < 4; i++) {
151        CFNumberRef value = static_cast<CFNumberRef>(CFArrayGetValueAtIndex(colorArray, i));
152        if (CFGetTypeID(value) != CFNumberGetTypeID())
153            return false;
154
155        float component;
156        CFNumberGetValue(value, kCFNumberFloatType, &component);
157        componentArray[i] = component;
158    }
159
160    color = makeRGBA32FromFloats(componentArray[1], componentArray[2], componentArray[3], componentArray[0]);
161    return true;
162}
163
164void InbandTextTrackPrivateAVF::processCueAttributes(CFAttributedStringRef attributedString, GenericCueData& cueData)
165{
166    // Some of the attributes we translate into per-cue WebVTT settings are are repeated on each part of an attributed string so only
167    // process the first instance of each.
168    enum AttributeFlags {
169        Line = 1 << 0,
170        Position = 1 << 1,
171        Size = 1 << 2,
172        Vertical = 1 << 3,
173        Align = 1 << 4,
174        FontName = 1 << 5
175    };
176    unsigned processed = 0;
177
178    StringBuilder content;
179    String attributedStringValue = CFAttributedStringGetString(attributedString);
180    CFIndex length = attributedStringValue.length();
181    if (!length)
182        return;
183
184    CFRange effectiveRange = CFRangeMake(0, 0);
185    while ((effectiveRange.location + effectiveRange.length) < length) {
186
187        CFDictionaryRef attributes = CFAttributedStringGetAttributes(attributedString, effectiveRange.location + effectiveRange.length, &effectiveRange);
188        if (!attributes)
189            continue;
190
191        StringBuilder tagStart;
192        CFStringRef valueString;
193        String tagEnd;
194        CFIndex attributeCount = CFDictionaryGetCount(attributes);
195        Vector<const void*> keys(attributeCount);
196        Vector<const void*> values(attributeCount);
197        CFDictionaryGetKeysAndValues(attributes, keys.data(), values.data());
198
199        for (CFIndex i = 0; i < attributeCount; ++i) {
200            CFStringRef key = static_cast<CFStringRef>(keys[i]);
201            CFTypeRef value = values[i];
202            if (CFGetTypeID(key) != CFStringGetTypeID() || !CFStringGetLength(key))
203                continue;
204
205            if (CFStringCompare(key, kCMTextMarkupAttribute_Alignment, 0) == kCFCompareEqualTo) {
206                valueString = static_cast<CFStringRef>(value);
207                if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
208                    continue;
209                if (processed & Align)
210                    continue;
211                processed |= Align;
212
213                if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Start, 0) == kCFCompareEqualTo)
214                    cueData.setAlign(GenericCueData::Start);
215                else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_Middle, 0) == kCFCompareEqualTo)
216                    cueData.setAlign(GenericCueData::Middle);
217                else if (CFStringCompare(valueString, kCMTextMarkupAlignmentType_End, 0) == kCFCompareEqualTo)
218                    cueData.setAlign(GenericCueData::End);
219                else
220                    ASSERT_NOT_REACHED();
221
222                continue;
223            }
224
225            if (CFStringCompare(key, kCMTextMarkupAttribute_BoldStyle, 0) == kCFCompareEqualTo) {
226                if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
227                    continue;
228
229                tagStart.append("<b>");
230                tagEnd = "</b>" + tagEnd;
231                continue;
232            }
233
234            if (CFStringCompare(key, kCMTextMarkupAttribute_ItalicStyle, 0) == kCFCompareEqualTo) {
235                if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
236                    continue;
237
238                tagStart.append("<i>");
239                tagEnd = "</i>" + tagEnd;
240                continue;
241            }
242
243            if (CFStringCompare(key, kCMTextMarkupAttribute_UnderlineStyle, 0) == kCFCompareEqualTo) {
244                if (static_cast<CFBooleanRef>(value) != kCFBooleanTrue)
245                    continue;
246
247                tagStart.append("<u>");
248                tagEnd = "</u>" + tagEnd;
249                continue;
250            }
251
252            if (CFStringCompare(key, kCMTextMarkupAttribute_OrthogonalLinePositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) {
253                if (CFGetTypeID(value) != CFNumberGetTypeID())
254                    continue;
255                if (processed & Line)
256                    continue;
257                processed |= Line;
258
259                CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
260                double line;
261                CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &line);
262                cueData.setLine(line);
263                continue;
264            }
265
266            if (CFStringCompare(key, kCMTextMarkupAttribute_TextPositionPercentageRelativeToWritingDirection, 0) == kCFCompareEqualTo) {
267                if (CFGetTypeID(value) != CFNumberGetTypeID())
268                    continue;
269                if (processed & Position)
270                    continue;
271                processed |= Position;
272
273                CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
274                double position;
275                CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &position);
276                cueData.setPosition(position);
277                continue;
278            }
279
280            if (CFStringCompare(key, kCMTextMarkupAttribute_WritingDirectionSizePercentage, 0) == kCFCompareEqualTo) {
281                if (CFGetTypeID(value) != CFNumberGetTypeID())
282                    continue;
283                if (processed & Size)
284                    continue;
285                processed |= Size;
286
287                CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
288                double size;
289                CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &size);
290                cueData.setSize(size);
291                continue;
292            }
293
294            if (CFStringCompare(key, kCMTextMarkupAttribute_VerticalLayout, 0) == kCFCompareEqualTo) {
295                valueString = static_cast<CFStringRef>(value);
296                if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
297                    continue;
298
299                if (CFStringCompare(valueString, kCMTextVerticalLayout_LeftToRight, 0) == kCFCompareEqualTo)
300                    tagStart.append(leftToRightMark);
301                else if (CFStringCompare(valueString, kCMTextVerticalLayout_RightToLeft, 0) == kCFCompareEqualTo)
302                    tagStart.append(rightToLeftMark);
303                continue;
304            }
305
306            if (CFStringCompare(key, kCMTextMarkupAttribute_BaseFontSizePercentageRelativeToVideoHeight, 0) == kCFCompareEqualTo) {
307                if (CFGetTypeID(value) != CFNumberGetTypeID())
308                    continue;
309
310                CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
311                double baseFontSize;
312                CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &baseFontSize);
313                cueData.setBaseFontSize(baseFontSize);
314                continue;
315            }
316
317            if (CFStringCompare(key, kCMTextMarkupAttribute_RelativeFontSize, 0) == kCFCompareEqualTo) {
318                if (CFGetTypeID(value) != CFNumberGetTypeID())
319                    continue;
320
321                CFNumberRef valueNumber = static_cast<CFNumberRef>(value);
322                double relativeFontSize;
323                CFNumberGetValue(valueNumber, kCFNumberFloat64Type, &relativeFontSize);
324                cueData.setRelativeFontSize(relativeFontSize);
325                continue;
326            }
327
328            if (CFStringCompare(key, kCMTextMarkupAttribute_FontFamilyName, 0) == kCFCompareEqualTo) {
329                valueString = static_cast<CFStringRef>(value);
330                if (CFGetTypeID(valueString) != CFStringGetTypeID() || !CFStringGetLength(valueString))
331                    continue;
332                if (processed & FontName)
333                    continue;
334                processed |= FontName;
335
336                cueData.setFontName(valueString);
337                continue;
338            }
339
340            if (CFStringCompare(key, kCMTextMarkupAttribute_ForegroundColorARGB, 0) == kCFCompareEqualTo) {
341                CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
342                if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
343                    continue;
344
345                RGBA32 color;
346                if (!makeRGBA32FromARGBCFArray(arrayValue, color))
347                    continue;
348                cueData.setForegroundColor(color);
349            }
350
351            if (CFStringCompare(key, kCMTextMarkupAttribute_BackgroundColorARGB, 0) == kCFCompareEqualTo) {
352                CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
353                if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
354                    continue;
355
356                RGBA32 color;
357                if (!makeRGBA32FromARGBCFArray(arrayValue, color))
358                    continue;
359                cueData.setBackgroundColor(color);
360            }
361
362            if (CFStringCompare(key, kCMTextMarkupAttribute_CharacterBackgroundColorARGB, 0) == kCFCompareEqualTo) {
363                CFArrayRef arrayValue = static_cast<CFArrayRef>(value);
364                if (CFGetTypeID(arrayValue) != CFArrayGetTypeID())
365                    continue;
366
367                RGBA32 color;
368                if (!makeRGBA32FromARGBCFArray(arrayValue, color))
369                    continue;
370                cueData.setHighlightColor(color);
371            }
372        }
373
374        content.append(tagStart);
375        content.append(attributedStringValue.substring(effectiveRange.location, effectiveRange.length));
376        content.append(tagEnd);
377    }
378
379    if (content.length())
380        cueData.setContent(content.toString());
381}
382
383void InbandTextTrackPrivateAVF::processCue(CFArrayRef attributedStrings, CFArrayRef nativeSamples, double time)
384{
385    if (!client())
386        return;
387
388    processAttributedStrings(attributedStrings, time);
389    processNativeSamples(nativeSamples, time);
390}
391
392void InbandTextTrackPrivateAVF::processAttributedStrings(CFArrayRef attributedStrings, double time)
393{
394    LOG(Media, "InbandTextTrackPrivateAVF::processAttributedStrings - %li attributed strings at time %.2f\n", attributedStrings ? CFArrayGetCount(attributedStrings) : 0, time);
395
396    Vector<RefPtr<GenericCueData>> arrivingCues;
397    if (attributedStrings) {
398        CFIndex count = CFArrayGetCount(attributedStrings);
399        for (CFIndex i = 0; i < count; i++) {
400            CFAttributedStringRef attributedString = static_cast<CFAttributedStringRef>(CFArrayGetValueAtIndex(attributedStrings, i));
401
402            if (!attributedString || !CFAttributedStringGetLength(attributedString))
403                continue;
404
405            RefPtr<GenericCueData> cueData = GenericCueData::create();
406            processCueAttributes(attributedString, *cueData.get());
407            if (!cueData->content().length())
408                continue;
409
410            arrivingCues.append(cueData);
411
412            cueData->setStartTime(time);
413            cueData->setEndTime(std::numeric_limits<double>::infinity());
414
415            // AVFoundation cue "position" is to the center of the text so adjust relative to the edge because we will use it to
416            // set CSS "left".
417            if (cueData->position() >= 0 && cueData->size() > 0)
418                cueData->setPosition(cueData->position() - cueData->size() / 2);
419
420            LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - considering cue (\"%s\") for time = %.2f, position =  %.2f, line =  %.2f", this, cueData->content().utf8().data(), cueData->startTime(), cueData->position(), cueData->line());
421
422            cueData->setStatus(GenericCueData::Partial);
423        }
424    }
425
426    if (m_pendingCueStatus != None) {
427        // Cues do not have an explicit duration, they are displayed until the next "cue" (which might be empty) is emitted.
428        m_currentCueEndTime = time;
429
430        if (m_currentCueEndTime >= m_currentCueStartTime) {
431            for (auto& cueData : m_cues) {
432                // See if one of the newly-arrived cues is an extension of this cue.
433                Vector<RefPtr<GenericCueData>> nonExtensionCues;
434                for (auto& arrivingCue : arrivingCues) {
435                    if (!arrivingCue->doesExtendCueData(*cueData))
436                        nonExtensionCues.append(arrivingCue);
437                    else
438                        LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - found an extension cue (\"%s\") for time = %.2f, end = %.2f, position =  %.2f, line =  %.2f", this, arrivingCue->content().utf8().data(), arrivingCue->startTime(), arrivingCue->endTime(), arrivingCue->position(), arrivingCue->line());
439                }
440
441                bool currentCueIsExtended = (arrivingCues.size() != nonExtensionCues.size());
442
443                arrivingCues = nonExtensionCues;
444
445                if (currentCueIsExtended)
446                    continue;
447
448                if (m_pendingCueStatus == Valid) {
449                    cueData->setEndTime(m_currentCueEndTime);
450                    cueData->setStatus(GenericCueData::Complete);
451
452                    LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - updating cue \"%s\": start=%.2f, end=%.2f", this, cueData->content().utf8().data(), cueData->startTime(), m_currentCueEndTime);
453                    client()->updateGenericCue(this, cueData.get());
454                } else {
455                    // We have to assume that the implicit duration is invalid for cues delivered during a seek because the AVF decode pipeline may not
456                    // see every cue, so DO NOT update cue duration while seeking.
457                    LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - ignoring cue delivered during seek: start=%.2f, end=%.2f, content=\"%s\"", this, cueData->startTime(), m_currentCueEndTime, cueData->content().utf8().data());
458                }
459            }
460        } else
461            LOG(Media, "InbandTextTrackPrivateAVF::processCue negative length cue(s) ignored: start=%.2f, end=%.2f\n", m_currentCueStartTime, m_currentCueEndTime);
462
463        removeCompletedCues();
464    }
465
466    if (arrivingCues.isEmpty())
467        return;
468
469    m_currentCueStartTime = time;
470
471    for (auto& cueData : arrivingCues) {
472
473        m_cues.append(cueData);
474
475        LOG(Media, "InbandTextTrackPrivateAVF::processCue(%p) - adding cue \"%s\" for time = %.2f, end = %.2f, position =  %.2f, line =  %.2f", this, cueData->content().utf8().data(), cueData->startTime(), cueData->endTime(), cueData->position(), cueData->line());
476
477        client()->addGenericCue(this, cueData.release());
478    }
479
480    m_pendingCueStatus = seeking() ? DeliveredDuringSeek : Valid;
481}
482
483void InbandTextTrackPrivateAVF::beginSeeking()
484{
485    // Forget any partially accumulated cue data as the seek could be to a time outside of the cue's
486    // range, which will mean that the next cue delivered will result in the current cue getting the
487    // incorrect duration.
488    resetCueValues();
489    m_seeking = true;
490}
491
492void InbandTextTrackPrivateAVF::disconnect()
493{
494    m_owner = 0;
495    m_index = 0;
496}
497
498void InbandTextTrackPrivateAVF::removeCompletedCues()
499{
500    if (client()) {
501        long currentCue = m_cues.size() - 1;
502        for (; currentCue >= 0; --currentCue) {
503            if (m_cues[currentCue]->status() != GenericCueData::Complete)
504                continue;
505
506            LOG(Media, "InbandTextTrackPrivateAVF::removeCompletedCues(%p) - removing cue \"%s\": start=%.2f, end=%.2f", this, m_cues[currentCue]->content().utf8().data(), m_cues[currentCue]->startTime(), m_cues[currentCue]->endTime());
507
508            m_cues.remove(currentCue);
509        }
510    }
511
512    if (m_cues.isEmpty())
513        m_pendingCueStatus = None;
514
515    m_currentCueStartTime = 0;
516    m_currentCueEndTime = 0;
517}
518
519void InbandTextTrackPrivateAVF::resetCueValues()
520{
521    if (m_currentCueEndTime && m_cues.size())
522        LOG(Media, "InbandTextTrackPrivateAVF::resetCueValues flushing data for cues: start=%.2f\n", m_currentCueStartTime);
523
524    if (client()) {
525        for (size_t i = 0; i < m_cues.size(); i++)
526            client()->removeGenericCue(this, m_cues[i].get());
527    }
528
529    m_cues.resize(0);
530    m_pendingCueStatus = None;
531    m_currentCueStartTime = 0;
532    m_currentCueEndTime = 0;
533}
534
535void InbandTextTrackPrivateAVF::setMode(InbandTextTrackPrivate::Mode newMode)
536{
537    if (!m_owner)
538        return;
539
540    InbandTextTrackPrivate::Mode oldMode = mode();
541    InbandTextTrackPrivate::setMode(newMode);
542
543    if (oldMode == newMode)
544        return;
545
546    m_owner->trackModeChanged();
547}
548
549void InbandTextTrackPrivateAVF::processNativeSamples(CFArrayRef nativeSamples, double presentationTime)
550{
551#if PLATFORM(WIN)
552    UNUSED_PARAM(nativeSamples);
553    UNUSED_PARAM(presentationTime);
554    ASSERT_NOT_REACHED();
555#else
556    if (!nativeSamples)
557        return;
558
559    CFIndex count = CFArrayGetCount(nativeSamples);
560    if (!count)
561        return;
562
563    LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples - %li sample buffers at time %.2f\n", count, presentationTime);
564
565    for (CFIndex i = 0; i < count; i++) {
566
567        CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)CFArrayGetValueAtIndex(nativeSamples, i);
568        if (!sampleBuffer)
569            continue;
570
571        CMSampleTimingInfo timingInfo;
572        OSStatus status = CMSampleBufferGetSampleTimingInfo(sampleBuffer, i, &timingInfo);
573        if (status) {
574            LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - CMSampleBufferGetSampleTimingInfo returned error %x for sample %li", this, static_cast<int>(status), i);
575            continue;
576        }
577
578        CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
579        size_t bufferLength = CMBlockBufferGetDataLength(blockBuffer);
580        if (bufferLength < ISOBox::boxHeaderSize()) {
581            LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: CMSampleBuffer size length unexpectedly small (%zu)!!", this, bufferLength);
582            continue;
583        }
584
585        m_sampleInputBuffer.resize(m_sampleInputBuffer.size() + bufferLength);
586        CMBlockBufferCopyDataBytes(blockBuffer, 0, bufferLength, m_sampleInputBuffer.data() + m_sampleInputBuffer.size() - bufferLength);
587
588        RefPtr<ArrayBuffer> buffer = ArrayBuffer::create(m_sampleInputBuffer.data(), m_sampleInputBuffer.size());
589
590        String type = ISOBox::peekType(buffer.get());
591        size_t boxLength = ISOBox::peekLength(buffer.get());
592        if (boxLength > buffer->byteLength()) {
593            LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - ERROR: chunk '%s' size (%zu) larger than buffer length (%u)!!", this, type.utf8().data(), boxLength, buffer->byteLength());
594            continue;
595        }
596
597        LOG(Media, "InbandTextTrackPrivateAVF::processNativeSamples(%p) - chunk type = '%s', size = %zu", this, type.utf8().data(), boxLength);
598
599        if (type == ISOWebVTTCue::boxType()) {
600#if !PLATFORM(WIN)
601            ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), toMediaTime(timingInfo.duration), buffer.get());
602#else
603            ISOWebVTTCue cueData = ISOWebVTTCue(MediaTime::createWithDouble(presentationTime), MediaTime::createWithDouble(CMTimeGetSeconds(timingInfo.duration)), buffer.get());
604#endif
605            LOG(Media, "    sample presentation time = %.2f, duration = %.2f", cueData.presentationTime().toDouble(), cueData.duration().toDouble());
606            LOG(Media, "    id = \"%s\", settings = \"%s\", cue text = \"%s\"", cueData.id().utf8().data(), cueData.settings().utf8().data(), cueData.cueText().utf8().data());
607            LOG(Media, "    sourceID = \"%s\", originalStartTime = \"%s\"", cueData.sourceID().utf8().data(), cueData.originalStartTime().utf8().data());
608
609            client()->parseWebVTTCueData(this, cueData);
610        }
611
612        do {
613            if (m_haveReportedVTTHeader)
614                break;
615
616            CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
617            if (!formatDescription)
618                break;
619
620            CFDictionaryRef extensions = CMFormatDescriptionGetExtensions(formatDescription);
621            if (!extensions)
622                break;
623
624            CFDictionaryRef sampleDescriptionExtensions = static_cast<CFDictionaryRef>(CFDictionaryGetValue(extensions, kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms));
625            if (!sampleDescriptionExtensions)
626                break;
627
628            CFDataRef webvttHeaderData = static_cast<CFDataRef>(CFDictionaryGetValue(sampleDescriptionExtensions, CFSTR("vttC")));
629            if (!webvttHeaderData)
630                break;
631
632            unsigned length = CFDataGetLength(webvttHeaderData);
633            if (!length)
634                break;
635
636            // A WebVTT header is terminated by "One or more WebVTT line terminators" so append two line feeds to make sure the parser
637            // reccognized this string as a full header.
638            StringBuilder header;
639            header.append(reinterpret_cast<const unsigned char*>(CFDataGetBytePtr(webvttHeaderData)), length);
640            header.append("\n\n");
641
642            LOG(Media, "    vtt header = \n%s", header.toString().utf8().data());
643            client()->parseWebVTTFileHeader(this, header.toString());
644            m_haveReportedVTTHeader = true;
645        } while (0);
646
647        m_sampleInputBuffer.remove(0, boxLength);
648    }
649#endif
650}
651
652} // namespace WebCore
653
654#endif // ENABLE(VIDEO) && (USE(AVFOUNDATION) || PLATFORM(IOS))
655