1/*
2 *  Copyright (C) 2011, 2012 Igalia S.L
3 *  Copyright (C) 2011 Zan Dobersek  <zandobersek@gmail.com>
4 *
5 *  This library is free software; you can redistribute it and/or
6 *  modify it under the terms of the GNU Lesser General Public
7 *  License as published by the Free Software Foundation; either
8 *  version 2 of the License, or (at your option) any later version.
9 *
10 *  This library is distributed in the hope that it will be useful,
11 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 *  Lesser General Public License for more details.
14 *
15 *  You should have received a copy of the GNU Lesser General Public
16 *  License along with this library; if not, write to the Free Software
17 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
18 */
19
20#include "config.h"
21
22#if ENABLE(WEB_AUDIO)
23
24#include "AudioFileReader.h"
25
26#include "AudioBus.h"
27#include "GStreamerVersioning.h"
28
29#if PLATFORM(QT)
30// Clear out offending Qt macro so the following header, gio.h, can be included.
31// https://bugs.webkit.org/show_bug.cgi?id=95081
32#undef signals
33#endif
34
35#include <gio/gio.h>
36#include <gst/app/gstappsink.h>
37#include <gst/gst.h>
38#include <gst/pbutils/pbutils.h>
39#include <wtf/Noncopyable.h>
40#include <wtf/PassOwnPtr.h>
41#include <wtf/gobject/GOwnPtr.h>
42#include <wtf/gobject/GRefPtr.h>
43
44#ifdef GST_API_VERSION_1
45#include <gst/audio/audio.h>
46#else
47#include <gst/audio/multichannel.h>
48#endif
49
50#ifdef GST_API_VERSION_1
51static const char* gDecodebinName = "decodebin";
52#else
53static const char* gDecodebinName = "decodebin2";
54#endif
55
56namespace WebCore {
57
58class AudioFileReader {
59    WTF_MAKE_NONCOPYABLE(AudioFileReader);
60public:
61    AudioFileReader(const char* filePath);
62    AudioFileReader(const void* data, size_t dataSize);
63    ~AudioFileReader();
64
65    PassRefPtr<AudioBus> createBus(float sampleRate, bool mixToMono);
66
67#ifdef GST_API_VERSION_1
68    GstFlowReturn handleSample(GstAppSink*);
69#else
70    GstFlowReturn handleBuffer(GstAppSink*);
71#endif
72    gboolean handleMessage(GstMessage*);
73    void handleNewDeinterleavePad(GstPad*);
74    void deinterleavePadsConfigured();
75    void plugDeinterleave(GstPad*);
76    void decodeAudioForBusCreation();
77
78private:
79    const void* m_data;
80    size_t m_dataSize;
81    const char* m_filePath;
82
83    float m_sampleRate;
84    GstBufferList* m_frontLeftBuffers;
85    GstBufferList* m_frontRightBuffers;
86
87#ifndef GST_API_VERSION_1
88    GstBufferListIterator* m_frontLeftBuffersIterator;
89    GstBufferListIterator* m_frontRightBuffersIterator;
90#endif
91
92    GstElement* m_pipeline;
93    unsigned m_channelSize;
94    GRefPtr<GstElement> m_decodebin;
95    GRefPtr<GstElement> m_deInterleave;
96    GRefPtr<GMainLoop> m_loop;
97    bool m_errorOccurred;
98};
99
100static void copyGstreamerBuffersToAudioChannel(GstBufferList* buffers, AudioChannel* audioChannel)
101{
102#ifdef GST_API_VERSION_1
103    float* destination = audioChannel->mutableData();
104    unsigned bufferCount = gst_buffer_list_length(buffers);
105    for (unsigned i = 0; i < bufferCount; ++i) {
106        GstBuffer* buffer = gst_buffer_list_get(buffers, i);
107        ASSERT(buffer);
108        gsize bufferSize = gst_buffer_get_size(buffer);
109        gst_buffer_extract(buffer, 0, destination, bufferSize);
110        destination += bufferSize / sizeof(float);
111    }
112#else
113    GstBufferListIterator* iter = gst_buffer_list_iterate(buffers);
114    gst_buffer_list_iterator_next_group(iter);
115    GstBuffer* buffer = gst_buffer_list_iterator_merge_group(iter);
116    if (buffer) {
117        memcpy(audioChannel->mutableData(), reinterpret_cast<float*>(GST_BUFFER_DATA(buffer)), GST_BUFFER_SIZE(buffer));
118        gst_buffer_unref(buffer);
119    }
120
121    gst_buffer_list_iterator_free(iter);
122#endif
123}
124
125static GstFlowReturn onAppsinkPullRequiredCallback(GstAppSink* sink, gpointer userData)
126{
127#ifdef GST_API_VERSION_1
128    return static_cast<AudioFileReader*>(userData)->handleSample(sink);
129#else
130    return static_cast<AudioFileReader*>(userData)->handleBuffer(sink);
131#endif
132}
133
134gboolean messageCallback(GstBus*, GstMessage* message, AudioFileReader* reader)
135{
136    return reader->handleMessage(message);
137}
138
139static void onGStreamerDeinterleavePadAddedCallback(GstElement*, GstPad* pad, AudioFileReader* reader)
140{
141    reader->handleNewDeinterleavePad(pad);
142}
143
144static void onGStreamerDeinterleaveReadyCallback(GstElement*, AudioFileReader* reader)
145{
146    reader->deinterleavePadsConfigured();
147}
148
149static void onGStreamerDecodebinPadAddedCallback(GstElement*, GstPad* pad, AudioFileReader* reader)
150{
151    reader->plugDeinterleave(pad);
152}
153
154gboolean enteredMainLoopCallback(gpointer userData)
155{
156    AudioFileReader* reader = reinterpret_cast<AudioFileReader*>(userData);
157    reader->decodeAudioForBusCreation();
158    return FALSE;
159}
160
161AudioFileReader::AudioFileReader(const char* filePath)
162    : m_data(0)
163    , m_dataSize(0)
164    , m_filePath(filePath)
165    , m_channelSize(0)
166    , m_errorOccurred(false)
167{
168}
169
170AudioFileReader::AudioFileReader(const void* data, size_t dataSize)
171    : m_data(data)
172    , m_dataSize(dataSize)
173    , m_filePath(0)
174    , m_channelSize(0)
175    , m_errorOccurred(false)
176{
177}
178
179AudioFileReader::~AudioFileReader()
180{
181    if (m_pipeline) {
182        GRefPtr<GstBus> bus = webkitGstPipelineGetBus(GST_PIPELINE(m_pipeline));
183        ASSERT(bus);
184        g_signal_handlers_disconnect_by_func(bus.get(), reinterpret_cast<gpointer>(messageCallback), this);
185        gst_bus_remove_signal_watch(bus.get());
186
187        gst_element_set_state(m_pipeline, GST_STATE_NULL);
188        gst_object_unref(GST_OBJECT(m_pipeline));
189    }
190
191    if (m_decodebin) {
192        g_signal_handlers_disconnect_by_func(m_decodebin.get(), reinterpret_cast<gpointer>(onGStreamerDecodebinPadAddedCallback), this);
193        m_decodebin.clear();
194    }
195
196    if (m_deInterleave) {
197        g_signal_handlers_disconnect_by_func(m_deInterleave.get(), reinterpret_cast<gpointer>(onGStreamerDeinterleavePadAddedCallback), this);
198        g_signal_handlers_disconnect_by_func(m_deInterleave.get(), reinterpret_cast<gpointer>(onGStreamerDeinterleaveReadyCallback), this);
199        m_deInterleave.clear();
200    }
201
202#ifndef GST_API_VERSION_1
203    gst_buffer_list_iterator_free(m_frontLeftBuffersIterator);
204    gst_buffer_list_iterator_free(m_frontRightBuffersIterator);
205#endif
206    gst_buffer_list_unref(m_frontLeftBuffers);
207    gst_buffer_list_unref(m_frontRightBuffers);
208}
209
210#ifdef GST_API_VERSION_1
211GstFlowReturn AudioFileReader::handleSample(GstAppSink* sink)
212{
213    GstSample* sample = gst_app_sink_pull_sample(sink);
214    if (!sample)
215        return GST_FLOW_ERROR;
216
217    GstBuffer* buffer = gst_sample_get_buffer(sample);
218    if (!buffer) {
219        gst_sample_unref(sample);
220        return GST_FLOW_ERROR;
221    }
222
223    GstCaps* caps = gst_sample_get_caps(sample);
224    if (!caps) {
225        gst_sample_unref(sample);
226        return GST_FLOW_ERROR;
227    }
228
229    GstAudioInfo info;
230    gst_audio_info_from_caps(&info, caps);
231    int frames = GST_CLOCK_TIME_TO_FRAMES(GST_BUFFER_DURATION(buffer), GST_AUDIO_INFO_RATE(&info));
232
233    // Check the first audio channel. The buffer is supposed to store
234    // data of a single channel anyway.
235    switch (GST_AUDIO_INFO_POSITION(&info, 0)) {
236    case GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT:
237        gst_buffer_list_add(m_frontLeftBuffers, gst_buffer_ref(buffer));
238        m_channelSize += frames;
239        break;
240    case GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT:
241        gst_buffer_list_add(m_frontRightBuffers, gst_buffer_ref(buffer));
242        break;
243    default:
244        break;
245    }
246
247    gst_sample_unref(sample);
248    return GST_FLOW_OK;
249
250}
251#endif
252
253#ifndef GST_API_VERSION_1
254GstFlowReturn AudioFileReader::handleBuffer(GstAppSink* sink)
255{
256    GstBuffer* buffer = gst_app_sink_pull_buffer(sink);
257    if (!buffer)
258        return GST_FLOW_ERROR;
259
260    GstCaps* caps = gst_buffer_get_caps(buffer);
261    GstStructure* structure = gst_caps_get_structure(caps, 0);
262
263    gint channels = 0;
264    if (!gst_structure_get_int(structure, "channels", &channels) || !channels) {
265        gst_caps_unref(caps);
266        gst_buffer_unref(buffer);
267        return GST_FLOW_ERROR;
268    }
269
270    gint sampleRate = 0;
271    if (!gst_structure_get_int(structure, "rate", &sampleRate) || !sampleRate) {
272        gst_caps_unref(caps);
273        gst_buffer_unref(buffer);
274        return GST_FLOW_ERROR;
275    }
276
277    gint width = 0;
278    if (!gst_structure_get_int(structure, "width", &width) || !width) {
279        gst_caps_unref(caps);
280        gst_buffer_unref(buffer);
281        return GST_FLOW_ERROR;
282    }
283
284    GstClockTime duration = (static_cast<guint64>(GST_BUFFER_SIZE(buffer)) * 8 * GST_SECOND) / (sampleRate * channels * width);
285    int frames = GST_CLOCK_TIME_TO_FRAMES(duration, sampleRate);
286
287    // Check the first audio channel. The buffer is supposed to store
288    // data of a single channel anyway.
289    GstAudioChannelPosition* positions = gst_audio_get_channel_positions(structure);
290    switch (positions[0]) {
291    case GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT:
292        gst_buffer_list_iterator_add(m_frontLeftBuffersIterator, buffer);
293        m_channelSize += frames;
294        break;
295    case GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT:
296        gst_buffer_list_iterator_add(m_frontRightBuffersIterator, buffer);
297        break;
298    default:
299        gst_buffer_unref(buffer);
300        break;
301    }
302
303    g_free(positions);
304    gst_caps_unref(caps);
305    return GST_FLOW_OK;
306}
307#endif
308
309gboolean AudioFileReader::handleMessage(GstMessage* message)
310{
311    GOwnPtr<GError> error;
312    GOwnPtr<gchar> debug;
313
314    switch (GST_MESSAGE_TYPE(message)) {
315    case GST_MESSAGE_EOS:
316        g_main_loop_quit(m_loop.get());
317        break;
318    case GST_MESSAGE_WARNING:
319        gst_message_parse_warning(message, &error.outPtr(), &debug.outPtr());
320        g_warning("Warning: %d, %s. Debug output: %s", error->code,  error->message, debug.get());
321        break;
322    case GST_MESSAGE_ERROR:
323        gst_message_parse_error(message, &error.outPtr(), &debug.outPtr());
324        g_warning("Error: %d, %s. Debug output: %s", error->code,  error->message, debug.get());
325        m_errorOccurred = true;
326        g_main_loop_quit(m_loop.get());
327        break;
328    default:
329        break;
330    }
331    return TRUE;
332}
333
334void AudioFileReader::handleNewDeinterleavePad(GstPad* pad)
335{
336    // A new pad for a planar channel was added in deinterleave. Plug
337    // in an appsink so we can pull the data from each
338    // channel. Pipeline looks like:
339    // ... deinterleave ! queue ! appsink.
340    GstElement* queue = gst_element_factory_make("queue", 0);
341    GstElement* sink = gst_element_factory_make("appsink", 0);
342
343    GstAppSinkCallbacks callbacks;
344    callbacks.eos = 0;
345    callbacks.new_preroll = 0;
346#ifdef GST_API_VERSION_1
347    callbacks.new_sample = onAppsinkPullRequiredCallback;
348#else
349    callbacks.new_buffer_list = 0;
350    callbacks.new_buffer = onAppsinkPullRequiredCallback;
351#endif
352    gst_app_sink_set_callbacks(GST_APP_SINK(sink), &callbacks, this, 0);
353
354    g_object_set(sink, "sync", FALSE, NULL);
355
356    gst_bin_add_many(GST_BIN(m_pipeline), queue, sink, NULL);
357
358    GstPad* sinkPad = gst_element_get_static_pad(queue, "sink");
359    gst_pad_link_full(pad, sinkPad, GST_PAD_LINK_CHECK_NOTHING);
360    gst_object_unref(GST_OBJECT(sinkPad));
361
362    gst_element_link_pads_full(queue, "src", sink, "sink", GST_PAD_LINK_CHECK_NOTHING);
363
364    gst_element_set_state(queue, GST_STATE_READY);
365    gst_element_set_state(sink, GST_STATE_READY);
366}
367
368void AudioFileReader::deinterleavePadsConfigured()
369{
370    // All deinterleave src pads are now available, let's roll to
371    // PLAYING so data flows towards the sinks and it can be retrieved.
372    gst_element_set_state(m_pipeline, GST_STATE_PLAYING);
373}
374
375void AudioFileReader::plugDeinterleave(GstPad* pad)
376{
377    // A decodebin pad was added, plug in a deinterleave element to
378    // separate each planar channel. Sub pipeline looks like
379    // ... decodebin2 ! audioconvert ! audioresample ! capsfilter ! deinterleave.
380    GstElement* audioConvert  = gst_element_factory_make("audioconvert", 0);
381    GstElement* audioResample = gst_element_factory_make("audioresample", 0);
382    GstElement* capsFilter = gst_element_factory_make("capsfilter", 0);
383    m_deInterleave = gst_element_factory_make("deinterleave", "deinterleave");
384
385    g_object_set(m_deInterleave.get(), "keep-positions", TRUE, NULL);
386    g_signal_connect(m_deInterleave.get(), "pad-added", G_CALLBACK(onGStreamerDeinterleavePadAddedCallback), this);
387    g_signal_connect(m_deInterleave.get(), "no-more-pads", G_CALLBACK(onGStreamerDeinterleaveReadyCallback), this);
388
389    GstCaps* caps = getGstAudioCaps(2, m_sampleRate);
390    g_object_set(capsFilter, "caps", caps, NULL);
391    gst_caps_unref(caps);
392
393    gst_bin_add_many(GST_BIN(m_pipeline), audioConvert, audioResample, capsFilter, m_deInterleave.get(), NULL);
394
395    GstPad* sinkPad = gst_element_get_static_pad(audioConvert, "sink");
396    gst_pad_link_full(pad, sinkPad, GST_PAD_LINK_CHECK_NOTHING);
397    gst_object_unref(GST_OBJECT(sinkPad));
398
399    gst_element_link_pads_full(audioConvert, "src", audioResample, "sink", GST_PAD_LINK_CHECK_NOTHING);
400    gst_element_link_pads_full(audioResample, "src", capsFilter, "sink", GST_PAD_LINK_CHECK_NOTHING);
401    gst_element_link_pads_full(capsFilter, "src", m_deInterleave.get(), "sink", GST_PAD_LINK_CHECK_NOTHING);
402
403    gst_element_sync_state_with_parent(audioConvert);
404    gst_element_sync_state_with_parent(audioResample);
405    gst_element_sync_state_with_parent(capsFilter);
406    gst_element_sync_state_with_parent(m_deInterleave.get());
407}
408
409void AudioFileReader::decodeAudioForBusCreation()
410{
411    // Build the pipeline (giostreamsrc | filesrc) ! decodebin2
412    // A deinterleave element is added once a src pad becomes available in decodebin.
413    m_pipeline = gst_pipeline_new(0);
414
415    GRefPtr<GstBus> bus = webkitGstPipelineGetBus(GST_PIPELINE(m_pipeline));
416    ASSERT(bus);
417    gst_bus_add_signal_watch(bus.get());
418    g_signal_connect(bus.get(), "message", G_CALLBACK(messageCallback), this);
419
420    GstElement* source;
421    if (m_data) {
422        ASSERT(m_dataSize);
423        source = gst_element_factory_make("giostreamsrc", 0);
424        GRefPtr<GInputStream> memoryStream = adoptGRef(g_memory_input_stream_new_from_data(m_data, m_dataSize, 0));
425        g_object_set(source, "stream", memoryStream.get(), NULL);
426    } else {
427        source = gst_element_factory_make("filesrc", 0);
428        g_object_set(source, "location", m_filePath, NULL);
429    }
430
431    m_decodebin = gst_element_factory_make(gDecodebinName, "decodebin");
432    g_signal_connect(m_decodebin.get(), "pad-added", G_CALLBACK(onGStreamerDecodebinPadAddedCallback), this);
433
434    gst_bin_add_many(GST_BIN(m_pipeline), source, m_decodebin.get(), NULL);
435    gst_element_link_pads_full(source, "src", m_decodebin.get(), "sink", GST_PAD_LINK_CHECK_NOTHING);
436    gst_element_set_state(m_pipeline, GST_STATE_PAUSED);
437}
438
439PassRefPtr<AudioBus> AudioFileReader::createBus(float sampleRate, bool mixToMono)
440{
441    m_sampleRate = sampleRate;
442
443    m_frontLeftBuffers = gst_buffer_list_new();
444    m_frontRightBuffers = gst_buffer_list_new();
445
446#ifndef GST_API_VERSION_1
447    m_frontLeftBuffersIterator = gst_buffer_list_iterate(m_frontLeftBuffers);
448    gst_buffer_list_iterator_add_group(m_frontLeftBuffersIterator);
449
450    m_frontRightBuffersIterator = gst_buffer_list_iterate(m_frontRightBuffers);
451    gst_buffer_list_iterator_add_group(m_frontRightBuffersIterator);
452#endif
453
454    GRefPtr<GMainContext> context = adoptGRef(g_main_context_new());
455    g_main_context_push_thread_default(context.get());
456    m_loop = adoptGRef(g_main_loop_new(context.get(), FALSE));
457
458    // Start the pipeline processing just after the loop is started.
459    GRefPtr<GSource> timeoutSource = adoptGRef(g_timeout_source_new(0));
460    g_source_attach(timeoutSource.get(), context.get());
461    g_source_set_callback(timeoutSource.get(), reinterpret_cast<GSourceFunc>(enteredMainLoopCallback), this, 0);
462
463    g_main_loop_run(m_loop.get());
464    g_main_context_pop_thread_default(context.get());
465
466    if (m_errorOccurred)
467        return 0;
468
469    unsigned channels = mixToMono ? 1 : 2;
470    RefPtr<AudioBus> audioBus = AudioBus::create(channels, m_channelSize, true);
471    audioBus->setSampleRate(m_sampleRate);
472
473    copyGstreamerBuffersToAudioChannel(m_frontLeftBuffers, audioBus->channel(0));
474    if (!mixToMono)
475        copyGstreamerBuffersToAudioChannel(m_frontRightBuffers, audioBus->channel(1));
476
477    return audioBus;
478}
479
480PassRefPtr<AudioBus> createBusFromAudioFile(const char* filePath, bool mixToMono, float sampleRate)
481{
482    return AudioFileReader(filePath).createBus(sampleRate, mixToMono);
483}
484
485PassRefPtr<AudioBus> createBusFromInMemoryAudioFile(const void* data, size_t dataSize, bool mixToMono, float sampleRate)
486{
487    return AudioFileReader(data, dataSize).createBus(sampleRate, mixToMono);
488}
489
490} // WebCore
491
492#endif // ENABLE(WEB_AUDIO)
493