1/*
2 *  Copyright (C) 2011, 2012 Igalia S.L
3 *
4 *  This library is free software; you can redistribute it and/or
5 *  modify it under the terms of the GNU Lesser General Public
6 *  License as published by the Free Software Foundation; either
7 *  version 2 of the License, or (at your option) any later version.
8 *
9 *  This library is distributed in the hope that it will be useful,
10 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
11 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12 *  Lesser General Public License for more details.
13 *
14 *  You should have received a copy of the GNU Lesser General Public
15 *  License along with this library; if not, write to the Free Software
16 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
17 */
18
19#include "config.h"
20
21#include "WebKitWebAudioSourceGStreamer.h"
22
23#if ENABLE(WEB_AUDIO) && USE(GSTREAMER)
24
25#include "AudioBus.h"
26#include "AudioIOCallback.h"
27#include "GRefPtrGStreamer.h"
28#include "GStreamerUtilities.h"
29#include <gst/audio/audio.h>
30#include <gst/pbutils/pbutils.h>
31#include <wtf/gobject/GUniquePtr.h>
32
33using namespace WebCore;
34
35typedef struct _WebKitWebAudioSrcClass   WebKitWebAudioSrcClass;
36typedef struct _WebKitWebAudioSourcePrivate WebKitWebAudioSourcePrivate;
37
38struct _WebKitWebAudioSrc {
39    GstBin parent;
40
41    WebKitWebAudioSourcePrivate* priv;
42};
43
44struct _WebKitWebAudioSrcClass {
45    GstBinClass parentClass;
46};
47
48#define WEBKIT_WEB_AUDIO_SRC_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE((obj), WEBKIT_TYPE_WEBAUDIO_SRC, WebKitWebAudioSourcePrivate))
49struct _WebKitWebAudioSourcePrivate {
50    gfloat sampleRate;
51    AudioBus* bus;
52    AudioIOCallback* provider;
53    guint framesToPull;
54
55    GRefPtr<GstElement> interleave;
56    GRefPtr<GstElement> wavEncoder;
57
58    GRefPtr<GstTask> task;
59    GRecMutex mutex;
60
61    GSList* pads; // List of queue sink pads. One queue for each planar audio channel.
62    GstPad* sourcePad; // src pad of the element, interleaved wav data is pushed to it.
63
64    bool newStreamEventPending;
65    GstSegment segment;
66};
67
68enum {
69    PROP_RATE = 1,
70    PROP_BUS,
71    PROP_PROVIDER,
72    PROP_FRAMES
73};
74
75static GstStaticPadTemplate srcTemplate = GST_STATIC_PAD_TEMPLATE("src",
76                                                                  GST_PAD_SRC,
77                                                                  GST_PAD_ALWAYS,
78                                                                  GST_STATIC_CAPS("audio/x-wav"));
79
80GST_DEBUG_CATEGORY_STATIC(webkit_web_audio_src_debug);
81#define GST_CAT_DEFAULT webkit_web_audio_src_debug
82
83static void webKitWebAudioSrcConstructed(GObject*);
84static void webKitWebAudioSrcFinalize(GObject*);
85static void webKitWebAudioSrcSetProperty(GObject*, guint propertyId, const GValue*, GParamSpec*);
86static void webKitWebAudioSrcGetProperty(GObject*, guint propertyId, GValue*, GParamSpec*);
87static GstStateChangeReturn webKitWebAudioSrcChangeState(GstElement*, GstStateChange);
88static void webKitWebAudioSrcLoop(WebKitWebAudioSrc*);
89
90static GstCaps* getGStreamerMonoAudioCaps(float sampleRate)
91{
92    return gst_caps_new_simple("audio/x-raw", "rate", G_TYPE_INT, static_cast<int>(sampleRate),
93        "channels", G_TYPE_INT, 1,
94        "format", G_TYPE_STRING, gst_audio_format_to_string(GST_AUDIO_FORMAT_F32),
95        "layout", G_TYPE_STRING, "non-interleaved", NULL);
96}
97
98static GstAudioChannelPosition webKitWebAudioGStreamerChannelPosition(int channelIndex)
99{
100    GstAudioChannelPosition position = GST_AUDIO_CHANNEL_POSITION_NONE;
101
102    switch (channelIndex) {
103    case AudioBus::ChannelLeft:
104        position = GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT;
105        break;
106    case AudioBus::ChannelRight:
107        position = GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT;
108        break;
109    case AudioBus::ChannelCenter:
110        position = GST_AUDIO_CHANNEL_POSITION_FRONT_CENTER;
111        break;
112    case AudioBus::ChannelLFE:
113        position = GST_AUDIO_CHANNEL_POSITION_LFE1;
114        break;
115    case AudioBus::ChannelSurroundLeft:
116        position = GST_AUDIO_CHANNEL_POSITION_REAR_LEFT;
117        break;
118    case AudioBus::ChannelSurroundRight:
119        position = GST_AUDIO_CHANNEL_POSITION_REAR_RIGHT;
120        break;
121    default:
122        break;
123    };
124
125    return position;
126}
127
128#define webkit_web_audio_src_parent_class parent_class
129G_DEFINE_TYPE_WITH_CODE(WebKitWebAudioSrc, webkit_web_audio_src, GST_TYPE_BIN, GST_DEBUG_CATEGORY_INIT(webkit_web_audio_src_debug, \
130                            "webkitwebaudiosrc", \
131                            0, \
132                            "webaudiosrc element"));
133
134static void webkit_web_audio_src_class_init(WebKitWebAudioSrcClass* webKitWebAudioSrcClass)
135{
136    GObjectClass* objectClass = G_OBJECT_CLASS(webKitWebAudioSrcClass);
137    GstElementClass* elementClass = GST_ELEMENT_CLASS(webKitWebAudioSrcClass);
138
139    gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&srcTemplate));
140    gst_element_class_set_metadata(elementClass, "WebKit WebAudio source element", "Source", "Handles WebAudio data from WebCore", "Philippe Normand <pnormand@igalia.com>");
141
142    objectClass->constructed = webKitWebAudioSrcConstructed;
143    objectClass->finalize = webKitWebAudioSrcFinalize;
144    elementClass->change_state = webKitWebAudioSrcChangeState;
145
146    objectClass->set_property = webKitWebAudioSrcSetProperty;
147    objectClass->get_property = webKitWebAudioSrcGetProperty;
148
149    GParamFlags flags = static_cast<GParamFlags>(G_PARAM_CONSTRUCT_ONLY | G_PARAM_READWRITE);
150    g_object_class_install_property(objectClass,
151                                    PROP_RATE,
152                                    g_param_spec_float("rate", "rate",
153                                                       "Sample rate", G_MINDOUBLE, G_MAXDOUBLE,
154                                                       44100.0, flags));
155
156    g_object_class_install_property(objectClass,
157                                    PROP_BUS,
158                                    g_param_spec_pointer("bus", "bus",
159                                                         "Bus", flags));
160
161    g_object_class_install_property(objectClass,
162                                    PROP_PROVIDER,
163                                    g_param_spec_pointer("provider", "provider",
164                                                         "Provider", flags));
165
166    g_object_class_install_property(objectClass,
167                                    PROP_FRAMES,
168                                    g_param_spec_uint("frames", "frames",
169                                                      "Number of audio frames to pull at each iteration",
170                                                      0, G_MAXUINT8, 128, flags));
171
172    g_type_class_add_private(webKitWebAudioSrcClass, sizeof(WebKitWebAudioSourcePrivate));
173}
174
175static void webkit_web_audio_src_init(WebKitWebAudioSrc* src)
176{
177    WebKitWebAudioSourcePrivate* priv = G_TYPE_INSTANCE_GET_PRIVATE(src, WEBKIT_TYPE_WEB_AUDIO_SRC, WebKitWebAudioSourcePrivate);
178    src->priv = priv;
179    new (priv) WebKitWebAudioSourcePrivate();
180
181    priv->sourcePad = webkitGstGhostPadFromStaticTemplate(&srcTemplate, "src", 0);
182    gst_element_add_pad(GST_ELEMENT(src), priv->sourcePad);
183
184    priv->provider = 0;
185    priv->bus = 0;
186
187    priv->newStreamEventPending = true;
188    gst_segment_init(&priv->segment, GST_FORMAT_TIME);
189
190    g_rec_mutex_init(&priv->mutex);
191    priv->task = gst_task_new(reinterpret_cast<GstTaskFunction>(webKitWebAudioSrcLoop), src, 0);
192
193    gst_task_set_lock(priv->task.get(), &priv->mutex);
194}
195
196static void webKitWebAudioSrcConstructed(GObject* object)
197{
198    WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
199    WebKitWebAudioSourcePrivate* priv = src->priv;
200
201    ASSERT(priv->bus);
202    ASSERT(priv->provider);
203    ASSERT(priv->sampleRate);
204
205    priv->interleave = gst_element_factory_make("interleave", 0);
206    priv->wavEncoder = gst_element_factory_make("wavenc", 0);
207
208    if (!priv->interleave) {
209        GST_ERROR_OBJECT(src, "Failed to create interleave");
210        return;
211    }
212
213    if (!priv->wavEncoder) {
214        GST_ERROR_OBJECT(src, "Failed to create wavenc");
215        return;
216    }
217
218    gst_bin_add_many(GST_BIN(src), priv->interleave.get(), priv->wavEncoder.get(), NULL);
219    gst_element_link_pads_full(priv->interleave.get(), "src", priv->wavEncoder.get(), "sink", GST_PAD_LINK_CHECK_NOTHING);
220
221    // For each channel of the bus create a new upstream branch for interleave, like:
222    // queue ! capsfilter ! audioconvert. which is plugged to a new interleave request sinkpad.
223    for (unsigned channelIndex = 0; channelIndex < priv->bus->numberOfChannels(); channelIndex++) {
224        GUniquePtr<gchar> queueName(g_strdup_printf("webaudioQueue%u", channelIndex));
225        GstElement* queue = gst_element_factory_make("queue", queueName.get());
226        GstElement* capsfilter = gst_element_factory_make("capsfilter", 0);
227        GstElement* audioconvert = gst_element_factory_make("audioconvert", 0);
228
229        GRefPtr<GstCaps> monoCaps = adoptGRef(getGStreamerMonoAudioCaps(priv->sampleRate));
230
231        GstAudioInfo info;
232        gst_audio_info_from_caps(&info, monoCaps.get());
233        GST_AUDIO_INFO_POSITION(&info, 0) = webKitWebAudioGStreamerChannelPosition(channelIndex);
234        GRefPtr<GstCaps> caps = adoptGRef(gst_audio_info_to_caps(&info));
235        g_object_set(capsfilter, "caps", caps.get(), NULL);
236
237        // Configure the queue for minimal latency.
238        g_object_set(queue, "max-size-buffers", static_cast<guint>(1), NULL);
239
240        GstPad* pad = gst_element_get_static_pad(queue, "sink");
241        priv->pads = g_slist_prepend(priv->pads, pad);
242
243        gst_bin_add_many(GST_BIN(src), queue, capsfilter, audioconvert, NULL);
244        gst_element_link_pads_full(queue, "src", capsfilter, "sink", GST_PAD_LINK_CHECK_NOTHING);
245        gst_element_link_pads_full(capsfilter, "src", audioconvert, "sink", GST_PAD_LINK_CHECK_NOTHING);
246        gst_element_link_pads_full(audioconvert, "src", priv->interleave.get(), 0, GST_PAD_LINK_CHECK_NOTHING);
247
248    }
249    priv->pads = g_slist_reverse(priv->pads);
250
251    // wavenc's src pad is the only visible pad of our element.
252    GRefPtr<GstPad> targetPad = adoptGRef(gst_element_get_static_pad(priv->wavEncoder.get(), "src"));
253    gst_ghost_pad_set_target(GST_GHOST_PAD(priv->sourcePad), targetPad.get());
254}
255
256static void webKitWebAudioSrcFinalize(GObject* object)
257{
258    WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
259    WebKitWebAudioSourcePrivate* priv = src->priv;
260
261    g_rec_mutex_clear(&priv->mutex);
262
263    g_slist_free_full(priv->pads, reinterpret_cast<GDestroyNotify>(gst_object_unref));
264
265    priv->~WebKitWebAudioSourcePrivate();
266    GST_CALL_PARENT(G_OBJECT_CLASS, finalize, ((GObject* )(src)));
267}
268
269static void webKitWebAudioSrcSetProperty(GObject* object, guint propertyId, const GValue* value, GParamSpec* pspec)
270{
271    WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
272    WebKitWebAudioSourcePrivate* priv = src->priv;
273
274    switch (propertyId) {
275    case PROP_RATE:
276        priv->sampleRate = g_value_get_float(value);
277        break;
278    case PROP_BUS:
279        priv->bus = static_cast<AudioBus*>(g_value_get_pointer(value));
280        break;
281    case PROP_PROVIDER:
282        priv->provider = static_cast<AudioIOCallback*>(g_value_get_pointer(value));
283        break;
284    case PROP_FRAMES:
285        priv->framesToPull = g_value_get_uint(value);
286        break;
287    default:
288        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, pspec);
289        break;
290    }
291}
292
293static void webKitWebAudioSrcGetProperty(GObject* object, guint propertyId, GValue* value, GParamSpec* pspec)
294{
295    WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(object);
296    WebKitWebAudioSourcePrivate* priv = src->priv;
297
298    switch (propertyId) {
299    case PROP_RATE:
300        g_value_set_float(value, priv->sampleRate);
301        break;
302    case PROP_BUS:
303        g_value_set_pointer(value, priv->bus);
304        break;
305    case PROP_PROVIDER:
306        g_value_set_pointer(value, priv->provider);
307        break;
308    case PROP_FRAMES:
309        g_value_set_uint(value, priv->framesToPull);
310        break;
311    default:
312        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, pspec);
313        break;
314    }
315}
316
317static void webKitWebAudioSrcLoop(WebKitWebAudioSrc* src)
318{
319    WebKitWebAudioSourcePrivate* priv = src->priv;
320
321    ASSERT(priv->bus);
322    ASSERT(priv->provider);
323    if (!priv->provider || !priv->bus)
324        return;
325
326    GSList* channelBufferList = 0;
327    register int i;
328    unsigned bufferSize = priv->framesToPull * sizeof(float);
329    for (i = g_slist_length(priv->pads) - 1; i >= 0; i--) {
330        GstBuffer* channelBuffer = gst_buffer_new_and_alloc(bufferSize);
331        ASSERT(channelBuffer);
332        channelBufferList = g_slist_prepend(channelBufferList, channelBuffer);
333        GstMapInfo info;
334        gst_buffer_map(channelBuffer, &info, GST_MAP_READ);
335        priv->bus->setChannelMemory(i, reinterpret_cast<float*>(info.data), priv->framesToPull);
336        gst_buffer_unmap(channelBuffer, &info);
337    }
338
339    // FIXME: Add support for local/live audio input.
340    priv->provider->render(0, priv->bus, priv->framesToPull);
341
342    GSList* padsIt = priv->pads;
343    GSList* buffersIt = channelBufferList;
344
345#if GST_CHECK_VERSION(1, 2, 0)
346    guint groupId = 0;
347    if (priv->newStreamEventPending)
348        groupId = gst_util_group_id_next();
349#endif
350
351    for (i = 0; padsIt && buffersIt; padsIt = g_slist_next(padsIt), buffersIt = g_slist_next(buffersIt), ++i) {
352        GstPad* pad = static_cast<GstPad*>(padsIt->data);
353        GstBuffer* channelBuffer = static_cast<GstBuffer*>(buffersIt->data);
354
355        // Send stream-start, segment and caps events downstream, along with the first buffer.
356        if (priv->newStreamEventPending) {
357            GRefPtr<GstElement> queue = adoptGRef(gst_pad_get_parent_element(pad));
358            GRefPtr<GstPad> sinkPad = adoptGRef(gst_element_get_static_pad(queue.get(), "sink"));
359            GUniquePtr<gchar> queueName(gst_element_get_name(queue.get()));
360            GUniquePtr<gchar> streamId(g_strdup_printf("webaudio/%s", queueName.get()));
361            GstEvent* streamStartEvent = gst_event_new_stream_start(streamId.get());
362#if GST_CHECK_VERSION(1, 2, 0)
363            gst_event_set_group_id(streamStartEvent, groupId);
364#endif
365            gst_pad_send_event(sinkPad.get(), streamStartEvent);
366
367            GRefPtr<GstCaps> monoCaps = adoptGRef(getGStreamerMonoAudioCaps(priv->sampleRate));
368            GstAudioInfo info;
369            gst_audio_info_from_caps(&info, monoCaps.get());
370            GST_AUDIO_INFO_POSITION(&info, 0) = webKitWebAudioGStreamerChannelPosition(i);
371            GRefPtr<GstCaps> capsWithChannelPosition = adoptGRef(gst_audio_info_to_caps(&info));
372            gst_pad_send_event(sinkPad.get(), gst_event_new_caps(capsWithChannelPosition.get()));
373
374            gst_pad_send_event(sinkPad.get(), gst_event_new_segment(&priv->segment));
375        }
376
377        GstFlowReturn ret = gst_pad_chain(pad, channelBuffer);
378        if (ret != GST_FLOW_OK)
379            GST_ELEMENT_ERROR(src, CORE, PAD, ("Internal WebAudioSrc error"), ("Failed to push buffer on %s:%s flow: %s", GST_DEBUG_PAD_NAME(pad), gst_flow_get_name(ret)));
380    }
381
382    priv->newStreamEventPending = false;
383
384    g_slist_free(channelBufferList);
385}
386
387static GstStateChangeReturn webKitWebAudioSrcChangeState(GstElement* element, GstStateChange transition)
388{
389    GstStateChangeReturn returnValue = GST_STATE_CHANGE_SUCCESS;
390    WebKitWebAudioSrc* src = WEBKIT_WEB_AUDIO_SRC(element);
391
392    switch (transition) {
393    case GST_STATE_CHANGE_NULL_TO_READY:
394        if (!src->priv->interleave) {
395            gst_element_post_message(element, gst_missing_element_message_new(element, "interleave"));
396            GST_ELEMENT_ERROR(src, CORE, MISSING_PLUGIN, (0), ("no interleave"));
397            return GST_STATE_CHANGE_FAILURE;
398        }
399        if (!src->priv->wavEncoder) {
400            gst_element_post_message(element, gst_missing_element_message_new(element, "wavenc"));
401            GST_ELEMENT_ERROR(src, CORE, MISSING_PLUGIN, (0), ("no wavenc"));
402            return GST_STATE_CHANGE_FAILURE;
403        }
404        break;
405    default:
406        break;
407    }
408
409    returnValue = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition);
410    if (UNLIKELY(returnValue == GST_STATE_CHANGE_FAILURE)) {
411        GST_DEBUG_OBJECT(src, "State change failed");
412        return returnValue;
413    }
414
415    switch (transition) {
416    case GST_STATE_CHANGE_READY_TO_PAUSED:
417        GST_DEBUG_OBJECT(src, "READY->PAUSED");
418        if (!gst_task_start(src->priv->task.get()))
419            returnValue = GST_STATE_CHANGE_FAILURE;
420        break;
421    case GST_STATE_CHANGE_PAUSED_TO_READY:
422        src->priv->newStreamEventPending = true;
423        GST_DEBUG_OBJECT(src, "PAUSED->READY");
424        if (!gst_task_join(src->priv->task.get()))
425            returnValue = GST_STATE_CHANGE_FAILURE;
426        break;
427    default:
428        break;
429    }
430
431    return returnValue;
432}
433
434#endif // ENABLE(WEB_AUDIO) && USE(GSTREAMER)
435