1/* 2 * Copyright (C) 2011, 2012 Igalia S.L 3 * Copyright (C) 2011 Zan Dobersek <zandobersek@gmail.com> 4 * 5 * This library is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU Lesser General Public 7 * License as published by the Free Software Foundation; either 8 * version 2 of the License, or (at your option) any later version. 9 * 10 * This library is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * Lesser General Public License for more details. 14 * 15 * You should have received a copy of the GNU Lesser General Public 16 * License along with this library; if not, write to the Free Software 17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 18 */ 19 20#include "config.h" 21 22#if ENABLE(WEB_AUDIO) 23 24#include "AudioFileReader.h" 25 26#include "AudioBus.h" 27#include "GStreamerVersioning.h" 28 29#if PLATFORM(QT) 30// Clear out offending Qt macro so the following header, gio.h, can be included. 31// https://bugs.webkit.org/show_bug.cgi?id=95081 32#undef signals 33#endif 34 35#include <gio/gio.h> 36#include <gst/app/gstappsink.h> 37#include <gst/gst.h> 38#include <gst/pbutils/pbutils.h> 39#include <wtf/Noncopyable.h> 40#include <wtf/PassOwnPtr.h> 41#include <wtf/gobject/GOwnPtr.h> 42#include <wtf/gobject/GRefPtr.h> 43 44#ifdef GST_API_VERSION_1 45#include <gst/audio/audio.h> 46#else 47#include <gst/audio/multichannel.h> 48#endif 49 50#ifdef GST_API_VERSION_1 51static const char* gDecodebinName = "decodebin"; 52#else 53static const char* gDecodebinName = "decodebin2"; 54#endif 55 56namespace WebCore { 57 58class AudioFileReader { 59 WTF_MAKE_NONCOPYABLE(AudioFileReader); 60public: 61 AudioFileReader(const char* filePath); 62 AudioFileReader(const void* data, size_t dataSize); 63 ~AudioFileReader(); 64 65 PassRefPtr<AudioBus> createBus(float sampleRate, bool mixToMono); 66 67#ifdef GST_API_VERSION_1 68 GstFlowReturn handleSample(GstAppSink*); 69#else 70 GstFlowReturn handleBuffer(GstAppSink*); 71#endif 72 gboolean handleMessage(GstMessage*); 73 void handleNewDeinterleavePad(GstPad*); 74 void deinterleavePadsConfigured(); 75 void plugDeinterleave(GstPad*); 76 void decodeAudioForBusCreation(); 77 78private: 79 const void* m_data; 80 size_t m_dataSize; 81 const char* m_filePath; 82 83 float m_sampleRate; 84 GstBufferList* m_frontLeftBuffers; 85 GstBufferList* m_frontRightBuffers; 86 87#ifndef GST_API_VERSION_1 88 GstBufferListIterator* m_frontLeftBuffersIterator; 89 GstBufferListIterator* m_frontRightBuffersIterator; 90#endif 91 92 GstElement* m_pipeline; 93 unsigned m_channelSize; 94 GRefPtr<GstElement> m_decodebin; 95 GRefPtr<GstElement> m_deInterleave; 96 GRefPtr<GMainLoop> m_loop; 97 bool m_errorOccurred; 98}; 99 100static void copyGstreamerBuffersToAudioChannel(GstBufferList* buffers, AudioChannel* audioChannel) 101{ 102#ifdef GST_API_VERSION_1 103 float* destination = audioChannel->mutableData(); 104 unsigned bufferCount = gst_buffer_list_length(buffers); 105 for (unsigned i = 0; i < bufferCount; ++i) { 106 GstBuffer* buffer = gst_buffer_list_get(buffers, i); 107 ASSERT(buffer); 108 gsize bufferSize = gst_buffer_get_size(buffer); 109 gst_buffer_extract(buffer, 0, destination, bufferSize); 110 destination += bufferSize / sizeof(float); 111 } 112#else 113 GstBufferListIterator* iter = gst_buffer_list_iterate(buffers); 114 gst_buffer_list_iterator_next_group(iter); 115 GstBuffer* buffer = gst_buffer_list_iterator_merge_group(iter); 116 if (buffer) { 117 memcpy(audioChannel->mutableData(), reinterpret_cast<float*>(GST_BUFFER_DATA(buffer)), GST_BUFFER_SIZE(buffer)); 118 gst_buffer_unref(buffer); 119 } 120 121 gst_buffer_list_iterator_free(iter); 122#endif 123} 124 125static GstFlowReturn onAppsinkPullRequiredCallback(GstAppSink* sink, gpointer userData) 126{ 127#ifdef GST_API_VERSION_1 128 return static_cast<AudioFileReader*>(userData)->handleSample(sink); 129#else 130 return static_cast<AudioFileReader*>(userData)->handleBuffer(sink); 131#endif 132} 133 134gboolean messageCallback(GstBus*, GstMessage* message, AudioFileReader* reader) 135{ 136 return reader->handleMessage(message); 137} 138 139static void onGStreamerDeinterleavePadAddedCallback(GstElement*, GstPad* pad, AudioFileReader* reader) 140{ 141 reader->handleNewDeinterleavePad(pad); 142} 143 144static void onGStreamerDeinterleaveReadyCallback(GstElement*, AudioFileReader* reader) 145{ 146 reader->deinterleavePadsConfigured(); 147} 148 149static void onGStreamerDecodebinPadAddedCallback(GstElement*, GstPad* pad, AudioFileReader* reader) 150{ 151 reader->plugDeinterleave(pad); 152} 153 154gboolean enteredMainLoopCallback(gpointer userData) 155{ 156 AudioFileReader* reader = reinterpret_cast<AudioFileReader*>(userData); 157 reader->decodeAudioForBusCreation(); 158 return FALSE; 159} 160 161AudioFileReader::AudioFileReader(const char* filePath) 162 : m_data(0) 163 , m_dataSize(0) 164 , m_filePath(filePath) 165 , m_channelSize(0) 166 , m_errorOccurred(false) 167{ 168} 169 170AudioFileReader::AudioFileReader(const void* data, size_t dataSize) 171 : m_data(data) 172 , m_dataSize(dataSize) 173 , m_filePath(0) 174 , m_channelSize(0) 175 , m_errorOccurred(false) 176{ 177} 178 179AudioFileReader::~AudioFileReader() 180{ 181 if (m_pipeline) { 182 GRefPtr<GstBus> bus = webkitGstPipelineGetBus(GST_PIPELINE(m_pipeline)); 183 ASSERT(bus); 184 g_signal_handlers_disconnect_by_func(bus.get(), reinterpret_cast<gpointer>(messageCallback), this); 185 gst_bus_remove_signal_watch(bus.get()); 186 187 gst_element_set_state(m_pipeline, GST_STATE_NULL); 188 gst_object_unref(GST_OBJECT(m_pipeline)); 189 } 190 191 if (m_decodebin) { 192 g_signal_handlers_disconnect_by_func(m_decodebin.get(), reinterpret_cast<gpointer>(onGStreamerDecodebinPadAddedCallback), this); 193 m_decodebin.clear(); 194 } 195 196 if (m_deInterleave) { 197 g_signal_handlers_disconnect_by_func(m_deInterleave.get(), reinterpret_cast<gpointer>(onGStreamerDeinterleavePadAddedCallback), this); 198 g_signal_handlers_disconnect_by_func(m_deInterleave.get(), reinterpret_cast<gpointer>(onGStreamerDeinterleaveReadyCallback), this); 199 m_deInterleave.clear(); 200 } 201 202#ifndef GST_API_VERSION_1 203 gst_buffer_list_iterator_free(m_frontLeftBuffersIterator); 204 gst_buffer_list_iterator_free(m_frontRightBuffersIterator); 205#endif 206 gst_buffer_list_unref(m_frontLeftBuffers); 207 gst_buffer_list_unref(m_frontRightBuffers); 208} 209 210#ifdef GST_API_VERSION_1 211GstFlowReturn AudioFileReader::handleSample(GstAppSink* sink) 212{ 213 GstSample* sample = gst_app_sink_pull_sample(sink); 214 if (!sample) 215 return GST_FLOW_ERROR; 216 217 GstBuffer* buffer = gst_sample_get_buffer(sample); 218 if (!buffer) { 219 gst_sample_unref(sample); 220 return GST_FLOW_ERROR; 221 } 222 223 GstCaps* caps = gst_sample_get_caps(sample); 224 if (!caps) { 225 gst_sample_unref(sample); 226 return GST_FLOW_ERROR; 227 } 228 229 GstAudioInfo info; 230 gst_audio_info_from_caps(&info, caps); 231 int frames = GST_CLOCK_TIME_TO_FRAMES(GST_BUFFER_DURATION(buffer), GST_AUDIO_INFO_RATE(&info)); 232 233 // Check the first audio channel. The buffer is supposed to store 234 // data of a single channel anyway. 235 switch (GST_AUDIO_INFO_POSITION(&info, 0)) { 236 case GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT: 237 gst_buffer_list_add(m_frontLeftBuffers, gst_buffer_ref(buffer)); 238 m_channelSize += frames; 239 break; 240 case GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT: 241 gst_buffer_list_add(m_frontRightBuffers, gst_buffer_ref(buffer)); 242 break; 243 default: 244 break; 245 } 246 247 gst_sample_unref(sample); 248 return GST_FLOW_OK; 249 250} 251#endif 252 253#ifndef GST_API_VERSION_1 254GstFlowReturn AudioFileReader::handleBuffer(GstAppSink* sink) 255{ 256 GstBuffer* buffer = gst_app_sink_pull_buffer(sink); 257 if (!buffer) 258 return GST_FLOW_ERROR; 259 260 GstCaps* caps = gst_buffer_get_caps(buffer); 261 GstStructure* structure = gst_caps_get_structure(caps, 0); 262 263 gint channels = 0; 264 if (!gst_structure_get_int(structure, "channels", &channels) || !channels) { 265 gst_caps_unref(caps); 266 gst_buffer_unref(buffer); 267 return GST_FLOW_ERROR; 268 } 269 270 gint sampleRate = 0; 271 if (!gst_structure_get_int(structure, "rate", &sampleRate) || !sampleRate) { 272 gst_caps_unref(caps); 273 gst_buffer_unref(buffer); 274 return GST_FLOW_ERROR; 275 } 276 277 gint width = 0; 278 if (!gst_structure_get_int(structure, "width", &width) || !width) { 279 gst_caps_unref(caps); 280 gst_buffer_unref(buffer); 281 return GST_FLOW_ERROR; 282 } 283 284 GstClockTime duration = (static_cast<guint64>(GST_BUFFER_SIZE(buffer)) * 8 * GST_SECOND) / (sampleRate * channels * width); 285 int frames = GST_CLOCK_TIME_TO_FRAMES(duration, sampleRate); 286 287 // Check the first audio channel. The buffer is supposed to store 288 // data of a single channel anyway. 289 GstAudioChannelPosition* positions = gst_audio_get_channel_positions(structure); 290 switch (positions[0]) { 291 case GST_AUDIO_CHANNEL_POSITION_FRONT_LEFT: 292 gst_buffer_list_iterator_add(m_frontLeftBuffersIterator, buffer); 293 m_channelSize += frames; 294 break; 295 case GST_AUDIO_CHANNEL_POSITION_FRONT_RIGHT: 296 gst_buffer_list_iterator_add(m_frontRightBuffersIterator, buffer); 297 break; 298 default: 299 gst_buffer_unref(buffer); 300 break; 301 } 302 303 g_free(positions); 304 gst_caps_unref(caps); 305 return GST_FLOW_OK; 306} 307#endif 308 309gboolean AudioFileReader::handleMessage(GstMessage* message) 310{ 311 GOwnPtr<GError> error; 312 GOwnPtr<gchar> debug; 313 314 switch (GST_MESSAGE_TYPE(message)) { 315 case GST_MESSAGE_EOS: 316 g_main_loop_quit(m_loop.get()); 317 break; 318 case GST_MESSAGE_WARNING: 319 gst_message_parse_warning(message, &error.outPtr(), &debug.outPtr()); 320 g_warning("Warning: %d, %s. Debug output: %s", error->code, error->message, debug.get()); 321 break; 322 case GST_MESSAGE_ERROR: 323 gst_message_parse_error(message, &error.outPtr(), &debug.outPtr()); 324 g_warning("Error: %d, %s. Debug output: %s", error->code, error->message, debug.get()); 325 m_errorOccurred = true; 326 g_main_loop_quit(m_loop.get()); 327 break; 328 default: 329 break; 330 } 331 return TRUE; 332} 333 334void AudioFileReader::handleNewDeinterleavePad(GstPad* pad) 335{ 336 // A new pad for a planar channel was added in deinterleave. Plug 337 // in an appsink so we can pull the data from each 338 // channel. Pipeline looks like: 339 // ... deinterleave ! queue ! appsink. 340 GstElement* queue = gst_element_factory_make("queue", 0); 341 GstElement* sink = gst_element_factory_make("appsink", 0); 342 343 GstAppSinkCallbacks callbacks; 344 callbacks.eos = 0; 345 callbacks.new_preroll = 0; 346#ifdef GST_API_VERSION_1 347 callbacks.new_sample = onAppsinkPullRequiredCallback; 348#else 349 callbacks.new_buffer_list = 0; 350 callbacks.new_buffer = onAppsinkPullRequiredCallback; 351#endif 352 gst_app_sink_set_callbacks(GST_APP_SINK(sink), &callbacks, this, 0); 353 354 g_object_set(sink, "sync", FALSE, NULL); 355 356 gst_bin_add_many(GST_BIN(m_pipeline), queue, sink, NULL); 357 358 GstPad* sinkPad = gst_element_get_static_pad(queue, "sink"); 359 gst_pad_link_full(pad, sinkPad, GST_PAD_LINK_CHECK_NOTHING); 360 gst_object_unref(GST_OBJECT(sinkPad)); 361 362 gst_element_link_pads_full(queue, "src", sink, "sink", GST_PAD_LINK_CHECK_NOTHING); 363 364 gst_element_set_state(queue, GST_STATE_READY); 365 gst_element_set_state(sink, GST_STATE_READY); 366} 367 368void AudioFileReader::deinterleavePadsConfigured() 369{ 370 // All deinterleave src pads are now available, let's roll to 371 // PLAYING so data flows towards the sinks and it can be retrieved. 372 gst_element_set_state(m_pipeline, GST_STATE_PLAYING); 373} 374 375void AudioFileReader::plugDeinterleave(GstPad* pad) 376{ 377 // A decodebin pad was added, plug in a deinterleave element to 378 // separate each planar channel. Sub pipeline looks like 379 // ... decodebin2 ! audioconvert ! audioresample ! capsfilter ! deinterleave. 380 GstElement* audioConvert = gst_element_factory_make("audioconvert", 0); 381 GstElement* audioResample = gst_element_factory_make("audioresample", 0); 382 GstElement* capsFilter = gst_element_factory_make("capsfilter", 0); 383 m_deInterleave = gst_element_factory_make("deinterleave", "deinterleave"); 384 385 g_object_set(m_deInterleave.get(), "keep-positions", TRUE, NULL); 386 g_signal_connect(m_deInterleave.get(), "pad-added", G_CALLBACK(onGStreamerDeinterleavePadAddedCallback), this); 387 g_signal_connect(m_deInterleave.get(), "no-more-pads", G_CALLBACK(onGStreamerDeinterleaveReadyCallback), this); 388 389 GstCaps* caps = getGstAudioCaps(2, m_sampleRate); 390 g_object_set(capsFilter, "caps", caps, NULL); 391 gst_caps_unref(caps); 392 393 gst_bin_add_many(GST_BIN(m_pipeline), audioConvert, audioResample, capsFilter, m_deInterleave.get(), NULL); 394 395 GstPad* sinkPad = gst_element_get_static_pad(audioConvert, "sink"); 396 gst_pad_link_full(pad, sinkPad, GST_PAD_LINK_CHECK_NOTHING); 397 gst_object_unref(GST_OBJECT(sinkPad)); 398 399 gst_element_link_pads_full(audioConvert, "src", audioResample, "sink", GST_PAD_LINK_CHECK_NOTHING); 400 gst_element_link_pads_full(audioResample, "src", capsFilter, "sink", GST_PAD_LINK_CHECK_NOTHING); 401 gst_element_link_pads_full(capsFilter, "src", m_deInterleave.get(), "sink", GST_PAD_LINK_CHECK_NOTHING); 402 403 gst_element_sync_state_with_parent(audioConvert); 404 gst_element_sync_state_with_parent(audioResample); 405 gst_element_sync_state_with_parent(capsFilter); 406 gst_element_sync_state_with_parent(m_deInterleave.get()); 407} 408 409void AudioFileReader::decodeAudioForBusCreation() 410{ 411 // Build the pipeline (giostreamsrc | filesrc) ! decodebin2 412 // A deinterleave element is added once a src pad becomes available in decodebin. 413 m_pipeline = gst_pipeline_new(0); 414 415 GRefPtr<GstBus> bus = webkitGstPipelineGetBus(GST_PIPELINE(m_pipeline)); 416 ASSERT(bus); 417 gst_bus_add_signal_watch(bus.get()); 418 g_signal_connect(bus.get(), "message", G_CALLBACK(messageCallback), this); 419 420 GstElement* source; 421 if (m_data) { 422 ASSERT(m_dataSize); 423 source = gst_element_factory_make("giostreamsrc", 0); 424 GRefPtr<GInputStream> memoryStream = adoptGRef(g_memory_input_stream_new_from_data(m_data, m_dataSize, 0)); 425 g_object_set(source, "stream", memoryStream.get(), NULL); 426 } else { 427 source = gst_element_factory_make("filesrc", 0); 428 g_object_set(source, "location", m_filePath, NULL); 429 } 430 431 m_decodebin = gst_element_factory_make(gDecodebinName, "decodebin"); 432 g_signal_connect(m_decodebin.get(), "pad-added", G_CALLBACK(onGStreamerDecodebinPadAddedCallback), this); 433 434 gst_bin_add_many(GST_BIN(m_pipeline), source, m_decodebin.get(), NULL); 435 gst_element_link_pads_full(source, "src", m_decodebin.get(), "sink", GST_PAD_LINK_CHECK_NOTHING); 436 gst_element_set_state(m_pipeline, GST_STATE_PAUSED); 437} 438 439PassRefPtr<AudioBus> AudioFileReader::createBus(float sampleRate, bool mixToMono) 440{ 441 m_sampleRate = sampleRate; 442 443 m_frontLeftBuffers = gst_buffer_list_new(); 444 m_frontRightBuffers = gst_buffer_list_new(); 445 446#ifndef GST_API_VERSION_1 447 m_frontLeftBuffersIterator = gst_buffer_list_iterate(m_frontLeftBuffers); 448 gst_buffer_list_iterator_add_group(m_frontLeftBuffersIterator); 449 450 m_frontRightBuffersIterator = gst_buffer_list_iterate(m_frontRightBuffers); 451 gst_buffer_list_iterator_add_group(m_frontRightBuffersIterator); 452#endif 453 454 GRefPtr<GMainContext> context = adoptGRef(g_main_context_new()); 455 g_main_context_push_thread_default(context.get()); 456 m_loop = adoptGRef(g_main_loop_new(context.get(), FALSE)); 457 458 // Start the pipeline processing just after the loop is started. 459 GRefPtr<GSource> timeoutSource = adoptGRef(g_timeout_source_new(0)); 460 g_source_attach(timeoutSource.get(), context.get()); 461 g_source_set_callback(timeoutSource.get(), reinterpret_cast<GSourceFunc>(enteredMainLoopCallback), this, 0); 462 463 g_main_loop_run(m_loop.get()); 464 g_main_context_pop_thread_default(context.get()); 465 466 if (m_errorOccurred) 467 return 0; 468 469 unsigned channels = mixToMono ? 1 : 2; 470 RefPtr<AudioBus> audioBus = AudioBus::create(channels, m_channelSize, true); 471 audioBus->setSampleRate(m_sampleRate); 472 473 copyGstreamerBuffersToAudioChannel(m_frontLeftBuffers, audioBus->channel(0)); 474 if (!mixToMono) 475 copyGstreamerBuffersToAudioChannel(m_frontRightBuffers, audioBus->channel(1)); 476 477 return audioBus; 478} 479 480PassRefPtr<AudioBus> createBusFromAudioFile(const char* filePath, bool mixToMono, float sampleRate) 481{ 482 return AudioFileReader(filePath).createBus(sampleRate, mixToMono); 483} 484 485PassRefPtr<AudioBus> createBusFromInMemoryAudioFile(const void* data, size_t dataSize, bool mixToMono, float sampleRate) 486{ 487 return AudioFileReader(data, dataSize).createBus(sampleRate, mixToMono); 488} 489 490} // WebCore 491 492#endif // ENABLE(WEB_AUDIO) 493