1/* 2 * Copyright (C) 2007 OpenedHand 3 * Copyright (C) 2007 Alp Toker <alp@atoker.com> 4 * Copyright (C) 2009, 2010, 2011, 2012 Igalia S.L 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 19 */ 20 21/* 22 * 23 * WebKitVideoSink is a GStreamer sink element that triggers 24 * repaints in the WebKit GStreamer media player for the 25 * current video buffer. 26 */ 27 28#include "config.h" 29#include "VideoSinkGStreamer.h" 30 31#if ENABLE(VIDEO) && USE(GSTREAMER) 32#include "GRefPtrGStreamer.h" 33#include "GStreamerUtilities.h" 34#include "IntSize.h" 35#include <glib.h> 36#include <gst/gst.h> 37#include <gst/video/gstvideometa.h> 38#include <wtf/OwnPtr.h> 39#include <wtf/gobject/GMainLoopSource.h> 40#include <wtf/gobject/GMutexLocker.h> 41 42using namespace WebCore; 43 44// CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant. 45#if G_BYTE_ORDER == G_LITTLE_ENDIAN 46#define GST_CAPS_FORMAT "{ BGRx, BGRA }" 47#else 48#define GST_CAPS_FORMAT "{ xRGB, ARGB }" 49#endif 50#if GST_CHECK_VERSION(1, 1, 0) 51#define GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, GST_CAPS_FORMAT) ";" 52#else 53#define GST_FEATURED_CAPS 54#endif 55 56#define WEBKIT_VIDEO_SINK_PAD_CAPS GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE(GST_CAPS_FORMAT) 57 58static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS)); 59 60 61GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug); 62#define GST_CAT_DEFAULT webkitVideoSinkDebug 63 64enum { 65 REPAINT_REQUESTED, 66 LAST_SIGNAL 67}; 68 69enum { 70 PROP_0, 71 PROP_CAPS 72}; 73 74static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, }; 75 76struct _WebKitVideoSinkPrivate { 77 GstBuffer* buffer; 78 GMainLoopSource timeoutSource; 79 GMutex* bufferMutex; 80 GCond* dataCondition; 81 82 GstVideoInfo info; 83 84 GstCaps* currentCaps; 85 86 // If this is TRUE all processing should finish ASAP 87 // This is necessary because there could be a race between 88 // unlock() and render(), where unlock() wins, signals the 89 // GCond, then render() tries to render a frame although 90 // everything else isn't running anymore. This will lead 91 // to deadlocks because render() holds the stream lock. 92 // 93 // Protected by the buffer mutex 94 bool unlocked; 95}; 96 97#define webkit_video_sink_parent_class parent_class 98G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink")); 99 100 101static void webkit_video_sink_init(WebKitVideoSink* sink) 102{ 103 sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate); 104 new (sink->priv) WebKitVideoSinkPrivate(); 105 sink->priv->dataCondition = new GCond; 106 g_cond_init(sink->priv->dataCondition); 107 sink->priv->bufferMutex = new GMutex; 108 g_mutex_init(sink->priv->bufferMutex); 109 110 gst_video_info_init(&sink->priv->info); 111} 112 113static void webkitVideoSinkTimeoutCallback(WebKitVideoSink* sink) 114{ 115 WebKitVideoSinkPrivate* priv = sink->priv; 116 117 GMutexLocker lock(priv->bufferMutex); 118 GstBuffer* buffer = priv->buffer; 119 priv->buffer = 0; 120 121 if (!buffer || priv->unlocked || UNLIKELY(!GST_IS_BUFFER(buffer))) { 122 g_cond_signal(priv->dataCondition); 123 return; 124 } 125 126 g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, buffer); 127 gst_buffer_unref(buffer); 128 g_cond_signal(priv->dataCondition); 129} 130 131static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer) 132{ 133 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink); 134 WebKitVideoSinkPrivate* priv = sink->priv; 135 136 GMutexLocker lock(priv->bufferMutex); 137 138 if (priv->unlocked) 139 return GST_FLOW_OK; 140 141 priv->buffer = gst_buffer_ref(buffer); 142 143 // The video info structure is valid only if the sink handled an allocation query. 144 GstVideoFormat format = GST_VIDEO_INFO_FORMAT(&priv->info); 145 if (format == GST_VIDEO_FORMAT_UNKNOWN) { 146 gst_buffer_unref(buffer); 147 return GST_FLOW_ERROR; 148 } 149 150#if !(USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS)) 151 // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't. 152 // Here we convert to Cairo's ARGB. 153 if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) { 154 // Because GstBaseSink::render() only owns the buffer reference in the 155 // method scope we can't use gst_buffer_make_writable() here. Also 156 // The buffer content should not be changed here because the same buffer 157 // could be passed multiple times to this method (in theory). 158 159 GstBuffer* newBuffer = WebCore::createGstBuffer(buffer); 160 161 // Check if allocation failed. 162 if (UNLIKELY(!newBuffer)) { 163 gst_buffer_unref(buffer); 164 return GST_FLOW_ERROR; 165 } 166 167 // We don't use Color::premultipliedARGBFromColor() here because 168 // one function call per video pixel is just too expensive: 169 // For 720p/PAL for example this means 1280*720*25=23040000 170 // function calls per second! 171 GstVideoFrame sourceFrame; 172 GstVideoFrame destinationFrame; 173 174 if (!gst_video_frame_map(&sourceFrame, &priv->info, buffer, GST_MAP_READ)) { 175 gst_buffer_unref(buffer); 176 gst_buffer_unref(newBuffer); 177 return GST_FLOW_ERROR; 178 } 179 if (!gst_video_frame_map(&destinationFrame, &priv->info, newBuffer, GST_MAP_WRITE)) { 180 gst_video_frame_unmap(&sourceFrame); 181 gst_buffer_unref(buffer); 182 gst_buffer_unref(newBuffer); 183 return GST_FLOW_ERROR; 184 } 185 186 const guint8* source = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&sourceFrame, 0)); 187 guint8* destination = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&destinationFrame, 0)); 188 189 for (int x = 0; x < GST_VIDEO_FRAME_HEIGHT(&sourceFrame); x++) { 190 for (int y = 0; y < GST_VIDEO_FRAME_WIDTH(&sourceFrame); y++) { 191#if G_BYTE_ORDER == G_LITTLE_ENDIAN 192 unsigned short alpha = source[3]; 193 destination[0] = (source[0] * alpha + 128) / 255; 194 destination[1] = (source[1] * alpha + 128) / 255; 195 destination[2] = (source[2] * alpha + 128) / 255; 196 destination[3] = alpha; 197#else 198 unsigned short alpha = source[0]; 199 destination[0] = alpha; 200 destination[1] = (source[1] * alpha + 128) / 255; 201 destination[2] = (source[2] * alpha + 128) / 255; 202 destination[3] = (source[3] * alpha + 128) / 255; 203#endif 204 source += 4; 205 destination += 4; 206 } 207 } 208 209 gst_video_frame_unmap(&sourceFrame); 210 gst_video_frame_unmap(&destinationFrame); 211 gst_buffer_unref(buffer); 212 buffer = priv->buffer = newBuffer; 213 } 214#endif 215 216 // This should likely use a lower priority, but glib currently starves 217 // lower priority sources. 218 // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830. 219 gst_object_ref(sink); 220 priv->timeoutSource.schedule("[WebKit] webkitVideoSinkTimeoutCallback", std::function<void()>(std::bind(webkitVideoSinkTimeoutCallback, sink)), G_PRIORITY_DEFAULT, 221 [sink] { gst_object_unref(sink); }); 222 223 g_cond_wait(priv->dataCondition, priv->bufferMutex); 224 return GST_FLOW_OK; 225} 226 227static void webkitVideoSinkDispose(GObject* object) 228{ 229 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object); 230 WebKitVideoSinkPrivate* priv = sink->priv; 231 232 if (priv->dataCondition) { 233 g_cond_clear(priv->dataCondition); 234 delete priv->dataCondition; 235 priv->dataCondition = 0; 236 } 237 238 if (priv->bufferMutex) { 239 g_mutex_clear(priv->bufferMutex); 240 delete priv->bufferMutex; 241 priv->bufferMutex = 0; 242 } 243 244 G_OBJECT_CLASS(parent_class)->dispose(object); 245} 246 247static void webkitVideoSinkFinalize(GObject* object) 248{ 249 WEBKIT_VIDEO_SINK(object)->priv->~WebKitVideoSinkPrivate(); 250 G_OBJECT_CLASS(parent_class)->finalize(object); 251} 252 253static void webkitVideoSinkGetProperty(GObject* object, guint propertyId, GValue* value, GParamSpec* parameterSpec) 254{ 255 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object); 256 WebKitVideoSinkPrivate* priv = sink->priv; 257 258 switch (propertyId) { 259 case PROP_CAPS: { 260 GstCaps* caps = priv->currentCaps; 261 if (caps) 262 gst_caps_ref(caps); 263 g_value_take_boxed(value, caps); 264 break; 265 } 266 default: 267 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, parameterSpec); 268 } 269} 270 271static void unlockBufferMutex(WebKitVideoSinkPrivate* priv) 272{ 273 GMutexLocker lock(priv->bufferMutex); 274 275 if (priv->buffer) { 276 gst_buffer_unref(priv->buffer); 277 priv->buffer = 0; 278 } 279 280 priv->unlocked = true; 281 282 g_cond_signal(priv->dataCondition); 283} 284 285static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink) 286{ 287 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink); 288 289 unlockBufferMutex(sink->priv); 290 291 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE); 292} 293 294static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink) 295{ 296 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv; 297 298 { 299 GMutexLocker lock(priv->bufferMutex); 300 priv->unlocked = false; 301 } 302 303 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE); 304} 305 306static gboolean webkitVideoSinkStop(GstBaseSink* baseSink) 307{ 308 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv; 309 310 unlockBufferMutex(priv); 311 312 if (priv->currentCaps) { 313 gst_caps_unref(priv->currentCaps); 314 priv->currentCaps = 0; 315 } 316 317 return TRUE; 318} 319 320static gboolean webkitVideoSinkStart(GstBaseSink* baseSink) 321{ 322 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv; 323 324 GMutexLocker lock(priv->bufferMutex); 325 priv->unlocked = false; 326 return TRUE; 327} 328 329static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps) 330{ 331 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink); 332 WebKitVideoSinkPrivate* priv = sink->priv; 333 334 GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps); 335 336 GstVideoInfo videoInfo; 337 gst_video_info_init(&videoInfo); 338 if (!gst_video_info_from_caps(&videoInfo, caps)) { 339 GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps); 340 return FALSE; 341 } 342 343 priv->info = videoInfo; 344 gst_caps_replace(&priv->currentCaps, caps); 345 return TRUE; 346} 347 348static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query) 349{ 350 GstCaps* caps; 351 gst_query_parse_allocation(query, &caps, 0); 352 if (!caps) 353 return FALSE; 354 355 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink); 356 if (!gst_video_info_from_caps(&sink->priv->info, caps)) 357 return FALSE; 358 359 gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, 0); 360 gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, 0); 361#if GST_CHECK_VERSION(1, 1, 0) 362 gst_query_add_allocation_meta(query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, 0); 363#endif 364 return TRUE; 365} 366 367static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass) 368{ 369 GObjectClass* gobjectClass = G_OBJECT_CLASS(klass); 370 GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass); 371 GstElementClass* elementClass = GST_ELEMENT_CLASS(klass); 372 373 gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate)); 374 gst_element_class_set_metadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to WebKit", "Igalia, Alp Toker <alp@atoker.com>"); 375 376 g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate)); 377 378 gobjectClass->dispose = webkitVideoSinkDispose; 379 gobjectClass->finalize = webkitVideoSinkFinalize; 380 gobjectClass->get_property = webkitVideoSinkGetProperty; 381 382 baseSinkClass->unlock = webkitVideoSinkUnlock; 383 baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop; 384 baseSinkClass->render = webkitVideoSinkRender; 385 baseSinkClass->preroll = webkitVideoSinkRender; 386 baseSinkClass->stop = webkitVideoSinkStop; 387 baseSinkClass->start = webkitVideoSinkStart; 388 baseSinkClass->set_caps = webkitVideoSinkSetCaps; 389 baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation; 390 391 g_object_class_install_property(gobjectClass, PROP_CAPS, 392 g_param_spec_boxed("current-caps", "Current-Caps", "Current caps", GST_TYPE_CAPS, G_PARAM_READABLE)); 393 394 webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested", 395 G_TYPE_FROM_CLASS(klass), 396 static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION), 397 0, // Class offset 398 0, // Accumulator 399 0, // Accumulator data 400 g_cclosure_marshal_generic, 401 G_TYPE_NONE, // Return type 402 1, // Only one parameter 403 GST_TYPE_BUFFER); 404} 405 406 407GstElement* webkitVideoSinkNew() 408{ 409 return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0)); 410} 411 412#endif // ENABLE(VIDEO) && USE(GSTREAMER) 413