1/*
2 *  Copyright (C) 2007 OpenedHand
3 *  Copyright (C) 2007 Alp Toker <alp@atoker.com>
4 *  Copyright (C) 2009, 2010, 2011, 2012 Igalia S.L
5 *
6 *  This library is free software; you can redistribute it and/or
7 *  modify it under the terms of the GNU Lesser General Public
8 *  License as published by the Free Software Foundation; either
9 *  version 2 of the License, or (at your option) any later version.
10 *
11 *  This library is distributed in the hope that it will be useful,
12 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 *  Lesser General Public License for more details.
15 *
16 *  You should have received a copy of the GNU Lesser General Public
17 *  License along with this library; if not, write to the Free Software
18 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
19 */
20
21/*
22 *
23 * WebKitVideoSink is a GStreamer sink element that triggers
24 * repaints in the WebKit GStreamer media player for the
25 * current video buffer.
26 */
27
28#include "config.h"
29#include "VideoSinkGStreamer.h"
30
31#if ENABLE(VIDEO) && USE(GSTREAMER)
32#include "GRefPtrGStreamer.h"
33#include "GStreamerUtilities.h"
34#include "IntSize.h"
35#include <glib.h>
36#include <gst/gst.h>
37#include <gst/video/gstvideometa.h>
38#include <wtf/OwnPtr.h>
39#include <wtf/gobject/GMainLoopSource.h>
40#include <wtf/gobject/GMutexLocker.h>
41
42using namespace WebCore;
43
44// CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
45#if G_BYTE_ORDER == G_LITTLE_ENDIAN
46#define GST_CAPS_FORMAT "{ BGRx, BGRA }"
47#else
48#define GST_CAPS_FORMAT "{ xRGB, ARGB }"
49#endif
50#if GST_CHECK_VERSION(1, 1, 0)
51#define GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, GST_CAPS_FORMAT) ";"
52#else
53#define GST_FEATURED_CAPS
54#endif
55
56#define WEBKIT_VIDEO_SINK_PAD_CAPS GST_FEATURED_CAPS GST_VIDEO_CAPS_MAKE(GST_CAPS_FORMAT)
57
58static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS));
59
60
61GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug);
62#define GST_CAT_DEFAULT webkitVideoSinkDebug
63
64enum {
65    REPAINT_REQUESTED,
66    LAST_SIGNAL
67};
68
69enum {
70    PROP_0,
71    PROP_CAPS
72};
73
74static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, };
75
76struct _WebKitVideoSinkPrivate {
77    GstBuffer* buffer;
78    GMainLoopSource timeoutSource;
79    GMutex* bufferMutex;
80    GCond* dataCondition;
81
82    GstVideoInfo info;
83
84    GstCaps* currentCaps;
85
86    // If this is TRUE all processing should finish ASAP
87    // This is necessary because there could be a race between
88    // unlock() and render(), where unlock() wins, signals the
89    // GCond, then render() tries to render a frame although
90    // everything else isn't running anymore. This will lead
91    // to deadlocks because render() holds the stream lock.
92    //
93    // Protected by the buffer mutex
94    bool unlocked;
95};
96
97#define webkit_video_sink_parent_class parent_class
98G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink"));
99
100
101static void webkit_video_sink_init(WebKitVideoSink* sink)
102{
103    sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
104    new (sink->priv) WebKitVideoSinkPrivate();
105    sink->priv->dataCondition = new GCond;
106    g_cond_init(sink->priv->dataCondition);
107    sink->priv->bufferMutex = new GMutex;
108    g_mutex_init(sink->priv->bufferMutex);
109
110    gst_video_info_init(&sink->priv->info);
111}
112
113static void webkitVideoSinkTimeoutCallback(WebKitVideoSink* sink)
114{
115    WebKitVideoSinkPrivate* priv = sink->priv;
116
117    GMutexLocker lock(priv->bufferMutex);
118    GstBuffer* buffer = priv->buffer;
119    priv->buffer = 0;
120
121    if (!buffer || priv->unlocked || UNLIKELY(!GST_IS_BUFFER(buffer))) {
122        g_cond_signal(priv->dataCondition);
123        return;
124    }
125
126    g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, buffer);
127    gst_buffer_unref(buffer);
128    g_cond_signal(priv->dataCondition);
129}
130
131static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer)
132{
133    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
134    WebKitVideoSinkPrivate* priv = sink->priv;
135
136    GMutexLocker lock(priv->bufferMutex);
137
138    if (priv->unlocked)
139        return GST_FLOW_OK;
140
141    priv->buffer = gst_buffer_ref(buffer);
142
143    // The video info structure is valid only if the sink handled an allocation query.
144    GstVideoFormat format = GST_VIDEO_INFO_FORMAT(&priv->info);
145    if (format == GST_VIDEO_FORMAT_UNKNOWN) {
146        gst_buffer_unref(buffer);
147        return GST_FLOW_ERROR;
148    }
149
150#if !(USE(TEXTURE_MAPPER_GL) && !USE(COORDINATED_GRAPHICS))
151    // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
152    // Here we convert to Cairo's ARGB.
153    if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
154        // Because GstBaseSink::render() only owns the buffer reference in the
155        // method scope we can't use gst_buffer_make_writable() here. Also
156        // The buffer content should not be changed here because the same buffer
157        // could be passed multiple times to this method (in theory).
158
159        GstBuffer* newBuffer = WebCore::createGstBuffer(buffer);
160
161        // Check if allocation failed.
162        if (UNLIKELY(!newBuffer)) {
163            gst_buffer_unref(buffer);
164            return GST_FLOW_ERROR;
165        }
166
167        // We don't use Color::premultipliedARGBFromColor() here because
168        // one function call per video pixel is just too expensive:
169        // For 720p/PAL for example this means 1280*720*25=23040000
170        // function calls per second!
171        GstVideoFrame sourceFrame;
172        GstVideoFrame destinationFrame;
173
174        if (!gst_video_frame_map(&sourceFrame, &priv->info, buffer, GST_MAP_READ)) {
175            gst_buffer_unref(buffer);
176            gst_buffer_unref(newBuffer);
177            return GST_FLOW_ERROR;
178        }
179        if (!gst_video_frame_map(&destinationFrame, &priv->info, newBuffer, GST_MAP_WRITE)) {
180            gst_video_frame_unmap(&sourceFrame);
181            gst_buffer_unref(buffer);
182            gst_buffer_unref(newBuffer);
183            return GST_FLOW_ERROR;
184        }
185
186        const guint8* source = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&sourceFrame, 0));
187        guint8* destination = static_cast<guint8*>(GST_VIDEO_FRAME_PLANE_DATA(&destinationFrame, 0));
188
189        for (int x = 0; x < GST_VIDEO_FRAME_HEIGHT(&sourceFrame); x++) {
190            for (int y = 0; y < GST_VIDEO_FRAME_WIDTH(&sourceFrame); y++) {
191#if G_BYTE_ORDER == G_LITTLE_ENDIAN
192                unsigned short alpha = source[3];
193                destination[0] = (source[0] * alpha + 128) / 255;
194                destination[1] = (source[1] * alpha + 128) / 255;
195                destination[2] = (source[2] * alpha + 128) / 255;
196                destination[3] = alpha;
197#else
198                unsigned short alpha = source[0];
199                destination[0] = alpha;
200                destination[1] = (source[1] * alpha + 128) / 255;
201                destination[2] = (source[2] * alpha + 128) / 255;
202                destination[3] = (source[3] * alpha + 128) / 255;
203#endif
204                source += 4;
205                destination += 4;
206            }
207        }
208
209        gst_video_frame_unmap(&sourceFrame);
210        gst_video_frame_unmap(&destinationFrame);
211        gst_buffer_unref(buffer);
212        buffer = priv->buffer = newBuffer;
213    }
214#endif
215
216    // This should likely use a lower priority, but glib currently starves
217    // lower priority sources.
218    // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830.
219    gst_object_ref(sink);
220    priv->timeoutSource.schedule("[WebKit] webkitVideoSinkTimeoutCallback", std::function<void()>(std::bind(webkitVideoSinkTimeoutCallback, sink)), G_PRIORITY_DEFAULT,
221        [sink] { gst_object_unref(sink); });
222
223    g_cond_wait(priv->dataCondition, priv->bufferMutex);
224    return GST_FLOW_OK;
225}
226
227static void webkitVideoSinkDispose(GObject* object)
228{
229    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
230    WebKitVideoSinkPrivate* priv = sink->priv;
231
232    if (priv->dataCondition) {
233        g_cond_clear(priv->dataCondition);
234        delete priv->dataCondition;
235        priv->dataCondition = 0;
236    }
237
238    if (priv->bufferMutex) {
239        g_mutex_clear(priv->bufferMutex);
240        delete priv->bufferMutex;
241        priv->bufferMutex = 0;
242    }
243
244    G_OBJECT_CLASS(parent_class)->dispose(object);
245}
246
247static void webkitVideoSinkFinalize(GObject* object)
248{
249    WEBKIT_VIDEO_SINK(object)->priv->~WebKitVideoSinkPrivate();
250    G_OBJECT_CLASS(parent_class)->finalize(object);
251}
252
253static void webkitVideoSinkGetProperty(GObject* object, guint propertyId, GValue* value, GParamSpec* parameterSpec)
254{
255    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
256    WebKitVideoSinkPrivate* priv = sink->priv;
257
258    switch (propertyId) {
259    case PROP_CAPS: {
260        GstCaps* caps = priv->currentCaps;
261        if (caps)
262            gst_caps_ref(caps);
263        g_value_take_boxed(value, caps);
264        break;
265    }
266    default:
267        G_OBJECT_WARN_INVALID_PROPERTY_ID(object, propertyId, parameterSpec);
268    }
269}
270
271static void unlockBufferMutex(WebKitVideoSinkPrivate* priv)
272{
273    GMutexLocker lock(priv->bufferMutex);
274
275    if (priv->buffer) {
276        gst_buffer_unref(priv->buffer);
277        priv->buffer = 0;
278    }
279
280    priv->unlocked = true;
281
282    g_cond_signal(priv->dataCondition);
283}
284
285static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink)
286{
287    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
288
289    unlockBufferMutex(sink->priv);
290
291    return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE);
292}
293
294static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink)
295{
296    WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
297
298    {
299        GMutexLocker lock(priv->bufferMutex);
300        priv->unlocked = false;
301    }
302
303    return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE);
304}
305
306static gboolean webkitVideoSinkStop(GstBaseSink* baseSink)
307{
308    WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
309
310    unlockBufferMutex(priv);
311
312    if (priv->currentCaps) {
313        gst_caps_unref(priv->currentCaps);
314        priv->currentCaps = 0;
315    }
316
317    return TRUE;
318}
319
320static gboolean webkitVideoSinkStart(GstBaseSink* baseSink)
321{
322    WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
323
324    GMutexLocker lock(priv->bufferMutex);
325    priv->unlocked = false;
326    return TRUE;
327}
328
329static gboolean webkitVideoSinkSetCaps(GstBaseSink* baseSink, GstCaps* caps)
330{
331    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
332    WebKitVideoSinkPrivate* priv = sink->priv;
333
334    GST_DEBUG_OBJECT(sink, "Current caps %" GST_PTR_FORMAT ", setting caps %" GST_PTR_FORMAT, priv->currentCaps, caps);
335
336    GstVideoInfo videoInfo;
337    gst_video_info_init(&videoInfo);
338    if (!gst_video_info_from_caps(&videoInfo, caps)) {
339        GST_ERROR_OBJECT(sink, "Invalid caps %" GST_PTR_FORMAT, caps);
340        return FALSE;
341    }
342
343    priv->info = videoInfo;
344    gst_caps_replace(&priv->currentCaps, caps);
345    return TRUE;
346}
347
348static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query)
349{
350    GstCaps* caps;
351    gst_query_parse_allocation(query, &caps, 0);
352    if (!caps)
353        return FALSE;
354
355    WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
356    if (!gst_video_info_from_caps(&sink->priv->info, caps))
357        return FALSE;
358
359    gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, 0);
360    gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, 0);
361#if GST_CHECK_VERSION(1, 1, 0)
362    gst_query_add_allocation_meta(query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, 0);
363#endif
364    return TRUE;
365}
366
367static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
368{
369    GObjectClass* gobjectClass = G_OBJECT_CLASS(klass);
370    GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass);
371    GstElementClass* elementClass = GST_ELEMENT_CLASS(klass);
372
373    gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate));
374    gst_element_class_set_metadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to WebKit", "Igalia, Alp Toker <alp@atoker.com>");
375
376    g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
377
378    gobjectClass->dispose = webkitVideoSinkDispose;
379    gobjectClass->finalize = webkitVideoSinkFinalize;
380    gobjectClass->get_property = webkitVideoSinkGetProperty;
381
382    baseSinkClass->unlock = webkitVideoSinkUnlock;
383    baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop;
384    baseSinkClass->render = webkitVideoSinkRender;
385    baseSinkClass->preroll = webkitVideoSinkRender;
386    baseSinkClass->stop = webkitVideoSinkStop;
387    baseSinkClass->start = webkitVideoSinkStart;
388    baseSinkClass->set_caps = webkitVideoSinkSetCaps;
389    baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation;
390
391    g_object_class_install_property(gobjectClass, PROP_CAPS,
392        g_param_spec_boxed("current-caps", "Current-Caps", "Current caps", GST_TYPE_CAPS, G_PARAM_READABLE));
393
394    webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
395            G_TYPE_FROM_CLASS(klass),
396            static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
397            0, // Class offset
398            0, // Accumulator
399            0, // Accumulator data
400            g_cclosure_marshal_generic,
401            G_TYPE_NONE, // Return type
402            1, // Only one parameter
403            GST_TYPE_BUFFER);
404}
405
406
407GstElement* webkitVideoSinkNew()
408{
409    return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
410}
411
412#endif // ENABLE(VIDEO) && USE(GSTREAMER)
413