GStreamer backend changes for media probing API.

QGstreamerPlayerSession: Using GStreamer buffer probes
to access media data.

Change-Id: Ibc056283fdedaebba90456cc4e86ab63eae5f5f7
Reviewed-by: Michael Goddard <michael.goddard@nokia.com>

src/gsttools/qgstutils.cpp

@@ -45,6 +45,7 @@
 #include <QtCore/qbytearray.h>
 #include <QtCore/qvariant.h>
 #include <QtCore/qsize.h>
+#include <qaudioformat.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -143,7 +144,7 @@ QSize QGstUtils::capsResolution(const GstCaps *caps)
 
 /*!
   Returns aspect ratio corrected resolution of \a caps.
-  If caps doesn't have a valid size, and ampty QSize is returned.
+  If caps doesn't have a valid size, an empty QSize is returned.
 */
 QSize QGstUtils::capsCorrectedResolution(const GstCaps *caps)
 {
@@ -166,4 +167,88 @@ QSize QGstUtils::capsCorrectedResolution(const GstCaps *caps)
     return size;
 }
 
+/*!
+ *Returns audio format for caps.
+  If caps doesn't have a valid audio format, an empty QAudioFormat is returned.
+*/
+
+QAudioFormat QGstUtils::audioFormatForCaps(const GstCaps *caps)
+{
+    const GstStructure *structure = gst_caps_get_structure(caps, 0);
+
+    QAudioFormat format;
+
+    if (qstrcmp(gst_structure_get_name(structure), "audio/x-raw-int") == 0) {
+
+        format.setCodec("audio/pcm");
+
+        int endianness = 0;
+        gst_structure_get_int(structure, "endianness", &endianness);
+        if (endianness == 1234)
+            format.setByteOrder(QAudioFormat::LittleEndian);
+        else if (endianness == 4321)
+            format.setByteOrder(QAudioFormat::BigEndian);
+
+        gboolean isSigned = FALSE;
+        gst_structure_get_boolean(structure, "signed", &isSigned);
+        if (isSigned)
+            format.setSampleType(QAudioFormat::SignedInt);
+        else
+            format.setSampleType(QAudioFormat::UnSignedInt);
+
+        // Number of bits allocated per sample.
+        int width = 0;
+        gst_structure_get_int(structure, "width", &width);
+
+        // The number of bits used per sample. This must be less than or equal to the width.
+        int depth = 0;
+        gst_structure_get_int(structure, "depth", &depth);
+
+        if (width != depth) {
+            // Unsupported sample layout.
+            return QAudioFormat();
+        }
+        format.setSampleSize(width);
+
+        int rate = 0;
+        gst_structure_get_int(structure, "rate", &rate);
+        format.setSampleRate(rate);
+
+        int channels = 0;
+        gst_structure_get_int(structure, "channels", &channels);
+        format.setChannelCount(channels);
+
+    } else if (qstrcmp(gst_structure_get_name(structure), "audio/x-raw-float") == 0) {
+
+        format.setCodec("audio/pcm");
+
+        int endianness = 0;
+        gst_structure_get_int(structure, "endianness", &endianness);
+        if (endianness == 1234)
+            format.setByteOrder(QAudioFormat::LittleEndian);
+        else if (endianness == 4321)
+            format.setByteOrder(QAudioFormat::BigEndian);
+
+        format.setSampleType(QAudioFormat::Float);
+
+        int width = 0;
+        gst_structure_get_int(structure, "width", &width);
+
+        format.setSampleSize(width);
+
+        int rate = 0;
+        gst_structure_get_int(structure, "rate", &rate);
+        format.setSampleRate(rate);
+
+        int channels = 0;
+        gst_structure_get_int(structure, "channels", &channels);
+        format.setChannelCount(channels);
+
+    } else {
+        return QAudioFormat();
+    }
+
+    return format;
+}
+
 QT_END_NAMESPACE

src/gsttools/qvideosurfacegstsink.cpp

@@ -199,16 +199,7 @@ GstFlowReturn QVideoSurfaceGstDelegate::render(GstBuffer *buffer)
             m_format.frameSize(),
             m_format.pixelFormat());
 
-    qint64 startTime = GST_BUFFER_TIMESTAMP(buffer);
-
-    if (startTime >= 0) {
-        m_frame.setStartTime(startTime/G_GINT64_CONSTANT (1000000));
-
-        qint64 duration = GST_BUFFER_DURATION(buffer);
-
-        if (duration >= 0)
-            m_frame.setEndTime((startTime + duration)/G_GINT64_CONSTANT (1000000));
-    }
+    QVideoSurfaceGstSink::setFrameTimeStamps(&m_frame, buffer);
 
     m_renderReturn = GST_FLOW_OK;
 
@@ -685,6 +676,18 @@ QVideoSurfaceFormat QVideoSurfaceGstSink::formatForCaps(GstCaps *caps, int *byte
     return QVideoSurfaceFormat();
 }
 
+void QVideoSurfaceGstSink::setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer)
+{
+    qint64 startTime = GST_BUFFER_TIMESTAMP(buffer);
+    if (startTime >= 0) {
+        frame->setStartTime(startTime/G_GINT64_CONSTANT (1000000));
+
+        qint64 duration = GST_BUFFER_DURATION(buffer);
+        if (duration >= 0)
+            frame->setEndTime((startTime + duration)/G_GINT64_CONSTANT (1000000));
+    }
+}
+
 void QVideoSurfaceGstSink::handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d)
 {
     Q_UNUSED(o);

src/multimedia/gsttools_headers/qgstutils_p.h

@@ -55,6 +55,7 @@
 
 #include <QtCore/qmap.h>
 #include <gst/gst.h>
+#include <qaudioformat.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -67,6 +68,7 @@ namespace QGstUtils {
 
     QSize capsResolution(const GstCaps *caps);
     QSize capsCorrectedResolution(const GstCaps *caps);
+    QAudioFormat audioFormatForCaps(const GstCaps *caps);
 }
 
 QT_END_NAMESPACE

src/multimedia/gsttools_headers/qvideosurfacegstsink_p.h

@@ -135,6 +135,7 @@ public:
 
     static QVideoSurfaceGstSink *createSink(QAbstractVideoSurface *surface);
     static QVideoSurfaceFormat formatForCaps(GstCaps *caps, int *bytesPerLine = 0);
+    static void setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer);
 
     static void handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d);
 

src/plugins/gstreamer/gstreamer.pro

@@ -63,6 +63,8 @@ HEADERS += \
     qgstreamervideoinputdevicecontrol.h \
     gstvideoconnector.h \
     qgstcodecsinfo.h \
+    qgstreamervideoprobecontrol.h \
+    qgstreameraudioprobecontrol.h \
 
 SOURCES += \
     qgstreamervideorendererinterface.cpp \
@@ -72,6 +74,8 @@ SOURCES += \
     qgstreamervideoinputdevicecontrol.cpp \
     qgstcodecsinfo.cpp \
     gstvideoconnector.c \
+    qgstreamervideoprobecontrol.cpp \
+    qgstreameraudioprobecontrol.cpp \
 
 
 contains(config_test_xvideo, yes):!isEmpty(QT.widgets.name): {

src/plugins/gstreamer/mediaplayer/qgstreamerplayerservice.cpp

@@ -64,6 +64,8 @@
 #endif
 
 #include "qgstreamerstreamscontrol.h"
+#include "qgstreameraudioprobecontrol.h"
+#include "qgstreamervideoprobecontrol.h"
 
 #include <qmediaplaylistnavigator.h>
 #include <qmediaplaylist.h>
@@ -115,6 +117,24 @@ QMediaControl *QGstreamerPlayerService::requestControl(const char *name)
     if (qstrcmp(name,QMediaStreamsControl_iid) == 0)
         return m_streamsControl;
 
+    if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) {
+        if (m_session) {
+            QGstreamerVideoProbeControl *probe = new QGstreamerVideoProbeControl(this);
+            m_session->addProbe(probe);
+            return probe;
+        }
+        return 0;
+    }
+
+    if (qstrcmp(name,QMediaAudioProbeControl_iid) == 0) {
+        if (m_session) {
+            QGstreamerAudioProbeControl *probe = new QGstreamerAudioProbeControl(this);
+            m_session->addProbe(probe);
+            return probe;
+        }
+        return 0;
+    }
+
     if (!m_videoOutput) {
         if (qstrcmp(name, QVideoRendererControl_iid) == 0)
             m_videoOutput = m_videoRenderer;
@@ -140,6 +160,22 @@ void QGstreamerPlayerService::releaseControl(QMediaControl *control)
         m_videoOutput = 0;
         m_control->setVideoOutput(0);
     }
+
+    QGstreamerVideoProbeControl* videoProbe = qobject_cast<QGstreamerVideoProbeControl*>(control);
+    if (videoProbe) {
+        if (m_session)
+            m_session->removeProbe(videoProbe);
+        delete videoProbe;
+        return;
+    }
+
+    QGstreamerAudioProbeControl* audioProbe = qobject_cast<QGstreamerAudioProbeControl*>(control);
+    if (audioProbe) {
+        if (m_session)
+            m_session->removeProbe(audioProbe);
+        delete audioProbe;
+        return;
+    }
 }
 
 QT_END_NAMESPACE

src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.cpp

@@ -42,6 +42,8 @@
 #include "qgstreamerplayersession.h"
 #include <private/qgstreamerbushelper_p.h>
 
+#include "qgstreameraudioprobecontrol.h"
+#include "qgstreamervideoprobecontrol.h"
 #include "qgstreamervideorendererinterface.h"
 #include "gstvideoconnector.h"
 #include <private/qgstutils_p.h>
@@ -89,6 +91,7 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
      m_videoSink(0),
      m_pendingVideoSink(0),
      m_nullVideoSink(0),
+     m_audioSink(0),
      m_bus(0),
      m_videoOutput(0),
      m_renderer(0),
@@ -96,6 +99,8 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
 #if defined(HAVE_GST_APPSRC)
      m_appSrc(0),
 #endif
+     m_videoBufferProbeId(-1),
+     m_audioBufferProbeId(-1),
      m_volume(100),
      m_playbackRate(1.0),
      m_muted(false),
@@ -128,6 +133,12 @@ QGstreamerPlayerSession::QGstreamerPlayerSession(QObject *parent)
         flags |= GST_PLAY_FLAG_NATIVE_VIDEO;
 #endif
         g_object_set(G_OBJECT(m_playbin), "flags", flags, NULL);
+
+        m_audioSink = gst_element_factory_make("autoaudiosink", "audiosink");
+        if (m_audioSink) {
+            g_object_set(G_OBJECT(m_playbin), "audio-sink", m_audioSink, NULL);
+            addAudioBufferProbe();
+        }
     } else {
         m_usePlaybin2 = false;
         m_playbin = gst_element_factory_make("playbin", NULL);
@@ -181,6 +192,9 @@ QGstreamerPlayerSession::~QGstreamerPlayerSession()
     if (m_playbin) {
         stop();
 
+        removeVideoBufferProbe();
+        removeAudioBufferProbe();
+
         delete m_busHelper;
         gst_object_unref(GST_OBJECT(m_bus));
         gst_object_unref(GST_OBJECT(m_playbin));
@@ -511,6 +525,7 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
         qDebug() << "The pipeline has not started yet, pending state:" << m_pendingState;
 #endif
         //the pipeline has not started yet
+        flushVideoProbes();
         m_pendingVideoSink = 0;
         gst_element_set_state(m_videoSink, GST_STATE_NULL);
         gst_element_set_state(m_playbin, GST_STATE_NULL);
@@ -522,6 +537,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
             gst_element_unlink(m_videoIdentity, m_videoSink);
         }
 
+        removeVideoBufferProbe();
+
         gst_bin_remove(GST_BIN(m_videoOutputBin), m_videoSink);
 
         m_videoSink = videoSink;
@@ -544,6 +561,8 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
             g_object_set(G_OBJECT(m_videoSink), "show-preroll-frame", value, NULL);
         }
 
+        addVideoBufferProbe();
+
         switch (m_pendingState) {
         case QMediaPlayer::PausedState:
             gst_element_set_state(m_playbin, GST_STATE_PAUSED);
@@ -554,6 +573,9 @@ void QGstreamerPlayerSession::setVideoRenderer(QObject *videoOutput)
         default:
             break;
         }
+
+        resumeVideoProbes();
+
     } else {
         if (m_pendingVideoSink) {
 #ifdef DEBUG_PLAYBIN
@@ -630,6 +652,8 @@ void QGstreamerPlayerSession::finishVideoOutputChange()
         gst_element_unlink(m_videoIdentity, m_videoSink);
     }
 
+    removeVideoBufferProbe();
+
     gst_bin_remove(GST_BIN(m_videoOutputBin), m_videoSink);
 
     m_videoSink = m_pendingVideoSink;
@@ -637,6 +661,8 @@ void QGstreamerPlayerSession::finishVideoOutputChange()
 
     gst_bin_add(GST_BIN(m_videoOutputBin), m_videoSink);
 
+    addVideoBufferProbe();
+
     m_usingColorspaceElement = false;
     bool linked = gst_element_link(m_videoIdentity, m_videoSink);
     if (!linked) {
@@ -682,10 +708,16 @@ void QGstreamerPlayerSession::finishVideoOutputChange()
 
     gst_element_set_state(m_videoSink, state);
 
+    if (state == GST_STATE_NULL)
+        flushVideoProbes();
+
     // Set state change that was deferred due the video output
     // change being pending
     gst_element_set_state(m_playbin, state);
 
+    if (state != GST_STATE_NULL)
+        resumeVideoProbes();
+
     //don't have to wait here, it will unblock eventually
     if (gst_pad_is_blocked(srcPad))
         gst_pad_set_blocked_async(srcPad, false, &block_pad_cb, 0);
@@ -766,8 +798,10 @@ bool QGstreamerPlayerSession::play()
             m_pendingState = m_state = QMediaPlayer::StoppedState;
 
             emit stateChanged(m_state);
-        } else
+        } else {
+            resumeVideoProbes();
             return true;
+        }
     }
 
     return false;
@@ -789,6 +823,7 @@ bool QGstreamerPlayerSession::pause()
 
             emit stateChanged(m_state);
         } else {
+            resumeVideoProbes();
             return true;
         }
     }
@@ -803,9 +838,11 @@ void QGstreamerPlayerSession::stop()
 #endif
     m_everPlayed = false;
     if (m_playbin) {
+
         if (m_renderer)
             m_renderer->stopRenderer();
 
+        flushVideoProbes();
         gst_element_set_state(m_playbin, GST_STATE_NULL);
 
         m_lastPosition = 0;
@@ -1621,4 +1658,140 @@ void QGstreamerPlayerSession::showPrerollFrames(bool enabled)
     }
 }
 
+void QGstreamerPlayerSession::addProbe(QGstreamerVideoProbeControl* probe)
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+
+    if (m_videoProbes.contains(probe))
+        return;
+
+    m_videoProbes.append(probe);
+}
+
+void QGstreamerPlayerSession::removeProbe(QGstreamerVideoProbeControl* probe)
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    m_videoProbes.removeOne(probe);
+    // Do not emit flush signal in this case.
+    // Assume user releases any outstanding references to video frames.
+}
+
+gboolean QGstreamerPlayerSession::padVideoBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data)
+{
+    Q_UNUSED(pad);
+
+    QGstreamerPlayerSession *session = reinterpret_cast<QGstreamerPlayerSession*>(user_data);
+    QMutexLocker locker(&session->m_videoProbeMutex);
+
+    if (session->m_videoProbes.isEmpty())
+        return TRUE;
+
+    foreach (QGstreamerVideoProbeControl* probe, session->m_videoProbes)
+        probe->bufferProbed(buffer);
+
+    return TRUE;
+}
+
+void QGstreamerPlayerSession::addProbe(QGstreamerAudioProbeControl* probe)
+{
+    QMutexLocker locker(&m_audioProbeMutex);
+
+    if (m_audioProbes.contains(probe))
+        return;
+
+    m_audioProbes.append(probe);
+}
+
+void QGstreamerPlayerSession::removeProbe(QGstreamerAudioProbeControl* probe)
+{
+    QMutexLocker locker(&m_audioProbeMutex);
+    m_audioProbes.removeOne(probe);
+}
+
+gboolean QGstreamerPlayerSession::padAudioBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data)
+{
+    Q_UNUSED(pad);
+
+    QGstreamerPlayerSession *session = reinterpret_cast<QGstreamerPlayerSession*>(user_data);
+    QMutexLocker locker(&session->m_audioProbeMutex);
+
+    if (session->m_audioProbes.isEmpty())
+        return TRUE;
+
+    foreach (QGstreamerAudioProbeControl* probe, session->m_audioProbes)
+        probe->bufferProbed(buffer);
+
+    return TRUE;
+}
+
+void QGstreamerPlayerSession::removeVideoBufferProbe()
+{
+    if (m_videoBufferProbeId == -1)
+        return;
+
+    if (!m_videoSink) {
+        m_videoBufferProbeId = -1;
+        return;
+    }
+
+    GstPad *pad = gst_element_get_static_pad(m_videoSink, "sink");
+    if (pad)
+        gst_pad_remove_buffer_probe(pad, m_videoBufferProbeId);
+
+    m_videoBufferProbeId = -1;
+}
+
+void QGstreamerPlayerSession::addVideoBufferProbe()
+{
+    Q_ASSERT(m_videoBufferProbeId == -1);
+    if (!m_videoSink)
+        return;
+
+    GstPad *pad = gst_element_get_static_pad(m_videoSink, "sink");
+    if (pad)
+        m_videoBufferProbeId = gst_pad_add_buffer_probe(pad, G_CALLBACK(padVideoBufferProbe), this);
+}
+
+void QGstreamerPlayerSession::removeAudioBufferProbe()
+{
+    if (m_audioBufferProbeId == -1)
+        return;
+
+    if (!m_audioSink) {
+        m_audioBufferProbeId = -1;
+        return;
+    }
+
+    GstPad *pad = gst_element_get_static_pad(m_audioSink, "sink");
+    if (pad)
+        gst_pad_remove_buffer_probe(pad, m_audioBufferProbeId);
+
+    m_audioBufferProbeId = -1;
+}
+
+void QGstreamerPlayerSession::addAudioBufferProbe()
+{
+    Q_ASSERT(m_audioBufferProbeId == -1);
+    if (!m_audioSink)
+        return;
+
+    GstPad *pad = gst_element_get_static_pad(m_audioSink, "sink");
+    if (pad)
+        m_audioBufferProbeId = gst_pad_add_buffer_probe(pad, G_CALLBACK(padAudioBufferProbe), this);
+}
+
+void QGstreamerPlayerSession::flushVideoProbes()
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    foreach (QGstreamerVideoProbeControl* probe, m_videoProbes)
+        probe->startFlushing();
+}
+
+void QGstreamerPlayerSession::resumeVideoProbes()
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    foreach (QGstreamerVideoProbeControl* probe, m_videoProbes)
+        probe->stopFlushing();
+}
+
 QT_END_NAMESPACE

src/plugins/gstreamer/mediaplayer/qgstreamerplayersession.h

@@ -43,11 +43,13 @@
 #define QGSTREAMERPLAYERSESSION_H
 
 #include <QObject>
+#include <QtCore/qmutex.h>
 #include <QtNetwork/qnetworkrequest.h>
 #include "qgstreamerplayercontrol.h"
 #include <private/qgstreamerbushelper_p.h>
 #include <qmediaplayer.h>
 #include <qmediastreamscontrol.h>
+#include <qaudioformat.h>
 
 #if defined(HAVE_GST_APPSRC)
 #include "qgstappsrc.h"
@@ -61,6 +63,8 @@ class QGstreamerBusHelper;
 class QGstreamerMessage;
 
 class QGstreamerVideoRendererInterface;
+class QGstreamerVideoProbeControl;
+class QGstreamerAudioProbeControl;
 
 class QGstreamerPlayerSession : public QObject,
                                 public QGstreamerBusMessageFilter
@@ -119,6 +123,14 @@ public:
 
     bool isLiveSource() const;
 
+    void addProbe(QGstreamerVideoProbeControl* probe);
+    void removeProbe(QGstreamerVideoProbeControl* probe);
+    static gboolean padVideoBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data);
+
+    void addProbe(QGstreamerAudioProbeControl* probe);
+    void removeProbe(QGstreamerAudioProbeControl* probe);
+    static gboolean padAudioBufferProbe(GstPad *pad, GstBuffer *buffer, gpointer user_data);
+
 public slots:
     void loadFromUri(const QNetworkRequest &url);
     void loadFromStream(const QNetworkRequest &url, QIODevice *stream);
@@ -169,6 +181,13 @@ private:
     static void handleElementAdded(GstBin *bin, GstElement *element, QGstreamerPlayerSession *session);
     void processInvalidMedia(QMediaPlayer::Error errorCode, const QString& errorString);
 
+    void removeVideoBufferProbe();
+    void addVideoBufferProbe();
+    void removeAudioBufferProbe();
+    void addAudioBufferProbe();
+    void flushVideoProbes();
+    void resumeVideoProbes();
+
     QNetworkRequest m_request;
     QMediaPlayer::State m_state;
     QMediaPlayer::State m_pendingState;
@@ -184,6 +203,8 @@ private:
     GstElement* m_pendingVideoSink;
     GstElement* m_nullVideoSink;
 
+    GstElement* m_audioSink;
+
     GstBus* m_bus;
     QObject *m_videoOutput;
     QGstreamerVideoRendererInterface *m_renderer;
@@ -199,6 +220,13 @@ private:
     QList<QMediaStreamsControl::StreamType> m_streamTypes;
     QMap<QMediaStreamsControl::StreamType, int> m_playbin2StreamOffset;
 
+    QList<QGstreamerVideoProbeControl*> m_videoProbes;
+    QMutex m_videoProbeMutex;
+    int m_videoBufferProbeId;
+
+    QList<QGstreamerAudioProbeControl*> m_audioProbes;
+    QMutex m_audioProbeMutex;
+    int m_audioBufferProbeId;
 
     int m_volume;
     qreal m_playbackRate;

src/plugins/gstreamer/qgstreameraudioprobecontrol.cpp

@@ -0,0 +1,86 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qgstreameraudioprobecontrol.h"
+#include <private/qgstutils_p.h>
+
+QGstreamerAudioProbeControl::QGstreamerAudioProbeControl(QObject *parent)
+    : QMediaAudioProbeControl(parent)
+{
+
+}
+
+QGstreamerAudioProbeControl::~QGstreamerAudioProbeControl()
+{
+
+}
+
+void QGstreamerAudioProbeControl::bufferProbed(GstBuffer* buffer)
+{
+    GstCaps* caps = gst_buffer_get_caps(buffer);
+    if (!caps)
+        return;
+
+    QAudioFormat format = QGstUtils::audioFormatForCaps(caps);
+    gst_caps_unref(caps);
+    if (!format.isValid())
+        return;
+
+    QAudioBuffer audioBuffer = QAudioBuffer(QByteArray((const char*)buffer->data, buffer->size), format);
+
+    {
+        QMutexLocker locker(&m_bufferMutex);
+        m_pendingBuffer = audioBuffer;
+        QMetaObject::invokeMethod(this, "bufferProbed", Qt::QueuedConnection);
+    }
+}
+
+void QGstreamerAudioProbeControl::bufferProbed()
+{
+    QAudioBuffer audioBuffer;
+    {
+        QMutexLocker locker(&m_bufferMutex);
+        if (!m_pendingBuffer.isValid())
+            return;
+        audioBuffer = m_pendingBuffer;
+    }
+    emit audioBufferProbed(audioBuffer);
+}

src/plugins/gstreamer/qgstreameraudioprobecontrol.h

@@ -0,0 +1,71 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QGSTREAMERAUDIOPROBECONTROL_H
+#define QGSTREAMERAUDIOPROBECONTROL_H
+
+#include <gst/gst.h>
+#include <qmediaaudioprobecontrol.h>
+#include <QtCore/qmutex.h>
+#include "qaudiobuffer.h"
+
+QT_BEGIN_NAMESPACE
+
+class QGstreamerAudioProbeControl : public QMediaAudioProbeControl
+{
+    Q_OBJECT
+public:
+    explicit QGstreamerAudioProbeControl(QObject *parent);
+    virtual ~QGstreamerAudioProbeControl();
+
+    void bufferProbed(GstBuffer* buffer);
+
+private slots:
+    void bufferProbed();
+
+private:
+    QAudioBuffer m_pendingBuffer;
+    QMutex m_bufferMutex;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGSTREAMERAUDIOPROBECONTROL_H

src/plugins/gstreamer/qgstreamervideoprobecontrol.cpp

@@ -0,0 +1,117 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qgstreamervideoprobecontrol.h"
+#include <private/qvideosurfacegstsink_p.h>
+#include <private/qgstvideobuffer_p.h>
+
+QGstreamerVideoProbeControl::QGstreamerVideoProbeControl(QObject *parent)
+    : QMediaVideoProbeControl(parent)
+    , m_flushing(false)
+    , m_frameProbed(false)
+{
+
+}
+
+QGstreamerVideoProbeControl::~QGstreamerVideoProbeControl()
+{
+
+}
+
+void QGstreamerVideoProbeControl::startFlushing()
+{
+    m_flushing = true;
+
+    {
+        QMutexLocker locker(&m_frameMutex);
+        m_pendingFrame = QVideoFrame();
+    }
+
+    // only emit flush if at least one frame was probed
+    if (m_frameProbed)
+        emit flush();
+}
+
+void QGstreamerVideoProbeControl::stopFlushing()
+{
+    m_flushing = false;
+}
+
+void QGstreamerVideoProbeControl::bufferProbed(GstBuffer* buffer)
+{
+    if (m_flushing)
+        return;
+
+    GstCaps* caps = gst_buffer_get_caps(buffer);
+    if (!caps)
+        return;
+
+    int bytesPerLine = 0;
+    QVideoSurfaceFormat format = QVideoSurfaceGstSink::formatForCaps(caps, &bytesPerLine);
+    gst_caps_unref(caps);
+    if (!format.isValid() || !bytesPerLine)
+        return;
+
+    QVideoFrame frame = QVideoFrame(new QGstVideoBuffer(buffer, bytesPerLine),
+                                    format.frameSize(), format.pixelFormat());
+
+    QVideoSurfaceGstSink::setFrameTimeStamps(&frame, buffer);
+
+    m_frameProbed = true;
+
+    {
+        QMutexLocker locker(&m_frameMutex);
+        m_pendingFrame = frame;
+        QMetaObject::invokeMethod(this, "frameProbed", Qt::QueuedConnection);
+    }
+}
+
+void QGstreamerVideoProbeControl::frameProbed()
+{
+    QVideoFrame frame;
+    {
+        QMutexLocker locker(&m_frameMutex);
+        if (!m_pendingFrame.isValid())
+            return;
+        frame = m_pendingFrame;
+    }
+    emit videoFrameProbed(frame);
+}

src/plugins/gstreamer/qgstreamervideoprobecontrol.h

@@ -0,0 +1,75 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QGSTREAMERVIDEOPROBECONTROL_H
+#define QGSTREAMERVIDEOPROBECONTROL_H
+
+#include <gst/gst.h>
+#include <qmediavideoprobecontrol.h>
+#include <QtCore/qmutex.h>
+#include "qvideoframe.h"
+
+QT_BEGIN_NAMESPACE
+
+class QGstreamerVideoProbeControl : public QMediaVideoProbeControl
+{
+    Q_OBJECT
+public:
+    explicit QGstreamerVideoProbeControl(QObject *parent);
+    virtual ~QGstreamerVideoProbeControl();
+
+    void bufferProbed(GstBuffer* buffer);
+    void startFlushing();
+    void stopFlushing();
+
+private slots:
+    void frameProbed();
+
+private:
+    bool m_flushing;
+    bool m_frameProbed; // true if at least one frame was probed
+    QVideoFrame m_pendingFrame;
+    QMutex m_frameMutex;
+};
+
+QT_END_NAMESPACE
+
+#endif // QGSTREAMERVIDEOPROBECONTROL_H

tests/auto/integration/qmediaplayerbackend/tst_qmediaplayerbackend.cpp

@@ -44,6 +44,8 @@
 #include <qabstractvideosurface.h>
 #include "qmediaservice.h"
 #include "qmediaplayer.h"
+#include "qaudioprobe.h"
+#include "qvideoprobe.h"
 
 //TESTED_COMPONENT=src/multimedia
 
@@ -78,6 +80,7 @@ private slots:
     void volumeAcrossFiles_data();
     void volumeAcrossFiles();
     void seekPauseSeek();
+    void probes();
 
 private:
     //one second local wav file
@@ -92,7 +95,7 @@ class TestVideoSurface : public QAbstractVideoSurface
 {
     Q_OBJECT
 public:
-    explicit TestVideoSurface() { }
+    TestVideoSurface() { }
 
     //video surface
     QList<QVideoFrame::PixelFormat> supportedPixelFormats(
@@ -102,12 +105,26 @@ public:
     void stop();
     bool present(const QVideoFrame &frame);
 
-    QList<QVideoFrame>& frameList() { return m_frameList; }
-
-private:
     QList<QVideoFrame> m_frameList;
 };
 
+class ProbeDataHandler : public QObject
+{
+    Q_OBJECT
+
+public:
+    ProbeDataHandler() : isVideoFlushCalled(false) { }
+
+    QList<QVideoFrame> m_frameList;
+    QList<QAudioBuffer> m_bufferList;
+    bool isVideoFlushCalled;
+
+public slots:
+    void processFrame(const QVideoFrame&);
+    void processBuffer(const QAudioBuffer&);
+    void flushVideo();
+    void flushAudio();
+};
 
 void tst_QMediaPlayerBackend::init()
 {
@@ -453,21 +470,21 @@ void tst_QMediaPlayerBackend::seekPauseSeek()
 
     player.setMedia(QUrl::fromLocalFile(videoFile.absoluteFilePath()));
     QCOMPARE(player.state(), QMediaPlayer::StoppedState);
-    QVERIFY(surface->frameList().isEmpty()); // frame must not appear until we call pause() or play()
+    QVERIFY(surface->m_frameList.isEmpty()); // frame must not appear until we call pause() or play()
 
     positionSpy.clear();
     player.setPosition((qint64)7000);
     QTRY_VERIFY(!positionSpy.isEmpty() && qAbs(player.position() - (qint64)7000) < (qint64)500);
     QCOMPARE(player.state(), QMediaPlayer::StoppedState);
     QTest::qWait(250); // wait a bit to ensure the frame is not rendered
-    QVERIFY(surface->frameList().isEmpty()); // still no frame, we must call pause() or play() to see a frame
+    QVERIFY(surface->m_frameList.isEmpty()); // still no frame, we must call pause() or play() to see a frame
 
     player.pause();
     QTRY_COMPARE(player.state(), QMediaPlayer::PausedState); // it might take some time for the operation to be completed
-    QTRY_COMPARE(surface->frameList().size(), 1); // we must see a frame at position 7000 here
+    QTRY_COMPARE(surface->m_frameList.size(), 1); // we must see a frame at position 7000 here
 
     {
-        QVideoFrame frame = surface->frameList().back();
+        QVideoFrame frame = surface->m_frameList.back();
         QVERIFY(qAbs(frame.startTime() - (qint64)7000) < (qint64)500);
         QCOMPARE(frame.width(), 160);
         QCOMPARE(frame.height(), 120);
@@ -486,10 +503,10 @@ void tst_QMediaPlayerBackend::seekPauseSeek()
     player.setPosition((qint64)12000);
     QTRY_VERIFY(!positionSpy.isEmpty() && qAbs(player.position() - (qint64)12000) < (qint64)500);
     QCOMPARE(player.state(), QMediaPlayer::PausedState);
-    QCOMPARE(surface->frameList().size(), 2);
+    QCOMPARE(surface->m_frameList.size(), 2);
 
     {
-        QVideoFrame frame = surface->frameList().back();
+        QVideoFrame frame = surface->m_frameList.back();
         QVERIFY(qAbs(frame.startTime() - (qint64)12000) < (qint64)500);
         QCOMPARE(frame.width(), 160);
         QCOMPARE(frame.height(), 120);
@@ -504,6 +521,41 @@ void tst_QMediaPlayerBackend::seekPauseSeek()
     }
 }
 
+void tst_QMediaPlayerBackend::probes()
+{
+    QMediaPlayer *player = new QMediaPlayer;
+
+    TestVideoSurface *surface = new TestVideoSurface;
+    player->setVideoOutput(surface);
+
+    QVideoProbe *videoProbe = new QVideoProbe;
+    QAudioProbe *audioProbe = new QAudioProbe;
+
+    ProbeDataHandler probeHandler;
+    connect(videoProbe, SIGNAL(videoFrameProbed(const QVideoFrame&)), &probeHandler, SLOT(processFrame(QVideoFrame)));
+    connect(videoProbe, SIGNAL(flush()), &probeHandler, SLOT(flushVideo()));
+    connect(audioProbe, SIGNAL(audioBufferProbed(const QAudioBuffer&)), &probeHandler, SLOT(processBuffer(QAudioBuffer)));
+    connect(audioProbe, SIGNAL(flush()), &probeHandler, SLOT(flushAudio()));
+
+    QVERIFY(videoProbe->setSource(player));
+    QVERIFY(audioProbe->setSource(player));
+
+    QFileInfo videoFile(QLatin1String(TESTDATA_DIR "testdata/colors.mp4"));
+    QVERIFY(videoFile.exists());
+    player->setMedia(QUrl::fromLocalFile(videoFile.absoluteFilePath()));
+    QTRY_COMPARE(player->mediaStatus(), QMediaPlayer::LoadedMedia);
+
+    player->pause();
+    QTRY_COMPARE(surface->m_frameList.size(), 1);
+    QVERIFY(!probeHandler.m_frameList.isEmpty());
+    QTRY_VERIFY(!probeHandler.m_bufferList.isEmpty());
+
+    delete player;
+    QTRY_VERIFY(probeHandler.isVideoFlushCalled);
+    delete videoProbe;
+    delete audioProbe;
+}
+
 QList<QVideoFrame::PixelFormat> TestVideoSurface::supportedPixelFormats(
         QAbstractVideoBuffer::HandleType handleType) const
 {
@@ -538,6 +590,26 @@ bool TestVideoSurface::present(const QVideoFrame &frame)
 }
 
 
+void ProbeDataHandler::processFrame(const QVideoFrame &frame)
+{
+    m_frameList.append(frame);
+}
+
+void ProbeDataHandler::processBuffer(const QAudioBuffer &buffer)
+{
+    m_bufferList.append(buffer);
+}
+
+void ProbeDataHandler::flushVideo()
+{
+    isVideoFlushCalled = true;
+}
+
+void ProbeDataHandler::flushAudio()
+{
+
+}
+
 QTEST_MAIN(tst_QMediaPlayerBackend)
 #include "tst_qmediaplayerbackend.moc"