WMF: re-enabled video probes and made it more robust.

Fixed the way the custom MF Transform (getting the frames) works:
- Recreate it whenever we load a new media
- During media type negotiation between nodes, the MFT should support
  the same types as the video sink supports
- Allow input and output types to be changed as many times as needed,
  otherwise the topology cannot be resolved in some cases

Change-Id: I7ca77e1a3dee83643f1a97f2e6ada9c5c0e88309
Reviewed-by: Lars Knoll <lars.knoll@digia.com>

src/plugins/wmf/mftvideo.cpp

@@ -47,6 +47,7 @@
 #include <uuids.h>
 #include <InitGuid.h>
 #include <d3d9.h>
+#include <qdebug.h>
 
 // This MFT sends all samples it processes to connected video probes.
 // Sample is sent to probes in ProcessInput.
@@ -70,6 +71,9 @@ MFTransform::~MFTransform()
 
     if (m_outputType)
         m_outputType->Release();
+
+    for (int i = 0; i < m_mediaTypes.size(); ++i)
+        m_mediaTypes[i]->Release();
 }
 
 void MFTransform::addProbe(MFVideoProbeControl *probe)
@@ -88,6 +92,14 @@ void MFTransform::removeProbe(MFVideoProbeControl *probe)
     m_videoProbes.removeOne(probe);
 }
 
+void MFTransform::addSupportedMediaType(IMFMediaType *type)
+{
+    if (!type)
+        return;
+    QMutexLocker locker(&m_mutex);
+    m_mediaTypes.append(type);
+}
+
 STDMETHODIMP MFTransform::QueryInterface(REFIID riid, void** ppv)
 {
     if (!ppv)
@@ -151,6 +163,8 @@ STDMETHODIMP MFTransform::GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInput
 
 STDMETHODIMP MFTransform::GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo)
 {
+    QMutexLocker locker(&m_mutex);
+
     if (dwInputStreamID > 0)
         return MF_E_INVALIDSTREAMNUMBER;
 
@@ -167,6 +181,8 @@ STDMETHODIMP MFTransform::GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_ST
 
 STDMETHODIMP MFTransform::GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo)
 {
+    QMutexLocker locker(&m_mutex);
+
     if (dwOutputStreamID > 0)
         return MF_E_INVALIDSTREAMNUMBER;
 
@@ -243,16 +259,27 @@ STDMETHODIMP MFTransform::SetInputType(DWORD dwInputStreamID, IMFMediaType *pTyp
 
     QMutexLocker locker(&m_mutex);
 
-    DWORD flags = 0;
+    if (m_sample)
-    if (m_outputType && m_outputType->IsEqual(pType, &flags) != S_OK) {
+        return MF_E_TRANSFORM_CANNOT_CHANGE_MEDIATYPE_WHILE_PROCESSING;
+
+    if (!isMediaTypeSupported(pType))
+        return MF_E_INVALIDMEDIATYPE;
+
+    DWORD flags = 0;
+    if (pType && !m_inputType && m_outputType && m_outputType->IsEqual(pType, &flags) != S_OK)
         return MF_E_INVALIDMEDIATYPE;
-    }
 
     if (dwFlags == MFT_SET_TYPE_TEST_ONLY)
         return pType ? S_OK : E_POINTER;
 
-    if (m_inputType)
+    if (m_inputType) {
         m_inputType->Release();
+        // Input type has changed, discard output type (if it's set) so it's reset later on
+        if (m_outputType &&  m_outputType->IsEqual(pType, &flags) != S_OK) {
+            m_outputType->Release();
+            m_outputType = 0;
+        }
+    }
 
     m_inputType = pType;
 
@@ -269,16 +296,27 @@ STDMETHODIMP MFTransform::SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pT
 
     QMutexLocker locker(&m_mutex);
 
-    DWORD flags = 0;
+    if (m_sample)
-    if (m_inputType && m_inputType->IsEqual(pType, &flags) != S_OK) {
+        return MF_E_TRANSFORM_CANNOT_CHANGE_MEDIATYPE_WHILE_PROCESSING;
+
+    if (!isMediaTypeSupported(pType))
+        return MF_E_INVALIDMEDIATYPE;
+
+    DWORD flags = 0;
+    if (pType && !m_outputType && m_inputType && m_inputType->IsEqual(pType, &flags) != S_OK)
         return MF_E_INVALIDMEDIATYPE;
-    }
 
     if (dwFlags == MFT_SET_TYPE_TEST_ONLY)
         return pType ? S_OK : E_POINTER;
 
-    if (m_outputType)
+    if (m_outputType) {
         m_outputType->Release();
+        // Output type has changed, discard input type (if it's set) so it's reset later on
+        if (m_inputType &&  m_inputType->IsEqual(pType, &flags) != S_OK) {
+            m_inputType->Release();
+            m_inputType = 0;
+        }
+    }
 
     m_outputType = pType;
 
@@ -645,3 +683,19 @@ QByteArray MFTransform::dataFromBuffer(IMFMediaBuffer *buffer, int height, int *
 
     return array;
 }
+
+bool MFTransform::isMediaTypeSupported(IMFMediaType *type)
+{
+    // if the list is empty, it supports all formats
+    if (!type || m_mediaTypes.isEmpty())
+        return true;
+
+    for (int i = 0; i < m_mediaTypes.size(); ++i) {
+        DWORD flags = 0;
+        m_mediaTypes.at(i)->IsEqual(type, &flags);
+        if (flags & MF_MEDIATYPE_EQUAL_FORMAT_TYPES)
+            return true;
+    }
+
+    return false;
+}

src/plugins/wmf/mftvideo.h

@@ -61,6 +61,8 @@ public:
     void addProbe(MFVideoProbeControl* probe);
     void removeProbe(MFVideoProbeControl* probe);
 
+    void addSupportedMediaType(IMFMediaType *type);
+
     // IUnknown methods
     STDMETHODIMP QueryInterface(REFIID iid, void** ppv);
     STDMETHODIMP_(ULONG) AddRef();
@@ -97,6 +99,7 @@ private:
     static QVideoSurfaceFormat videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine);
     QVideoFrame makeVideoFrame();
     QByteArray dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine);
+    bool isMediaTypeSupported(IMFMediaType *type);
 
     long m_cRef;
     IMFMediaType *m_inputType;
@@ -104,6 +107,8 @@ private:
     IMFSample *m_sample;
     QMutex m_mutex;
 
+    QList<IMFMediaType*> m_mediaTypes;
+
     QList<MFVideoProbeControl*> m_videoProbes;
     QMutex m_videoProbeMutex;
 

src/plugins/wmf/player/mfplayerservice.cpp

@@ -116,12 +116,11 @@ QMediaControl* MFPlayerService::requestControl(const char *name)
         }
         return 0;
     } else if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) {
-        // FIXME!! Disabled in Qt 5.0 because it is unstable
+        if (m_session) {
-//        if (m_session) {
+            MFVideoProbeControl *probe = new MFVideoProbeControl(this);
-//            MFVideoProbeControl *probe = new MFVideoProbeControl(this);
+            m_session->addProbe(probe);
-//            m_session->addProbe(probe);
+            return probe;
-//            return probe;
+        }
-//        }
         return 0;
     }
 

src/plugins/wmf/player/mfplayersession.cpp

@@ -436,7 +436,6 @@ MFPlayerSession::MFPlayerSession(MFPlayerService *playerService)
     m_request.rate = 1.0f;
 
     m_audioSampleGrabber = new AudioSampleGrabberCallback;
-    m_videoProbeMFT = new MFTransform;
 }
 
 void MFPlayerSession::close()
@@ -472,6 +471,11 @@ void MFPlayerSession::close()
         m_sourceResolver->Release();
         m_sourceResolver = 0;
     }
+    if (m_videoProbeMFT) {
+        m_videoProbeMFT->Release();
+        m_videoProbeMFT = 0;
+    }
+
 
     if (m_session)
         m_session->Release();
@@ -493,18 +497,26 @@ void MFPlayerSession::removeProbe(MFAudioProbeControl *probe)
 
 void MFPlayerSession::addProbe(MFVideoProbeControl* probe)
 {
+    if (m_videoProbes.contains(probe))
+        return;
+
+    m_videoProbes.append(probe);
+
+    if (m_videoProbeMFT)
         m_videoProbeMFT->addProbe(probe);
 }
 
 void MFPlayerSession::removeProbe(MFVideoProbeControl* probe)
 {
+    m_videoProbes.removeOne(probe);
+
+    if (m_videoProbeMFT)
         m_videoProbeMFT->removeProbe(probe);
 }
 
 MFPlayerSession::~MFPlayerSession()
 {
     m_audioSampleGrabber->Release();
-    m_videoProbeMFT->Release();
 }
 
 
@@ -988,73 +1000,92 @@ IMFTopology *MFPlayerSession::insertMFT(IMFTopology *topology, TOPOID outputNode
         if (FAILED(topoLoader->Load(topology, &resolvedTopology, NULL)))
             break;
 
-// FIXME!! VideoProbe disabled in Qt 5.0 because it is unstable.
-//         Commented out the following code to skip inserting the transform node
-//         getting the video frames.
-
         // Get all output nodes and search for video output node.
-//        if (FAILED(resolvedTopology->GetOutputNodeCollection(&outputNodes)))
+        if (FAILED(resolvedTopology->GetOutputNodeCollection(&outputNodes)))
-//            break;
+            break;
 
-//        DWORD elementCount = 0;
+        DWORD elementCount = 0;
-//        if (FAILED(outputNodes->GetElementCount(&elementCount)))
+        if (FAILED(outputNodes->GetElementCount(&elementCount)))
-//            break;
+            break;
 
-//        for (DWORD n = 0; n < elementCount; n++) {
+        for (DWORD n = 0; n < elementCount; n++) {
-//            IUnknown *element = 0;
+            IUnknown *element = 0;
-//            IMFTopologyNode *node = 0;
+            IMFTopologyNode *node = 0;
-//            IMFTopologyNode *inputNode = 0;
+            IUnknown *outputObject = 0;
-//            IMFTopologyNode *mftNode = 0;
+            IMFMediaTypeHandler *videoSink = 0;
+            IMFTopologyNode *inputNode = 0;
+            IMFTopologyNode *mftNode = 0;
 
-//            do {
+            do {
-//                if (FAILED(outputNodes->GetElement(n, &element)))
+                if (FAILED(outputNodes->GetElement(n, &element)))
-//                    break;
+                    break;
 
-//                if (FAILED(element->QueryInterface(IID_IMFTopologyNode, (void**)&node)))
+                if (FAILED(element->QueryInterface(IID_IMFTopologyNode, (void**)&node)))
-//                    break;
+                    break;
 
-//                TOPOID id;
+                TOPOID id;
-//                if (FAILED(node->GetTopoNodeID(&id)))
+                if (FAILED(node->GetTopoNodeID(&id)))
-//                    break;
+                    break;
 
-//                if (id != outputNodeId)
+                if (id != outputNodeId)
-//                    break;
+                    break;
 
-//                // Insert MFT between the output node and the node connected to it.
+                // Use output supported media types for the MFT
-//                DWORD outputIndex = 0;
+                if (FAILED(node->GetObject(&outputObject)))
-//                if (FAILED(node->GetInput(0, &inputNode, &outputIndex)))
+                    break;
-//                    break;
 
-//                if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &mftNode)))
+                if (FAILED(outputObject->QueryInterface(IID_IMFMediaTypeHandler, (void**)&videoSink)))
-//                    break;
+                    break;
 
-//                if (FAILED(mftNode->SetObject(m_videoProbeMFT)))
+                DWORD mtCount;
-//                    break;
+                if (FAILED(videoSink->GetMediaTypeCount(&mtCount)))
+                    break;
 
-//                if (FAILED(resolvedTopology->AddNode(mftNode)))
+                for (DWORD i = 0; i < mtCount; ++i) {
-//                    break;
+                    IMFMediaType *type = 0;
+                    if (SUCCEEDED(videoSink->GetMediaTypeByIndex(i, &type)))
+                        m_videoProbeMFT->addSupportedMediaType(type);
+                }
 
-//                if (FAILED(inputNode->ConnectOutput(0, mftNode, 0)))
+                // Insert MFT between the output node and the node connected to it.
-//                    break;
+                DWORD outputIndex = 0;
+                if (FAILED(node->GetInput(0, &inputNode, &outputIndex)))
+                    break;
 
-//                if (FAILED(mftNode->ConnectOutput(0, node, 0)))
+                if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &mftNode)))
-//                    break;
+                    break;
 
-//                isNewTopology = true;
+                if (FAILED(mftNode->SetObject(m_videoProbeMFT)))
-//            } while (false);
+                    break;
 
-//            if (mftNode)
+                if (FAILED(resolvedTopology->AddNode(mftNode)))
-//                mftNode->Release();
+                    break;
-//            if (inputNode)
-//                inputNode->Release();
-//            if (node)
-//                node->Release();
-//            if (element)
-//                element->Release();
 
-//            if (isNewTopology)
+                if (FAILED(inputNode->ConnectOutput(0, mftNode, 0)))
-//                break;
+                    break;
-//        }
+
+                if (FAILED(mftNode->ConnectOutput(0, node, 0)))
+                    break;
+
+                isNewTopology = true;
+            } while (false);
+
+            if (mftNode)
+                mftNode->Release();
+            if (inputNode)
+                inputNode->Release();
+            if (node)
+                node->Release();
+            if (element)
+                element->Release();
+            if (videoSink)
+                videoSink->Release();
+            if (outputObject)
+                outputObject->Release();
+
+            if (isNewTopology)
+                break;
+        }
     } while (false);
 
     if (outputNodes)
@@ -1181,6 +1212,10 @@ void MFPlayerSession::createSession()
     QObject::connect(m_sourceResolver, SIGNAL(mediaSourceReady()), this, SLOT(handleMediaSourceReady()));
     QObject::connect(m_sourceResolver, SIGNAL(error(long)), this, SLOT(handleSourceError(long)));
 
+    m_videoProbeMFT = new MFTransform;
+    for (int i = 0; i < m_videoProbes.size(); ++i)
+        m_videoProbeMFT->addProbe(m_videoProbes.at(i));
+
     Q_ASSERT(m_session == NULL);
     HRESULT hr = MFCreateMediaSession(NULL, &m_session);
     if (FAILED(hr)) {
@@ -1569,7 +1604,9 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent)
         break;
     case MESourceUnknown:
         changeStatus(QMediaPlayer::InvalidMedia);
+        break;
     case MEError:
+        changeStatus(QMediaPlayer::UnknownMediaStatus);
         qWarning() << "handleSessionEvent: serious error = " << hrStatus;
         emit error(QMediaPlayer::ResourceError, tr("Media session serious error."), true);
         break;

src/plugins/wmf/player/mfplayersession.h

@@ -232,6 +232,7 @@ private:
 
     IMFTopology *insertMFT(IMFTopology *topology, TOPOID outputNodeId);
     MFTransform *m_videoProbeMFT;
+    QList<MFVideoProbeControl*> m_videoProbes;
 };