Videoprobe implementation for wmf backend.

Change-Id: Ia597af428764229a76c0059ae7a57eb302aee63d
Reviewed-by: Dmytro Poplavskiy <dmytro.poplavskiy@nokia.com>
Reviewed-by: Michael Goddard <michael.goddard@nokia.com>

src/plugins/wmf/mftvideo.cpp

@@ -0,0 +1,647 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "mftvideo.h"
+#include "mfvideoprobecontrol.h"
+#include <private/qmemoryvideobuffer_p.h>
+#include <Mferror.h>
+#include <strmif.h>
+#include <uuids.h>
+#include <InitGuid.h>
+#include <d3d9.h>
+
+// This MFT sends all samples it processes to connected video probes.
+// Sample is sent to probes in ProcessInput.
+// In ProcessOutput this MFT simply returns the original sample.
+
+// The implementation is based on a boilerplate from the MF SDK example.
+
+MFTransform::MFTransform():
+    m_cRef(1),
+    m_inputType(0),
+    m_outputType(0),
+    m_sample(0),
+    m_bytesPerLine(0)
+{
+}
+
+MFTransform::~MFTransform()
+{
+    if (m_inputType)
+        m_inputType->Release();
+
+    if (m_outputType)
+        m_outputType->Release();
+}
+
+void MFTransform::addProbe(MFVideoProbeControl *probe)
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+
+    if (m_videoProbes.contains(probe))
+        return;
+
+    m_videoProbes.append(probe);
+}
+
+void MFTransform::removeProbe(MFVideoProbeControl *probe)
+{
+    QMutexLocker locker(&m_videoProbeMutex);
+    m_videoProbes.removeOne(probe);
+}
+
+STDMETHODIMP MFTransform::QueryInterface(REFIID riid, void** ppv)
+{
+    if (!ppv)
+        return E_POINTER;
+    if (riid == IID_IMFTransform) {
+        *ppv = static_cast<IMFTransform*>(this);
+    } else if (riid == IID_IUnknown) {
+        *ppv = static_cast<IUnknown*>(this);
+    } else {
+        *ppv =  NULL;
+        return E_NOINTERFACE;
+    }
+    AddRef();
+    return S_OK;
+}
+
+STDMETHODIMP_(ULONG) MFTransform::AddRef()
+{
+    return InterlockedIncrement(&m_cRef);
+}
+
+STDMETHODIMP_(ULONG) MFTransform::Release()
+{
+    ULONG cRef = InterlockedDecrement(&m_cRef);
+    if (cRef == 0) {
+        delete this;
+    }
+    return cRef;
+}
+
+STDMETHODIMP MFTransform::GetStreamLimits(DWORD *pdwInputMinimum, DWORD *pdwInputMaximum, DWORD *pdwOutputMinimum, DWORD *pdwOutputMaximum)
+{
+    if (!pdwInputMinimum || !pdwInputMaximum || !pdwOutputMinimum || !pdwOutputMaximum)
+        return E_POINTER;
+    *pdwInputMinimum = 1;
+    *pdwInputMaximum = 1;
+    *pdwOutputMinimum = 1;
+    *pdwOutputMaximum = 1;
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetStreamCount(DWORD *pcInputStreams, DWORD *pcOutputStreams)
+{
+    if (!pcInputStreams || !pcOutputStreams)
+        return E_POINTER;
+
+    *pcInputStreams = 1;
+    *pcOutputStreams = 1;
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInputIDs, DWORD dwOutputIDArraySize, DWORD *pdwOutputIDs)
+{
+    // streams are numbered consecutively
+    Q_UNUSED(dwInputIDArraySize);
+    Q_UNUSED(pdwInputIDs);
+    Q_UNUSED(dwOutputIDArraySize);
+    Q_UNUSED(pdwOutputIDs);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo)
+{
+    if (dwInputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    if (!pStreamInfo)
+        return E_POINTER;
+
+    pStreamInfo->cbSize = 0;
+    pStreamInfo->hnsMaxLatency = 0;
+    pStreamInfo->dwFlags = MFT_INPUT_STREAM_WHOLE_SAMPLES | MFT_INPUT_STREAM_SINGLE_SAMPLE_PER_BUFFER;
+    pStreamInfo->cbMaxLookahead = 0;
+    pStreamInfo->cbAlignment = 0;
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo)
+{
+    if (dwOutputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    if (!pStreamInfo)
+        return E_POINTER;
+
+    pStreamInfo->cbSize = 0;
+    pStreamInfo->dwFlags = MFT_OUTPUT_STREAM_WHOLE_SAMPLES | MFT_OUTPUT_STREAM_SINGLE_SAMPLE_PER_BUFFER;
+    pStreamInfo->cbAlignment = 0;
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetAttributes(IMFAttributes **pAttributes)
+{
+    // This MFT does not support attributes.
+    Q_UNUSED(pAttributes);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::GetInputStreamAttributes(DWORD dwInputStreamID, IMFAttributes **pAttributes)
+{
+    // This MFT does not support input stream attributes.
+    Q_UNUSED(dwInputStreamID);
+    Q_UNUSED(pAttributes);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::GetOutputStreamAttributes(DWORD dwOutputStreamID, IMFAttributes **pAttributes)
+{
+    // This MFT does not support output stream attributes.
+    Q_UNUSED(dwOutputStreamID);
+    Q_UNUSED(pAttributes);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::DeleteInputStream(DWORD dwStreamID)
+{
+    // This MFT has a fixed number of input streams.
+    Q_UNUSED(dwStreamID);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::AddInputStreams(DWORD cStreams, DWORD *adwStreamIDs)
+{
+    // This MFT has a fixed number of input streams.
+    Q_UNUSED(cStreams);
+    Q_UNUSED(adwStreamIDs);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::GetInputAvailableType(DWORD dwInputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType)
+{
+    // This MFT does not have a list of preferred input types
+    Q_UNUSED(dwInputStreamID);
+    Q_UNUSED(dwTypeIndex);
+    Q_UNUSED(ppType);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::GetOutputAvailableType(DWORD dwOutputStreamID,DWORD dwTypeIndex, IMFMediaType **ppType)
+{
+    // This MFT does not have a list of preferred output types
+    Q_UNUSED(dwOutputStreamID);
+    Q_UNUSED(dwTypeIndex);
+    Q_UNUSED(ppType);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::SetInputType(DWORD dwInputStreamID, IMFMediaType *pType, DWORD dwFlags)
+{
+    if (dwInputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    QMutexLocker locker(&m_mutex);
+
+    DWORD flags = 0;
+    if (m_outputType && m_outputType->IsEqual(pType, &flags) != S_OK) {
+        return MF_E_INVALIDMEDIATYPE;
+    }
+
+    if (dwFlags == MFT_SET_TYPE_TEST_ONLY)
+        return pType ? S_OK : E_POINTER;
+
+    if (m_inputType)
+        m_inputType->Release();
+
+    m_inputType = pType;
+
+    if (m_inputType)
+        m_inputType->AddRef();
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pType, DWORD dwFlags)
+{
+    if (dwOutputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    QMutexLocker locker(&m_mutex);
+
+    DWORD flags = 0;
+    if (m_inputType && m_inputType->IsEqual(pType, &flags) != S_OK) {
+        return MF_E_INVALIDMEDIATYPE;
+    }
+
+    if (dwFlags == MFT_SET_TYPE_TEST_ONLY)
+        return pType ? S_OK : E_POINTER;
+
+    if (m_outputType)
+        m_outputType->Release();
+
+    m_outputType = pType;
+
+    if (m_outputType) {
+        m_outputType->AddRef();
+        m_format = videoFormatForMFMediaType(m_outputType, &m_bytesPerLine);
+    }
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetInputCurrentType(DWORD dwInputStreamID, IMFMediaType **ppType)
+{
+    if (dwInputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    if (ppType == NULL)
+        return E_POINTER;
+
+    QMutexLocker locker(&m_mutex);
+
+    if (!m_inputType)
+        return MF_E_TRANSFORM_TYPE_NOT_SET;
+
+    *ppType = m_inputType;
+    (*ppType)->AddRef();
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetOutputCurrentType(DWORD dwOutputStreamID, IMFMediaType **ppType)
+{
+    if (dwOutputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    if (ppType == NULL)
+        return E_POINTER;
+
+    QMutexLocker locker(&m_mutex);
+
+    if (!m_outputType) {
+        if (m_inputType) {
+            *ppType = m_inputType;
+            (*ppType)->AddRef();
+            return S_OK;
+        }
+        return MF_E_TRANSFORM_TYPE_NOT_SET;
+    }
+
+    *ppType = m_outputType;
+    (*ppType)->AddRef();
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetInputStatus(DWORD dwInputStreamID, DWORD *pdwFlags)
+{
+    if (dwInputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    if (!pdwFlags)
+        return E_POINTER;
+
+    QMutexLocker locker(&m_mutex);
+
+    if (!m_inputType)
+        return MF_E_TRANSFORM_TYPE_NOT_SET;
+
+    if (m_sample)
+        *pdwFlags = 0;
+    else
+        *pdwFlags = MFT_INPUT_STATUS_ACCEPT_DATA;
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::GetOutputStatus(DWORD *pdwFlags)
+{
+    if (!pdwFlags)
+        return E_POINTER;
+
+    QMutexLocker locker(&m_mutex);
+
+    if (!m_outputType)
+        return MF_E_TRANSFORM_TYPE_NOT_SET;
+
+    if (m_sample)
+        *pdwFlags = MFT_OUTPUT_STATUS_SAMPLE_READY;
+    else
+        *pdwFlags = 0;
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::SetOutputBounds(LONGLONG hnsLowerBound, LONGLONG hnsUpperBound)
+{
+    Q_UNUSED(hnsLowerBound);
+    Q_UNUSED(hnsUpperBound);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::ProcessEvent(DWORD dwInputStreamID, IMFMediaEvent *pEvent)
+{
+    // This MFT ignores all events, and the pipeline should send all events downstream.
+    Q_UNUSED(dwInputStreamID);
+    Q_UNUSED(pEvent);
+    return E_NOTIMPL;
+}
+
+STDMETHODIMP MFTransform::ProcessMessage(MFT_MESSAGE_TYPE eMessage, ULONG_PTR ulParam)
+{
+    Q_UNUSED(ulParam);
+
+    HRESULT hr = S_OK;
+
+    switch (eMessage)
+    {
+    case MFT_MESSAGE_COMMAND_FLUSH:
+        hr = OnFlush();
+        break;
+
+    case MFT_MESSAGE_COMMAND_DRAIN:
+        // Drain: Tells the MFT not to accept any more input until
+        // all of the pending output has been processed. That is our
+        // default behevior already, so there is nothing to do.
+        break;
+
+    case MFT_MESSAGE_SET_D3D_MANAGER:
+        // The pipeline should never send this message unless the MFT
+        // has the MF_SA_D3D_AWARE attribute set to TRUE. However, if we
+        // do get this message, it's invalid and we don't implement it.
+        hr = E_NOTIMPL;
+        break;
+
+    // The remaining messages do not require any action from this MFT.
+    case MFT_MESSAGE_NOTIFY_BEGIN_STREAMING:
+    case MFT_MESSAGE_NOTIFY_END_STREAMING:
+    case MFT_MESSAGE_NOTIFY_END_OF_STREAM:
+    case MFT_MESSAGE_NOTIFY_START_OF_STREAM:
+        break;
+    }
+
+    return hr;
+}
+
+STDMETHODIMP MFTransform::ProcessInput(DWORD dwInputStreamID, IMFSample *pSample, DWORD dwFlags)
+{
+    if (dwInputStreamID > 0)
+        return MF_E_INVALIDSTREAMNUMBER;
+
+    if (dwFlags != 0)
+        return E_INVALIDARG; // dwFlags is reserved and must be zero.
+
+    QMutexLocker locker(&m_mutex);
+
+    if (!m_inputType || !m_outputType)
+        return MF_E_TRANSFORM_TYPE_NOT_SET;
+
+    if (m_sample)
+        return MF_E_NOTACCEPTING;
+
+    // Validate the number of buffers. There should only be a single buffer to hold the video frame.
+    DWORD dwBufferCount = 0;
+    HRESULT hr = pSample->GetBufferCount(&dwBufferCount);
+    if (FAILED(hr))
+        return hr;
+
+    if (dwBufferCount == 0)
+        return E_FAIL;
+
+    if (dwBufferCount > 1)
+        return MF_E_SAMPLE_HAS_TOO_MANY_BUFFERS;
+
+    m_sample = pSample;
+    m_sample->AddRef();
+
+    QMutexLocker lockerProbe(&m_videoProbeMutex);
+
+    if (!m_videoProbes.isEmpty()) {
+        QVideoFrame frame = makeVideoFrame();
+
+        foreach (MFVideoProbeControl* probe, m_videoProbes)
+            probe->bufferProbed(frame);
+    }
+
+    return S_OK;
+}
+
+STDMETHODIMP MFTransform::ProcessOutput(DWORD dwFlags, DWORD cOutputBufferCount, MFT_OUTPUT_DATA_BUFFER *pOutputSamples, DWORD *pdwStatus)
+{
+    if (dwFlags != 0)
+        return E_INVALIDARG;
+
+    if (pOutputSamples == NULL || pdwStatus == NULL)
+        return E_POINTER;
+
+    if (cOutputBufferCount != 1)
+        return E_INVALIDARG;
+
+    QMutexLocker locker(&m_mutex);
+
+    if (!m_sample)
+        return MF_E_TRANSFORM_NEED_MORE_INPUT;
+
+    if (pOutputSamples[0].pSample)
+        pOutputSamples[0].pSample->Release();
+
+    pOutputSamples[0].pSample = m_sample;
+    pOutputSamples[0].pSample->AddRef();
+
+    pOutputSamples[0].dwStatus = 0;
+    *pdwStatus = 0;
+
+    m_sample->Release();
+    m_sample = 0;
+
+    return S_OK;
+}
+
+HRESULT MFTransform::OnFlush()
+{
+    QMutexLocker locker(&m_mutex);
+
+    if (m_sample) {
+        m_sample->Release();
+        m_sample = 0;
+    }
+    return S_OK;
+}
+
+QVideoFrame::PixelFormat MFTransform::formatFromSubtype(const GUID& subtype)
+{
+    if (subtype == MFVideoFormat_ARGB32)
+        return QVideoFrame::Format_ARGB32;
+    else if (subtype == MFVideoFormat_RGB32)
+        return QVideoFrame::Format_RGB32;
+    else if (subtype == MFVideoFormat_RGB24)
+        return QVideoFrame::Format_RGB24;
+    else if (subtype == MFVideoFormat_RGB565)
+        return QVideoFrame::Format_RGB565;
+    else if (subtype == MFVideoFormat_RGB555)
+        return QVideoFrame::Format_RGB555;
+    else if (subtype == MFVideoFormat_AYUV)
+        return QVideoFrame::Format_AYUV444;
+    else if (subtype == MFVideoFormat_I420)
+        return QVideoFrame::Format_YUV420P;
+    else if (subtype == MFVideoFormat_UYVY)
+        return QVideoFrame::Format_UYVY;
+    else if (subtype == MFVideoFormat_YV12)
+        return QVideoFrame::Format_YV12;
+    else if (subtype == MFVideoFormat_NV12)
+        return QVideoFrame::Format_NV12;
+
+    return QVideoFrame::Format_Invalid;
+}
+
+QVideoSurfaceFormat MFTransform::videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine)
+{
+    UINT32 stride;
+    if (FAILED(mediaType->GetUINT32(MF_MT_DEFAULT_STRIDE, &stride))) {
+        *bytesPerLine = 0;
+        return QVideoSurfaceFormat();
+    }
+
+    *bytesPerLine = (int)stride;
+
+    QSize size;
+    UINT32 width, height;
+    if (FAILED(MFGetAttributeSize(mediaType, MF_MT_FRAME_SIZE, &width, &height)))
+        return QVideoSurfaceFormat();
+
+    size.setWidth(width);
+    size.setHeight(height);
+
+    GUID subtype = GUID_NULL;
+    if (FAILED(mediaType->GetGUID(MF_MT_SUBTYPE, &subtype)))
+        return QVideoSurfaceFormat();
+
+    QVideoFrame::PixelFormat pixelFormat = formatFromSubtype(subtype);
+    QVideoSurfaceFormat format(size, pixelFormat);
+
+    UINT32 num, den;
+    if (SUCCEEDED(MFGetAttributeRatio(mediaType, MF_MT_PIXEL_ASPECT_RATIO, &num, &den))) {
+        format.setPixelAspectRatio(num, den);
+    }
+    if (SUCCEEDED(MFGetAttributeRatio(mediaType, MF_MT_FRAME_RATE, &num, &den))) {
+        format.setFrameRate(qreal(num)/den);
+    }
+
+    return format;
+}
+
+QVideoFrame MFTransform::makeVideoFrame()
+{
+    QVideoFrame frame;
+
+    if (!m_format.isValid())
+        return frame;
+
+    IMFMediaBuffer *buffer = 0;
+
+    do {
+        if (FAILED(m_sample->ConvertToContiguousBuffer(&buffer)))
+            break;
+
+        QByteArray array = dataFromBuffer(buffer, m_format.frameHeight(), &m_bytesPerLine);
+        if (array.isEmpty())
+            break;
+
+        // Wrapping IMFSample or IMFMediaBuffer in a QVideoFrame is not possible because we cannot hold
+        // IMFSample for a "long" time without affecting the rest of the topology.
+        // If IMFSample is held for more than 5 frames decoder starts to reuse it even though it hasn't been released it yet.
+        // That is why we copy data from IMFMediaBuffer here.
+        frame = QVideoFrame(new QMemoryVideoBuffer(array, m_bytesPerLine), m_format.frameSize(), m_format.pixelFormat());
+
+        LONGLONG startTime = -1;
+        if (SUCCEEDED(m_sample->GetSampleTime(&startTime))) {
+            frame.setStartTime(startTime);
+
+            LONGLONG duration = -1;
+            if (SUCCEEDED(m_sample->GetSampleDuration(&duration)))
+                frame.setEndTime(startTime + duration);
+        }
+    } while (false);
+
+    if (buffer)
+        buffer->Release();
+
+    return frame;
+}
+
+QByteArray MFTransform::dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine)
+{
+    QByteArray array;
+    BYTE *bytes;
+    DWORD length;
+    HRESULT hr = buffer->Lock(&bytes, NULL, &length);
+    if (SUCCEEDED(hr)) {
+        array = QByteArray((const char *)bytes, (int)length);
+        buffer->Unlock();
+    } else {
+        // try to lock as Direct3DSurface
+        IDirect3DSurface9 *surface = 0;
+        do {
+            if (FAILED(MFGetService(buffer, MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void**)&surface)))
+                break;
+
+            D3DLOCKED_RECT rect;
+            if (FAILED(surface->LockRect(&rect, NULL, D3DLOCK_READONLY)))
+                break;
+
+            if (bytesPerLine)
+                *bytesPerLine = (int)rect.Pitch;
+
+            array = QByteArray((const char *)rect.pBits, rect.Pitch * height);
+            surface->UnlockRect();
+        } while (false);
+
+        if (surface) {
+            surface->Release();
+            surface = 0;
+        }
+    }
+
+    return array;
+}

src/plugins/wmf/mftvideo.h

@@ -0,0 +1,114 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef MFTRANSFORM_H
+#define MFTRANSFORM_H
+
+#include <mfapi.h>
+#include <mfidl.h>
+#include <QtCore/qlist.h>
+#include <QtCore/qmutex.h>
+#include <QtMultimedia/qvideosurfaceformat.h>
+
+QT_USE_NAMESPACE
+
+class MFVideoProbeControl;
+
+class MFTransform: public IMFTransform
+{
+public:
+    MFTransform();
+    ~MFTransform();
+
+    void addProbe(MFVideoProbeControl* probe);
+    void removeProbe(MFVideoProbeControl* probe);
+
+    // IUnknown methods
+    STDMETHODIMP QueryInterface(REFIID iid, void** ppv);
+    STDMETHODIMP_(ULONG) AddRef();
+    STDMETHODIMP_(ULONG) Release();
+
+    // IMFTransform methods
+    STDMETHODIMP GetStreamLimits(DWORD *pdwInputMinimum, DWORD *pdwInputMaximum, DWORD *pdwOutputMinimum, DWORD *pdwOutputMaximum);
+    STDMETHODIMP GetStreamCount(DWORD *pcInputStreams, DWORD *pcOutputStreams);
+    STDMETHODIMP GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInputIDs, DWORD dwOutputIDArraySize, DWORD *pdwOutputIDs);
+    STDMETHODIMP GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo);
+    STDMETHODIMP GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo);
+    STDMETHODIMP GetAttributes(IMFAttributes **pAttributes);
+    STDMETHODIMP GetInputStreamAttributes(DWORD dwInputStreamID, IMFAttributes **pAttributes);
+    STDMETHODIMP GetOutputStreamAttributes(DWORD dwOutputStreamID, IMFAttributes **pAttributes);
+    STDMETHODIMP DeleteInputStream(DWORD dwStreamID);
+    STDMETHODIMP AddInputStreams(DWORD cStreams, DWORD *adwStreamIDs);
+    STDMETHODIMP GetInputAvailableType(DWORD dwInputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType);
+    STDMETHODIMP GetOutputAvailableType(DWORD dwOutputStreamID,DWORD dwTypeIndex, IMFMediaType **ppType);
+    STDMETHODIMP SetInputType(DWORD dwInputStreamID, IMFMediaType *pType, DWORD dwFlags);
+    STDMETHODIMP SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pType, DWORD dwFlags);
+    STDMETHODIMP GetInputCurrentType(DWORD dwInputStreamID, IMFMediaType **ppType);
+    STDMETHODIMP GetOutputCurrentType(DWORD dwOutputStreamID, IMFMediaType **ppType);
+    STDMETHODIMP GetInputStatus(DWORD dwInputStreamID, DWORD *pdwFlags);
+    STDMETHODIMP GetOutputStatus(DWORD *pdwFlags);
+    STDMETHODIMP SetOutputBounds(LONGLONG hnsLowerBound, LONGLONG hnsUpperBound);
+    STDMETHODIMP ProcessEvent(DWORD dwInputStreamID, IMFMediaEvent *pEvent);
+    STDMETHODIMP ProcessMessage(MFT_MESSAGE_TYPE eMessage, ULONG_PTR ulParam);
+    STDMETHODIMP ProcessInput(DWORD dwInputStreamID, IMFSample *pSample, DWORD dwFlags);
+    STDMETHODIMP ProcessOutput(DWORD dwFlags, DWORD cOutputBufferCount, MFT_OUTPUT_DATA_BUFFER *pOutputSamples, DWORD *pdwStatus);
+
+private:
+    HRESULT OnFlush();
+    static QVideoFrame::PixelFormat formatFromSubtype(const GUID& subtype);
+    static QVideoSurfaceFormat videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine);
+    QVideoFrame makeVideoFrame();
+    QByteArray dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine);
+
+    long m_cRef;
+    IMFMediaType *m_inputType;
+    IMFMediaType *m_outputType;
+    IMFSample *m_sample;
+    QMutex m_mutex;
+
+    QList<MFVideoProbeControl*> m_videoProbes;
+    QMutex m_videoProbeMutex;
+
+    QVideoSurfaceFormat m_format;
+    int m_bytesPerLine;
+};
+
+#endif

src/plugins/wmf/player/mfplayerservice.cpp

@@ -50,6 +50,7 @@
 #include "mfvideorenderercontrol.h"
 #include "mfaudioendpointcontrol.h"
 #include "mfaudioprobecontrol.h"
+#include "mfvideoprobecontrol.h"
 #include "mfplayerservice.h"
 #include "mfplayersession.h"
 #include "mfmetadatacontrol.h"
@@ -113,6 +114,13 @@ QMediaControl* MFPlayerService::requestControl(const char *name)
             return probe;
         }
         return 0;
+    } else if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) {
+        if (m_session) {
+            MFVideoProbeControl *probe = new MFVideoProbeControl(this);
+            m_session->addProbe(probe);
+            return probe;
+        }
+        return 0;
     }
 
     return 0;
@@ -141,6 +149,14 @@ void MFPlayerService::releaseControl(QMediaControl *control)
         delete audioProbe;
         return;
     }
+
+    MFVideoProbeControl* videoProbe = qobject_cast<MFVideoProbeControl*>(control);
+    if (videoProbe) {
+        if (m_session)
+            m_session->removeProbe(videoProbe);
+        delete videoProbe;
+        return;
+    }
 }
 
 MFAudioEndpointControl* MFPlayerService::audioEndpointControl() const

src/plugins/wmf/player/mfplayersession.cpp

@@ -64,6 +64,7 @@
 #include <nserror.h>
 #include "sourceresolver.h"
 #include "samplegrabber.h"
+#include "mftvideo.h"
 
 //#define DEBUG_MEDIAFOUNDATION
 //#define TEST_STREAMING
@@ -419,6 +420,7 @@ MFPlayerSession::MFPlayerSession(MFPlayerService *playerService)
     , m_mediaTypes(0)
     , m_audioSampleGrabber(0)
     , m_audioSampleGrabberNode(0)
+    , m_videoProbeMFT(0)
 {
     m_hCloseEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
     m_sourceResolver = new SourceResolver();
@@ -442,6 +444,7 @@ MFPlayerSession::MFPlayerSession(MFPlayerService *playerService)
     m_varStart.uhVal.QuadPart = 0;
 
     m_audioSampleGrabber = new AudioSampleGrabberCallback;
+    m_videoProbeMFT = new MFTransform;
 }
 
 void MFPlayerSession::close()
@@ -485,9 +488,20 @@ void MFPlayerSession::removeProbe(MFAudioProbeControl *probe)
     m_audioSampleGrabber->removeProbe(probe);
 }
 
+void MFPlayerSession::addProbe(MFVideoProbeControl* probe)
+{
+    m_videoProbeMFT->addProbe(probe);
+}
+
+void MFPlayerSession::removeProbe(MFVideoProbeControl* probe)
+{
+    m_videoProbeMFT->removeProbe(probe);
+}
+
 MFPlayerSession::~MFPlayerSession()
 {
     m_audioSampleGrabber->Release();
+    m_videoProbeMFT->Release();
 }
 
 
@@ -581,6 +595,9 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat
         return;
     }
 
+    // Remember output node id for a first video stream
+    TOPOID outputNodeId = -1;
+
     // For each stream, create the topology nodes and add them to the topology.
     DWORD succeededCount = 0;
     for (DWORD i = 0; i < cSourceStreams; i++)
@@ -595,12 +612,17 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat
             if (sourceNode) {
                 IMFTopologyNode *outputNode = addOutputNode(streamDesc, mediaType, topology, 0);
                 if (outputNode) {
-                    // Only install audio sample grabber for the first audio stream.
+                    bool connected = false;
-                    if (mediaType != Audio || m_audioSampleGrabberNode || !setupAudioSampleGrabber(topology, sourceNode, outputNode)) {
+                    if (mediaType == Audio) {
-                        hr = sourceNode->ConnectOutput(0, outputNode, 0);
+                        if (!m_audioSampleGrabberNode)
-                    } else {
+                            connected = setupAudioSampleGrabber(topology, sourceNode, outputNode);
-                        hr = S_OK;
+                    } else if (mediaType == Video && outputNodeId == -1) {
+                        // Remember video output node ID.
+                        outputNode->GetTopoNodeID(&outputNodeId);
                     }
+
+                    if (!connected)
+                        hr = sourceNode->ConnectOutput(0, outputNode, 0);
                     if (FAILED(hr)) {
                         emit error(QMediaPlayer::FormatError, tr("Unable to play some stream"), false);
                     }
@@ -627,6 +649,10 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat
         changeStatus(QMediaPlayer::InvalidMedia);
         emit error(QMediaPlayer::ResourceError, tr("Unable to play"), true);
     } else {
+        if (outputNodeId != -1) {
+            topology = insertMFT(topology, outputNodeId);
+        }
+
         hr = m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology);
         if (FAILED(hr)) {
             changeStatus(QMediaPlayer::UnknownMediaStatus);
@@ -837,6 +863,204 @@ QAudioFormat MFPlayerSession::audioFormatForMFMediaType(IMFMediaType *mediaType)
     return format;
 }
 
+// BindOutputNode
+// Sets the IMFStreamSink pointer on an output node.
+// IMFActivate pointer in the output node must be converted to an
+// IMFStreamSink pointer before the topology loader resolves the topology.
+HRESULT BindOutputNode(IMFTopologyNode *pNode)
+{
+    IUnknown *nodeObject = NULL;
+    IMFActivate *activate = NULL;
+    IMFStreamSink *stream = NULL;
+    IMFMediaSink *sink = NULL;
+
+    HRESULT hr = pNode->GetObject(&nodeObject);
+    if (FAILED(hr))
+        return hr;
+
+    hr = nodeObject->QueryInterface(IID_PPV_ARGS(&activate));
+    if (SUCCEEDED(hr)) {
+        DWORD dwStreamID = 0;
+
+        // Try to create the media sink.
+        hr = activate->ActivateObject(IID_PPV_ARGS(&sink));
+        if (SUCCEEDED(hr))
+           dwStreamID = MFGetAttributeUINT32(pNode, MF_TOPONODE_STREAMID, 0);
+
+        if (SUCCEEDED(hr)) {
+            // First check if the media sink already has a stream sink with the requested ID.
+            hr = sink->GetStreamSinkById(dwStreamID, &stream);
+            if (FAILED(hr)) {
+                // Create the stream sink.
+                hr = sink->AddStreamSink(dwStreamID, NULL, &stream);
+            }
+        }
+
+        // Replace the node's object pointer with the stream sink.
+        if (SUCCEEDED(hr)) {
+            hr = pNode->SetObject(stream);
+        }
+    } else {
+        hr = nodeObject->QueryInterface(IID_PPV_ARGS(&stream));
+    }
+
+    if (nodeObject)
+        nodeObject->Release();
+    if (activate)
+        activate->Release();
+    if (stream)
+        stream->Release();
+    if (sink)
+        sink->Release();
+    return hr;
+}
+
+// BindOutputNodes
+// Sets the IMFStreamSink pointers on all of the output nodes in a topology.
+HRESULT BindOutputNodes(IMFTopology *pTopology)
+{
+    DWORD cNodes = 0;
+
+    IMFCollection *collection = NULL;
+    IUnknown *element = NULL;
+    IMFTopologyNode *node = NULL;
+
+    // Get the collection of output nodes.
+    HRESULT hr = pTopology->GetOutputNodeCollection(&collection);
+
+    // Enumerate all of the nodes in the collection.
+    if (SUCCEEDED(hr))
+        hr = collection->GetElementCount(&cNodes);
+
+    if (SUCCEEDED(hr)) {
+        for (DWORD i = 0; i < cNodes; i++) {
+            hr = collection->GetElement(i, &element);
+            if (FAILED(hr))
+                break;
+
+            hr = element->QueryInterface(IID_IMFTopologyNode, (void**)&node);
+            if (FAILED(hr))
+                break;
+
+            // Bind this node.
+            hr = BindOutputNode(node);
+            if (FAILED(hr))
+                break;
+        }
+    }
+
+    if (collection)
+        collection->Release();
+    if (element)
+        element->Release();
+    if (node)
+        node->Release();
+    return hr;
+}
+
+// This method binds output nodes to complete the topology,
+// then loads the topology and inserts MFT between the output node
+// and a filter connected to the output node.
+IMFTopology *MFPlayerSession::insertMFT(IMFTopology *topology, TOPOID outputNodeId)
+{
+    bool isNewTopology = false;
+
+    IMFTopoLoader *topoLoader = 0;
+    IMFTopology *resolvedTopology = 0;
+    IMFCollection *outputNodes = 0;
+
+    do {
+        if (FAILED(BindOutputNodes(topology)))
+            break;
+
+        if (FAILED(MFCreateTopoLoader(&topoLoader)))
+            break;
+
+        if (FAILED(topoLoader->Load(topology, &resolvedTopology, NULL)))
+            break;
+
+        // Get all output nodes and search for video output node.
+        if (FAILED(resolvedTopology->GetOutputNodeCollection(&outputNodes)))
+            break;
+
+        DWORD elementCount = 0;
+        if (FAILED(outputNodes->GetElementCount(&elementCount)))
+            break;
+
+        for (DWORD n = 0; n < elementCount; n++) {
+            IUnknown *element = 0;
+            IMFTopologyNode *node = 0;
+            IMFTopologyNode *inputNode = 0;
+            IMFTopologyNode *mftNode = 0;
+
+            do {
+                if (FAILED(outputNodes->GetElement(n, &element)))
+                    break;
+
+                if (FAILED(element->QueryInterface(IID_IMFTopologyNode, (void**)&node)))
+                    break;
+
+                TOPOID id;
+                if (FAILED(node->GetTopoNodeID(&id)))
+                    break;
+
+                if (id != outputNodeId)
+                    break;
+
+                // Insert MFT between the output node and the node connected to it.
+                DWORD outputIndex = 0;
+                if (FAILED(node->GetInput(0, &inputNode, &outputIndex)))
+                    break;
+
+                if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &mftNode)))
+                    break;
+
+                if (FAILED(mftNode->SetObject(m_videoProbeMFT)))
+                    break;
+
+                if (FAILED(resolvedTopology->AddNode(mftNode)))
+                    break;
+
+                if (FAILED(inputNode->ConnectOutput(0, mftNode, 0)))
+                    break;
+
+                if (FAILED(mftNode->ConnectOutput(0, node, 0)))
+                    break;
+
+                isNewTopology = true;
+            } while (false);
+
+            if (mftNode)
+                mftNode->Release();
+            if (inputNode)
+                inputNode->Release();
+            if (node)
+                node->Release();
+            if (element)
+                element->Release();
+
+            if (isNewTopology)
+                break;
+        }
+    } while (false);
+
+    if (outputNodes)
+        outputNodes->Release();
+
+    if (topoLoader)
+        topoLoader->Release();
+
+    if (isNewTopology) {
+        topology->Release();
+        return resolvedTopology;
+    }
+
+    if (resolvedTopology)
+        resolvedTopology->Release();
+
+    return topology;
+}
+
 void MFPlayerSession::stop(bool immediate)
 {
 #ifdef DEBUG_MEDIAFOUNDATION

src/plugins/wmf/player/mfplayersession.h

@@ -55,6 +55,7 @@
 #include <QtCore/qurl.h>
 #include <QtCore/qwaitcondition.h>
 #include <QtMultimedia/qaudioformat.h>
+#include <QtMultimedia/qvideosurfaceformat.h>
 
 QT_BEGIN_NAMESPACE
 class QMediaContent;
@@ -72,7 +73,9 @@ class MFPlayerControl;
 class MFMetaDataControl;
 class MFPlayerService;
 class AudioSampleGrabberCallback;
+class MFTransform;
 class MFAudioProbeControl;
+class MFVideoProbeControl;
 
 class MFPlayerSession : public QObject, public IMFAsyncCallback
 {
@@ -120,6 +123,8 @@ public:
 
     void addProbe(MFAudioProbeControl* probe);
     void removeProbe(MFAudioProbeControl* probe);
+    void addProbe(MFVideoProbeControl* probe);
+    void removeProbe(MFVideoProbeControl* probe);
 
 Q_SIGNALS:
     void error(QMediaPlayer::Error error, QString errorString, bool isFatal);
@@ -223,6 +228,9 @@ private:
     QAudioFormat audioFormatForMFMediaType(IMFMediaType *mediaType) const;
     AudioSampleGrabberCallback *m_audioSampleGrabber;
     IMFTopologyNode *m_audioSampleGrabberNode;
+
+    IMFTopology *insertMFT(IMFTopology *topology, TOPOID outputNodeId);
+    MFTransform *m_videoProbeMFT;
 };
 
 

src/plugins/wmf/player/mfvideoprobecontrol.cpp

@@ -0,0 +1,56 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "mfvideoprobecontrol.h"
+
+MFVideoProbeControl::MFVideoProbeControl(QObject *parent)
+    : QMediaVideoProbeControl(parent)
+{
+}
+
+MFVideoProbeControl::~MFVideoProbeControl()
+{
+}
+
+void MFVideoProbeControl::bufferProbed(const QVideoFrame& frame)
+{
+    QMetaObject::invokeMethod(this, "videoFrameProbed", Qt::QueuedConnection, Q_ARG(QVideoFrame, frame));
+}

src/plugins/wmf/player/mfvideoprobecontrol.h

@@ -0,0 +1,61 @@
+/****************************************************************************
+**
+** Copyright (C) 2012 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: http://www.qt-project.org/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** GNU Lesser General Public License Usage
+** This file may be used under the terms of the GNU Lesser General Public
+** License version 2.1 as published by the Free Software Foundation and
+** appearing in the file LICENSE.LGPL included in the packaging of this
+** file. Please review the following information to ensure the GNU Lesser
+** General Public License version 2.1 requirements will be met:
+** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights. These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU General
+** Public License version 3.0 as published by the Free Software Foundation
+** and appearing in the file LICENSE.GPL included in the packaging of this
+** file. Please review the following information to ensure the GNU General
+** Public License version 3.0 requirements will be met:
+** http://www.gnu.org/copyleft/gpl.html.
+**
+** Other Usage
+** Alternatively, this file may be used in accordance with the terms and
+** conditions contained in a signed written agreement between you and Nokia.
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef MFVIDEOPROBECONTROL_H
+#define MFVIDEOPROBECONTROL_H
+
+#include <qmediavideoprobecontrol.h>
+#include <QtCore/qmutex.h>
+#include <qvideoframe.h>
+
+QT_USE_NAMESPACE
+
+class MFVideoProbeControl : public QMediaVideoProbeControl
+{
+    Q_OBJECT
+public:
+    explicit MFVideoProbeControl(QObject *parent);
+    virtual ~MFVideoProbeControl();
+
+    void bufferProbed(const QVideoFrame& frame);
+};
+
+#endif

src/plugins/wmf/player/player.pri

@@ -11,7 +11,8 @@ HEADERS += \
     $$PWD/mfvideorenderercontrol.h \
     $$PWD/mfaudioendpointcontrol.h \
     $$PWD/mfmetadatacontrol.h \
-    $$PWD/mfaudioprobecontrol.h
+    $$PWD/mfaudioprobecontrol.h \
+    $$PWD/mfvideoprobecontrol.h
 
 SOURCES += \
     $$PWD/mfplayerservice.cpp \
@@ -20,7 +21,8 @@ SOURCES += \
     $$PWD/mfvideorenderercontrol.cpp \
     $$PWD/mfaudioendpointcontrol.cpp \
     $$PWD/mfmetadatacontrol.cpp \
-    $$PWD/mfaudioprobecontrol.cpp
+    $$PWD/mfaudioprobecontrol.cpp \
+    $$PWD/mfvideoprobecontrol.cpp
 
 !isEmpty(QT.widgets.name):!simulator {
     HEADERS += $$PWD/evr9videowindowcontrol.h

src/plugins/wmf/wmf.pro

@@ -19,13 +19,15 @@ HEADERS += \
     wmfserviceplugin.h \
     mfstream.h \
     sourceresolver.h \
-    samplegrabber.h
+    samplegrabber.h \
+    mftvideo.h
 
 SOURCES += \
     wmfserviceplugin.cpp \
     mfstream.cpp \
     sourceresolver.cpp \
-    samplegrabber.cpp
+    samplegrabber.cpp \
+    mftvideo.cpp
 
 include (player/player.pri)
 include (decoder/decoder.pri)