winrt: Add camera video probe controls

[ChangLog][multimedia][winrt] The WinRT backend now supports QVideoProbes
on camera objects.

Task-number: QTBUG-46228
Change-Id: I7850c5ec6f61e5824064d4be8afc8a0b55d05806
Reviewed-by: Andrew Knight <andrew.knight@intopalo.com>

src/plugins/winrt/qwinrtcameracontrol.cpp

@@ -630,6 +630,7 @@ void QWinRTCameraControl::setState(QCamera::State state)
         emit stateChanged(d->state);
         d->status = QCamera::ActiveStatus;
         emit statusChanged(d->status);
+        d->mediaSink->RequestSample();
         break;
     }
     case QCamera::LoadedState: {

src/plugins/winrt/qwinrtcameraservice.cpp

@@ -37,6 +37,8 @@
 #include "qwinrtcameraservice.h"
 #include "qwinrtcameracontrol.h"
 #include "qwinrtcamerainfocontrol.h"
+#include "qwinrtvideoprobecontrol.h"
+#include "qwinrtcameravideorenderercontrol.h"
 
 #include <QtCore/QCoreApplication>
 #include <QtCore/qfunctions_winrt.h>
@@ -47,6 +49,7 @@
 #include <QtMultimedia/QImageEncoderControl>
 #include <QtMultimedia/QCameraFocusControl>
 #include <QtMultimedia/QCameraLocksControl>
+#include <QtMultimedia/QMediaVideoProbeControl>
 
 QT_BEGIN_NAMESPACE
 
@@ -98,12 +101,17 @@ QMediaControl *QWinRTCameraService::requestControl(const char *name)
     if (qstrcmp(name, QCameraLocksControl_iid) == 0)
         return d->cameraControl->cameraLocksControl();
 
+    if (qstrcmp(name, QMediaVideoProbeControl_iid) == 0)
+        return new QWinRTVideoProbeControl(qobject_cast<QWinRTCameraVideoRendererControl *>(d->cameraControl->videoRenderer()));
+
     return nullptr;
 }
 
 void QWinRTCameraService::releaseControl(QMediaControl *control)
 {
-    Q_UNUSED(control);
+    Q_ASSERT(control);
+    if (QWinRTVideoProbeControl *videoProbe = qobject_cast<QWinRTVideoProbeControl *>(control))
+        videoProbe->deleteLater();
 }
 
 QT_END_NAMESPACE

src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp

@@ -39,6 +39,7 @@
 #include <QtCore/qfunctions_winrt.h>
 #include <QtCore/QSize>
 #include <QtCore/QVector>
+#include <QVideoFrame>
 
 #include <d3d11.h>
 #include <mfapi.h>
@@ -47,6 +48,60 @@ using namespace Microsoft::WRL;
 
 QT_BEGIN_NAMESPACE
 
+class QWinRTCameraVideoBuffer : public QAbstractVideoBuffer
+{
+public:
+    QWinRTCameraVideoBuffer(IMF2DBuffer *buffer, int size)
+        : QAbstractVideoBuffer(NoHandle)
+        , currentMode(NotMapped)
+        , buffer(buffer)
+        , size(size)
+    {
+    }
+
+    ~QWinRTCameraVideoBuffer()
+    {
+        unmap();
+    }
+
+    MapMode mapMode() const Q_DECL_OVERRIDE
+    {
+        return currentMode;
+    }
+
+    uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) Q_DECL_OVERRIDE
+    {
+        if (currentMode != NotMapped || mode == NotMapped)
+            return nullptr;
+
+        BYTE *bytes;
+        LONG stride;
+        HRESULT hr = buffer->Lock2D(&bytes, &stride);
+        RETURN_IF_FAILED("Failed to lock camera frame buffer", nullptr);
+
+        if (bytesPerLine)
+            *bytesPerLine = stride;
+        if (numBytes)
+            *numBytes = size;
+        currentMode = mode;
+        return bytes;
+    }
+
+    void unmap() Q_DECL_OVERRIDE
+    {
+        if (currentMode == NotMapped)
+            return;
+        HRESULT hr = buffer->Unlock2D();
+        RETURN_VOID_IF_FAILED("Failed to unlock camera frame buffer");
+        currentMode = NotMapped;
+    }
+
+private:
+    ComPtr<IMF2DBuffer> buffer;
+    MapMode currentMode;
+    int size;
+};
+
 class D3DVideoBlitter
 {
 public:
@@ -143,11 +198,52 @@ public:
     ComPtr<IMF2DBuffer> buffers[CAMERA_SAMPLE_QUEUE_SIZE];
     QAtomicInteger<quint16> writeIndex;
     QAtomicInteger<quint16> readIndex;
+    QVideoFrame::PixelFormat cameraSampleformat;
+    int cameraSampleSize;
+    uint videoProbesCounter;
+    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer);
+    ComPtr<IMF2DBuffer> dequeueBuffer();
 };
 
+bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer)
+{
+    ComPtr<ID3D11Texture2D> sourceTexture;
+    ComPtr<IMFDXGIBuffer> dxgiBuffer;
+    HRESULT hr = buffer.As(&dxgiBuffer);
+    Q_ASSERT_SUCCEEDED(hr);
+    hr = dxgiBuffer->GetResource(IID_PPV_ARGS(&sourceTexture));
+    if (FAILED(hr)) {
+        qErrnoWarning(hr, "The video frame does not support texture output");
+        cameraSampleformat = QVideoFrame::Format_Invalid;
+        return false;
+    }
+    D3D11_TEXTURE2D_DESC desc;
+    sourceTexture->GetDesc(&desc);
+    switch (desc.Format) {
+    case DXGI_FORMAT_R8G8B8A8_TYPELESS:
+        cameraSampleformat = QVideoFrame::Format_ARGB32;
+        break;
+    case DXGI_FORMAT_NV12:
+        cameraSampleformat = QVideoFrame::Format_NV12;
+        break;
+    default:
+        cameraSampleformat = QVideoFrame::Format_Invalid;
+        qErrnoWarning("Unsupported camera probe format.");
+        return false;
+    }
+    DWORD pcbLength;
+    hr = buffer->GetContiguousLength(&pcbLength);
+    Q_ASSERT_SUCCEEDED(hr);
+    cameraSampleSize = pcbLength;
+    return true;
+}
+
 QWinRTCameraVideoRendererControl::QWinRTCameraVideoRendererControl(const QSize &size, QObject *parent)
     : QWinRTAbstractVideoRendererControl(size, parent), d_ptr(new QWinRTCameraVideoRendererControlPrivate)
 {
+    Q_D(QWinRTCameraVideoRendererControl);
+    d->cameraSampleformat = QVideoFrame::Format_User;
+    d->videoProbesCounter = 0;
 }
 
 QWinRTCameraVideoRendererControl::~QWinRTCameraVideoRendererControl()
@@ -158,22 +254,15 @@ QWinRTCameraVideoRendererControl::~QWinRTCameraVideoRendererControl()
 bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target)
 {
     Q_D(QWinRTCameraVideoRendererControl);
-
+    ComPtr<IMF2DBuffer> buffer = d->dequeueBuffer();
-    const quint16 readIndex = d->readIndex;
+    if (!buffer) {
-    if (readIndex == d->writeIndex) {
         emit bufferRequested();
         return false;
     }
 
-    HRESULT hr;
-    ComPtr<IMF2DBuffer> buffer = d->buffers[readIndex];
-    Q_ASSERT(buffer);
-    d->buffers[readIndex].Reset();
-    d->readIndex = (readIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE;
-
     ComPtr<ID3D11Texture2D> sourceTexture;
     ComPtr<IMFDXGIBuffer> dxgiBuffer;
-    hr = buffer.As(&dxgiBuffer);
+    HRESULT hr = buffer.As(&dxgiBuffer);
     Q_ASSERT_SUCCEEDED(hr);
     hr = dxgiBuffer->GetResource(IID_PPV_ARGS(&sourceTexture));
     if (FAILED(hr)) {
@@ -196,11 +285,41 @@ void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer)
 {
     Q_D(QWinRTCameraVideoRendererControl);
     Q_ASSERT(buffer);
+
+    if (d->videoProbesCounter > 0) {
+        if (d->cameraSampleformat == QVideoFrame::Format_User)
+            d->getCameraSampleInfo(buffer);
+
+        if (d->cameraSampleformat != QVideoFrame::Format_Invalid) {
+            QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize);
+            QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat);
+            emit videoFrameProbed(frame);
+        }
+    }
+
     const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE;
     if (d->readIndex == writeIndex) // Drop new sample if queue is full
         return;
     d->buffers[d->writeIndex] = buffer;
     d->writeIndex = writeIndex;
+
+    if (!surface()) {
+        d->dequeueBuffer();
+        emit bufferRequested();
+    }
+}
+
+ComPtr<IMF2DBuffer> QWinRTCameraVideoRendererControlPrivate::dequeueBuffer()
+{
+    const quint16 currentReadIndex = readIndex;
+    if (currentReadIndex == writeIndex)
+        return nullptr;
+
+    ComPtr<IMF2DBuffer> buffer = buffers[currentReadIndex];
+    Q_ASSERT(buffer);
+    buffers[currentReadIndex].Reset();
+    readIndex = (currentReadIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE;
+    return buffer;
 }
 
 void QWinRTCameraVideoRendererControl::discardBuffers()
@@ -211,4 +330,17 @@ void QWinRTCameraVideoRendererControl::discardBuffers()
         buffer.Reset();
 }
 
+void QWinRTCameraVideoRendererControl::incrementProbe()
+{
+    Q_D(QWinRTCameraVideoRendererControl);
+    ++d->videoProbesCounter;
+}
+
+void QWinRTCameraVideoRendererControl::decrementProbe()
+{
+    Q_D(QWinRTCameraVideoRendererControl);
+    Q_ASSERT(d->videoProbesCounter > 0);
+    --d->videoProbesCounter;
+}
+
 QT_END_NAMESPACE

src/plugins/winrt/qwinrtcameravideorenderercontrol.h

@@ -39,10 +39,13 @@
 
 #include "qwinrtabstractvideorenderercontrol.h"
 
+#include <QVideoFrame>
+
 struct IMF2DBuffer;
 
 QT_BEGIN_NAMESPACE
 
+class QWinRTVideoProbeControl;
 class QVideoSurfaceFormat;
 class QWinRTCameraVideoRendererControlPrivate;
 class QWinRTCameraVideoRendererControl : public QWinRTAbstractVideoRendererControl
@@ -55,9 +58,12 @@ public:
     bool render(ID3D11Texture2D *texture) Q_DECL_OVERRIDE;
     void queueBuffer(IMF2DBuffer *buffer);
     void discardBuffers();
+    void incrementProbe();
+    void decrementProbe();
 
 signals:
     void bufferRequested();
+    void videoFrameProbed(const QVideoFrame &frame);
 
 private:
     QScopedPointer<QWinRTCameraVideoRendererControlPrivate> d_ptr;

src/plugins/winrt/qwinrtvideoprobecontrol.cpp

@@ -0,0 +1,55 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd and/or its subsidiary(-ies).
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#include "qwinrtvideoprobecontrol.h"
+#include "qwinrtcameravideorenderercontrol.h"
+
+QT_BEGIN_NAMESPACE
+
+QWinRTVideoProbeControl::QWinRTVideoProbeControl(QWinRTCameraVideoRendererControl *parent)
+    : QMediaVideoProbeControl(parent)
+{
+    QObject::connect(parent, &QWinRTCameraVideoRendererControl::videoFrameProbed,
+                     this, &QMediaVideoProbeControl::videoFrameProbed, Qt::QueuedConnection);
+    parent->incrementProbe();
+}
+
+QWinRTVideoProbeControl::~QWinRTVideoProbeControl()
+{
+    if (QWinRTCameraVideoRendererControl *renderer = qobject_cast<QWinRTCameraVideoRendererControl *>(parent()))
+        renderer->decrementProbe();
+}
+
+QT_END_NAMESPACE

src/plugins/winrt/qwinrtvideoprobecontrol.h

@@ -0,0 +1,54 @@
+/****************************************************************************
+**
+** Copyright (C) 2015 The Qt Company Ltd and/or its subsidiary(-ies).
+** Contact: http://www.qt.io/licensing/
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL3$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see http://www.qt.io/terms-conditions. For further
+** information use the contact form at http://www.qt.io/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 3 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPLv3 included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 3 requirements
+** will be met: https://www.gnu.org/licenses/lgpl.html.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 2.0 or later as published by the Free
+** Software Foundation and appearing in the file LICENSE.GPL included in
+** the packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 2.0 requirements will be
+** met: http://www.gnu.org/licenses/gpl-2.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+#ifndef QWINRTVIDEOPROBECONTROL_H
+#define QWINRTVIDEOPROBECONTROL_H
+
+#include <qmediavideoprobecontrol.h>
+
+QT_BEGIN_NAMESPACE
+
+class QWinRTCameraVideoRendererControl;
+class QWinRTVideoProbeControl : public QMediaVideoProbeControl
+{
+    Q_OBJECT
+public:
+    explicit QWinRTVideoProbeControl(QWinRTCameraVideoRendererControl *parent);
+    ~QWinRTVideoProbeControl();
+};
+
+QT_END_NAMESPACE
+
+#endif // QWINRTVIDEOPROBECONTROL_H

src/plugins/winrt/winrt.pro

@@ -21,7 +21,8 @@ HEADERS += \
     qwinrtmediaplayerservice.h \
     qwinrtplayerrenderercontrol.h \
     qwinrtserviceplugin.h \
-    qwinrtvideodeviceselectorcontrol.h
+    qwinrtvideodeviceselectorcontrol.h \
+    qwinrtvideoprobecontrol.h
 
 SOURCES += \
     qwinrtabstractvideorenderercontrol.cpp \
@@ -37,7 +38,8 @@ SOURCES += \
     qwinrtmediaplayerservice.cpp \
     qwinrtplayerrenderercontrol.cpp \
     qwinrtserviceplugin.cpp \
-    qwinrtvideodeviceselectorcontrol.cpp
+    qwinrtvideodeviceselectorcontrol.cpp \
+    qwinrtvideoprobecontrol.cpp
 
 OTHER_FILES += \
     winrt.json