winrt: Introduce DirectX pipeline bypass.

Qt Windows Runtime camera uses DirectVideo to convert NV12 format
texture to BGRA format texture. As the EGL Node can draw using NV12
already, allow video render control to choose which path to take. By
default use DirectVideo. Bypass can be used as fallback when
DirectVideo cannot be used or is not working.

Task-Id: QTBUG-48331
Change-Id: I0cb87a7c4523bfb60610e6b41ab3fb05aff092a1
Reviewed-by: Andrew Knight <andrew.knight@intopalo.com>

src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp

@@ -182,6 +182,8 @@ public:
 
     QThread renderThread;
     bool active;
+    QWinRTAbstractVideoRendererControl::BlitMode blitMode;
+    CRITICAL_SECTION mutex;
 };
 
 ID3D11Device *QWinRTAbstractVideoRendererControl::d3dDevice()
@@ -212,6 +214,8 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi
     d->eglConfig = 0;
     d->eglSurface = EGL_NO_SURFACE;
     d->active = false;
+    d->blitMode = DirectVideo;
+    InitializeCriticalSectionEx(&d->mutex, 0, 0);
 
     connect(&d->renderThread, &QThread::started,
             this, &QWinRTAbstractVideoRendererControl::syncAndRender,
@@ -220,7 +224,11 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi
 
 QWinRTAbstractVideoRendererControl::~QWinRTAbstractVideoRendererControl()
 {
+    Q_D(QWinRTAbstractVideoRendererControl);
+    CriticalSectionLocker locker(&d->mutex);
     shutdown();
+    DeleteCriticalSection(&d->mutex);
+    eglDestroySurface(d->eglDisplay, d->eglSurface);
 }
 
 QAbstractVideoSurface *QWinRTAbstractVideoRendererControl::surface() const
@@ -244,7 +252,8 @@ void QWinRTAbstractVideoRendererControl::syncAndRender()
     forever {
         if (currentThread->isInterruptionRequested())
             break;
-
+        {
+            CriticalSectionLocker lock(&d->mutex);
             HRESULT hr;
             if (d->dirtyState == TextureDirty) {
                 CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1);
@@ -263,13 +272,26 @@ void QWinRTAbstractVideoRendererControl::syncAndRender()
             hr = g->output->WaitForVBlank();
             CONTINUE_IF_FAILED("Failed to wait for vertical blank");
 
-        if (!render(d->texture.Get()))
+            bool success = false;
+            switch (d->blitMode) {
+            case DirectVideo:
+                success = render(d->texture.Get());
+                break;
+            case MediaFoundation:
+                success = dequeueFrame(&d->presentFrame);
+                break;
+            default:
+                success = false;
+            }
+
+            if (!success)
                 continue;
 
             // Queue to the control's thread for presentation
             present.invoke(this, Qt::QueuedConnection);
             currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents);
         }
+    }
 
     // All done, exit render loop
     currentThread->quit();
@@ -326,7 +348,44 @@ void QWinRTAbstractVideoRendererControl::setActive(bool active)
         d->surface->stop();
 }
 
-void QWinRTAbstractVideoRendererControl::present()
+QWinRTAbstractVideoRendererControl::BlitMode QWinRTAbstractVideoRendererControl::blitMode() const
+{
+    Q_D(const QWinRTAbstractVideoRendererControl);
+    return d->blitMode;
+}
+
+void QWinRTAbstractVideoRendererControl::setBlitMode(QWinRTAbstractVideoRendererControl::BlitMode mode)
+{
+    Q_D(QWinRTAbstractVideoRendererControl);
+    CriticalSectionLocker lock(&d->mutex);
+
+    if (d->blitMode == mode)
+        return;
+
+    d->blitMode = mode;
+    d->dirtyState = d->blitMode == MediaFoundation ? NotDirty : TextureDirty;
+
+    if (d->blitMode == DirectVideo)
+        return;
+
+    if (d->texture) {
+        d->texture.Reset();
+        d->shareHandle = 0;
+    }
+
+    if (d->eglSurface) {
+        eglDestroySurface(d->eglDisplay, d->eglSurface);
+        d->eglSurface = EGL_NO_SURFACE;
+    }
+}
+
+bool QWinRTAbstractVideoRendererControl::dequeueFrame(QVideoFrame *frame)
+{
+    Q_UNUSED(frame)
+    return false;
+}
+
+void QWinRTAbstractVideoRendererControl::textureToFrame()
 {
     Q_D(QWinRTAbstractVideoRendererControl);
 
@@ -387,6 +446,13 @@ void QWinRTAbstractVideoRendererControl::present()
 
         d->dirtyState = NotDirty;
     }
+}
+
+void QWinRTAbstractVideoRendererControl::present()
+{
+    Q_D(QWinRTAbstractVideoRendererControl);
+    if (d->blitMode == DirectVideo)
+        textureToFrame();
 
     // Present the frame
     d->surface->present(d->presentFrame);

src/plugins/winrt/qwinrtabstractvideorenderercontrol.h

@@ -40,6 +40,8 @@
 #include <QtMultimedia/QVideoRendererControl>
 #include <QtMultimedia/QVideoSurfaceFormat>
 
+#include <qt_windows.h>
+
 struct ID3D11Device;
 struct ID3D11Texture2D;
 
@@ -53,6 +55,11 @@ public:
     explicit QWinRTAbstractVideoRendererControl(const QSize &size, QObject *parent = 0);
     ~QWinRTAbstractVideoRendererControl();
 
+    enum BlitMode {
+        DirectVideo,
+        MediaFoundation
+    };
+
     QAbstractVideoSurface *surface() const Q_DECL_OVERRIDE;
     void setSurface(QAbstractVideoSurface *surface) Q_DECL_OVERRIDE;
 
@@ -63,7 +70,11 @@ public:
 
     void setActive(bool active);
 
+    BlitMode blitMode() const;
+    void setBlitMode(BlitMode mode);
+
     virtual bool render(ID3D11Texture2D *texture) = 0;
+    virtual bool dequeueFrame(QVideoFrame *frame);
 
     static ID3D11Device *d3dDevice();
 
@@ -74,12 +85,29 @@ private slots:
     void syncAndRender();
 
 private:
+    void textureToFrame();
     Q_INVOKABLE void present();
 
     QScopedPointer<QWinRTAbstractVideoRendererControlPrivate> d_ptr;
     Q_DECLARE_PRIVATE(QWinRTAbstractVideoRendererControl)
 };
 
+class CriticalSectionLocker
+{
+public:
+    CriticalSectionLocker(CRITICAL_SECTION *section)
+        : m_section(section)
+    {
+        EnterCriticalSection(m_section);
+    }
+    ~CriticalSectionLocker()
+    {
+        LeaveCriticalSection(m_section);
+    }
+private:
+    CRITICAL_SECTION *m_section;
+};
+
 QT_END_NAMESPACE
 
 #endif // QWINRTABSTRACTVIDEORENDERERCONTROL_H

src/plugins/winrt/qwinrtcameracontrol.cpp

@@ -198,22 +198,6 @@ private:
     ComPtr<IRegionOfInterest> regionOfInterest;
 };
 
-class CriticalSectionLocker
-{
-public:
-    CriticalSectionLocker(CRITICAL_SECTION *section)
-        : m_section(section)
-    {
-        EnterCriticalSection(m_section);
-    }
-    ~CriticalSectionLocker()
-    {
-        LeaveCriticalSection(m_section);
-    }
-private:
-    CRITICAL_SECTION *m_section;
-};
-
 class MediaStream : public RuntimeClass<RuntimeClassFlags<WinRtClassicComMix>, IMFStreamSink, IMFMediaEventGenerator, IMFMediaTypeHandler>
 {
     enum Flags { NoFlag = 0, BufferLockRequired = 1 };

src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp

@@ -201,12 +201,15 @@ public:
     QVideoFrame::PixelFormat cameraSampleformat;
     int cameraSampleSize;
     uint videoProbesCounter;
-    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer);
+    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer,
+                             QWinRTAbstractVideoRendererControl::BlitMode *mode);
     ComPtr<IMF2DBuffer> dequeueBuffer();
 };
 
-bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer)
+bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer,
+                                                                  QWinRTAbstractVideoRendererControl::BlitMode *mode)
 {
+    Q_ASSERT(mode);
     ComPtr<ID3D11Texture2D> sourceTexture;
     ComPtr<IMFDXGIBuffer> dxgiBuffer;
     HRESULT hr = buffer.As(&dxgiBuffer);
@@ -219,6 +222,10 @@ bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<I
     }
     D3D11_TEXTURE2D_DESC desc;
     sourceTexture->GetDesc(&desc);
+
+    if (!(desc.MiscFlags & D3D11_RESOURCE_MISC_SHARED))
+        *mode = QWinRTAbstractVideoRendererControl::MediaFoundation;
+
     switch (desc.Format) {
     case DXGI_FORMAT_R8G8B8A8_TYPELESS:
         cameraSampleformat = QVideoFrame::Format_ARGB32;
@@ -281,21 +288,40 @@ bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target)
     return true;
 }
 
+bool QWinRTCameraVideoRendererControl::dequeueFrame(QVideoFrame *frame)
+{
+    Q_ASSERT(frame);
+    Q_D(QWinRTCameraVideoRendererControl);
+
+    ComPtr<IMF2DBuffer> buffer = d->dequeueBuffer();
+    if (!buffer || d->cameraSampleformat == QVideoFrame::Format_Invalid) {
+        emit bufferRequested();
+        return false;
+    }
+
+    QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer.Get(), d->cameraSampleSize);
+    *frame = QVideoFrame(videoBuffer, size(), d->cameraSampleformat);
+
+    emit bufferRequested();
+    return true;
+}
+
 void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer)
 {
     Q_D(QWinRTCameraVideoRendererControl);
     Q_ASSERT(buffer);
 
-    if (d->videoProbesCounter > 0) {
+    if (d->cameraSampleformat == QVideoFrame::Format_User) {
-        if (d->cameraSampleformat == QVideoFrame::Format_User)
+        BlitMode mode = blitMode();
-            d->getCameraSampleInfo(buffer);
+        d->getCameraSampleInfo(buffer, &mode);
+        setBlitMode(mode);
+    }
 
-        if (d->cameraSampleformat != QVideoFrame::Format_Invalid) {
+    if (d->videoProbesCounter > 0 && d->cameraSampleformat != QVideoFrame::Format_Invalid) {
         QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize);
         QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat);
         emit videoFrameProbed(frame);
     }
-    }
 
     const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE;
     if (d->readIndex == writeIndex) // Drop new sample if queue is full

src/plugins/winrt/qwinrtcameravideorenderercontrol.h

@@ -56,6 +56,7 @@ public:
     ~QWinRTCameraVideoRendererControl();
 
     bool render(ID3D11Texture2D *texture) Q_DECL_OVERRIDE;
+    bool dequeueFrame(QVideoFrame *frame) Q_DECL_OVERRIDE;
     void queueBuffer(IMF2DBuffer *buffer);
     void discardBuffers();
     void incrementProbe();