winrt: Introduce DirectX pipeline bypass.

Qt Windows Runtime camera uses DirectVideo to convert NV12 format
texture to BGRA format texture. As the EGL Node can draw using NV12
already, allow video render control to choose which path to take. By
default use DirectVideo. Bypass can be used as fallback when
DirectVideo cannot be used or is not working.

Task-Id: QTBUG-48331
Change-Id: I0cb87a7c4523bfb60610e6b41ab3fb05aff092a1
Reviewed-by: Andrew Knight <andrew.knight@intopalo.com>

src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp

@@ -182,6 +182,8 @@ public:
 
     QThread renderThread;
     bool active;
+    QWinRTAbstractVideoRendererControl::BlitMode blitMode;
+    CRITICAL_SECTION mutex;
 };
 
 ID3D11Device *QWinRTAbstractVideoRendererControl::d3dDevice()
@@ -212,6 +214,8 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi
     d->eglConfig = 0;
     d->eglSurface = EGL_NO_SURFACE;
     d->active = false;
+    d->blitMode = DirectVideo;
+    InitializeCriticalSectionEx(&d->mutex, 0, 0);
 
     connect(&d->renderThread, &QThread::started,
             this, &QWinRTAbstractVideoRendererControl::syncAndRender,
@@ -220,7 +224,11 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi
 
 QWinRTAbstractVideoRendererControl::~QWinRTAbstractVideoRendererControl()
 {
+    Q_D(QWinRTAbstractVideoRendererControl);
+    CriticalSectionLocker locker(&d->mutex);
     shutdown();
+    DeleteCriticalSection(&d->mutex);
+    eglDestroySurface(d->eglDisplay, d->eglSurface);
 }
 
 QAbstractVideoSurface *QWinRTAbstractVideoRendererControl::surface() const
@@ -244,31 +252,45 @@ void QWinRTAbstractVideoRendererControl::syncAndRender()
     forever {
         if (currentThread->isInterruptionRequested())
             break;
+        {
+            CriticalSectionLocker lock(&d->mutex);
+            HRESULT hr;
+            if (d->dirtyState == TextureDirty) {
+                CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1);
+                desc.BindFlags |= D3D11_BIND_RENDER_TARGET;
+                desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
+                hr = g->device->CreateTexture2D(&desc, NULL, d->texture.ReleaseAndGetAddressOf());
+                BREAK_IF_FAILED("Failed to get create video texture");
+                ComPtr<IDXGIResource> resource;
+                hr = d->texture.As(&resource);
+                BREAK_IF_FAILED("Failed to cast texture to resource");
+                hr = resource->GetSharedHandle(&d->shareHandle);
+                BREAK_IF_FAILED("Failed to get texture share handle");
+                d->dirtyState = SurfaceDirty;
+            }
 
-        HRESULT hr;
+            hr = g->output->WaitForVBlank();
-        if (d->dirtyState == TextureDirty) {
+            CONTINUE_IF_FAILED("Failed to wait for vertical blank");
-            CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1);
+
-            desc.BindFlags |= D3D11_BIND_RENDER_TARGET;
+            bool success = false;
-            desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED;
+            switch (d->blitMode) {
-            hr = g->device->CreateTexture2D(&desc, NULL, d->texture.ReleaseAndGetAddressOf());
+            case DirectVideo:
-            BREAK_IF_FAILED("Failed to get create video texture");
+                success = render(d->texture.Get());
-            ComPtr<IDXGIResource> resource;
+                break;
-            hr = d->texture.As(&resource);
+            case MediaFoundation:
-            BREAK_IF_FAILED("Failed to cast texture to resource");
+                success = dequeueFrame(&d->presentFrame);
-            hr = resource->GetSharedHandle(&d->shareHandle);
+                break;
-            BREAK_IF_FAILED("Failed to get texture share handle");
+            default:
-            d->dirtyState = SurfaceDirty;
+                success = false;
+            }
+
+            if (!success)
+                continue;
+
+            // Queue to the control's thread for presentation
+            present.invoke(this, Qt::QueuedConnection);
+            currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents);
         }
-
-        hr = g->output->WaitForVBlank();
-        CONTINUE_IF_FAILED("Failed to wait for vertical blank");
-
-        if (!render(d->texture.Get()))
-            continue;
-
-        // Queue to the control's thread for presentation
-        present.invoke(this, Qt::QueuedConnection);
-        currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents);
     }
 
     // All done, exit render loop
@@ -326,7 +348,44 @@ void QWinRTAbstractVideoRendererControl::setActive(bool active)
         d->surface->stop();
 }
 
-void QWinRTAbstractVideoRendererControl::present()
+QWinRTAbstractVideoRendererControl::BlitMode QWinRTAbstractVideoRendererControl::blitMode() const
+{
+    Q_D(const QWinRTAbstractVideoRendererControl);
+    return d->blitMode;
+}
+
+void QWinRTAbstractVideoRendererControl::setBlitMode(QWinRTAbstractVideoRendererControl::BlitMode mode)
+{
+    Q_D(QWinRTAbstractVideoRendererControl);
+    CriticalSectionLocker lock(&d->mutex);
+
+    if (d->blitMode == mode)
+        return;
+
+    d->blitMode = mode;
+    d->dirtyState = d->blitMode == MediaFoundation ? NotDirty : TextureDirty;
+
+    if (d->blitMode == DirectVideo)
+        return;
+
+    if (d->texture) {
+        d->texture.Reset();
+        d->shareHandle = 0;
+    }
+
+    if (d->eglSurface) {
+        eglDestroySurface(d->eglDisplay, d->eglSurface);
+        d->eglSurface = EGL_NO_SURFACE;
+    }
+}
+
+bool QWinRTAbstractVideoRendererControl::dequeueFrame(QVideoFrame *frame)
+{
+    Q_UNUSED(frame)
+    return false;
+}
+
+void QWinRTAbstractVideoRendererControl::textureToFrame()
 {
     Q_D(QWinRTAbstractVideoRendererControl);
 
@@ -387,6 +446,13 @@ void QWinRTAbstractVideoRendererControl::present()
 
         d->dirtyState = NotDirty;
     }
+}
+
+void QWinRTAbstractVideoRendererControl::present()
+{
+    Q_D(QWinRTAbstractVideoRendererControl);
+    if (d->blitMode == DirectVideo)
+        textureToFrame();
 
     // Present the frame
     d->surface->present(d->presentFrame);

src/plugins/winrt/qwinrtabstractvideorenderercontrol.h

@@ -40,6 +40,8 @@
 #include <QtMultimedia/QVideoRendererControl>
 #include <QtMultimedia/QVideoSurfaceFormat>
 
+#include <qt_windows.h>
+
 struct ID3D11Device;
 struct ID3D11Texture2D;
 
@@ -53,6 +55,11 @@ public:
     explicit QWinRTAbstractVideoRendererControl(const QSize &size, QObject *parent = 0);
     ~QWinRTAbstractVideoRendererControl();
 
+    enum BlitMode {
+        DirectVideo,
+        MediaFoundation
+    };
+
     QAbstractVideoSurface *surface() const Q_DECL_OVERRIDE;
     void setSurface(QAbstractVideoSurface *surface) Q_DECL_OVERRIDE;
 
@@ -63,7 +70,11 @@ public:
 
     void setActive(bool active);
 
+    BlitMode blitMode() const;
+    void setBlitMode(BlitMode mode);
+
     virtual bool render(ID3D11Texture2D *texture) = 0;
+    virtual bool dequeueFrame(QVideoFrame *frame);
 
     static ID3D11Device *d3dDevice();
 
@@ -74,12 +85,29 @@ private slots:
     void syncAndRender();
 
 private:
+    void textureToFrame();
     Q_INVOKABLE void present();
 
     QScopedPointer<QWinRTAbstractVideoRendererControlPrivate> d_ptr;
     Q_DECLARE_PRIVATE(QWinRTAbstractVideoRendererControl)
 };
 
+class CriticalSectionLocker
+{
+public:
+    CriticalSectionLocker(CRITICAL_SECTION *section)
+        : m_section(section)
+    {
+        EnterCriticalSection(m_section);
+    }
+    ~CriticalSectionLocker()
+    {
+        LeaveCriticalSection(m_section);
+    }
+private:
+    CRITICAL_SECTION *m_section;
+};
+
 QT_END_NAMESPACE
 
 #endif // QWINRTABSTRACTVIDEORENDERERCONTROL_H

src/plugins/winrt/qwinrtcameracontrol.cpp

@@ -198,22 +198,6 @@ private:
     ComPtr<IRegionOfInterest> regionOfInterest;
 };
 
-class CriticalSectionLocker
-{
-public:
-    CriticalSectionLocker(CRITICAL_SECTION *section)
-        : m_section(section)
-    {
-        EnterCriticalSection(m_section);
-    }
-    ~CriticalSectionLocker()
-    {
-        LeaveCriticalSection(m_section);
-    }
-private:
-    CRITICAL_SECTION *m_section;
-};
-
 class MediaStream : public RuntimeClass<RuntimeClassFlags<WinRtClassicComMix>, IMFStreamSink, IMFMediaEventGenerator, IMFMediaTypeHandler>
 {
     enum Flags { NoFlag = 0, BufferLockRequired = 1 };

src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp

@@ -201,12 +201,15 @@ public:
     QVideoFrame::PixelFormat cameraSampleformat;
     int cameraSampleSize;
     uint videoProbesCounter;
-    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer);
+    bool getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer,
+                             QWinRTAbstractVideoRendererControl::BlitMode *mode);
     ComPtr<IMF2DBuffer> dequeueBuffer();
 };
 
-bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer)
+bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<IMF2DBuffer> &buffer,
+                                                                  QWinRTAbstractVideoRendererControl::BlitMode *mode)
 {
+    Q_ASSERT(mode);
     ComPtr<ID3D11Texture2D> sourceTexture;
     ComPtr<IMFDXGIBuffer> dxgiBuffer;
     HRESULT hr = buffer.As(&dxgiBuffer);
@@ -219,6 +222,10 @@ bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr<I
     }
     D3D11_TEXTURE2D_DESC desc;
     sourceTexture->GetDesc(&desc);
+
+    if (!(desc.MiscFlags & D3D11_RESOURCE_MISC_SHARED))
+        *mode = QWinRTAbstractVideoRendererControl::MediaFoundation;
+
     switch (desc.Format) {
     case DXGI_FORMAT_R8G8B8A8_TYPELESS:
         cameraSampleformat = QVideoFrame::Format_ARGB32;
@@ -281,20 +288,39 @@ bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target)
     return true;
 }
 
+bool QWinRTCameraVideoRendererControl::dequeueFrame(QVideoFrame *frame)
+{
+    Q_ASSERT(frame);
+    Q_D(QWinRTCameraVideoRendererControl);
+
+    ComPtr<IMF2DBuffer> buffer = d->dequeueBuffer();
+    if (!buffer || d->cameraSampleformat == QVideoFrame::Format_Invalid) {
+        emit bufferRequested();
+        return false;
+    }
+
+    QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer.Get(), d->cameraSampleSize);
+    *frame = QVideoFrame(videoBuffer, size(), d->cameraSampleformat);
+
+    emit bufferRequested();
+    return true;
+}
+
 void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer)
 {
     Q_D(QWinRTCameraVideoRendererControl);
     Q_ASSERT(buffer);
 
-    if (d->videoProbesCounter > 0) {
+    if (d->cameraSampleformat == QVideoFrame::Format_User) {
-        if (d->cameraSampleformat == QVideoFrame::Format_User)
+        BlitMode mode = blitMode();
-            d->getCameraSampleInfo(buffer);
+        d->getCameraSampleInfo(buffer, &mode);
+        setBlitMode(mode);
+    }
 
-        if (d->cameraSampleformat != QVideoFrame::Format_Invalid) {
+    if (d->videoProbesCounter > 0 && d->cameraSampleformat != QVideoFrame::Format_Invalid) {
-            QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize);
+        QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize);
-            QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat);
+        QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat);
-            emit videoFrameProbed(frame);
+        emit videoFrameProbed(frame);
-        }
     }
 
     const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE;

src/plugins/winrt/qwinrtcameravideorenderercontrol.h

@@ -56,6 +56,7 @@ public:
     ~QWinRTCameraVideoRendererControl();
 
     bool render(ID3D11Texture2D *texture) Q_DECL_OVERRIDE;
+    bool dequeueFrame(QVideoFrame *frame) Q_DECL_OVERRIDE;
     void queueBuffer(IMF2DBuffer *buffer);
     void discardBuffers();
     void incrementProbe();