From e489e7b5b1bf41101cb95785fd1aeb30e3a2f282 Mon Sep 17 00:00:00 2001 From: Samuel Nevala Date: Wed, 23 Sep 2015 15:18:39 +0300 Subject: [PATCH] winrt: Introduce DirectX pipeline bypass. Qt Windows Runtime camera uses DirectVideo to convert NV12 format texture to BGRA format texture. As the EGL Node can draw using NV12 already, allow video render control to choose which path to take. By default use DirectVideo. Bypass can be used as fallback when DirectVideo cannot be used or is not working. Task-Id: QTBUG-48331 Change-Id: I0cb87a7c4523bfb60610e6b41ab3fb05aff092a1 Reviewed-by: Andrew Knight --- .../qwinrtabstractvideorenderercontrol.cpp | 114 ++++++++++++++---- .../qwinrtabstractvideorenderercontrol.h | 28 +++++ src/plugins/winrt/qwinrtcameracontrol.cpp | 16 --- .../qwinrtcameravideorenderercontrol.cpp | 46 +++++-- .../winrt/qwinrtcameravideorenderercontrol.h | 1 + 5 files changed, 155 insertions(+), 50 deletions(-) diff --git a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp index 4a225e9c..cf62843f 100644 --- a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp +++ b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.cpp @@ -182,6 +182,8 @@ public: QThread renderThread; bool active; + QWinRTAbstractVideoRendererControl::BlitMode blitMode; + CRITICAL_SECTION mutex; }; ID3D11Device *QWinRTAbstractVideoRendererControl::d3dDevice() @@ -212,6 +214,8 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi d->eglConfig = 0; d->eglSurface = EGL_NO_SURFACE; d->active = false; + d->blitMode = DirectVideo; + InitializeCriticalSectionEx(&d->mutex, 0, 0); connect(&d->renderThread, &QThread::started, this, &QWinRTAbstractVideoRendererControl::syncAndRender, @@ -220,7 +224,11 @@ QWinRTAbstractVideoRendererControl::QWinRTAbstractVideoRendererControl(const QSi QWinRTAbstractVideoRendererControl::~QWinRTAbstractVideoRendererControl() { + Q_D(QWinRTAbstractVideoRendererControl); + CriticalSectionLocker locker(&d->mutex); shutdown(); + DeleteCriticalSection(&d->mutex); + eglDestroySurface(d->eglDisplay, d->eglSurface); } QAbstractVideoSurface *QWinRTAbstractVideoRendererControl::surface() const @@ -244,31 +252,45 @@ void QWinRTAbstractVideoRendererControl::syncAndRender() forever { if (currentThread->isInterruptionRequested()) break; + { + CriticalSectionLocker lock(&d->mutex); + HRESULT hr; + if (d->dirtyState == TextureDirty) { + CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1); + desc.BindFlags |= D3D11_BIND_RENDER_TARGET; + desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; + hr = g->device->CreateTexture2D(&desc, NULL, d->texture.ReleaseAndGetAddressOf()); + BREAK_IF_FAILED("Failed to get create video texture"); + ComPtr resource; + hr = d->texture.As(&resource); + BREAK_IF_FAILED("Failed to cast texture to resource"); + hr = resource->GetSharedHandle(&d->shareHandle); + BREAK_IF_FAILED("Failed to get texture share handle"); + d->dirtyState = SurfaceDirty; + } - HRESULT hr; - if (d->dirtyState == TextureDirty) { - CD3D11_TEXTURE2D_DESC desc(DXGI_FORMAT_B8G8R8A8_UNORM, d->format.frameWidth(), d->format.frameHeight(), 1, 1); - desc.BindFlags |= D3D11_BIND_RENDER_TARGET; - desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; - hr = g->device->CreateTexture2D(&desc, NULL, d->texture.ReleaseAndGetAddressOf()); - BREAK_IF_FAILED("Failed to get create video texture"); - ComPtr resource; - hr = d->texture.As(&resource); - BREAK_IF_FAILED("Failed to cast texture to resource"); - hr = resource->GetSharedHandle(&d->shareHandle); - BREAK_IF_FAILED("Failed to get texture share handle"); - d->dirtyState = SurfaceDirty; + hr = g->output->WaitForVBlank(); + CONTINUE_IF_FAILED("Failed to wait for vertical blank"); + + bool success = false; + switch (d->blitMode) { + case DirectVideo: + success = render(d->texture.Get()); + break; + case MediaFoundation: + success = dequeueFrame(&d->presentFrame); + break; + default: + success = false; + } + + if (!success) + continue; + + // Queue to the control's thread for presentation + present.invoke(this, Qt::QueuedConnection); + currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents); } - - hr = g->output->WaitForVBlank(); - CONTINUE_IF_FAILED("Failed to wait for vertical blank"); - - if (!render(d->texture.Get())) - continue; - - // Queue to the control's thread for presentation - present.invoke(this, Qt::QueuedConnection); - currentThread->eventDispatcher()->processEvents(QEventLoop::AllEvents); } // All done, exit render loop @@ -326,7 +348,44 @@ void QWinRTAbstractVideoRendererControl::setActive(bool active) d->surface->stop(); } -void QWinRTAbstractVideoRendererControl::present() +QWinRTAbstractVideoRendererControl::BlitMode QWinRTAbstractVideoRendererControl::blitMode() const +{ + Q_D(const QWinRTAbstractVideoRendererControl); + return d->blitMode; +} + +void QWinRTAbstractVideoRendererControl::setBlitMode(QWinRTAbstractVideoRendererControl::BlitMode mode) +{ + Q_D(QWinRTAbstractVideoRendererControl); + CriticalSectionLocker lock(&d->mutex); + + if (d->blitMode == mode) + return; + + d->blitMode = mode; + d->dirtyState = d->blitMode == MediaFoundation ? NotDirty : TextureDirty; + + if (d->blitMode == DirectVideo) + return; + + if (d->texture) { + d->texture.Reset(); + d->shareHandle = 0; + } + + if (d->eglSurface) { + eglDestroySurface(d->eglDisplay, d->eglSurface); + d->eglSurface = EGL_NO_SURFACE; + } +} + +bool QWinRTAbstractVideoRendererControl::dequeueFrame(QVideoFrame *frame) +{ + Q_UNUSED(frame) + return false; +} + +void QWinRTAbstractVideoRendererControl::textureToFrame() { Q_D(QWinRTAbstractVideoRendererControl); @@ -387,6 +446,13 @@ void QWinRTAbstractVideoRendererControl::present() d->dirtyState = NotDirty; } +} + +void QWinRTAbstractVideoRendererControl::present() +{ + Q_D(QWinRTAbstractVideoRendererControl); + if (d->blitMode == DirectVideo) + textureToFrame(); // Present the frame d->surface->present(d->presentFrame); diff --git a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h index ed8b76a6..70227c53 100644 --- a/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h +++ b/src/plugins/winrt/qwinrtabstractvideorenderercontrol.h @@ -40,6 +40,8 @@ #include #include +#include + struct ID3D11Device; struct ID3D11Texture2D; @@ -53,6 +55,11 @@ public: explicit QWinRTAbstractVideoRendererControl(const QSize &size, QObject *parent = 0); ~QWinRTAbstractVideoRendererControl(); + enum BlitMode { + DirectVideo, + MediaFoundation + }; + QAbstractVideoSurface *surface() const Q_DECL_OVERRIDE; void setSurface(QAbstractVideoSurface *surface) Q_DECL_OVERRIDE; @@ -63,7 +70,11 @@ public: void setActive(bool active); + BlitMode blitMode() const; + void setBlitMode(BlitMode mode); + virtual bool render(ID3D11Texture2D *texture) = 0; + virtual bool dequeueFrame(QVideoFrame *frame); static ID3D11Device *d3dDevice(); @@ -74,12 +85,29 @@ private slots: void syncAndRender(); private: + void textureToFrame(); Q_INVOKABLE void present(); QScopedPointer d_ptr; Q_DECLARE_PRIVATE(QWinRTAbstractVideoRendererControl) }; +class CriticalSectionLocker +{ +public: + CriticalSectionLocker(CRITICAL_SECTION *section) + : m_section(section) + { + EnterCriticalSection(m_section); + } + ~CriticalSectionLocker() + { + LeaveCriticalSection(m_section); + } +private: + CRITICAL_SECTION *m_section; +}; + QT_END_NAMESPACE #endif // QWINRTABSTRACTVIDEORENDERERCONTROL_H diff --git a/src/plugins/winrt/qwinrtcameracontrol.cpp b/src/plugins/winrt/qwinrtcameracontrol.cpp index 6abf3a05..b503007c 100644 --- a/src/plugins/winrt/qwinrtcameracontrol.cpp +++ b/src/plugins/winrt/qwinrtcameracontrol.cpp @@ -198,22 +198,6 @@ private: ComPtr regionOfInterest; }; -class CriticalSectionLocker -{ -public: - CriticalSectionLocker(CRITICAL_SECTION *section) - : m_section(section) - { - EnterCriticalSection(m_section); - } - ~CriticalSectionLocker() - { - LeaveCriticalSection(m_section); - } -private: - CRITICAL_SECTION *m_section; -}; - class MediaStream : public RuntimeClass, IMFStreamSink, IMFMediaEventGenerator, IMFMediaTypeHandler> { enum Flags { NoFlag = 0, BufferLockRequired = 1 }; diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp index 6c5575a1..d42cc577 100644 --- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp +++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp @@ -201,12 +201,15 @@ public: QVideoFrame::PixelFormat cameraSampleformat; int cameraSampleSize; uint videoProbesCounter; - bool getCameraSampleInfo(const ComPtr &buffer); + bool getCameraSampleInfo(const ComPtr &buffer, + QWinRTAbstractVideoRendererControl::BlitMode *mode); ComPtr dequeueBuffer(); }; -bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr &buffer) +bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtr &buffer, + QWinRTAbstractVideoRendererControl::BlitMode *mode) { + Q_ASSERT(mode); ComPtr sourceTexture; ComPtr dxgiBuffer; HRESULT hr = buffer.As(&dxgiBuffer); @@ -219,6 +222,10 @@ bool QWinRTCameraVideoRendererControlPrivate::getCameraSampleInfo(const ComPtrGetDesc(&desc); + + if (!(desc.MiscFlags & D3D11_RESOURCE_MISC_SHARED)) + *mode = QWinRTAbstractVideoRendererControl::MediaFoundation; + switch (desc.Format) { case DXGI_FORMAT_R8G8B8A8_TYPELESS: cameraSampleformat = QVideoFrame::Format_ARGB32; @@ -281,20 +288,39 @@ bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target) return true; } +bool QWinRTCameraVideoRendererControl::dequeueFrame(QVideoFrame *frame) +{ + Q_ASSERT(frame); + Q_D(QWinRTCameraVideoRendererControl); + + ComPtr buffer = d->dequeueBuffer(); + if (!buffer || d->cameraSampleformat == QVideoFrame::Format_Invalid) { + emit bufferRequested(); + return false; + } + + QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer.Get(), d->cameraSampleSize); + *frame = QVideoFrame(videoBuffer, size(), d->cameraSampleformat); + + emit bufferRequested(); + return true; +} + void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer) { Q_D(QWinRTCameraVideoRendererControl); Q_ASSERT(buffer); - if (d->videoProbesCounter > 0) { - if (d->cameraSampleformat == QVideoFrame::Format_User) - d->getCameraSampleInfo(buffer); + if (d->cameraSampleformat == QVideoFrame::Format_User) { + BlitMode mode = blitMode(); + d->getCameraSampleInfo(buffer, &mode); + setBlitMode(mode); + } - if (d->cameraSampleformat != QVideoFrame::Format_Invalid) { - QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize); - QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat); - emit videoFrameProbed(frame); - } + if (d->videoProbesCounter > 0 && d->cameraSampleformat != QVideoFrame::Format_Invalid) { + QWinRTCameraVideoBuffer *videoBuffer = new QWinRTCameraVideoBuffer(buffer, d->cameraSampleSize); + QVideoFrame frame(videoBuffer, size(), d->cameraSampleformat); + emit videoFrameProbed(frame); } const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE; diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.h b/src/plugins/winrt/qwinrtcameravideorenderercontrol.h index 122418de..76bff5e0 100644 --- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.h +++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.h @@ -56,6 +56,7 @@ public: ~QWinRTCameraVideoRendererControl(); bool render(ID3D11Texture2D *texture) Q_DECL_OVERRIDE; + bool dequeueFrame(QVideoFrame *frame) Q_DECL_OVERRIDE; void queueBuffer(IMF2DBuffer *buffer); void discardBuffers(); void incrementProbe();