From 6fe3061c1f6b7438d20c7bfb92c7652cedc5b049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Str=C3=B8mme?= Date: Thu, 16 Apr 2015 15:00:30 +0200 Subject: [PATCH 01/10] Restore ContentCamera.qml The file was removed in 5c3a5cf8106e1b873924b296c792448c33ee4df1 but left the description, documentation and parts of the functionality unchanged. This change adds the camera functionality back into the example. Change-Id: I3bfdd95f8322796d446c571a4e074ce98e5443dd Reviewed-by: Yoann Lopes --- .../qml/qmlvideofx/ContentCamera.qml | 43 +++++++++++++++++++ .../video/qmlvideofx/qmlvideofx.qrc | 1 + 2 files changed, 44 insertions(+) create mode 100644 examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml diff --git a/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml b/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml new file mode 100644 index 00000000..ef5a3d4a --- /dev/null +++ b/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml @@ -0,0 +1,43 @@ +/**************************************************************************** +** +** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Qt Mobility Components. +** +** $QT_BEGIN_LICENSE:LGPL21$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 or version 3 as published by the Free +** Software Foundation and appearing in the file LICENSE.LGPLv21 and +** LICENSE.LGPLv3 included in the packaging of this file. Please review the +** following information to ensure the GNU Lesser General Public License +** requirements will be met: https://www.gnu.org/licenses/lgpl.html and +** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +import QtQuick 2.1 +import QtMultimedia 5.0 + +VideoOutput { + source: camera + + Camera { + id: camera + } +} diff --git a/examples/multimedia/video/qmlvideofx/qmlvideofx.qrc b/examples/multimedia/video/qmlvideofx/qmlvideofx.qrc index e7978e39..f3ad2770 100644 --- a/examples/multimedia/video/qmlvideofx/qmlvideofx.qrc +++ b/examples/multimedia/video/qmlvideofx/qmlvideofx.qrc @@ -3,6 +3,7 @@ images/qt-logo.png qml/qmlvideofx/Button.qml qml/qmlvideofx/Content.qml + qml/qmlvideofx/ContentCamera.qml qml/qmlvideofx/ContentImage.qml qml/qmlvideofx/ContentVideo.qml qml/qmlvideofx/Divider.qml From 483da26351b6b62a73785b592673bce9577d1a88 Mon Sep 17 00:00:00 2001 From: Oswald Buddenhagen Date: Mon, 20 Apr 2015 12:28:05 +0200 Subject: [PATCH 02/10] Bump version Change-Id: I8c772e6048ed08abf98c0aef4b731653b3957ba4 --- .qmake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.qmake.conf b/.qmake.conf index 35f43d51..60b46503 100644 --- a/.qmake.conf +++ b/.qmake.conf @@ -1,4 +1,4 @@ load(qt_build_config) CONFIG += qt_example_installs -MODULE_VERSION = 5.4.2 +MODULE_VERSION = 5.4.3 From d44f5734bf945b9b3d57f2bc9c73243343c21518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Str=C3=B8mme?= Date: Fri, 17 Apr 2015 12:36:48 +0200 Subject: [PATCH 03/10] Remove warning about missing qmake variable on Android. gstreamer on Android is not supported, so we don't need to check for it. Change-Id: I7ccde2b9878f9f435828195a5ac1b76a9a6985d1 Reviewed-by: Yoann Lopes --- qtmultimedia.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qtmultimedia.pro b/qtmultimedia.pro index 21d72d09..1225ffb6 100644 --- a/qtmultimedia.pro +++ b/qtmultimedia.pro @@ -14,7 +14,7 @@ win32 { qtCompileTest(avfoundation) } else:qnx { qtCompileTest(mmrenderer) -} else { +} else:!android { contains(QT_CONFIG, alsa):qtCompileTest(alsa) contains(QT_CONFIG, pulseaudio):qtCompileTest(pulseaudio) From 61033aa420b4d34107e3cc67e8751506bc4ab745 Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Wed, 22 Apr 2015 12:42:38 +0300 Subject: [PATCH 04/10] winrt: fix camera sample queue thread safety Replace sample buffer QVector with C array to avoid reallocations. The resource needs to be protected, so use atomic indexes to prevent writing into the same array element that is being read. Task-number: QTBUG-45667 Change-Id: Ifd30dd128765ea4794fe8614f25ef596bba891ee Reviewed-by: Andrew Knight Reviewed-by: Maurice Kalinowski --- .../qwinrtcameravideorenderercontrol.cpp | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp index 74afabaf..bcae8a20 100644 --- a/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp +++ b/src/plugins/winrt/qwinrtcameravideorenderercontrol.cpp @@ -132,11 +132,14 @@ private: ComPtr m_outputView; }; +#define CAMERA_SAMPLE_QUEUE_SIZE 5 class QWinRTCameraVideoRendererControlPrivate { public: QScopedPointer blitter; - QVector> buffers; + ComPtr buffers[CAMERA_SAMPLE_QUEUE_SIZE]; + QAtomicInteger writeIndex; + QAtomicInteger readIndex; }; QWinRTCameraVideoRendererControl::QWinRTCameraVideoRendererControl(const QSize &size, QObject *parent) @@ -153,13 +156,17 @@ bool QWinRTCameraVideoRendererControl::render(ID3D11Texture2D *target) { Q_D(QWinRTCameraVideoRendererControl); - if (d->buffers.isEmpty()) { + const quint16 readIndex = d->readIndex; + if (readIndex == d->writeIndex) { emit bufferRequested(); return false; } HRESULT hr; - ComPtr buffer = d->buffers.takeFirst(); + ComPtr buffer = d->buffers[readIndex]; + Q_ASSERT(buffer); + d->buffers[readIndex].Reset(); + d->readIndex = (readIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE; ComPtr sourceTexture; ComPtr dxgiBuffer; @@ -186,11 +193,17 @@ void QWinRTCameraVideoRendererControl::queueBuffer(IMF2DBuffer *buffer) { Q_D(QWinRTCameraVideoRendererControl); Q_ASSERT(buffer); - d->buffers.append(buffer); + const quint16 writeIndex = (d->writeIndex + 1) % CAMERA_SAMPLE_QUEUE_SIZE; + if (d->readIndex == writeIndex) // Drop new sample if queue is full + return; + d->buffers[d->writeIndex] = buffer; + d->writeIndex = writeIndex; } void QWinRTCameraVideoRendererControl::discardBuffers() { Q_D(QWinRTCameraVideoRendererControl); - d->buffers.clear(); + d->writeIndex = d->readIndex = 0; + for (ComPtr &buffer : d->buffers) + buffer.Reset(); } From d910b6d63ff0ca74b5af004a83c437fa4db190c0 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Tue, 21 Apr 2015 18:49:29 +0200 Subject: [PATCH 05/10] AVFoundation: correctly detect the default audio capture device. Use AVCaptureDevice::defaultDeviceWithMediaType instead of the first device in the list of available devices. Change-Id: I436921f99280a28d7158d345cd977a874cfb8968 Reviewed-by: Timur Pocheptsov Reviewed-by: Christian Stromme --- .../avfoundation/camera/avfaudioinputselectorcontrol.h | 1 + .../avfoundation/camera/avfaudioinputselectorcontrol.mm | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.h b/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.h index 28188e27..2f499f0a 100644 --- a/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.h +++ b/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.h @@ -69,6 +69,7 @@ private: QString m_activeInput; bool m_dirty; + QString m_defaultDevice; QStringList m_devices; QMap m_deviceDescriptions; }; diff --git a/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.mm b/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.mm index 4a2f068f..51a4eb4e 100644 --- a/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.mm +++ b/src/plugins/avfoundation/camera/avfaudioinputselectorcontrol.mm @@ -60,8 +60,11 @@ AVFAudioInputSelectorControl::AVFAudioInputSelectorControl(AVFCameraService *ser QString::fromUtf8([[device localizedName] UTF8String])); } - if (m_devices.size() > 0) - m_activeInput = m_devices.first(); + AVCaptureDevice *defaultDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + if (defaultDevice) { + m_defaultDevice = QString::fromUtf8([defaultDevice.uniqueID UTF8String]); + m_activeInput = m_defaultDevice; + } } AVFAudioInputSelectorControl::~AVFAudioInputSelectorControl() @@ -80,7 +83,7 @@ QString AVFAudioInputSelectorControl::inputDescription(const QString &name) cons QString AVFAudioInputSelectorControl::defaultInput() const { - return m_devices.size() > 0 ? m_devices.first() : QString(); + return m_defaultDevice; } QString AVFAudioInputSelectorControl::activeInput() const From bcaec9624c9124dc256ad963d5735496e219414a Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Tue, 21 Apr 2015 18:07:19 +0200 Subject: [PATCH 06/10] AVFoundation: fix microphone permission when using the camera. The microphone permission was always requested when using the camera, even when not actually using the microphone, which can only happen when recording a video. The permission request is triggered by adding an audio AVCaptureDeviceInput to the AVCaptureSession, which was done when setting the camera to LoadedState. This is now done when setting the camera mode to CaptureVideo. Task-number: QTBUG-45659 Change-Id: I3692797128cfb70ba5ccbc7a36b6955471039e80 Reviewed-by: Timur Pocheptsov --- .../avfoundation/camera/avfcamerasession.h | 3 +- .../avfoundation/camera/avfcamerasession.mm | 38 ++---------- .../camera/avfmediarecordercontrol.h | 5 +- .../camera/avfmediarecordercontrol.mm | 62 ++++++++++++++++--- 4 files changed, 63 insertions(+), 45 deletions(-) diff --git a/src/plugins/avfoundation/camera/avfcamerasession.h b/src/plugins/avfoundation/camera/avfcamerasession.h index 8ce7461d..0a2ca214 100644 --- a/src/plugins/avfoundation/camera/avfcamerasession.h +++ b/src/plugins/avfoundation/camera/avfcamerasession.h @@ -92,7 +92,7 @@ Q_SIGNALS: private: static void updateCameraDevices(); - void attachInputDevices(); + void attachVideoInputDevice(); static int m_defaultCameraIndex; static QList m_cameraDevices; @@ -105,7 +105,6 @@ private: AVCaptureSession *m_captureSession; AVCaptureDeviceInput *m_videoInput; - AVCaptureDeviceInput *m_audioInput; AVFCameraSessionObserver *m_observer; }; diff --git a/src/plugins/avfoundation/camera/avfcamerasession.mm b/src/plugins/avfoundation/camera/avfcamerasession.mm index 4d4b2f65..d0bb0e51 100644 --- a/src/plugins/avfoundation/camera/avfcamerasession.mm +++ b/src/plugins/avfoundation/camera/avfcamerasession.mm @@ -147,7 +147,6 @@ AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent) , m_state(QCamera::UnloadedState) , m_active(false) , m_videoInput(nil) - , m_audioInput(nil) { m_captureSession = [[AVCaptureSession alloc] init]; m_observer = [[AVFCameraSessionObserver alloc] initWithCameraSession:this]; @@ -163,11 +162,6 @@ AVFCameraSession::~AVFCameraSession() [m_videoInput release]; } - if (m_audioInput) { - [m_captureSession removeInput:m_audioInput]; - [m_audioInput release]; - } - [m_observer release]; [m_captureSession release]; } @@ -286,10 +280,9 @@ void AVFCameraSession::setState(QCamera::State newState) QCamera::State oldState = m_state; m_state = newState; - //attach audio and video inputs during Unloaded->Loaded transition - if (oldState == QCamera::UnloadedState) { - attachInputDevices(); - } + //attach video input during Unloaded->Loaded transition + if (oldState == QCamera::UnloadedState) + attachVideoInputDevice(); if (m_state == QCamera::ActiveState) { Q_EMIT readyToConfigureConnections(); @@ -331,7 +324,7 @@ void AVFCameraSession::processSessionStopped() } } -void AVFCameraSession::attachInputDevices() +void AVFCameraSession::attachVideoInputDevice() { //Attach video input device: if (m_service->videoDeviceControl()->isDirty()) { @@ -359,29 +352,6 @@ void AVFCameraSession::attachInputDevices() } } } - - //Attach audio input device: - if (m_service->audioInputSelectorControl()->isDirty()) { - if (m_audioInput) { - [m_captureSession removeInput:m_audioInput]; - [m_audioInput release]; - m_audioInput = 0; - } - - AVCaptureDevice *audioDevice = m_service->audioInputSelectorControl()->createCaptureDevice(); - - NSError *error = nil; - m_audioInput = [AVCaptureDeviceInput - deviceInputWithDevice:audioDevice - error:&error]; - - if (!m_audioInput) { - qWarning() << "Failed to create audio device input"; - } else { - [m_audioInput retain]; - [m_captureSession addInput:m_audioInput]; - } - } } #include "moc_avfcamerasession.cpp" diff --git a/src/plugins/avfoundation/camera/avfmediarecordercontrol.h b/src/plugins/avfoundation/camera/avfmediarecordercontrol.h index a4c7b7f4..fb181a67 100644 --- a/src/plugins/avfoundation/camera/avfmediarecordercontrol.h +++ b/src/plugins/avfoundation/camera/avfmediarecordercontrol.h @@ -46,6 +46,7 @@ QT_BEGIN_NAMESPACE class AVFCameraSession; class AVFCameraControl; +class AVFAudioInputSelectorControl; class AVFCameraService; class AVFMediaRecorderControl : public QMediaRecorderControl @@ -78,11 +79,12 @@ public Q_SLOTS: void handleRecordingFailed(const QString &message); private Q_SLOTS: - void reconnectMovieOutput(); + void setupSessionForCapture(); void updateStatus(); private: AVFCameraControl *m_cameraControl; + AVFAudioInputSelectorControl *m_audioInputControl; AVFCameraSession *m_session; bool m_connected; @@ -96,6 +98,7 @@ private: bool m_muted; qreal m_volume; + AVCaptureDeviceInput *m_audioInput; AVCaptureMovieFileOutput *m_movieOutput; AVFMediaRecorderDelegate *m_recorderDelagate; AVFStorageLocation m_storageLocation; diff --git a/src/plugins/avfoundation/camera/avfmediarecordercontrol.mm b/src/plugins/avfoundation/camera/avfmediarecordercontrol.mm index d65d238d..3e855b46 100644 --- a/src/plugins/avfoundation/camera/avfmediarecordercontrol.mm +++ b/src/plugins/avfoundation/camera/avfmediarecordercontrol.mm @@ -44,6 +44,7 @@ #include "avfcamerasession.h" #include "avfcameraservice.h" #include "avfcameracontrol.h" +#include "avfaudioinputselectorcontrol.h" #include #include @@ -122,6 +123,7 @@ QT_USE_NAMESPACE AVFMediaRecorderControl::AVFMediaRecorderControl(AVFCameraService *service, QObject *parent) : QMediaRecorderControl(parent) , m_cameraControl(service->cameraControl()) + , m_audioInputControl(service->audioInputSelectorControl()) , m_session(service->session()) , m_connected(false) , m_state(QMediaRecorder::StoppedState) @@ -130,21 +132,29 @@ AVFMediaRecorderControl::AVFMediaRecorderControl(AVFCameraService *service, QObj , m_recordingFinished(false) , m_muted(false) , m_volume(1.0) + , m_audioInput(nil) { m_movieOutput = [[AVCaptureMovieFileOutput alloc] init]; m_recorderDelagate = [[AVFMediaRecorderDelegate alloc] initWithRecorder:this]; connect(m_cameraControl, SIGNAL(stateChanged(QCamera::State)), SLOT(updateStatus())); connect(m_cameraControl, SIGNAL(statusChanged(QCamera::Status)), SLOT(updateStatus())); - connect(m_cameraControl, SIGNAL(captureModeChanged(QCamera::CaptureModes)), SLOT(reconnectMovieOutput())); - - reconnectMovieOutput(); + connect(m_cameraControl, SIGNAL(captureModeChanged(QCamera::CaptureModes)), SLOT(setupSessionForCapture())); + connect(m_session, SIGNAL(readyToConfigureConnections()), SLOT(setupSessionForCapture())); + connect(m_session, SIGNAL(stateChanged(QCamera::State)), SLOT(setupSessionForCapture())); } AVFMediaRecorderControl::~AVFMediaRecorderControl() { - if (m_movieOutput) + if (m_movieOutput) { [m_session->captureSession() removeOutput:m_movieOutput]; + [m_movieOutput release]; + } + + if (m_audioInput) { + [m_session->captureSession() removeInput:m_audioInput]; + [m_audioInput release]; + } [m_recorderDelagate release]; } @@ -315,13 +325,39 @@ void AVFMediaRecorderControl::handleRecordingFailed(const QString &message) Q_EMIT error(QMediaRecorder::ResourceError, message); } -void AVFMediaRecorderControl::reconnectMovieOutput() +void AVFMediaRecorderControl::setupSessionForCapture() { //adding movie output causes high CPU usage even when while recording is not active, - //connect it only while video capture mode is enabled + //connect it only while video capture mode is enabled. + // Similarly, connect the Audio input only in that mode, since it's only necessary + // when recording anyway. Adding an Audio input will trigger the microphone permission + // request on iOS, but it shoudn't do so until we actually try to record. AVCaptureSession *captureSession = m_session->captureSession(); - if (!m_connected && m_cameraControl->captureMode().testFlag(QCamera::CaptureVideo)) { + if (!m_connected + && m_cameraControl->captureMode().testFlag(QCamera::CaptureVideo) + && m_session->state() != QCamera::UnloadedState) { + + // Add audio input + // Allow recording even if something wrong happens with the audio input initialization + AVCaptureDevice *audioDevice = m_audioInputControl->createCaptureDevice(); + if (!audioDevice) { + qWarning("No audio input device available"); + } else { + NSError *error = nil; + m_audioInput = [AVCaptureDeviceInput deviceInputWithDevice:audioDevice error:&error]; + + if (!m_audioInput) { + qWarning() << "Failed to create audio device input"; + } else if (![captureSession canAddInput:m_audioInput]) { + qWarning() << "Could not connect the audio input"; + m_audioInput = 0; + } else { + [m_audioInput retain]; + [captureSession addInput:m_audioInput]; + } + } + if ([captureSession canAddOutput:m_movieOutput]) { [captureSession addOutput:m_movieOutput]; m_connected = true; @@ -329,8 +365,18 @@ void AVFMediaRecorderControl::reconnectMovieOutput() Q_EMIT error(QMediaRecorder::ResourceError, tr("Could not connect the video recorder")); qWarning() << "Could not connect the video recorder"; } - } else if (m_connected && !m_cameraControl->captureMode().testFlag(QCamera::CaptureVideo)) { + } else if (m_connected + && (!m_cameraControl->captureMode().testFlag(QCamera::CaptureVideo) + || m_session->state() != QCamera::ActiveState)) { + [captureSession removeOutput:m_movieOutput]; + + if (m_audioInput) { + [captureSession removeInput:m_audioInput]; + [m_audioInput release]; + m_audioInput = nil; + } + m_connected = false; } From 20dbf8490e8226e96898d9b0ef6edd5f9a87bff2 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Thu, 23 Apr 2015 14:52:55 +0200 Subject: [PATCH 07/10] Added 5.4.2 change file. Change-Id: Ib9d829e92343d5230875c37bee7b2bf912b1d304 Reviewed-by: Christian Stromme --- dist/changes-5.4.2 | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 dist/changes-5.4.2 diff --git a/dist/changes-5.4.2 b/dist/changes-5.4.2 new file mode 100644 index 00000000..07e55302 --- /dev/null +++ b/dist/changes-5.4.2 @@ -0,0 +1,64 @@ +Qt 5.4.2 is a bug-fix release. It maintains both forward and backward +compatibility (source and binary) with Qt 5.4.0 and 5.4.1. + +For more details, refer to the online documentation included in this +distribution. The documentation is also available online: + + http://qt-project.org/doc/qt-5.4 + +The Qt version 5.4 series is binary compatible with the 5.3.x series. +Applications compiled for 5.3 will continue to run with 5.4. + +Some of the changes listed in this file include issue tracking numbers +corresponding to tasks in the Qt Bug Tracker: + + http://bugreports.qt-project.org/ + +Each of these identifiers can be entered in the bug tracker to obtain more +information about a particular change. + +**************************************************************************** +* Library * +**************************************************************************** + +QtMultimedia +------------ + + - QCameraInfo::availableCameras() and QtMultimedia.availableCameras (QML) + now return an up-to-date list when a camera is added or removed from the + system. + - Fixed memory leak in QMediaPlaylist::load(). + +**************************************************************************** +* Platform Specific Changes * +**************************************************************************** + +Android +------- + + - [QTBUG-37525] Fixed VideoOutput not working on some devices when used + with a Camera. + - [QTBUG-44383] Fixed setting a URL with special characters on a + MediaPlayer. + +iOS +--- + + - [QTBUG-44790] Fixed crash when using QAudioOutput/QAudioInput on iOS 5. + - [QTBUG-45659] Using QCamera doesn't always prompt for microphone + permission anymore, it now only does so when using the CaptureVideo mode. + +Linux +----- + + - [QTBUG-42326] Fixed regression causing software devices to not be + included in QAudioDeviceInfo::availableDevices() when using the ALSA + backend. + +Windows +------- + + - [QTBUG-32746] QMediaPlayer doesn't resume playback anymore when seeking + an audio media while paused. + - [QTBUG-42648] Greatly improved QAudioDeviceInfo supported formats detection. + - [QTBUG-44305] QAudioDeviceInfo: Fixed memory leak in availableDevices(). From fe13f5bb05ef278eb5109a9518118dda7ddb0b9d Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Tue, 28 Apr 2015 18:35:38 +0200 Subject: [PATCH 08/10] Minor refactor of built-in QSGVideoNodes. - Load shader source from resource files. - Correctly report material types: each material can use different shaders depending on the video pixel format but it was reporting a unique material type. This was causing the node to keep using the same shader even if its pixel format changed. Change-Id: Ib903ecd6e7dd1dd56d7cefe255ab7049933df17d Reviewed-by: Gunnar Sletta --- .../qsgvideonode_rgb.cpp | 79 ++--- .../qsgvideonode_texture.cpp | 82 ++--- .../qsgvideonode_yuv.cpp | 280 +++++++----------- .../qtmultimediaquicktools.pro | 14 + .../qtmultimediaquicktools.qrc | 13 + .../shaders/biplanaryuvvideo.frag | 14 + .../shaders/biplanaryuvvideo.vert | 13 + .../shaders/biplanaryuvvideo_swizzle.frag | 14 + .../shaders/rgbvideo.frag | 8 + .../shaders/rgbvideo.vert | 9 + .../shaders/rgbvideo_padded.vert | 10 + .../shaders/rgbvideo_swizzle.frag | 8 + .../shaders/triplanaryuvvideo.frag | 18 ++ .../shaders/triplanaryuvvideo.vert | 16 + 14 files changed, 292 insertions(+), 286 deletions(-) create mode 100644 src/qtmultimediaquicktools/qtmultimediaquicktools.qrc create mode 100644 src/qtmultimediaquicktools/shaders/biplanaryuvvideo.frag create mode 100644 src/qtmultimediaquicktools/shaders/biplanaryuvvideo.vert create mode 100644 src/qtmultimediaquicktools/shaders/biplanaryuvvideo_swizzle.frag create mode 100644 src/qtmultimediaquicktools/shaders/rgbvideo.frag create mode 100644 src/qtmultimediaquicktools/shaders/rgbvideo.vert create mode 100644 src/qtmultimediaquicktools/shaders/rgbvideo_padded.vert create mode 100644 src/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag create mode 100644 src/qtmultimediaquicktools/shaders/triplanaryuvvideo.frag create mode 100644 src/qtmultimediaquicktools/shaders/triplanaryuvvideo.vert diff --git a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp b/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp index 659dc8bd..0f682d1e 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_rgb.cpp @@ -68,14 +68,15 @@ QSGVideoNode *QSGVideoNodeFactory_RGB::createNode(const QVideoSurfaceFormat &for class QSGVideoMaterialShader_RGB : public QSGMaterialShader { public: - QSGVideoMaterialShader_RGB(QVideoFrame::PixelFormat pixelFormat) + QSGVideoMaterialShader_RGB() : QSGMaterialShader(), m_id_matrix(-1), m_id_width(-1), m_id_rgbTexture(-1), - m_id_opacity(-1), - m_pixelFormat(pixelFormat) + m_id_opacity(-1) { + setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo_padded.vert")); + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag")); } void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial); @@ -90,54 +91,6 @@ public: } protected: - - virtual const char *vertexShader() const { - const char *shader = - "uniform highp mat4 qt_Matrix; \n" - "uniform highp float width; \n" - "attribute highp vec4 qt_VertexPosition; \n" - "attribute highp vec2 qt_VertexTexCoord; \n" - "varying highp vec2 qt_TexCoord; \n" - "void main() { \n" - " qt_TexCoord = qt_VertexTexCoord * vec2(width, 1);\n" - " gl_Position = qt_Matrix * qt_VertexPosition; \n" - "}"; - return shader; - } - - virtual const char *fragmentShader() const { - static const char *shader = - "uniform sampler2D rgbTexture;" - "uniform lowp float opacity;" - "" - "varying highp vec2 qt_TexCoord;" - "" - "void main()" - "{" - " gl_FragColor = texture2D(rgbTexture, qt_TexCoord) * opacity;" - "}"; - - static const char *colorsSwapShader = - "uniform sampler2D rgbTexture;" - "uniform lowp float opacity;" - "" - "varying highp vec2 qt_TexCoord;" - "" - "void main()" - "{" - " gl_FragColor = vec4(texture2D(rgbTexture, qt_TexCoord).bgr, 1.0) * opacity;" - "}"; - - - switch (m_pixelFormat) { - case QVideoFrame::Format_RGB32: - case QVideoFrame::Format_ARGB32: - return colorsSwapShader; - default: - return shader; - } - } - virtual void initialize() { m_id_matrix = program()->uniformLocation("qt_Matrix"); m_id_width = program()->uniformLocation("width"); @@ -149,7 +102,16 @@ protected: int m_id_width; int m_id_rgbTexture; int m_id_opacity; - QVideoFrame::PixelFormat m_pixelFormat; +}; + +class QSGVideoMaterialShader_RGB_swizzle : public QSGVideoMaterialShader_RGB +{ +public: + QSGVideoMaterialShader_RGB_swizzle() + : QSGVideoMaterialShader_RGB() + { + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag")); + } }; @@ -172,12 +134,13 @@ public: } virtual QSGMaterialType *type() const { - static QSGMaterialType theType; - return &theType; + static QSGMaterialType normalType, swizzleType; + return needsSwizzling() ? &swizzleType : &normalType; } virtual QSGMaterialShader *createShader() const { - return new QSGVideoMaterialShader_RGB(m_format.pixelFormat()); + return needsSwizzling() ? new QSGVideoMaterialShader_RGB_swizzle + : new QSGVideoMaterialShader_RGB; } virtual int compare(const QSGMaterial *other) const { @@ -263,6 +226,12 @@ public: GLuint m_textureId; qreal m_opacity; GLfloat m_width; + +private: + bool needsSwizzling() const { + return m_format.pixelFormat() == QVideoFrame::Format_RGB32 + || m_format.pixelFormat() == QVideoFrame::Format_ARGB32; + } }; diff --git a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp index 3dc845c8..fd8d8719 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_texture.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_texture.cpp @@ -69,10 +69,11 @@ QSGVideoNode *QSGVideoNodeFactory_Texture::createNode(const QVideoSurfaceFormat class QSGVideoMaterialShader_Texture : public QSGMaterialShader { public: - QSGVideoMaterialShader_Texture(QVideoFrame::PixelFormat pixelFormat) - : QSGMaterialShader(), - m_pixelFormat(pixelFormat) + QSGVideoMaterialShader_Texture() + : QSGMaterialShader() { + setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.vert")); + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo.frag")); } void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial); @@ -87,56 +88,6 @@ public: } protected: - - virtual const char *vertexShader() const { - const char *shader = - "uniform highp mat4 qt_Matrix; \n" - "attribute highp vec4 qt_VertexPosition; \n" - "attribute highp vec2 qt_VertexTexCoord; \n" - "varying highp vec2 qt_TexCoord; \n" - "void main() { \n" - " qt_TexCoord = qt_VertexTexCoord; \n" - " gl_Position = qt_Matrix * qt_VertexPosition; \n" - "}"; - return shader; - } - - virtual const char *fragmentShader() const { - static const char *shader = - "uniform sampler2D rgbTexture;" - "uniform lowp float opacity;" - "" - "varying highp vec2 qt_TexCoord;" - "" - "void main()" - "{" - " gl_FragColor = texture2D(rgbTexture, qt_TexCoord) * opacity;" - "}"; - - static const char *colorsSwapShader = - "uniform sampler2D rgbTexture;" - "uniform lowp float opacity;" - "" - "varying highp vec2 qt_TexCoord;" - "" - "void main()" - "{" - " gl_FragColor = vec4(texture2D(rgbTexture, qt_TexCoord).bgr, 1.0) * opacity;" - "}"; - - if (!QMediaOpenGLHelper::isANGLE()) { - switch (m_pixelFormat) { - case QVideoFrame::Format_RGB32: - case QVideoFrame::Format_ARGB32: - return colorsSwapShader; - default: - break; - } - } - - return shader; - } - virtual void initialize() { m_id_matrix = program()->uniformLocation("qt_Matrix"); m_id_Texture = program()->uniformLocation("rgbTexture"); @@ -146,7 +97,16 @@ protected: int m_id_matrix; int m_id_Texture; int m_id_opacity; - QVideoFrame::PixelFormat m_pixelFormat; +}; + +class QSGVideoMaterialShader_Texture_swizzle : public QSGVideoMaterialShader_Texture +{ +public: + QSGVideoMaterialShader_Texture_swizzle() + : QSGVideoMaterialShader_Texture() + { + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag")); + } }; @@ -167,12 +127,13 @@ public: } virtual QSGMaterialType *type() const { - static QSGMaterialType theType; - return &theType; + static QSGMaterialType normalType, swizzleType; + return needsSwizzling() ? &swizzleType : &normalType; } virtual QSGMaterialShader *createShader() const { - return new QSGVideoMaterialShader_Texture(m_format.pixelFormat()); + return needsSwizzling() ? new QSGVideoMaterialShader_Texture_swizzle + : new QSGVideoMaterialShader_Texture; } virtual int compare(const QSGMaterial *other) const { @@ -220,6 +181,13 @@ public: QVideoSurfaceFormat m_format; GLuint m_textureId; qreal m_opacity; + +private: + bool needsSwizzling() const { + return !QMediaOpenGLHelper::isANGLE() + && (m_format.pixelFormat() == QVideoFrame::Format_RGB32 + || m_format.pixelFormat() == QVideoFrame::Format_ARGB32); + } }; diff --git a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp index fc40d659..9da1023b 100644 --- a/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp +++ b/src/qtmultimediaquicktools/qsgvideonode_yuv.cpp @@ -62,10 +62,17 @@ QSGVideoNode *QSGVideoNodeFactory_YUV::createNode(const QVideoSurfaceFormat &for } -class QSGVideoMaterialShader_YUV420 : public QSGMaterialShader +class QSGVideoMaterialShader_YUV_BiPlanar : public QSGMaterialShader { public: - void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial); + QSGVideoMaterialShader_YUV_BiPlanar() + : QSGMaterialShader() + { + setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/biplanaryuvvideo.vert")); + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/biplanaryuvvideo.frag")); + } + + virtual void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial); virtual char const *const *attributeNames() const { static const char *names[] = { @@ -77,133 +84,58 @@ public: } protected: - - virtual const char *vertexShader() const { - const char *shader = - "uniform highp mat4 qt_Matrix; \n" - "uniform highp float yWidth; \n" - "uniform highp float uvWidth; \n" - "attribute highp vec4 qt_VertexPosition; \n" - "attribute highp vec2 qt_VertexTexCoord; \n" - "varying highp vec2 yTexCoord; \n" - "varying highp vec2 uvTexCoord; \n" - "void main() { \n" - " yTexCoord = qt_VertexTexCoord * vec2(yWidth, 1);\n" - " uvTexCoord = qt_VertexTexCoord * vec2(uvWidth, 1);\n" - " gl_Position = qt_Matrix * qt_VertexPosition; \n" - "}"; - return shader; - } - - virtual const char *fragmentShader() const { - static const char *shader = - "uniform sampler2D yTexture;" - "uniform sampler2D uTexture;" - "uniform sampler2D vTexture;" - "uniform mediump mat4 colorMatrix;" - "uniform lowp float opacity;" - "" - "varying highp vec2 yTexCoord;" - "varying highp vec2 uvTexCoord;" - "" - "void main()" - "{" - " mediump float Y = texture2D(yTexture, yTexCoord).r;" - " mediump float U = texture2D(uTexture, uvTexCoord).r;" - " mediump float V = texture2D(vTexture, uvTexCoord).r;" - " mediump vec4 color = vec4(Y, U, V, 1.);" - " gl_FragColor = colorMatrix * color * opacity;" - "}"; - return shader; - } - virtual void initialize() { m_id_matrix = program()->uniformLocation("qt_Matrix"); - m_id_yWidth = program()->uniformLocation("yWidth"); - m_id_uvWidth = program()->uniformLocation("uvWidth"); - m_id_yTexture = program()->uniformLocation("yTexture"); - m_id_uTexture = program()->uniformLocation("uTexture"); - m_id_vTexture = program()->uniformLocation("vTexture"); + m_id_plane1Width = program()->uniformLocation("plane1Width"); + m_id_plane2Width = program()->uniformLocation("plane2Width"); + m_id_plane1Texture = program()->uniformLocation("plane1Texture"); + m_id_plane2Texture = program()->uniformLocation("plane2Texture"); m_id_colorMatrix = program()->uniformLocation("colorMatrix"); m_id_opacity = program()->uniformLocation("opacity"); } int m_id_matrix; - int m_id_yWidth; - int m_id_uvWidth; - int m_id_yTexture; - int m_id_uTexture; - int m_id_vTexture; + int m_id_plane1Width; + int m_id_plane2Width; + int m_id_plane1Texture; + int m_id_plane2Texture; int m_id_colorMatrix; int m_id_opacity; }; -class QSGVideoMaterialShader_NV_12_21 : public QSGVideoMaterialShader_YUV420 + +class QSGVideoMaterialShader_YUV_BiPlanar_swizzle : public QSGVideoMaterialShader_YUV_BiPlanar { public: - QSGVideoMaterialShader_NV_12_21(bool isNV21) : m_isNV21(isNV21) { + QSGVideoMaterialShader_YUV_BiPlanar_swizzle() + : QSGVideoMaterialShader_YUV_BiPlanar() + { + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/biplanaryuvvideo_swizzle.frag")); + } +}; + + +class QSGVideoMaterialShader_YUV_TriPlanar : public QSGVideoMaterialShader_YUV_BiPlanar +{ +public: + QSGVideoMaterialShader_YUV_TriPlanar() + : QSGVideoMaterialShader_YUV_BiPlanar() + { + setShaderSourceFile(QOpenGLShader::Vertex, QStringLiteral(":/qtmultimediaquicktools/shaders/triplanaryuvvideo.vert")); + setShaderSourceFile(QOpenGLShader::Fragment, QStringLiteral(":/qtmultimediaquicktools/shaders/triplanaryuvvideo.frag")); } virtual void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial); protected: - - virtual const char *vertexShader() const { - const char *shader = - "uniform highp mat4 qt_Matrix; \n" - "uniform highp float yWidth; \n" - "attribute highp vec4 qt_VertexPosition; \n" - "attribute highp vec2 qt_VertexTexCoord; \n" - "varying highp vec2 yTexCoord; \n" - "void main() { \n" - " yTexCoord = qt_VertexTexCoord * vec2(yWidth, 1);\n" - " gl_Position = qt_Matrix * qt_VertexPosition; \n" - "}"; - return shader; - } - - virtual const char *fragmentShader() const { - static const char *shaderNV12 = - "uniform sampler2D yTexture; \n" - "uniform sampler2D uvTexture; \n" - "uniform mediump mat4 colorMatrix; \n" - "uniform lowp float opacity; \n" - "varying highp vec2 yTexCoord; \n" - "void main() \n" - "{ \n" - " mediump float Y = texture2D(yTexture, yTexCoord).r; \n" - " mediump vec2 UV = texture2D(uvTexture, yTexCoord).ra; \n" - " mediump vec4 color = vec4(Y, UV.x, UV.y, 1.); \n" - " gl_FragColor = colorMatrix * color * opacity; \n" - "}"; - - static const char *shaderNV21 = - "uniform sampler2D yTexture; \n" - "uniform sampler2D uvTexture; \n" - "uniform mediump mat4 colorMatrix; \n" - "uniform lowp float opacity; \n" - "varying highp vec2 yTexCoord; \n" - "void main() \n" - "{ \n" - " mediump float Y = texture2D(yTexture, yTexCoord).r; \n" - " mediump vec2 UV = texture2D(uvTexture, yTexCoord).ar; \n" - " mediump vec4 color = vec4(Y, UV.x, UV.y, 1.); \n" - " gl_FragColor = colorMatrix * color * opacity; \n" - "}"; - return m_isNV21 ? shaderNV21 : shaderNV12; - } - virtual void initialize() { - m_id_yTexture = program()->uniformLocation("yTexture"); - m_id_uTexture = program()->uniformLocation("uvTexture"); - m_id_matrix = program()->uniformLocation("qt_Matrix"); - m_id_yWidth = program()->uniformLocation("yWidth"); - m_id_colorMatrix = program()->uniformLocation("colorMatrix"); - m_id_opacity = program()->uniformLocation("opacity"); + m_id_plane3Width = program()->uniformLocation("plane3Width"); + m_id_plane3Texture = program()->uniformLocation("plane3Texture"); + QSGVideoMaterialShader_YUV_BiPlanar::initialize(); } -private: - bool m_isNV21; + int m_id_plane3Width; + int m_id_plane3Texture; }; @@ -213,22 +145,28 @@ public: QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format); ~QSGVideoMaterial_YUV(); - bool isNV12_21() const { - const QVideoFrame::PixelFormat pf = m_format.pixelFormat(); - return pf == QVideoFrame::Format_NV12 || pf == QVideoFrame::Format_NV21; - } - virtual QSGMaterialType *type() const { - static QSGMaterialType theType; - return &theType; + static QSGMaterialType biPlanarType, biPlanarSwizzleType, triPlanarType; + + switch (m_format.pixelFormat()) { + case QVideoFrame::Format_NV12: + return &biPlanarType; + case QVideoFrame::Format_NV21: + return &biPlanarSwizzleType; + default: // Currently: YUV420P and YV12 + return &triPlanarType; + } } virtual QSGMaterialShader *createShader() const { - const QVideoFrame::PixelFormat pf = m_format.pixelFormat(); - if (isNV12_21()) - return new QSGVideoMaterialShader_NV_12_21(pf == QVideoFrame::Format_NV21); - - return new QSGVideoMaterialShader_YUV420; + switch (m_format.pixelFormat()) { + case QVideoFrame::Format_NV12: + return new QSGVideoMaterialShader_YUV_BiPlanar; + case QVideoFrame::Format_NV21: + return new QSGVideoMaterialShader_YUV_BiPlanar_swizzle; + default: // Currently: YUV420P and YV12 + return new QSGVideoMaterialShader_YUV_TriPlanar; + } } virtual int compare(const QSGMaterial *other) const { @@ -236,13 +174,10 @@ public: int d = m_textureIds[0] - m->m_textureIds[0]; if (d) return d; - - d = m_textureIds[1] - m->m_textureIds[1]; - - if (m_textureIds.size() == 2 || d != 0) + else if ((d = m_textureIds[1] - m->m_textureIds[1]) != 0) return d; - - return m_textureIds[2] - m->m_textureIds[2]; + else + return m_textureIds[2] - m->m_textureIds[2]; } void updateBlending() { @@ -259,12 +194,12 @@ public: QVideoSurfaceFormat m_format; QSize m_textureSize; + int m_planeCount; - QVector m_textureIds; + GLuint m_textureIds[3]; + GLfloat m_planeWidth[3]; qreal m_opacity; - GLfloat m_yWidth; - GLfloat m_uvWidth; QMatrix4x4 m_colorMatrix; QVideoFrame m_frame; @@ -273,11 +208,23 @@ public: QSGVideoMaterial_YUV::QSGVideoMaterial_YUV(const QVideoSurfaceFormat &format) : m_format(format), - m_opacity(1.0), - m_yWidth(1.0), - m_uvWidth(1.0) + m_opacity(1.0) { - m_textureIds.resize(isNV12_21() ? 2 : 3); + memset(m_textureIds, 0, sizeof(m_textureIds)); + + switch (format.pixelFormat()) { + case QVideoFrame::Format_NV12: + case QVideoFrame::Format_NV21: + m_planeCount = 2; + break; + case QVideoFrame::Format_YUV420P: + case QVideoFrame::Format_YV12: + m_planeCount = 3; + break; + default: + m_planeCount = 1; + break; + } switch (format.yCbCrColorSpace()) { case QVideoSurfaceFormat::YCbCr_JPEG: @@ -310,7 +257,7 @@ QSGVideoMaterial_YUV::~QSGVideoMaterial_YUV() { if (!m_textureSize.isEmpty()) { if (QOpenGLContext *current = QOpenGLContext::currentContext()) - current->functions()->glDeleteTextures(m_textureIds.size(), &m_textureIds[0]); + current->functions()->glDeleteTextures(m_planeCount, m_textureIds); else qWarning() << "QSGVideoMaterial_YUV: Cannot obtain GL context, unable to delete textures"; } @@ -328,8 +275,8 @@ void QSGVideoMaterial_YUV::bind() // Frame has changed size, recreate textures... if (m_textureSize != m_frame.size()) { if (!m_textureSize.isEmpty()) - functions->glDeleteTextures(m_textureIds.size(), &m_textureIds[0]); - functions->glGenTextures(m_textureIds.size(), &m_textureIds[0]); + functions->glDeleteTextures(m_planeCount, m_textureIds); + functions->glGenTextures(m_planeCount, m_textureIds); m_textureSize = m_frame.size(); } @@ -337,24 +284,25 @@ void QSGVideoMaterial_YUV::bind() functions->glGetIntegerv(GL_UNPACK_ALIGNMENT, &previousAlignment); functions->glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - if (isNV12_21()) { + if (m_format.pixelFormat() == QVideoFrame::Format_NV12 + || m_format.pixelFormat() == QVideoFrame::Format_NV21) { const int y = 0; const int uv = 1; - m_yWidth = qreal(fw) / m_frame.bytesPerLine(y); - m_uvWidth = m_yWidth; + m_planeWidth[0] = m_planeWidth[1] = qreal(fw) / m_frame.bytesPerLine(y); functions->glActiveTexture(GL_TEXTURE1); bindTexture(m_textureIds[1], m_frame.bytesPerLine(uv) / 2, fh / 2, m_frame.bits(uv), GL_LUMINANCE_ALPHA); functions->glActiveTexture(GL_TEXTURE0); // Finish with 0 as default texture unit bindTexture(m_textureIds[0], m_frame.bytesPerLine(y), fh, m_frame.bits(y), GL_LUMINANCE); - } else { + + } else { // YUV420P || YV12 const int y = 0; const int u = m_frame.pixelFormat() == QVideoFrame::Format_YUV420P ? 1 : 2; const int v = m_frame.pixelFormat() == QVideoFrame::Format_YUV420P ? 2 : 1; - m_yWidth = qreal(fw) / m_frame.bytesPerLine(y); - m_uvWidth = qreal(fw) / (2 * m_frame.bytesPerLine(u)); + m_planeWidth[0] = qreal(fw) / m_frame.bytesPerLine(y); + m_planeWidth[1] = m_planeWidth[2] = qreal(fw) / (2 * m_frame.bytesPerLine(u)); functions->glActiveTexture(GL_TEXTURE1); bindTexture(m_textureIds[1], m_frame.bytesPerLine(u), fh / 2, m_frame.bits(u), GL_LUMINANCE); @@ -370,14 +318,10 @@ void QSGVideoMaterial_YUV::bind() m_frame = QVideoFrame(); } else { - functions->glActiveTexture(GL_TEXTURE1); - functions->glBindTexture(GL_TEXTURE_2D, m_textureIds[1]); - if (!isNV12_21()) { - functions->glActiveTexture(GL_TEXTURE2); - functions->glBindTexture(GL_TEXTURE_2D, m_textureIds[2]); + for (int i = 0; i < m_planeCount; ++i) { + functions->glActiveTexture(GL_TEXTURE0 + i); + functions->glBindTexture(GL_TEXTURE_2D, m_textureIds[i]); } - functions->glActiveTexture(GL_TEXTURE0); // Finish with 0 as default texture unit - functions->glBindTexture(GL_TEXTURE_2D, m_textureIds[0]); } } @@ -411,22 +355,21 @@ void QSGVideoNode_YUV::setCurrentFrame(const QVideoFrame &frame, FrameFlags) markDirty(DirtyMaterial); } -void QSGVideoMaterialShader_YUV420::updateState(const RenderState &state, - QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) +void QSGVideoMaterialShader_YUV_BiPlanar::updateState(const RenderState &state, + QSGMaterial *newMaterial, + QSGMaterial *oldMaterial) { Q_UNUSED(oldMaterial); QSGVideoMaterial_YUV *mat = static_cast(newMaterial); - program()->setUniformValue(m_id_yTexture, 0); - program()->setUniformValue(m_id_uTexture, 1); - program()->setUniformValue(m_id_vTexture, 2); + program()->setUniformValue(m_id_plane1Texture, 0); + program()->setUniformValue(m_id_plane2Texture, 1); mat->bind(); program()->setUniformValue(m_id_colorMatrix, mat->m_colorMatrix); - program()->setUniformValue(m_id_yWidth, mat->m_yWidth); - program()->setUniformValue(m_id_uvWidth, mat->m_uvWidth); + program()->setUniformValue(m_id_plane1Width, mat->m_planeWidth[0]); + program()->setUniformValue(m_id_plane2Width, mat->m_planeWidth[1]); if (state.isOpacityDirty()) { mat->m_opacity = state.opacity(); program()->setUniformValue(m_id_opacity, GLfloat(mat->m_opacity)); @@ -435,26 +378,15 @@ void QSGVideoMaterialShader_YUV420::updateState(const RenderState &state, program()->setUniformValue(m_id_matrix, state.combinedMatrix()); } -void QSGVideoMaterialShader_NV_12_21::updateState(const RenderState &state, - QSGMaterial *newMaterial, - QSGMaterial *oldMaterial) +void QSGVideoMaterialShader_YUV_TriPlanar::updateState(const RenderState &state, + QSGMaterial *newMaterial, + QSGMaterial *oldMaterial) { - Q_UNUSED(oldMaterial); + QSGVideoMaterialShader_YUV_BiPlanar::updateState(state, newMaterial, oldMaterial); QSGVideoMaterial_YUV *mat = static_cast(newMaterial); - program()->setUniformValue(m_id_yTexture, 0); - program()->setUniformValue(m_id_uTexture, 1); - - mat->bind(); - - program()->setUniformValue(m_id_colorMatrix, mat->m_colorMatrix); - program()->setUniformValue(m_id_yWidth, mat->m_yWidth); - if (state.isOpacityDirty()) { - mat->m_opacity = state.opacity(); - program()->setUniformValue(m_id_opacity, GLfloat(mat->m_opacity)); - } - if (state.isMatrixDirty()) - program()->setUniformValue(m_id_matrix, state.combinedMatrix()); + program()->setUniformValue(m_id_plane3Texture, 2); + program()->setUniformValue(m_id_plane3Width, mat->m_planeWidth[2]); } QT_END_NAMESPACE diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro index 917a5ac2..61c5f7b2 100644 --- a/src/qtmultimediaquicktools/qtmultimediaquicktools.pro +++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.pro @@ -32,3 +32,17 @@ HEADERS += \ qsgvideonode_yuv.h \ qsgvideonode_rgb.h \ qsgvideonode_texture.h + +RESOURCES += \ + qtmultimediaquicktools.qrc + +OTHER_FILES += \ + shaders/rgbvideo.vert \ + shaders/rgbvideo_padded.vert \ + shaders/rgbvideo.frag \ + shaders/rgbvideo_swizzle.frag \ + shaders/biplanaryuvvideo.vert \ + shaders/biplanaryuvvideo.frag \ + shaders/biplanaryuvvideo_swizzle.frag \ + shaders/triplanaryuvvideo.vert \ + shaders/triplanaryuvvideo.frag diff --git a/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc new file mode 100644 index 00000000..b0a7f078 --- /dev/null +++ b/src/qtmultimediaquicktools/qtmultimediaquicktools.qrc @@ -0,0 +1,13 @@ + + + shaders/rgbvideo.vert + shaders/rgbvideo.frag + shaders/rgbvideo_swizzle.frag + shaders/rgbvideo_padded.vert + shaders/biplanaryuvvideo.frag + shaders/biplanaryuvvideo.vert + shaders/biplanaryuvvideo_swizzle.frag + shaders/triplanaryuvvideo.frag + shaders/triplanaryuvvideo.vert + + diff --git a/src/qtmultimediaquicktools/shaders/biplanaryuvvideo.frag b/src/qtmultimediaquicktools/shaders/biplanaryuvvideo.frag new file mode 100644 index 00000000..2ede9108 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/biplanaryuvvideo.frag @@ -0,0 +1,14 @@ +uniform sampler2D plane1Texture; +uniform sampler2D plane2Texture; +uniform mediump mat4 colorMatrix; +uniform lowp float opacity; +varying highp vec2 plane1TexCoord; +varying highp vec2 plane2TexCoord; + +void main() +{ + mediump float Y = texture2D(plane1Texture, plane1TexCoord).r; + mediump vec2 UV = texture2D(plane2Texture, plane2TexCoord).ra; + mediump vec4 color = vec4(Y, UV.x, UV.y, 1.); + gl_FragColor = colorMatrix * color * opacity; +} diff --git a/src/qtmultimediaquicktools/shaders/biplanaryuvvideo.vert b/src/qtmultimediaquicktools/shaders/biplanaryuvvideo.vert new file mode 100644 index 00000000..d01b6907 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/biplanaryuvvideo.vert @@ -0,0 +1,13 @@ +uniform highp mat4 qt_Matrix; +uniform highp float plane1Width; +uniform highp float plane2Width; +attribute highp vec4 qt_VertexPosition; +attribute highp vec2 qt_VertexTexCoord; +varying highp vec2 plane1TexCoord; +varying highp vec2 plane2TexCoord; + +void main() { + plane1TexCoord = qt_VertexTexCoord * vec2(plane1Width, 1); + plane2TexCoord = qt_VertexTexCoord * vec2(plane2Width, 1); + gl_Position = qt_Matrix * qt_VertexPosition; +} diff --git a/src/qtmultimediaquicktools/shaders/biplanaryuvvideo_swizzle.frag b/src/qtmultimediaquicktools/shaders/biplanaryuvvideo_swizzle.frag new file mode 100644 index 00000000..560a0d15 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/biplanaryuvvideo_swizzle.frag @@ -0,0 +1,14 @@ +uniform sampler2D plane1Texture; +uniform sampler2D plane2Texture; +uniform mediump mat4 colorMatrix; +uniform lowp float opacity; +varying highp vec2 plane1TexCoord; +varying highp vec2 plane2TexCoord; + +void main() +{ + mediump float Y = texture2D(plane1Texture, plane1TexCoord).r; + mediump vec2 UV = texture2D(plane2Texture, plane2TexCoord).ar; + mediump vec4 color = vec4(Y, UV.x, UV.y, 1.); + gl_FragColor = colorMatrix * color * opacity; +} diff --git a/src/qtmultimediaquicktools/shaders/rgbvideo.frag b/src/qtmultimediaquicktools/shaders/rgbvideo.frag new file mode 100644 index 00000000..673eb4bf --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rgbvideo.frag @@ -0,0 +1,8 @@ +uniform sampler2D rgbTexture; +uniform lowp float opacity; +varying highp vec2 qt_TexCoord; + +void main() +{ + gl_FragColor = texture2D(rgbTexture, qt_TexCoord) * opacity; +} diff --git a/src/qtmultimediaquicktools/shaders/rgbvideo.vert b/src/qtmultimediaquicktools/shaders/rgbvideo.vert new file mode 100644 index 00000000..19915f2f --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rgbvideo.vert @@ -0,0 +1,9 @@ +uniform highp mat4 qt_Matrix; +attribute highp vec4 qt_VertexPosition; +attribute highp vec2 qt_VertexTexCoord; +varying highp vec2 qt_TexCoord; + +void main() { + qt_TexCoord = qt_VertexTexCoord; + gl_Position = qt_Matrix * qt_VertexPosition; +} diff --git a/src/qtmultimediaquicktools/shaders/rgbvideo_padded.vert b/src/qtmultimediaquicktools/shaders/rgbvideo_padded.vert new file mode 100644 index 00000000..5c7c41d1 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rgbvideo_padded.vert @@ -0,0 +1,10 @@ +uniform highp mat4 qt_Matrix; +uniform highp float width; +attribute highp vec4 qt_VertexPosition; +attribute highp vec2 qt_VertexTexCoord; +varying highp vec2 qt_TexCoord; + +void main() { + qt_TexCoord = qt_VertexTexCoord * vec2(width, 1); + gl_Position = qt_Matrix * qt_VertexPosition; +} diff --git a/src/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag b/src/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag new file mode 100644 index 00000000..f01dc86a --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/rgbvideo_swizzle.frag @@ -0,0 +1,8 @@ +uniform sampler2D rgbTexture; +uniform lowp float opacity; +varying highp vec2 qt_TexCoord; + +void main() +{ + gl_FragColor = vec4(texture2D(rgbTexture, qt_TexCoord).bgr, 1.0) * opacity; +} diff --git a/src/qtmultimediaquicktools/shaders/triplanaryuvvideo.frag b/src/qtmultimediaquicktools/shaders/triplanaryuvvideo.frag new file mode 100644 index 00000000..dad8bb5a --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/triplanaryuvvideo.frag @@ -0,0 +1,18 @@ +uniform sampler2D plane1Texture; +uniform sampler2D plane2Texture; +uniform sampler2D plane3Texture; +uniform mediump mat4 colorMatrix; +uniform lowp float opacity; + +varying highp vec2 plane1TexCoord; +varying highp vec2 plane2TexCoord; +varying highp vec2 plane3TexCoord; + +void main() +{ + mediump float Y = texture2D(plane1Texture, plane1TexCoord).r; + mediump float U = texture2D(plane2Texture, plane2TexCoord).r; + mediump float V = texture2D(plane3Texture, plane3TexCoord).r; + mediump vec4 color = vec4(Y, U, V, 1.); + gl_FragColor = colorMatrix * color * opacity; +} diff --git a/src/qtmultimediaquicktools/shaders/triplanaryuvvideo.vert b/src/qtmultimediaquicktools/shaders/triplanaryuvvideo.vert new file mode 100644 index 00000000..d76b3f35 --- /dev/null +++ b/src/qtmultimediaquicktools/shaders/triplanaryuvvideo.vert @@ -0,0 +1,16 @@ +uniform highp mat4 qt_Matrix; +uniform highp float plane1Width; +uniform highp float plane2Width; +uniform highp float plane3Width; +attribute highp vec4 qt_VertexPosition; +attribute highp vec2 qt_VertexTexCoord; +varying highp vec2 plane1TexCoord; +varying highp vec2 plane2TexCoord; +varying highp vec2 plane3TexCoord; + +void main() { + plane1TexCoord = qt_VertexTexCoord * vec2(plane1Width, 1); + plane2TexCoord = qt_VertexTexCoord * vec2(plane2Width, 1); + plane3TexCoord = qt_VertexTexCoord * vec2(plane3Width, 1); + gl_Position = qt_Matrix * qt_VertexPosition; +} From 0559f645bbe928e9a54666356409fce444e80c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Str=C3=B8mme?= Date: Mon, 27 Apr 2015 16:24:18 +0200 Subject: [PATCH 09/10] Android: Don't delete the media recorder object twice. In QAndroidCaptureSession::stop() we call restartViewFinder() which eventually calls QAndroidCaptureSession::stop() again, but this time the media recorder object is already released. Task-number: QTBUG-45637 Change-Id: I943c423398a99d98ccda1063fc16e47cba470deb Reviewed-by: Yoann Lopes --- .../mediacapture/qandroidcapturesession.cpp | 28 ++++++++----------- .../src/mediacapture/qandroidcapturesession.h | 2 +- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp b/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp index ae6991a3..3e3b17c7 100644 --- a/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp +++ b/src/plugins/android/src/mediacapture/qandroidcapturesession.cpp @@ -151,23 +151,19 @@ void QAndroidCaptureSession::setState(QMediaRecorder::State state) stop(); break; case QMediaRecorder::RecordingState: - if (!start()) - return; + start(); break; case QMediaRecorder::PausedState: // Not supported by Android API qWarning("QMediaRecorder::PausedState is not supported on Android"); - return; + break; } - - m_state = state; - emit stateChanged(m_state); } -bool QAndroidCaptureSession::start() +void QAndroidCaptureSession::start() { if (m_state == QMediaRecorder::RecordingState || m_status != QMediaRecorder::LoadedStatus) - return false; + return; setStatus(QMediaRecorder::StartingStatus); @@ -225,13 +221,13 @@ bool QAndroidCaptureSession::start() if (!m_mediaRecorder->prepare()) { emit error(QMediaRecorder::FormatError, QLatin1String("Unable to prepare the media recorder.")); restartViewfinder(); - return false; + return; } if (!m_mediaRecorder->start()) { emit error(QMediaRecorder::FormatError, QLatin1String("Unable to start the media recorder.")); restartViewfinder(); - return false; + return; } m_elapsedTime.start(); @@ -241,22 +237,21 @@ bool QAndroidCaptureSession::start() if (m_cameraSession) m_cameraSession->setReadyForCapture(false); - return true; + m_state = QMediaRecorder::RecordingState; + emit stateChanged(m_state); } void QAndroidCaptureSession::stop(bool error) { - if (m_state == QMediaRecorder::StoppedState) + if (m_state == QMediaRecorder::StoppedState || m_mediaRecorder == 0) return; setStatus(QMediaRecorder::FinalizingStatus); m_mediaRecorder->stop(); - m_notifyTimer.stop(); updateDuration(); m_elapsedTime.invalidate(); - m_mediaRecorder->release(); delete m_mediaRecorder; m_mediaRecorder = 0; @@ -279,6 +274,9 @@ void QAndroidCaptureSession::stop(bool error) m_actualOutputLocation = m_usedOutputLocation; emit actualLocationChanged(m_actualOutputLocation); } + + m_state = QMediaRecorder::StoppedState; + emit stateChanged(m_state); } void QAndroidCaptureSession::setStatus(QMediaRecorder::Status status) @@ -541,8 +539,6 @@ void QAndroidCaptureSession::onError(int what, int extra) Q_UNUSED(what) Q_UNUSED(extra) stop(true); - m_state = QMediaRecorder::StoppedState; - emit stateChanged(m_state); emit error(QMediaRecorder::ResourceError, QLatin1String("Unknown error.")); } diff --git a/src/plugins/android/src/mediacapture/qandroidcapturesession.h b/src/plugins/android/src/mediacapture/qandroidcapturesession.h index f96e9766..c17f081d 100644 --- a/src/plugins/android/src/mediacapture/qandroidcapturesession.h +++ b/src/plugins/android/src/mediacapture/qandroidcapturesession.h @@ -130,7 +130,7 @@ private: CaptureProfile getProfile(int id); - bool start(); + void start(); void stop(bool error = false); void setStatus(QMediaRecorder::Status status); From 0761f2d53f6e6d2b482326442100693542142a52 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Tue, 5 May 2015 13:30:41 +0200 Subject: [PATCH 10/10] Update license header in qmlvideofx example. Change-Id: Icb525176a8f00cc1a7414e68a7452452d1ef3da0 Reviewed-by: Yoann Lopes --- .../qmlvideofx/qml/qmlvideofx/ContentCamera.qml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml b/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml index ef5a3d4a..f78b9916 100644 --- a/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml +++ b/examples/multimedia/video/qmlvideofx/qml/qmlvideofx/ContentCamera.qml @@ -1,7 +1,7 @@ /**************************************************************************** ** -** Copyright (C) 2014 Digia Plc and/or its subsidiary(-ies). -** Contact: http://www.qt-project.org/legal +** Copyright (C) 2015 The Qt Company Ltd. +** Contact: http://www.qt.io/licensing/ ** ** This file is part of the Qt Mobility Components. ** @@ -10,9 +10,9 @@ ** Licensees holding valid commercial Qt licenses may use this file in ** accordance with the commercial license agreement provided with the ** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and Digia. For licensing terms and -** conditions see http://qt.digia.com/licensing. For further information -** use the contact form at http://qt.digia.com/contact-us. +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see http://www.qt.io/terms-conditions. For further +** information use the contact form at http://www.qt.io/contact-us. ** ** GNU Lesser General Public License Usage ** Alternatively, this file may be used under the terms of the GNU Lesser @@ -23,8 +23,8 @@ ** requirements will be met: https://www.gnu.org/licenses/lgpl.html and ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. ** -** In addition, as a special exception, Digia gives you certain additional -** rights. These rights are described in the Digia Qt LGPL Exception +** As a special exception, The Qt Company gives you certain additional +** rights. These rights are described in The Qt Company LGPL Exception ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. ** ** $QT_END_LICENSE$