From ded4d7b006180d12f4f2e7642877ffc6586a5331 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Mon, 24 Aug 2015 15:06:53 +0200 Subject: [PATCH 01/12] Fix possible crashes in QDeclarativeRadioData. It would crash when accessing any of its properties when there is no radio backend available. Task-number: QTBUG-47859 Change-Id: Ic7c0a70ca3e9a4768359b3429ea812caece09fce Reviewed-by: Christian Stromme --- .../multimedia/qdeclarativeradiodata.cpp | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/src/imports/multimedia/qdeclarativeradiodata.cpp b/src/imports/multimedia/qdeclarativeradiodata.cpp index 51d3bb87..f272e8d0 100644 --- a/src/imports/multimedia/qdeclarativeradiodata.cpp +++ b/src/imports/multimedia/qdeclarativeradiodata.cpp @@ -129,7 +129,10 @@ QDeclarativeRadioData::~QDeclarativeRadioData() */ QDeclarativeRadioData::Availability QDeclarativeRadioData::availability() const { - return Availability(m_radioData->availability()); + if (m_radioData) + return Availability(m_radioData->availability()); + + return Unavailable; } @@ -141,7 +144,10 @@ QDeclarativeRadioData::Availability QDeclarativeRadioData::availability() const */ QString QDeclarativeRadioData::stationId() const { - return m_radioData->stationId(); + if (m_radioData) + return m_radioData->stationId(); + + return QString(); } /*! @@ -206,7 +212,10 @@ QString QDeclarativeRadioData::stationId() const */ QDeclarativeRadioData::ProgramType QDeclarativeRadioData::programType() const { - return static_cast(m_radioData->programType()); + if (m_radioData) + return static_cast(m_radioData->programType()); + + return Undefined; } /*! @@ -216,7 +225,10 @@ QDeclarativeRadioData::ProgramType QDeclarativeRadioData::programType() const */ QString QDeclarativeRadioData::programTypeName() const { - return m_radioData->programTypeName(); + if (m_radioData) + return m_radioData->programTypeName(); + + return QString(); } /*! @@ -226,7 +238,10 @@ QString QDeclarativeRadioData::programTypeName() const */ QString QDeclarativeRadioData::stationName() const { - return m_radioData->stationName(); + if (m_radioData) + return m_radioData->stationName(); + + return QString(); } /*! @@ -238,7 +253,10 @@ QString QDeclarativeRadioData::stationName() const */ QString QDeclarativeRadioData::radioText() const { - return m_radioData->radioText(); + if (m_radioData) + return m_radioData->radioText(); + + return QString(); } /*! @@ -250,12 +268,16 @@ QString QDeclarativeRadioData::radioText() const */ bool QDeclarativeRadioData::alternativeFrequenciesEnabled() const { - return m_radioData->isAlternativeFrequenciesEnabled(); + if (m_radioData) + return m_radioData->isAlternativeFrequenciesEnabled(); + + return false; } void QDeclarativeRadioData::setAlternativeFrequenciesEnabled(bool enabled) { - m_radioData->setAlternativeFrequenciesEnabled(enabled); + if (m_radioData) + m_radioData->setAlternativeFrequenciesEnabled(enabled); } void QDeclarativeRadioData::_q_programTypeChanged(QRadioData::ProgramType programType) From 4eb4a3ada360af152b9403e8a88eb76b9474c4ba Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Thu, 13 Aug 2015 18:18:44 +0200 Subject: [PATCH 02/12] Fix QML Camera::supportedViewfinderFrameRateRanges(). Pass the resolution argument as a QJSValue instead of a QSize. This allows to be more flexible and doesn't require the QML argument to be an actual QML 'size' value. It can be any object with the 'width' and 'height' properties. Added missing auto-tests for supportedViewfinderResolutions() and supportedViewfinderFrameRateRanges(). Change-Id: I6c8ae72e6dab8c9d12bbada5b8e7f45e96e9289d Task-number: QTBUG-47630 Reviewed-by: Simon Hausmann --- src/imports/multimedia/qdeclarativecamera.cpp | 9 +- src/imports/multimedia/qdeclarativecamera_p.h | 3 +- .../tst_qdeclarativecamera.qml | 154 ++++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) diff --git a/src/imports/multimedia/qdeclarativecamera.cpp b/src/imports/multimedia/qdeclarativecamera.cpp index 5930b20f..eac1c0c9 100644 --- a/src/imports/multimedia/qdeclarativecamera.cpp +++ b/src/imports/multimedia/qdeclarativecamera.cpp @@ -1013,12 +1013,17 @@ QJSValue QDeclarativeCamera::supportedViewfinderResolutions(qreal minimumFrameRa \since 5.5 */ -QJSValue QDeclarativeCamera::supportedViewfinderFrameRateRanges(const QSize &resolution) +QJSValue QDeclarativeCamera::supportedViewfinderFrameRateRanges(const QJSValue &resolution) { QQmlEngine *engine = qmlEngine(this); QCameraViewfinderSettings settings; - settings.setResolution(resolution); + if (!resolution.isUndefined()) { + QJSValue width = resolution.property(QStringLiteral("width")); + QJSValue height = resolution.property(QStringLiteral("height")); + if (width.isNumber() && height.isNumber()) + settings.setResolution(width.toInt(), height.toInt()); + } QList frameRateRanges = m_camera->supportedViewfinderFrameRateRanges(settings); QJSValue supportedFrameRateRanges = engine->newArray(frameRateRanges.count()); diff --git a/src/imports/multimedia/qdeclarativecamera_p.h b/src/imports/multimedia/qdeclarativecamera_p.h index 623460c5..b3f199a5 100644 --- a/src/imports/multimedia/qdeclarativecamera_p.h +++ b/src/imports/multimedia/qdeclarativecamera_p.h @@ -57,6 +57,7 @@ #include #include #include +#include QT_BEGIN_NAMESPACE @@ -297,7 +298,7 @@ public Q_SLOTS: Q_REVISION(2) QJSValue supportedViewfinderResolutions(qreal minimumFrameRate = 0.0, qreal maximumFrameRate = 0.0); - Q_REVISION(2) QJSValue supportedViewfinderFrameRateRanges(const QSize &resolution = QSize()); + Q_REVISION(2) QJSValue supportedViewfinderFrameRateRanges(const QJSValue &resolution = QJSValue()); Q_SIGNALS: void errorChanged(); diff --git a/tests/auto/unit/qdeclarativecamera/tst_qdeclarativecamera.qml b/tests/auto/unit/qdeclarativecamera/tst_qdeclarativecamera.qml index 833ca522..02e32f4c 100644 --- a/tests/auto/unit/qdeclarativecamera/tst_qdeclarativecamera.qml +++ b/tests/auto/unit/qdeclarativecamera/tst_qdeclarativecamera.qml @@ -144,4 +144,158 @@ TestCase { cameraLoader.sourceComponent = undefined; } + + function test_supportedViewfinderResolutions_data() { + // see mockcameraviewfindersettingscontrol.h for expected values + + return [ + { + tag: "all", + minimumFrameRate: 0, maximumFrameRate: 0, + expectedResolutions: [ + { width: 320, height: 240 }, + { width: 640, height: 480 }, + { width: 1280, height: 720 }, + { width: 1920, height: 1080 } + ] + }, + { + tag: "invalid minimumFrameRate", + minimumFrameRate: 2, maximumFrameRate: 0, + expectedResolutions: [ ] + }, + { + tag: "minimumFrameRate=5", + minimumFrameRate: 5, maximumFrameRate: 0, + expectedResolutions: [ + { width: 1920, height: 1080 } + ] + }, + { + tag: "minimumFrameRate=10", + minimumFrameRate: 10, maximumFrameRate: 0, + expectedResolutions: [ + { width: 1280, height: 720 } + ] + }, + { + tag: "minimumFrameRate=30", + minimumFrameRate: 30, maximumFrameRate: 0, + expectedResolutions: [ + { width: 320, height: 240 }, + { width: 640, height: 480 }, + { width: 1280, height: 720 } + ] + }, + { + tag: "invalid maximumFrameRate", + minimumFrameRate: 0, maximumFrameRate: 2, + expectedResolutions: [ ] + }, + { + tag: "maximumFrameRate=10", + minimumFrameRate: 0, maximumFrameRate: 10, + expectedResolutions: [ + { width: 1280, height: 720 }, + { width: 1920, height: 1080 } + ] + }, + { + tag: "minimumFrameRate=10, maximumFrameRate=10", + minimumFrameRate: 10, maximumFrameRate: 10, + expectedResolutions: [ + { width: 1280, height: 720 } + ] + }, + { + tag: "minimumFrameRate=30, maximumFrameRate=30", + minimumFrameRate: 30, maximumFrameRate: 30, + expectedResolutions: [ + { width: 320, height: 240 }, + { width: 640, height: 480 }, + { width: 1280, height: 720 } + ] + } + ] + } + + function test_supportedViewfinderResolutions(data) { + cameraLoader.sourceComponent = cameraComponent; + var camera = cameraLoader.item; + + var actualResolutions = camera.supportedViewfinderResolutions(data.minimumFrameRate, data.maximumFrameRate); + compare(actualResolutions.length, data.expectedResolutions.length); + for (var i = 0; i < actualResolutions.length; ++i) { + compare(actualResolutions[i].width, data.expectedResolutions[i].width); + compare(actualResolutions[i].height, data.expectedResolutions[i].height); + } + + cameraLoader.sourceComponent = undefined; + } + + function test_supportedViewfinderFrameRateRanges_data() { + // see mockcameraviewfindersettingscontrol.h for expected values + return [ + { + tag: "all", + expectedFrameRateRanges: [ + { minimumFrameRate: 5, maximumFrameRate: 10 }, + { minimumFrameRate: 10, maximumFrameRate: 10 }, + { minimumFrameRate: 30, maximumFrameRate: 30 } + ] + }, + { + tag: "invalid", + resolution: { width: 452472, height: 444534 }, + expectedFrameRateRanges: [ ] + }, + { + tag: "320, 240", + resolution: { width: 320, height: 240 }, + expectedFrameRateRanges: [ + { minimumFrameRate: 30, maximumFrameRate: 30 } + ] + }, + { + tag: "1280, 720", + resolution: { width: 1280, height: 720 }, + expectedFrameRateRanges: [ + { minimumFrameRate: 10, maximumFrameRate: 10 }, + { minimumFrameRate: 30, maximumFrameRate: 30 } + ] + }, + { + tag: "1920, 1080", + resolution: { width: 1920, height: 1080 }, + expectedFrameRateRanges: [ + { minimumFrameRate: 5, maximumFrameRate: 10 } + ] + } + ] + } + + function test_supportedViewfinderFrameRateRanges(data) { + cameraLoader.sourceComponent = cameraComponent; + var camera = cameraLoader.item; + + // Pass the resolution as an object + var actualFrameRateRanges = camera.supportedViewfinderFrameRateRanges(data.resolution); + compare(actualFrameRateRanges.length, data.expectedFrameRateRanges.length); + for (var i = 0; i < actualFrameRateRanges.length; ++i) { + compare(actualFrameRateRanges[i].minimumFrameRate, data.expectedFrameRateRanges[i].minimumFrameRate); + compare(actualFrameRateRanges[i].maximumFrameRate, data.expectedFrameRateRanges[i].maximumFrameRate); + } + + // Pass the resolution as a size + if (typeof data.resolution !== 'undefined') { + actualFrameRateRanges = camera.supportedViewfinderFrameRateRanges(Qt.size(data.resolution.width, data.resolution.height)); + compare(actualFrameRateRanges.length, data.expectedFrameRateRanges.length); + for (i = 0; i < actualFrameRateRanges.length; ++i) { + compare(actualFrameRateRanges[i].minimumFrameRate, data.expectedFrameRateRanges[i].minimumFrameRate); + compare(actualFrameRateRanges[i].maximumFrameRate, data.expectedFrameRateRanges[i].maximumFrameRate); + } + } + + cameraLoader.sourceComponent = undefined; + } } From edc415be476c7ef59f00914b2a6dfc6896d7a34d Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Tue, 25 Aug 2015 14:09:16 +0200 Subject: [PATCH 03/12] Fix crash in QSoundEffect when using it from a non-GUI thread. The internal QIODevice used as data source was not set as child of the parent QSoundEffectPrivate. If moveToThread() was called on the QSoundEffect, the QIODevice would still receive events on the main thread, leading to race conditions. Task-number: QTBUG-46359 Change-Id: I180da2fb498108b316fd9b5b5cc84376b360fa3f Reviewed-by: Christian Stromme --- src/multimedia/audio/qsoundeffect_qaudio_p.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/multimedia/audio/qsoundeffect_qaudio_p.cpp b/src/multimedia/audio/qsoundeffect_qaudio_p.cpp index c4776b32..e95e4e52 100644 --- a/src/multimedia/audio/qsoundeffect_qaudio_p.cpp +++ b/src/multimedia/audio/qsoundeffect_qaudio_p.cpp @@ -297,6 +297,7 @@ void QSoundEffectPrivate::setCategory(const QString &category) } PrivateSoundSource::PrivateSoundSource(QSoundEffectPrivate* s): + QIODevice(s), m_loopCount(1), m_runningCount(0), m_playing(false), From 29ce8886b3bbc306a2ad52a4bc4ea89b3401658a Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Mon, 31 Aug 2015 16:50:53 +0200 Subject: [PATCH 04/12] Android: allow to map video buffers. Since we internally use a fbo, use QOpenGLFramebufferObject::toImage() to grab the pixel data and enable the use of map() on frames retrieved using QAbstractVideoFilter. Change-Id: If96e992e12e26091524913bb24926fa21d9d58cc Reviewed-by: Laszlo Agocs --- .../src/common/qandroidvideorendercontrol.cpp | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp index bf95c42f..cd9c9d1d 100644 --- a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp +++ b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp @@ -73,29 +73,60 @@ public: : QAbstractVideoBuffer(GLTextureHandle) , m_control(control) , m_textureUpdated(false) + , m_mapMode(NotMapped) { } virtual ~AndroidTextureVideoBuffer() {} - MapMode mapMode() const { return NotMapped; } - uchar *map(MapMode, int*, int*) { return 0; } - void unmap() {} + MapMode mapMode() const { return m_mapMode; } + + uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) + { + if (m_mapMode == NotMapped && mode == ReadOnly) { + updateFrame(); + m_mapMode = mode; + m_image = m_control->m_fbo->toImage(); + + if (numBytes) + *numBytes = m_image.byteCount(); + + if (bytesPerLine) + *bytesPerLine = m_image.bytesPerLine(); + + return m_image.bits(); + } else { + return 0; + } + } + + void unmap() + { + m_image = QImage(); + m_mapMode = NotMapped; + } QVariant handle() const + { + AndroidTextureVideoBuffer *that = const_cast(this); + that->updateFrame(); + return m_control->m_fbo->texture(); + } + +private: + void updateFrame() { if (!m_textureUpdated) { // update the video texture (called from the render thread) m_control->renderFrameToFbo(); m_textureUpdated = true; } - - return m_control->m_fbo->texture(); } -private: - mutable QAndroidVideoRendererControl *m_control; - mutable bool m_textureUpdated; + QAndroidVideoRendererControl *m_control; + bool m_textureUpdated; + MapMode m_mapMode; + QImage m_image; }; QAndroidVideoRendererControl::QAndroidVideoRendererControl(QObject *parent) From 95fa47f7472ac9bcb99df98bcd02c9a1907b6f3a Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Thu, 3 Sep 2015 15:48:37 +0200 Subject: [PATCH 05/12] AVFoundation: fix playback rate. Calling play on the AVPlayer internally resets the playback rate, which means any playback rate set before calling play was ignored. We now always call setRate to start playback to avoid resetting the rate with play. Aslo fixed the playbackRateChanged() signal that was never emitted. Task-number: QTBUG-45570 Change-Id: I3a77e1db31c57f1e3491287bdf977731b9d73509 Reviewed-by: Christian Stromme --- .../mediaplayer/avfmediaplayercontrol.mm | 2 ++ .../mediaplayer/avfmediaplayersession.h | 1 + .../mediaplayer/avfmediaplayersession.mm | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm index 1a4defac..ca11ef29 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm @@ -63,6 +63,8 @@ void AVFMediaPlayerControl::setSession(AVFMediaPlayerSession *session) connect(m_session, SIGNAL(audioAvailableChanged(bool)), this, SIGNAL(audioAvailableChanged(bool))); connect(m_session, SIGNAL(videoAvailableChanged(bool)), this, SIGNAL(videoAvailableChanged(bool))); connect(m_session, SIGNAL(error(int,QString)), this, SIGNAL(error(int,QString))); + connect(m_session, &AVFMediaPlayerSession::playbackRateChanged, + this, &AVFMediaPlayerControl::playbackRateChanged); } QMediaPlayer::State AVFMediaPlayerControl::state() const diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h index 5e224bb5..decab1b3 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h @@ -106,6 +106,7 @@ Q_SIGNALS: void mutedChanged(bool muted); void audioAvailableChanged(bool audioAvailable); void videoAvailableChanged(bool videoAvailable); + void playbackRateChanged(qreal rate); void error(int error, const QString &errorString); private: diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index 5ef1f9ba..c0e81321 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -602,10 +602,10 @@ void AVFMediaPlayerSession::setPlaybackRate(qreal rate) m_rate = rate; AVPlayer *player = [(AVFMediaPlayerSessionObserver*)m_observer player]; - - if (player != 0 && m_state == QMediaPlayer::PlayingState) { + if (player && m_state == QMediaPlayer::PlayingState) [player setRate:m_rate]; - } + + Q_EMIT playbackRateChanged(m_rate); } void AVFMediaPlayerSession::setPosition(qint64 pos) @@ -655,8 +655,10 @@ void AVFMediaPlayerSession::play() processLoadStateChange(); } - if (m_mediaStatus == QMediaPlayer::LoadedMedia || m_mediaStatus == QMediaPlayer::BufferedMedia) - [[(AVFMediaPlayerSessionObserver*)m_observer player] play]; + if (m_mediaStatus == QMediaPlayer::LoadedMedia || m_mediaStatus == QMediaPlayer::BufferedMedia) { + // Setting the rate starts playback + [[(AVFMediaPlayerSessionObserver*)m_observer player] setRate:m_rate]; + } //processLoadStateChange(); Q_EMIT stateChanged(m_state); @@ -697,8 +699,8 @@ void AVFMediaPlayerSession::stop() return; m_state = QMediaPlayer::StoppedState; - m_rate = 0.0f; - [[(AVFMediaPlayerSessionObserver*)m_observer player] setRate:m_rate]; + // AVPlayer doesn't have stop(), only pause() and play(). + [[(AVFMediaPlayerSessionObserver*)m_observer player] pause]; setPosition(0); if (m_videoOutput) { @@ -821,8 +823,8 @@ void AVFMediaPlayerSession::processLoadStateChange() newStatus = isPlaying ? QMediaPlayer::BufferedMedia : QMediaPlayer::LoadedMedia; if (m_state == QMediaPlayer::PlayingState && [(AVFMediaPlayerSessionObserver*)m_observer player]) { + // Setting the rate is enough to start playback, no need to call play() [[(AVFMediaPlayerSessionObserver*)m_observer player] setRate:m_rate]; - [[(AVFMediaPlayerSessionObserver*)m_observer player] play]; } } From 6a6a69b7d8343a3cdd9daff60b9288eb361148c9 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Thu, 3 Sep 2015 17:02:26 +0200 Subject: [PATCH 06/12] AVFoundation: fix initial volume. The volume level set before loading a media was never actually set on AVPlayer. Regression introduced by 4e07ff99 on OSX. Task-number: QTBUG-48154 Change-Id: I599e3d55b35d7196aebc4753a367a29049f99d33 Reviewed-by: Christian Stromme --- .../mediaplayer/avfmediaplayersession.h | 3 ++ .../mediaplayer/avfmediaplayersession.mm | 47 ++++++++++--------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h index decab1b3..1e675c02 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h @@ -80,6 +80,8 @@ public: qreal playbackRate() const; + inline bool isVolumeSupported() const { return m_volumeSupported; } + public Q_SLOTS: void setPlaybackRate(qreal rate); @@ -159,6 +161,7 @@ private: QMediaContent m_resources; ResourceHandler m_resourceHandler; + const bool m_volumeSupported; bool m_muted; bool m_tryingAsync; int m_volume; diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index c0e81321..d9d705c6 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -222,11 +222,11 @@ static void *AVFMediaPlayerSessionObserverCurrentItemObservationContext = &AVFMe m_player = [AVPlayer playerWithPlayerItem:m_playerItem]; [m_player retain]; -#if defined(Q_OS_OSX) //Set the initial volume on new player object - if (self.session) - m_player.volume = m_session->volume() / 100.0f; -#endif + if (self.session && self.session->isVolumeSupported()) { + [m_player setVolume:m_session->volume() / 100.0f]; + [m_player setMuted:m_session->isMuted()]; + } //Create a new player layer if we don't have one already if (!m_playerLayer) @@ -382,6 +382,11 @@ AVFMediaPlayerSession::AVFMediaPlayerSession(AVFMediaPlayerService *service, QOb , m_state(QMediaPlayer::StoppedState) , m_mediaStatus(QMediaPlayer::NoMedia) , m_mediaStream(0) +#ifdef Q_OS_IOS + , m_volumeSupported(QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_7_0) +#else + , m_volumeSupported(true) +#endif , m_muted(false) , m_tryingAsync(false) , m_volume(100) @@ -718,21 +723,20 @@ void AVFMediaPlayerSession::setVolume(int volume) qDebug() << Q_FUNC_INFO << volume; #endif - if (m_volume == volume) - return; - - AVPlayer *player = [(AVFMediaPlayerSessionObserver*)m_observer player]; - if (!player) - return; - - if (![player respondsToSelector:@selector(setVolume:)]) { + if (!m_volumeSupported) { qWarning("%s not implemented, requires iOS 7 or later", Q_FUNC_INFO); return; } - [player setVolume:volume / 100.0f]; + if (m_volume == volume) + return; + m_volume = volume; + AVPlayer *player = [(AVFMediaPlayerSessionObserver*)m_observer player]; + if (player) + [player setVolume:volume / 100.0f]; + Q_EMIT volumeChanged(m_volume); } @@ -741,22 +745,21 @@ void AVFMediaPlayerSession::setMuted(bool muted) #ifdef QT_DEBUG_AVF qDebug() << Q_FUNC_INFO << muted; #endif - if (m_muted == muted) - return; - AVPlayer *player = [(AVFMediaPlayerSessionObserver*)m_observer player]; - if (!player) - return; - - // iOS: setMuted exists since iOS 7.0, thus check if it exists - if (![player respondsToSelector:@selector(setMuted:)]) { + if (!m_volumeSupported) { qWarning("%s not implemented, requires iOS 7 or later", Q_FUNC_INFO); return; } - [player setMuted:muted]; + if (m_muted == muted) + return; + m_muted = muted; + AVPlayer *player = [(AVFMediaPlayerSessionObserver*)m_observer player]; + if (player) + [player setMuted:muted]; + Q_EMIT mutedChanged(muted); } From bede0a838ddafadafc400f20c00a687e753ea638 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Thu, 3 Sep 2015 18:33:17 +0200 Subject: [PATCH 07/12] AVFoundation: store seek requests before media is loaded. And actually seek once the media is loaded. Task-number: QTBUG-48057 Change-Id: I9446a1e66a48f9a94c039be9af81689ed04bc56c Reviewed-by: Christian Stromme --- .../mediaplayer/avfmediaplayersession.h | 1 + .../mediaplayer/avfmediaplayersession.mm | 31 ++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h index 1e675c02..024f380c 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h @@ -166,6 +166,7 @@ private: bool m_tryingAsync; int m_volume; qreal m_rate; + qint64 m_requestedPosition; qint64 m_duration; bool m_videoAvailable; diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index d9d705c6..1184451a 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -391,6 +391,7 @@ AVFMediaPlayerSession::AVFMediaPlayerSession(AVFMediaPlayerService *service, QOb , m_tryingAsync(false) , m_volume(100) , m_rate(1.0) + , m_requestedPosition(-1) , m_duration(0) , m_videoAvailable(false) , m_audioAvailable(false) @@ -468,6 +469,8 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st setAudioAvailable(false); setVideoAvailable(false); + m_requestedPosition = -1; + Q_EMIT positionChanged(position()); QMediaPlayer::MediaStatus oldMediaStatus = m_mediaStatus; @@ -479,7 +482,6 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st if (m_mediaStatus != oldMediaStatus) Q_EMIT mediaStatusChanged(m_mediaStatus); - Q_EMIT positionChanged(position()); return; } else { @@ -487,6 +489,7 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st if (m_mediaStatus != oldMediaStatus) Q_EMIT mediaStatusChanged(m_mediaStatus); } + //Load AVURLAsset //initialize asset using content's URL NSString *urlString = [NSString stringWithUTF8String:content.canonicalUrl().toEncoded().constData()]; @@ -499,7 +502,7 @@ qint64 AVFMediaPlayerSession::position() const AVPlayerItem *playerItem = [(AVFMediaPlayerSessionObserver*)m_observer playerItem]; if (!playerItem) - return 0; + return m_requestedPosition != -1 ? m_requestedPosition : 0; CMTime time = [playerItem currentTime]; return static_cast(float(time.value) / float(time.timescale) * 1000.0f); @@ -619,14 +622,23 @@ void AVFMediaPlayerSession::setPosition(qint64 pos) qDebug() << Q_FUNC_INFO << pos; #endif - if ( !isSeekable() || pos == position()) + if (pos == position()) return; AVPlayerItem *playerItem = [(AVFMediaPlayerSessionObserver*)m_observer playerItem]; - - if (!playerItem) + if (!playerItem) { + m_requestedPosition = pos; + Q_EMIT positionChanged(m_requestedPosition); return; + } else if (!isSeekable()) { + if (m_requestedPosition != -1) { + m_requestedPosition = -1; + Q_EMIT positionChanged(position()); + } + return; + } + pos = qMax(qint64(0), pos); if (duration() > 0) pos = qMin(pos, duration()); @@ -823,6 +835,11 @@ void AVFMediaPlayerSession::processLoadStateChange() if (m_duration != currentDuration) Q_EMIT durationChanged(m_duration = currentDuration); + if (m_requestedPosition != -1) { + setPosition(m_requestedPosition); + m_requestedPosition = -1; + } + newStatus = isPlaying ? QMediaPlayer::BufferedMedia : QMediaPlayer::LoadedMedia; if (m_state == QMediaPlayer::PlayingState && [(AVFMediaPlayerSessionObserver*)m_observer player]) { @@ -842,6 +859,10 @@ void AVFMediaPlayerSession::processPositionChange() void AVFMediaPlayerSession::processMediaLoadError() { + if (m_requestedPosition != -1) { + m_requestedPosition = -1; + Q_EMIT positionChanged(position()); + } Q_EMIT error(QMediaPlayer::FormatError, tr("Failed to load media")); Q_EMIT mediaStatusChanged(m_mediaStatus = QMediaPlayer::InvalidMedia); Q_EMIT stateChanged(m_state = QMediaPlayer::StoppedState); From fdac8d9c74185e39a46581be8519501251b88fc8 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Thu, 3 Sep 2015 18:42:52 +0200 Subject: [PATCH 08/12] AVFoundation: correctly unload current media. When loading a new media, we should unload the previous media. This makes sure all properties set right after loading a new media (e.g. position, volume) are correctly set on the new AVPlayer and not on the old one. Change-Id: I4cd71b785ccdb4cd0772cedffc3c25665f402776 Reviewed-by: Christian Stromme --- .../mediaplayer/avfmediaplayersession.mm | 45 ++++++------------- 1 file changed, 14 insertions(+), 31 deletions(-) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index 1184451a..c2760193 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -121,19 +121,28 @@ static void *AVFMediaPlayerSessionObserverCurrentItemObservationContext = &AVFMe - (void) unloadMedia { - if (m_player) - [m_player setRate:0.0]; if (m_playerItem) { [m_playerItem removeObserver:self forKeyPath:AVF_STATUS_KEY]; [[NSNotificationCenter defaultCenter] removeObserver:self name:AVPlayerItemDidPlayToEndTimeNotification - object:m_playerItem]; + object:m_playerItem]; [[NSNotificationCenter defaultCenter] removeObserver:self name:AVPlayerItemTimeJumpedNotification object:m_playerItem]; m_playerItem = 0; } + if (m_player) { + [m_player setRate:0.0]; + [m_player removeObserver:self forKeyPath:AVF_CURRENT_ITEM_KEY]; + [m_player removeObserver:self forKeyPath:AVF_RATE_KEY]; + [m_player release]; + m_player = 0; + } + if (m_playerLayer) { + [m_playerLayer release]; + m_playerLayer = 0; + } } - (void) prepareToPlayAsset:(AVURLAsset *)asset @@ -203,21 +212,6 @@ static void *AVFMediaPlayerSessionObserverCurrentItemObservationContext = &AVFMe name:AVPlayerItemTimeJumpedNotification object:m_playerItem]; - - //Clean up old player if we have one - if (m_player) { - [m_player setRate:0.0]; - [m_player removeObserver:self forKeyPath:AVF_CURRENT_ITEM_KEY]; - [m_player removeObserver:self forKeyPath:AVF_RATE_KEY]; - [m_player release]; - m_player = 0; - - if (m_playerLayer) { - [m_playerLayer release]; - m_playerLayer = 0; //Will have been released - } - } - //Get a new AVPlayer initialized to play the specified player item. m_player = [AVPlayer playerWithPlayerItem:m_playerItem]; [m_player retain]; @@ -354,18 +348,6 @@ static void *AVFMediaPlayerSessionObserverCurrentItemObservationContext = &AVFMe #endif [self unloadMedia]; - if (m_player) { - [m_player removeObserver:self forKeyPath:AVF_CURRENT_ITEM_KEY]; - [m_player removeObserver:self forKeyPath:AVF_RATE_KEY]; - [m_player release]; - m_player = 0; - } - - if (m_playerLayer) { - [m_playerLayer release]; - m_playerLayer = 0; - } - if (m_URL) { [m_URL release]; } @@ -464,6 +446,8 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st qDebug() << Q_FUNC_INFO << content.canonicalUrl(); #endif + [(AVFMediaPlayerSessionObserver*)m_observer unloadMedia]; + m_resources = content; m_mediaStream = stream; @@ -475,7 +459,6 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st QMediaPlayer::MediaStatus oldMediaStatus = m_mediaStatus; if (content.isNull() || content.canonicalUrl().isEmpty()) { - [(AVFMediaPlayerSessionObserver*)m_observer unloadMedia]; m_mediaStatus = QMediaPlayer::NoMedia; if (m_state != QMediaPlayer::StoppedState) Q_EMIT stateChanged(m_state = QMediaPlayer::StoppedState); From e9cf6670fe61f67c8f16ed0393086c07f88582dd Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Fri, 4 Sep 2015 14:59:55 +0200 Subject: [PATCH 09/12] AVFoundation: fix rendering when player reached EndOfMedia. Once the media player reached the end of the media, it shouldn't be rendering frames anymore. This could happen when seeking after reaching the end. The output layer is now cleared on EndOfMedia to make sure that doesn't happen. Change-Id: I18a21eaff6c63a2bd54d4c2953f89eb1722f66d5 Reviewed-by: Christian Stromme --- .../avfoundation/mediaplayer/avfmediaplayersession.mm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index c2760193..50ec1ffa 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -767,6 +767,11 @@ void AVFMediaPlayerSession::processEOS() Q_EMIT positionChanged(position()); m_mediaStatus = QMediaPlayer::EndOfMedia; + // At this point, frames should not be rendered anymore. + // Clear the output layer to make sure of that. + if (m_videoOutput) + m_videoOutput->setLayer(0); + Q_EMIT stateChanged(m_state = QMediaPlayer::StoppedState); Q_EMIT mediaStatusChanged(m_mediaStatus); } From 9c5292927e9f9668671f8e83ecdcc271515b9bf4 Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Fri, 4 Sep 2015 18:04:51 +0200 Subject: [PATCH 10/12] AVFoundation: implemented QMediaPlayer::seekable. Change-Id: Iaca8daa2460062954497b3e510dd1828953c80fd Reviewed-by: Christian Stromme --- .../mediaplayer/avfmediaplayercontrol.mm | 2 ++ .../mediaplayer/avfmediaplayersession.h | 3 +++ .../mediaplayer/avfmediaplayersession.mm | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm index ca11ef29..1f4b42bf 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayercontrol.mm @@ -65,6 +65,8 @@ void AVFMediaPlayerControl::setSession(AVFMediaPlayerSession *session) connect(m_session, SIGNAL(error(int,QString)), this, SIGNAL(error(int,QString))); connect(m_session, &AVFMediaPlayerSession::playbackRateChanged, this, &AVFMediaPlayerControl::playbackRateChanged); + connect(m_session, &AVFMediaPlayerSession::seekableChanged, + this, &AVFMediaPlayerControl::seekableChanged); } QMediaPlayer::State AVFMediaPlayerControl::state() const diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h index 024f380c..18bcb23e 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.h @@ -109,6 +109,7 @@ Q_SIGNALS: void audioAvailableChanged(bool audioAvailable); void videoAvailableChanged(bool videoAvailable); void playbackRateChanged(qreal rate); + void seekableChanged(bool seekable); void error(int error, const QString &errorString); private: @@ -151,6 +152,7 @@ private: void setAudioAvailable(bool available); void setVideoAvailable(bool available); + void setSeekable(bool seekable); AVFMediaPlayerService *m_service; AVFVideoOutput *m_videoOutput; @@ -171,6 +173,7 @@ private: qint64 m_duration; bool m_videoAvailable; bool m_audioAvailable; + bool m_seekable; void *m_observer; }; diff --git a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm index 50ec1ffa..53cdebd8 100644 --- a/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm +++ b/src/plugins/avfoundation/mediaplayer/avfmediaplayersession.mm @@ -377,6 +377,7 @@ AVFMediaPlayerSession::AVFMediaPlayerSession(AVFMediaPlayerService *service, QOb , m_duration(0) , m_videoAvailable(false) , m_audioAvailable(false) + , m_seekable(false) { m_observer = [[AVFMediaPlayerSessionObserver alloc] initWithMediaPlayerSession:this]; } @@ -453,6 +454,7 @@ void AVFMediaPlayerSession::setMedia(const QMediaContent &content, QIODevice *st setAudioAvailable(false); setVideoAvailable(false); + setSeekable(false); m_requestedPosition = -1; Q_EMIT positionChanged(position()); @@ -554,7 +556,16 @@ bool AVFMediaPlayerSession::isVideoAvailable() const bool AVFMediaPlayerSession::isSeekable() const { - return true; + return m_seekable; +} + +void AVFMediaPlayerSession::setSeekable(bool seekable) +{ + if (m_seekable == seekable) + return; + + m_seekable = seekable; + Q_EMIT seekableChanged(seekable); } QMediaTimeRange AVFMediaPlayerSession::availablePlaybackRanges() const @@ -806,6 +817,8 @@ void AVFMediaPlayerSession::processLoadStateChange() } } + setSeekable([[playerItem seekableTimeRanges] count] > 0); + // Get the native size of the video, and reset the bounds of the player layer AVPlayerLayer *playerLayer = [(AVFMediaPlayerSessionObserver*)m_observer playerLayer]; if (videoTrack && playerLayer) { From 99f6cf5f282c0d3d7192d5e87b745873e159c93d Mon Sep 17 00:00:00 2001 From: Marco Benelli Date: Tue, 15 Sep 2015 13:37:57 +0200 Subject: [PATCH 11/12] Fixed revisions of camera-related meta objects. Change-Id: I91fe90122d53ce12322cecac8282ef9622869b71 Reviewed-by: Yoann Lopes --- src/imports/multimedia/multimedia.cpp | 6 +++--- src/imports/multimedia/plugins.qmltypes | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/imports/multimedia/multimedia.cpp b/src/imports/multimedia/multimedia.cpp index 652359f3..4b31068e 100644 --- a/src/imports/multimedia/multimedia.cpp +++ b/src/imports/multimedia/multimedia.cpp @@ -97,7 +97,7 @@ public: trUtf8("CameraImageProcessing is provided by Camera")); // 5.2 types - qmlRegisterRevision(uri, 5, 2); + qmlRegisterType(uri, 5, 2, "VideoOutput"); // 5.3 types // Nothing changed, but adding "import QtMultimedia 5.3" in QML will fail unless at @@ -106,13 +106,13 @@ public: // 5.4 types qmlRegisterSingletonType(uri, 5, 4, "QtMultimedia", multimedia_global_object); - qmlRegisterRevision(uri, 5, 4); + qmlRegisterType(uri, 5, 4, "Camera"); qmlRegisterUncreatableType(uri, 5, 4, "CameraViewfinder", trUtf8("CameraViewfinder is provided by Camera")); // 5.5 types qmlRegisterUncreatableType(uri, 5, 5, "CameraImageProcessing", trUtf8("CameraImageProcessing is provided by Camera")); - qmlRegisterRevision(uri, 5, 5); + qmlRegisterType(uri, 5, 5, "Camera"); qmlRegisterType(); qmlRegisterType(); diff --git a/src/imports/multimedia/plugins.qmltypes b/src/imports/multimedia/plugins.qmltypes index d01c988b..53d7cb43 100644 --- a/src/imports/multimedia/plugins.qmltypes +++ b/src/imports/multimedia/plugins.qmltypes @@ -1,4 +1,4 @@ -import QtQuick.tooling 1.1 +import QtQuick.tooling 1.2 // This file describes the plugin-supplied types contained in the library. // It is used for QML tooling purposes only. @@ -7,6 +7,7 @@ import QtQuick.tooling 1.1 // 'qmlplugindump -nonrelocatable QtMultimedia 5.5' Module { + dependencies: ["QtQuick 2.0"] Component { name: "QAbstractVideoFilter" prototype: "QObject" @@ -244,8 +245,12 @@ Module { Component { name: "QDeclarativeCamera" prototype: "QObject" - exports: ["QtMultimedia/Camera 5.0"] - exportMetaObjectRevisions: [0] + exports: [ + "QtMultimedia/Camera 5.0", + "QtMultimedia/Camera 5.4", + "QtMultimedia/Camera 5.5" + ] + exportMetaObjectRevisions: [0, 1, 2] Enum { name: "Position" values: { @@ -517,7 +522,7 @@ Module { name: "supportedViewfinderFrameRateRanges" revision: 2 type: "QJSValue" - Parameter { name: "resolution"; type: "QSize" } + Parameter { name: "resolution"; type: "QJSValue" } } Method { name: "supportedViewfinderFrameRateRanges"; revision: 2; type: "QJSValue" } } @@ -1497,8 +1502,11 @@ Module { name: "QDeclarativeVideoOutput" defaultProperty: "data" prototype: "QQuickItem" - exports: ["QtMultimedia/VideoOutput 5.0"] - exportMetaObjectRevisions: [0] + exports: [ + "QtMultimedia/VideoOutput 5.0", + "QtMultimedia/VideoOutput 5.2" + ] + exportMetaObjectRevisions: [0, 2] Enum { name: "FillMode" values: { From 46a83d5b86a792e62f14674e27c5d95695f2b44d Mon Sep 17 00:00:00 2001 From: Yoann Lopes Date: Fri, 29 May 2015 18:02:26 +0200 Subject: [PATCH 12/12] AVFoundation: render camera frames using OpenGL textures on iOS. OpenGL textures can easily be created from a CVImageBuffer using Apple APIs. It avoids having to map the buffer to main memory and therefore greatly improves rendering performances. We could do the same on OSX, but there, the textures are always of the GL_TEXTURE_RECTANGLE target type and changes need to be done to the QVideoFrame API and to the video node implementations to support that. Change-Id: I6dde7e8d7a27460e41523cd474c3c741affc1480 Reviewed-by: James Turner Reviewed-by: Christian Stromme --- .../camera/avfcamerarenderercontrol.h | 9 ++ .../camera/avfcamerarenderercontrol.mm | 137 ++++++++++++++---- .../avfcameraviewfindersettingscontrol.mm | 27 ++-- 3 files changed, 135 insertions(+), 38 deletions(-) diff --git a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.h b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.h index b8f92d9c..2d74e251 100644 --- a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.h +++ b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.h @@ -63,6 +63,8 @@ public: AVCaptureVideoDataOutput *videoDataOutput() const; + bool supportsTextures() const { return m_supportsTextures; } + #ifdef Q_OS_IOS AVFCaptureFramesDelegate *captureDelegate() const; void resetCaptureDelegate() const; @@ -81,11 +83,18 @@ private: AVFCameraSession *m_cameraSession; AVCaptureVideoDataOutput *m_videoDataOutput; + bool m_supportsTextures; bool m_needsHorizontalMirroring; +#ifdef Q_OS_IOS + CVOpenGLESTextureCacheRef m_textureCache; +#endif + QVideoFrame m_lastViewfinderFrame; QMutex m_vfMutex; dispatch_queue_t m_delegateQueue; + + friend class CVImageVideoBuffer; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm index 52954128..924c62d9 100644 --- a/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm +++ b/src/plugins/avfoundation/camera/avfcamerarenderercontrol.mm @@ -38,6 +38,10 @@ #include "avfcameraservice.h" #include "avfcameradebug.h" +#ifdef Q_OS_IOS +#include +#endif + #include #include @@ -45,20 +49,32 @@ QT_USE_NAMESPACE -class CVPixelBufferVideoBuffer : public QAbstractPlanarVideoBuffer +class CVImageVideoBuffer : public QAbstractPlanarVideoBuffer { - friend class CVPixelBufferVideoBufferPrivate; public: - CVPixelBufferVideoBuffer(CVPixelBufferRef buffer) + CVImageVideoBuffer(CVImageBufferRef buffer, AVFCameraRendererControl *renderer) +#ifndef Q_OS_IOS : QAbstractPlanarVideoBuffer(NoHandle) +#else + : QAbstractPlanarVideoBuffer(renderer->supportsTextures() + && CVPixelBufferGetPixelFormatType(buffer) == kCVPixelFormatType_32BGRA + ? GLTextureHandle : NoHandle) + , m_texture(0) +#endif , m_buffer(buffer) + , m_renderer(renderer) , m_mode(NotMapped) { CVPixelBufferRetain(m_buffer); } - virtual ~CVPixelBufferVideoBuffer() + ~CVImageVideoBuffer() { + CVImageVideoBuffer::unmap(); +#ifdef Q_OS_IOS + if (m_texture) + CFRelease(m_texture); +#endif CVPixelBufferRelease(m_buffer); } @@ -78,7 +94,9 @@ public: // For a bi-planar format we have to set the parameters correctly: if (mode != QAbstractVideoBuffer::NotMapped && m_mode == QAbstractVideoBuffer::NotMapped) { - CVPixelBufferLockBaseAddress(m_buffer, 0); + CVPixelBufferLockBaseAddress(m_buffer, mode == QAbstractVideoBuffer::ReadOnly + ? kCVPixelBufferLock_ReadOnly + : 0); if (numBytes) *numBytes = CVPixelBufferGetDataSize(m_buffer); @@ -103,8 +121,9 @@ public: uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) { if (mode != NotMapped && m_mode == NotMapped) { - CVPixelBufferLockBaseAddress(m_buffer, 0); - + CVPixelBufferLockBaseAddress(m_buffer, mode == QAbstractVideoBuffer::ReadOnly + ? kCVPixelBufferLock_ReadOnly + : 0); if (numBytes) *numBytes = CVPixelBufferGetDataSize(m_buffer); @@ -121,13 +140,63 @@ public: void unmap() { if (m_mode != NotMapped) { + CVPixelBufferUnlockBaseAddress(m_buffer, m_mode == QAbstractVideoBuffer::ReadOnly + ? kCVPixelBufferLock_ReadOnly + : 0); m_mode = NotMapped; - CVPixelBufferUnlockBaseAddress(m_buffer, 0); } } + QVariant handle() const + { +#ifdef Q_OS_IOS + // Called from the render thread, so there is a current OpenGL context + + if (!m_renderer->m_textureCache) { + CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, + NULL, + [EAGLContext currentContext], + NULL, + &m_renderer->m_textureCache); + + if (err != kCVReturnSuccess) + qWarning("Error creating texture cache"); + } + + if (m_renderer->m_textureCache && !m_texture) { + CVOpenGLESTextureCacheFlush(m_renderer->m_textureCache, 0); + + CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + m_renderer->m_textureCache, + m_buffer, + NULL, + GL_TEXTURE_2D, + GL_RGBA, + CVPixelBufferGetWidth(m_buffer), + CVPixelBufferGetHeight(m_buffer), + GL_BGRA, + GL_UNSIGNED_BYTE, + 0, + &m_texture); + if (err != kCVReturnSuccess) + qWarning("Error creating texture from buffer"); + } + + if (m_texture) + return CVOpenGLESTextureGetName(m_texture); + else + return 0; +#else + return QVariant(); +#endif + } + private: - CVPixelBufferRef m_buffer; +#ifdef Q_OS_IOS + mutable CVOpenGLESTextureRef m_texture; +#endif + CVImageBufferRef m_buffer; + AVFCameraRendererControl *m_renderer; MapMode m_mode; }; @@ -171,13 +240,25 @@ private: int width = CVPixelBufferGetWidth(imageBuffer); int height = CVPixelBufferGetHeight(imageBuffer); - QVideoFrame::PixelFormat format = - AVFCameraViewfinderSettingsControl2::QtPixelFormatFromCVFormat(CVPixelBufferGetPixelFormatType(imageBuffer)); + QVideoFrame::PixelFormat format; + +#ifdef Q_OS_IOS + bool useTexture = m_renderer->supportsTextures() + && CVPixelBufferGetPixelFormatType(imageBuffer) == kCVPixelFormatType_32BGRA; + + if (useTexture) + format = QVideoFrame::Format_BGRA32; + else +#endif + format = AVFCameraViewfinderSettingsControl2::QtPixelFormatFromCVFormat(CVPixelBufferGetPixelFormatType(imageBuffer)); if (format == QVideoFrame::Format_Invalid) return; - QVideoFrame frame(new CVPixelBufferVideoBuffer(imageBuffer), QSize(width, height), format); + QVideoFrame frame(new CVImageVideoBuffer(imageBuffer, m_renderer), + QSize(width, height), + format); + m_renderer->syncHandleViewfinderFrame(frame); } @@ -187,7 +268,11 @@ private: AVFCameraRendererControl::AVFCameraRendererControl(QObject *parent) : QVideoRendererControl(parent) , m_surface(0) + , m_supportsTextures(false) , m_needsHorizontalMirroring(false) +#ifdef Q_OS_IOS + , m_textureCache(0) +#endif { m_viewfinderFramesDelegate = [[AVFCaptureFramesDelegate alloc] initWithRenderer:this]; } @@ -198,6 +283,10 @@ AVFCameraRendererControl::~AVFCameraRendererControl() [m_viewfinderFramesDelegate release]; if (m_delegateQueue) dispatch_release(m_delegateQueue); +#ifdef Q_OS_IOS + if (m_textureCache) + CFRelease(m_textureCache); +#endif } QAbstractVideoSurface *AVFCameraRendererControl::surface() const @@ -209,6 +298,11 @@ void AVFCameraRendererControl::setSurface(QAbstractVideoSurface *surface) { if (m_surface != surface) { m_surface = surface; +#ifdef Q_OS_IOS + m_supportsTextures = m_surface + ? m_surface->supportedPixelFormats(QAbstractVideoBuffer::GLTextureHandle).contains(QVideoFrame::Format_BGRA32) + : false; +#endif Q_EMIT surfaceChanged(surface); } } @@ -261,21 +355,6 @@ void AVFCameraRendererControl::syncHandleViewfinderFrame(const QVideoFrame &fram m_lastViewfinderFrame = frame; - if (m_needsHorizontalMirroring) { - m_lastViewfinderFrame.map(QAbstractVideoBuffer::ReadOnly); - - // no deep copy - QImage image(m_lastViewfinderFrame.bits(), - m_lastViewfinderFrame.size().width(), - m_lastViewfinderFrame.size().height(), - m_lastViewfinderFrame.bytesPerLine(), - QImage::Format_RGB32); - - QImage mirrored = image.mirrored(true, false); - - m_lastViewfinderFrame.unmap(); - m_lastViewfinderFrame = QVideoFrame(mirrored); - } if (m_cameraSession && m_lastViewfinderFrame.isValid()) m_cameraSession->onCameraFrameFetched(m_lastViewfinderFrame); } @@ -315,7 +394,9 @@ void AVFCameraRendererControl::handleViewfinderFrame() } if (!m_surface->isActive()) { - QVideoSurfaceFormat format(frame.size(), frame.pixelFormat()); + QVideoSurfaceFormat format(frame.size(), frame.pixelFormat(), frame.handleType()); + if (m_needsHorizontalMirroring) + format.setProperty("mirrored", true); if (!m_surface->start(format)) { qWarning() << "Failed to start viewfinder m_surface, format:" << format; diff --git a/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm b/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm index b32c0cd6..05f28898 100644 --- a/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm +++ b/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm @@ -573,22 +573,29 @@ void AVFCameraViewfinderSettingsControl2::applySettings() if (!convertPixelFormatIfSupported(m_settings.pixelFormat(), avfPixelFormat)) { // If the the pixel format is not specified or invalid, pick the preferred video surface // format, or if no surface is set, the preferred capture device format + const QVector deviceFormats = viewfinderPixelFormats(); - QList surfaceFormats; - if (m_service->videoOutput() && m_service->videoOutput()->surface()) - surfaceFormats = m_service->videoOutput()->surface()->supportedPixelFormats(); + QVideoFrame::PixelFormat pickedFormat = deviceFormats.first(); - QVideoFrame::PixelFormat format = deviceFormats.first(); + QAbstractVideoSurface *surface = m_service->videoOutput() ? m_service->videoOutput()->surface() + : 0; + if (surface) { + if (m_service->videoOutput()->supportsTextures()) { + pickedFormat = QVideoFrame::Format_ARGB32; + } else { + QList surfaceFormats = m_service->videoOutput()->surface()->supportedPixelFormats(); - for (int i = 0; i < surfaceFormats.count(); ++i) { - const QVideoFrame::PixelFormat surfaceFormat = surfaceFormats.at(i); - if (deviceFormats.contains(surfaceFormat)) { - format = surfaceFormat; - break; + for (int i = 0; i < surfaceFormats.count(); ++i) { + const QVideoFrame::PixelFormat surfaceFormat = surfaceFormats.at(i); + if (deviceFormats.contains(surfaceFormat)) { + pickedFormat = surfaceFormat; + break; + } + } } } - CVPixelFormatFromQtFormat(format, avfPixelFormat); + CVPixelFormatFromQtFormat(pickedFormat, avfPixelFormat); } if (avfPixelFormat != 0) {