Android: QVideoProbe support for camera

QMediaVideoProbeControl sublclass added to capture service to make
QVideoProbe work with Android camera.

[ChangeLog][QtMultimedia][Android] QVideoProbe support for camera

Task-number: QTBUG-35416
Change-Id: I14d0a0e8abd14ee8f577e2901b976b8ed46eb320
Reviewed-by: Yoann Lopes <yoann.lopes@digia.com>

src/plugins/android/jar/src/org/qtproject/qt5/android/multimedia/QtCamera.java

@@ -60,6 +60,7 @@ public class QtCamera implements Camera.ShutterCallback,
     private int m_actualPreviewBuffer = 0;
     private final ReentrantLock m_buffersLock = new ReentrantLock();
     private boolean m_isReleased = false;
+    private boolean m_fetchEachFrame = false;
 
     private static final String TAG = "Qt Camera";
 
@@ -141,6 +142,11 @@ public class QtCamera implements Camera.ShutterCallback,
         }
     }
 
+    public void fetchEachFrame(boolean fetch)
+    {
+        m_fetchEachFrame = fetch;
+    }
+
     public void startPreview()
     {
         Camera.Size previewSize = m_camera.getParameters().getPreviewSize();
@@ -233,6 +239,10 @@ public class QtCamera implements Camera.ShutterCallback,
     public void onPreviewFrame(byte[] data, Camera camera)
     {
         m_buffersLock.lock();
+
+        if (data != null && m_fetchEachFrame)
+            notifyFrameFetched(m_cameraId, data);
+
         if (data == m_cameraPreviewFirstBuffer)
             m_actualPreviewBuffer = 1;
         else if (data == m_cameraPreviewSecondBuffer)
@@ -252,4 +262,5 @@ public class QtCamera implements Camera.ShutterCallback,
     private static native void notifyAutoFocusComplete(int id, boolean success);
     private static native void notifyPictureExposed(int id);
     private static native void notifyPictureCaptured(int id, byte[] data);
+    private static native void notifyFrameFetched(int id, byte[] data);
 }

src/plugins/android/src/mediacapture/mediacapture.pri

@@ -21,7 +21,8 @@ SOURCES += \
     $$PWD/qandroidaudioencodersettingscontrol.cpp \
     $$PWD/qandroidmediacontainercontrol.cpp \
     $$PWD/qandroidvideoencodersettingscontrol.cpp \
-    $$PWD/qandroidaudioinputselectorcontrol.cpp
+    $$PWD/qandroidaudioinputselectorcontrol.cpp \
+    $$PWD/qandroidmediavideoprobecontrol.cpp
 
 HEADERS += \
     $$PWD/qandroidcaptureservice.h \
@@ -44,4 +45,5 @@ HEADERS += \
     $$PWD/qandroidaudioencodersettingscontrol.h \
     $$PWD/qandroidmediacontainercontrol.h \
     $$PWD/qandroidvideoencodersettingscontrol.h \
-    $$PWD/qandroidaudioinputselectorcontrol.h
+    $$PWD/qandroidaudioinputselectorcontrol.h \
+    $$PWD/qandroidmediavideoprobecontrol.h

src/plugins/android/src/mediacapture/qandroidcamerasession.cpp

@@ -44,11 +44,13 @@
 #include "jcamera.h"
 #include "jmultimediautils.h"
 #include "qandroidvideooutput.h"
+#include "qandroidmediavideoprobecontrol.h"
 #include "qandroidmultimediautils.h"
 #include <QtConcurrent/qtconcurrentrun.h>
 #include <qfile.h>
 #include <qguiapplication.h>
 #include <qdebug.h>
+#include <qvideoframe.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -183,6 +185,9 @@ bool QAndroidCameraSession::open()
     if (m_camera) {
         connect(m_camera, SIGNAL(pictureExposed()), this, SLOT(onCameraPictureExposed()));
         connect(m_camera, SIGNAL(previewFetched(QByteArray)), this, SLOT(onCameraPreviewFetched(QByteArray)));
+        connect(m_camera, SIGNAL(frameFetched(QByteArray)),
+                this, SLOT(onCameraFrameFetched(QByteArray)),
+                Qt::DirectConnection);
         connect(m_camera, SIGNAL(pictureCaptured(QByteArray)), this, SLOT(onCameraPictureCaptured(QByteArray)));
         connect(m_camera, SIGNAL(previewStarted()), this, SLOT(onCameraPreviewStarted()));
         connect(m_camera, SIGNAL(previewStopped()), this, SLOT(onCameraPreviewStopped()));
@@ -200,6 +205,8 @@ bool QAndroidCameraSession::open()
         if (m_camera->getPreviewFormat() != JCamera::NV21)
             m_camera->setPreviewFormat(JCamera::NV21);
 
+        m_camera->fetchEachFrame(m_videoProbes.count());
+
         emit opened();
     } else {
         m_status = QCamera::UnavailableStatus;
@@ -364,6 +371,25 @@ int QAndroidCameraSession::currentCameraRotation() const
     return rotation;
 }
 
+void QAndroidCameraSession::addProbe(QAndroidMediaVideoProbeControl *probe)
+{
+    m_videoProbesMutex.lock();
+    if (probe)
+        m_videoProbes << probe;
+    if (m_camera)
+        m_camera->fetchEachFrame(m_videoProbes.count());
+    m_videoProbesMutex.unlock();
+}
+
+void QAndroidCameraSession::removeProbe(QAndroidMediaVideoProbeControl *probe)
+{
+    m_videoProbesMutex.lock();
+    m_videoProbes.remove(probe);
+    if (m_camera)
+        m_camera->fetchEachFrame(m_videoProbes.count());
+    m_videoProbesMutex.unlock();
+}
+
 void QAndroidCameraSession::applyImageSettings()
 {
     if (!m_camera || !m_imageSettingsDirty)
@@ -513,6 +539,19 @@ void QAndroidCameraSession::onCameraPreviewFetched(const QByteArray &preview)
     }
 }
 
+void QAndroidCameraSession::onCameraFrameFetched(const QByteArray &frame)
+{
+    m_videoProbesMutex.lock();
+    if (frame.size() && m_videoProbes.count()) {
+        QVideoFrame videoFrame(new DataVideoBuffer(frame),
+                               m_camera->previewSize(),
+                               QVideoFrame::Format_NV21);
+        foreach (QAndroidMediaVideoProbeControl *probe, m_videoProbes)
+            probe->newFrameProbed(videoFrame);
+    }
+    m_videoProbesMutex.unlock();
+}
+
 void QAndroidCameraSession::onCameraPictureCaptured(const QByteArray &data)
 {
     if (!m_captureCanceled) {
@@ -592,11 +631,16 @@ void QAndroidCameraSession::processCapturedImage(int id,
 }
 
 void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data, int rotation)
+{
+    emit imageCaptured(id, prepareImageFromPreviewData(data, rotation));
+}
+
+QImage QAndroidCameraSession::prepareImageFromPreviewData(const QByteArray &data, int rotation)
 {
     QSize frameSize = m_camera->previewSize();
-    QImage preview(frameSize, QImage::Format_ARGB32);
+    QImage result(frameSize, QImage::Format_ARGB32);
     qt_convert_NV21_to_ARGB32((const uchar *)data.constData(),
-                              (quint32 *)preview.bits(),
+                              (quint32 *)result.bits(),
                               frameSize.width(),
                               frameSize.height());
 
@@ -610,9 +654,9 @@ void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data,
 
     transform.rotate(rotation);
 
-    preview = preview.transformed(transform);
+    result = result.transformed(transform);
 
-    emit imageCaptured(id, preview);
+    return result;
 }
 
 void QAndroidCameraSession::onVideoOutputReady(bool ready)

src/plugins/android/src/mediacapture/qandroidcamerasession.h

@@ -45,12 +45,15 @@
 #include <qcamera.h>
 #include <qmediaencodersettings.h>
 #include <QCameraImageCapture>
+#include <QSet>
+#include <QMutex>
 #include "qandroidmediastoragelocation.h"
 
 QT_BEGIN_NAMESPACE
 
 class JCamera;
 class QAndroidVideoOutput;
+class QAndroidMediaVideoProbeControl;
 
 class QAndroidCameraSession : public QObject
 {
@@ -90,6 +93,9 @@ public:
 
     int currentCameraRotation() const;
 
+    void addProbe(QAndroidMediaVideoProbeControl *probe);
+    void removeProbe(QAndroidMediaVideoProbeControl *probe);
+
 Q_SIGNALS:
     void statusChanged(QCamera::Status status);
     void stateChanged(QCamera::State);
@@ -114,6 +120,7 @@ private Q_SLOTS:
 
     void onCameraPictureExposed();
     void onCameraPreviewFetched(const QByteArray &preview);
+    void onCameraFrameFetched(const QByteArray &frame);
     void onCameraPictureCaptured(const QByteArray &data);
     void onCameraPreviewStarted();
     void onCameraPreviewStopped();
@@ -127,6 +134,7 @@ private:
 
     void applyImageSettings();
     void processPreviewImage(int id, const QByteArray &data, int rotation);
+    QImage prepareImageFromPreviewData(const QByteArray &data, int rotation);
     void processCapturedImage(int id,
                               const QByteArray &data,
                               const QSize &resolution,
@@ -156,6 +164,9 @@ private:
     QString m_currentImageCaptureFileName;
 
     QAndroidMediaStorageLocation m_mediaStorageLocation;
+
+    QSet<QAndroidMediaVideoProbeControl *> m_videoProbes;
+    QMutex m_videoProbesMutex;
 };
 
 QT_END_NAMESPACE

src/plugins/android/src/mediacapture/qandroidcaptureservice.cpp

@@ -61,6 +61,7 @@
 #include "qandroidaudioencodersettingscontrol.h"
 #include "qandroidvideoencodersettingscontrol.h"
 #include "qandroidmediacontainercontrol.h"
+#include "qandroidmediavideoprobecontrol.h"
 
 #include <qmediaserviceproviderplugin.h>
 
@@ -201,16 +202,37 @@ QMediaControl *QAndroidCaptureService::requestControl(const char *name)
         return m_videoRendererControl;
     }
 
+    if (qstrcmp(name,QMediaVideoProbeControl_iid) == 0) {
+        QAndroidMediaVideoProbeControl *videoProbe = 0;
+        if (m_cameraSession) {
+            videoProbe = new QAndroidMediaVideoProbeControl(this);
+            m_cameraSession->addProbe(videoProbe);
+        }
+        return videoProbe;
+    }
+
     return 0;
 }
 
 void QAndroidCaptureService::releaseControl(QMediaControl *control)
 {
-    if (control && control == m_videoRendererControl) {
+    if (control) {
+        if (control == m_videoRendererControl) {
             m_cameraSession->setVideoPreview(0);
             delete m_videoRendererControl;
             m_videoRendererControl = 0;
+            return;
+        }
+
+        QAndroidMediaVideoProbeControl *videoProbe = qobject_cast<QAndroidMediaVideoProbeControl *>(control);
+        if (videoProbe) {
+            if (m_cameraSession)
+                m_cameraSession->removeProbe(videoProbe);
+            delete videoProbe;
+            return;
         }
     }
 
+}
+
 QT_END_NAMESPACE

src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.cpp

@@ -0,0 +1,59 @@
+/****************************************************************************
+**
+** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
+** Copyright (C) 2013 Integrated Computer Solutions, Inc
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qandroidmediavideoprobecontrol.h"
+#include <qvideoframe.h>
+
+QAndroidMediaVideoProbeControl::QAndroidMediaVideoProbeControl(QObject *parent) :
+    QMediaVideoProbeControl(parent)
+{
+}
+
+QAndroidMediaVideoProbeControl::~QAndroidMediaVideoProbeControl()
+{
+
+}
+
+void QAndroidMediaVideoProbeControl::newFrameProbed(const QVideoFrame &frame)
+{
+    emit videoFrameProbed(frame);
+}

src/plugins/android/src/mediacapture/qandroidmediavideoprobecontrol.h

@@ -0,0 +1,59 @@
+/****************************************************************************
+**
+** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies).
+** Copyright (C) 2013 Integrated Computer Solutions, Inc
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QANDROIDMEDIAVIDEOPROBECONTROL_H
+#define QANDROIDMEDIAVIDEOPROBECONTROL_H
+
+#include <qmediavideoprobecontrol.h>
+
+class QAndroidMediaVideoProbeControl : public QMediaVideoProbeControl
+{
+    Q_OBJECT
+public:
+    explicit QAndroidMediaVideoProbeControl(QObject *parent = 0);
+    virtual ~QAndroidMediaVideoProbeControl();
+
+    void newFrameProbed(const QVideoFrame& frame);
+
+};
+
+#endif // QANDROIDMEDIAVIDEOPROBECONTROL_H

src/plugins/android/src/wrappers/jcamera.cpp

@@ -111,6 +111,21 @@ static void notifyPictureCaptured(JNIEnv *env, jobject, int id, jbyteArray data)
     }
 }
 
+static void notifyFrameFetched(JNIEnv *env, jobject, int id, jbyteArray data)
+{
+    g_objectMapMutex.lock();
+    JCamera *obj = g_objectMap.value(id, 0);
+    g_objectMapMutex.unlock();
+    if (obj) {
+        QByteArray bytes;
+        int arrayLength = env->GetArrayLength(data);
+        bytes.resize(arrayLength);
+        env->GetByteArrayRegion(data, 0, arrayLength, (jbyte*)bytes.data());
+
+        Q_EMIT obj->frameFetched(bytes);
+    }
+}
+
 class JCameraInstantiator : public QObject
 {
     Q_OBJECT
@@ -201,6 +216,7 @@ class JCameraWorker : public QObject, public QJNIObjectPrivate
     Q_INVOKABLE void startPreview();
     Q_INVOKABLE void stopPreview();
 
+    Q_INVOKABLE void fetchEachFrame(bool fetch);
     Q_INVOKABLE void fetchLastPreviewFrame();
 
     Q_INVOKABLE void applyParameters();
@@ -571,6 +587,11 @@ void JCamera::takePicture()
     QMetaObject::invokeMethod(d, "callVoidMethod", Q_ARG(QByteArray, "takePicture"));
 }
 
+void JCamera::fetchEachFrame(bool fetch)
+{
+    QMetaObject::invokeMethod(d, "fetchEachFrame", Q_ARG(bool, fetch));
+}
+
 void JCamera::fetchLastPreviewFrame()
 {
     QMetaObject::invokeMethod(d, "fetchLastPreviewFrame");
@@ -1165,6 +1186,11 @@ void JCameraWorker::stopPreview()
     emit previewStopped();
 }
 
+void JCameraWorker::fetchEachFrame(bool fetch)
+{
+    callMethod<void>("fetchEachFrame", "(Z)V", fetch);
+}
+
 void JCameraWorker::fetchLastPreviewFrame()
 {
     QJNIEnvironmentPrivate env;
@@ -1224,7 +1250,8 @@ void JCameraWorker::callVoidMethod(const QByteArray &methodName)
 static JNINativeMethod methods[] = {
     {"notifyAutoFocusComplete", "(IZ)V", (void *)notifyAutoFocusComplete},
     {"notifyPictureExposed", "(I)V", (void *)notifyPictureExposed},
-    {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured}
+    {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured},
+    {"notifyFrameFetched", "(I[B)V", (void *)notifyFrameFetched}
 };
 
 bool JCamera::initJNI(JNIEnv *env)

src/plugins/android/src/wrappers/jcamera.h

@@ -155,6 +155,7 @@ public:
 
     void takePicture();
 
+    void fetchEachFrame(bool fetch);
     void fetchLastPreviewFrame();
     QJNIObjectPrivate getCameraObject();
 
@@ -173,6 +174,7 @@ Q_SIGNALS:
     void pictureExposed();
     void pictureCaptured(const QByteArray &data);
     void previewFetched(const QByteArray &preview);
+    void frameFetched(const QByteArray &frame);
 
 private:
     JCamera(int cameraId, jobject cam, QThread *workerThread);