Android: minor refactor of the camera frame callback.

Change-Id: I6b281c9b2d02cf223e66e04e31fdd0268aa277fc
Reviewed-by: Christian Stromme <christian.stromme@theqtcompany.com>

src/plugins/android/jar/src/org/qtproject/qt5/android/multimedia/QtCameraListener.java

@@ -38,63 +38,87 @@ import android.graphics.ImageFormat;
 import android.graphics.SurfaceTexture;
 import android.util.Log;
 import java.lang.Math;
-import java.util.concurrent.locks.ReentrantLock;
 
 public class QtCameraListener implements Camera.ShutterCallback,
                                          Camera.PictureCallback,
                                          Camera.AutoFocusCallback,
                                          Camera.PreviewCallback
 {
-    private int m_cameraId = -1;
-    private byte[][] m_cameraPreviewBuffer = null;
-    private volatile int m_actualPreviewBuffer = 0;
-    private final ReentrantLock m_buffersLock = new ReentrantLock();
-    private boolean m_fetchEachFrame = false;
-
     private static final String TAG = "Qt Camera";
 
+    private static final int BUFFER_POOL_SIZE = 2;
+
+    private int m_cameraId = -1;
+
+    private boolean m_notifyNewFrames = false;
+    private byte[][] m_previewBuffers = null;
+    private byte[] m_lastPreviewBuffer = null;
+    private Camera.Size m_previewSize = null;
+
     private QtCameraListener(int id)
     {
         m_cameraId = id;
     }
 
-    public void preparePreviewBuffer(Camera camera)
+    public void notifyNewFrames(boolean notify)
     {
-        Camera.Size previewSize = camera.getParameters().getPreviewSize();
+        m_notifyNewFrames = notify;
-        double bytesPerPixel = ImageFormat.getBitsPerPixel(camera.getParameters().getPreviewFormat()) / 8.0;
-        int bufferSizeNeeded = (int)Math.ceil(bytesPerPixel*previewSize.width*previewSize.height);
-        m_buffersLock.lock();
-        if (m_cameraPreviewBuffer == null || m_cameraPreviewBuffer[0].length < bufferSizeNeeded)
-            m_cameraPreviewBuffer = new byte[2][bufferSizeNeeded];
-        m_buffersLock.unlock();
     }
 
-    public void fetchEachFrame(boolean fetch)
+    public byte[] lastPreviewBuffer()
     {
-        m_fetchEachFrame = fetch;
+        return m_lastPreviewBuffer;
     }
 
-    public byte[] lockAndFetchPreviewBuffer()
+    public int previewWidth()
     {
-        //This method should always be followed by unlockPreviewBuffer()
+        if (m_previewSize == null)
-        //This method is not just a getter. It also marks last preview as already seen one.
+            return -1;
-        //We should reset actualBuffer flag here to make sure we will not use old preview with future captures
+
-        byte[] result = null;
+        return m_previewSize.width;
-        m_buffersLock.lock();
-        result = m_cameraPreviewBuffer[(m_actualPreviewBuffer == 1) ? 0 : 1];
-        m_actualPreviewBuffer = 0;
-        return result;
     }
 
-    public void unlockPreviewBuffer()
+    public int previewHeight()
     {
-        if (m_buffersLock.isHeldByCurrentThread())
+        if (m_previewSize == null)
-            m_buffersLock.unlock();
+            return -1;
+
+        return m_previewSize.height;
     }
 
-    public byte[] callbackBuffer()
+    public void setupPreviewCallback(Camera camera)
     {
-        return m_cameraPreviewBuffer[(m_actualPreviewBuffer == 1) ? 1 : 0];
+        // Clear previous callback (also clears added buffers)
+        m_lastPreviewBuffer = null;
+        camera.setPreviewCallbackWithBuffer(null);
+
+        final Camera.Parameters params = camera.getParameters();
+        m_previewSize = params.getPreviewSize();
+        double bytesPerPixel = ImageFormat.getBitsPerPixel(params.getPreviewFormat()) / 8.0;
+        int bufferSizeNeeded = (int) Math.ceil(bytesPerPixel * m_previewSize.width * m_previewSize.height);
+
+        // We could keep the same buffers when they are already bigger than the required size
+        // but the Android doc says the size must match, so in doubt just replace them.
+        if (m_previewBuffers == null || m_previewBuffers[0].length != bufferSizeNeeded)
+            m_previewBuffers = new byte[BUFFER_POOL_SIZE][bufferSizeNeeded];
+
+        // Add callback and queue all buffers
+        camera.setPreviewCallbackWithBuffer(this);
+        for (byte[] buffer : m_previewBuffers)
+            camera.addCallbackBuffer(buffer);
+    }
+
+    @Override
+    public void onPreviewFrame(byte[] data, Camera camera)
+    {
+        // Re-enqueue the last buffer
+        if (m_lastPreviewBuffer != null)
+            camera.addCallbackBuffer(m_lastPreviewBuffer);
+
+        m_lastPreviewBuffer = data;
+
+        if (data != null && m_notifyNewFrames)
+            notifyNewPreviewFrame(m_cameraId, data, m_previewSize.width, m_previewSize.height);
     }
 
     @Override
@@ -109,24 +133,6 @@ public class QtCameraListener implements Camera.ShutterCallback,
         notifyPictureCaptured(m_cameraId, data);
     }
 
-    @Override
-    public void onPreviewFrame(byte[] data, Camera camera)
-    {
-        m_buffersLock.lock();
-
-        if (data != null && m_fetchEachFrame)
-            notifyFrameFetched(m_cameraId, data);
-
-        if (data == m_cameraPreviewBuffer[0])
-            m_actualPreviewBuffer = 1;
-        else if (data == m_cameraPreviewBuffer[1])
-            m_actualPreviewBuffer = 2;
-        else
-            m_actualPreviewBuffer = 0;
-        camera.addCallbackBuffer(m_cameraPreviewBuffer[(m_actualPreviewBuffer == 1) ? 1 : 0]);
-        m_buffersLock.unlock();
-    }
-
     @Override
     public void onAutoFocus(boolean success, Camera camera)
     {
@@ -136,5 +142,5 @@ public class QtCameraListener implements Camera.ShutterCallback,
     private static native void notifyAutoFocusComplete(int id, boolean success);
     private static native void notifyPictureExposed(int id);
     private static native void notifyPictureCaptured(int id, byte[] data);
-    private static native void notifyFrameFetched(int id, byte[] data);
+    private static native void notifyNewPreviewFrame(int id, byte[] data, int width, int height);
 }

src/plugins/android/src/mediacapture/qandroidcamerasession.cpp

@@ -206,9 +206,10 @@ bool QAndroidCameraSession::open()
 
     if (m_camera) {
         connect(m_camera, SIGNAL(pictureExposed()), this, SLOT(onCameraPictureExposed()));
-        connect(m_camera, SIGNAL(previewFetched(QByteArray)), this, SLOT(onCameraPreviewFetched(QByteArray)));
+        connect(m_camera, SIGNAL(lastPreviewFrameFetched(QByteArray,int,int)),
-        connect(m_camera, SIGNAL(frameFetched(QByteArray)),
+                this, SLOT(onLastPreviewFrameFetched(QByteArray,int,int)));
-                this, SLOT(onCameraFrameFetched(QByteArray)),
+        connect(m_camera, SIGNAL(newPreviewFrame(QByteArray,int,int)),
+                this, SLOT(onNewPreviewFrame(QByteArray,int,int)),
                 Qt::DirectConnection);
         connect(m_camera, SIGNAL(pictureCaptured(QByteArray)), this, SLOT(onCameraPictureCaptured(QByteArray)));
         connect(m_camera, SIGNAL(previewStarted()), this, SLOT(onCameraPreviewStarted()));
@@ -221,7 +222,7 @@ bool QAndroidCameraSession::open()
         if (m_camera->getPreviewFormat() != AndroidCamera::NV21)
             m_camera->setPreviewFormat(AndroidCamera::NV21);
 
-        m_camera->fetchEachFrame(m_videoProbes.count());
+        m_camera->notifyNewFrames(m_videoProbes.count());
 
         emit opened();
     } else {
@@ -410,7 +411,7 @@ void QAndroidCameraSession::addProbe(QAndroidMediaVideoProbeControl *probe)
     if (probe)
         m_videoProbes << probe;
     if (m_camera)
-        m_camera->fetchEachFrame(m_videoProbes.count());
+        m_camera->notifyNewFrames(m_videoProbes.count());
     m_videoProbesMutex.unlock();
 }
 
@@ -419,7 +420,7 @@ void QAndroidCameraSession::removeProbe(QAndroidMediaVideoProbeControl *probe)
     m_videoProbesMutex.lock();
     m_videoProbes.remove(probe);
     if (m_camera)
-        m_camera->fetchEachFrame(m_videoProbes.count());
+        m_camera->notifyNewFrames(m_videoProbes.count());
     m_videoProbesMutex.unlock();
 }
 
@@ -562,25 +563,54 @@ void QAndroidCameraSession::onCameraPictureExposed()
     m_camera->fetchLastPreviewFrame();
 }
 
-void QAndroidCameraSession::onCameraPreviewFetched(const QByteArray &preview)
+void QAndroidCameraSession::onLastPreviewFrameFetched(const QByteArray &preview, int width, int height)
 {
     if (preview.size()) {
         QtConcurrent::run(this, &QAndroidCameraSession::processPreviewImage,
                           m_currentImageCaptureId,
                           preview,
+                          width,
+                          height,
                           m_camera->getRotation());
     }
 }
 
-void QAndroidCameraSession::onCameraFrameFetched(const QByteArray &frame)
+void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data, int width, int height, int rotation)
+{
+    emit imageCaptured(id, prepareImageFromPreviewData(data, width, height, rotation));
+}
+
+QImage QAndroidCameraSession::prepareImageFromPreviewData(const QByteArray &data, int width, int height, int rotation)
+{
+    QImage result(width, height, QImage::Format_ARGB32);
+    qt_convert_NV21_to_ARGB32((const uchar *)data.constData(),
+                              (quint32 *)result.bits(),
+                              width,
+                              height);
+
+    QTransform transform;
+
+    // Preview display of front-facing cameras is flipped horizontally, but the frame data
+    // we get here is not. Flip it ourselves if the camera is front-facing to match what the user
+    // sees on the viewfinder.
+    if (m_camera->getFacing() == AndroidCamera::CameraFacingFront)
+        transform.scale(-1, 1);
+
+    transform.rotate(rotation);
+
+    result = result.transformed(transform);
+
+    return result;
+}
+
+void QAndroidCameraSession::onNewPreviewFrame(const QByteArray &frame, int width, int height)
 {
     m_videoProbesMutex.lock();
     if (frame.size() && m_videoProbes.count()) {
-        const QSize frameSize = m_camera->previewSize();
         // Bytes per line should be only for the first plane. For NV21, the Y plane has 8 bits
         // per sample, so bpl == width
-        QVideoFrame videoFrame(new DataVideoBuffer(frame, frameSize.width()),
+        QVideoFrame videoFrame(new DataVideoBuffer(frame, width),
-                               frameSize,
+                               QSize(width, height),
                                QVideoFrame::Format_NV21);
         foreach (QAndroidMediaVideoProbeControl *probe, m_videoProbes)
             probe->newFrameProbed(videoFrame);
@@ -666,35 +696,6 @@ void QAndroidCameraSession::processCapturedImage(int id,
     }
 }
 
-void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data, int rotation)
-{
-    emit imageCaptured(id, prepareImageFromPreviewData(data, rotation));
-}
-
-QImage QAndroidCameraSession::prepareImageFromPreviewData(const QByteArray &data, int rotation)
-{
-    QSize frameSize = m_camera->previewSize();
-    QImage result(frameSize, QImage::Format_ARGB32);
-    qt_convert_NV21_to_ARGB32((const uchar *)data.constData(),
-                              (quint32 *)result.bits(),
-                              frameSize.width(),
-                              frameSize.height());
-
-    QTransform transform;
-
-    // Preview display of front-facing cameras is flipped horizontally, but the frame data
-    // we get here is not. Flip it ourselves if the camera is front-facing to match what the user
-    // sees on the viewfinder.
-    if (m_camera->getFacing() == AndroidCamera::CameraFacingFront)
-        transform.scale(-1, 1);
-
-    transform.rotate(rotation);
-
-    result = result.transformed(transform);
-
-    return result;
-}
-
 void QAndroidCameraSession::onVideoOutputReady(bool ready)
 {
     if (ready && m_state == QCamera::ActiveState)

src/plugins/android/src/mediacapture/qandroidcamerasession.h

@@ -113,9 +113,9 @@ private Q_SLOTS:
     void onApplicationStateChanged(Qt::ApplicationState state);
 
     void onCameraPictureExposed();
-    void onCameraPreviewFetched(const QByteArray &preview);
-    void onCameraFrameFetched(const QByteArray &frame);
     void onCameraPictureCaptured(const QByteArray &data);
+    void onLastPreviewFrameFetched(const QByteArray &preview, int width, int height);
+    void onNewPreviewFrame(const QByteArray &frame, int width, int height);
     void onCameraPreviewStarted();
     void onCameraPreviewStopped();
 
@@ -129,8 +129,8 @@ private:
     void stopPreview();
 
     void applyImageSettings();
-    void processPreviewImage(int id, const QByteArray &data, int rotation);
+    void processPreviewImage(int id, const QByteArray &data, int width, int height, int rotation);
-    QImage prepareImageFromPreviewData(const QByteArray &data, int rotation);
+    QImage prepareImageFromPreviewData(const QByteArray &data, int width, int height, int rotation);
     void processCapturedImage(int id,
                               const QByteArray &data,
                               const QSize &resolution,

src/plugins/android/src/wrappers/jni/androidcamera.cpp

@@ -114,7 +114,7 @@ static void notifyPictureCaptured(JNIEnv *env, jobject, int id, jbyteArray data)
     }
 }
 
-static void notifyFrameFetched(JNIEnv *env, jobject, int id, jbyteArray data)
+static void notifyNewPreviewFrame(JNIEnv *env, jobject, int id, jbyteArray data, int width, int height)
 {
     QMutexLocker locker(&g_cameraMapMutex);
     AndroidCamera *obj = g_cameraMap->value(id, 0);
@@ -123,7 +123,7 @@ static void notifyFrameFetched(JNIEnv *env, jobject, int id, jbyteArray data)
         QByteArray bytes(arrayLength, Qt::Uninitialized);
         env->GetByteArrayRegion(data, 0, arrayLength, (jbyte*)bytes.data());
 
-        Q_EMIT obj->frameFetched(bytes);
+        Q_EMIT obj->newPreviewFrame(bytes, width, height);
     }
 }
 
@@ -205,7 +205,7 @@ public:
     Q_INVOKABLE void takePicture();
 
     Q_INVOKABLE void setupPreviewFrameCallback();
-    Q_INVOKABLE void fetchEachFrame(bool fetch);
+    Q_INVOKABLE void notifyNewFrames(bool notify);
     Q_INVOKABLE void fetchLastPreviewFrame();
 
     Q_INVOKABLE void applyParameters();
@@ -230,7 +230,7 @@ Q_SIGNALS:
 
     void whiteBalanceChanged();
 
-    void previewFetched(const QByteArray &preview);
+    void lastPreviewFrameFetched(const QByteArray &preview, int width, int height);
 };
 
 AndroidCamera::AndroidCamera(AndroidCameraPrivate *d, QThread *worker)
@@ -248,7 +248,7 @@ AndroidCamera::AndroidCamera(AndroidCameraPrivate *d, QThread *worker)
     connect(d, &AndroidCameraPrivate::previewStopped, this, &AndroidCamera::previewStopped);
     connect(d, &AndroidCameraPrivate::autoFocusStarted, this, &AndroidCamera::autoFocusStarted);
     connect(d, &AndroidCameraPrivate::whiteBalanceChanged, this, &AndroidCamera::whiteBalanceChanged);
-    connect(d, &AndroidCameraPrivate::previewFetched, this, &AndroidCamera::previewFetched);
+    connect(d, &AndroidCameraPrivate::lastPreviewFrameFetched, this, &AndroidCamera::lastPreviewFrameFetched);
 }
 
 AndroidCamera::~AndroidCamera()
@@ -640,10 +640,10 @@ void AndroidCamera::setupPreviewFrameCallback()
     QMetaObject::invokeMethod(d, "setupPreviewFrameCallback");
 }
 
-void AndroidCamera::fetchEachFrame(bool fetch)
+void AndroidCamera::notifyNewFrames(bool notify)
 {
     Q_D(AndroidCamera);
-    QMetaObject::invokeMethod(d, "fetchEachFrame", Q_ARG(bool, fetch));
+    QMetaObject::invokeMethod(d, "notifyNewFrames", Q_ARG(bool, notify));
 }
 
 void AndroidCamera::fetchLastPreviewFrame()
@@ -1337,41 +1337,32 @@ void AndroidCameraPrivate::takePicture()
 
 void AndroidCameraPrivate::setupPreviewFrameCallback()
 {
-    //We need to clear preview buffers queue here, but there is no method to do it
+    m_cameraListener.callMethod<void>("setupPreviewCallback", "(Landroid/hardware/Camera;)V", m_camera.object());
-    //Though just resetting preview callback do the trick
-    m_camera.callMethod<void>("setPreviewCallbackWithBuffer",
-                              "(Landroid/hardware/Camera$PreviewCallback;)V",
-                              jobject(0));
-    m_cameraListener.callMethod<void>("preparePreviewBuffer", "(Landroid/hardware/Camera;)V", m_camera.object());
-    QJNIObjectPrivate buffer = m_cameraListener.callObjectMethod<jbyteArray>("callbackBuffer");
-    m_camera.callMethod<void>("addCallbackBuffer", "([B)V", buffer.object());
-    m_camera.callMethod<void>("setPreviewCallbackWithBuffer",
-                              "(Landroid/hardware/Camera$PreviewCallback;)V",
-                              m_cameraListener.object());
 }
 
-void AndroidCameraPrivate::fetchEachFrame(bool fetch)
+void AndroidCameraPrivate::notifyNewFrames(bool notify)
 {
-    m_cameraListener.callMethod<void>("fetchEachFrame", "(Z)V", fetch);
+    m_cameraListener.callMethod<void>("notifyNewFrames", "(Z)V", notify);
 }
 
 void AndroidCameraPrivate::fetchLastPreviewFrame()
 {
     QJNIEnvironmentPrivate env;
-    QJNIObjectPrivate data = m_cameraListener.callObjectMethod("lockAndFetchPreviewBuffer", "()[B");
+    QJNIObjectPrivate data = m_cameraListener.callObjectMethod("lastPreviewBuffer", "()[B");
-    if (!data.isValid()) {
+
-        m_cameraListener.callMethod<void>("unlockPreviewBuffer");
+    if (!data.isValid())
         return;
-    }
+
     const int arrayLength = env->GetArrayLength(static_cast<jbyteArray>(data.object()));
     QByteArray bytes(arrayLength, Qt::Uninitialized);
     env->GetByteArrayRegion(static_cast<jbyteArray>(data.object()),
                             0,
                             arrayLength,
                             reinterpret_cast<jbyte *>(bytes.data()));
-    m_cameraListener.callMethod<void>("unlockPreviewBuffer");
 
-    emit previewFetched(bytes);
+    emit lastPreviewFrameFetched(bytes,
+                                 m_cameraListener.callMethod<jint>("previewWidth"),
+                                 m_cameraListener.callMethod<jint>("previewHeight"));
 }
 
 void AndroidCameraPrivate::applyParameters()
@@ -1416,7 +1407,7 @@ bool AndroidCamera::initJNI(JNIEnv *env)
         {"notifyAutoFocusComplete", "(IZ)V", (void *)notifyAutoFocusComplete},
         {"notifyPictureExposed", "(I)V", (void *)notifyPictureExposed},
         {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured},
-        {"notifyFrameFetched", "(I[B)V", (void *)notifyFrameFetched}
+        {"notifyNewPreviewFrame", "(I[BII)V", (void *)notifyNewPreviewFrame}
     };
 
     if (clazz && env->RegisterNatives(clazz,

src/plugins/android/src/wrappers/jni/androidcamera.h

@@ -156,7 +156,7 @@ public:
     void takePicture();
 
     void setupPreviewFrameCallback();
-    void fetchEachFrame(bool fetch);
+    void notifyNewFrames(bool notify);
     void fetchLastPreviewFrame();
     QJNIObjectPrivate getCameraObject();
 
@@ -177,8 +177,8 @@ Q_SIGNALS:
 
     void pictureExposed();
     void pictureCaptured(const QByteArray &data);
-    void previewFetched(const QByteArray &preview);
+    void lastPreviewFrameFetched(const QByteArray &preview, int width, int height);
-    void frameFetched(const QByteArray &frame);
+    void newPreviewFrame(const QByteArray &frame, int width, int height);
 
 private:
     AndroidCamera(AndroidCameraPrivate *d, QThread *worker);