slapin/qtmultimedia
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Blackberry: Grab viewfinder frames from native window

Browse Source 
Since the conversion from NV12 to RGB on the CPU does not scale
for larger photo/video resolutions, this patch uses a different
approach. It uses the low-level screen API to grab screenshots
of the native viewfinder window and provides them as QImage to
the QAbstractVideoSurface. Even for large resolutions this
is quite performant.

Change-Id: I59a7cbe6850b3b07575ea10026f3180cfd22e935
Reviewed-by: Thomas McGuire <thomas.mcguire@kdab.com>
This commit is contained in:
Tobias Koenig
2013-02-13 11:57:02 +01:00
committed by The Qt Project
parent 13ecd7171b
commit 31b454b8d6
6 changed files with 453 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/plugins/blackberry/blackberry.pro
View File

@@ -10,6 +10,7 @@ LIBS += -lscreen
HEADERS += bbserviceplugin.h
SOURCES += bbserviceplugin.cpp
include(common/common.pri)
include(camera/camera.pri)
include(mediaplayer/mediaplayer.pri)

182
src/plugins/blackberry/camera/bbcamerasession.cpp
View File

@@ -42,6 +42,7 @@
#include "bbcameraorientationhandler.h"
#include "bbcameraviewfindersettingscontrol.h"
#include "windowgrabber.h"
#include <QAbstractVideoSurface>
#include <QBuffer>
@@ -123,10 +124,13 @@ BbCameraSession::BbCameraSession(QObject *parent)
, m_videoState(QMediaRecorder::StoppedState)
, m_videoStatus(QMediaRecorder::LoadedStatus)
, m_handle(CAMERA_HANDLE_INVALID)
, m_windowGrabber(new WindowGrabber(this))
{
connect(this, SIGNAL(statusChanged(QCamera::Status)), SLOT(updateReadyForCapture()));
connect(this, SIGNAL(captureModeChanged(QCamera::CaptureModes)), SLOT(updateReadyForCapture()));
connect(m_orientationHandler, SIGNAL(orientationChanged(int)), SLOT(deviceOrientationChanged(int)));
connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage)), SLOT(viewfinderFrameGrabbed(QImage)));
}
BbCameraSession::~BbCameraSession()
@@ -626,8 +630,15 @@ void BbCameraSession::applyVideoSettings()
Q_ASSERT(viewfinderResolution.isValid());
const QByteArray windowId = QString().sprintf("qcamera_vf_%p", this).toLatin1();
m_windowGrabber->setWindowId(windowId);
const QByteArray windowGroupId = m_windowGrabber->windowGroupId();
camera_error_t result = CAMERA_EOK;
result = camera_set_videovf_property(m_handle,
CAMERA_IMGPROP_WIN_GROUPID, windowGroupId.data(),
CAMERA_IMGPROP_WIN_ID, windowId.data(),
CAMERA_IMGPROP_WIDTH, viewfinderResolution.width(),
CAMERA_IMGPROP_HEIGHT, viewfinderResolution.height(),
CAMERA_IMGPROP_ROTATION, 360 - m_nativeCameraOrientation);
@@ -743,125 +754,6 @@ void BbCameraSession::setAudioSettings(const QAudioEncoderSettings &settings)
m_audioEncoderSettings = settings;
}
static QImage convertFrameToImage(camera_buffer_t *buffer)
{
if (buffer->frametype != CAMERA_FRAMETYPE_NV12)
return QImage();
const unsigned int width = buffer->framedesc.nv12.width;
const unsigned int height = buffer->framedesc.nv12.height;
/**
* Copying the data from the buffer into our own data array and working
* on this copy is actually faster than working on the buffer.
* Looks like we hit some cache misses here, since the stride inside the
* NV12 frame is really large (4096) in comparison to the actual image width (768)
*/
const unsigned long long size = width*height + width*height/2;
unsigned char *data = new unsigned char[size];
unsigned char *source = buffer->framebuf;
unsigned char *dest = data;
for (uint row = 0; row < height; ++row) {
memcpy(dest, source, width);
source += buffer->framedesc.nv12.stride;
dest += width;
}
source = buffer->framebuf + buffer->framedesc.nv12.uv_offset;
for (uint row = 0; row < height/2; ++row) {
memcpy(dest, source, width);
source += buffer->framedesc.nv12.uv_stride;
dest += width;
}
QImage image(width, height, QImage::Format_RGB32);
unsigned char *dataPtr = data;
unsigned char *uvDataPtr = 0;
unsigned int uv_base_offset = width*height;
int yValue = 0;
int uValue = 0;
int vValue = 0;
int bValue = 0;
int gValue = 0;
int rValue = 0;
unsigned char *rowStart = 0;
unsigned char *imageDest = 0;
for (unsigned int y = 0; y < height; ++y) {
imageDest = const_cast<unsigned char*>(image.constScanLine(y));
rowStart = data + (uv_base_offset + (width*qFloor(y/2.0)));
for (unsigned int x = 0; x < width; ++x) {
uvDataPtr = rowStart + (qFloor(x/2)*2);
yValue = ((*dataPtr++) - 16) * 1.164;
uValue = ((*uvDataPtr++) - 128);
vValue = ((*uvDataPtr) - 128);
bValue = yValue + 2.018 * uValue;
gValue = yValue - 0.813 * vValue - 0.391 * uValue;
rValue = yValue + 1.596 * vValue;
*imageDest = qBound(0, bValue, 255);
imageDest++;
*imageDest = qBound(0, gValue, 255);
imageDest++;
*imageDest = qBound(0, rValue, 255);
imageDest++;
*imageDest = 255;
imageDest++;
}
}
delete [] data;
return image;
}
void BbCameraSession::handlePhotoViewFinderData(camera_buffer_t *buffer)
{
QTransform transform;
transform.rotate(m_nativeCameraOrientation);
const QImage frame = convertFrameToImage(buffer).transformed(transform);
QMutexLocker locker(&m_surfaceMutex);
if (m_surface) {
if (frame.size() != m_surface->surfaceFormat().frameSize()) {
m_surface->stop();
m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_RGB32));
}
QVideoFrame videoFrame(frame);
m_surface->present(videoFrame);
}
}
void BbCameraSession::handleVideoViewFinderData(camera_buffer_t *buffer)
{
QTransform transform;
transform.rotate(m_nativeCameraOrientation);
const QImage frame = convertFrameToImage(buffer).transformed(transform);
QMutexLocker locker(&m_surfaceMutex);
if (m_surface) {
if (frame.size() != m_surface->surfaceFormat().frameSize()) {
m_surface->stop();
m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_RGB32));
}
QVideoFrame videoFrame(frame);
m_surface->present(videoFrame);
}
}
void BbCameraSession::updateReadyForCapture()
{
emit readyForCaptureChanged(isReadyForCapture());
@@ -981,6 +873,27 @@ void BbCameraSession::handleCameraPowerUp()
startViewFinder();
}
void BbCameraSession::viewfinderFrameGrabbed(const QImage &image)
{
QTransform transform;
transform.rotate(m_nativeCameraOrientation);
const QImage frame = image.copy().transformed(transform);
QMutexLocker locker(&m_surfaceMutex);
if (m_surface) {
if (frame.size() != m_surface->surfaceFormat().frameSize()) {
m_surface->stop();
m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32));
}
QVideoFrame videoFrame(frame);
m_surface->present(videoFrame);
}
}
bool BbCameraSession::openCamera()
{
if (m_handle != CAMERA_HANDLE_INVALID) // camera is already open
@@ -1047,22 +960,6 @@ void BbCameraSession::closeCamera()
emit statusChanged(m_status);
}
static void photoViewFinderDataCallback(camera_handle_t handle, camera_buffer_t *buffer, void *context)
{
Q_UNUSED(handle)
BbCameraSession *session = static_cast<BbCameraSession*>(context);
session->handlePhotoViewFinderData(buffer);
}
static void videoViewFinderDataCallback(camera_handle_t handle, camera_buffer_t *buffer, void *context)
{
Q_UNUSED(handle)
BbCameraSession *session = static_cast<BbCameraSession*>(context);
session->handleVideoViewFinderData(buffer);
}
static void viewFinderStatusCallback(camera_handle_t handle, camera_devstatus_t status, uint16_t value, void *context)
{
Q_UNUSED(handle)
@@ -1084,10 +981,10 @@ bool BbCameraSession::startViewFinder()
QSize viewfinderResolution;
camera_error_t result = CAMERA_EOK;
if (m_captureMode & QCamera::CaptureStillImage) {
result = camera_start_photo_viewfinder(m_handle, photoViewFinderDataCallback, viewFinderStatusCallback, this);
result = camera_start_photo_viewfinder(m_handle, 0, viewFinderStatusCallback, this);
viewfinderResolution = currentViewfinderResolution(QCamera::CaptureStillImage);
} else if (m_captureMode & QCamera::CaptureVideo) {
result = camera_start_video_viewfinder(m_handle, videoViewFinderDataCallback, viewFinderStatusCallback, this);
result = camera_start_video_viewfinder(m_handle, 0, viewFinderStatusCallback, this);
viewfinderResolution = currentViewfinderResolution(QCamera::CaptureVideo);
}
@@ -1125,7 +1022,7 @@ bool BbCameraSession::startViewFinder()
m_surfaceMutex.lock();
if (m_surface) {
const bool ok = m_surface->start(QVideoSurfaceFormat(rotatedSize, QVideoFrame::Format_RGB32));
const bool ok = m_surface->start(QVideoSurfaceFormat(rotatedSize, QVideoFrame::Format_ARGB32));
if (!ok)
qWarning() << "Unable to start camera viewfinder surface";
}
@@ -1139,6 +1036,8 @@ bool BbCameraSession::startViewFinder()
void BbCameraSession::stopViewFinder()
{
m_windowGrabber->stop();
m_status = QCamera::StoppingStatus;
emit statusChanged(m_status);
@@ -1188,7 +1087,14 @@ void BbCameraSession::applyConfiguration()
Q_ASSERT(viewfinderResolution.isValid());
const QByteArray windowId = QString().sprintf("qcamera_vf_%p", this).toLatin1();
m_windowGrabber->setWindowId(windowId);
const QByteArray windowGroupId = m_windowGrabber->windowGroupId();
camera_error_t result = camera_set_photovf_property(m_handle,
CAMERA_IMGPROP_WIN_GROUPID, windowGroupId.data(),
CAMERA_IMGPROP_WIN_ID, windowId.data(),
CAMERA_IMGPROP_WIDTH, viewfinderResolution.width(),
CAMERA_IMGPROP_HEIGHT, viewfinderResolution.height(),
CAMERA_IMGPROP_FORMAT, CAMERA_FRAMETYPE_NV12,

8
src/plugins/blackberry/camera/bbcamerasession.h
View File

@@ -57,6 +57,7 @@
QT_BEGIN_NAMESPACE
class BbCameraOrientationHandler;
class WindowGrabber;
class BbCameraSession : public QObject
{
@@ -137,10 +138,6 @@ public:
QAudioEncoderSettings audioSettings() const;
void setAudioSettings(const QAudioEncoderSettings &settings);
// methods invoked from BB10 camera API callbacks in separated thread
void handlePhotoViewFinderData(camera_buffer_t*);
void handleVideoViewFinderData(camera_buffer_t*);
Q_SIGNALS:
// camera control
void statusChanged(QCamera::Status);
@@ -178,6 +175,7 @@ private slots:
void handleVideoRecordingResumed();
void deviceOrientationChanged(int);
void handleCameraPowerUp();
void viewfinderFrameGrabbed(const QImage &image);
private:
bool openCamera();
@@ -227,6 +225,8 @@ private:
BbMediaStorageLocation m_mediaStorageLocation;
camera_handle_t m_handle;
WindowGrabber* m_windowGrabber;
};
QDebug operator<<(QDebug debug, camera_error_t error);

7
src/plugins/blackberry/common/common.pri Normal file
View File

@@ -0,0 +1,7 @@
INCLUDEPATH += $$PWD
HEADERS += \
$$PWD/windowgrabber.h
SOURCES += \
$$PWD/windowgrabber.cpp

291
src/plugins/blackberry/common/windowgrabber.cpp Normal file
View File

@@ -0,0 +1,291 @@
/****************************************************************************
**
** Copyright (C) 2013 Research In Motion
** Contact: http://www.qt-project.org/legal
**
** This file is part of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia. For licensing terms and
** conditions see http://qt.digia.com/licensing. For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights. These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include "windowgrabber.h"
#include <QAbstractEventDispatcher>
#include <QDebug>
#include <QGuiApplication>
#include <QImage>
#include <qpa/qplatformnativeinterface.h>
#include <bps/screen.h>
#include <errno.h>
QT_BEGIN_NAMESPACE
WindowGrabber::WindowGrabber(QObject *parent)
: QObject(parent),
m_screenBuffer(0),
m_active(false),
m_screenContextInitialized(false),
m_screenPixmapInitialized(false),
m_screenPixmapBufferInitialized(false)
{
// grab the window frame with 60 frames per second
m_timer.setInterval(1000/60);
connect(&m_timer, SIGNAL(timeout()), SLOT(grab()));
QCoreApplication::eventDispatcher()->installNativeEventFilter(this);
}
WindowGrabber::~WindowGrabber()
{
QCoreApplication::eventDispatcher()->removeNativeEventFilter(this);
}
void WindowGrabber::setFrameRate(int frameRate)
{
m_timer.setInterval(1000/frameRate);
}
void WindowGrabber::setWindowId(const QByteArray &windowId)
{
m_windowId = windowId;
}
void WindowGrabber::start()
{
int result = 0;
result = screen_create_context(&m_screenContext, SCREEN_APPLICATION_CONTEXT);
if (result != 0) {
qWarning() << "WindowGrabber: cannot create screen context:" << strerror(errno);
return;
} else {
m_screenContextInitialized = true;
}
result = screen_create_pixmap(&m_screenPixmap, m_screenContext);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot create pixmap:" << strerror(errno);
return;
} else {
m_screenPixmapInitialized = true;
}
const int usage = SCREEN_USAGE_READ | SCREEN_USAGE_NATIVE;
result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_USAGE, &usage);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot set pixmap usage:" << strerror(errno);
return;
}
const int format = SCREEN_FORMAT_RGBA8888;
result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_FORMAT, &format);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot set pixmap format:" << strerror(errno);
return;
}
int size[2] = { 0, 0 };
result = screen_get_window_property_iv(m_window, SCREEN_PROPERTY_SIZE, size);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot get window size:" << strerror(errno);
return;
}
m_screenBufferWidth = size[0];
m_screenBufferHeight = size[1];
result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_BUFFER_SIZE, size);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot set pixmap size:" << strerror(errno);
return;
}
result = screen_create_pixmap_buffer(m_screenPixmap);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot create pixmap buffer:" << strerror(errno);
return;
}
result = screen_get_pixmap_property_pv(m_screenPixmap, SCREEN_PROPERTY_RENDER_BUFFERS, (void**)&m_screenPixmapBuffer);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot get pixmap buffer:" << strerror(errno);
return;
} else {
m_screenPixmapBufferInitialized = true;
}
result = screen_get_buffer_property_pv(m_screenPixmapBuffer, SCREEN_PROPERTY_POINTER, (void**)&m_screenBuffer);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot get pixmap buffer pointer:" << strerror(errno);
return;
}
result = screen_get_buffer_property_iv(m_screenPixmapBuffer, SCREEN_PROPERTY_STRIDE, &m_screenBufferStride);
if (result != 0) {
cleanup();
qWarning() << "WindowGrabber: cannot get pixmap buffer stride:" << strerror(errno);
return;
}
m_timer.start();
m_active = true;
}
void WindowGrabber::stop()
{
if (!m_active)
return;
cleanup();
m_timer.stop();
m_active = false;
}
void WindowGrabber::pause()
{
m_timer.stop();
}
void WindowGrabber::resume()
{
if (!m_active)
return;
m_timer.start();
}
bool WindowGrabber::nativeEventFilter(const QByteArray&, void *message, long*)
{
bps_event_t * const event = static_cast<bps_event_t *>(message);
if (event && bps_event_get_domain(event) == screen_get_domain()) {
const screen_event_t screen_event = screen_event_get_event(event);
int eventType;
if (screen_get_event_property_iv(screen_event, SCREEN_PROPERTY_TYPE, &eventType) != 0) {
qWarning() << "WindowGrabber: Failed to query screen event type";
return false;
}
if (eventType != SCREEN_EVENT_CREATE)
return false;
screen_window_t window = 0;
if (screen_get_event_property_pv(screen_event, SCREEN_PROPERTY_WINDOW, (void**)&window) != 0) {
qWarning() << "WindowGrabber: Failed to query window property";
return false;
}
const int maxIdStrLength = 128;
char idString[maxIdStrLength];
if (screen_get_window_property_cv(window, SCREEN_PROPERTY_ID_STRING, maxIdStrLength, idString) != 0) {
qWarning() << "WindowGrabber: Failed to query window ID string";
return false;
}
if (m_windowId == idString) {
m_window = window;
start();
}
}
return false;
}
QByteArray WindowGrabber::windowGroupId() const
{
QWindow *window = QGuiApplication::allWindows().isEmpty() ? 0 : QGuiApplication::allWindows().first();
if (!window)
return QByteArray();
QPlatformNativeInterface * const nativeInterface = QGuiApplication::platformNativeInterface();
if (!nativeInterface) {
qWarning() << "WindowGrabber: Unable to get platform native interface";
return QByteArray();
}
const char * const groupIdData = static_cast<const char *>(
nativeInterface->nativeResourceForWindow("windowGroup", window));
if (!groupIdData) {
qWarning() << "WindowGrabber: Unable to find window group for window" << window;
return QByteArray();
}
return QByteArray(groupIdData);
}
void WindowGrabber::grab()
{
const int result = screen_read_window(m_window, m_screenPixmapBuffer, 0, 0, 0);
if (result != 0)
return;
const QImage frame((unsigned char*)m_screenBuffer, m_screenBufferWidth, m_screenBufferHeight,
m_screenBufferStride, QImage::Format_ARGB32);
emit frameGrabbed(frame);
}
void WindowGrabber::cleanup()
{
if (m_screenPixmapBufferInitialized) {
screen_destroy_buffer(m_screenPixmapBuffer);
m_screenPixmapBufferInitialized = false;
}
if (m_screenPixmapInitialized) {
screen_destroy_pixmap(m_screenPixmap);
m_screenPixmapInitialized = false;
}
if (m_screenContextInitialized) {
screen_destroy_context(m_screenContext);
m_screenContextInitialized = false;
}
}
QT_END_NAMESPACE

106
src/plugins/blackberry/common/windowgrabber.h Normal file
View File

@@ -0,0 +1,106 @@
/****************************************************************************
**
** Copyright (C) 2013 Research In Motion
** Contact: http://www.qt-project.org/legal
**
** This file is part of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and Digia. For licensing terms and
** conditions see http://qt.digia.com/licensing. For further information
** use the contact form at http://qt.digia.com/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 2.1 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 2.1 requirements
** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
**
** In addition, as a special exception, Digia gives you certain additional
** rights. These rights are described in the Digia Qt LGPL Exception
** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3.0 as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL included in the
** packaging of this file. Please review the following information to
** ensure the GNU General Public License version 3.0 requirements will be
** met: http://www.gnu.org/copyleft/gpl.html.
**
**
** $QT_END_LICENSE$
**
****************************************************************************/
#ifndef WINDOWGRABBER_H
#define WINDOWGRABBER_H
#include <QAbstractNativeEventFilter>
#include <QObject>
#include <QTimer>
#include <screen/screen.h>
QT_BEGIN_NAMESPACE
class WindowGrabber : public QObject, public QAbstractNativeEventFilter
{
Q_OBJECT
public:
explicit WindowGrabber(QObject *parent = 0);
~WindowGrabber();
void setFrameRate(int frameRate);
void setWindowId(const QByteArray &windowId);
void start();
void stop();
void pause();
void resume();
bool nativeEventFilter(const QByteArray &eventType, void *message, long *result) Q_DECL_OVERRIDE;
QByteArray windowGroupId() const;
signals:
void frameGrabbed(const QImage &frame);
private slots:
void grab();
private:
void cleanup();
QTimer m_timer;
QByteArray m_windowId;
screen_window_t m_window;
screen_context_t m_screenContext;
screen_pixmap_t m_screenPixmap;
screen_buffer_t m_screenPixmapBuffer;
char* m_screenBuffer;
int m_screenBufferWidth;
int m_screenBufferHeight;
int m_screenBufferStride;
bool m_active : 1;
bool m_screenContextInitialized : 1;
bool m_screenPixmapInitialized : 1;
bool m_screenPixmapBufferInitialized : 1;
};
QT_END_NAMESPACE
#endif