QNX: Remove most of the CPU overhead for video rendering

This patch uses the GL_OES_EGL_image extension to create a OpenGL Texture handle
for a libscreen pixmap. If the extension is not available it uses the "old"
technique as fallback where the image data is copied into a QImage.

This reduces the CPU load by more than 70% and allows HD videos to be played jitter-free.

Task-number: QTBUG-37752

Change-Id: I4cad22c39390e4cf9eb5be5f0bfe446544a11b9e
Reviewed-by: Bernd Weimer <bweimer@blackberry.com>
Reviewed-by: Sean Harmer <sean.harmer@kdab.com>

src/plugins/qnx/camera/bbcamerasession.cpp

@@ -140,7 +140,7 @@ BbCameraSession::BbCameraSession(QObject *parent)
     connect(this, SIGNAL(captureModeChanged(QCamera::CaptureModes)), SLOT(updateReadyForCapture()));
     connect(m_orientationHandler, SIGNAL(orientationChanged(int)), SLOT(deviceOrientationChanged(int)));
 
-    connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage)), SLOT(viewfinderFrameGrabbed(QImage)));
+    connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage, int)), SLOT(viewfinderFrameGrabbed(QImage)));
 }
 
 BbCameraSession::~BbCameraSession()

src/plugins/qnx/common/windowgrabber.cpp

@@ -47,6 +47,8 @@
 #include <QImage>
 #include <qpa/qplatformnativeinterface.h>
 
+#include <QOpenGLContext>
+
 #ifdef Q_OS_BLACKBERRY
 #include <bps/event.h>
 #include <bps/screen.h>
@@ -57,13 +59,15 @@ QT_BEGIN_NAMESPACE
 
 WindowGrabber::WindowGrabber(QObject *parent)
     : QObject(parent),
-      m_screenBuffer(0),
       m_screenBufferWidth(-1),
       m_screenBufferHeight(-1),
       m_active(false),
       m_screenContextInitialized(false),
-      m_screenPixmapInitialized(false),
-      m_screenPixmapBufferInitialized(false)
+      m_screenPixmapBuffersInitialized(false),
+      m_currentFrame(0),
+      m_eglImageSupported(false),
+      m_eglImagesInitialized(false),
+      m_eglImageCheck(false)
 {
     // grab the window frame with 60 frames per second
     m_timer.setInterval(1000/60);
@@ -76,6 +80,11 @@ WindowGrabber::WindowGrabber(QObject *parent)
 WindowGrabber::~WindowGrabber()
 {
     QCoreApplication::eventDispatcher()->removeNativeEventFilter(this);
+    if (eglImagesInitialized()) {
+        glDeleteTextures(2, imgTextures);
+        eglDestroyImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), img[0]);
+        eglDestroyImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), img[1]);
+    }
 }
 
 void WindowGrabber::setFrameRate(int frameRate)
@@ -83,6 +92,37 @@ void WindowGrabber::setFrameRate(int frameRate)
     m_timer.setInterval(1000/frameRate);
 }
 
+void WindowGrabber::createEglImages()
+{
+    // Do nothing if either egl images are not supported, the screen context is not valid
+    // or the images are already created
+    if (!eglImageSupported() || !m_screenContextInitialized || eglImagesInitialized())
+        return;
+
+    glGenTextures(2, imgTextures);
+    glBindTexture(GL_TEXTURE_2D, imgTextures[0]);
+    img[0] = eglCreateImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), EGL_NO_CONTEXT,
+                                        EGL_NATIVE_PIXMAP_KHR,
+                                        m_screenPixmaps[0],
+                                        0);
+    glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, img[0]);
+
+    glBindTexture(GL_TEXTURE_2D, imgTextures[1]);
+    img[1] = eglCreateImageKHR(eglGetDisplay(EGL_DEFAULT_DISPLAY), EGL_NO_CONTEXT,
+                                        EGL_NATIVE_PIXMAP_KHR,
+                                        m_screenPixmaps[1],
+                                        0);
+
+    glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, img[1]);
+
+    if (img[0] == 0 || img[1] == 0) {
+        qWarning() << "Failed to create KHR images" << img[0] << img[1] << strerror(errno) << errno;
+        m_eglImageSupported = false;
+    } else {
+        m_eglImagesInitialized = true;
+    }
+}
+
 void WindowGrabber::setWindowId(const QByteArray &windowId)
 {
     m_windowId = windowId;
@@ -90,6 +130,9 @@ void WindowGrabber::setWindowId(const QByteArray &windowId)
 
 void WindowGrabber::start()
 {
+    if (m_active)
+        return;
+
     int result = 0;
 
 #ifdef Q_OS_BLACKBERRY_TABLET
@@ -124,30 +167,21 @@ void WindowGrabber::start()
         m_screenContextInitialized = true;
     }
 
-    result = screen_create_pixmap(&m_screenPixmap, m_screenContext);
+    result = screen_create_pixmap(&m_screenPixmaps[0], m_screenContext);
+    result = screen_create_pixmap(&m_screenPixmaps[1], m_screenContext);
     if (result != 0) {
         cleanup();
-        qWarning() << "WindowGrabber: cannot create pixmap:" << strerror(errno);
-        return;
-    } else {
-        m_screenPixmapInitialized = true;
-    }
-
-    const int usage = SCREEN_USAGE_READ | SCREEN_USAGE_NATIVE;
-    result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_USAGE, &usage);
-    if (result != 0) {
-        cleanup();
-        qWarning() << "WindowGrabber: cannot set pixmap usage:" << strerror(errno);
+        qWarning() << "WindowGrabber: cannot create pixmaps:" << strerror(errno);
         return;
     }
 
-    const int format = SCREEN_FORMAT_RGBA8888;
-    result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_FORMAT, &format);
-    if (result != 0) {
-        cleanup();
-        qWarning() << "WindowGrabber: cannot set pixmap format:" << strerror(errno);
-        return;
-    }
+    const int usage = SCREEN_USAGE_NATIVE;
+    result = screen_set_pixmap_property_iv(m_screenPixmaps[0], SCREEN_PROPERTY_USAGE, &usage);
+    result |= screen_set_pixmap_property_iv(m_screenPixmaps[1], SCREEN_PROPERTY_USAGE, &usage);
+
+    const int format = SCREEN_FORMAT_RGBX8888;
+    screen_set_pixmap_property_iv(m_screenPixmaps[0], SCREEN_PROPERTY_FORMAT, &format);
+    screen_set_pixmap_property_iv(m_screenPixmaps[1], SCREEN_PROPERTY_FORMAT, &format);
 
     int size[2] = { 0, 0 };
 
@@ -172,37 +206,51 @@ void WindowGrabber::updateFrameSize()
 {
     int size[2] = { m_screenBufferWidth, m_screenBufferHeight };
 
-    int result = screen_set_pixmap_property_iv(m_screenPixmap, SCREEN_PROPERTY_BUFFER_SIZE, size);
-    if (result != 0) {
-        cleanup();
-        qWarning() << "WindowGrabber: cannot set pixmap size:" << strerror(errno);
-        return;
-    }
+    screen_set_pixmap_property_iv(m_screenPixmaps[0], SCREEN_PROPERTY_BUFFER_SIZE, size);
+    if (eglImageSupported())
+        screen_set_pixmap_property_iv(m_screenPixmaps[1], SCREEN_PROPERTY_BUFFER_SIZE, size);
+
+    int result = screen_create_pixmap_buffer(m_screenPixmaps[0]);
+    if (eglImageSupported())
+        result |= screen_create_pixmap_buffer(m_screenPixmaps[1]);
 
-    result = screen_create_pixmap_buffer(m_screenPixmap);
     if (result != 0) {
         cleanup();
         qWarning() << "WindowGrabber: cannot create pixmap buffer:" << strerror(errno);
         return;
+    } else {
+        m_screenPixmapBuffersInitialized = true;
+    }
+
+    result = screen_get_pixmap_property_pv(m_screenPixmaps[0], SCREEN_PROPERTY_RENDER_BUFFERS,
+            (void**)&m_screenPixmapBuffers[0]);
+    if (eglImageSupported()) {
+        result |= screen_get_pixmap_property_pv(m_screenPixmaps[1], SCREEN_PROPERTY_RENDER_BUFFERS,
+                (void**)&m_screenPixmapBuffers[1]);
     }
 
-    result = screen_get_pixmap_property_pv(m_screenPixmap, SCREEN_PROPERTY_RENDER_BUFFERS, (void**)&m_screenPixmapBuffer);
     if (result != 0) {
         cleanup();
         qWarning() << "WindowGrabber: cannot get pixmap buffer:" << strerror(errno);
         return;
-    } else {
-        m_screenPixmapBufferInitialized = true;
     }
 
-    result = screen_get_buffer_property_pv(m_screenPixmapBuffer, SCREEN_PROPERTY_POINTER, (void**)&m_screenBuffer);
+    result = screen_get_buffer_property_pv(m_screenPixmapBuffers[0], SCREEN_PROPERTY_POINTER,
+            (void**)&m_screenBuffers[0]);
+    if (eglImageSupported()) {
+        result |= screen_get_buffer_property_pv(m_screenPixmapBuffers[1], SCREEN_PROPERTY_POINTER,
+                (void**)&m_screenBuffers[1]);
+    }
+
     if (result != 0) {
         cleanup();
         qWarning() << "WindowGrabber: cannot get pixmap buffer pointer:" << strerror(errno);
         return;
     }
 
-    result = screen_get_buffer_property_iv(m_screenPixmapBuffer, SCREEN_PROPERTY_STRIDE, &m_screenBufferStride);
+    result = screen_get_buffer_property_iv(m_screenPixmapBuffers[0], SCREEN_PROPERTY_STRIDE,
+            &m_screenBufferStride);
+
     if (result != 0) {
         cleanup();
         qWarning() << "WindowGrabber: cannot get pixmap buffer stride:" << strerror(errno);
@@ -310,8 +358,40 @@ QByteArray WindowGrabber::windowGroupId() const
     return QByteArray(groupIdData);
 }
 
+bool WindowGrabber::eglImageSupported()
+{
+    return m_eglImageSupported;
+}
+
+void WindowGrabber::checkForEglImageExtension()
+{
+    QOpenGLContext *m_context = QOpenGLContext::currentContext();
+    if (!m_context) //Should not happen, because we are called from the render thread
+        return;
+
+    QByteArray eglExtensions = QByteArray(eglQueryString(eglGetDisplay(EGL_DEFAULT_DISPLAY),
+                                                         EGL_EXTENSIONS));
+    m_eglImageSupported = m_context->hasExtension(QByteArrayLiteral("GL_OES_EGL_image"))
+                          && eglExtensions.contains(QByteArrayLiteral("EGL_KHR_image"));
+
+    m_eglImageCheck = true;
+}
+
+bool WindowGrabber::eglImagesInitialized()
+{
+    return m_eglImagesInitialized;
+}
+
 void WindowGrabber::grab()
 {
+    if (!m_eglImageCheck) // We did not check for egl images yet
+        return;
+
+    if (eglImageSupported())
+        m_currentFrame = (m_currentFrame + 1) % 2;
+    else
+        m_currentFrame = 0;
+
     int size[2] = { 0, 0 };
 
     int result = screen_get_window_property_iv(m_window, SCREEN_PROPERTY_SOURCE_SIZE, size);
@@ -324,40 +404,33 @@ void WindowGrabber::grab()
     if (m_screenBufferWidth != size[0] || m_screenBufferHeight != size[1]) {
         // The source viewport size changed, so we have to adapt our buffers
 
-        if (m_screenPixmapBufferInitialized) {
-            screen_destroy_pixmap_buffer(m_screenPixmap);
-            m_screenPixmapBufferInitialized = false;
+        if (m_screenPixmapBuffersInitialized) {
+            screen_destroy_pixmap_buffer(m_screenPixmaps[0]);
+            if (eglImageSupported())
+                screen_destroy_pixmap_buffer(m_screenPixmaps[1]);
         }
 
         m_screenBufferWidth = size[0];
         m_screenBufferHeight = size[1];
 
         updateFrameSize();
+        m_eglImagesInitialized = false;
     }
 
     const int rect[] = { 0, 0, m_screenBufferWidth, m_screenBufferHeight };
-    result = screen_read_window(m_window, m_screenPixmapBuffer, 1, rect, 0);
+    result = screen_read_window(m_window, m_screenPixmapBuffers[m_currentFrame], 1, rect, 0);
     if (result != 0)
         return;
 
-    const QImage frame((unsigned char*)m_screenBuffer, m_screenBufferWidth, m_screenBufferHeight,
-                       m_screenBufferStride, QImage::Format_ARGB32);
+    const QImage frame((unsigned char*)m_screenBuffers[m_currentFrame], m_screenBufferWidth,
+                       m_screenBufferHeight, m_screenBufferStride, QImage::Format_ARGB32);
 
-    emit frameGrabbed(frame);
+    emit frameGrabbed(frame, imgTextures[m_currentFrame]);
 }
 
 void WindowGrabber::cleanup()
 {
-    if (m_screenPixmapBufferInitialized) {
-        screen_destroy_buffer(m_screenPixmapBuffer);
-        m_screenPixmapBufferInitialized = false;
-    }
-
-    if (m_screenPixmapInitialized) {
-        screen_destroy_pixmap(m_screenPixmap);
-        m_screenPixmapInitialized = false;
-    }
-
+    //We only need to destroy the context as it frees all resources associated with it
     if (m_screenContextInitialized) {
         screen_destroy_context(m_screenContext);
         m_screenContextInitialized = false;

src/plugins/qnx/common/windowgrabber.h

@@ -41,6 +41,12 @@
 #ifndef WINDOWGRABBER_H
 #define WINDOWGRABBER_H
 
+#define EGL_EGLEXT_PROTOTYPES = 1
+#define GL_GLEXT_PROTOTYPES = 1
+#include <EGL/egl.h>
+#include <GLES2/gl2.h>
+#include <GLES2/gl2ext.h>
+#include <EGL/eglext.h>
 #include <QAbstractNativeEventFilter>
 #include <QObject>
 #include <QTimer>
@@ -59,6 +65,8 @@ public:
 
     void setFrameRate(int frameRate);
 
+    void createEglImages();
+
     void setWindowId(const QByteArray &windowId);
 
     void start();
@@ -73,8 +81,12 @@ public:
 
     QByteArray windowGroupId() const;
 
+    bool eglImageSupported();
+    void checkForEglImageExtension();
+    bool eglImagesInitialized();
+
 signals:
-    void frameGrabbed(const QImage &frame);
+    void frameGrabbed(const QImage &frame, int);
 
 private slots:
     void grab();
@@ -89,10 +101,10 @@ private:
 
     screen_window_t m_window;
     screen_context_t m_screenContext;
-    screen_pixmap_t m_screenPixmap;
-    screen_buffer_t m_screenPixmapBuffer;
+    screen_pixmap_t m_screenPixmaps[2];
+    screen_buffer_t m_screenPixmapBuffers[2];
 
-    char* m_screenBuffer;
+    char *m_screenBuffers[2];
 
     int m_screenBufferWidth;
     int m_screenBufferHeight;
@@ -100,8 +112,13 @@ private:
 
     bool m_active : 1;
     bool m_screenContextInitialized : 1;
-    bool m_screenPixmapInitialized : 1;
-    bool m_screenPixmapBufferInitialized : 1;
+    bool m_screenPixmapBuffersInitialized : 1;
+    int m_currentFrame;
+    EGLImageKHR img[2];
+    GLuint imgTextures[2];
+    bool m_eglImageSupported : 1;
+    bool m_eglImagesInitialized : 1;
+    bool m_eglImageCheck : 1; // We must not send a grabed frame before this is true
 };
 
 QT_END_NAMESPACE

src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.cpp

@@ -46,6 +46,7 @@
 #include <QCoreApplication>
 #include <QDebug>
 #include <QVideoSurfaceFormat>
+#include <QOpenGLContext>
 
 #include <mm/renderer.h>
 
@@ -59,7 +60,7 @@ MmRendererPlayerVideoRendererControl::MmRendererPlayerVideoRendererControl(QObje
     , m_context(0)
     , m_videoId(-1)
 {
-    connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage)), SLOT(frameGrabbed(QImage)));
+    connect(m_windowGrabber, SIGNAL(frameGrabbed(QImage, int)), SLOT(frameGrabbed(QImage, int)));
 }
 
 MmRendererPlayerVideoRendererControl::~MmRendererPlayerVideoRendererControl()
@@ -75,6 +76,10 @@ QAbstractVideoSurface *MmRendererPlayerVideoRendererControl::surface() const
 void MmRendererPlayerVideoRendererControl::setSurface(QAbstractVideoSurface *surface)
 {
     m_surface = QPointer<QAbstractVideoSurface>(surface);
+    if (QOpenGLContext::currentContext())
+        m_windowGrabber->checkForEglImageExtension();
+    else
+        m_surface->setProperty("_q_GLThreadCallback", QVariant::fromValue<QObject*>(this));
 }
 
 void MmRendererPlayerVideoRendererControl::attachDisplay(mmr_context_t *context)
@@ -139,20 +144,86 @@ void MmRendererPlayerVideoRendererControl::resume()
     m_windowGrabber->resume();
 }
 
-void MmRendererPlayerVideoRendererControl::frameGrabbed(const QImage &frame)
+class BBTextureBuffer : public QAbstractVideoBuffer
+{
+public:
+    BBTextureBuffer(int handle) :
+        QAbstractVideoBuffer(QAbstractVideoBuffer::GLTextureHandle)
+    {
+        m_handle = handle;
+    }
+    MapMode mapMode() const {
+        return QAbstractVideoBuffer::ReadWrite;
+    }
+    void unmap() {
+
+    }
+    uchar *map(MapMode mode, int * numBytes, int * bytesPerLine) {
+        Q_UNUSED(mode);
+        Q_UNUSED(numBytes);
+        Q_UNUSED(bytesPerLine);
+        return 0;
+    }
+    QVariant handle() const {
+        return m_handle;
+    }
+private:
+    int m_handle;
+};
+
+void MmRendererPlayerVideoRendererControl::frameGrabbed(const QImage &frame, int handle)
 {
     if (m_surface) {
         if (!m_surface->isActive()) {
-            m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32));
+            if (m_windowGrabber->eglImageSupported()) {
+                if (QOpenGLContext::currentContext())
+                    m_windowGrabber->createEglImages();
+                else
+                    m_surface->setProperty("_q_GLThreadCallback", QVariant::fromValue<QObject*>(this));
+
+                m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_BGR32,
+                                                 QAbstractVideoBuffer::GLTextureHandle));
+            } else {
+                m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32));
+            }
         } else {
             if (m_surface->surfaceFormat().frameSize() != frame.size()) {
+                QAbstractVideoBuffer::HandleType type = m_surface->surfaceFormat().handleType();
                 m_surface->stop();
-                m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32));
+                if (type != QAbstractVideoBuffer::NoHandle) {
+                    m_surface->setProperty("_q_GLThreadCallback", QVariant::fromValue<QObject*>(this));
+                    m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_BGR32,
+                                                     QAbstractVideoBuffer::GLTextureHandle));
+                } else {
+                    m_surface->start(QVideoSurfaceFormat(frame.size(), QVideoFrame::Format_ARGB32));
+                }
             }
         }
 
-        m_surface->present(frame.copy());
+        // Depending on the support of EGL images on the current platform we either pass a texture
+        // handle or a copy of the image data
+        if (m_surface->surfaceFormat().handleType() != QAbstractVideoBuffer::NoHandle) {
+            if (m_windowGrabber->eglImagesInitialized() &&
+                    m_surface->property("_q_GLThreadCallback") != 0)
+                m_surface->setProperty("_q_GLThreadCallback", 0);
+
+
+            BBTextureBuffer *textBuffer = new BBTextureBuffer(handle);
+            QVideoFrame actualFrame(textBuffer, frame.size(), QVideoFrame::Format_BGR32);
+            m_surface->present(actualFrame);
+        } else {
+            m_surface->present(frame.copy());
+        }
     }
 }
 
+void MmRendererPlayerVideoRendererControl::customEvent(QEvent *e)
+{
+    // This is running in the render thread (OpenGL enabled)
+    if (e->type() == QEvent::User)
+        m_windowGrabber->checkForEglImageExtension();
+    else if (e->type() == QEvent::User + 1)
+        m_windowGrabber->createEglImages();
+}
+
 QT_END_NAMESPACE

src/plugins/qnx/mediaplayer/mmrendererplayervideorenderercontrol.h

@@ -67,8 +67,10 @@ public:
     void pause();
     void resume();
 
+    void customEvent(QEvent *) Q_DECL_OVERRIDE;
+
 private Q_SLOTS:
-    void frameGrabbed(const QImage &frame);
+    void frameGrabbed(const QImage &frame, int);
 
 private:
     QPointer<QAbstractVideoSurface> m_surface;

src/qtmultimediaquicktools/qdeclarativevideooutput_render.cpp

@@ -201,6 +201,16 @@ QSGNode *QDeclarativeVideoRendererBackend::updatePaintNode(QSGNode *oldNode,
             obj->event(&ev);
         }
     }
+#if defined (Q_OS_QNX) // On QNX we need to be called back again for creating the egl images
+    else {
+        // Internal mechanism to call back the surface renderer from the QtQuick render thread
+        QObject *obj = m_surface->property("_q_GLThreadCallback").value<QObject*>();
+        if (obj) {
+            QEvent ev(static_cast<QEvent::Type>(QEvent::User + 1));
+            obj->event(&ev);
+        }
+    }
+#endif
 
     if (m_frameChanged) {
         if (videoNode && videoNode->pixelFormat() != m_frame.pixelFormat()) {