AVFoundation: Use CoreAnimation to render video to QVideoWidget

Previously a QGLWidget was used as a target for the
AVFVideoFrameRenderer.  This was uncessary as it is possible to render
directly on top of the QWidget using the CoreAnimation Framework.

Change-Id: I08923c85fd56c8874c1d8c187ae5145e220fab92
Reviewed-by: Richard Moe Gustavsen <richard.gustavsen@digia.com>

src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h

@@ -56,15 +56,11 @@ class QOpenGLFramebufferObject;
 class QWindow;
 class QOpenGLContext;
 class QAbstractVideoSurface;
-class QGLWidget;
 
 class AVFVideoFrameRenderer : public QObject
 {
 public:
     AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent = 0);
-#ifndef QT_NO_WIDGETS
-    AVFVideoFrameRenderer(QGLWidget *glWidget, const QSize &size, QObject *parent = 0);
-#endif
 
     virtual ~AVFVideoFrameRenderer();
 
@@ -76,9 +72,6 @@ private:
     void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo);
 
     CARenderer *m_videoLayerRenderer;
-#ifndef QT_NO_WIDGETS
-    QGLWidget *m_glWidget;
-#endif
     QAbstractVideoSurface *m_surface;
     QOpenGLFramebufferObject *m_fbo[2];
     QWindow *m_offscreenSurface;

src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm

@@ -45,10 +45,6 @@
 #include <QtGui/QOpenGLFramebufferObject>
 #include <QtGui/QWindow>
 
-#ifndef QT_NO_WIDGETS
-#include <QtOpenGL/QGLWidget>
-#endif
-
 #ifdef QT_DEBUG_AVF
 #include <QtCore/qdebug.h>
 #endif
@@ -76,31 +72,6 @@ AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QOb
     m_offscreenSurface->setGeometry(0, 0, 1, 1);
     m_offscreenSurface->create();
 }
-#ifndef QT_NO_WIDGETS
-AVFVideoFrameRenderer::AVFVideoFrameRenderer(QGLWidget *glWidget, const QSize &size, QObject *parent)
-    : QObject(parent)
-    , m_videoLayerRenderer(0)
-    , m_glWidget(glWidget)
-    , m_surface(0)
-    , m_offscreenSurface(0)
-    , m_glContext(0)
-    , m_targetSize(size)
-    , m_currentBuffer(1)
-    , m_isContextShared(true)
-{
-    m_fbo[0] = 0;
-    m_fbo[1] = 0;
-
-    //Create Hidden QWindow surface to create context in this thread
-    m_offscreenSurface = new QWindow();
-    m_offscreenSurface->setSurfaceType(QWindow::OpenGLSurface);
-    //Needs geometry to be a valid surface, but size is not important
-    m_offscreenSurface->setGeometry(0, 0, 1, 1);
-    m_offscreenSurface->create();
-
-
-}
-#endif
 
 AVFVideoFrameRenderer::~AVFVideoFrameRenderer()
 {
@@ -168,10 +139,6 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay
         if (m_surface) {
             //QOpenGLContext *renderThreadContext = 0;
             shareContext = qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>());
-#ifndef QT_NO_WIDGETS
-        } else {
-            shareContext = m_glWidget->context()->contextHandle();
-#endif
         }
         m_glContext = new QOpenGLContext();
         m_glContext->setFormat(m_offscreenSurface->requestedFormat());

src/plugins/avfoundation/mediaplayer/avfvideowidget.h

@@ -42,43 +42,44 @@
 #ifndef AVFVIDEOWIDGET_H
 #define AVFVIDEOWIDGET_H
 
-#include <QtOpenGL/QGLWidget>
+#include <QtWidgets/QWidget>
-#include <QtGui/QMatrix4x4>
+
+@class AVPlayerLayer;
+#if defined(Q_OS_OSX)
+@class NSView;
+#else
+@class UIView;
+#endif
 
 QT_BEGIN_NAMESPACE
 
-class QOpenGLShaderProgram;
+class AVFVideoWidget : public QWidget
-
-class AVFVideoWidget : public QGLWidget
 {
 public:
-    AVFVideoWidget(QWidget *parent, const QGLFormat &format);
+    AVFVideoWidget(QWidget *parent);
     virtual ~AVFVideoWidget();
 
-    void initializeGL();
-    void resizeGL(int w, int h);
-    void paintGL();
-
-    void setTexture(GLuint texture);
-
     QSize sizeHint() const;
-    void setNativeSize(const QSize &size);
+    Qt::AspectRatioMode aspectRatioMode() const;
-
     void setAspectRatioMode(Qt::AspectRatioMode mode);
+    void setPlayerLayer(AVPlayerLayer *layer);
+
+protected:
+    void resizeEvent(QResizeEvent *);
+    void paintEvent(QPaintEvent *);
 
 private:
-    QRect displayRect() const;
+    void updateAspectRatio();
+    void updatePlayerLayerBounds(const QSize &size);
 
-    GLuint m_textureId;
     QSize m_nativeSize;
     Qt::AspectRatioMode m_aspectRatioMode;
-
+    AVPlayerLayer *m_playerLayer;
-    QOpenGLShaderProgram *m_shaderProgram;
+#if defined(Q_OS_OSX)
-    QMatrix4x4 m_transformMatrix;
+    NSView *m_nativeView;
-
+#else
-    int m_matrixLocation;
+    UIView *m_nativeView;
-    int m_vertexCoordEntry;
+#endif
-    int m_textureCoordEntry;
 };
 
 QT_END_NAMESPACE

src/plugins/avfoundation/mediaplayer/avfvideowidget.mm

@@ -41,15 +41,19 @@
 
 #include "avfvideowidget.h"
 #include <QtCore/QDebug>
-#include <QtGui/QOpenGLShaderProgram>
+
+#include <AVFoundation/AVFoundation.h>
+#include <QtGui/QResizeEvent>
+#include <QtGui/QPaintEvent>
+#include <QtGui/QPainter>
 
 QT_USE_NAMESPACE
 
-AVFVideoWidget::AVFVideoWidget(QWidget *parent, const QGLFormat &format)
+AVFVideoWidget::AVFVideoWidget(QWidget *parent)
-    : QGLWidget(format, parent)
+    : QWidget(parent)
-    , m_textureId(0)
     , m_aspectRatioMode(Qt::KeepAspectRatio)
-    , m_shaderProgram(0)
+    , m_playerLayer(0)
+    , m_nativeView(0)
 {
     setAutoFillBackground(false);
 }
@@ -59,113 +63,9 @@ AVFVideoWidget::~AVFVideoWidget()
 #ifdef QT_DEBUG_AVF
     qDebug() << Q_FUNC_INFO;
 #endif
-    delete m_shaderProgram;
-}
 
-void AVFVideoWidget::initializeGL()
+    if (m_playerLayer)
-{
+        [m_playerLayer release];
-    glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
-
-    m_shaderProgram = new QOpenGLShaderProgram;
-
-    static const char *textureVertexProgram =
-            "uniform highp mat4 matrix;\n"
-            "attribute highp vec3 vertexCoordEntry;\n"
-            "attribute highp vec2 textureCoordEntry;\n"
-            "varying highp vec2 textureCoord;\n"
-            "void main() {\n"
-            "   textureCoord = textureCoordEntry;\n"
-            "   gl_Position = matrix * vec4(vertexCoordEntry, 1);\n"
-            "}\n";
-
-    static const char *textureFragmentProgram =
-            "uniform sampler2D texture;\n"
-            "varying highp vec2 textureCoord;\n"
-            "void main() {\n"
-            "   gl_FragColor = texture2D(texture, textureCoord);\n"
-            "}\n";
-
-    m_shaderProgram->addShaderFromSourceCode(QOpenGLShader::Vertex, textureVertexProgram);
-    m_shaderProgram->addShaderFromSourceCode(QOpenGLShader::Fragment, textureFragmentProgram);
-    m_shaderProgram->link();
-}
-
-void AVFVideoWidget::resizeGL(int w, int h)
-{
-    glViewport(0, 0, GLsizei(w), GLsizei(h));
-    updateGL();
-}
-
-void AVFVideoWidget::paintGL()
-{
-    glClear(GL_COLOR_BUFFER_BIT);
-    if (!m_textureId)
-        return;
-
-    QRect targetRect = displayRect();
-    GLfloat x1 = targetRect.left();
-    GLfloat x2 = targetRect.right();
-    GLfloat y1 = targetRect.bottom();
-    GLfloat y2 = targetRect.top();
-    GLfloat zValue = 0;
-
-    const GLfloat textureCoordinates[] = {
-        0, 0,
-        1, 0,
-        1, 1,
-        0, 1
-    };
-
-    const GLfloat vertexCoordinates[] = {
-        x1, y1, zValue,
-        x2, y1, zValue,
-        x2, y2, zValue,
-        x1, y2, zValue
-    };
-
-    //Set matrix to transfrom geometry values into gl coordinate space.
-    m_transformMatrix.setToIdentity();
-    m_transformMatrix.scale( 2.0f / size().width(), 2.0f / size().height() );
-    m_transformMatrix.translate(-size().width() / 2.0f, -size().height() / 2.0f);
-
-    m_shaderProgram->bind();
-
-    m_vertexCoordEntry = m_shaderProgram->attributeLocation("vertexCoordEntry");
-    m_textureCoordEntry = m_shaderProgram->attributeLocation("textureCoordEntry");
-    m_matrixLocation = m_shaderProgram->uniformLocation("matrix");
-
-    //attach the data!
-    glEnableVertexAttribArray(m_vertexCoordEntry);
-    glEnableVertexAttribArray(m_textureCoordEntry);
-
-    glVertexAttribPointer(m_vertexCoordEntry, 3, GL_FLOAT, GL_FALSE, 0, vertexCoordinates);
-    glVertexAttribPointer(m_textureCoordEntry, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinates);
-    m_shaderProgram->setUniformValue(m_matrixLocation, m_transformMatrix);
-
-    glBindTexture(GL_TEXTURE_2D, m_textureId);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-
-    glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
-
-    glBindTexture(GL_TEXTURE_2D, 0);
-
-    glDisableVertexAttribArray(m_vertexCoordEntry);
-    glDisableVertexAttribArray(m_textureCoordEntry);
-
-    m_shaderProgram->release();
-}
-
-void AVFVideoWidget::setTexture(GLuint texture)
-{
-    m_textureId = texture;
-
-    if (isVisible()) {
-        makeCurrent();
-        updateGL();
-    }
 }
 
 QSize AVFVideoWidget::sizeHint() const
@@ -173,29 +73,104 @@ QSize AVFVideoWidget::sizeHint() const
     return m_nativeSize;
 }
 
-void AVFVideoWidget::setNativeSize(const QSize &size)
+Qt::AspectRatioMode AVFVideoWidget::aspectRatioMode() const
 {
-    m_nativeSize = size;
+    return m_aspectRatioMode;
 }
 
 void AVFVideoWidget::setAspectRatioMode(Qt::AspectRatioMode mode)
 {
     if (m_aspectRatioMode != mode) {
         m_aspectRatioMode = mode;
-        update();
+
+        updateAspectRatio();
     }
 }
 
-QRect AVFVideoWidget::displayRect() const
+void AVFVideoWidget::setPlayerLayer(AVPlayerLayer *layer)
 {
-    QRect displayRect = rect();
+    if (m_playerLayer == layer)
+        return;
 
-    if (m_aspectRatioMode == Qt::KeepAspectRatio) {
+    if (!m_nativeView) {
-        QSize size = m_nativeSize;
+        //make video widget a native window
-        size.scale(displayRect.size(), Qt::KeepAspectRatio);
+#if defined(Q_OS_OSX)
+        m_nativeView = (NSView*)this->winId();
+        [m_nativeView setWantsLayer:YES];
+#else
+        m_nativeView = (UIView*)this->winId();
+#endif
+    }
 
-        displayRect = QRect(QPoint(0, 0), size);
+    if (m_playerLayer) {
-        displayRect.moveCenter(rect().center());
+        [m_playerLayer removeFromSuperlayer];
+        [m_playerLayer release];
     }
-    return displayRect;
+
+    m_playerLayer = layer;
+
+    CALayer *nativeLayer = [m_nativeView layer];
+
+    if (layer) {
+        [layer retain];
+
+        m_nativeSize = QSize(m_playerLayer.bounds.size.width,
+                             m_playerLayer.bounds.size.height);
+
+        updateAspectRatio();
+        [nativeLayer addSublayer:m_playerLayer];
+        updatePlayerLayerBounds(this->size());
+    }
+
+    NSArray *sublayers = [nativeLayer sublayers];
+    qDebug() << "playerlayer: " << "at z:" << [m_playerLayer zPosition]
+                << " frame: " << m_playerLayer.frame.size.width << "x"  << m_playerLayer.frame.size.height;
+    qDebug() << "superlayer: " << "at z:" << [nativeLayer zPosition]
+                << " frame: " << nativeLayer.frame.size.width << "x"  << nativeLayer.frame.size.height;
+    int i = 0;
+    for (CALayer *layer in sublayers) {
+        qDebug() << "layer " << i << ": at z:" << [layer zPosition]
+                    << " frame: " << layer.frame.size.width << "x"  << layer.frame.size.height;
+        i++;
+    }
+
+
+}
+
+void AVFVideoWidget::resizeEvent(QResizeEvent *event)
+{
+    updatePlayerLayerBounds(event->size());
+    QWidget::resizeEvent(event);
+}
+
+void AVFVideoWidget::paintEvent(QPaintEvent *event)
+{
+    QPainter painter(this);
+    painter.fillRect(rect(), Qt::black);
+
+    QWidget::paintEvent(event);
+}
+
+void AVFVideoWidget::updateAspectRatio()
+{
+    if (m_playerLayer) {
+        switch (m_aspectRatioMode) {
+        case Qt::IgnoreAspectRatio:
+            [m_playerLayer setVideoGravity:AVLayerVideoGravityResize];
+            break;
+        case Qt::KeepAspectRatio:
+            [m_playerLayer setVideoGravity:AVLayerVideoGravityResizeAspect];
+            break;
+        case Qt::KeepAspectRatioByExpanding:
+            [m_playerLayer setVideoGravity:AVLayerVideoGravityResizeAspectFill];
+            break;
+        default:
+            break;
+        }
+    }
+}
+
+void AVFVideoWidget::updatePlayerLayerBounds(const QSize &size)
+{
+    m_playerLayer.bounds = CGRectMake(0.0f, 0.0f, (float)size.width(), (float)size.height());
 }

src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.h

@@ -45,13 +45,11 @@
 #include <qvideowidgetcontrol.h>
 #include "avfvideooutput.h"
 
-#import <CoreVideo/CVBase.h>
+@class AVPlayerLayer;
 
 QT_BEGIN_NAMESPACE
 
-class AVFDisplayLink;
 class AVFVideoWidget;
-class AVFVideoFrameRenderer;
 
 class AVFVideoWidgetControl : public QVideoWidgetControl, public AVFVideoOutput
 {
@@ -83,24 +81,15 @@ public:
     int saturation() const;
     void setSaturation(int saturation);
 
-private Q_SLOTS:
-    void updateVideoFrame(const CVTimeStamp &ts);
-
 private:
-    void setupVideoOutput();
-
-    AVFDisplayLink *m_displayLink;
     AVFVideoWidget *m_videoWidget;
-    AVFVideoFrameRenderer *m_frameRenderer;
+
-    QSize m_nativeSize;
-    Qt::AspectRatioMode m_aspectRatioMode;
     bool m_fullscreen;
     int m_brightness;
     int m_contrast;
     int m_hue;
     int m_saturation;
 
-    void *m_playerLayer;
 };
 
 QT_END_NAMESPACE

src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.mm

@@ -40,10 +40,7 @@
 ****************************************************************************/
 
 #include "avfvideowidgetcontrol.h"
-
 #include "avfvideowidget.h"
-#include "avfvideoframerenderer.h"
-#include "avfdisplaylink.h"
 
 #ifdef QT_DEBUG_AVF
 #include <QtCore/QDebug>
@@ -55,22 +52,13 @@ QT_USE_NAMESPACE
 
 AVFVideoWidgetControl::AVFVideoWidgetControl(QObject *parent)
     : QVideoWidgetControl(parent)
-    , m_frameRenderer(0)
-    , m_aspectRatioMode(Qt::KeepAspectRatio)
     , m_fullscreen(false)
     , m_brightness(0)
     , m_contrast(0)
     , m_hue(0)
     , m_saturation(0)
-    , m_playerLayer(0)
 {
-    QGLFormat format = QGLFormat::defaultFormat();
+    m_videoWidget = new AVFVideoWidget(0);
-    format.setSwapInterval(1); // Vertical sync (avoid tearing)
-    format.setDoubleBuffer(true);
-    m_videoWidget = new AVFVideoWidget(0, format);
-
-    m_displayLink = new AVFDisplayLink(this);
-    connect(m_displayLink, SIGNAL(tick(CVTimeStamp)), this, SLOT(updateVideoFrame(CVTimeStamp)));
 }
 
 AVFVideoWidgetControl::~AVFVideoWidgetControl()
@@ -78,10 +66,6 @@ AVFVideoWidgetControl::~AVFVideoWidgetControl()
 #ifdef QT_DEBUG_AVF
     qDebug() << Q_FUNC_INFO;
 #endif
-    m_displayLink->stop();
-    if (m_playerLayer)
-        [(AVPlayerLayer*)m_playerLayer release];
-
     delete m_videoWidget;
 }
 
@@ -91,26 +75,8 @@ void AVFVideoWidgetControl::setLayer(void *playerLayer)
     qDebug() << Q_FUNC_INFO << playerLayer;
 #endif
 
-    if (m_playerLayer == playerLayer)
+    m_videoWidget->setPlayerLayer((AVPlayerLayer*)playerLayer);
-        return;
 
-    [(AVPlayerLayer*)playerLayer retain];
-    [(AVPlayerLayer*)m_playerLayer release];
-
-    m_playerLayer = playerLayer;
-
-    //If there is no layer to render, stop scheduling updates
-    if (m_playerLayer == 0) {
-        m_displayLink->stop();
-        return;
-    }
-
-    setupVideoOutput();
-
-    //make sure we schedule updates
-    if (!m_displayLink->isActive()) {
-        m_displayLink->start();
-    }
 }
 
 QWidget *AVFVideoWidgetControl::videoWidget()
@@ -130,12 +96,11 @@ void AVFVideoWidgetControl::setFullScreen(bool fullScreen)
 
 Qt::AspectRatioMode AVFVideoWidgetControl::aspectRatioMode() const
 {
-    return m_aspectRatioMode;
+    return m_videoWidget->aspectRatioMode();
 }
 
 void AVFVideoWidgetControl::setAspectRatioMode(Qt::AspectRatioMode mode)
 {
-    m_aspectRatioMode = mode;
     m_videoWidget->setAspectRatioMode(mode);
 }
 
@@ -179,41 +144,4 @@ void AVFVideoWidgetControl::setSaturation(int saturation)
     m_saturation = saturation;
 }
 
-void AVFVideoWidgetControl::updateVideoFrame(const CVTimeStamp &ts)
+#include "moc_avfvideowidgetcontrol.cpp"
-{
-    Q_UNUSED(ts)
-
-    AVPlayerLayer *playerLayer = (AVPlayerLayer*)m_playerLayer;
-
-    if (!playerLayer) {
-        qWarning("updateVideoFrame called without AVPlayerLayer (which shouldn't happen)");
-        return;
-    }
-
-    //Don't try to render a layer that is not ready
-    if (!playerLayer.readyForDisplay)
-        return;
-
-    GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer);
-
-    //Make sure we have a valid texture
-    if (textureId == 0) {
-        qWarning("renderLayerToTexture failed");
-        return;
-    }
-
-    m_videoWidget->setTexture(textureId);
-}
-
-void AVFVideoWidgetControl::setupVideoOutput()
-{
-    CGRect layerBounds = [(AVPlayerLayer*)m_playerLayer bounds];
-    m_nativeSize = QSize(layerBounds.size.width, layerBounds.size.height);
-    m_videoWidget->setNativeSize(m_nativeSize);
-
-    if (m_frameRenderer)
-        delete m_frameRenderer;
-
-    m_frameRenderer = new AVFVideoFrameRenderer(m_videoWidget, m_nativeSize, this);
-}
-

src/plugins/avfoundation/mediaplayer/mediaplayer.pro

@@ -44,7 +44,7 @@ OBJECTIVE_SOURCES += \
     LIBS += -framework QuartzCore -framework AppKit
 
     qtHaveModule(widgets) {
-        QT += multimediawidgets-private opengl
+        QT += multimediawidgets-private
         HEADERS += \
             avfvideowidgetcontrol.h \
             avfvideowidget.h