slapin/qtmultimedia
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

AVFoundation: Use CoreAnimation to render video to QVideoWidget

Browse Source 
Previously a QGLWidget was used as a target for the
AVFVideoFrameRenderer.  This was uncessary as it is possible to render
directly on top of the QWidget using the CoreAnimation Framework.

Change-Id: I08923c85fd56c8874c1d8c187ae5145e220fab92
Reviewed-by: Richard Moe Gustavsen <richard.gustavsen@digia.com>
This commit is contained in:
Andy Nichols
2013-08-15 15:08:35 +02:00
committed by The Qt Project
parent 17a700292a
commit 51d0f852b9
7 changed files with 129 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.h
View File

@@ -56,15 +56,11 @@ class QOpenGLFramebufferObject;
class QWindow;
class QOpenGLContext;
class QAbstractVideoSurface;
class QGLWidget;
class AVFVideoFrameRenderer : public QObject
{
public:
AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QObject *parent = 0);
#ifndef QT_NO_WIDGETS
AVFVideoFrameRenderer(QGLWidget *glWidget, const QSize &size, QObject *parent = 0);
#endif
virtual ~AVFVideoFrameRenderer();
@@ -76,9 +72,6 @@ private:
void renderLayerToFBO(AVPlayerLayer *layer, QOpenGLFramebufferObject *fbo);
CARenderer *m_videoLayerRenderer;
#ifndef QT_NO_WIDGETS
QGLWidget *m_glWidget;
#endif
QAbstractVideoSurface *m_surface;
QOpenGLFramebufferObject *m_fbo[2];
QWindow *m_offscreenSurface;

33
src/plugins/avfoundation/mediaplayer/avfvideoframerenderer.mm
View File

@@ -45,10 +45,6 @@
#include <QtGui/QOpenGLFramebufferObject>
#include <QtGui/QWindow>
#ifndef QT_NO_WIDGETS
#include <QtOpenGL/QGLWidget>
#endif
#ifdef QT_DEBUG_AVF
#include <QtCore/qdebug.h>
#endif
@@ -76,31 +72,6 @@ AVFVideoFrameRenderer::AVFVideoFrameRenderer(QAbstractVideoSurface *surface, QOb
m_offscreenSurface->setGeometry(0, 0, 1, 1);
m_offscreenSurface->create();
}
#ifndef QT_NO_WIDGETS
AVFVideoFrameRenderer::AVFVideoFrameRenderer(QGLWidget *glWidget, const QSize &size, QObject *parent)
: QObject(parent)
, m_videoLayerRenderer(0)
, m_glWidget(glWidget)
, m_surface(0)
, m_offscreenSurface(0)
, m_glContext(0)
, m_targetSize(size)
, m_currentBuffer(1)
, m_isContextShared(true)
{
m_fbo[0] = 0;
m_fbo[1] = 0;
//Create Hidden QWindow surface to create context in this thread
m_offscreenSurface = new QWindow();
m_offscreenSurface->setSurfaceType(QWindow::OpenGLSurface);
//Needs geometry to be a valid surface, but size is not important
m_offscreenSurface->setGeometry(0, 0, 1, 1);
m_offscreenSurface->create();
}
#endif
AVFVideoFrameRenderer::~AVFVideoFrameRenderer()
{
@@ -168,10 +139,6 @@ QOpenGLFramebufferObject *AVFVideoFrameRenderer::initRenderer(AVPlayerLayer *lay
if (m_surface) {
//QOpenGLContext *renderThreadContext = 0;
shareContext = qobject_cast<QOpenGLContext*>(m_surface->property("GLContext").value<QObject*>());
#ifndef QT_NO_WIDGETS
} else {
shareContext = m_glWidget->context()->contextHandle();
#endif
}
m_glContext = new QOpenGLContext();
m_glContext->setFormat(m_offscreenSurface->requestedFormat());

47
src/plugins/avfoundation/mediaplayer/avfvideowidget.h
View File

@@ -42,43 +42,44 @@
#ifndef AVFVIDEOWIDGET_H
#define AVFVIDEOWIDGET_H
#include <QtOpenGL/QGLWidget>
#include <QtGui/QMatrix4x4>
#include <QtWidgets/QWidget>
@class AVPlayerLayer;
#if defined(Q_OS_OSX)
@class NSView;
#else
@class UIView;
#endif
QT_BEGIN_NAMESPACE
class QOpenGLShaderProgram;
class AVFVideoWidget : public QGLWidget
class AVFVideoWidget : public QWidget
{
public:
AVFVideoWidget(QWidget *parent, const QGLFormat &format);
AVFVideoWidget(QWidget *parent);
virtual ~AVFVideoWidget();
void initializeGL();
void resizeGL(int w, int h);
void paintGL();
void setTexture(GLuint texture);
QSize sizeHint() const;
void setNativeSize(const QSize &size);
Qt::AspectRatioMode aspectRatioMode() const;
void setAspectRatioMode(Qt::AspectRatioMode mode);
void setPlayerLayer(AVPlayerLayer *layer);
protected:
void resizeEvent(QResizeEvent *);
void paintEvent(QPaintEvent *);
private:
QRect displayRect() const;
void updateAspectRatio();
void updatePlayerLayerBounds(const QSize &size);
GLuint m_textureId;
QSize m_nativeSize;
Qt::AspectRatioMode m_aspectRatioMode;
QOpenGLShaderProgram *m_shaderProgram;
QMatrix4x4 m_transformMatrix;
int m_matrixLocation;
int m_vertexCoordEntry;
int m_textureCoordEntry;
AVPlayerLayer *m_playerLayer;
#if defined(Q_OS_OSX)
NSView *m_nativeView;
#else
UIView *m_nativeView;
#endif
};
QT_END_NAMESPACE

221
src/plugins/avfoundation/mediaplayer/avfvideowidget.mm
View File

@@ -41,15 +41,19 @@
#include "avfvideowidget.h"
#include <QtCore/QDebug>
#include <QtGui/QOpenGLShaderProgram>
#include <AVFoundation/AVFoundation.h>
#include <QtGui/QResizeEvent>
#include <QtGui/QPaintEvent>
#include <QtGui/QPainter>
QT_USE_NAMESPACE
AVFVideoWidget::AVFVideoWidget(QWidget *parent, const QGLFormat &format)
: QGLWidget(format, parent)
, m_textureId(0)
AVFVideoWidget::AVFVideoWidget(QWidget *parent)
: QWidget(parent)
, m_aspectRatioMode(Qt::KeepAspectRatio)
, m_shaderProgram(0)
, m_playerLayer(0)
, m_nativeView(0)
{
setAutoFillBackground(false);
}
@@ -59,113 +63,9 @@ AVFVideoWidget::~AVFVideoWidget()
#ifdef QT_DEBUG_AVF
qDebug() << Q_FUNC_INFO;
#endif
delete m_shaderProgram;
}
void AVFVideoWidget::initializeGL()
{
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
m_shaderProgram = new QOpenGLShaderProgram;
static const char *textureVertexProgram =
"uniform highp mat4 matrix;\n"
"attribute highp vec3 vertexCoordEntry;\n"
"attribute highp vec2 textureCoordEntry;\n"
"varying highp vec2 textureCoord;\n"
"void main() {\n"
" textureCoord = textureCoordEntry;\n"
" gl_Position = matrix * vec4(vertexCoordEntry, 1);\n"
"}\n";
static const char *textureFragmentProgram =
"uniform sampler2D texture;\n"
"varying highp vec2 textureCoord;\n"
"void main() {\n"
" gl_FragColor = texture2D(texture, textureCoord);\n"
"}\n";
m_shaderProgram->addShaderFromSourceCode(QOpenGLShader::Vertex, textureVertexProgram);
m_shaderProgram->addShaderFromSourceCode(QOpenGLShader::Fragment, textureFragmentProgram);
m_shaderProgram->link();
}
void AVFVideoWidget::resizeGL(int w, int h)
{
glViewport(0, 0, GLsizei(w), GLsizei(h));
updateGL();
}
void AVFVideoWidget::paintGL()
{
glClear(GL_COLOR_BUFFER_BIT);
if (!m_textureId)
return;
QRect targetRect = displayRect();
GLfloat x1 = targetRect.left();
GLfloat x2 = targetRect.right();
GLfloat y1 = targetRect.bottom();
GLfloat y2 = targetRect.top();
GLfloat zValue = 0;
const GLfloat textureCoordinates[] = {
0, 0,
1, 0,
1, 1,
0, 1
};
const GLfloat vertexCoordinates[] = {
x1, y1, zValue,
x2, y1, zValue,
x2, y2, zValue,
x1, y2, zValue
};
//Set matrix to transfrom geometry values into gl coordinate space.
m_transformMatrix.setToIdentity();
m_transformMatrix.scale( 2.0f / size().width(), 2.0f / size().height() );
m_transformMatrix.translate(-size().width() / 2.0f, -size().height() / 2.0f);
m_shaderProgram->bind();
m_vertexCoordEntry = m_shaderProgram->attributeLocation("vertexCoordEntry");
m_textureCoordEntry = m_shaderProgram->attributeLocation("textureCoordEntry");
m_matrixLocation = m_shaderProgram->uniformLocation("matrix");
//attach the data!
glEnableVertexAttribArray(m_vertexCoordEntry);
glEnableVertexAttribArray(m_textureCoordEntry);
glVertexAttribPointer(m_vertexCoordEntry, 3, GL_FLOAT, GL_FALSE, 0, vertexCoordinates);
glVertexAttribPointer(m_textureCoordEntry, 2, GL_FLOAT, GL_FALSE, 0, textureCoordinates);
m_shaderProgram->setUniformValue(m_matrixLocation, m_transformMatrix);
glBindTexture(GL_TEXTURE_2D, m_textureId);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
glBindTexture(GL_TEXTURE_2D, 0);
glDisableVertexAttribArray(m_vertexCoordEntry);
glDisableVertexAttribArray(m_textureCoordEntry);
m_shaderProgram->release();
}
void AVFVideoWidget::setTexture(GLuint texture)
{
m_textureId = texture;
if (isVisible()) {
makeCurrent();
updateGL();
}
if (m_playerLayer)
[m_playerLayer release];
}
QSize AVFVideoWidget::sizeHint() const
@@ -173,29 +73,104 @@ QSize AVFVideoWidget::sizeHint() const
return m_nativeSize;
}
void AVFVideoWidget::setNativeSize(const QSize &size)
Qt::AspectRatioMode AVFVideoWidget::aspectRatioMode() const
{
m_nativeSize = size;
return m_aspectRatioMode;
}
void AVFVideoWidget::setAspectRatioMode(Qt::AspectRatioMode mode)
{
if (m_aspectRatioMode != mode) {
m_aspectRatioMode = mode;
update();
updateAspectRatio();
}
}
QRect AVFVideoWidget::displayRect() const
void AVFVideoWidget::setPlayerLayer(AVPlayerLayer *layer)
{
QRect displayRect = rect();
if (m_playerLayer == layer)
return;
if (m_aspectRatioMode == Qt::KeepAspectRatio) {
QSize size = m_nativeSize;
size.scale(displayRect.size(), Qt::KeepAspectRatio);
displayRect = QRect(QPoint(0, 0), size);
displayRect.moveCenter(rect().center());
if (!m_nativeView) {
//make video widget a native window
#if defined(Q_OS_OSX)
m_nativeView = (NSView*)this->winId();
[m_nativeView setWantsLayer:YES];
#else
m_nativeView = (UIView*)this->winId();
#endif
}
return displayRect;
if (m_playerLayer) {
[m_playerLayer removeFromSuperlayer];
[m_playerLayer release];
}
m_playerLayer = layer;
CALayer *nativeLayer = [m_nativeView layer];
if (layer) {
[layer retain];
m_nativeSize = QSize(m_playerLayer.bounds.size.width,
m_playerLayer.bounds.size.height);
updateAspectRatio();
[nativeLayer addSublayer:m_playerLayer];
updatePlayerLayerBounds(this->size());
}
NSArray *sublayers = [nativeLayer sublayers];
qDebug() << "playerlayer: " << "at z:" << [m_playerLayer zPosition]
<< " frame: " << m_playerLayer.frame.size.width << "x" << m_playerLayer.frame.size.height;
qDebug() << "superlayer: " << "at z:" << [nativeLayer zPosition]
<< " frame: " << nativeLayer.frame.size.width << "x" << nativeLayer.frame.size.height;
int i = 0;
for (CALayer *layer in sublayers) {
qDebug() << "layer " << i << ": at z:" << [layer zPosition]
<< " frame: " << layer.frame.size.width << "x" << layer.frame.size.height;
i++;
}
}
void AVFVideoWidget::resizeEvent(QResizeEvent *event)
{
updatePlayerLayerBounds(event->size());
QWidget::resizeEvent(event);
}
void AVFVideoWidget::paintEvent(QPaintEvent *event)
{
QPainter painter(this);
painter.fillRect(rect(), Qt::black);
QWidget::paintEvent(event);
}
void AVFVideoWidget::updateAspectRatio()
{
if (m_playerLayer) {
switch (m_aspectRatioMode) {
case Qt::IgnoreAspectRatio:
[m_playerLayer setVideoGravity:AVLayerVideoGravityResize];
break;
case Qt::KeepAspectRatio:
[m_playerLayer setVideoGravity:AVLayerVideoGravityResizeAspect];
break;
case Qt::KeepAspectRatioByExpanding:
[m_playerLayer setVideoGravity:AVLayerVideoGravityResizeAspectFill];
break;
default:
break;
}
}
}
void AVFVideoWidget::updatePlayerLayerBounds(const QSize &size)
{
m_playerLayer.bounds = CGRectMake(0.0f, 0.0f, (float)size.width(), (float)size.height());
}

15
src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.h
View File

@@ -45,13 +45,11 @@
#include <qvideowidgetcontrol.h>
#include "avfvideooutput.h"
#import <CoreVideo/CVBase.h>
@class AVPlayerLayer;
QT_BEGIN_NAMESPACE
class AVFDisplayLink;
class AVFVideoWidget;
class AVFVideoFrameRenderer;
class AVFVideoWidgetControl : public QVideoWidgetControl, public AVFVideoOutput
{
@@ -83,24 +81,15 @@ public:
int saturation() const;
void setSaturation(int saturation);
private Q_SLOTS:
void updateVideoFrame(const CVTimeStamp &ts);
private:
void setupVideoOutput();
AVFDisplayLink *m_displayLink;
AVFVideoWidget *m_videoWidget;
AVFVideoFrameRenderer *m_frameRenderer;
QSize m_nativeSize;
Qt::AspectRatioMode m_aspectRatioMode;
bool m_fullscreen;
int m_brightness;
int m_contrast;
int m_hue;
int m_saturation;
void *m_playerLayer;
};
QT_END_NAMESPACE

80
src/plugins/avfoundation/mediaplayer/avfvideowidgetcontrol.mm
View File

@@ -40,10 +40,7 @@
****************************************************************************/
#include "avfvideowidgetcontrol.h"
#include "avfvideowidget.h"
#include "avfvideoframerenderer.h"
#include "avfdisplaylink.h"
#ifdef QT_DEBUG_AVF
#include <QtCore/QDebug>
@@ -55,22 +52,13 @@ QT_USE_NAMESPACE
AVFVideoWidgetControl::AVFVideoWidgetControl(QObject *parent)
: QVideoWidgetControl(parent)
, m_frameRenderer(0)
, m_aspectRatioMode(Qt::KeepAspectRatio)
, m_fullscreen(false)
, m_brightness(0)
, m_contrast(0)
, m_hue(0)
, m_saturation(0)
, m_playerLayer(0)
{
QGLFormat format = QGLFormat::defaultFormat();
format.setSwapInterval(1); // Vertical sync (avoid tearing)
format.setDoubleBuffer(true);
m_videoWidget = new AVFVideoWidget(0, format);
m_displayLink = new AVFDisplayLink(this);
connect(m_displayLink, SIGNAL(tick(CVTimeStamp)), this, SLOT(updateVideoFrame(CVTimeStamp)));
m_videoWidget = new AVFVideoWidget(0);
}
AVFVideoWidgetControl::~AVFVideoWidgetControl()
@@ -78,10 +66,6 @@ AVFVideoWidgetControl::~AVFVideoWidgetControl()
#ifdef QT_DEBUG_AVF
qDebug() << Q_FUNC_INFO;
#endif
m_displayLink->stop();
if (m_playerLayer)
[(AVPlayerLayer*)m_playerLayer release];
delete m_videoWidget;
}
@@ -91,26 +75,8 @@ void AVFVideoWidgetControl::setLayer(void *playerLayer)
qDebug() << Q_FUNC_INFO << playerLayer;
#endif
if (m_playerLayer == playerLayer)
return;
m_videoWidget->setPlayerLayer((AVPlayerLayer*)playerLayer);
[(AVPlayerLayer*)playerLayer retain];
[(AVPlayerLayer*)m_playerLayer release];
m_playerLayer = playerLayer;
//If there is no layer to render, stop scheduling updates
if (m_playerLayer == 0) {
m_displayLink->stop();
return;
}
setupVideoOutput();
//make sure we schedule updates
if (!m_displayLink->isActive()) {
m_displayLink->start();
}
}
QWidget *AVFVideoWidgetControl::videoWidget()
@@ -130,12 +96,11 @@ void AVFVideoWidgetControl::setFullScreen(bool fullScreen)
Qt::AspectRatioMode AVFVideoWidgetControl::aspectRatioMode() const
{
return m_aspectRatioMode;
return m_videoWidget->aspectRatioMode();
}
void AVFVideoWidgetControl::setAspectRatioMode(Qt::AspectRatioMode mode)
{
m_aspectRatioMode = mode;
m_videoWidget->setAspectRatioMode(mode);
}
@@ -179,41 +144,4 @@ void AVFVideoWidgetControl::setSaturation(int saturation)
m_saturation = saturation;
}
void AVFVideoWidgetControl::updateVideoFrame(const CVTimeStamp &ts)
{
Q_UNUSED(ts)
AVPlayerLayer *playerLayer = (AVPlayerLayer*)m_playerLayer;
if (!playerLayer) {
qWarning("updateVideoFrame called without AVPlayerLayer (which shouldn't happen)");
return;
}
//Don't try to render a layer that is not ready
if (!playerLayer.readyForDisplay)
return;
GLuint textureId = m_frameRenderer->renderLayerToTexture(playerLayer);
//Make sure we have a valid texture
if (textureId == 0) {
qWarning("renderLayerToTexture failed");
return;
}
m_videoWidget->setTexture(textureId);
}
void AVFVideoWidgetControl::setupVideoOutput()
{
CGRect layerBounds = [(AVPlayerLayer*)m_playerLayer bounds];
m_nativeSize = QSize(layerBounds.size.width, layerBounds.size.height);
m_videoWidget->setNativeSize(m_nativeSize);
if (m_frameRenderer)
delete m_frameRenderer;
m_frameRenderer = new AVFVideoFrameRenderer(m_videoWidget, m_nativeSize, this);
}
#include "moc_avfvideowidgetcontrol.cpp"

2
src/plugins/avfoundation/mediaplayer/mediaplayer.pro
View File

@@ -44,7 +44,7 @@ OBJECTIVE_SOURCES += \
LIBS += -framework QuartzCore -framework AppKit
qtHaveModule(widgets) {
QT += multimediawidgets-private opengl
QT += multimediawidgets-private
HEADERS += \
avfvideowidgetcontrol.h \
avfvideowidget.h