Fix a segfault (buffer overflow for win32 only).
[vlc.git] / modules / video_output / opengllayer.m
1 /*****************************************************************************
2  * opengl.c: CAOpenGLLayer (Mac OS X) video output. Display a video output in
3  * a layer. The layer will register itself to the drawable object stored in 
4  * the "drawable" variable. 
5  *****************************************************************************
6  * Copyright (C) 2004-2009 the VideoLAN team
7  * $Id$
8  *
9  * Authors: Cyril Deguet <asmax@videolan.org>
10  *          Gildas Bazin <gbazin@videolan.org>
11  *          Eric Petit <titer@m0k.org>
12  *          Cedric Cocquebert <cedric.cocquebert@supelec.fr>
13  *          Pierre d'Herbemont <pdherbemont # videolan.org>
14  *
15  * This program is free software; you can redistribute it and/or modify
16  * it under the terms of the GNU General Public License as published by
17  * the Free Software Foundation; either version 2 of the License, or
18  * (at your option) any later version.
19  *
20  * This program is distributed in the hope that it will be useful,
21  * but WITHOUT ANY WARRANTY; without even the implied warranty of
22  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
23  * GNU General Public License for more details.
24  *
25  * You should have received a copy of the GNU General Public License
26  * along with this program; if not, write to the Free Software
27  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
28  *****************************************************************************/
29
30 /*****************************************************************************
31  * Preamble
32  *****************************************************************************/
33 #include <errno.h>                                                 /* ENOMEM */
34
35 #ifdef HAVE_CONFIG_H
36 # include "config.h"
37 #endif
38
39 #include <vlc_common.h>
40 #include <vlc_plugin.h>
41 #include <vlc_vout.h>
42
43 #import <QuartzCore/QuartzCore.h>
44 #import <Cocoa/Cocoa.h>
45 #import <OpenGL/OpenGL.h>
46
47 /* On OS X, use GL_TEXTURE_RECTANGLE_EXT instead of GL_TEXTURE_2D.
48    This allows sizes which are not powers of 2 */
49 #define VLCGL_TARGET GL_TEXTURE_RECTANGLE_EXT
50
51 /* OS X OpenGL supports YUV. Hehe. */
52 #define VLCGL_FORMAT GL_YCBCR_422_APPLE
53 #define VLCGL_TYPE   GL_UNSIGNED_SHORT_8_8_APPLE
54
55 /* RV32 */
56 #define VLCGL_RGB_FORMAT GL_RGBA
57 #define VLCGL_RGB_TYPE GL_UNSIGNED_BYTE
58
59 /* YUY2 */
60 #ifndef YCBCR_MESA
61 #define YCBCR_MESA 0x8757
62 #endif
63 #ifndef UNSIGNED_SHORT_8_8_MESA
64 #define UNSIGNED_SHORT_8_8_MESA 0x85BA
65 #endif
66 #define VLCGL_YUV_FORMAT YCBCR_MESA
67 #define VLCGL_YUV_TYPE UNSIGNED_SHORT_8_8_MESA
68
69
70 #ifndef GL_CLAMP_TO_EDGE
71 #   define GL_CLAMP_TO_EDGE 0x812F
72 #endif
73
74 @interface VLCVideoView : NSObject
75 - (void)addVoutLayer:(CALayer *)layer;
76 @end
77
78 /*****************************************************************************
79  * Vout interface
80  *****************************************************************************/
81 static int  CreateVout   ( vlc_object_t * );
82 static void DestroyVout  ( vlc_object_t * );
83 static int  Init         ( vout_thread_t * );
84 static void End          ( vout_thread_t * );
85 static int  Manage       ( vout_thread_t * );
86 static void Render       ( vout_thread_t *, picture_t * );
87 static void DisplayVideo ( vout_thread_t *, picture_t * );
88 static int  Control      ( vout_thread_t *, int, va_list );
89
90 static int InitTextures  ( vout_thread_t * );
91
92 vlc_module_begin ()
93     set_shortname( "OpenGLLayer" )
94     set_category( CAT_VIDEO )
95     set_subcategory( SUBCAT_VIDEO_VOUT )
96     set_description( N_("Core Animation OpenGL Layer (Mac OS X)") )
97     set_capability( "video output", 20 )
98     add_shortcut( "opengllayer" )
99     set_callbacks( CreateVout, DestroyVout )
100 vlc_module_end ()
101
102 @interface VLCVoutLayer : CAOpenGLLayer {
103     vout_thread_t * p_vout;
104 }
105 + (id)layerWithVout:(vout_thread_t*)_p_vout; 
106 @end
107
108 /*****************************************************************************
109  * vout_sys_t: video output method descriptor
110  *****************************************************************************
111  * This structure is part of the video output thread descriptor.
112  * It describes the OpenGL specific properties of the output thread.
113  *****************************************************************************/
114 struct vout_sys_t
115 {
116     vout_thread_t * p_vout;
117
118     uint8_t    *pp_buffer[2]; /* one last rendered, one to be rendered */
119     int         i_index;
120     bool  b_frame_available;
121     
122     CGLContextObj glContext;
123
124     int         i_tex_width;
125     int         i_tex_height;
126     GLuint      p_textures[2];
127
128     NSAutoreleasePool *autorealease_pool;
129     VLCVoutLayer * o_layer;
130     id          o_cocoa_container;
131 };
132
133 /*****************************************************************************
134  * CreateVout: This function allocates and initializes the OpenGL vout method.
135  *****************************************************************************/
136 static int CreateVout( vlc_object_t *p_this )
137 {
138     vout_thread_t *p_vout = (vout_thread_t *)p_this;
139     vout_sys_t *p_sys;
140     char * psz;
141
142     /* Allocate structure */
143     p_vout->p_sys = p_sys = calloc( 1, sizeof( vout_sys_t ) );
144     if( p_sys == NULL )
145         return VLC_EGENERIC;
146
147     p_sys->i_tex_width  = p_vout->fmt_in.i_width;
148     p_sys->i_tex_height = p_vout->fmt_in.i_height;
149
150     msg_Dbg( p_vout, "Texture size: %dx%d", p_sys->i_tex_width,
151              p_sys->i_tex_height );
152
153     p_vout->pf_init = Init;
154     p_vout->pf_end = End;
155     p_vout->pf_manage = Manage;
156     p_vout->pf_render = Render;
157     p_vout->pf_display = DisplayVideo;
158     p_vout->pf_control = Control;
159
160     return VLC_SUCCESS;
161 }
162
163 /*****************************************************************************
164  * Init: initialize the OpenGL video thread output method
165  *****************************************************************************/
166 static int Init( vout_thread_t *p_vout )
167 {
168     vout_sys_t *p_sys = p_vout->p_sys;
169     int i_pixel_pitch;
170     vlc_value_t val;
171
172 #if ( defined( WORDS_BIGENDIAN ) && VLCGL_FORMAT == GL_YCBCR_422_APPLE ) || (VLCGL_FORMAT == YCBCR_MESA)
173     p_vout->output.i_chroma = VLC_CODEC_YUYV;
174     i_pixel_pitch = 2;
175 #elif (VLCGL_FORMAT == GL_YCBCR_422_APPLE)
176     p_vout->output.i_chroma = VLC_CODEC_UYVY;
177     i_pixel_pitch = 2;
178 #endif
179
180     /* Since OpenGL can do rescaling for us, stick to the default
181      * coordinates and aspect. */
182     p_vout->output.i_width  = p_vout->render.i_width;
183     p_vout->output.i_height = p_vout->render.i_height;
184     p_vout->output.i_aspect = p_vout->render.i_aspect;
185
186     /* We do need a drawable to work properly */
187     vlc_value_t value_drawable;
188     var_Create( p_vout, "drawable-gl", VLC_VAR_DOINHERIT );
189     var_Get( p_vout, "drawable-gl", &value_drawable );
190
191     p_vout->p_sys->o_cocoa_container = (id) value_drawable.i_int;
192     
193     p_vout->fmt_out = p_vout->fmt_in;
194     p_vout->fmt_out.i_chroma = p_vout->output.i_chroma;
195
196     /* We know the chroma, allocate two buffer which will be used
197      * directly by the decoder */
198     int i;
199     for( i = 0; i < 2; i++ )
200     {
201         p_sys->pp_buffer[i] =
202             malloc( p_sys->i_tex_width * p_sys->i_tex_height * i_pixel_pitch );
203         if( !p_sys->pp_buffer[i] )
204             return VLC_EGENERIC;
205     }
206     p_sys->b_frame_available = false;
207     p_sys->i_index = 0;
208
209     p_vout->p_picture[0].i_planes = 1;
210     p_vout->p_picture[0].p->p_pixels = p_sys->pp_buffer[p_sys->i_index];
211     p_vout->p_picture[0].p->i_lines = p_vout->output.i_height;
212     p_vout->p_picture[0].p->i_visible_lines = p_vout->output.i_height;
213     p_vout->p_picture[0].p->i_pixel_pitch = i_pixel_pitch;
214     p_vout->p_picture[0].p->i_pitch = p_vout->output.i_width *
215         p_vout->p_picture[0].p->i_pixel_pitch;
216     p_vout->p_picture[0].p->i_visible_pitch = p_vout->output.i_width *
217         p_vout->p_picture[0].p->i_pixel_pitch;
218
219     p_vout->p_picture[0].i_status = DESTROYED_PICTURE;
220     p_vout->p_picture[0].i_type   = DIRECT_PICTURE;
221
222     PP_OUTPUTPICTURE[ 0 ] = &p_vout->p_picture[0];
223
224     I_OUTPUTPICTURES = 1;
225     p_sys->autorealease_pool = [[NSAutoreleasePool alloc] init];
226
227     [VLCVoutLayer performSelectorOnMainThread:@selector(autoinitInVout:)
228                              withObject:[NSValue valueWithPointer:p_vout]
229                              waitUntilDone:YES];
230
231     return 0;
232 }
233
234 /*****************************************************************************
235  * End: terminate GLX video thread output method
236  *****************************************************************************/
237 static void End( vout_thread_t *p_vout )
238 {
239     vout_sys_t *p_sys = p_vout->p_sys;
240
241     p_vout->p_sys->b_frame_available = false;
242
243     [p_vout->p_sys->o_cocoa_container performSelectorOnMainThread:@selector(removeVoutLayer:) withObject:p_vout->p_sys->o_layer waitUntilDone:YES];
244
245     // Should be done automatically
246     [p_sys->o_layer release];
247     [p_sys->autorealease_pool release];
248
249     /* Free the texture buffer*/
250     free( p_sys->pp_buffer[0] );
251     free( p_sys->pp_buffer[1] );
252 }
253
254 /*****************************************************************************
255  * Destroy: destroy GLX video thread output method
256  *****************************************************************************
257  * Terminate an output method created by CreateVout
258  *****************************************************************************/
259 static void DestroyVout( vlc_object_t *p_this )
260 {
261     vout_thread_t *p_vout = (vout_thread_t *)p_this;
262     vout_sys_t *p_sys = p_vout->p_sys;
263     free( p_sys );
264 }
265
266 /*****************************************************************************
267  * Manage: handle Sys events
268  *****************************************************************************
269  * This function should be called regularly by video output thread. It returns
270  * a non null value if an error occurred.
271  *****************************************************************************/
272 static int Manage( vout_thread_t *p_vout )
273 {
274     vout_sys_t *p_sys = p_vout->p_sys;
275
276     return VLC_SUCCESS;
277 }
278
279 /*****************************************************************************
280  * Render: render previously calculated output
281  *****************************************************************************/
282 static void Render( vout_thread_t *p_vout, picture_t *p_pic )
283 {
284     vout_sys_t *p_sys = p_vout->p_sys;
285
286     @synchronized( p_sys->o_layer ) /* Make sure the p_sys->glContext isn't edited */
287     {
288         if( p_sys->glContext )
289         {
290             CGLLockContext(p_sys->glContext);
291             CGLSetCurrentContext(p_sys->glContext);
292             int i_new_index;
293             i_new_index = ( p_sys->i_index + 1 ) & 1;
294
295
296             /* Update the texture */
297             glBindTexture( VLCGL_TARGET, p_sys->p_textures[i_new_index] );
298             glTexSubImage2D( VLCGL_TARGET, 0, 0, 0,
299                          p_vout->fmt_out.i_width,
300                          p_vout->fmt_out.i_height,
301                          VLCGL_FORMAT, VLCGL_TYPE, p_sys->pp_buffer[i_new_index] );
302
303             /* Bind to the previous texture for drawing */
304             glBindTexture( VLCGL_TARGET, p_sys->p_textures[p_sys->i_index] );
305
306             /* Switch buffers */
307             p_sys->i_index = i_new_index;
308             p_pic->p->p_pixels = p_sys->pp_buffer[p_sys->i_index];
309             CGLUnlockContext(p_sys->glContext);
310             
311             p_sys->b_frame_available = true;
312         }
313     }
314
315     /* Give a buffer where the image will be rendered */
316     p_pic->p->p_pixels = p_sys->pp_buffer[p_sys->i_index];
317 }
318
319 /*****************************************************************************
320  * DisplayVideo: displays previously rendered output
321  *****************************************************************************/
322 static void DisplayVideo( vout_thread_t *p_vout, picture_t *p_pic )
323 {
324     vout_sys_t *p_sys = p_vout->p_sys;
325     
326     [p_sys->o_layer performSelectorOnMainThread:@selector(display)
327                     withObject:nil waitUntilDone:YES];
328 }
329
330 /*****************************************************************************
331  * Control: control facility for the vout
332  *****************************************************************************/
333 static int Control( vout_thread_t *p_vout, int i_query, va_list args )
334 {
335     vout_sys_t *p_sys = p_vout->p_sys;
336
337     if( p_sys->p_vout->pf_control )
338         return p_sys->p_vout->pf_control( p_sys->p_vout, i_query, args );
339     return VLC_EGENERIC;
340 }
341
342 /*****************************************************************************
343  * InitTextures
344  *****************************************************************************/
345 static int InitTextures( vout_thread_t *p_vout )
346 {
347     vout_sys_t *p_sys = p_vout->p_sys;
348     int i_index;
349
350     glDeleteTextures( 2, p_sys->p_textures );
351     glGenTextures( 2, p_sys->p_textures );
352
353     for( i_index = 0; i_index < 2; i_index++ )
354     {
355         glBindTexture( VLCGL_TARGET, p_sys->p_textures[i_index] );
356
357         /* Set the texture parameters */
358         glTexParameterf( VLCGL_TARGET, GL_TEXTURE_PRIORITY, 1.0 );
359
360         glTexParameteri( VLCGL_TARGET, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE );
361         glTexParameteri( VLCGL_TARGET, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE );
362
363         glTexParameteri( VLCGL_TARGET, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
364         glTexParameteri( VLCGL_TARGET, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
365
366         glTexEnvf( GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE );
367
368         /* Note: It seems that we can't bypass those, and even
369          * disabled they are used. They are the cause of the flickering */
370
371         /* Tell the driver not to make a copy of the texture but to use
372            our buffer */
373         glEnable( GL_UNPACK_CLIENT_STORAGE_APPLE );
374         glPixelStorei( GL_UNPACK_CLIENT_STORAGE_APPLE, GL_TRUE );
375
376         /* Use AGP texturing */
377         glTexParameteri( VLCGL_TARGET, GL_TEXTURE_STORAGE_HINT_APPLE, GL_STORAGE_SHARED_APPLE );
378
379         /* Call glTexImage2D only once, and use glTexSubImage2D later */
380         glTexImage2D( VLCGL_TARGET, 0, 4, p_sys->i_tex_width,
381                       p_sys->i_tex_height, 0, VLCGL_FORMAT, VLCGL_TYPE,
382                       p_sys->pp_buffer[i_index] );
383     }
384
385     return 0;
386 }
387
388 /*****************************************************************************
389  * @implementation VLCVoutLayer
390  */
391 @implementation VLCVoutLayer
392
393 /*****************************************************************************
394  * autoinitInVout: Called from the video thread to create a layer.
395  * The created layer is stored in the p_vout. We do that way because, cocoa
396  * doesn't support layer creation on non-main thread.
397  *****************************************************************************/
398 + (void)autoinitInVout:(NSValue*)arg
399 {
400     vout_thread_t * p_vout = [arg pointerValue];
401     p_vout->p_sys->o_layer = [[VLCVoutLayer layerWithVout:p_vout] retain];
402     [p_vout->p_sys->o_cocoa_container addVoutLayer:p_vout->p_sys->o_layer];
403 }
404
405 + (id)layerWithVout:(vout_thread_t*)_p_vout 
406 {
407     VLCVoutLayer* me = [[[self alloc] init] autorelease];
408     if( me )
409     {
410         me->p_vout = _p_vout;
411         me.asynchronous = NO;
412         me.bounds = CGRectMake( 0.0, 0.0, 
413                                 (float)_p_vout->fmt_in.i_visible_width * _p_vout->fmt_in.i_sar_num,
414                                 (float)_p_vout->fmt_in.i_visible_height * _p_vout->fmt_in.i_sar_den );
415     }
416     return me;
417 }
418
419 - (BOOL)canDrawInCGLContext:(CGLContextObj)glContext pixelFormat:(CGLPixelFormatObj)pixelFormat forLayerTime:(CFTimeInterval)timeInterval displayTime:(const CVTimeStamp *)timeStamp
420 {
421     /* Only draw the frame if we have a frame that was previously rendered */
422         return p_vout->p_sys->b_frame_available; // Flag is cleared by drawInCGLContext:pixelFormat:forLayerTime:displayTime:
423 }
424
425 - (void)drawInCGLContext:(CGLContextObj)glContext pixelFormat:(CGLPixelFormatObj)pixelFormat forLayerTime:(CFTimeInterval)timeInterval displayTime:(const CVTimeStamp *)timeStamp
426 {
427     CGLLockContext( glContext );
428     CGLSetCurrentContext( glContext );
429
430     float f_width, f_height, f_x, f_y;
431
432     f_x = (float)p_vout->fmt_out.i_x_offset;
433     f_y = (float)p_vout->fmt_out.i_y_offset;
434     f_width = (float)p_vout->fmt_out.i_x_offset +
435               (float)p_vout->fmt_out.i_visible_width;
436     f_height = (float)p_vout->fmt_out.i_y_offset +
437                (float)p_vout->fmt_out.i_visible_height;
438
439     glClear( GL_COLOR_BUFFER_BIT );
440
441     glEnable( VLCGL_TARGET );
442     glBegin( GL_POLYGON );
443     glTexCoord2f( f_x, f_y ); glVertex2f( -1.0, 1.0 );
444     glTexCoord2f( f_width, f_y ); glVertex2f( 1.0, 1.0 );
445     glTexCoord2f( f_width, f_height ); glVertex2f( 1.0, -1.0 );
446     glTexCoord2f( f_x, f_height ); glVertex2f( -1.0, -1.0 );
447     glEnd();
448
449     glDisable( VLCGL_TARGET );
450
451     glFlush();
452
453     CGLUnlockContext( glContext );
454 }
455
456 - (CGLContextObj)copyCGLContextForPixelFormat:(CGLPixelFormatObj)pixelFormat
457 {
458     CGLContextObj context = [super copyCGLContextForPixelFormat:pixelFormat];
459
460     CGLLockContext( context );
461
462     CGLSetCurrentContext( context );
463
464     /* Swap buffers only during the vertical retrace of the monitor.
465     http://developer.apple.com/documentation/GraphicsImaging/
466     Conceptual/OpenGL/chap5/chapter_5_section_44.html */
467
468     GLint params = 1;
469     CGLSetParameter( CGLGetCurrentContext(), kCGLCPSwapInterval,
470                      &params );
471
472     InitTextures( p_vout );
473
474     glDisable( GL_BLEND );
475     glDisable( GL_DEPTH_TEST );
476     glDepthMask( GL_FALSE );
477     glDisable( GL_CULL_FACE) ;
478     glClearColor( 0.0f, 0.0f, 0.0f, 1.0f );
479     glClear( GL_COLOR_BUFFER_BIT );
480
481     CGLUnlockContext( context );
482     @synchronized( self )
483     {
484         p_vout->p_sys->glContext = context;
485     }
486
487     return context;
488 }
489
490 - (void)releaseCGLContext:(CGLContextObj)glContext
491 {
492     @synchronized( self )
493     {
494         p_vout->p_sys->glContext = nil;
495     }
496
497     CGLLockContext( glContext );
498     CGLSetCurrentContext( glContext );
499
500     glDeleteTextures( 2, p_vout->p_sys->p_textures );
501
502     CGLUnlockContext( glContext );
503 }
504 @end