Fix a segfault (buffer overflow for win32 only).
[vlc.git] / modules / video_output / caca.c
1 /*****************************************************************************
2  * caca.c: Color ASCII Art video output plugin using libcaca
3  *****************************************************************************
4  * Copyright (C) 2003-2009 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Sam Hocevar <sam@zoy.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * Preamble
26  *****************************************************************************/
27
28 #ifdef HAVE_CONFIG_H
29 # include "config.h"
30 #endif
31
32 #include <vlc_common.h>
33 #include <vlc_plugin.h>
34 #include <vlc_vout.h>
35 #include <vlc_interface.h>
36 #include <vlc_playlist.h>
37 #include <vlc_keys.h>
38
39 #include <caca.h>
40
41 #ifndef CACA_API_VERSION_1
42     /* Upward compatibility macros */
43     typedef char cucul_canvas_t;
44     typedef struct caca_bitmap cucul_dither_t;
45     typedef char caca_display_t;
46 #   define CUCUL_COLOR_DEFAULT CACA_COLOR_LIGHTGRAY
47 #   define CUCUL_COLOR_BLACK CACA_COLOR_BLACK
48 #   define cucul_clear_canvas(x) caca_clear()
49 #   define cucul_create_canvas(x,y) "" /* kinda hacky */
50 #   define cucul_create_dither caca_create_bitmap
51 #   define cucul_dither_bitmap(x,y,z,t,u,v,w) caca_draw_bitmap(y,z,t,u,v,w)
52 #   define cucul_free_dither caca_free_bitmap
53 #   define cucul_free_canvas(x)
54 #   define cucul_get_canvas_width(x) caca_get_width()
55 #   define cucul_get_canvas_height(x) caca_get_height()
56 #   define cucul_set_color(x,y,z) caca_set_color(y,z)
57 #   define caca_create_display(x) (caca_init() ? NULL : "") /* hacky, too */
58 #   define caca_free_display(x) caca_end()
59 #   define caca_get_event(x,y,z,t) *(z) = caca_get_event(y)
60 #   define caca_refresh_display(x) caca_refresh()
61 #   define caca_set_display_title(x,y) caca_set_window_title(y)
62 #endif
63
64 /*****************************************************************************
65  * Local prototypes
66  *****************************************************************************/
67 static int  Create    ( vlc_object_t * );
68 static void Destroy   ( vlc_object_t * );
69
70 static int  Init      ( vout_thread_t * );
71 static void End       ( vout_thread_t * );
72 static int  Manage    ( vout_thread_t * );
73 static void Render    ( vout_thread_t *, picture_t * );
74 static void Display   ( vout_thread_t *, picture_t * );
75
76 /*****************************************************************************
77  * Module descriptor
78  *****************************************************************************/
79 vlc_module_begin ()
80     set_shortname( "Caca" )
81     set_category( CAT_VIDEO )
82     set_subcategory( SUBCAT_VIDEO_VOUT )
83     set_description( N_("Color ASCII art video output") )
84     set_capability( "video output", 12 )
85     set_callbacks( Create, Destroy )
86 vlc_module_end ()
87
88 /*****************************************************************************
89  * vout_sys_t: libcaca video output method descriptor
90  *****************************************************************************
91  * This structure is part of the video output thread descriptor.
92  * It describes the libcaca specific properties of an output thread.
93  *****************************************************************************/
94 struct vout_sys_t
95 {
96     cucul_canvas_t *p_cv;
97     caca_display_t *p_dp;
98     cucul_dither_t *p_dither;
99 };
100
101 /*****************************************************************************
102  * Create: allocates libcaca video output thread
103  *****************************************************************************
104  * This function initializes libcaca vout method.
105  *****************************************************************************/
106 static int Create( vlc_object_t *p_this )
107 {
108     vout_thread_t *p_vout = (vout_thread_t *)p_this;
109
110 #if defined( WIN32 ) && !defined( UNDER_CE )
111     CONSOLE_SCREEN_BUFFER_INFO csbiInfo;
112     SMALL_RECT rect;
113     COORD coord;
114     HANDLE hstdout;
115
116     if( !AllocConsole() )
117     {
118         msg_Err( p_vout, "cannot create console" );
119         return VLC_EGENERIC;
120     }
121
122     hstdout =
123         CreateConsoleScreenBuffer( GENERIC_READ | GENERIC_WRITE,
124                                    FILE_SHARE_READ | FILE_SHARE_WRITE,
125                                    NULL, CONSOLE_TEXTMODE_BUFFER, NULL );
126     if( !hstdout || hstdout == INVALID_HANDLE_VALUE )
127     {
128         msg_Err( p_vout, "cannot create screen buffer" );
129         FreeConsole();
130         return VLC_EGENERIC;
131     }
132
133     if( !SetConsoleActiveScreenBuffer( hstdout) )
134     {
135         msg_Err( p_vout, "cannot set active screen buffer" );
136         FreeConsole();
137         return VLC_EGENERIC;
138     }
139
140     coord = GetLargestConsoleWindowSize( hstdout );
141     msg_Dbg( p_vout, "SetConsoleWindowInfo: %ix%i", coord.X, coord.Y );
142
143     /* Force size for now */
144     coord.X = 100;
145     coord.Y = 40;
146
147     if( !SetConsoleScreenBufferSize( hstdout, coord ) )
148         msg_Warn( p_vout, "SetConsoleScreenBufferSize %i %i",
149                   coord.X, coord.Y );
150
151     /* Get the current screen buffer size and window position. */
152     if( GetConsoleScreenBufferInfo( hstdout, &csbiInfo ) )
153     {
154         rect.Top = 0; rect.Left = 0;
155         rect.Right = csbiInfo.dwMaximumWindowSize.X - 1;
156         rect.Bottom = csbiInfo.dwMaximumWindowSize.Y - 1;
157         if( !SetConsoleWindowInfo( hstdout, TRUE, &rect ) )
158             msg_Dbg( p_vout, "SetConsoleWindowInfo failed: %ix%i",
159                      rect.Right, rect.Bottom );
160     }
161 #endif
162
163     /* Allocate structure */
164     p_vout->p_sys = malloc( sizeof( vout_sys_t ) );
165     if( p_vout->p_sys == NULL )
166     {
167 #if defined( WIN32 ) && !defined( UNDER_CE )
168         FreeConsole();
169 #endif
170         return VLC_ENOMEM;
171     }
172
173     p_vout->p_sys->p_cv = cucul_create_canvas(0, 0);
174     if( !p_vout->p_sys->p_cv )
175     {
176         msg_Err( p_vout, "cannot initialize libcucul" );
177 #if defined( WIN32 ) && !defined( UNDER_CE )
178         FreeConsole();
179 #endif
180         free( p_vout->p_sys );
181         return VLC_EGENERIC;
182     }
183
184     p_vout->p_sys->p_dp = caca_create_display( p_vout->p_sys->p_cv );
185     if( !p_vout->p_sys->p_dp )
186     {
187         msg_Err( p_vout, "cannot initialize libcaca" );
188         cucul_free_canvas( p_vout->p_sys->p_cv );
189 #if defined( WIN32 ) && !defined( UNDER_CE )
190         FreeConsole();
191 #endif
192         free( p_vout->p_sys );
193         return VLC_EGENERIC;
194     }
195
196     caca_set_display_title( p_vout->p_sys->p_dp,
197                             VOUT_TITLE " - Colour AsCii Art (caca)" );
198
199     p_vout->pf_init = Init;
200     p_vout->pf_end = End;
201     p_vout->pf_manage = Manage;
202     p_vout->pf_render = Render;
203     p_vout->pf_display = Display;
204
205     return VLC_SUCCESS;
206 }
207
208 /*****************************************************************************
209  * Init: initialize libcaca video output thread
210  *****************************************************************************/
211 static int Init( vout_thread_t *p_vout )
212 {
213     int i_index;
214     picture_t *p_pic = NULL;
215
216     I_OUTPUTPICTURES = 0;
217
218     p_vout->output.i_chroma = VLC_CODEC_RGB32;
219     p_vout->output.i_width = p_vout->render.i_width;
220     p_vout->output.i_height = p_vout->render.i_height;
221     p_vout->output.i_aspect = p_vout->render.i_aspect;
222
223     p_vout->output.i_rmask = 0x00ff0000;
224     p_vout->output.i_gmask = 0x0000ff00;
225     p_vout->output.i_bmask = 0x000000ff;
226
227     /* Create the libcaca dither object */
228     p_vout->p_sys->p_dither = cucul_create_dither
229                        ( 32, p_vout->output.i_width, p_vout->output.i_height,
230                          4 * ((p_vout->output.i_width + 15) & ~15),
231                          p_vout->output.i_rmask, p_vout->output.i_gmask,
232                          p_vout->output.i_bmask, 0x00000000 );
233
234     if( !p_vout->p_sys->p_dither )
235     {
236         msg_Err( p_vout, "could not create libcaca dither object" );
237         return VLC_EGENERIC;
238     }
239
240     /* Find an empty picture slot */
241     for( i_index = 0 ; i_index < VOUT_MAX_PICTURES ; i_index++ )
242     {
243         if( p_vout->p_picture[ i_index ].i_status == FREE_PICTURE )
244         {
245             p_pic = p_vout->p_picture + i_index;
246             break;
247         }
248     }
249
250     if( p_pic == NULL )
251     {
252         return VLC_EGENERIC;
253     }
254
255     /* Allocate the picture */
256     p_pic->p->i_lines = p_vout->output.i_height;
257     p_pic->p->i_visible_lines = p_vout->output.i_height;
258     p_pic->p->i_pitch = 4 * ((p_vout->output.i_width + 15) & ~15);
259     p_pic->p->i_pixel_pitch = 4;
260     p_pic->p->i_visible_pitch = 4 * p_vout->output.i_width;
261     p_pic->i_planes = 1;
262     p_pic->p->p_pixels = malloc( p_pic->p->i_pitch * p_pic->p->i_lines );
263
264     p_pic->i_status = DESTROYED_PICTURE;
265     p_pic->i_type   = DIRECT_PICTURE;
266
267     PP_OUTPUTPICTURE[ I_OUTPUTPICTURES ] = p_pic;
268     I_OUTPUTPICTURES++;
269
270     return VLC_SUCCESS;
271 }
272
273 /*****************************************************************************
274  * End: terminate libcaca video output thread
275  *****************************************************************************/
276 static void End( vout_thread_t *p_vout )
277 {
278     cucul_free_dither( p_vout->p_sys->p_dither );
279 }
280
281 /*****************************************************************************
282  * Destroy: destroy libcaca video output thread
283  *****************************************************************************
284  * Terminate an output method created by AaCreateOutputMethod
285  *****************************************************************************/
286 static void Destroy( vlc_object_t *p_this )
287 {
288     vout_thread_t *p_vout = (vout_thread_t *)p_this;
289
290     caca_free_display( p_vout->p_sys->p_dp );
291     cucul_free_canvas( p_vout->p_sys->p_cv );
292
293 #if defined( WIN32 ) && !defined( UNDER_CE )
294     FreeConsole();
295 #endif
296
297     free( p_vout->p_sys );
298 }
299
300 /*****************************************************************************
301  * Manage: handle libcaca events
302  *****************************************************************************
303  * This function should be called regularly by video output thread. It manages
304  * console events. It returns a non null value on error.
305  *****************************************************************************/
306 static int Manage( vout_thread_t *p_vout )
307 {
308 #ifdef CACA_API_VERSION_1
309     struct caca_event ev;
310 #else
311     int ev;
312 #endif
313
314     while( caca_get_event(p_vout->p_sys->p_dp, CACA_EVENT_ANY, &ev, 0) )
315     {
316         playlist_t *p_playlist;
317         vlc_value_t val;
318
319 #ifdef CACA_API_VERSION_1
320 #ifdef CACA_EVENT_OPAQUE
321         switch( caca_get_event_type( &ev ) )
322 #else
323         switch( ev.type )
324 #endif /* CACA_EVENT_OPAQUE */
325 #else
326         switch( ev )
327 #endif
328         {
329         case CACA_EVENT_KEY_RELEASE:
330 #ifdef CACA_API_VERSION_1
331 #ifdef CACA_EVENT_OPAQUE
332             switch( caca_get_event_key_ch( &ev ) )
333 #else
334             switch( ev.data.key.ch )
335 #endif /* CACA_EVENT_OPAQUE */
336 #else
337             switch( ev & 0x00ffffff )
338 #endif
339             {
340             case 'q':
341                 val.i_int = KEY_MODIFIER_CTRL | 'q';
342                 break;
343             case ' ':
344                 val.i_int = KEY_SPACE;
345                 break;
346             default:
347                 continue;
348             }
349
350             var_Set( p_vout->p_libvlc, "key-pressed", val );
351             break;
352         case CACA_EVENT_RESIZE:
353             /* Acknowledge the resize */
354             caca_refresh_display( p_vout->p_sys->p_dp );
355             break;
356 #ifdef CACA_API_VERSION_1
357         case  CACA_EVENT_MOUSE_MOTION:
358             val.i_int =
359 #ifdef CACA_EVENT_OPAQUE
360                 caca_get_event_mouse_x( &ev )
361 #else
362                 ev.data.mouse.x
363 #endif /* CACA_EVENT_OPAQUE */
364                 * p_vout->render.i_width
365                          / cucul_get_canvas_width( p_vout->p_sys->p_cv );
366             var_Set( p_vout, "mouse-x", val );
367             val.i_int =
368 #ifdef CACA_EVENT_OPAQUE
369                 caca_get_event_mouse_y( &ev ) 
370 #else
371                 ev.data.mouse.y
372 #endif /* CACA_EVENT_OPAQUE */
373                 * p_vout->render.i_height
374                          / cucul_get_canvas_height( p_vout->p_sys->p_cv );
375             var_Set( p_vout, "mouse-y", val );
376             var_SetBool( p_vout, "mouse-moved", true );
377             break;
378         case CACA_EVENT_MOUSE_RELEASE:
379             var_SetBool( p_vout, "mouse-clicked", true );
380             break;
381         case CACA_EVENT_QUIT:
382         {
383             p_playlist = pl_Hold( p_vout );
384             if( p_playlist )
385             {
386                 playlist_Stop( p_playlist );
387                 pl_Release( p_vout );
388             }
389             libvlc_Quit( p_vout->p_libvlc );
390             break;
391         }
392 #endif
393         default:
394             break;
395         }
396     }
397
398     return VLC_SUCCESS;
399 }
400
401 /*****************************************************************************
402  * Render: render previously calculated output
403  *****************************************************************************/
404 static void Render( vout_thread_t *p_vout, picture_t *p_pic )
405 {
406     cucul_set_color_ansi( p_vout->p_sys->p_cv,
407                      CUCUL_COLOR_DEFAULT, CUCUL_COLOR_BLACK );
408     cucul_clear_canvas( p_vout->p_sys->p_cv );
409     cucul_dither_bitmap( p_vout->p_sys->p_cv, 0, 0,
410                          cucul_get_canvas_width( p_vout->p_sys->p_cv ) - 1,
411                          cucul_get_canvas_height( p_vout->p_sys->p_cv ) - 1,
412                          p_vout->p_sys->p_dither, p_pic->p->p_pixels );
413 }
414
415 /*****************************************************************************
416  * Display: displays previously rendered output
417  *****************************************************************************/
418 static void Display( vout_thread_t *p_vout, picture_t *p_pic )
419 {
420     VLC_UNUSED(p_pic);
421     caca_refresh_display( p_vout->p_sys->p_dp );
422 }
423