Use the right declaration for threaded functions.
[vlc.git] / modules / video_filter / remoteosd.c
1 /*****************************************************************************
2  * remoteosd.c: remote osd over vnc filter module
3  *****************************************************************************
4  * Copyright (C) 2007-2008 Matthias Bauer
5  * $Id$
6  *
7  * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implid warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
26  * menu generated by a streaming server as overlay for the streamed video.
27  *
28  * The streaming server that implements this is the ffnetdev plugin for VDR.
29  * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
30  * software.
31  * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
32  * video over the network instead of hardware video outputs.
33  * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
34  *
35  * In fact this video-filter is a simple VNC client that could be also used to
36  * connect to a real VNC host.
37  * Only 8-bit color is supported at the moment.
38  * Using password protected VNC hosts is supported but not recommended, because
39  * you need to insert the used password in the plugin configuration page of
40  * VLC configuration in plain text and it's saved in plain text.
41  *****************************************************************************/
42
43 //#define VNC_DEBUG
44
45 /*****************************************************************************
46  * Preamble
47  *****************************************************************************/
48
49 #ifdef HAVE_CONFIG_H
50 # include "config.h"
51 #endif
52
53 #include <vlc_common.h>
54 #include <vlc_plugin.h>
55 #include <vlc_vout.h>
56
57 #include "vlc_filter.h"
58 #include "filter_common.h"
59 #include "vlc_image.h"
60 #include "vlc_osd.h"
61 #include "vlc_keys.h"
62
63 #include <vlc_network.h>
64 #include <gcrypt.h>              /* to encrypt password */
65 #include <vlc_gcrypt.h>
66
67 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
68
69 /*****************************************************************************
70  * Module descriptor
71  *****************************************************************************/
72 #define READ_BUFFER_SIZE 1000000
73
74 #define RMTOSD_HOST_TEXT N_("VNC Host")
75 #define RMTOSD_HOST_LONGTEXT N_( \
76     "VNC hostname or IP address." )
77
78 #define RMTOSD_PORT_TEXT N_("VNC Port")
79 #define RMTOSD_PORT_LONGTEXT N_( \
80     "VNC portnumber." )
81
82 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
83 #define RMTOSD_PASSWORD_LONGTEXT N_( \
84     "VNC password." )
85
86 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
87 #define RMTOSD_UPDATE_LONGTEXT N_( \
88     "In this interval an update from VNC is requested, default every 300 ms. ")
89
90 #define RMTOSD_POLL_TEXT N_("VNC polling")
91 #define RMTOSD_POLL_LONGTEXT N_( \
92     "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
93
94 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
95 #define RMTOSD_MOUSE_LONGTEXT N_( \
96     "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
97
98 #define RMTOSD_KEYS_TEXT N_("Key events")
99 #define RMTOSD_KEYS_LONGTEXT N_( \
100     "Send key events to VNC host." )
101
102 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
103 #define RMTOSD_ALPHA_LONGTEXT N_( \
104     "The transparency of the OSD VNC can be changed by giving a value " \
105     "between 0 and 255. A lower value specifies more transparency a higher " \
106     "means less transparency. The default is being not transparent " \
107     "(value 255) the minimum is fully transparent (value 0)." )
108
109 #define RMTOSD_CFG "rmtosd-"
110
111 #define RMTOSD_UPDATE_MIN     200
112 #define RMTOSD_UPDATE_DEFAULT 1000
113 #define RMTOSD_UPDATE_MAX     300
114
115 static int  CreateFilter ( vlc_object_t * );
116 static void DestroyFilter( vlc_object_t * );
117
118 vlc_module_begin();
119     set_description( N_("Remote-OSD over VNC") );
120     set_capability( "sub filter", 100 );
121     set_shortname( N_("Remote-OSD") );
122     set_category( CAT_VIDEO );
123     set_subcategory( SUBCAT_VIDEO_SUBPIC );
124     add_shortcut( "rmtosd" );
125     set_callbacks( CreateFilter, DestroyFilter );
126
127     add_string( RMTOSD_CFG "host", "myvdr", NULL, RMTOSD_HOST_TEXT,
128         RMTOSD_HOST_LONGTEXT, false );
129     add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF, NULL,
130         RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false );
131     add_password( RMTOSD_CFG "password", "", NULL, RMTOSD_PASSWORD_TEXT,
132         RMTOSD_PASSWORD_LONGTEXT, false );
133     add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
134         RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, NULL, RMTOSD_UPDATE_TEXT,
135         RMTOSD_UPDATE_LONGTEXT, true );
136     add_bool( RMTOSD_CFG "vnc-polling", 0, NULL,
137               RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false );
138     add_bool( RMTOSD_CFG "mouse-events", 0, NULL,
139               RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false );
140     add_bool( RMTOSD_CFG "key-events", 0, NULL,
141               RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false );
142     add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255, NULL,
143         RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true );
144
145 vlc_module_end();
146
147
148 /*****************************************************************************
149  * Local prototypes
150  *****************************************************************************/
151 #define CHALLENGESIZE 16
152 #define MAX_VNC_SERVER_NAME_LENGTH 255
153
154 /* subfilter functions */
155 static subpicture_t *Filter( filter_t *, mtime_t );
156
157 static int MouseEvent ( vlc_object_t *p_this, char const *psz_var,
158                         vlc_value_t oldval, vlc_value_t newval, void *p_data );
159
160 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
161                      vlc_value_t oldval, vlc_value_t newval, void *p_data );
162
163 static void stop_osdvnc ( filter_t *p_filter );
164
165 static void* vnc_worker_thread ( vlc_object_t *p_thread_obj );
166
167 static void* update_request_thread( vlc_object_t *p_thread_obj );
168
169 static bool open_vnc_connection ( filter_t *p_filter );
170
171 static bool handshaking ( filter_t *p_filter );
172
173 static bool process_server_message ( filter_t *p_filter,
174                                      rfbServerToClientMsg *msg );
175
176 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
177                                int r, int g, int b );
178
179 static inline bool fill_rect( filter_sys_t* p_sys,
180                               uint16_t i_x, uint16_t i_y,
181                               uint16_t i_w, uint16_t i_h,
182                               uint8_t i_color );
183 static inline bool copy_rect( filter_sys_t* p_sys,
184                               uint16_t i_x, uint16_t i_y,
185                               uint16_t i_w, uint16_t i_h,
186                               uint16_t i_sx, uint16_t i_sy );
187
188
189 static inline bool raw_line(  filter_sys_t* p_sys,
190                               uint16_t i_x, uint16_t i_y,
191                               uint16_t i_w );
192
193 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
194
195
196 /*****************************************************************************
197  * Sub filter code
198  *****************************************************************************/
199
200 /*****************************************************************************
201  * Local prototypes
202  *****************************************************************************/
203 struct filter_sys_t
204 {
205     VLC_COMMON_MEMBERS
206
207     bool          b_need_update;       /* VNC picture is updated, do update the OSD*/
208     mtime_t       i_vnc_poll_interval; /* Update the OSD menu every n ms */
209
210     uint8_t       i_alpha;             /* alpha transparency value */
211
212     char          *psz_host;           /* VNC host */
213     int           i_port;
214
215     char          *psz_passwd;         /* VNC password */
216
217     bool          b_vnc_poll;          /* Activate VNC polling ? */
218     bool          b_vnc_mouse_events;  /* Send MouseEvents ? */
219     bool          b_vnc_key_events;    /* Send KeyEvents ? */
220
221     bool          b_connection_active; /* Handshaking finished ? */
222
223     vlc_mutex_t   lock;                /* To lock for read/write on picture */
224
225     picture_t     *p_pic;              /* The picture with OSD data from VNC */
226
227     vout_thread_t *p_vout;             /* Pointer to video-out thread */
228
229     int           i_socket;            /* Socket used for VNC */
230
231     uint16_t      i_vnc_width;          /* The with of the VNC screen */
232     uint16_t      i_vnc_height;         /* The height of the VNC screen */
233         uint32_t      i_vnc_pixels;         /* The pixels of the VNC screen */
234
235     bool    b_alpha_from_vnc;    /* Special ffnetdev alpha feature enabled ? */
236
237     char          read_buffer[READ_BUFFER_SIZE];
238
239     bool          b_continue;
240
241     vlc_object_t* p_worker_thread;
242
243     uint8_t       ar_color_table_yuv[256][4];
244 };
245
246 /*****************************************************************************
247  * CreateFilter: Create the filter and open the definition file
248  *****************************************************************************/
249 static int CreateFilter ( vlc_object_t *p_this )
250 {
251     filter_t *p_filter = (filter_t *)p_this;
252     filter_sys_t *p_sys = NULL;
253
254     msg_Dbg( p_filter, "Creating vnc osd filter..." );
255
256     p_filter->p_sys = p_sys = malloc( sizeof(*p_sys) );
257     if( !p_filter->p_sys )
258         return VLC_ENOMEM;
259     memset( p_sys, 0, sizeof(*p_sys) );
260
261     /* Populating struct */
262     vlc_mutex_init( &p_sys->lock );
263     p_sys->b_continue = true;
264     p_sys->i_socket = -1;
265     p_sys->p_pic = NULL;
266
267     p_sys->psz_host = var_CreateGetString( p_this, RMTOSD_CFG "host" );
268     if( EMPTY_STR(p_sys->psz_host) )
269     {
270         msg_Err( p_filter, "unable to get vnc host" );
271         goto error;
272     }
273
274     p_sys->psz_passwd = var_CreateGetString( p_this, RMTOSD_CFG "password" );
275     if( !p_sys->psz_passwd )
276     {
277         msg_Err( p_filter, "unable to get vnc password" );
278         goto error;
279     }
280
281     p_sys->i_port = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "port" );
282
283     p_sys->i_alpha = var_CreateGetIntegerCommand( p_this, RMTOSD_CFG "alpha" );
284
285     /* in miliseconds, 0 disables polling, should not be lower than 100 */
286     p_sys->i_vnc_poll_interval  = var_CreateGetIntegerCommand( p_this,
287                                                        RMTOSD_CFG "update" );
288     if ( p_sys->i_vnc_poll_interval < 100)
289     {
290        p_sys->i_vnc_poll_interval = 100;
291     }
292
293     for ( int i = 0; i < 256; i++ )
294     {
295         p_sys->ar_color_table_yuv[i][0] = 255;
296         p_sys->ar_color_table_yuv[i][1] = 255;
297         p_sys->ar_color_table_yuv[i][2] = 255;
298         p_sys->ar_color_table_yuv[i][3] = 255;
299     }
300
301     p_sys->b_vnc_poll = var_CreateGetBoolCommand( p_this,
302                                             RMTOSD_CFG "vnc-polling" );
303     p_sys->b_vnc_mouse_events = var_CreateGetBoolCommand( p_this,
304                                             RMTOSD_CFG "mouse-events" );
305     p_sys->b_vnc_key_events = var_CreateGetBoolCommand( p_this,
306                                             RMTOSD_CFG "key-events" );
307
308     /* Keep track of OSD Events */
309     p_sys->b_need_update  = false;
310
311     /* Attach subpicture filter callback */
312     p_filter->pf_sub_filter = Filter;
313
314     p_sys->p_vout = vlc_object_find( p_this, VLC_OBJECT_VOUT, FIND_PARENT );
315
316     if( p_sys->p_vout )
317     {
318         var_AddCallback( p_sys->p_vout, "mouse-moved",
319                          MouseEvent, p_this );
320         var_AddCallback( p_sys->p_vout, "mouse-button-down",
321                          MouseEvent, p_this );
322         var_AddCallback( p_sys->p_vout->p_libvlc, "key-pressed",
323                          KeyEvent, p_this );
324     }
325
326     es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_FOURCC( 's','p','u',' ' ) );
327     p_filter->fmt_out.i_priority = 0;
328
329     vlc_gcrypt_init();
330
331     /* create the vnc worker thread */
332     p_sys->p_worker_thread = vlc_object_create( p_this,
333                                                 sizeof( vlc_object_t ) );
334     vlc_object_attach( p_sys->p_worker_thread, p_this );
335     if( vlc_thread_create( p_sys->p_worker_thread, "vnc worker thread",
336                            vnc_worker_thread,
337                            VLC_THREAD_PRIORITY_LOW, false ) )
338     {
339         vlc_object_detach( p_sys->p_worker_thread );
340         vlc_object_release( p_sys->p_worker_thread );
341         msg_Err( p_filter, "cannot spawn vnc message reader thread" );
342         goto error;
343     }
344
345     msg_Dbg( p_filter, "osdvnc filter started" );
346
347     return VLC_SUCCESS;
348
349 error:
350     msg_Err( p_filter, "osdvnc filter discarded" );
351
352     stop_osdvnc( p_filter );
353
354     vlc_mutex_destroy( &p_sys->lock );
355     free( p_sys->psz_host );
356     free( p_sys->psz_passwd );
357     free( p_sys );
358
359     return VLC_EGENERIC;
360 }
361
362 /*****************************************************************************
363  * DestroyFilter: Make a clean exit of this plugin
364  *****************************************************************************/
365 static void DestroyFilter( vlc_object_t *p_this )
366 {
367     filter_t     *p_filter = (filter_t*)p_this;
368     filter_sys_t *p_sys = p_filter->p_sys;
369
370     msg_Dbg( p_filter, "DestroyFilter called." );
371
372     stop_osdvnc( p_filter );
373
374     if( p_sys->p_vout )
375     {
376         var_DelCallback( p_sys->p_vout, "mouse-moved",
377                          MouseEvent, p_this );
378         var_DelCallback( p_sys->p_vout, "mouse-button-down",
379                          MouseEvent, p_this );
380         var_DelCallback( p_sys->p_vout->p_libvlc, "key-pressed",
381                          KeyEvent, p_this );
382
383         vlc_object_release( p_sys->p_vout );
384     }
385
386     var_Destroy( p_this, RMTOSD_CFG "host" );
387     var_Destroy( p_this, RMTOSD_CFG "port" );
388     var_Destroy( p_this, RMTOSD_CFG "password" );
389     var_Destroy( p_this, RMTOSD_CFG "update" );
390     var_Destroy( p_this, RMTOSD_CFG "vnc-polling" );
391     var_Destroy( p_this, RMTOSD_CFG "mouse-events" );
392     var_Destroy( p_this, RMTOSD_CFG "key-events" );
393     var_Destroy( p_this, RMTOSD_CFG "alpha" );
394
395     vlc_mutex_destroy( &p_sys->lock );
396     free( p_sys->psz_host );
397     free( p_sys->psz_passwd );
398     free( p_sys );
399 }
400
401 static void stop_osdvnc ( filter_t *p_filter )
402 {
403     filter_sys_t *p_sys = p_filter->p_sys;
404
405     /* It will unlock socket reading */
406     vlc_object_kill( p_filter );
407
408     /* */
409     if( p_sys->p_worker_thread )
410     {
411         msg_Dbg( p_filter, "joining worker_thread" );
412         vlc_object_kill( p_sys->p_worker_thread );
413         vlc_thread_join( p_sys->p_worker_thread );
414         vlc_object_detach( p_sys->p_worker_thread );
415         vlc_object_release( p_sys->p_worker_thread );
416         msg_Dbg( p_filter, "released worker_thread" );
417     }
418
419     msg_Dbg( p_filter, "osdvnc stopped" );
420 }
421
422 static bool read_exact( filter_t *p_filter,
423                         int i_socket,
424                         char* p_readbuf,
425                         int i_bytes )
426 {
427     return i_bytes == net_Read( p_filter, i_socket, NULL,
428                                   (unsigned char*)p_readbuf,
429                                   i_bytes, true );
430 }
431
432
433 static bool write_exact( filter_t *p_filter,
434                          int i_socket,
435                          char* p_writebuf,
436                          int i_bytes )
437 {
438     return i_bytes == net_Write( p_filter, i_socket, NULL,
439                                   (unsigned char*)p_writebuf, i_bytes );
440 }
441
442 static bool open_vnc_connection ( filter_t *p_filter )
443 {
444     filter_sys_t *p_sys = p_filter->p_sys;
445
446     msg_Dbg( p_filter, "Open socket to vnc server on %s:%u.",
447               p_sys->psz_host, p_sys->i_port );
448
449     p_sys->i_socket = net_ConnectTCP( p_filter, p_sys->psz_host, p_sys->i_port );
450
451     if( p_sys->i_socket < 0 )
452     {
453         msg_Err( p_filter, "Could not open socket" );
454         return false;
455     }
456
457     msg_Dbg( p_filter, "socket is open." );
458
459     return true;
460 }
461
462 static bool handshaking ( filter_t *p_filter )
463 {
464     filter_sys_t *p_sys = p_filter->p_sys;
465
466     msg_Dbg( p_filter, "Reading protocol version" );
467
468     rfbProtocolVersionMsg pv;
469     if ( !read_exact( p_filter, p_sys->i_socket, pv,
470                       sz_rfbProtocolVersionMsg ) )
471     {
472         msg_Err( p_filter, "Could not read version message" );
473         return false;
474     }
475     pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
476
477     msg_Dbg( p_filter, "Server version is %s", pv );
478
479     strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
480
481     if( !write_exact(p_filter, p_sys->i_socket, pv,
482                      sz_rfbProtocolVersionMsg) )
483     {
484         msg_Err( p_filter, "Could not write version message" );
485         return false;
486     }
487
488     msg_Dbg( p_filter, "Reading authentication scheme" );
489     uint32_t i_authScheme;
490     if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authScheme, 4 ) )
491     {
492         msg_Err( p_filter, "Could not read authentication scheme" );
493         return false;
494     }
495     i_authScheme = htonl(i_authScheme);
496
497     msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
498     if ( i_authScheme == rfbConnFailed )
499     {
500         msg_Err( p_filter, "Connection rejected by server" );
501         return false;
502     }
503     if (i_authScheme == rfbVncAuth)
504     {
505         unsigned char challenge[CHALLENGESIZE];
506         if ( !read_exact( p_filter, p_sys->i_socket,
507                           (char*)challenge, CHALLENGESIZE ) )
508         {
509             msg_Err( p_filter, "Could not read password challenge" );
510             return false;
511         }
512
513         vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
514
515         if( !write_exact(p_filter, p_sys->i_socket,
516                          (char*)challenge, CHALLENGESIZE ) )
517         {
518             msg_Err( p_filter, "Could not write password" );
519             return false;
520         }
521         uint32_t i_authResult;
522         if( !read_exact( p_filter, p_sys->i_socket, (char*)&i_authResult, 4 ) )
523         {
524             msg_Err( p_filter, "Could not read authentication result" );
525             return false;
526         }
527         i_authResult = htonl(i_authResult);
528         if (i_authResult != rfbVncAuthOK)
529         {
530             msg_Err( p_filter, "VNC authentication failed" );
531             return false;
532         }
533     }
534
535     msg_Dbg( p_filter, "Writing client init message" );
536     rfbClientInitMsg ci;
537     ci.shared = 1;
538     if( !write_exact( p_filter, p_sys->i_socket,
539                       (char*)&ci, sz_rfbClientInitMsg ) )
540     {
541         msg_Err( p_filter, "Could not write client init message" );
542         return false;
543     }
544
545     msg_Dbg( p_filter, "Reading server init message" );
546     rfbServerInitMsg si;
547     if( !read_exact( p_filter, p_sys->i_socket,
548                      (char*)&si, sz_rfbServerInitMsg ) )
549     {
550         msg_Err( p_filter, "Could not read server init message" );
551         return false;
552     }
553     si.framebufferWidth = htons(si.framebufferWidth);
554     si.framebufferHeight = htons(si.framebufferHeight);
555     si.format.redMax = htons(si.format.redMax);
556     si.format.greenMax = htons(si.format.greenMax);
557     si.format.blueMax = htons(si.format.blueMax);
558
559     p_sys->i_vnc_width = si.framebufferWidth;
560     p_sys->i_vnc_height = si.framebufferHeight;
561
562     msg_Dbg( p_filter, "Servers preferred pixelformat: "
563                         "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
564                         si.framebufferWidth,
565                         si.framebufferHeight,
566                         si.format.redMax,
567                         si.format.greenMax,
568                         si.format.blueMax,
569                         si.format.bitsPerPixel,
570                         si.format.depth,
571                         si.format.trueColour ? "TrueColor" : "Not-TrueColor");
572
573     uint32_t i_nameLength = htonl(si.nameLength);
574     if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
575     {
576         msg_Err( p_filter, "Server name too long" );
577         return false;
578     }
579     char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
580
581     msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
582     if( !read_exact( p_filter, p_sys->i_socket, s_ServerName, i_nameLength ) )
583     {
584         msg_Err( p_filter, "Could not read server name" );
585         return false;
586     }
587     s_ServerName[i_nameLength] = '\0';
588
589     if( strcmp( s_ServerName, "VDR-OSD") == 0 )
590     {
591         msg_Dbg( p_filter, "Server is a VDR" );
592         p_sys->b_alpha_from_vnc = true;
593     }
594     else
595     {
596         msg_Dbg( p_filter, "Server is a normal VNC" );
597         p_sys->b_alpha_from_vnc = false;
598     }
599
600
601     msg_Dbg( p_filter, "Server init message read properly" );
602     msg_Dbg( p_filter, "Server name is %s", s_ServerName );
603
604     msg_Dbg( p_filter, "Writing SetPixelFormat message" );
605
606     rfbSetPixelFormatMsg sp;
607     sp.type = rfbSetPixelFormat;
608     sp.pad1 = sp.pad2 = 0;
609     sp.format.bitsPerPixel = 8;
610     sp.format.depth = 8 ;
611     sp.format.bigEndian = 1;
612     sp.format.trueColour = 0;
613     sp.format.redMax = htons(31);
614     sp.format.greenMax = htons(31);
615     sp.format.blueMax = htons(31);
616     sp.format.redShift = 10;
617     sp.format.greenShift = 5;
618     sp.format.blueShift = 0;
619     sp.format.pad1 = sp.format.pad2 = 0;
620
621     if( !write_exact( p_filter, p_sys->i_socket,
622                       (char*)&sp, sz_rfbSetPixelFormatMsg) )
623     {
624         msg_Err( p_filter, "Could not write SetPixelFormat message" );
625         return false;
626     }
627
628     msg_Dbg( p_filter, "Writing SetEncodings message" );
629
630     rfbSetEncodingsMsg se;
631     se.type = rfbSetEncodings;
632     se.pad = 0;
633     se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
634
635     if( !write_exact( p_filter, p_sys->i_socket,
636                       (char*)&se, sz_rfbSetEncodingsMsg) )
637     {
638         msg_Err( p_filter, "Could not write SetEncodings message begin" );
639         return false;
640     }
641
642     uint32_t i_encoding;
643
644     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
645     i_encoding = htonl(rfbEncodingCopyRect);
646     if( !write_exact( p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
647     {
648         msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
649         return false;
650     }
651
652     msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
653     i_encoding = htonl(rfbEncodingRRE);
654     if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
655     {
656         msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
657         return false;
658     }
659
660     if( p_sys->b_alpha_from_vnc )
661     {
662         msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
663         i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
664                                   * before we swap it */
665         if( !write_exact(p_filter, p_sys->i_socket, (char*)&i_encoding, 4) )
666         {
667             msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
668             return false;
669         }
670     }
671     return true;
672
673 }
674
675 static void* vnc_worker_thread( vlc_object_t *p_thread_obj )
676 {
677     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
678     filter_sys_t *p_sys = p_filter->p_sys;
679     vlc_object_t *p_update_request_thread;
680
681     msg_Dbg( p_filter, "VNC worker thread started" );
682
683     if( !open_vnc_connection ( p_filter ) )
684     {
685         msg_Err( p_filter, "Could not connect to vnc host" );
686         goto exit;
687     }
688
689     if( !handshaking ( p_filter ) )
690     {
691         msg_Err( p_filter, "Error occured while handshaking vnc host" );
692         goto exit;
693     }
694
695     p_sys->b_connection_active = true; /* to enable sending key
696                                             * and mouse events to host */
697
698     /* Create an empty picture for VNC the data */
699     vlc_mutex_lock( &p_sys->lock );
700     p_sys->p_pic = picture_New( VLC_FOURCC('Y','U','V','A'),
701                                 p_sys->i_vnc_width, p_sys->i_vnc_height, VOUT_ASPECT_FACTOR );
702     if( !p_sys->p_pic )
703     {
704         vlc_mutex_unlock( &p_sys->lock );
705         goto exit;
706     }
707         p_sys->i_vnc_pixels = p_sys->i_vnc_width * p_sys->i_vnc_height;
708
709     vlc_mutex_unlock( &p_sys->lock );
710
711     /* create the update request thread */
712     p_update_request_thread = vlc_object_create( p_filter,
713                                                  sizeof( vlc_object_t ) );
714     vlc_object_attach( p_update_request_thread, p_filter );
715     if( vlc_thread_create( p_update_request_thread,
716                            "vnc update request thread",
717                            update_request_thread,
718                            VLC_THREAD_PRIORITY_LOW, false ) )
719     {
720         vlc_object_detach( p_update_request_thread );
721         vlc_object_release( p_update_request_thread );
722         msg_Err( p_filter, "cannot spawn vnc update request thread" );
723         goto exit;
724     }
725
726     /* connection is initialized, now read and handle server messages */
727     while( vlc_object_alive( p_thread_obj ) )
728     {
729         rfbServerToClientMsg msg;
730         int i_msgSize;
731
732         memset( &msg, 0, sizeof(msg) );
733
734         if( !read_exact(p_filter, p_sys->i_socket, (char*)&msg, 1 ) )
735         {
736             msg_Err( p_filter, "Error while waiting for next server message");
737             break;
738         }
739         switch (msg.type)
740         {
741         case rfbFramebufferUpdate:
742             i_msgSize = sz_rfbFramebufferUpdateMsg;
743             break;
744         case rfbSetColourMapEntries:
745             i_msgSize = sz_rfbSetColourMapEntriesMsg;
746             break;
747         case rfbBell:
748             i_msgSize = sz_rfbBellMsg;
749             break;
750         case rfbServerCutText:
751             i_msgSize = sz_rfbServerCutTextMsg;
752             break;
753         case rfbReSizeFrameBuffer:
754             i_msgSize = sz_rfbReSizeFrameBufferMsg;
755             break;
756         default:
757             i_msgSize = 0;
758             msg_Err( p_filter, "Invalid message %u received", msg.type );
759             break;
760         }
761
762         if( i_msgSize <= 0 )
763             break;
764
765         if( --i_msgSize > 0 )
766         {
767             if ( !read_exact( p_filter, p_sys->i_socket,
768                               ((char*)&msg)+1, i_msgSize ) )
769             {
770                 msg_Err( p_filter, "Error while reading message of type %u",
771                          msg.type );
772                 break;
773             }
774         }
775         process_server_message( p_filter, &msg);
776     }
777
778     msg_Dbg( p_filter, "joining update_request_thread" );
779     vlc_object_kill( p_update_request_thread );
780     vlc_thread_join( p_update_request_thread );
781     vlc_object_detach( p_update_request_thread );
782     vlc_object_release( p_update_request_thread );
783     msg_Dbg( p_filter, "released update_request_thread" );
784
785 exit:
786
787     vlc_mutex_lock( &p_sys->lock );
788     p_sys->b_connection_active = false;
789
790     if (p_sys->i_socket >= 0)
791         net_Close(p_sys->i_socket);
792
793     if( p_sys->p_pic )
794         picture_Release( p_sys->p_pic );
795
796     /* It will hide the subtitle */
797     p_sys->b_continue = false;
798     p_sys->b_need_update = true;
799     vlc_mutex_unlock( &p_sys->lock );
800
801     msg_Dbg( p_filter, "VNC message reader thread ended" );
802     return NULL;
803 }
804
805 static void* update_request_thread( vlc_object_t *p_thread_obj )
806 {
807     filter_t* p_filter = (filter_t*)(p_thread_obj->p_parent);
808     filter_sys_t *p_sys = p_filter->p_sys;
809
810     msg_Dbg( p_filter, "VNC update request thread started" );
811
812     rfbFramebufferUpdateRequestMsg udr;
813     udr.type = rfbFramebufferUpdateRequest;
814     udr.incremental = 0;
815     udr.x = 0;
816     udr.y = 0;
817     udr.w = htons(p_sys->i_vnc_width);
818     udr.h = htons(p_sys->i_vnc_height);
819
820     if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
821            sz_rfbFramebufferUpdateRequestMsg) == false)
822     {
823         msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
824         p_sys->b_continue = false;
825         return NULL;
826     }
827
828     udr.incremental = 1;
829     mtime_t i_poll_interval_microsec = p_sys->i_vnc_poll_interval * 1000;
830
831     if( p_sys->b_vnc_poll)
832     {
833         while( vlc_object_alive( p_thread_obj ) )
834         {
835             msleep( i_poll_interval_microsec );
836             if( write_exact(p_filter, p_sys->i_socket, (char*)&udr,
837                    sz_rfbFramebufferUpdateRequestMsg) == false)
838             {
839                 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
840                 break;
841             }
842         }
843         p_sys->b_continue = false;
844     }
845     else
846     {
847         msg_Dbg( p_filter, "VNC polling disabled." );
848     }
849
850     msg_Dbg( p_filter, "VNC update request thread ended" );
851     return NULL;
852 }
853
854 static bool process_server_message ( filter_t *p_filter,
855                                      rfbServerToClientMsg *msg )
856 {
857     filter_sys_t *p_sys = p_filter->p_sys;
858
859     switch (msg->type)
860     {
861     case rfbFramebufferUpdate:
862         {
863             msg->fu.nRects = htons(msg->fu.nRects);
864             rfbFramebufferUpdateRectHeader hdr;
865
866             for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
867             {
868                 if (!read_exact(p_filter, p_sys->i_socket, (char*)&hdr,
869                     sz_rfbFramebufferUpdateRectHeader ) )
870                 {
871                     msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
872                     return false;
873                 }
874                 hdr.r.x = htons(hdr.r.x);
875                 hdr.r.y = htons(hdr.r.y);
876                 hdr.r.w = htons(hdr.r.w);
877                 hdr.r.h = htons(hdr.r.h);
878                 hdr.encoding = htonl(hdr.encoding);
879
880                 switch (hdr.encoding)
881                 {
882                 case rfbEncodingRaw:
883                     {
884                         int i_line;
885                         for (i_line = 0; i_line < hdr.r.h; i_line++)
886                         {
887                             if ( !read_exact( p_filter, p_sys->i_socket,
888                                     p_sys->read_buffer, hdr.r.w ) )
889                             {
890                                 msg_Err( p_filter,
891                                  "Could not read FrameBufferUpdate line data" );
892                                return false;
893                             }
894                             vlc_mutex_lock( &p_sys->lock );
895                             if ( !raw_line( p_sys, hdr.r.x,
896                                             hdr.r.y + i_line,
897                                             hdr.r.w ) )
898                             {
899                                 msg_Err( p_filter, "raw_line failed." );
900                                 vlc_mutex_unlock( &p_sys->lock );
901                                 return false;
902                             }
903                             vlc_mutex_unlock( &p_sys->lock );
904                         }
905                     }
906                     break;
907
908                 case rfbEncodingCopyRect:
909                     {
910                         rfbCopyRect rect;
911
912                         if ( !read_exact( p_filter, p_sys->i_socket,
913                                           (char*)&rect,
914                                           sz_rfbCopyRect ) )
915                         {
916                             msg_Err( p_filter, "Could not read rfbCopyRect" );
917                             return false;
918                         }
919                         rect.srcX = htons( rect.srcX );
920                         rect.srcY = htons( rect.srcY );
921
922                         vlc_mutex_lock( &p_sys->lock );
923                         if ( !copy_rect( p_sys,
924                                          hdr.r.x,   hdr.r.y,
925                                          hdr.r.w,   hdr.r.h,
926                                          rect.srcX, rect.srcY ) )
927                         {
928                             msg_Err( p_filter, "copy_rect failed." );
929                             vlc_mutex_unlock( &p_sys->lock );
930                             return false;
931                         }
932                         vlc_mutex_unlock( &p_sys->lock );
933                     }
934                     break;
935
936                 case rfbEncodingRRE:
937                     {
938                         rfbRREHeader rrehdr;
939                         if ( !read_exact( p_filter, p_sys->i_socket,
940                                           (char*)&rrehdr,
941                                           sz_rfbRREHeader ) )
942                         {
943                             msg_Err( p_filter, "Could not read rfbRREHeader" );
944                             return false;
945                         }
946                         uint8_t i_pixcolor;
947                         if ( !read_exact(p_filter, p_sys->i_socket,
948                                          (char*)&i_pixcolor, 1 ) )
949                         {
950                             msg_Err( p_filter, "Could not read RRE pixcolor" );
951                             return false;
952                         }
953
954                         vlc_mutex_lock( &p_sys->lock );
955                         if ( !fill_rect( p_sys,
956                                         hdr.r.x, hdr.r.y,
957                                         hdr.r.w, hdr.r.h,
958                                         i_pixcolor) )
959                         {
960                             msg_Err( p_filter, "main fill_rect failed." );
961                             vlc_mutex_unlock( &p_sys->lock );
962                             return false;
963                         }
964                         vlc_mutex_unlock( &p_sys->lock );
965
966                         rrehdr.nSubrects = htonl(rrehdr.nSubrects);
967
968                         int i_datasize = rrehdr.nSubrects *
969                                      ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
970                         if ( i_datasize > READ_BUFFER_SIZE )
971                         {
972                             msg_Err( p_filter, "Buffer too small, "
973                                      "need %u bytes", i_datasize );
974                             return false;
975                         }
976                         if ( !read_exact( p_filter, p_sys->i_socket,
977                                        p_sys->read_buffer, i_datasize ) )
978                         {
979                             msg_Err( p_filter,
980                                      "Could not read RRE subrect data" );
981                             return false;
982                         }
983
984                         uint32_t i_subrect;
985                         rfbRectangle* p_subrect;
986                         int i_offset = 0;
987                         vlc_mutex_lock( &p_sys->lock );
988                         for ( i_subrect = 0;
989                               i_subrect < rrehdr.nSubrects; i_subrect++)
990                         {
991                             i_pixcolor = p_sys->read_buffer[i_offset];
992                             i_offset += sizeof(i_pixcolor);
993                             p_subrect =
994                                (rfbRectangle*)(p_sys->read_buffer + i_offset);
995                             i_offset += sz_rfbRectangle;
996
997                             if (!fill_rect( p_sys,
998                                             htons(p_subrect->x) + hdr.r.x,
999                                             htons(p_subrect->y) + hdr.r.y,
1000                                             htons(p_subrect->w),
1001                                             htons(p_subrect->h),
1002                                             i_pixcolor) )
1003                             {
1004                                 msg_Err( p_filter,
1005                                     "subrect %u fill_rect failed.", i_subrect );
1006                                 vlc_mutex_unlock( &p_sys->lock );
1007                                 return false;
1008                             }
1009                         }
1010                         vlc_mutex_unlock( &p_sys->lock );
1011                     }
1012                     break;
1013
1014                 }
1015
1016             }
1017             vlc_mutex_lock( &p_sys->lock );
1018             p_sys->b_need_update = true;
1019             vlc_mutex_unlock( &p_sys->lock );
1020         }
1021         return true;
1022
1023     case rfbSetColourMapEntries:
1024         {
1025             msg->scme.nColours = htons(msg->scme.nColours);
1026             msg->scme.firstColour = htons(msg->scme.firstColour);
1027             int i_datasize;
1028             if ( p_sys->b_alpha_from_vnc == true )
1029             {
1030                 i_datasize = 2 * msg->scme.nColours * 4;
1031             }
1032             else
1033             {
1034                 i_datasize = 2 * msg->scme.nColours * 3;
1035             }
1036             if ( i_datasize > READ_BUFFER_SIZE )
1037             {
1038                 msg_Err( p_filter, "Buffer too small, need %u bytes",
1039                                    i_datasize );
1040                 return false;
1041             }
1042
1043             if ( !read_exact( p_filter, p_sys->i_socket,
1044                               p_sys->read_buffer, i_datasize ) )
1045             {
1046                 msg_Err( p_filter, "Could not read color map data" );
1047                 return false;
1048             }
1049
1050             uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
1051             uint16_t i_offset = 0;
1052             i_alpha = 255;
1053
1054             for (int i = 0; i < msg->scme.nColours; i++)
1055             {
1056                 i_color_index = i+msg->scme.firstColour;
1057                 if ( p_sys->b_alpha_from_vnc == true )
1058                 {
1059                     i_alpha = p_sys->read_buffer[i_offset];
1060                     i_offset += 2;
1061                 }
1062                 i_red   = p_sys->read_buffer[i_offset];
1063                 i_offset += 2;
1064                 i_green = p_sys->read_buffer[i_offset];
1065                 i_offset += 2;
1066                 i_blue  = p_sys->read_buffer[i_offset];
1067                 i_offset += 2;
1068                 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
1069                             &p_sys->ar_color_table_yuv[i_color_index][1],
1070                             &p_sys->ar_color_table_yuv[i_color_index][2],
1071                             i_red,
1072                             i_green,
1073                             i_blue );
1074                 p_sys->ar_color_table_yuv[i][3] = i_alpha;
1075             }
1076         }
1077         return true;
1078
1079     case rfbBell:
1080         msg_Err( p_filter, "rfbBell received" );
1081         return true;
1082
1083     case rfbServerCutText:
1084         msg->sct.length = htons(msg->sct.length);
1085         if ( msg->sct.length > READ_BUFFER_SIZE )
1086         {
1087             msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
1088             return false;
1089         }
1090         if ( !read_exact(p_filter, p_sys->i_socket,
1091                          p_sys->read_buffer, msg->sct.length ) )
1092         {
1093             msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
1094             return false;
1095         }
1096         return true;
1097
1098     case rfbReSizeFrameBuffer:
1099         msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
1100         return false;
1101
1102     default:
1103         msg_Err( p_filter, "Invalid message %u received", msg->type );
1104         return false;
1105     }
1106     return false;
1107 }
1108
1109 /****************************************************************************
1110  * Filter: the whole thing
1111  ****************************************************************************
1112  * This function outputs subpictures at regular time intervals.
1113  ****************************************************************************/
1114 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
1115 {
1116     filter_sys_t *p_sys = p_filter->p_sys;
1117     subpicture_t *p_spu;
1118     subpicture_region_t *p_region;
1119     video_format_t fmt;
1120     picture_t *p_pic;
1121
1122     if( !p_sys->b_need_update )
1123     {
1124         return NULL;
1125     }
1126
1127     vlc_mutex_lock( &p_sys->lock );
1128
1129     p_pic = p_sys->p_pic;
1130
1131     if( !p_pic )
1132     {
1133         vlc_mutex_unlock( &p_sys->lock );
1134         return NULL;
1135     }
1136
1137     /* Allocate the subpicture internal data. */
1138     p_spu = filter_NewSubpicture( p_filter );
1139     if( !p_spu )
1140     {
1141         vlc_mutex_unlock( &p_sys->lock );
1142         return NULL;
1143     }
1144
1145     p_spu->b_absolute = false;
1146     p_spu->i_start = date;
1147     p_spu->i_stop = 0;
1148     p_spu->b_ephemer = true;
1149
1150     if( !p_sys->b_continue )
1151         p_spu->i_stop = p_spu->i_start + 1;
1152
1153     /* Create new SPU region */
1154     memset( &fmt, 0, sizeof(video_format_t) );
1155     fmt.i_chroma = VLC_FOURCC('Y','U','V','A');
1156     fmt.i_aspect = VOUT_ASPECT_FACTOR;
1157     fmt.i_sar_num = fmt.i_sar_den = 1;
1158     fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1159     fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1160     fmt.i_x_offset = fmt.i_y_offset = 0;
1161     p_region = p_spu->pf_create_region( VLC_OBJECT(p_filter), &fmt );
1162     if( !p_region )
1163     {
1164         msg_Err( p_filter, "cannot allocate SPU region" );
1165         p_filter->pf_sub_buffer_del( p_filter, p_spu );
1166         vlc_mutex_unlock( &p_sys->lock );
1167         return NULL;
1168     }
1169
1170     vout_CopyPicture( p_filter, &p_region->picture, p_pic );
1171
1172     p_sys->b_need_update = false;
1173
1174     vlc_mutex_unlock( &p_sys->lock );
1175
1176     /* set to one of the 9 relative locations */
1177     p_region->i_align = 0; /* Center */
1178     p_spu->b_absolute = false;
1179
1180
1181     p_spu->i_x = 0;
1182     p_spu->i_y = 0;
1183     p_spu->i_original_picture_width = 0; /*Let vout core do the horizontal scaling */
1184     p_spu->i_original_picture_height = fmt.i_height;
1185
1186     p_spu->p_region = p_region;
1187
1188     p_spu->i_alpha = ( p_sys->i_alpha );
1189
1190     return p_spu;
1191 }
1192
1193
1194 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1195                                int r, int g, int b )
1196 {
1197     *y = ( ( (  66 * r + 129 * g +  25 * b + 128 ) >> 8 ) + 16 );
1198     *u =   ( ( -38 * r -  74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1199     *v =   ( ( 112 * r -  94 * g -  18 * b + 128 ) >> 8 ) + 128 ;
1200 }
1201
1202 static inline bool fill_rect( filter_sys_t* p_sys,
1203                               uint16_t i_x, uint16_t i_y,
1204                               uint16_t i_w, uint16_t i_h,
1205                               uint8_t i_color)
1206 {
1207     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1208     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1209     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1210     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1211     int i_pitch = p_outY->i_pitch;
1212     int i_lines = p_outY->i_lines;
1213     if ( i_x + i_w > i_pitch)
1214         return false;
1215     if ( i_y + i_h > i_lines)
1216         return false;
1217     int i_line_offset = i_y * i_pitch;
1218     uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1219     uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1220     uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1221     uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1222     for( int i_line = 0; i_line < i_h; i_line++ )
1223     {
1224         for( int i_column = 0; i_column < i_w; i_column++ )
1225         {
1226             int i_total_offset = i_line_offset + i_x + i_column;
1227             p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1228             p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1229             p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1230             p_outA->p_pixels[ i_total_offset ] = i_alpha;
1231         }
1232         i_line_offset += i_pitch;
1233     }
1234     return true;
1235 }
1236
1237 static inline bool copy_rect( filter_sys_t* p_sys,
1238                               uint16_t i_x, uint16_t i_y,
1239                               uint16_t i_w, uint16_t i_h,
1240                               uint16_t i_sx, uint16_t i_sy )
1241 {
1242     plane_t *p_Y = p_sys->p_pic->p+Y_PLANE;
1243     plane_t *p_U = p_sys->p_pic->p+U_PLANE;
1244     plane_t *p_V = p_sys->p_pic->p+V_PLANE;
1245     plane_t *p_A = p_sys->p_pic->p+A_PLANE;
1246
1247     int i_pitch = p_Y->i_pitch;
1248     int i_lines = p_Y->i_lines;
1249
1250     fprintf( stderr, "copy_rect: (%d,%d)+(%d,%d) -> (%d,%d)\n", i_x, i_y, i_w, i_h, i_sx, i_sy );
1251
1252     if( i_x + i_w > i_pitch || i_sx + i_w > i_pitch )
1253         return false;
1254     if( i_y + i_h > i_lines || i_sy + i_h > i_lines)
1255         return false;
1256
1257     if( i_w <= 0 || i_h <= 0 )
1258         return true;
1259
1260     uint8_t *pb_buffer = calloc( i_w * i_h, 4 );
1261     if( !pb_buffer )
1262         return false;
1263
1264     for( int i_line = 0; i_line < i_h; i_line++ )
1265     {
1266         for( int i_column = 0; i_column < i_w; i_column++ )
1267         {
1268             const int i_src_offset = ( i_sy + i_line ) * i_pitch + i_sx + i_column;
1269             const int i_tmp_offset = (    0 + i_line ) *     i_w +    0 + i_column;
1270
1271             pb_buffer[4*i_tmp_offset + 0] = p_Y->p_pixels[i_src_offset];
1272             pb_buffer[4*i_tmp_offset + 1] = p_U->p_pixels[i_src_offset];
1273             pb_buffer[4*i_tmp_offset + 2] = p_V->p_pixels[i_src_offset];
1274             pb_buffer[4*i_tmp_offset + 3] = p_A->p_pixels[i_src_offset];
1275         }
1276     }
1277
1278     for( int i_line = 0; i_line < i_h; i_line++ )
1279     {
1280         for( int i_column = 0; i_column < i_w; i_column++ )
1281         {
1282             const int i_tmp_offset = (   0 + i_line ) *     i_w +   0 + i_column;
1283             const int i_dst_offset = ( i_y + i_line ) * i_pitch + i_x + i_column;
1284
1285             p_Y->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 0];
1286             p_U->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 1];
1287             p_V->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 2];
1288             p_A->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 3];
1289         }
1290     }
1291     free( pb_buffer );
1292     return true;
1293
1294 }
1295
1296 static inline bool raw_line(  filter_sys_t* p_sys,
1297                               uint16_t i_x, uint16_t i_y,
1298                               uint16_t i_w )
1299 {
1300     plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1301     plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1302     plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1303     plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1304     int i_pitch = p_outY->i_pitch;
1305     int i_lines = p_outY->i_lines;
1306     if ( i_x + i_w > i_pitch)
1307         return false;
1308     if ( i_y > i_lines)
1309         return false;
1310
1311     int i_line_offset = i_y * i_pitch + i_x;
1312
1313     for( int i_column = 0; i_column < i_w; i_column++ )
1314     {
1315         int i_offset = i_line_offset + i_column;
1316         uint8_t i_color = p_sys->read_buffer[i_column];
1317         p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1318         p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1319         p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1320         p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1321     }
1322
1323     return true;
1324 }
1325
1326
1327 /*****************************************************************************
1328  * MouseEvent: callback for mouse events
1329  *****************************************************************************/
1330 static int MouseEvent( vlc_object_t *p_this, char const *psz_var,
1331                        vlc_value_t oldval, vlc_value_t newval, void *p_data )
1332 {
1333     VLC_UNUSED(oldval); VLC_UNUSED(newval); VLC_UNUSED(psz_var);
1334
1335     filter_t *p_filter = (filter_t *)p_data;
1336     filter_sys_t *p_sys = p_filter->p_sys;
1337
1338     if( !p_sys->b_vnc_mouse_events )
1339         return VLC_SUCCESS;
1340
1341     vout_thread_t *p_vout = (vout_thread_t*)p_sys->p_vout;
1342     int i_x, i_y;
1343     int i_v;
1344
1345
1346     i_v = var_GetInteger( p_sys->p_vout, "mouse-button-down" );
1347     i_y = var_GetInteger( p_sys->p_vout, "mouse-y" );
1348     i_x = var_GetInteger( p_sys->p_vout, "mouse-x" );
1349
1350     vlc_mutex_lock( &p_sys->lock );
1351
1352     const int v_h = p_vout->fmt_in.i_visible_height;
1353     const int v_w = p_sys->i_vnc_width * v_h / p_sys->i_vnc_height;
1354     const int v_x = (p_vout->fmt_in.i_visible_width-v_w)/2;
1355
1356     i_x -= v_x;
1357
1358     if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1359     {
1360         vlc_mutex_unlock( &p_sys->lock );
1361         msg_Dbg( p_this, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1362         return VLC_SUCCESS;
1363     }
1364     if( !p_sys->b_connection_active )
1365     {
1366         vlc_mutex_unlock( &p_sys->lock );
1367         return VLC_SUCCESS;
1368     }
1369
1370 #ifdef VNC_DEBUG
1371     msg_Dbg( p_this, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1372 #endif
1373
1374     /* */
1375     i_x = i_x * p_sys->i_vnc_width / v_w;
1376     i_y = i_y * p_sys->i_vnc_height / v_h;
1377
1378     /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1379     rfbPointerEventMsg ev;
1380     ev.type = rfbPointerEvent;
1381     ev.buttonMask = i_v;
1382     ev.x = htons(i_x);
1383     ev.y = htons(i_y);
1384
1385     write_exact( p_filter, p_sys->i_socket,
1386                  (char*)&ev, sz_rfbPointerEventMsg);
1387
1388     vlc_mutex_unlock( &p_sys->lock );
1389
1390     return VLC_SUCCESS;
1391 }
1392
1393 /*****************************************************************************
1394  * KeyEvent: callback for keyboard events
1395  *****************************************************************************/
1396 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1397                      vlc_value_t oldval, vlc_value_t newval, void *p_data )
1398 {
1399     VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1400
1401     filter_t *p_filter = (filter_t *)p_data;
1402     filter_sys_t *p_sys = p_filter->p_sys;
1403
1404     if( !p_sys->b_vnc_key_events )
1405         return VLC_SUCCESS;
1406
1407     msg_Dbg( p_this, "key pressed (%d) ", newval.i_int );
1408
1409     if ( !newval.i_int )
1410     {
1411         msg_Err( p_this, "Received invalid key event %d", newval.i_int );
1412         return VLC_EGENERIC;
1413     }
1414
1415     vlc_mutex_lock( &p_sys->lock );
1416     if( !p_sys->b_connection_active )
1417     {
1418         vlc_mutex_unlock( &p_sys->lock );
1419         return VLC_SUCCESS;
1420     }
1421
1422     uint32_t i_key32 = newval.i_int;
1423     i_key32 = htonl(i_key32);
1424     rfbKeyEventMsg ev;
1425
1426     ev.type = rfbKeyEvent;
1427     ev.down = 1;
1428     ev.pad = 0;
1429
1430     /* first key-down for modifier-keys */
1431     if (newval.i_int & KEY_MODIFIER_CTRL)
1432     {
1433         ev.key = 0xffe3;
1434         write_exact( p_filter, p_sys->i_socket,
1435                      (char*)&ev, sz_rfbKeyEventMsg);
1436     }
1437     if (newval.i_int & KEY_MODIFIER_SHIFT)
1438     {
1439         ev.key = 0xffe1;
1440         write_exact( p_filter, p_sys->i_socket,
1441                      (char*)&ev, sz_rfbKeyEventMsg);
1442     }
1443     if (newval.i_int & KEY_MODIFIER_ALT)
1444     {
1445         ev.key = 0xffe9;
1446         write_exact( p_filter, p_sys->i_socket,
1447                      (char*)&ev, sz_rfbKeyEventMsg);
1448     }
1449
1450     /* then key-down for the pressed key */
1451     ev.key = i_key32;
1452     write_exact( p_filter, p_sys->i_socket,
1453                  (char*)&ev, sz_rfbKeyEventMsg);
1454
1455     ev.down = 0;
1456
1457     /* then key-up for the pressed key */
1458     write_exact( p_filter, p_sys->i_socket,
1459                  (char*)&ev, sz_rfbKeyEventMsg);
1460
1461     /* last key-down for modifier-keys */
1462     if (newval.i_int & KEY_MODIFIER_CTRL)
1463     {
1464         ev.key = 0xffe3;
1465         write_exact( p_filter, p_sys->i_socket,
1466                      (char*)&ev, sz_rfbKeyEventMsg);
1467     }
1468     if (newval.i_int & KEY_MODIFIER_SHIFT)
1469     {
1470         ev.key = 0xffe1;
1471         write_exact( p_filter, p_sys->i_socket,
1472                      (char*)&ev, sz_rfbKeyEventMsg);
1473     }
1474     if (newval.i_int & KEY_MODIFIER_ALT)
1475     {
1476        ev.key = 0xffe9;
1477        write_exact( p_filter, p_sys->i_socket,
1478                     (char*)&ev, sz_rfbKeyEventMsg);
1479     }
1480     vlc_mutex_unlock( &p_sys->lock );
1481
1482     return VLC_SUCCESS;
1483 }
1484
1485 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1486 {
1487     unsigned char key[8];
1488     unsigned int i;
1489
1490     for (i = 0; i < 8; i++)
1491         key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1492
1493     gcry_cipher_hd_t ctx;
1494     gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1495
1496     /* reverse bits of the key */
1497     for( i = 0 ; i < 8 ; i ++ )
1498         key[i] =
1499             (key[i] >> 7) +
1500             (((key[i] >> 6) & 0x01 ) << 1 ) +
1501             (((key[i] >> 5) & 0x01 ) << 2 ) +
1502             (((key[i] >> 4) & 0x01 ) << 3 ) +
1503             (((key[i] >> 3) & 0x01 ) << 4 ) +
1504             (((key[i] >> 2) & 0x01 ) << 5 ) +
1505             (((key[i] >> 1) & 0x01 ) << 6 ) +
1506             ((key[i] & 0x01) << 7 );
1507
1508     gcry_cipher_setkey( ctx, key, 8 );
1509     gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1510     gcry_cipher_close( ctx );
1511 }
1512