video_filter: fix warnings
[vlc.git] / modules / video_filter / gradient.c
1 /*****************************************************************************
2  * gradient.c : Gradient and edge detection video effects plugin for vlc
3  *****************************************************************************
4  * Copyright (C) 2000-2006 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Samuel Hocevar <sam@zoy.org>
8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23  *****************************************************************************/
24
25 /*****************************************************************************
26  * Preamble
27  *****************************************************************************/
28
29 #include <math.h>                                            /* sin(), cos() */
30
31 #ifdef HAVE_CONFIG_H
32 # include "config.h"
33 #endif
34
35 #include <vlc/vlc.h>
36 #include <vlc_sout.h>
37 #include <vlc_vout.h>
38
39 #include "vlc_filter.h"
40
41 enum { GRADIENT, EDGE, HOUGH };
42
43 /*****************************************************************************
44  * Local prototypes
45  *****************************************************************************/
46 static int  Create    ( vlc_object_t * );
47 static void Destroy   ( vlc_object_t * );
48
49 static picture_t *Filter( filter_t *, picture_t * );
50 static int GradientCallback( vlc_object_t *, char const *,
51                              vlc_value_t, vlc_value_t,
52                              void * );
53
54 static void FilterGradient( filter_t *, picture_t *, picture_t * );
55 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
56 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
57
58 /*****************************************************************************
59  * Module descriptor
60  *****************************************************************************/
61 #define MODE_TEXT N_("Distort mode")
62 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
63
64 #define GRADIENT_TEXT N_("Gradient image type")
65 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
66         "turn the image to white while 1 will keep colors." )
67
68 #define CARTOON_TEXT N_("Apply cartoon effect")
69 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
70     "\"gradient\" and \"edge\".")
71
72 static const char *mode_list[] = { "gradient", "edge", "hough" };
73 static const char *mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
74
75 #define FILTER_PREFIX "gradient-"
76
77 vlc_module_begin();
78     set_description( _("Gradient video filter") );
79     set_shortname( _( "Gradient" ));
80     set_capability( "video filter2", 0 );
81     set_category( CAT_VIDEO );
82     set_subcategory( SUBCAT_VIDEO_VFILTER );
83
84     add_string( FILTER_PREFIX "mode", "gradient", NULL,
85                 MODE_TEXT, MODE_LONGTEXT, VLC_FALSE );
86         change_string_list( mode_list, mode_list_text, 0 );
87
88     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
89                 GRADIENT_TEXT, GRADIENT_LONGTEXT, VLC_FALSE );
90     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
91                 CARTOON_TEXT, CARTOON_LONGTEXT, VLC_FALSE );
92
93     add_shortcut( "gradient" );
94     set_callbacks( Create, Destroy );
95 vlc_module_end();
96
97 static const char *ppsz_filter_options[] = {
98     "mode", "type", "cartoon", NULL
99 };
100
101 /*****************************************************************************
102  * vout_sys_t: Distort video output method descriptor
103  *****************************************************************************
104  * This structure is part of the video output thread descriptor.
105  * It describes the Distort specific properties of an output thread.
106  *****************************************************************************/
107 struct filter_sys_t
108 {
109     int i_mode;
110
111     /* For the gradient mode */
112     int i_gradient_type;
113     vlc_bool_t b_cartoon;
114
115     uint32_t *p_buf32;
116     uint32_t *p_buf32_bis;
117     uint8_t *p_buf8;
118
119     /* For hough mode */
120     int *p_pre_hough;
121 };
122
123 /*****************************************************************************
124  * Create: allocates Distort video thread output method
125  *****************************************************************************
126  * This function allocates and initializes a Distort vout method.
127  *****************************************************************************/
128 static int Create( vlc_object_t *p_this )
129 {
130     filter_t *p_filter = (filter_t *)p_this;
131     char *psz_method;
132
133     /* Allocate structure */
134     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
135     if( p_filter->p_sys == NULL )
136     {
137         msg_Err( p_filter, "out of memory" );
138         return VLC_ENOMEM;
139     }
140
141     p_filter->pf_video_filter = Filter;
142
143     p_filter->p_sys->p_pre_hough = NULL;
144
145     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
146                    p_filter->p_cfg );
147
148     if( !(psz_method =
149         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
150     {
151         msg_Err( p_filter, "configuration variable "
152                  FILTER_PREFIX "mode empty" );
153         p_filter->p_sys->i_mode = GRADIENT;
154     }
155     else
156     {
157         if( !strcmp( psz_method, "gradient" ) )
158         {
159             p_filter->p_sys->i_mode = GRADIENT;
160         }
161         else if( !strcmp( psz_method, "edge" ) )
162         {
163             p_filter->p_sys->i_mode = EDGE;
164         }
165         else if( !strcmp( psz_method, "hough" ) )
166         {
167             p_filter->p_sys->i_mode = HOUGH;
168         }
169         else
170         {
171             msg_Err( p_filter, "no valid gradient mode provided (%s)", psz_method );
172             p_filter->p_sys->i_mode = GRADIENT;
173         }
174     }
175     free( psz_method );
176
177     p_filter->p_sys->i_gradient_type =
178         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
179     p_filter->p_sys->b_cartoon =
180         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
181
182     var_AddCallback( p_filter, FILTER_PREFIX "mode",
183                      GradientCallback, p_filter->p_sys );
184     var_AddCallback( p_filter, FILTER_PREFIX "type",
185                      GradientCallback, p_filter->p_sys );
186     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
187                      GradientCallback, p_filter->p_sys );
188
189     p_filter->p_sys->p_buf32 = NULL;
190     p_filter->p_sys->p_buf32_bis = NULL;
191     p_filter->p_sys->p_buf8 = NULL;
192
193     return VLC_SUCCESS;
194 }
195
196 /*****************************************************************************
197  * Destroy: destroy Distort video thread output method
198  *****************************************************************************
199  * Terminate an output method created by DistortCreateOutputMethod
200  *****************************************************************************/
201 static void Destroy( vlc_object_t *p_this )
202 {
203     filter_t *p_filter = (filter_t *)p_this;
204
205     free( p_filter->p_sys->p_buf32 );
206     free( p_filter->p_sys->p_buf32_bis );
207     free( p_filter->p_sys->p_buf8 );
208     free( p_filter->p_sys->p_pre_hough );
209
210     free( p_filter->p_sys );
211 }
212
213 /*****************************************************************************
214  * Render: displays previously rendered output
215  *****************************************************************************
216  * This function send the currently rendered image to Distort image, waits
217  * until it is displayed and switch the two rendering buffers, preparing next
218  * frame.
219  *****************************************************************************/
220 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
221 {
222     picture_t *p_outpic;
223
224     if( !p_pic ) return NULL;
225
226     p_outpic = p_filter->pf_vout_buffer_new( p_filter );
227     if( !p_outpic )
228     {
229         msg_Warn( p_filter, "can't get output picture" );
230         if( p_pic->pf_release )
231             p_pic->pf_release( p_pic );
232         return NULL;
233     }
234
235     switch( p_filter->p_sys->i_mode )
236     {
237         case EDGE:
238             FilterEdge( p_filter, p_pic, p_outpic );
239             break;
240
241         case GRADIENT:
242             FilterGradient( p_filter, p_pic, p_outpic );
243             break;
244
245         case HOUGH:
246             FilterHough( p_filter, p_pic, p_outpic );
247             break;
248
249         default:
250             break;
251     }
252
253     p_outpic->date = p_pic->date;
254     p_outpic->b_force = p_pic->b_force;
255     p_outpic->i_nb_fields = p_pic->i_nb_fields;
256     p_outpic->b_progressive = p_pic->b_progressive;
257     p_outpic->b_top_field_first = p_pic->b_top_field_first;
258
259     if( p_pic->pf_release )
260         p_pic->pf_release( p_pic );
261
262     return p_outpic;
263 }
264
265 /*****************************************************************************
266  * Gaussian Convolution
267  *****************************************************************************
268  *    Gaussian convolution ( sigma == 1.4 )
269  *
270  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
271  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
272  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
273  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
274  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
275  *****************************************************************************/
276 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
277 {
278     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
279     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
280     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
281     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
282
283     int x,y;
284     for( y = 2; y < i_num_lines - 2; y++ )
285     {
286         for( x = 2; x < i_src_visible - 2; x++ )
287         {
288             p_smooth[y*i_src_visible+x] = (uint32_t)(
289               /* 2 rows up */
290                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
291               + ((p_inpix[(y-2)*i_src_pitch+x-1]
292               +   p_inpix[(y-2)*i_src_pitch+x]
293               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
294               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
295               /* 1 row up */
296               + ((p_inpix[(y-1)*i_src_pitch+x-2]
297               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
298               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
299               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
300               +   p_inpix[(y-1)*i_src_pitch+x+2]
301               /* */
302               +   p_inpix[y*i_src_pitch+x-2]
303               + ( p_inpix[y*i_src_pitch+x-1]*3 )
304               + ( p_inpix[y*i_src_pitch+x]<<2 )
305               + ( p_inpix[y*i_src_pitch+x+1]*3 )
306               +   p_inpix[y*i_src_pitch+x+2]
307               /* 1 row down */
308               +   p_inpix[(y+1)*i_src_pitch+x-2]
309               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
310               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
311               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
312               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
313               /* 2 rows down */
314               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
315               + ((p_inpix[(y+2)*i_src_pitch+x-1]
316               +   p_inpix[(y+2)*i_src_pitch+x]
317               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
318               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
319               ) >> 6 /* 115 */;
320         }
321     }
322 }
323
324 /*****************************************************************************
325  * FilterGradient: Sobel
326  *****************************************************************************/
327 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
328                                                 picture_t *p_outpic )
329 {
330     int x, y;
331     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
332     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
333     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
334     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
335
336     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
337     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
338
339     uint32_t *p_smooth;
340     if( !p_filter->p_sys->p_buf32 )
341         p_filter->p_sys->p_buf32 =
342         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
343     p_smooth = p_filter->p_sys->p_buf32;
344
345     if( !p_smooth ) return;
346
347     if( p_filter->p_sys->b_cartoon )
348     {
349         p_filter->p_libvlc->pf_memcpy( p_outpic->p[U_PLANE].p_pixels,
350             p_inpic->p[U_PLANE].p_pixels,
351             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
352         p_filter->p_libvlc->pf_memcpy( p_outpic->p[V_PLANE].p_pixels,
353             p_inpic->p[V_PLANE].p_pixels,
354             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
355     }
356     else
357     {
358         p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
359             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
360         p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
361             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
362     }
363
364     GaussianConvolution( p_inpic, p_smooth );
365
366     /* Sobel gradient
367
368      | -1 0 1 |     |  1  2  1 |
369      | -2 0 2 | and |  0  0  0 |
370      | -1 0 1 |     | -1 -2 -1 | */
371
372 #define FOR                                                     \
373     for( y = 1; y < i_num_lines - 1; y++ )                      \
374     {                                                           \
375         for( x = 1; x < i_src_visible - 1; x++ )                \
376         {                                                       \
377             const uint32_t a =                                  \
378             (                                                   \
379               abs(                                              \
380                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
381                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
382                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
383                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
384                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
385                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
386               )                                                 \
387             +                                                   \
388               abs(                                              \
389                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
390                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
391                + ( ( p_smooth[y*i_src_visible+x-1]              \
392                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
393                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
394                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
395               )                                                 \
396             );
397     if( p_filter->p_sys->i_gradient_type )
398     {
399         if( p_filter->p_sys->b_cartoon )
400         {
401             FOR
402             if( a > 60 )
403             {
404                 p_outpix[y*i_dst_pitch+x] = 0x00;
405             }
406             else
407             {
408                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
409                     p_outpix[y*i_dst_pitch+x] =
410                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
411                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
412                     p_outpix[y*i_dst_pitch+x] =
413                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
414                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
415                     p_outpix[y*i_dst_pitch+x] =
416                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
417                 else
418                     p_outpix[y*i_dst_pitch+x] =
419                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
420             }
421             }}
422         }
423         else
424         {
425             FOR
426             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
427             }}
428         }
429     }
430     else
431     {
432         FOR
433         if( a>>8 )
434             p_outpix[y*i_dst_pitch+x] = 0;
435         else
436             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
437         }}
438     }
439 #undef FOR
440 }
441
442 /*****************************************************************************
443  * FilterEdge: Canny edge detection algorithm
444  *****************************************************************************
445  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
446  * (well ... my implementation isn't really the canny algorithm ... but some
447  * ideas are the same)
448  *****************************************************************************/
449 /* angle : | */
450 #define THETA_Y 0
451 /* angle : - */
452 #define THETA_X 1
453 /* angle : / */
454 #define THETA_P 2
455 /* angle : \ */
456 #define THETA_M 3
457 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
458                                             picture_t *p_outpic )
459 {
460     int x, y;
461
462     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
463     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
464     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
465     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
466
467     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
468     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
469
470     uint32_t *p_smooth;
471     uint32_t *p_grad;
472     uint8_t *p_theta;
473
474     if( !p_filter->p_sys->p_buf32 )
475         p_filter->p_sys->p_buf32 =
476         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
477     p_smooth = p_filter->p_sys->p_buf32;
478
479     if( !p_filter->p_sys->p_buf32_bis )
480         p_filter->p_sys->p_buf32_bis =
481         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
482     p_grad = p_filter->p_sys->p_buf32_bis;
483
484     if( !p_filter->p_sys->p_buf8 )
485         p_filter->p_sys->p_buf8 =
486         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
487     p_theta = p_filter->p_sys->p_buf8;
488
489     if( !p_smooth || !p_grad || !p_theta ) return;
490
491     if( p_filter->p_sys->b_cartoon )
492     {
493         p_filter->p_libvlc->pf_memcpy( p_outpic->p[U_PLANE].p_pixels,
494             p_inpic->p[U_PLANE].p_pixels,
495             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
496         p_filter->p_libvlc->pf_memcpy( p_outpic->p[V_PLANE].p_pixels,
497             p_inpic->p[V_PLANE].p_pixels,
498             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
499     }
500     else
501     {
502         p_filter->p_libvlc->pf_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
503               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
504         p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
505             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
506         p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
507             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
508     }
509
510     GaussianConvolution( p_inpic, p_smooth );
511
512     /* Sobel gradient
513
514      | -1 0 1 |     |  1  2  1 |
515      | -2 0 2 | and |  0  0  0 |
516      | -1 0 1 |     | -1 -2 -1 | */
517
518     for( y = 1; y < i_num_lines - 1; y++ )
519     {
520         for( x = 1; x < i_src_visible - 1; x++ )
521         {
522
523             const int gradx =
524                  ( p_smooth[(y-1)*i_src_visible+x-1]
525                    - p_smooth[(y+1)*i_src_visible+x-1] )
526                + ( ( p_smooth[(y-1)*i_src_visible+x]
527                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
528                + ( p_smooth[(y-1)*i_src_visible+x+1]
529                    - p_smooth[(y+1)*i_src_visible+x+1] );
530             const int grady =
531                  ( p_smooth[(y-1)*i_src_visible+x-1]
532                    - p_smooth[(y-1)*i_src_visible+x+1] )
533                + ( ( p_smooth[y*i_src_visible+x-1]
534                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
535                + ( p_smooth[(y+1)*i_src_visible+x-1]
536                    - p_smooth[(y+1)*i_src_visible+x+1] );
537
538             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
539
540             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
541              * tan( 26,565051177 ) = 0.5
542              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
543              * tan( 63,434948823 ) 2 */
544             if( (grady<<1) > gradx )
545                 p_theta[y*i_src_visible+x] = THETA_P;
546             else if( (grady<<1) < -gradx )
547                 p_theta[y*i_src_visible+x] = THETA_M;
548             else if( !gradx || abs(grady) > abs(gradx)<<1 )
549                 p_theta[y*i_src_visible+x] = THETA_Y;
550             else
551                 p_theta[y*i_src_visible+x] = THETA_X;
552         }
553     }
554
555     /* edge computing */
556     for( y = 1; y < i_num_lines - 1; y++ )
557     {
558         for( x = 1; x < i_src_visible - 1; x++ )
559         {
560             if( p_grad[y*i_src_visible+x] > 40 )
561             {
562                 switch( p_theta[y*i_src_visible+x] )
563                 {
564                     case THETA_Y:
565                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
566                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
567                         {
568                             p_outpix[y*i_dst_pitch+x] = 0;
569                             break;
570                         } else goto colorize;
571                     case THETA_P:
572                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
573                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
574                         {
575                             p_outpix[y*i_dst_pitch+x] = 0;
576                             break;
577                         } else goto colorize;
578                     case THETA_M:
579                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
580                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
581                         {
582                             p_outpix[y*i_dst_pitch+x] = 0;
583                             break;
584                         } else goto colorize;
585                     case THETA_X:
586                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
587                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
588                         {
589                             p_outpix[y*i_dst_pitch+x] = 0;
590                             break;
591                         } else goto colorize;
592                 }
593             }
594             else
595             {
596                 colorize:
597                 if( p_filter->p_sys->b_cartoon )
598                 {
599                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
600                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
601                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
602                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
603                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
604                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
605                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
606                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
607                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
608                     else
609                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
610                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
611                 }
612             }
613         }
614     }
615 }
616
617 /*****************************************************************************
618  * FilterHough
619  *****************************************************************************/
620 #define p_pre_hough p_filter->p_sys->p_pre_hough
621 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
622                                              picture_t *p_outpic )
623 {
624     int x, y, i;
625     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
626     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
627     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
628
629     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
630
631     int i_diag = sqrt( i_num_lines * i_num_lines +
632                         i_src_visible * i_src_visible);
633     int i_max, i_phi_max, i_rho, i_rho_max;
634     int i_nb_steps = 90;
635     double d_step = M_PI / i_nb_steps;
636     double d_sin;
637     double d_cos;
638     uint32_t *p_smooth;
639     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
640     if( ! p_hough ) return;
641     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
642     if( !p_smooth ) return;
643
644     if( ! p_pre_hough )
645     {
646         msg_Dbg(p_filter, "Starting precalculation");
647         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
648         if( ! p_pre_hough ) return;
649         for( i = 0 ; i < i_nb_steps ; i++)
650         {
651             d_sin = sin(d_step * i);
652             d_cos = cos(d_step * i);
653             for( y = 0 ; y < i_num_lines ; y++ )
654                 for( x = 0 ; x < i_src_visible ; x++ )
655                 {
656                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
657                         ceil(x*d_sin + y*d_cos);
658                 }
659         }
660         msg_Dbg(p_filter, "Precalculation done");
661     }
662
663     p_filter->p_libvlc->pf_memset( p_hough, 0,
664                                    i_diag * i_nb_steps * sizeof(int) );
665
666     p_filter->p_libvlc->pf_memcpy(
667         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
668         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
669     p_filter->p_libvlc->pf_memcpy(
670         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
671         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
672     p_filter->p_libvlc->pf_memcpy(
673         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
674         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
675
676     GaussianConvolution( p_inpic, p_smooth );
677
678     /* Sobel gradient
679
680      | -1 0 1 |     |  1  2  1 |
681      | -2 0 2 | and |  0  0  0 |
682      | -1 0 1 |     | -1 -2 -1 | */
683
684     i_max = 0;
685     i_rho_max = 0;
686     i_phi_max = 0;
687     for( y = 4; y < i_num_lines - 4; y++ )
688     {
689         for( x = 4; x < i_src_visible - 4; x++ )
690         {
691             uint32_t a =
692             (
693               abs(
694                 ( ( p_smooth[(y-1)*i_src_visible+x]
695                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
696                + ( p_smooth[(y-1)*i_src_visible+x-1]
697                    - p_smooth[(y+1)*i_src_visible+x-1] )
698                + ( p_smooth[(y-1)*i_src_visible+x+1]
699                    - p_smooth[(y+1)*i_src_visible+x+1] )
700               )
701             +
702               abs(
703                 ( ( p_smooth[y*i_src_visible+x-1]
704                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
705                + ( p_smooth[(y-1)*i_src_visible+x-1]
706                    - p_smooth[(y-1)*i_src_visible+x+1] )
707                + ( p_smooth[(y+1)*i_src_visible+x-1]
708                    - p_smooth[(y+1)*i_src_visible+x+1] )
709               )
710             );
711             if( a>>8 )
712             {
713                 for( i = 0 ; i < i_nb_steps ; i ++ )
714                 {
715                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
716                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
717                     {
718                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
719                         i_rho_max = i_rho;
720                         i_phi_max = i;
721                     }
722                 }
723             }
724         }
725     }
726
727     d_sin = sin(i_phi_max*d_step);
728     d_cos = cos(i_phi_max*d_step);
729     if( d_cos != 0 )
730     {
731         for( x = 0 ; x < i_src_visible ; x++ )
732         {
733             y = (i_rho_max - x * d_sin) / d_cos;
734             if( y >= 0 && y < i_num_lines )
735                 p_outpix[y*i_dst_pitch+x] = 255;
736         }
737     }
738
739     if( p_hough ) free( p_hough );
740     if( p_smooth ) free( p_smooth );
741 }
742 #undef p_pre_hough
743
744
745 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
746                              vlc_value_t oldval, vlc_value_t newval,
747                              void *p_data )
748 {
749     VLC_UNUSED(oldval);
750     filter_sys_t *p_sys = (filter_sys_t *)p_data;
751     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
752     {
753         if( !strcmp( newval.psz_string, "gradient" ) )
754         {
755             p_sys->i_mode = GRADIENT;
756         }
757         else if( !strcmp( newval.psz_string, "edge" ) )
758         {
759             p_sys->i_mode = EDGE;
760         }
761         else if( !strcmp( newval.psz_string, "hough" ) )
762         {
763             p_sys->i_mode = HOUGH;
764         }
765         else
766         {
767             msg_Err( p_this, "no valid gradient mode provided (%s)", newval.psz_string );
768             p_sys->i_mode = GRADIENT;
769         }
770     }
771     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
772     {
773         p_sys->i_gradient_type = newval.i_int;
774     }
775     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
776     {
777         p_sys->b_cartoon = newval.b_bool;
778     }
779     return VLC_SUCCESS;
780 }