994734389fbc040c5ef8032ecbf5f6be2bb292e5
[vlc.git] / modules / video_filter / gradient.c
1 /*****************************************************************************
2  * gradient.c : Gradient and edge detection video effects plugin for vlc
3  *****************************************************************************
4  * Copyright (C) 2000-2006 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Samuel Hocevar <sam@zoy.org>
8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23  *****************************************************************************/
24
25 /*****************************************************************************
26  * Preamble
27  *****************************************************************************/
28 #include <stdlib.h>                                      /* malloc(), free() */
29 #include <string.h>
30
31 #include <math.h>                                            /* sin(), cos() */
32
33 #include <vlc/vlc.h>
34 #include <vlc_sout.h>
35 #include <vlc_vout.h>
36
37 #include "vlc_filter.h"
38
39 enum { GRADIENT, EDGE, HOUGH };
40
41 /*****************************************************************************
42  * Local prototypes
43  *****************************************************************************/
44 static int  Create    ( vlc_object_t * );
45 static void Destroy   ( vlc_object_t * );
46
47 static picture_t *Filter( filter_t *, picture_t * );
48 static int GradientCallback( vlc_object_t *, char const *,
49                              vlc_value_t, vlc_value_t,
50                              void * );
51
52 static void FilterGradient( filter_t *, picture_t *, picture_t * );
53 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
54 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
55
56 /*****************************************************************************
57  * Module descriptor
58  *****************************************************************************/
59 #define MODE_TEXT N_("Distort mode")
60 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
61
62 #define GRADIENT_TEXT N_("Gradient image type")
63 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
64         "turn the image to white while 1 will keep colors." )
65
66 #define CARTOON_TEXT N_("Apply cartoon effect")
67 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
68     "\"gradient\" and \"edge\".")
69
70 static const char *mode_list[] = { "gradient", "edge", "hough" };
71 static const char *mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
72
73 #define FILTER_PREFIX "gradient-"
74
75 vlc_module_begin();
76     set_description( _("Gradient video filter") );
77     set_shortname( _( "Gradient" ));
78     set_capability( "video filter2", 0 );
79     set_category( CAT_VIDEO );
80     set_subcategory( SUBCAT_VIDEO_VFILTER );
81
82     add_string( FILTER_PREFIX "mode", "gradient", NULL,
83                 MODE_TEXT, MODE_LONGTEXT, VLC_FALSE );
84         change_string_list( mode_list, mode_list_text, 0 );
85
86     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
87                 GRADIENT_TEXT, GRADIENT_LONGTEXT, VLC_FALSE );
88     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
89                 CARTOON_TEXT, CARTOON_LONGTEXT, VLC_FALSE );
90
91     add_shortcut( "gradient" );
92     set_callbacks( Create, Destroy );
93 vlc_module_end();
94
95 static const char *ppsz_filter_options[] = {
96     "mode", "type", "cartoon", NULL
97 };
98
99 /*****************************************************************************
100  * vout_sys_t: Distort video output method descriptor
101  *****************************************************************************
102  * This structure is part of the video output thread descriptor.
103  * It describes the Distort specific properties of an output thread.
104  *****************************************************************************/
105 struct filter_sys_t
106 {
107     int i_mode;
108
109     /* For the gradient mode */
110     int i_gradient_type;
111     vlc_bool_t b_cartoon;
112
113     uint32_t *p_buf32;
114     uint32_t *p_buf32_bis;
115     uint8_t *p_buf8;
116
117     /* For hough mode */
118     int *p_pre_hough;
119 };
120
121 /*****************************************************************************
122  * Create: allocates Distort video thread output method
123  *****************************************************************************
124  * This function allocates and initializes a Distort vout method.
125  *****************************************************************************/
126 static int Create( vlc_object_t *p_this )
127 {
128     filter_t *p_filter = (filter_t *)p_this;
129     char *psz_method;
130
131     /* Allocate structure */
132     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
133     if( p_filter->p_sys == NULL )
134     {
135         msg_Err( p_filter, "out of memory" );
136         return VLC_ENOMEM;
137     }
138
139     p_filter->pf_video_filter = Filter;
140
141     p_filter->p_sys->p_pre_hough = NULL;
142
143     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
144                    p_filter->p_cfg );
145
146     if( !(psz_method =
147         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
148     {
149         msg_Err( p_filter, "configuration variable "
150                  FILTER_PREFIX "mode empty" );
151         p_filter->p_sys->i_mode = GRADIENT;
152     }
153     else
154     {
155         if( !strcmp( psz_method, "gradient" ) )
156         {
157             p_filter->p_sys->i_mode = GRADIENT;
158         }
159         else if( !strcmp( psz_method, "edge" ) )
160         {
161             p_filter->p_sys->i_mode = EDGE;
162         }
163         else if( !strcmp( psz_method, "hough" ) )
164         {
165             p_filter->p_sys->i_mode = HOUGH;
166         }
167         else
168         {
169             msg_Err( p_filter, "no valid gradient mode provided" );
170             p_filter->p_sys->i_mode = GRADIENT;
171         }
172     }
173     free( psz_method );
174
175     p_filter->p_sys->i_gradient_type =
176         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
177     p_filter->p_sys->b_cartoon =
178         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
179
180     var_AddCallback( p_filter, FILTER_PREFIX "mode",
181                      GradientCallback, p_filter->p_sys );
182     var_AddCallback( p_filter, FILTER_PREFIX "type",
183                      GradientCallback, p_filter->p_sys );
184     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
185                      GradientCallback, p_filter->p_sys );
186
187     p_filter->p_sys->p_buf32 = NULL;
188     p_filter->p_sys->p_buf32_bis = NULL;
189     p_filter->p_sys->p_buf8 = NULL;
190
191     return VLC_SUCCESS;
192 }
193
194 /*****************************************************************************
195  * Destroy: destroy Distort video thread output method
196  *****************************************************************************
197  * Terminate an output method created by DistortCreateOutputMethod
198  *****************************************************************************/
199 static void Destroy( vlc_object_t *p_this )
200 {
201     filter_t *p_filter = (filter_t *)p_this;
202
203     free( p_filter->p_sys->p_buf32 );
204     free( p_filter->p_sys->p_buf32_bis );
205     free( p_filter->p_sys->p_buf8 );
206     free( p_filter->p_sys->p_pre_hough );
207
208     free( p_filter->p_sys );
209 }
210
211 /*****************************************************************************
212  * Render: displays previously rendered output
213  *****************************************************************************
214  * This function send the currently rendered image to Distort image, waits
215  * until it is displayed and switch the two rendering buffers, preparing next
216  * frame.
217  *****************************************************************************/
218 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
219 {
220     picture_t *p_outpic;
221
222     if( !p_pic ) return NULL;
223
224     p_outpic = p_filter->pf_vout_buffer_new( p_filter );
225     if( !p_outpic )
226     {
227         msg_Warn( p_filter, "can't get output picture" );
228         if( p_pic->pf_release )
229             p_pic->pf_release( p_pic );
230         return NULL;
231     }
232
233     switch( p_filter->p_sys->i_mode )
234     {
235         case EDGE:
236             FilterEdge( p_filter, p_pic, p_outpic );
237             break;
238
239         case GRADIENT:
240             FilterGradient( p_filter, p_pic, p_outpic );
241             break;
242
243         case HOUGH:
244             FilterHough( p_filter, p_pic, p_outpic );
245             break;
246
247         default:
248             break;
249     }
250
251     p_outpic->date = p_pic->date;
252     p_outpic->b_force = p_pic->b_force;
253     p_outpic->i_nb_fields = p_pic->i_nb_fields;
254     p_outpic->b_progressive = p_pic->b_progressive;
255     p_outpic->b_top_field_first = p_pic->b_top_field_first;
256
257     if( p_pic->pf_release )
258         p_pic->pf_release( p_pic );
259
260     return p_outpic;
261 }
262
263 /*****************************************************************************
264  * Gaussian Convolution
265  *****************************************************************************
266  *    Gaussian convolution ( sigma == 1.4 )
267  *
268  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
269  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
270  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
271  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
272  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
273  *****************************************************************************/
274 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
275 {
276     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
277     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
278     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
279     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
280
281     int x,y;
282     for( y = 2; y < i_num_lines - 2; y++ )
283     {
284         for( x = 2; x < i_src_visible - 2; x++ )
285         {
286             p_smooth[y*i_src_visible+x] = (uint32_t)(
287               /* 2 rows up */
288                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
289               + ((p_inpix[(y-2)*i_src_pitch+x-1]
290               +   p_inpix[(y-2)*i_src_pitch+x]
291               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
292               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
293               /* 1 row up */
294               + ((p_inpix[(y-1)*i_src_pitch+x-2]
295               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
296               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
297               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
298               +   p_inpix[(y-1)*i_src_pitch+x+2]
299               /* */
300               +   p_inpix[y*i_src_pitch+x-2]
301               + ( p_inpix[y*i_src_pitch+x-1]*3 )
302               + ( p_inpix[y*i_src_pitch+x]<<2 )
303               + ( p_inpix[y*i_src_pitch+x+1]*3 )
304               +   p_inpix[y*i_src_pitch+x+2]
305               /* 1 row down */
306               +   p_inpix[(y+1)*i_src_pitch+x-2]
307               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
308               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
309               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
310               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
311               /* 2 rows down */
312               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
313               + ((p_inpix[(y+2)*i_src_pitch+x-1]
314               +   p_inpix[(y+2)*i_src_pitch+x]
315               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
316               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
317               ) >> 6 /* 115 */;
318         }
319     }
320 }
321
322 /*****************************************************************************
323  * FilterGradient: Sobel
324  *****************************************************************************/
325 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
326                                                 picture_t *p_outpic )
327 {
328     int x, y;
329     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
330     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
331     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
332     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
333
334     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
335     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
336
337     uint32_t *p_smooth;
338     if( !p_filter->p_sys->p_buf32 )
339         p_filter->p_sys->p_buf32 =
340         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
341     p_smooth = p_filter->p_sys->p_buf32;
342
343     if( !p_smooth ) return;
344
345     if( p_filter->p_sys->b_cartoon )
346     {
347         p_filter->p_libvlc->pf_memcpy( p_outpic->p[U_PLANE].p_pixels,
348             p_inpic->p[U_PLANE].p_pixels,
349             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
350         p_filter->p_libvlc->pf_memcpy( p_outpic->p[V_PLANE].p_pixels,
351             p_inpic->p[V_PLANE].p_pixels,
352             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
353     }
354     else
355     {
356         p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
357             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
358         p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
359             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
360     }
361
362     GaussianConvolution( p_inpic, p_smooth );
363
364     /* Sobel gradient
365
366      | -1 0 1 |     |  1  2  1 |
367      | -2 0 2 | and |  0  0  0 |
368      | -1 0 1 |     | -1 -2 -1 | */
369
370 #define FOR                                                     \
371     for( y = 1; y < i_num_lines - 1; y++ )                      \
372     {                                                           \
373         for( x = 1; x < i_src_visible - 1; x++ )                \
374         {                                                       \
375             const uint32_t a =                                  \
376             (                                                   \
377               abs(                                              \
378                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
379                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
380                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
381                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
382                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
383                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
384               )                                                 \
385             +                                                   \
386               abs(                                              \
387                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
388                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
389                + ( ( p_smooth[y*i_src_visible+x-1]              \
390                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
391                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
392                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
393               )                                                 \
394             );
395     if( p_filter->p_sys->i_gradient_type )
396     {
397         if( p_filter->p_sys->b_cartoon )
398         {
399             FOR
400             if( a > 60 )
401             {
402                 p_outpix[y*i_dst_pitch+x] = 0x00;
403             }
404             else
405             {
406                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
407                     p_outpix[y*i_dst_pitch+x] =
408                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
409                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
410                     p_outpix[y*i_dst_pitch+x] =
411                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
412                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
413                     p_outpix[y*i_dst_pitch+x] =
414                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
415                 else
416                     p_outpix[y*i_dst_pitch+x] =
417                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
418             }
419             }}
420         }
421         else
422         {
423             FOR
424             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
425             }}
426         }
427     }
428     else
429     {
430         FOR
431         if( a>>8 )
432             p_outpix[y*i_dst_pitch+x] = 0;
433         else
434             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
435         }}
436     }
437 #undef FOR
438 }
439
440 /*****************************************************************************
441  * FilterEdge: Canny edge detection algorithm
442  *****************************************************************************
443  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
444  * (well ... my implementation isn't really the canny algorithm ... but some
445  * ideas are the same)
446  *****************************************************************************/
447 /* angle : | */
448 #define THETA_Y 0
449 /* angle : - */
450 #define THETA_X 1
451 /* angle : / */
452 #define THETA_P 2
453 /* angle : \ */
454 #define THETA_M 3
455 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
456                                             picture_t *p_outpic )
457 {
458     int x, y;
459
460     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
461     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
462     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
463     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
464
465     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
466     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
467
468     uint32_t *p_smooth;
469     uint32_t *p_grad;
470     uint8_t *p_theta;
471
472     if( !p_filter->p_sys->p_buf32 )
473         p_filter->p_sys->p_buf32 =
474         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
475     p_smooth = p_filter->p_sys->p_buf32;
476
477     if( !p_filter->p_sys->p_buf32_bis )
478         p_filter->p_sys->p_buf32_bis =
479         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
480     p_grad = p_filter->p_sys->p_buf32_bis;
481
482     if( !p_filter->p_sys->p_buf8 )
483         p_filter->p_sys->p_buf8 =
484         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
485     p_theta = p_filter->p_sys->p_buf8;
486
487     if( !p_smooth || !p_grad || !p_theta ) return;
488
489     if( p_filter->p_sys->b_cartoon )
490     {
491         p_filter->p_libvlc->pf_memcpy( p_outpic->p[U_PLANE].p_pixels,
492             p_inpic->p[U_PLANE].p_pixels,
493             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
494         p_filter->p_libvlc->pf_memcpy( p_outpic->p[V_PLANE].p_pixels,
495             p_inpic->p[V_PLANE].p_pixels,
496             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
497     }
498     else
499     {
500         p_filter->p_libvlc->pf_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
501               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
502         p_filter->p_libvlc->pf_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
503             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
504         p_filter->p_libvlc->pf_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
505             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
506     }
507
508     GaussianConvolution( p_inpic, p_smooth );
509
510     /* Sobel gradient
511
512      | -1 0 1 |     |  1  2  1 |
513      | -2 0 2 | and |  0  0  0 |
514      | -1 0 1 |     | -1 -2 -1 | */
515
516     for( y = 1; y < i_num_lines - 1; y++ )
517     {
518         for( x = 1; x < i_src_visible - 1; x++ )
519         {
520
521             const int gradx =
522                  ( p_smooth[(y-1)*i_src_visible+x-1]
523                    - p_smooth[(y+1)*i_src_visible+x-1] )
524                + ( ( p_smooth[(y-1)*i_src_visible+x]
525                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
526                + ( p_smooth[(y-1)*i_src_visible+x+1]
527                    - p_smooth[(y+1)*i_src_visible+x+1] );
528             const int grady =
529                  ( p_smooth[(y-1)*i_src_visible+x-1]
530                    - p_smooth[(y-1)*i_src_visible+x+1] )
531                + ( ( p_smooth[y*i_src_visible+x-1]
532                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
533                + ( p_smooth[(y+1)*i_src_visible+x-1]
534                    - p_smooth[(y+1)*i_src_visible+x+1] );
535
536             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
537
538             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
539              * tan( 26,565051177 ) = 0.5
540              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
541              * tan( 63,434948823 ) 2 */
542             if( (grady<<1) > gradx )
543                 p_theta[y*i_src_visible+x] = THETA_P;
544             else if( (grady<<1) < -gradx )
545                 p_theta[y*i_src_visible+x] = THETA_M;
546             else if( !gradx || abs(grady) > abs(gradx)<<1 )
547                 p_theta[y*i_src_visible+x] = THETA_Y;
548             else
549                 p_theta[y*i_src_visible+x] = THETA_X;
550         }
551     }
552
553     /* edge computing */
554     for( y = 1; y < i_num_lines - 1; y++ )
555     {
556         for( x = 1; x < i_src_visible - 1; x++ )
557         {
558             if( p_grad[y*i_src_visible+x] > 40 )
559             {
560                 switch( p_theta[y*i_src_visible+x] )
561                 {
562                     case THETA_Y:
563                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
564                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
565                         {
566                             p_outpix[y*i_dst_pitch+x] = 0;
567                             break;
568                         } else goto colorize;
569                     case THETA_P:
570                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
571                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
572                         {
573                             p_outpix[y*i_dst_pitch+x] = 0;
574                             break;
575                         } else goto colorize;
576                     case THETA_M:
577                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
578                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
579                         {
580                             p_outpix[y*i_dst_pitch+x] = 0;
581                             break;
582                         } else goto colorize;
583                     case THETA_X:
584                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
585                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
586                         {
587                             p_outpix[y*i_dst_pitch+x] = 0;
588                             break;
589                         } else goto colorize;
590                 }
591             }
592             else
593             {
594                 colorize:
595                 if( p_filter->p_sys->b_cartoon )
596                 {
597                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
598                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
599                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
600                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
601                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
602                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
603                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
604                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
605                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
606                     else
607                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
608                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
609                 }
610             }
611         }
612     }
613 }
614
615 /*****************************************************************************
616  * FilterHough
617  *****************************************************************************/
618 #define p_pre_hough p_filter->p_sys->p_pre_hough
619 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
620                                              picture_t *p_outpic )
621 {
622     int x, y, i;
623     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
624     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
625     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
626
627     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
628
629     int i_diag = sqrt( i_num_lines * i_num_lines +
630                         i_src_visible * i_src_visible);
631     int i_max, i_phi_max, i_rho, i_rho_max;
632     int i_nb_steps = 90;
633     double d_step = M_PI / i_nb_steps;
634     double d_sin;
635     double d_cos;
636     uint32_t *p_smooth;
637     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
638     if( ! p_hough ) return;
639     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
640     if( !p_smooth ) return;
641
642     if( ! p_pre_hough )
643     {
644         msg_Dbg(p_filter, "Starting precalculation");
645         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
646         if( ! p_pre_hough ) return;
647         for( i = 0 ; i < i_nb_steps ; i++)
648         {
649             d_sin = sin(d_step * i);
650             d_cos = cos(d_step * i);
651             for( y = 0 ; y < i_num_lines ; y++ )
652                 for( x = 0 ; x < i_src_visible ; x++ )
653                 {
654                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
655                         ceil(x*d_sin + y*d_cos);
656                 }
657         }
658         msg_Dbg(p_filter, "Precalculation done");
659     }
660
661     p_filter->p_libvlc->pf_memset( p_hough, 0,
662                                    i_diag * i_nb_steps * sizeof(int) );
663
664     p_filter->p_libvlc->pf_memcpy(
665         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
666         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
667     p_filter->p_libvlc->pf_memcpy(
668         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
669         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
670     p_filter->p_libvlc->pf_memcpy(
671         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
672         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
673
674     GaussianConvolution( p_inpic, p_smooth );
675
676     /* Sobel gradient
677
678      | -1 0 1 |     |  1  2  1 |
679      | -2 0 2 | and |  0  0  0 |
680      | -1 0 1 |     | -1 -2 -1 | */
681
682     i_max = 0;
683     i_rho_max = 0;
684     i_phi_max = 0;
685     for( y = 4; y < i_num_lines - 4; y++ )
686     {
687         for( x = 4; x < i_src_visible - 4; x++ )
688         {
689             uint32_t a =
690             (
691               abs(
692                 ( ( p_smooth[(y-1)*i_src_visible+x]
693                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
694                + ( p_smooth[(y-1)*i_src_visible+x-1]
695                    - p_smooth[(y+1)*i_src_visible+x-1] )
696                + ( p_smooth[(y-1)*i_src_visible+x+1]
697                    - p_smooth[(y+1)*i_src_visible+x+1] )
698               )
699             +
700               abs(
701                 ( ( p_smooth[y*i_src_visible+x-1]
702                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
703                + ( p_smooth[(y-1)*i_src_visible+x-1]
704                    - p_smooth[(y-1)*i_src_visible+x+1] )
705                + ( p_smooth[(y+1)*i_src_visible+x-1]
706                    - p_smooth[(y+1)*i_src_visible+x+1] )
707               )
708             );
709             if( a>>8 )
710             {
711                 for( i = 0 ; i < i_nb_steps ; i ++ )
712                 {
713                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
714                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
715                     {
716                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
717                         i_rho_max = i_rho;
718                         i_phi_max = i;
719                     }
720                 }
721             }
722         }
723     }
724
725     d_sin = sin(i_phi_max*d_step);
726     d_cos = cos(i_phi_max*d_step);
727     if( d_cos != 0 )
728     {
729         for( x = 0 ; x < i_src_visible ; x++ )
730         {
731             y = (i_rho_max - x * d_sin) / d_cos;
732             if( y >= 0 && y < i_num_lines )
733                 p_outpix[y*i_dst_pitch+x] = 255;
734         }
735     }
736
737     if( p_hough ) free( p_hough );
738     if( p_smooth ) free( p_smooth );
739 }
740 #undef p_pre_hough
741
742
743 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
744                              vlc_value_t oldval, vlc_value_t newval,
745                              void *p_data )
746 {
747     filter_sys_t *p_sys = (filter_sys_t *)p_data;
748     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
749     {
750         if( !strcmp( newval.psz_string, "gradient" ) )
751         {
752             p_sys->i_mode = GRADIENT;
753         }
754         else if( !strcmp( newval.psz_string, "edge" ) )
755         {
756             p_sys->i_mode = EDGE;
757         }
758         else if( !strcmp( newval.psz_string, "hough" ) )
759         {
760             p_sys->i_mode = HOUGH;
761         }
762         else
763         {
764             msg_Err( p_this, "no valid gradient mode provided" );
765             p_sys->i_mode = GRADIENT;
766         }
767     }
768     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
769     {
770         p_sys->i_gradient_type = newval.i_int;
771     }
772     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
773     {
774         p_sys->b_cartoon = newval.b_bool;
775     }
776     return VLC_SUCCESS;
777 }