948cb4a04d318256b720203ed6ea0e8b54698f02
[vlc.git] / modules / video_filter / gradient.c
1 /*****************************************************************************
2  * gradient.c : Gradient and edge detection video effects plugin for vlc
3  *****************************************************************************
4  * Copyright (C) 2000-2008 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Samuel Hocevar <sam@zoy.org>
8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23  *****************************************************************************/
24
25 /*****************************************************************************
26  * Preamble
27  *****************************************************************************/
28
29 #ifdef HAVE_CONFIG_H
30 # include "config.h"
31 #endif
32
33 #include <math.h>                                            /* sin(), cos() */
34
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
37 #include <vlc_sout.h>
38 #include <vlc_vout.h>
39
40 #include "vlc_filter.h"
41 #include "filter_picture.h"
42
43 enum { GRADIENT, EDGE, HOUGH };
44
45 /*****************************************************************************
46  * Local prototypes
47  *****************************************************************************/
48 static int  Create    ( vlc_object_t * );
49 static void Destroy   ( vlc_object_t * );
50
51 static picture_t *Filter( filter_t *, picture_t * );
52 static int GradientCallback( vlc_object_t *, char const *,
53                              vlc_value_t, vlc_value_t,
54                              void * );
55
56 static void FilterGradient( filter_t *, picture_t *, picture_t * );
57 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
58 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
59
60 /*****************************************************************************
61  * Module descriptor
62  *****************************************************************************/
63 #define MODE_TEXT N_("Distort mode")
64 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
65
66 #define GRADIENT_TEXT N_("Gradient image type")
67 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
68         "turn the image to white while 1 will keep colors." )
69
70 #define CARTOON_TEXT N_("Apply cartoon effect")
71 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
72     "\"gradient\" and \"edge\".")
73
74 static const char *const mode_list[] = { "gradient", "edge", "hough" };
75 static const char *const mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
76
77 #define FILTER_PREFIX "gradient-"
78
79 vlc_module_begin ()
80     set_description( N_("Gradient video filter") )
81     set_shortname( N_( "Gradient" ))
82     set_capability( "video filter2", 0 )
83     set_category( CAT_VIDEO )
84     set_subcategory( SUBCAT_VIDEO_VFILTER )
85
86     add_string( FILTER_PREFIX "mode", "gradient", NULL,
87                 MODE_TEXT, MODE_LONGTEXT, false )
88         change_string_list( mode_list, mode_list_text, 0 )
89
90     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
91                 GRADIENT_TEXT, GRADIENT_LONGTEXT, false )
92     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
93                 CARTOON_TEXT, CARTOON_LONGTEXT, false )
94
95     add_shortcut( "gradient" )
96     set_callbacks( Create, Destroy )
97 vlc_module_end ()
98
99 static const char *const ppsz_filter_options[] = {
100     "mode", "type", "cartoon", NULL
101 };
102
103 /*****************************************************************************
104  * filter_sys_t: Distort video output method descriptor
105  *****************************************************************************
106  * This structure is part of the video output thread descriptor.
107  * It describes the Distort specific properties of an output thread.
108  *****************************************************************************/
109 struct filter_sys_t
110 {
111     vlc_mutex_t lock;
112     int i_mode;
113
114     /* For the gradient mode */
115     int i_gradient_type;
116     bool b_cartoon;
117
118     uint32_t *p_buf32;
119     uint32_t *p_buf32_bis;
120     uint8_t *p_buf8;
121
122     /* For hough mode */
123     int *p_pre_hough;
124 };
125
126 /*****************************************************************************
127  * Create: allocates Distort video thread output method
128  *****************************************************************************
129  * This function allocates and initializes a Distort vout method.
130  *****************************************************************************/
131 static int Create( vlc_object_t *p_this )
132 {
133     filter_t *p_filter = (filter_t *)p_this;
134     char *psz_method;
135
136     switch( p_filter->fmt_in.video.i_chroma )
137     {
138         CASE_PLANAR_YUV
139             break;
140
141         default:
142              msg_Err( p_filter, "Unsupported input chroma (%4s)",
143                       (char*)&(p_filter->fmt_in.video.i_chroma) );
144             return VLC_EGENERIC;
145     }
146
147     /* Allocate structure */
148     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
149     if( p_filter->p_sys == NULL )
150         return VLC_ENOMEM;
151
152     p_filter->pf_video_filter = Filter;
153
154     p_filter->p_sys->p_pre_hough = NULL;
155
156     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
157                    p_filter->p_cfg );
158
159     if( !(psz_method =
160         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
161     {
162         msg_Err( p_filter, "configuration variable "
163                  FILTER_PREFIX "mode empty" );
164         p_filter->p_sys->i_mode = GRADIENT;
165     }
166     else
167     {
168         if( !strcmp( psz_method, "gradient" ) )
169         {
170             p_filter->p_sys->i_mode = GRADIENT;
171         }
172         else if( !strcmp( psz_method, "edge" ) )
173         {
174             p_filter->p_sys->i_mode = EDGE;
175         }
176         else if( !strcmp( psz_method, "hough" ) )
177         {
178             p_filter->p_sys->i_mode = HOUGH;
179         }
180         else
181         {
182             msg_Err( p_filter, "no valid gradient mode provided (%s)", psz_method );
183             p_filter->p_sys->i_mode = GRADIENT;
184         }
185     }
186     free( psz_method );
187
188     p_filter->p_sys->i_gradient_type =
189         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
190     p_filter->p_sys->b_cartoon =
191         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
192
193     vlc_mutex_init( &p_filter->p_sys->lock );
194     var_AddCallback( p_filter, FILTER_PREFIX "mode",
195                      GradientCallback, p_filter->p_sys );
196     var_AddCallback( p_filter, FILTER_PREFIX "type",
197                      GradientCallback, p_filter->p_sys );
198     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
199                      GradientCallback, p_filter->p_sys );
200
201     p_filter->p_sys->p_buf32 = NULL;
202     p_filter->p_sys->p_buf32_bis = NULL;
203     p_filter->p_sys->p_buf8 = NULL;
204
205     return VLC_SUCCESS;
206 }
207
208 /*****************************************************************************
209  * Destroy: destroy Distort video thread output method
210  *****************************************************************************
211  * Terminate an output method created by DistortCreateOutputMethod
212  *****************************************************************************/
213 static void Destroy( vlc_object_t *p_this )
214 {
215     filter_t *p_filter = (filter_t *)p_this;
216     filter_sys_t *p_sys = p_filter->p_sys;
217
218     var_DelCallback( p_filter, FILTER_PREFIX "mode",
219                      GradientCallback, p_sys );
220     var_DelCallback( p_filter, FILTER_PREFIX "type",
221                      GradientCallback, p_sys );
222     var_DelCallback( p_filter, FILTER_PREFIX "cartoon",
223                      GradientCallback, p_sys );
224     vlc_mutex_destroy( &p_sys->lock );
225
226     free( p_sys->p_buf32 );
227     free( p_sys->p_buf32_bis );
228     free( p_sys->p_buf8 );
229     free( p_sys->p_pre_hough );
230
231     free( p_sys );
232 }
233
234 /*****************************************************************************
235  * Render: displays previously rendered output
236  *****************************************************************************
237  * This function send the currently rendered image to Distort image, waits
238  * until it is displayed and switch the two rendering buffers, preparing next
239  * frame.
240  *****************************************************************************/
241 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
242 {
243     picture_t *p_outpic;
244
245     if( !p_pic ) return NULL;
246
247     p_outpic = filter_NewPicture( p_filter );
248     if( !p_outpic )
249     {
250         picture_Release( p_pic );
251         return NULL;
252     }
253
254     vlc_mutex_lock( &p_filter->p_sys->lock );
255     switch( p_filter->p_sys->i_mode )
256     {
257         case EDGE:
258             FilterEdge( p_filter, p_pic, p_outpic );
259             break;
260
261         case GRADIENT:
262             FilterGradient( p_filter, p_pic, p_outpic );
263             break;
264
265         case HOUGH:
266             FilterHough( p_filter, p_pic, p_outpic );
267             break;
268
269         default:
270             break;
271     }
272     vlc_mutex_unlock( &p_filter->p_sys->lock );
273
274     return CopyInfoAndRelease( p_outpic, p_pic );
275 }
276
277 /*****************************************************************************
278  * Gaussian Convolution
279  *****************************************************************************
280  *    Gaussian convolution ( sigma == 1.4 )
281  *
282  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
283  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
284  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
285  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
286  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
287  *****************************************************************************/
288 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
289 {
290     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
291     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
292     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
293     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
294
295     int x,y;
296     for( y = 2; y < i_num_lines - 2; y++ )
297     {
298         for( x = 2; x < i_src_visible - 2; x++ )
299         {
300             p_smooth[y*i_src_visible+x] = (uint32_t)(
301               /* 2 rows up */
302                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
303               + ((p_inpix[(y-2)*i_src_pitch+x-1]
304               +   p_inpix[(y-2)*i_src_pitch+x]
305               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
306               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
307               /* 1 row up */
308               + ((p_inpix[(y-1)*i_src_pitch+x-2]
309               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
310               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
311               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
312               +   p_inpix[(y-1)*i_src_pitch+x+2]
313               /* */
314               +   p_inpix[y*i_src_pitch+x-2]
315               + ( p_inpix[y*i_src_pitch+x-1]*3 )
316               + ( p_inpix[y*i_src_pitch+x]<<2 )
317               + ( p_inpix[y*i_src_pitch+x+1]*3 )
318               +   p_inpix[y*i_src_pitch+x+2]
319               /* 1 row down */
320               +   p_inpix[(y+1)*i_src_pitch+x-2]
321               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
322               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
323               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
324               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
325               /* 2 rows down */
326               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
327               + ((p_inpix[(y+2)*i_src_pitch+x-1]
328               +   p_inpix[(y+2)*i_src_pitch+x]
329               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
330               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
331               ) >> 6 /* 115 */;
332         }
333     }
334 }
335
336 /*****************************************************************************
337  * FilterGradient: Sobel
338  *****************************************************************************/
339 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
340                                                 picture_t *p_outpic )
341 {
342     int x, y;
343     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
344     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
345     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
346     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
347
348     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
349     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
350
351     uint32_t *p_smooth;
352     if( !p_filter->p_sys->p_buf32 )
353         p_filter->p_sys->p_buf32 =
354         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
355     p_smooth = p_filter->p_sys->p_buf32;
356
357     if( !p_smooth ) return;
358
359     if( p_filter->p_sys->b_cartoon )
360     {
361         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
362             p_inpic->p[U_PLANE].p_pixels,
363             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
364         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
365             p_inpic->p[V_PLANE].p_pixels,
366             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
367     }
368     else
369     {
370         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
371             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
372         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
373             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
374     }
375
376     GaussianConvolution( p_inpic, p_smooth );
377
378     /* Sobel gradient
379
380      | -1 0 1 |     |  1  2  1 |
381      | -2 0 2 | and |  0  0  0 |
382      | -1 0 1 |     | -1 -2 -1 | */
383
384 #define FOR                                                     \
385     for( y = 1; y < i_num_lines - 1; y++ )                      \
386     {                                                           \
387         for( x = 1; x < i_src_visible - 1; x++ )                \
388         {                                                       \
389             const uint32_t a =                                  \
390             (                                                   \
391               abs(                                              \
392                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
393                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
394                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
395                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
396                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
397                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
398               )                                                 \
399             +                                                   \
400               abs(                                              \
401                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
402                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
403                + ( ( p_smooth[y*i_src_visible+x-1]              \
404                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
405                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
406                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
407               )                                                 \
408             );
409     if( p_filter->p_sys->i_gradient_type )
410     {
411         if( p_filter->p_sys->b_cartoon )
412         {
413             FOR
414             if( a > 60 )
415             {
416                 p_outpix[y*i_dst_pitch+x] = 0x00;
417             }
418             else
419             {
420                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
421                     p_outpix[y*i_dst_pitch+x] =
422                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
423                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
424                     p_outpix[y*i_dst_pitch+x] =
425                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
426                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
427                     p_outpix[y*i_dst_pitch+x] =
428                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
429                 else
430                     p_outpix[y*i_dst_pitch+x] =
431                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
432             }
433             }}
434         }
435         else
436         {
437             FOR
438             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
439             }}
440         }
441     }
442     else
443     {
444         FOR
445         if( a>>8 )
446             p_outpix[y*i_dst_pitch+x] = 0;
447         else
448             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
449         }}
450     }
451 #undef FOR
452 }
453
454 /*****************************************************************************
455  * FilterEdge: Canny edge detection algorithm
456  *****************************************************************************
457  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
458  * (well ... my implementation isn't really the canny algorithm ... but some
459  * ideas are the same)
460  *****************************************************************************/
461 /* angle : | */
462 #define THETA_Y 0
463 /* angle : - */
464 #define THETA_X 1
465 /* angle : / */
466 #define THETA_P 2
467 /* angle : \ */
468 #define THETA_M 3
469 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
470                                             picture_t *p_outpic )
471 {
472     int x, y;
473
474     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
475     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
476     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
477     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
478
479     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
480     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
481
482     uint32_t *p_smooth;
483     uint32_t *p_grad;
484     uint8_t *p_theta;
485
486     if( !p_filter->p_sys->p_buf32 )
487         p_filter->p_sys->p_buf32 =
488         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
489     p_smooth = p_filter->p_sys->p_buf32;
490
491     if( !p_filter->p_sys->p_buf32_bis )
492         p_filter->p_sys->p_buf32_bis =
493         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
494     p_grad = p_filter->p_sys->p_buf32_bis;
495
496     if( !p_filter->p_sys->p_buf8 )
497         p_filter->p_sys->p_buf8 =
498         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
499     p_theta = p_filter->p_sys->p_buf8;
500
501     if( !p_smooth || !p_grad || !p_theta ) return;
502
503     if( p_filter->p_sys->b_cartoon )
504     {
505         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
506             p_inpic->p[U_PLANE].p_pixels,
507             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
508         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
509             p_inpic->p[V_PLANE].p_pixels,
510             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
511     }
512     else
513     {
514         vlc_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
515               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
516         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
517             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
518         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
519             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
520     }
521
522     GaussianConvolution( p_inpic, p_smooth );
523
524     /* Sobel gradient
525
526      | -1 0 1 |     |  1  2  1 |
527      | -2 0 2 | and |  0  0  0 |
528      | -1 0 1 |     | -1 -2 -1 | */
529
530     for( y = 1; y < i_num_lines - 1; y++ )
531     {
532         for( x = 1; x < i_src_visible - 1; x++ )
533         {
534
535             const int gradx =
536                  ( p_smooth[(y-1)*i_src_visible+x-1]
537                    - p_smooth[(y+1)*i_src_visible+x-1] )
538                + ( ( p_smooth[(y-1)*i_src_visible+x]
539                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
540                + ( p_smooth[(y-1)*i_src_visible+x+1]
541                    - p_smooth[(y+1)*i_src_visible+x+1] );
542             const int grady =
543                  ( p_smooth[(y-1)*i_src_visible+x-1]
544                    - p_smooth[(y-1)*i_src_visible+x+1] )
545                + ( ( p_smooth[y*i_src_visible+x-1]
546                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
547                + ( p_smooth[(y+1)*i_src_visible+x-1]
548                    - p_smooth[(y+1)*i_src_visible+x+1] );
549
550             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
551
552             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
553              * tan( 26,565051177 ) = 0.5
554              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
555              * tan( 63,434948823 ) 2 */
556             if( (grady<<1) > gradx )
557                 p_theta[y*i_src_visible+x] = THETA_P;
558             else if( (grady<<1) < -gradx )
559                 p_theta[y*i_src_visible+x] = THETA_M;
560             else if( !gradx || abs(grady) > abs(gradx)<<1 )
561                 p_theta[y*i_src_visible+x] = THETA_Y;
562             else
563                 p_theta[y*i_src_visible+x] = THETA_X;
564         }
565     }
566
567     /* edge computing */
568     for( y = 1; y < i_num_lines - 1; y++ )
569     {
570         for( x = 1; x < i_src_visible - 1; x++ )
571         {
572             if( p_grad[y*i_src_visible+x] > 40 )
573             {
574                 switch( p_theta[y*i_src_visible+x] )
575                 {
576                     case THETA_Y:
577                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
578                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
579                         {
580                             p_outpix[y*i_dst_pitch+x] = 0;
581                             break;
582                         } else goto colorize;
583                     case THETA_P:
584                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
585                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
586                         {
587                             p_outpix[y*i_dst_pitch+x] = 0;
588                             break;
589                         } else goto colorize;
590                     case THETA_M:
591                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
592                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
593                         {
594                             p_outpix[y*i_dst_pitch+x] = 0;
595                             break;
596                         } else goto colorize;
597                     case THETA_X:
598                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
599                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
600                         {
601                             p_outpix[y*i_dst_pitch+x] = 0;
602                             break;
603                         } else goto colorize;
604                 }
605             }
606             else
607             {
608                 colorize:
609                 if( p_filter->p_sys->b_cartoon )
610                 {
611                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
612                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
613                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
614                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
615                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
616                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
617                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
618                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
619                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
620                     else
621                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
622                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
623                 }
624             }
625         }
626     }
627 }
628
629 /*****************************************************************************
630  * FilterHough
631  *****************************************************************************/
632 #define p_pre_hough p_filter->p_sys->p_pre_hough
633 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
634                                              picture_t *p_outpic )
635 {
636     int x, y, i;
637     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
638     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
639     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
640
641     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
642
643     int i_diag = sqrt( i_num_lines * i_num_lines +
644                         i_src_visible * i_src_visible);
645     int i_max, i_phi_max, i_rho, i_rho_max;
646     int i_nb_steps = 90;
647     double d_step = M_PI / i_nb_steps;
648     double d_sin;
649     double d_cos;
650     uint32_t *p_smooth;
651
652     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
653     if( ! p_hough ) return;
654
655     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
656     if( !p_smooth )
657     {
658         free( p_hough );
659         return;
660     }
661
662     if( ! p_pre_hough )
663     {
664         msg_Dbg(p_filter, "Starting precalculation");
665         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
666         if( ! p_pre_hough )
667         {
668             free( p_smooth );
669             free( p_hough );
670             return;
671         }
672         for( i = 0 ; i < i_nb_steps ; i++)
673         {
674             d_sin = sin(d_step * i);
675             d_cos = cos(d_step * i);
676             for( y = 0 ; y < i_num_lines ; y++ )
677                 for( x = 0 ; x < i_src_visible ; x++ )
678                 {
679                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
680                         ceil(x*d_sin + y*d_cos);
681                 }
682         }
683         msg_Dbg(p_filter, "Precalculation done");
684     }
685
686     vlc_memset( p_hough, 0, i_diag * i_nb_steps * sizeof(int) );
687
688     vlc_memcpy(
689         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
690         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
691     vlc_memcpy(
692         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
693         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
694     vlc_memcpy(
695         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
696         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
697
698     GaussianConvolution( p_inpic, p_smooth );
699
700     /* Sobel gradient
701
702      | -1 0 1 |     |  1  2  1 |
703      | -2 0 2 | and |  0  0  0 |
704      | -1 0 1 |     | -1 -2 -1 | */
705
706     i_max = 0;
707     i_rho_max = 0;
708     i_phi_max = 0;
709     for( y = 4; y < i_num_lines - 4; y++ )
710     {
711         for( x = 4; x < i_src_visible - 4; x++ )
712         {
713             uint32_t a =
714             (
715               abs(
716                 ( ( p_smooth[(y-1)*i_src_visible+x]
717                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
718                + ( p_smooth[(y-1)*i_src_visible+x-1]
719                    - p_smooth[(y+1)*i_src_visible+x-1] )
720                + ( p_smooth[(y-1)*i_src_visible+x+1]
721                    - p_smooth[(y+1)*i_src_visible+x+1] )
722               )
723             +
724               abs(
725                 ( ( p_smooth[y*i_src_visible+x-1]
726                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
727                + ( p_smooth[(y-1)*i_src_visible+x-1]
728                    - p_smooth[(y-1)*i_src_visible+x+1] )
729                + ( p_smooth[(y+1)*i_src_visible+x-1]
730                    - p_smooth[(y+1)*i_src_visible+x+1] )
731               )
732             );
733             if( a>>8 )
734             {
735                 for( i = 0 ; i < i_nb_steps ; i ++ )
736                 {
737                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
738                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
739                     {
740                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
741                         i_rho_max = i_rho;
742                         i_phi_max = i;
743                     }
744                 }
745             }
746         }
747     }
748
749     d_sin = sin(i_phi_max*d_step);
750     d_cos = cos(i_phi_max*d_step);
751     if( d_cos != 0 )
752     {
753         for( x = 0 ; x < i_src_visible ; x++ )
754         {
755             y = (i_rho_max - x * d_sin) / d_cos;
756             if( y >= 0 && y < i_num_lines )
757                 p_outpix[y*i_dst_pitch+x] = 255;
758         }
759     }
760
761     free( p_hough );
762     free( p_smooth );
763 }
764 #undef p_pre_hough
765
766
767 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
768                              vlc_value_t oldval, vlc_value_t newval,
769                              void *p_data )
770 {
771     VLC_UNUSED(oldval);
772     filter_sys_t *p_sys = (filter_sys_t *)p_data;
773
774     vlc_mutex_lock( &p_sys->lock );
775     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
776     {
777         if( !strcmp( newval.psz_string, "gradient" ) )
778         {
779             p_sys->i_mode = GRADIENT;
780         }
781         else if( !strcmp( newval.psz_string, "edge" ) )
782         {
783             p_sys->i_mode = EDGE;
784         }
785         else if( !strcmp( newval.psz_string, "hough" ) )
786         {
787             p_sys->i_mode = HOUGH;
788         }
789         else
790         {
791             msg_Err( p_this, "no valid gradient mode provided (%s)", newval.psz_string );
792             p_sys->i_mode = GRADIENT;
793         }
794     }
795     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
796     {
797         p_sys->i_gradient_type = newval.i_int;
798     }
799     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
800     {
801         p_sys->b_cartoon = newval.b_bool;
802     }
803     vlc_mutex_unlock( &p_sys->lock );
804
805     return VLC_SUCCESS;
806 }