c875bd16057513285a4c30227cd31200d6450ba9
[vlc.git] / modules / video_filter / gradient.c
1 /*****************************************************************************
2  * gradient.c : Gradient and edge detection video effects plugin for vlc
3  *****************************************************************************
4  * Copyright (C) 2000-2008 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Samuel Hocevar <sam@zoy.org>
8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23  *****************************************************************************/
24
25 /*****************************************************************************
26  * Preamble
27  *****************************************************************************/
28
29 #ifdef HAVE_CONFIG_H
30 # include "config.h"
31 #endif
32
33 #include <math.h>                                            /* sin(), cos() */
34
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
37 #include <vlc_sout.h>
38 #include <vlc_vout.h>
39
40 #include "vlc_filter.h"
41 #include "filter_picture.h"
42
43 enum { GRADIENT, EDGE, HOUGH };
44
45 /*****************************************************************************
46  * Local prototypes
47  *****************************************************************************/
48 static int  Create    ( vlc_object_t * );
49 static void Destroy   ( vlc_object_t * );
50
51 static picture_t *Filter( filter_t *, picture_t * );
52 static int GradientCallback( vlc_object_t *, char const *,
53                              vlc_value_t, vlc_value_t,
54                              void * );
55
56 static void FilterGradient( filter_t *, picture_t *, picture_t * );
57 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
58 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
59
60 /*****************************************************************************
61  * Module descriptor
62  *****************************************************************************/
63 #define MODE_TEXT N_("Distort mode")
64 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
65
66 #define GRADIENT_TEXT N_("Gradient image type")
67 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
68         "turn the image to white while 1 will keep colors." )
69
70 #define CARTOON_TEXT N_("Apply cartoon effect")
71 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
72     "\"gradient\" and \"edge\".")
73
74 static const char *const mode_list[] = { "gradient", "edge", "hough" };
75 static const char *const mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
76
77 #define FILTER_PREFIX "gradient-"
78
79 vlc_module_begin ()
80     set_description( N_("Gradient video filter") )
81     set_shortname( N_( "Gradient" ))
82     set_capability( "video filter2", 0 )
83     set_category( CAT_VIDEO )
84     set_subcategory( SUBCAT_VIDEO_VFILTER )
85
86     add_string( FILTER_PREFIX "mode", "gradient", NULL,
87                 MODE_TEXT, MODE_LONGTEXT, false )
88         change_string_list( mode_list, mode_list_text, 0 )
89
90     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
91                 GRADIENT_TEXT, GRADIENT_LONGTEXT, false )
92     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
93                 CARTOON_TEXT, CARTOON_LONGTEXT, false )
94
95     add_shortcut( "gradient" )
96     set_callbacks( Create, Destroy )
97 vlc_module_end ()
98
99 static const char *const ppsz_filter_options[] = {
100     "mode", "type", "cartoon", NULL
101 };
102
103 /*****************************************************************************
104  * filter_sys_t: Distort video output method descriptor
105  *****************************************************************************
106  * This structure is part of the video output thread descriptor.
107  * It describes the Distort specific properties of an output thread.
108  *****************************************************************************/
109 struct filter_sys_t
110 {
111     int i_mode;
112
113     /* For the gradient mode */
114     int i_gradient_type;
115     bool b_cartoon;
116
117     uint32_t *p_buf32;
118     uint32_t *p_buf32_bis;
119     uint8_t *p_buf8;
120
121     /* For hough mode */
122     int *p_pre_hough;
123 };
124
125 /*****************************************************************************
126  * Create: allocates Distort video thread output method
127  *****************************************************************************
128  * This function allocates and initializes a Distort vout method.
129  *****************************************************************************/
130 static int Create( vlc_object_t *p_this )
131 {
132     filter_t *p_filter = (filter_t *)p_this;
133     char *psz_method;
134
135     switch( p_filter->fmt_in.video.i_chroma )
136     {
137         CASE_PLANAR_YUV
138             break;
139
140         default:
141              msg_Err( p_filter, "Unsupported input chroma (%4s)",
142                       (char*)&(p_filter->fmt_in.video.i_chroma) );
143             return VLC_EGENERIC;
144     }
145
146     /* Allocate structure */
147     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
148     if( p_filter->p_sys == NULL )
149         return VLC_ENOMEM;
150
151     p_filter->pf_video_filter = Filter;
152
153     p_filter->p_sys->p_pre_hough = NULL;
154
155     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
156                    p_filter->p_cfg );
157
158     if( !(psz_method =
159         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
160     {
161         msg_Err( p_filter, "configuration variable "
162                  FILTER_PREFIX "mode empty" );
163         p_filter->p_sys->i_mode = GRADIENT;
164     }
165     else
166     {
167         if( !strcmp( psz_method, "gradient" ) )
168         {
169             p_filter->p_sys->i_mode = GRADIENT;
170         }
171         else if( !strcmp( psz_method, "edge" ) )
172         {
173             p_filter->p_sys->i_mode = EDGE;
174         }
175         else if( !strcmp( psz_method, "hough" ) )
176         {
177             p_filter->p_sys->i_mode = HOUGH;
178         }
179         else
180         {
181             msg_Err( p_filter, "no valid gradient mode provided (%s)", psz_method );
182             p_filter->p_sys->i_mode = GRADIENT;
183         }
184     }
185     free( psz_method );
186
187     p_filter->p_sys->i_gradient_type =
188         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
189     p_filter->p_sys->b_cartoon =
190         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
191
192     var_AddCallback( p_filter, FILTER_PREFIX "mode",
193                      GradientCallback, p_filter->p_sys );
194     var_AddCallback( p_filter, FILTER_PREFIX "type",
195                      GradientCallback, p_filter->p_sys );
196     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
197                      GradientCallback, p_filter->p_sys );
198
199     p_filter->p_sys->p_buf32 = NULL;
200     p_filter->p_sys->p_buf32_bis = NULL;
201     p_filter->p_sys->p_buf8 = NULL;
202
203     return VLC_SUCCESS;
204 }
205
206 /*****************************************************************************
207  * Destroy: destroy Distort video thread output method
208  *****************************************************************************
209  * Terminate an output method created by DistortCreateOutputMethod
210  *****************************************************************************/
211 static void Destroy( vlc_object_t *p_this )
212 {
213     filter_t *p_filter = (filter_t *)p_this;
214
215     free( p_filter->p_sys->p_buf32 );
216     free( p_filter->p_sys->p_buf32_bis );
217     free( p_filter->p_sys->p_buf8 );
218     free( p_filter->p_sys->p_pre_hough );
219
220     free( p_filter->p_sys );
221 }
222
223 /*****************************************************************************
224  * Render: displays previously rendered output
225  *****************************************************************************
226  * This function send the currently rendered image to Distort image, waits
227  * until it is displayed and switch the two rendering buffers, preparing next
228  * frame.
229  *****************************************************************************/
230 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
231 {
232     picture_t *p_outpic;
233
234     if( !p_pic ) return NULL;
235
236     p_outpic = filter_NewPicture( p_filter );
237     if( !p_outpic )
238     {
239         picture_Release( p_pic );
240         return NULL;
241     }
242
243     switch( p_filter->p_sys->i_mode )
244     {
245         case EDGE:
246             FilterEdge( p_filter, p_pic, p_outpic );
247             break;
248
249         case GRADIENT:
250             FilterGradient( p_filter, p_pic, p_outpic );
251             break;
252
253         case HOUGH:
254             FilterHough( p_filter, p_pic, p_outpic );
255             break;
256
257         default:
258             break;
259     }
260
261     return CopyInfoAndRelease( p_outpic, p_pic );
262 }
263
264 /*****************************************************************************
265  * Gaussian Convolution
266  *****************************************************************************
267  *    Gaussian convolution ( sigma == 1.4 )
268  *
269  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
270  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
271  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
272  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
273  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
274  *****************************************************************************/
275 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
276 {
277     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
278     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
279     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
280     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
281
282     int x,y;
283     for( y = 2; y < i_num_lines - 2; y++ )
284     {
285         for( x = 2; x < i_src_visible - 2; x++ )
286         {
287             p_smooth[y*i_src_visible+x] = (uint32_t)(
288               /* 2 rows up */
289                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
290               + ((p_inpix[(y-2)*i_src_pitch+x-1]
291               +   p_inpix[(y-2)*i_src_pitch+x]
292               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
293               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
294               /* 1 row up */
295               + ((p_inpix[(y-1)*i_src_pitch+x-2]
296               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
297               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
298               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
299               +   p_inpix[(y-1)*i_src_pitch+x+2]
300               /* */
301               +   p_inpix[y*i_src_pitch+x-2]
302               + ( p_inpix[y*i_src_pitch+x-1]*3 )
303               + ( p_inpix[y*i_src_pitch+x]<<2 )
304               + ( p_inpix[y*i_src_pitch+x+1]*3 )
305               +   p_inpix[y*i_src_pitch+x+2]
306               /* 1 row down */
307               +   p_inpix[(y+1)*i_src_pitch+x-2]
308               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
309               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
310               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
311               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
312               /* 2 rows down */
313               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
314               + ((p_inpix[(y+2)*i_src_pitch+x-1]
315               +   p_inpix[(y+2)*i_src_pitch+x]
316               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
317               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
318               ) >> 6 /* 115 */;
319         }
320     }
321 }
322
323 /*****************************************************************************
324  * FilterGradient: Sobel
325  *****************************************************************************/
326 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
327                                                 picture_t *p_outpic )
328 {
329     int x, y;
330     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
331     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
332     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
333     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
334
335     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
336     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
337
338     uint32_t *p_smooth;
339     if( !p_filter->p_sys->p_buf32 )
340         p_filter->p_sys->p_buf32 =
341         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
342     p_smooth = p_filter->p_sys->p_buf32;
343
344     if( !p_smooth ) return;
345
346     if( p_filter->p_sys->b_cartoon )
347     {
348         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
349             p_inpic->p[U_PLANE].p_pixels,
350             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
351         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
352             p_inpic->p[V_PLANE].p_pixels,
353             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
354     }
355     else
356     {
357         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
358             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
359         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
360             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
361     }
362
363     GaussianConvolution( p_inpic, p_smooth );
364
365     /* Sobel gradient
366
367      | -1 0 1 |     |  1  2  1 |
368      | -2 0 2 | and |  0  0  0 |
369      | -1 0 1 |     | -1 -2 -1 | */
370
371 #define FOR                                                     \
372     for( y = 1; y < i_num_lines - 1; y++ )                      \
373     {                                                           \
374         for( x = 1; x < i_src_visible - 1; x++ )                \
375         {                                                       \
376             const uint32_t a =                                  \
377             (                                                   \
378               abs(                                              \
379                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
380                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
381                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
382                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
383                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
384                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
385               )                                                 \
386             +                                                   \
387               abs(                                              \
388                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
389                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
390                + ( ( p_smooth[y*i_src_visible+x-1]              \
391                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
392                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
393                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
394               )                                                 \
395             );
396     if( p_filter->p_sys->i_gradient_type )
397     {
398         if( p_filter->p_sys->b_cartoon )
399         {
400             FOR
401             if( a > 60 )
402             {
403                 p_outpix[y*i_dst_pitch+x] = 0x00;
404             }
405             else
406             {
407                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
408                     p_outpix[y*i_dst_pitch+x] =
409                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
410                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
411                     p_outpix[y*i_dst_pitch+x] =
412                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
413                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
414                     p_outpix[y*i_dst_pitch+x] =
415                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
416                 else
417                     p_outpix[y*i_dst_pitch+x] =
418                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
419             }
420             }}
421         }
422         else
423         {
424             FOR
425             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
426             }}
427         }
428     }
429     else
430     {
431         FOR
432         if( a>>8 )
433             p_outpix[y*i_dst_pitch+x] = 0;
434         else
435             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
436         }}
437     }
438 #undef FOR
439 }
440
441 /*****************************************************************************
442  * FilterEdge: Canny edge detection algorithm
443  *****************************************************************************
444  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
445  * (well ... my implementation isn't really the canny algorithm ... but some
446  * ideas are the same)
447  *****************************************************************************/
448 /* angle : | */
449 #define THETA_Y 0
450 /* angle : - */
451 #define THETA_X 1
452 /* angle : / */
453 #define THETA_P 2
454 /* angle : \ */
455 #define THETA_M 3
456 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
457                                             picture_t *p_outpic )
458 {
459     int x, y;
460
461     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
462     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
463     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
464     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
465
466     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
467     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
468
469     uint32_t *p_smooth;
470     uint32_t *p_grad;
471     uint8_t *p_theta;
472
473     if( !p_filter->p_sys->p_buf32 )
474         p_filter->p_sys->p_buf32 =
475         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
476     p_smooth = p_filter->p_sys->p_buf32;
477
478     if( !p_filter->p_sys->p_buf32_bis )
479         p_filter->p_sys->p_buf32_bis =
480         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
481     p_grad = p_filter->p_sys->p_buf32_bis;
482
483     if( !p_filter->p_sys->p_buf8 )
484         p_filter->p_sys->p_buf8 =
485         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
486     p_theta = p_filter->p_sys->p_buf8;
487
488     if( !p_smooth || !p_grad || !p_theta ) return;
489
490     if( p_filter->p_sys->b_cartoon )
491     {
492         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
493             p_inpic->p[U_PLANE].p_pixels,
494             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
495         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
496             p_inpic->p[V_PLANE].p_pixels,
497             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
498     }
499     else
500     {
501         vlc_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
502               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
503         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
504             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
505         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
506             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
507     }
508
509     GaussianConvolution( p_inpic, p_smooth );
510
511     /* Sobel gradient
512
513      | -1 0 1 |     |  1  2  1 |
514      | -2 0 2 | and |  0  0  0 |
515      | -1 0 1 |     | -1 -2 -1 | */
516
517     for( y = 1; y < i_num_lines - 1; y++ )
518     {
519         for( x = 1; x < i_src_visible - 1; x++ )
520         {
521
522             const int gradx =
523                  ( p_smooth[(y-1)*i_src_visible+x-1]
524                    - p_smooth[(y+1)*i_src_visible+x-1] )
525                + ( ( p_smooth[(y-1)*i_src_visible+x]
526                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
527                + ( p_smooth[(y-1)*i_src_visible+x+1]
528                    - p_smooth[(y+1)*i_src_visible+x+1] );
529             const int grady =
530                  ( p_smooth[(y-1)*i_src_visible+x-1]
531                    - p_smooth[(y-1)*i_src_visible+x+1] )
532                + ( ( p_smooth[y*i_src_visible+x-1]
533                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
534                + ( p_smooth[(y+1)*i_src_visible+x-1]
535                    - p_smooth[(y+1)*i_src_visible+x+1] );
536
537             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
538
539             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
540              * tan( 26,565051177 ) = 0.5
541              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
542              * tan( 63,434948823 ) 2 */
543             if( (grady<<1) > gradx )
544                 p_theta[y*i_src_visible+x] = THETA_P;
545             else if( (grady<<1) < -gradx )
546                 p_theta[y*i_src_visible+x] = THETA_M;
547             else if( !gradx || abs(grady) > abs(gradx)<<1 )
548                 p_theta[y*i_src_visible+x] = THETA_Y;
549             else
550                 p_theta[y*i_src_visible+x] = THETA_X;
551         }
552     }
553
554     /* edge computing */
555     for( y = 1; y < i_num_lines - 1; y++ )
556     {
557         for( x = 1; x < i_src_visible - 1; x++ )
558         {
559             if( p_grad[y*i_src_visible+x] > 40 )
560             {
561                 switch( p_theta[y*i_src_visible+x] )
562                 {
563                     case THETA_Y:
564                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
565                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
566                         {
567                             p_outpix[y*i_dst_pitch+x] = 0;
568                             break;
569                         } else goto colorize;
570                     case THETA_P:
571                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
572                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
573                         {
574                             p_outpix[y*i_dst_pitch+x] = 0;
575                             break;
576                         } else goto colorize;
577                     case THETA_M:
578                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
579                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
580                         {
581                             p_outpix[y*i_dst_pitch+x] = 0;
582                             break;
583                         } else goto colorize;
584                     case THETA_X:
585                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
586                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
587                         {
588                             p_outpix[y*i_dst_pitch+x] = 0;
589                             break;
590                         } else goto colorize;
591                 }
592             }
593             else
594             {
595                 colorize:
596                 if( p_filter->p_sys->b_cartoon )
597                 {
598                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
599                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
600                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
601                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
602                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
603                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
604                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
605                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
606                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
607                     else
608                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
609                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
610                 }
611             }
612         }
613     }
614 }
615
616 /*****************************************************************************
617  * FilterHough
618  *****************************************************************************/
619 #define p_pre_hough p_filter->p_sys->p_pre_hough
620 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
621                                              picture_t *p_outpic )
622 {
623     int x, y, i;
624     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
625     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
626     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
627
628     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
629
630     int i_diag = sqrt( i_num_lines * i_num_lines +
631                         i_src_visible * i_src_visible);
632     int i_max, i_phi_max, i_rho, i_rho_max;
633     int i_nb_steps = 90;
634     double d_step = M_PI / i_nb_steps;
635     double d_sin;
636     double d_cos;
637     uint32_t *p_smooth;
638
639     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
640     if( ! p_hough ) return;
641
642     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
643     if( !p_smooth )
644     {
645         free( p_hough );
646         return;
647     }
648
649     if( ! p_pre_hough )
650     {
651         msg_Dbg(p_filter, "Starting precalculation");
652         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
653         if( ! p_pre_hough )
654         {
655             free( p_smooth );
656             free( p_hough );
657             return;
658         }
659         for( i = 0 ; i < i_nb_steps ; i++)
660         {
661             d_sin = sin(d_step * i);
662             d_cos = cos(d_step * i);
663             for( y = 0 ; y < i_num_lines ; y++ )
664                 for( x = 0 ; x < i_src_visible ; x++ )
665                 {
666                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
667                         ceil(x*d_sin + y*d_cos);
668                 }
669         }
670         msg_Dbg(p_filter, "Precalculation done");
671     }
672
673     vlc_memset( p_hough, 0, i_diag * i_nb_steps * sizeof(int) );
674
675     vlc_memcpy(
676         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
677         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
678     vlc_memcpy(
679         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
680         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
681     vlc_memcpy(
682         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
683         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
684
685     GaussianConvolution( p_inpic, p_smooth );
686
687     /* Sobel gradient
688
689      | -1 0 1 |     |  1  2  1 |
690      | -2 0 2 | and |  0  0  0 |
691      | -1 0 1 |     | -1 -2 -1 | */
692
693     i_max = 0;
694     i_rho_max = 0;
695     i_phi_max = 0;
696     for( y = 4; y < i_num_lines - 4; y++ )
697     {
698         for( x = 4; x < i_src_visible - 4; x++ )
699         {
700             uint32_t a =
701             (
702               abs(
703                 ( ( p_smooth[(y-1)*i_src_visible+x]
704                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
705                + ( p_smooth[(y-1)*i_src_visible+x-1]
706                    - p_smooth[(y+1)*i_src_visible+x-1] )
707                + ( p_smooth[(y-1)*i_src_visible+x+1]
708                    - p_smooth[(y+1)*i_src_visible+x+1] )
709               )
710             +
711               abs(
712                 ( ( p_smooth[y*i_src_visible+x-1]
713                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
714                + ( p_smooth[(y-1)*i_src_visible+x-1]
715                    - p_smooth[(y-1)*i_src_visible+x+1] )
716                + ( p_smooth[(y+1)*i_src_visible+x-1]
717                    - p_smooth[(y+1)*i_src_visible+x+1] )
718               )
719             );
720             if( a>>8 )
721             {
722                 for( i = 0 ; i < i_nb_steps ; i ++ )
723                 {
724                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
725                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
726                     {
727                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
728                         i_rho_max = i_rho;
729                         i_phi_max = i;
730                     }
731                 }
732             }
733         }
734     }
735
736     d_sin = sin(i_phi_max*d_step);
737     d_cos = cos(i_phi_max*d_step);
738     if( d_cos != 0 )
739     {
740         for( x = 0 ; x < i_src_visible ; x++ )
741         {
742             y = (i_rho_max - x * d_sin) / d_cos;
743             if( y >= 0 && y < i_num_lines )
744                 p_outpix[y*i_dst_pitch+x] = 255;
745         }
746     }
747
748     free( p_hough );
749     free( p_smooth );
750 }
751 #undef p_pre_hough
752
753
754 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
755                              vlc_value_t oldval, vlc_value_t newval,
756                              void *p_data )
757 {
758     VLC_UNUSED(oldval);
759     filter_sys_t *p_sys = (filter_sys_t *)p_data;
760     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
761     {
762         if( !strcmp( newval.psz_string, "gradient" ) )
763         {
764             p_sys->i_mode = GRADIENT;
765         }
766         else if( !strcmp( newval.psz_string, "edge" ) )
767         {
768             p_sys->i_mode = EDGE;
769         }
770         else if( !strcmp( newval.psz_string, "hough" ) )
771         {
772             p_sys->i_mode = HOUGH;
773         }
774         else
775         {
776             msg_Err( p_this, "no valid gradient mode provided (%s)", newval.psz_string );
777             p_sys->i_mode = GRADIENT;
778         }
779     }
780     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
781     {
782         p_sys->i_gradient_type = newval.i_int;
783     }
784     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
785     {
786         p_sys->b_cartoon = newval.b_bool;
787     }
788     return VLC_SUCCESS;
789 }