Remove unnedeeded msg_Error.
[vlc.git] / modules / video_filter / gradient.c
1 /*****************************************************************************
2  * gradient.c : Gradient and edge detection video effects plugin for vlc
3  *****************************************************************************
4  * Copyright (C) 2000-2008 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Samuel Hocevar <sam@zoy.org>
8  *          Antoine Cellerier <dionoea -at- videolan -dot- org>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
23  *****************************************************************************/
24
25 /*****************************************************************************
26  * Preamble
27  *****************************************************************************/
28
29 #include <math.h>                                            /* sin(), cos() */
30
31 #ifdef HAVE_CONFIG_H
32 # include "config.h"
33 #endif
34
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
37 #include <vlc_sout.h>
38 #include <vlc_vout.h>
39
40 #include "vlc_filter.h"
41 #include "filter_picture.h"
42
43 enum { GRADIENT, EDGE, HOUGH };
44
45 /*****************************************************************************
46  * Local prototypes
47  *****************************************************************************/
48 static int  Create    ( vlc_object_t * );
49 static void Destroy   ( vlc_object_t * );
50
51 static picture_t *Filter( filter_t *, picture_t * );
52 static int GradientCallback( vlc_object_t *, char const *,
53                              vlc_value_t, vlc_value_t,
54                              void * );
55
56 static void FilterGradient( filter_t *, picture_t *, picture_t * );
57 static void FilterEdge    ( filter_t *, picture_t *, picture_t * );
58 static void FilterHough   ( filter_t *, picture_t *, picture_t * );
59
60 /*****************************************************************************
61  * Module descriptor
62  *****************************************************************************/
63 #define MODE_TEXT N_("Distort mode")
64 #define MODE_LONGTEXT N_("Distort mode, one of \"gradient\", \"edge\" and \"hough\".")
65
66 #define GRADIENT_TEXT N_("Gradient image type")
67 #define GRADIENT_LONGTEXT N_("Gradient image type (0 or 1). 0 will " \
68         "turn the image to white while 1 will keep colors." )
69
70 #define CARTOON_TEXT N_("Apply cartoon effect")
71 #define CARTOON_LONGTEXT N_("Apply cartoon effect. It is only used by " \
72     "\"gradient\" and \"edge\".")
73
74 static const char *const mode_list[] = { "gradient", "edge", "hough" };
75 static const char *const mode_list_text[] = { N_("Gradient"), N_("Edge"), N_("Hough") };
76
77 #define FILTER_PREFIX "gradient-"
78
79 vlc_module_begin();
80     set_description( N_("Gradient video filter") );
81     set_shortname( N_( "Gradient" ));
82     set_capability( "video filter2", 0 );
83     set_category( CAT_VIDEO );
84     set_subcategory( SUBCAT_VIDEO_VFILTER );
85
86     add_string( FILTER_PREFIX "mode", "gradient", NULL,
87                 MODE_TEXT, MODE_LONGTEXT, false );
88         change_string_list( mode_list, mode_list_text, 0 );
89
90     add_integer_with_range( FILTER_PREFIX "type", 0, 0, 1, NULL,
91                 GRADIENT_TEXT, GRADIENT_LONGTEXT, false );
92     add_bool( FILTER_PREFIX "cartoon", 1, NULL,
93                 CARTOON_TEXT, CARTOON_LONGTEXT, false );
94
95     add_shortcut( "gradient" );
96     set_callbacks( Create, Destroy );
97 vlc_module_end();
98
99 static const char *const ppsz_filter_options[] = {
100     "mode", "type", "cartoon", NULL
101 };
102
103 /*****************************************************************************
104  * vout_sys_t: Distort video output method descriptor
105  *****************************************************************************
106  * This structure is part of the video output thread descriptor.
107  * It describes the Distort specific properties of an output thread.
108  *****************************************************************************/
109 struct filter_sys_t
110 {
111     int i_mode;
112
113     /* For the gradient mode */
114     int i_gradient_type;
115     bool b_cartoon;
116
117     uint32_t *p_buf32;
118     uint32_t *p_buf32_bis;
119     uint8_t *p_buf8;
120
121     /* For hough mode */
122     int *p_pre_hough;
123 };
124
125 /*****************************************************************************
126  * Create: allocates Distort video thread output method
127  *****************************************************************************
128  * This function allocates and initializes a Distort vout method.
129  *****************************************************************************/
130 static int Create( vlc_object_t *p_this )
131 {
132     filter_t *p_filter = (filter_t *)p_this;
133     char *psz_method;
134
135     switch( p_filter->fmt_in.video.i_chroma )
136     {
137         CASE_PLANAR_YUV
138             break;
139
140         default:
141              msg_Err( p_filter, "Unsupported input chroma (%4s)",
142                       (char*)&(p_filter->fmt_in.video.i_chroma) );
143             return VLC_EGENERIC;
144     }
145
146     /* Allocate structure */
147     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
148     if( p_filter->p_sys == NULL )
149         return VLC_ENOMEM;
150
151     p_filter->pf_video_filter = Filter;
152
153     p_filter->p_sys->p_pre_hough = NULL;
154
155     config_ChainParse( p_filter, FILTER_PREFIX, ppsz_filter_options,
156                    p_filter->p_cfg );
157
158     if( !(psz_method =
159         var_CreateGetNonEmptyStringCommand( p_filter, FILTER_PREFIX "mode" )) )
160     {
161         msg_Err( p_filter, "configuration variable "
162                  FILTER_PREFIX "mode empty" );
163         p_filter->p_sys->i_mode = GRADIENT;
164     }
165     else
166     {
167         if( !strcmp( psz_method, "gradient" ) )
168         {
169             p_filter->p_sys->i_mode = GRADIENT;
170         }
171         else if( !strcmp( psz_method, "edge" ) )
172         {
173             p_filter->p_sys->i_mode = EDGE;
174         }
175         else if( !strcmp( psz_method, "hough" ) )
176         {
177             p_filter->p_sys->i_mode = HOUGH;
178         }
179         else
180         {
181             msg_Err( p_filter, "no valid gradient mode provided (%s)", psz_method );
182             p_filter->p_sys->i_mode = GRADIENT;
183         }
184     }
185     free( psz_method );
186
187     p_filter->p_sys->i_gradient_type =
188         var_CreateGetIntegerCommand( p_filter, FILTER_PREFIX "type" );
189     p_filter->p_sys->b_cartoon =
190         var_CreateGetBoolCommand( p_filter, FILTER_PREFIX "cartoon" );
191
192     var_AddCallback( p_filter, FILTER_PREFIX "mode",
193                      GradientCallback, p_filter->p_sys );
194     var_AddCallback( p_filter, FILTER_PREFIX "type",
195                      GradientCallback, p_filter->p_sys );
196     var_AddCallback( p_filter, FILTER_PREFIX "cartoon",
197                      GradientCallback, p_filter->p_sys );
198
199     p_filter->p_sys->p_buf32 = NULL;
200     p_filter->p_sys->p_buf32_bis = NULL;
201     p_filter->p_sys->p_buf8 = NULL;
202
203     return VLC_SUCCESS;
204 }
205
206 /*****************************************************************************
207  * Destroy: destroy Distort video thread output method
208  *****************************************************************************
209  * Terminate an output method created by DistortCreateOutputMethod
210  *****************************************************************************/
211 static void Destroy( vlc_object_t *p_this )
212 {
213     filter_t *p_filter = (filter_t *)p_this;
214
215     free( p_filter->p_sys->p_buf32 );
216     free( p_filter->p_sys->p_buf32_bis );
217     free( p_filter->p_sys->p_buf8 );
218     free( p_filter->p_sys->p_pre_hough );
219
220     free( p_filter->p_sys );
221 }
222
223 /*****************************************************************************
224  * Render: displays previously rendered output
225  *****************************************************************************
226  * This function send the currently rendered image to Distort image, waits
227  * until it is displayed and switch the two rendering buffers, preparing next
228  * frame.
229  *****************************************************************************/
230 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
231 {
232     picture_t *p_outpic;
233
234     if( !p_pic ) return NULL;
235
236     p_outpic = p_filter->pf_vout_buffer_new( p_filter );
237     if( !p_outpic )
238     {
239         msg_Warn( p_filter, "can't get output picture" );
240         if( p_pic->pf_release )
241             p_pic->pf_release( p_pic );
242         return NULL;
243     }
244
245     switch( p_filter->p_sys->i_mode )
246     {
247         case EDGE:
248             FilterEdge( p_filter, p_pic, p_outpic );
249             break;
250
251         case GRADIENT:
252             FilterGradient( p_filter, p_pic, p_outpic );
253             break;
254
255         case HOUGH:
256             FilterHough( p_filter, p_pic, p_outpic );
257             break;
258
259         default:
260             break;
261     }
262
263     return CopyInfoAndRelease( p_outpic, p_pic );
264 }
265
266 /*****************************************************************************
267  * Gaussian Convolution
268  *****************************************************************************
269  *    Gaussian convolution ( sigma == 1.4 )
270  *
271  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
272  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
273  *    |  5 12 15 12  5  | ~ |  4 12 16 12  4 |
274  *    |  4  9 12  9  4  |   |  4  8 12  8  4 |
275  *    |  2  4  5  4  2  |   |  2  4  4  4  2 |
276  *****************************************************************************/
277 static void GaussianConvolution( picture_t *p_inpic, uint32_t *p_smooth )
278 {
279     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
280     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
281     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
282     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
283
284     int x,y;
285     for( y = 2; y < i_num_lines - 2; y++ )
286     {
287         for( x = 2; x < i_src_visible - 2; x++ )
288         {
289             p_smooth[y*i_src_visible+x] = (uint32_t)(
290               /* 2 rows up */
291                 ( p_inpix[(y-2)*i_src_pitch+x-2] )
292               + ((p_inpix[(y-2)*i_src_pitch+x-1]
293               +   p_inpix[(y-2)*i_src_pitch+x]
294               +   p_inpix[(y-2)*i_src_pitch+x+1])<<1 )
295               + ( p_inpix[(y-2)*i_src_pitch+x+2] )
296               /* 1 row up */
297               + ((p_inpix[(y-1)*i_src_pitch+x-2]
298               + ( p_inpix[(y-1)*i_src_pitch+x-1]<<1 )
299               + ( p_inpix[(y-1)*i_src_pitch+x]*3 )
300               + ( p_inpix[(y-1)*i_src_pitch+x+1]<<1 )
301               +   p_inpix[(y-1)*i_src_pitch+x+2]
302               /* */
303               +   p_inpix[y*i_src_pitch+x-2]
304               + ( p_inpix[y*i_src_pitch+x-1]*3 )
305               + ( p_inpix[y*i_src_pitch+x]<<2 )
306               + ( p_inpix[y*i_src_pitch+x+1]*3 )
307               +   p_inpix[y*i_src_pitch+x+2]
308               /* 1 row down */
309               +   p_inpix[(y+1)*i_src_pitch+x-2]
310               + ( p_inpix[(y+1)*i_src_pitch+x-1]<<1 )
311               + ( p_inpix[(y+1)*i_src_pitch+x]*3 )
312               + ( p_inpix[(y+1)*i_src_pitch+x+1]<<1 )
313               +   p_inpix[(y+1)*i_src_pitch+x+2] )<<1 )
314               /* 2 rows down */
315               + ( p_inpix[(y+2)*i_src_pitch+x-2] )
316               + ((p_inpix[(y+2)*i_src_pitch+x-1]
317               +   p_inpix[(y+2)*i_src_pitch+x]
318               +   p_inpix[(y+2)*i_src_pitch+x+1])<<1 )
319               + ( p_inpix[(y+2)*i_src_pitch+x+2] )
320               ) >> 6 /* 115 */;
321         }
322     }
323 }
324
325 /*****************************************************************************
326  * FilterGradient: Sobel
327  *****************************************************************************/
328 static void FilterGradient( filter_t *p_filter, picture_t *p_inpic,
329                                                 picture_t *p_outpic )
330 {
331     int x, y;
332     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
333     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
334     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
335     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
336
337     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
338     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
339
340     uint32_t *p_smooth;
341     if( !p_filter->p_sys->p_buf32 )
342         p_filter->p_sys->p_buf32 =
343         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
344     p_smooth = p_filter->p_sys->p_buf32;
345
346     if( !p_smooth ) return;
347
348     if( p_filter->p_sys->b_cartoon )
349     {
350         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
351             p_inpic->p[U_PLANE].p_pixels,
352             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
353         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
354             p_inpic->p[V_PLANE].p_pixels,
355             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
356     }
357     else
358     {
359         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
360             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
361         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
362             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
363     }
364
365     GaussianConvolution( p_inpic, p_smooth );
366
367     /* Sobel gradient
368
369      | -1 0 1 |     |  1  2  1 |
370      | -2 0 2 | and |  0  0  0 |
371      | -1 0 1 |     | -1 -2 -1 | */
372
373 #define FOR                                                     \
374     for( y = 1; y < i_num_lines - 1; y++ )                      \
375     {                                                           \
376         for( x = 1; x < i_src_visible - 1; x++ )                \
377         {                                                       \
378             const uint32_t a =                                  \
379             (                                                   \
380               abs(                                              \
381                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
382                    - p_smooth[(y+1)*i_src_visible+x-1] )        \
383                + ( ( p_smooth[(y-1)*i_src_visible+x]            \
384                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )   \
385                + ( p_smooth[(y-1)*i_src_visible+x+1]            \
386                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
387               )                                                 \
388             +                                                   \
389               abs(                                              \
390                  ( p_smooth[(y-1)*i_src_visible+x-1]            \
391                    - p_smooth[(y-1)*i_src_visible+x+1] )        \
392                + ( ( p_smooth[y*i_src_visible+x-1]              \
393                     - p_smooth[y*i_src_visible+x+1] ) <<1 )     \
394                + ( p_smooth[(y+1)*i_src_visible+x-1]            \
395                    - p_smooth[(y+1)*i_src_visible+x+1] )        \
396               )                                                 \
397             );
398     if( p_filter->p_sys->i_gradient_type )
399     {
400         if( p_filter->p_sys->b_cartoon )
401         {
402             FOR
403             if( a > 60 )
404             {
405                 p_outpix[y*i_dst_pitch+x] = 0x00;
406             }
407             else
408             {
409                 if( p_smooth[y*i_src_visible+x] > 0xa0 )
410                     p_outpix[y*i_dst_pitch+x] =
411                         0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
412                 else if( p_smooth[y*i_src_visible+x] > 0x70 )
413                     p_outpix[y*i_dst_pitch+x] =
414                         0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
415                 else if( p_smooth[y*i_src_visible+x] > 0x28 )
416                     p_outpix[y*i_dst_pitch+x] =
417                         0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
418                 else
419                     p_outpix[y*i_dst_pitch+x] =
420                         0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
421             }
422             }}
423         }
424         else
425         {
426             FOR
427             p_outpix[y*i_dst_pitch+x] = clip_uint8_vlc( a );
428             }}
429         }
430     }
431     else
432     {
433         FOR
434         if( a>>8 )
435             p_outpix[y*i_dst_pitch+x] = 0;
436         else
437             p_outpix[y*i_dst_pitch+x] = 0xff-(uint8_t)a;
438         }}
439     }
440 #undef FOR
441 }
442
443 /*****************************************************************************
444  * FilterEdge: Canny edge detection algorithm
445  *****************************************************************************
446  * http://fourier.eng.hmc.edu/e161/lectures/canny/node1.html
447  * (well ... my implementation isn't really the canny algorithm ... but some
448  * ideas are the same)
449  *****************************************************************************/
450 /* angle : | */
451 #define THETA_Y 0
452 /* angle : - */
453 #define THETA_X 1
454 /* angle : / */
455 #define THETA_P 2
456 /* angle : \ */
457 #define THETA_M 3
458 static void FilterEdge( filter_t *p_filter, picture_t *p_inpic,
459                                             picture_t *p_outpic )
460 {
461     int x, y;
462
463     const int i_src_pitch = p_inpic->p[Y_PLANE].i_pitch;
464     const int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
465     const int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
466     const int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
467
468     const uint8_t *p_inpix = p_inpic->p[Y_PLANE].p_pixels;
469     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
470
471     uint32_t *p_smooth;
472     uint32_t *p_grad;
473     uint8_t *p_theta;
474
475     if( !p_filter->p_sys->p_buf32 )
476         p_filter->p_sys->p_buf32 =
477         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
478     p_smooth = p_filter->p_sys->p_buf32;
479
480     if( !p_filter->p_sys->p_buf32_bis )
481         p_filter->p_sys->p_buf32_bis =
482         (uint32_t *)malloc( i_num_lines * i_src_visible * sizeof(uint32_t));
483     p_grad = p_filter->p_sys->p_buf32_bis;
484
485     if( !p_filter->p_sys->p_buf8 )
486         p_filter->p_sys->p_buf8 =
487         (uint8_t *)malloc( i_num_lines * i_src_visible * sizeof(uint8_t));
488     p_theta = p_filter->p_sys->p_buf8;
489
490     if( !p_smooth || !p_grad || !p_theta ) return;
491
492     if( p_filter->p_sys->b_cartoon )
493     {
494         vlc_memcpy( p_outpic->p[U_PLANE].p_pixels,
495             p_inpic->p[U_PLANE].p_pixels,
496             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
497         vlc_memcpy( p_outpic->p[V_PLANE].p_pixels,
498             p_inpic->p[V_PLANE].p_pixels,
499             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
500     }
501     else
502     {
503         vlc_memset( p_outpic->p[Y_PLANE].p_pixels, 0xff,
504               p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
505         vlc_memset( p_outpic->p[U_PLANE].p_pixels, 0x80,
506             p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
507         vlc_memset( p_outpic->p[V_PLANE].p_pixels, 0x80,
508             p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
509     }
510
511     GaussianConvolution( p_inpic, p_smooth );
512
513     /* Sobel gradient
514
515      | -1 0 1 |     |  1  2  1 |
516      | -2 0 2 | and |  0  0  0 |
517      | -1 0 1 |     | -1 -2 -1 | */
518
519     for( y = 1; y < i_num_lines - 1; y++ )
520     {
521         for( x = 1; x < i_src_visible - 1; x++ )
522         {
523
524             const int gradx =
525                  ( p_smooth[(y-1)*i_src_visible+x-1]
526                    - p_smooth[(y+1)*i_src_visible+x-1] )
527                + ( ( p_smooth[(y-1)*i_src_visible+x]
528                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
529                + ( p_smooth[(y-1)*i_src_visible+x+1]
530                    - p_smooth[(y+1)*i_src_visible+x+1] );
531             const int grady =
532                  ( p_smooth[(y-1)*i_src_visible+x-1]
533                    - p_smooth[(y-1)*i_src_visible+x+1] )
534                + ( ( p_smooth[y*i_src_visible+x-1]
535                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
536                + ( p_smooth[(y+1)*i_src_visible+x-1]
537                    - p_smooth[(y+1)*i_src_visible+x+1] );
538
539             p_grad[y*i_src_visible+x] = (uint32_t)(abs( gradx ) + abs( grady ));
540
541             /* tan( 22.5 ) = 0,414213562 .. * 128 = 53
542              * tan( 26,565051177 ) = 0.5
543              * tan( 45 + 22.5 ) = 2,414213562 .. * 128 = 309
544              * tan( 63,434948823 ) 2 */
545             if( (grady<<1) > gradx )
546                 p_theta[y*i_src_visible+x] = THETA_P;
547             else if( (grady<<1) < -gradx )
548                 p_theta[y*i_src_visible+x] = THETA_M;
549             else if( !gradx || abs(grady) > abs(gradx)<<1 )
550                 p_theta[y*i_src_visible+x] = THETA_Y;
551             else
552                 p_theta[y*i_src_visible+x] = THETA_X;
553         }
554     }
555
556     /* edge computing */
557     for( y = 1; y < i_num_lines - 1; y++ )
558     {
559         for( x = 1; x < i_src_visible - 1; x++ )
560         {
561             if( p_grad[y*i_src_visible+x] > 40 )
562             {
563                 switch( p_theta[y*i_src_visible+x] )
564                 {
565                     case THETA_Y:
566                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x]
567                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x] )
568                         {
569                             p_outpix[y*i_dst_pitch+x] = 0;
570                             break;
571                         } else goto colorize;
572                     case THETA_P:
573                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x-1]
574                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x+1] )
575                         {
576                             p_outpix[y*i_dst_pitch+x] = 0;
577                             break;
578                         } else goto colorize;
579                     case THETA_M:
580                         if(    p_grad[y*i_src_visible+x] > p_grad[(y-1)*i_src_visible+x+1]
581                             && p_grad[y*i_src_visible+x] > p_grad[(y+1)*i_src_visible+x-1] )
582                         {
583                             p_outpix[y*i_dst_pitch+x] = 0;
584                             break;
585                         } else goto colorize;
586                     case THETA_X:
587                         if(    p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x-1]
588                             && p_grad[y*i_src_visible+x] > p_grad[y*i_src_visible+x+1] )
589                         {
590                             p_outpix[y*i_dst_pitch+x] = 0;
591                             break;
592                         } else goto colorize;
593                 }
594             }
595             else
596             {
597                 colorize:
598                 if( p_filter->p_sys->b_cartoon )
599                 {
600                     if( p_smooth[y*i_src_visible+x] > 0xa0 )
601                         p_outpix[y*i_dst_pitch+x] = (uint8_t)
602                             0xff - ((0xff - p_inpix[y*i_src_pitch+x] )>>2);
603                     else if( p_smooth[y*i_src_visible+x] > 0x70 )
604                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
605                             0xa0 - ((0xa0 - p_inpix[y*i_src_pitch+x] )>>2);
606                     else if( p_smooth[y*i_src_visible+x] > 0x28 )
607                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
608                             0x70 - ((0x70 - p_inpix[y*i_src_pitch+x] )>>2);
609                     else
610                         p_outpix[y*i_dst_pitch+x] =(uint8_t)
611                             0x28 - ((0x28 - p_inpix[y*i_src_pitch+x] )>>2);
612                 }
613             }
614         }
615     }
616 }
617
618 /*****************************************************************************
619  * FilterHough
620  *****************************************************************************/
621 #define p_pre_hough p_filter->p_sys->p_pre_hough
622 static void FilterHough( filter_t *p_filter, picture_t *p_inpic,
623                                              picture_t *p_outpic )
624 {
625     int x, y, i;
626     int i_src_visible = p_inpic->p[Y_PLANE].i_visible_pitch;
627     int i_dst_pitch = p_outpic->p[Y_PLANE].i_pitch;
628     int i_num_lines = p_inpic->p[Y_PLANE].i_visible_lines;
629
630     uint8_t *p_outpix = p_outpic->p[Y_PLANE].p_pixels;
631
632     int i_diag = sqrt( i_num_lines * i_num_lines +
633                         i_src_visible * i_src_visible);
634     int i_max, i_phi_max, i_rho, i_rho_max;
635     int i_nb_steps = 90;
636     double d_step = M_PI / i_nb_steps;
637     double d_sin;
638     double d_cos;
639     uint32_t *p_smooth;
640     int *p_hough = malloc( i_diag * i_nb_steps * sizeof(int) );
641     if( ! p_hough ) return;
642     p_smooth = (uint32_t *)malloc( i_num_lines*i_src_visible*sizeof(uint32_t));
643     if( !p_smooth ) return;
644
645     if( ! p_pre_hough )
646     {
647         msg_Dbg(p_filter, "Starting precalculation");
648         p_pre_hough = malloc( i_num_lines*i_src_visible*i_nb_steps*sizeof(int));
649         if( ! p_pre_hough ) return;
650         for( i = 0 ; i < i_nb_steps ; i++)
651         {
652             d_sin = sin(d_step * i);
653             d_cos = cos(d_step * i);
654             for( y = 0 ; y < i_num_lines ; y++ )
655                 for( x = 0 ; x < i_src_visible ; x++ )
656                 {
657                     p_pre_hough[(i*i_num_lines+y)*i_src_visible + x] =
658                         ceil(x*d_sin + y*d_cos);
659                 }
660         }
661         msg_Dbg(p_filter, "Precalculation done");
662     }
663
664     vlc_memset( p_hough, 0, i_diag * i_nb_steps * sizeof(int) );
665
666     vlc_memcpy(
667         p_outpic->p[Y_PLANE].p_pixels, p_inpic->p[Y_PLANE].p_pixels,
668         p_outpic->p[Y_PLANE].i_lines * p_outpic->p[Y_PLANE].i_pitch );
669     vlc_memcpy(
670         p_outpic->p[U_PLANE].p_pixels, p_inpic->p[U_PLANE].p_pixels,
671         p_outpic->p[U_PLANE].i_lines * p_outpic->p[U_PLANE].i_pitch );
672     vlc_memcpy(
673         p_outpic->p[V_PLANE].p_pixels, p_inpic->p[V_PLANE].p_pixels,
674         p_outpic->p[V_PLANE].i_lines * p_outpic->p[V_PLANE].i_pitch );
675
676     GaussianConvolution( p_inpic, p_smooth );
677
678     /* Sobel gradient
679
680      | -1 0 1 |     |  1  2  1 |
681      | -2 0 2 | and |  0  0  0 |
682      | -1 0 1 |     | -1 -2 -1 | */
683
684     i_max = 0;
685     i_rho_max = 0;
686     i_phi_max = 0;
687     for( y = 4; y < i_num_lines - 4; y++ )
688     {
689         for( x = 4; x < i_src_visible - 4; x++ )
690         {
691             uint32_t a =
692             (
693               abs(
694                 ( ( p_smooth[(y-1)*i_src_visible+x]
695                     - p_smooth[(y+1)*i_src_visible+x] ) <<1 )
696                + ( p_smooth[(y-1)*i_src_visible+x-1]
697                    - p_smooth[(y+1)*i_src_visible+x-1] )
698                + ( p_smooth[(y-1)*i_src_visible+x+1]
699                    - p_smooth[(y+1)*i_src_visible+x+1] )
700               )
701             +
702               abs(
703                 ( ( p_smooth[y*i_src_visible+x-1]
704                     - p_smooth[y*i_src_visible+x+1] ) <<1 )
705                + ( p_smooth[(y-1)*i_src_visible+x-1]
706                    - p_smooth[(y-1)*i_src_visible+x+1] )
707                + ( p_smooth[(y+1)*i_src_visible+x-1]
708                    - p_smooth[(y+1)*i_src_visible+x+1] )
709               )
710             );
711             if( a>>8 )
712             {
713                 for( i = 0 ; i < i_nb_steps ; i ++ )
714                 {
715                     i_rho = p_pre_hough[(i*i_num_lines+y)*i_src_visible + x];
716                     if( p_hough[i_rho + i_diag/2 + i * i_diag]++ > i_max )
717                     {
718                         i_max = p_hough[i_rho + i_diag/2 + i * i_diag];
719                         i_rho_max = i_rho;
720                         i_phi_max = i;
721                     }
722                 }
723             }
724         }
725     }
726
727     d_sin = sin(i_phi_max*d_step);
728     d_cos = cos(i_phi_max*d_step);
729     if( d_cos != 0 )
730     {
731         for( x = 0 ; x < i_src_visible ; x++ )
732         {
733             y = (i_rho_max - x * d_sin) / d_cos;
734             if( y >= 0 && y < i_num_lines )
735                 p_outpix[y*i_dst_pitch+x] = 255;
736         }
737     }
738
739     free( p_hough );
740     free( p_smooth );
741 }
742 #undef p_pre_hough
743
744
745 static int GradientCallback( vlc_object_t *p_this, char const *psz_var,
746                              vlc_value_t oldval, vlc_value_t newval,
747                              void *p_data )
748 {
749     VLC_UNUSED(oldval);
750     filter_sys_t *p_sys = (filter_sys_t *)p_data;
751     if( !strcmp( psz_var, FILTER_PREFIX "mode" ) )
752     {
753         if( !strcmp( newval.psz_string, "gradient" ) )
754         {
755             p_sys->i_mode = GRADIENT;
756         }
757         else if( !strcmp( newval.psz_string, "edge" ) )
758         {
759             p_sys->i_mode = EDGE;
760         }
761         else if( !strcmp( newval.psz_string, "hough" ) )
762         {
763             p_sys->i_mode = HOUGH;
764         }
765         else
766         {
767             msg_Err( p_this, "no valid gradient mode provided (%s)", newval.psz_string );
768             p_sys->i_mode = GRADIENT;
769         }
770     }
771     else if( !strcmp( psz_var, FILTER_PREFIX "type" ) )
772     {
773         p_sys->i_gradient_type = newval.i_int;
774     }
775     else if( !strcmp( psz_var, FILTER_PREFIX "cartoon" ) )
776     {
777         p_sys->b_cartoon = newval.b_bool;
778     }
779     return VLC_SUCCESS;
780 }