* include/video_output.h, ALL: changed api for vout_Request()/vout_Create() to be...
[vlc.git] / modules / codec / ffmpeg / video_filter.c
1 /*****************************************************************************
2  * video filter: video filter doing chroma conversion and resizing
3  *               using the ffmpeg library
4  *****************************************************************************
5  * Copyright (C) 1999-2001 VideoLAN
6  * $Id$
7  *
8  * Authors: Gildas Bazin <gbazin@videolan.org>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111, USA.
23  *****************************************************************************/
24
25 /*****************************************************************************
26  * Preamble
27  *****************************************************************************/
28 #include <vlc/vlc.h>
29 #include <vlc/decoder.h>
30 #include "vlc_filter.h"
31
32 /* ffmpeg header */
33 #ifdef HAVE_FFMPEG_AVCODEC_H
34 #   include <ffmpeg/avcodec.h>
35 #else
36 #   include <avcodec.h>
37 #endif
38
39 #include "ffmpeg.h"
40
41 void E_(InitLibavcodec) ( vlc_object_t *p_object );
42 static int CheckInit( filter_t *p_filter );
43 static picture_t *Process( filter_t *p_filter, picture_t *p_pic );
44 static picture_t *Deinterlace( filter_t *p_filter, picture_t *p_pic );
45
46 /*****************************************************************************
47  * filter_sys_t : filter descriptor
48  *****************************************************************************/
49 struct filter_sys_t
50 {
51     vlc_bool_t b_resize;
52     vlc_bool_t b_convert;
53
54     es_format_t fmt_in;
55     int i_src_ffmpeg_chroma;
56     es_format_t fmt_out;
57     int i_dst_ffmpeg_chroma;
58
59     AVPicture tmp_pic;
60     ImgReSampleContext *p_rsc;
61 };
62
63 /*****************************************************************************
64  * OpenFilter: probe the filter and return score
65  *****************************************************************************/
66 int E_(OpenFilter)( vlc_object_t *p_this )
67 {
68     filter_t *p_filter = (filter_t*)p_this;
69     filter_sys_t *p_sys;
70     vlc_bool_t b_convert, b_resize;
71
72     /* Check if we can handle that formats */
73     if( E_(GetFfmpegChroma)( p_filter->fmt_in.video.i_chroma ) < 0 ||
74         E_(GetFfmpegChroma)( p_filter->fmt_out.video.i_chroma ) < 0 )
75     {
76         return VLC_EGENERIC;
77     }
78
79     b_resize =
80         p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width ||
81         p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height;
82     b_convert =
83         p_filter->fmt_in.video.i_chroma != p_filter->fmt_out.video.i_chroma;
84
85     if( !b_resize && !b_convert )
86     {
87         /* Nothing to do */
88         return VLC_EGENERIC;
89     }
90
91     /* Allocate the memory needed to store the decoder's structure */
92     if( ( p_filter->p_sys = p_sys =
93           (filter_sys_t *)malloc(sizeof(filter_sys_t)) ) == NULL )
94     {
95         msg_Err( p_filter, "out of memory" );
96         return VLC_EGENERIC;
97     }
98
99     /* Misc init */
100     p_sys->p_rsc = NULL;
101     p_sys->b_convert = b_convert;
102     p_sys->i_src_ffmpeg_chroma =
103         E_(GetFfmpegChroma)( p_filter->fmt_in.video.i_chroma );
104     p_sys->i_dst_ffmpeg_chroma =
105         E_(GetFfmpegChroma)( p_filter->fmt_out.video.i_chroma );
106     p_filter->pf_video_filter = Process;
107     es_format_Init( &p_sys->fmt_in, 0, 0 );
108     es_format_Init( &p_sys->fmt_out, 0, 0 );
109
110     if( CheckInit( p_filter ) != VLC_SUCCESS )
111     {
112         free( p_sys );
113         return VLC_EGENERIC;
114     }
115
116     avpicture_alloc( &p_sys->tmp_pic, p_sys->i_dst_ffmpeg_chroma,
117                      p_filter->fmt_in.video.i_width,
118                      p_filter->fmt_in.video.i_height );
119
120     msg_Dbg( p_filter, "input: %ix%i %4.4s -> %ix%i %4.4s",
121              p_filter->fmt_in.video.i_width, p_filter->fmt_in.video.i_height,
122              (char *)&p_filter->fmt_in.video.i_chroma,
123              p_filter->fmt_out.video.i_width, p_filter->fmt_out.video.i_height,
124              (char *)&p_filter->fmt_out.video.i_chroma );
125
126     /* libavcodec needs to be initialized for some chroma conversions */
127     E_(InitLibavcodec)(p_this);
128
129     return VLC_SUCCESS;
130 }
131
132 /*****************************************************************************
133  * CloseFilter: clean up the filter
134  *****************************************************************************/
135 void E_(CloseFilter)( vlc_object_t *p_this )
136 {
137     filter_t *p_filter = (filter_t*)p_this;
138     filter_sys_t *p_sys = p_filter->p_sys;
139
140     if( p_sys->p_rsc ) img_resample_close( p_sys->p_rsc );
141
142     avpicture_free( &p_sys->tmp_pic );
143
144     free( p_sys );
145 }
146
147 /*****************************************************************************
148  * CheckInit: Initialise filter when necessary
149  *****************************************************************************/
150 static int CheckInit( filter_t *p_filter )
151 {
152     filter_sys_t *p_sys = p_filter->p_sys;
153
154     if( p_filter->fmt_in.video.i_width != p_sys->fmt_in.video.i_width ||
155         p_filter->fmt_in.video.i_height != p_sys->fmt_in.video.i_height ||
156         p_filter->fmt_out.video.i_width != p_sys->fmt_out.video.i_width ||
157         p_filter->fmt_out.video.i_height != p_sys->fmt_out.video.i_height )
158     {
159         if( p_sys->p_rsc ) img_resample_close( p_sys->p_rsc );
160         p_sys->p_rsc = 0;
161
162         p_sys->b_resize =
163           p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width ||
164           p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height;
165
166         if( p_sys->b_resize )
167         {
168             p_sys->p_rsc = img_resample_init( p_filter->fmt_out.video.i_width,
169                                p_filter->fmt_out.video.i_height,
170                                p_filter->fmt_in.video.i_width,
171                                p_filter->fmt_in.video.i_height );
172
173             if( !p_sys->p_rsc )
174             {
175                 msg_Err( p_filter, "img_resample_init failed" );
176                 return VLC_EGENERIC;
177             }
178         }
179
180         p_sys->fmt_in = p_filter->fmt_in;
181         p_sys->fmt_out = p_filter->fmt_out;
182     }
183
184     return VLC_SUCCESS;
185 }
186
187 /*****************************************************************************
188  * Do the processing here
189  *****************************************************************************/
190 static picture_t *Process( filter_t *p_filter, picture_t *p_pic )
191 {
192     filter_sys_t *p_sys = p_filter->p_sys;
193     AVPicture src_pic, dest_pic, inter_pic;
194     picture_t *p_pic_dst;
195     vlc_bool_t b_resize = p_sys->b_resize;
196     int i;
197
198     /* Check if format properties changed */
199     if( CheckInit( p_filter ) != VLC_SUCCESS ) return 0;
200
201     /* Request output picture */
202     p_pic_dst = p_filter->pf_vout_buffer_new( p_filter );
203     if( !p_pic_dst )
204     {
205         msg_Warn( p_filter, "can't get output picture" );
206         p_pic->pf_release( p_pic );
207         return NULL;
208     }
209
210     /* Prepare the AVPictures for the conversion */
211     for( i = 0; i < p_pic->i_planes; i++ )
212     {
213         src_pic.data[i] = p_pic->p[i].p_pixels;
214         src_pic.linesize[i] = p_pic->p[i].i_pitch;
215     }
216     for( i = 0; i < p_pic_dst->i_planes; i++ )
217     {
218         dest_pic.data[i] = p_pic_dst->p[i].p_pixels;
219         dest_pic.linesize[i] = p_pic_dst->p[i].i_pitch;
220     }
221
222     /* Special cases */
223     if( p_filter->fmt_in.video.i_chroma == VLC_FOURCC('Y','V','1','2') ||
224         p_filter->fmt_in.video.i_chroma == VLC_FOURCC('Y','V','U','9') )
225     {
226         /* Invert U and V */
227         src_pic.data[1] = p_pic->p[2].p_pixels;
228         src_pic.data[2] = p_pic->p[1].p_pixels;
229     }
230     if( p_filter->fmt_out.video.i_chroma == VLC_FOURCC('Y','V','1','2') ||
231         p_filter->fmt_out.video.i_chroma == VLC_FOURCC('Y','V','U','9') )
232     {
233         /* Invert U and V */
234         dest_pic.data[1] = p_pic_dst->p[2].p_pixels;
235         dest_pic.data[2] = p_pic_dst->p[1].p_pixels;
236     }
237     if( p_sys->i_src_ffmpeg_chroma == PIX_FMT_RGB24 )
238         if( p_filter->fmt_in.video.i_bmask == 0x00ff0000 )
239             p_sys->i_src_ffmpeg_chroma = PIX_FMT_BGR24;
240
241 #if 0
242     if( p_sys->b_resize &&
243         p_filter->fmt_in.video.i_width * p_filter->fmt_in.video.i_height >
244         p_filter->fmt_out.video.i_width * p_filter->fmt_out.video.i_height )
245     {
246         img_resample( p_sys->p_rsc, &dest_pic, &p_sys->tmp_pic );
247         b_resize = 0;
248     }
249 #endif
250
251     if( p_sys->b_convert )
252     {
253         if( p_sys->b_resize ) inter_pic = p_sys->tmp_pic;
254         else inter_pic = dest_pic;
255
256         img_convert( &inter_pic, p_sys->i_dst_ffmpeg_chroma,
257                      &src_pic, p_sys->i_src_ffmpeg_chroma,
258                      p_filter->fmt_in.video.i_width,
259                      p_filter->fmt_in.video.i_height );
260
261         src_pic = inter_pic;
262     }
263
264     if( p_sys->b_resize && p_sys->p_rsc )
265     {
266         img_resample( p_sys->p_rsc, &dest_pic, &src_pic );
267     }
268
269     /* Special case for RV32 -> YUVA */
270     if( !p_sys->b_resize &&
271         p_filter->fmt_in.video.i_chroma == VLC_FOURCC('R','V','3','2') &&
272         p_filter->fmt_out.video.i_chroma == VLC_FOURCC('Y','U','V','A') )
273     {
274         uint8_t *p_src = p_pic->p[0].p_pixels;
275         int i_src_pitch = p_pic->p[0].i_pitch;
276         uint8_t *p_dst = p_pic_dst->p[3].p_pixels;
277         int i_dst_pitch = p_pic_dst->p[3].i_pitch;
278         int j;
279
280         for( i = 0; i < p_filter->fmt_out.video.i_height; i++ )
281         {
282             for( j = 0; j < p_filter->fmt_out.video.i_width; j++ )
283             {
284               p_dst[j] = p_src[j*4+3];
285             }
286             p_src += i_src_pitch;
287             p_dst += i_dst_pitch;
288         }
289     }
290     else if( p_filter->fmt_out.video.i_chroma == VLC_FOURCC('Y','U','V','A') )
291     {
292         /* Special case for YUVA */
293         memset( p_pic_dst->p[3].p_pixels, 0xFF,
294                 p_pic_dst->p[3].i_pitch * p_pic_dst->p[3].i_lines );
295     }
296
297     p_pic_dst->date = p_pic->date;
298     p_pic_dst->b_force = p_pic->b_force;
299     p_pic_dst->i_nb_fields = p_pic->i_nb_fields;
300     p_pic_dst->b_progressive = p_pic->b_progressive;
301     p_pic_dst->b_top_field_first = p_pic->b_top_field_first;
302
303     p_pic->pf_release( p_pic );
304     return p_pic_dst;
305 }
306
307 /*****************************************************************************
308  * OpenDeinterlace: probe the filter and return score
309  *****************************************************************************/
310 int E_(OpenDeinterlace)( vlc_object_t *p_this )
311 {
312     filter_t *p_filter = (filter_t*)p_this;
313     filter_sys_t *p_sys;
314
315     /* Check if we can handle that formats */
316     if( E_(GetFfmpegChroma)( p_filter->fmt_in.video.i_chroma ) < 0 )
317     {
318         return VLC_EGENERIC;
319     }
320
321     /* Allocate the memory needed to store the decoder's structure */
322     if( ( p_filter->p_sys = p_sys =
323           (filter_sys_t *)malloc(sizeof(filter_sys_t)) ) == NULL )
324     {
325         msg_Err( p_filter, "out of memory" );
326         return VLC_EGENERIC;
327     }
328
329     /* Misc init */
330     p_sys->i_src_ffmpeg_chroma =
331         E_(GetFfmpegChroma)( p_filter->fmt_in.video.i_chroma );
332     p_filter->pf_video_filter = Deinterlace;
333
334     msg_Dbg( p_filter, "deinterlacing" );
335
336     /* libavcodec needs to be initialized for some chroma conversions */
337     E_(InitLibavcodec)(p_this);
338
339     return VLC_SUCCESS;
340 }
341
342 /*****************************************************************************
343  * CloseDeinterlace: clean up the filter
344  *****************************************************************************/
345 void E_(CloseDeinterlace)( vlc_object_t *p_this )
346 {
347     filter_t *p_filter = (filter_t*)p_this;
348     filter_sys_t *p_sys = p_filter->p_sys;
349
350     free( p_sys );
351 }
352
353 /*****************************************************************************
354  * Do the processing here
355  *****************************************************************************/
356 static picture_t *Deinterlace( filter_t *p_filter, picture_t *p_pic )
357 {
358     filter_sys_t *p_sys = p_filter->p_sys;
359     AVPicture src_pic, dest_pic;
360     picture_t *p_pic_dst;
361     int i;
362
363     /* Request output picture */
364     p_pic_dst = p_filter->pf_vout_buffer_new( p_filter );
365     if( !p_pic_dst )
366     {
367         msg_Warn( p_filter, "can't get output picture" );
368         return NULL;
369     }
370
371     /* Prepare the AVPictures for the conversion */
372     for( i = 0; i < p_pic->i_planes; i++ )
373     {
374         src_pic.data[i] = p_pic->p[i].p_pixels;
375         src_pic.linesize[i] = p_pic->p[i].i_pitch;
376     }
377     for( i = 0; i < p_pic_dst->i_planes; i++ )
378     {
379         dest_pic.data[i] = p_pic_dst->p[i].p_pixels;
380         dest_pic.linesize[i] = p_pic_dst->p[i].i_pitch;
381     }
382
383     avpicture_deinterlace( &dest_pic, &src_pic, p_sys->i_src_ffmpeg_chroma,
384                            p_filter->fmt_in.video.i_width,
385                            p_filter->fmt_in.video.i_height );
386
387     p_pic_dst->date = p_pic->date;
388     p_pic_dst->b_force = p_pic->b_force;
389     p_pic_dst->i_nb_fields = p_pic->i_nb_fields;
390     p_pic_dst->b_progressive = VLC_TRUE;
391     p_pic_dst->b_top_field_first = p_pic->b_top_field_first;
392
393     p_pic->pf_release( p_pic );
394     return p_pic_dst;
395 }