Attempt to implement chroma conversion using swscale. This currently crashes on calli...
[vlc.git] / modules / codec / ffmpeg / chroma.c
1 /*****************************************************************************
2  * chroma.c: chroma conversion using ffmpeg library
3  *****************************************************************************
4  * Copyright (C) 1999-2001 the VideoLAN team
5  * $Id$
6  *
7  * Authors: Gildas Bazin <gbazin@videolan.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * Preamble
26  *****************************************************************************/
27 #include <vlc/vlc.h>
28 #include <vlc_vout.h>
29
30 #ifdef HAVE_FFMPEG_SWSCALE_H
31 #include <vlc_filter.h>
32 #endif
33
34 /* ffmpeg header */
35 #ifdef HAVE_FFMPEG_AVCODEC_H
36 #   include <ffmpeg/avcodec.h>
37 #else
38 #   include <avcodec.h>
39 #endif
40
41 #include "ffmpeg.h"
42
43 #if !defined(HAVE_FFMPEG_SWSCALE_H) && !defined(HAVE_LIBSWSCALE_TREE)
44 void E_(InitLibavcodec) ( vlc_object_t *p_object );
45 static void ChromaConversion( vout_thread_t *, picture_t *, picture_t * );
46
47 /*****************************************************************************
48  * chroma_sys_t: chroma method descriptor
49  *****************************************************************************
50  * This structure is part of the chroma transformation descriptor, it
51  * describes the chroma plugin specific properties.
52  *****************************************************************************/
53 struct chroma_sys_t
54 {
55     int i_src_vlc_chroma;
56     int i_src_ffmpeg_chroma;
57     int i_dst_vlc_chroma;
58     int i_dst_ffmpeg_chroma;
59     AVPicture tmp_pic;
60     ImgReSampleContext *p_rsc;
61 };
62
63 /*****************************************************************************
64  * OpenChroma: allocate a chroma function
65  *****************************************************************************
66  * This function allocates and initializes a chroma function
67  *****************************************************************************/
68 int E_(OpenChroma)( vlc_object_t *p_this )
69 {
70     vout_thread_t *p_vout = (vout_thread_t *)p_this;
71     int i_ffmpeg_chroma[2], i_vlc_chroma[2], i;
72
73     /*
74      * Check the source chroma first, then the destination chroma
75      */
76     i_vlc_chroma[0] = p_vout->render.i_chroma;
77     i_vlc_chroma[1] = p_vout->output.i_chroma;
78     for( i = 0; i < 2; i++ )
79     {
80         i_ffmpeg_chroma[i] = E_(GetFfmpegChroma)( i_vlc_chroma[i] );
81         if( i_ffmpeg_chroma[i] < 0 ) return VLC_EGENERIC;
82     }
83
84     p_vout->chroma.pf_convert = ChromaConversion;
85
86     p_vout->chroma.p_sys = malloc( sizeof( chroma_sys_t ) );
87     if( p_vout->chroma.p_sys == NULL )
88     {
89         return VLC_ENOMEM;
90     }
91
92     p_vout->chroma.p_sys->i_src_vlc_chroma = p_vout->render.i_chroma;
93     p_vout->chroma.p_sys->i_dst_vlc_chroma = p_vout->output.i_chroma;
94     p_vout->chroma.p_sys->i_src_ffmpeg_chroma = i_ffmpeg_chroma[0];
95     p_vout->chroma.p_sys->i_dst_ffmpeg_chroma = i_ffmpeg_chroma[1];
96
97     if( ( p_vout->render.i_height != p_vout->output.i_height ||
98           p_vout->render.i_width != p_vout->output.i_width ) &&
99         ( p_vout->chroma.p_sys->i_dst_vlc_chroma == VLC_FOURCC('I','4','2','0') ||
100           p_vout->chroma.p_sys->i_dst_vlc_chroma == VLC_FOURCC('Y','V','1','2') ))
101     {
102         msg_Dbg( p_vout, "preparing to resample picture" );
103         p_vout->chroma.p_sys->p_rsc =
104             img_resample_init( p_vout->output.i_width, p_vout->output.i_height,
105                                p_vout->render.i_width, p_vout->render.i_height );
106         avpicture_alloc( &p_vout->chroma.p_sys->tmp_pic,
107                          p_vout->chroma.p_sys->i_dst_ffmpeg_chroma,
108                          p_vout->render.i_width, p_vout->render.i_height );
109     }
110     else
111     {
112         msg_Dbg( p_vout, "no resampling" );
113         p_vout->chroma.p_sys->p_rsc = NULL;
114     }
115
116     /* libavcodec needs to be initialized for some chroma conversions */
117     E_(InitLibavcodec)(p_this);
118
119     return VLC_SUCCESS;
120 }
121
122 /*****************************************************************************
123  * ChromaConversion: actual chroma conversion function
124  *****************************************************************************/
125 static void ChromaConversion( vout_thread_t *p_vout,
126                               picture_t *p_src, picture_t *p_dest )
127 {
128     AVPicture src_pic;
129     AVPicture dest_pic;
130     int i;
131
132     /* Prepare the AVPictures for converion */
133     for( i = 0; i < p_src->i_planes; i++ )
134     {
135         src_pic.data[i] = p_src->p[i].p_pixels;
136         src_pic.linesize[i] = p_src->p[i].i_pitch;
137     }
138     for( i = 0; i < p_dest->i_planes; i++ )
139     {
140         dest_pic.data[i] = p_dest->p[i].p_pixels;
141         dest_pic.linesize[i] = p_dest->p[i].i_pitch;
142     }
143
144     /* Special cases */
145     if( p_vout->chroma.p_sys->i_src_vlc_chroma == VLC_FOURCC('Y','V','1','2') ||
146         p_vout->chroma.p_sys->i_src_vlc_chroma == VLC_FOURCC('Y','V','U','9') )
147     {
148         /* Invert U and V */
149         src_pic.data[1] = p_src->p[2].p_pixels;
150         src_pic.data[2] = p_src->p[1].p_pixels;
151     }
152     if( p_vout->chroma.p_sys->i_dst_vlc_chroma == VLC_FOURCC('Y','V','1','2') ||
153         p_vout->chroma.p_sys->i_dst_vlc_chroma == VLC_FOURCC('Y','V','U','9') )
154     {
155         /* Invert U and V */
156         dest_pic.data[1] = p_dest->p[2].p_pixels;
157         dest_pic.data[2] = p_dest->p[1].p_pixels;
158     }
159     if( p_vout->chroma.p_sys->i_src_ffmpeg_chroma == PIX_FMT_RGB24 )
160         if( p_vout->render.i_bmask == 0x00ff0000 )
161             p_vout->chroma.p_sys->i_src_ffmpeg_chroma = PIX_FMT_BGR24;
162
163     if( p_vout->chroma.p_sys->p_rsc )
164     {
165         img_convert( &p_vout->chroma.p_sys->tmp_pic,
166                      p_vout->chroma.p_sys->i_dst_ffmpeg_chroma,
167                      &src_pic, p_vout->chroma.p_sys->i_src_ffmpeg_chroma,
168                      p_vout->render.i_width, p_vout->render.i_height );
169         img_resample( p_vout->chroma.p_sys->p_rsc, &dest_pic,
170                       &p_vout->chroma.p_sys->tmp_pic );
171     }
172     else
173     {
174         img_convert( &dest_pic, p_vout->chroma.p_sys->i_dst_ffmpeg_chroma,
175                      &src_pic, p_vout->chroma.p_sys->i_src_ffmpeg_chroma,
176                      p_vout->render.i_width, p_vout->render.i_height );
177     }
178 }
179
180 /*****************************************************************************
181  * CloseChroma: free the chroma function
182  *****************************************************************************
183  * This function frees the previously allocated chroma function
184  *****************************************************************************/
185 void E_(CloseChroma)( vlc_object_t *p_this )
186 {
187     vout_thread_t *p_vout = (vout_thread_t *)p_this;
188     if( p_vout->chroma.p_sys->p_rsc )
189     {
190         img_resample_close( p_vout->chroma.p_sys->p_rsc );
191         avpicture_free( &p_vout->chroma.p_sys->tmp_pic );
192     }
193     free( p_vout->chroma.p_sys );
194 }
195 #else
196
197 static void ChromaConversion( vout_thread_t *, picture_t *, picture_t * );
198
199 /*****************************************************************************
200  * chroma_sys_t: chroma method descriptor
201  *****************************************************************************
202  * This structure is part of the chroma transformation descriptor, it
203  * describes the chroma plugin specific properties.
204  *****************************************************************************/
205 struct chroma_sys_t
206 {
207     filter_t *p_swscaler;
208 };
209
210 /*****************************************************************************
211  * Video Filter2 functions
212  *****************************************************************************/
213 struct filter_owner_sys_t
214 {
215     vout_thread_t *p_vout;
216 };
217
218 static picture_t *video_new_buffer_filter( filter_t *p_filter )
219 {
220     picture_t *p_picture;
221     vout_thread_t *p_vout = p_filter->p_owner->p_vout;
222
223     p_picture = vout_CreatePicture( p_vout, 0, 0, 0 );
224
225     return p_picture;
226 }
227
228 static void video_del_buffer_filter( filter_t *p_filter, picture_t *p_pic )
229 {
230     vout_DestroyPicture( p_filter->p_owner->p_vout, p_pic );
231 }
232
233 /*****************************************************************************
234  * OpenChroma: allocate a chroma function
235  *****************************************************************************
236  * This function allocates and initializes a chroma function
237  *****************************************************************************/
238 int E_(OpenChroma)( vlc_object_t *p_this )
239 {
240     vout_thread_t *p_vout = (vout_thread_t *)p_this;
241     chroma_sys_t  *p_sys = p_vout->chroma.p_sys;
242
243     p_vout->chroma.p_sys = p_sys = malloc( sizeof( chroma_sys_t ) );
244     if( p_vout->chroma.p_sys == NULL )
245     {
246         return VLC_ENOMEM;
247     }
248     p_vout->chroma.pf_convert = ChromaConversion;
249
250     p_sys->p_swscaler = vlc_object_create( p_vout, VLC_OBJECT_FILTER );
251     vlc_object_attach( p_sys->p_swscaler, p_vout );
252
253     p_sys->p_swscaler->pf_vout_buffer_new = video_new_buffer_filter;
254     p_sys->p_swscaler->pf_vout_buffer_del = video_del_buffer_filter;
255
256     p_sys->p_swscaler->fmt_out.video.i_x_offset =
257         p_sys->p_swscaler->fmt_out.video.i_y_offset = 0;
258     p_sys->p_swscaler->fmt_in.video = p_vout->fmt_in;
259     p_sys->p_swscaler->fmt_out.video = p_vout->fmt_out;
260     p_sys->p_swscaler->fmt_out.video.i_aspect = p_vout->render.i_aspect;
261     p_sys->p_swscaler->fmt_in.video.i_chroma = p_vout->render.i_chroma;
262     p_sys->p_swscaler->fmt_out.video.i_chroma = p_vout->output.i_chroma;
263
264     p_sys->p_swscaler->p_module = module_Need( p_sys->p_swscaler,
265                            "video filter2", 0, 0 );
266
267     if( p_sys->p_swscaler->p_module )
268     {
269         p_sys->p_swscaler->p_owner =
270             malloc( sizeof( filter_owner_sys_t ) );
271         if( p_sys->p_swscaler->p_owner )
272             p_sys->p_swscaler->p_owner->p_vout = p_vout;
273     }
274
275     if( !p_sys->p_swscaler->p_module || !p_sys->p_swscaler->p_owner )
276     {
277         vlc_object_detach( p_sys->p_swscaler );
278         vlc_object_destroy( p_sys->p_swscaler );
279         free( p_vout->chroma.p_sys );
280         return VLC_EGENERIC;
281     }
282
283     return VLC_SUCCESS;
284 }
285
286 /*****************************************************************************
287  * ChromaConversion: actual chroma conversion function
288  *****************************************************************************/
289 static void ChromaConversion( vout_thread_t *p_vout,
290                               picture_t *p_src, picture_t *p_dest )
291 {
292     chroma_sys_t *p_sys = (chroma_sys_t *) p_vout->chroma.p_sys;
293
294     if( p_sys && p_src && p_dest && 
295         p_sys->p_swscaler && p_sys->p_swscaler->p_module )
296     {
297         picture_t *p_pic;
298 #if 0
299         p_pic = (picture_t *) malloc( sizeof( picture_t ) );
300         if( p_pic )
301         {
302             memset( p_pic, 0, sizeof( picture_t ) );
303             vout_AllocatePicture( VLC_OBJECT(p_vout),
304                                   p_pic, p_vout->fmt_in.i_chroma,
305                                   p_vout->fmt_in.i_width,
306                                   p_vout->fmt_in.i_height,
307                                   p_vout->fmt_in.i_aspect );
308             if( !p_pic->i_planes )
309             {
310                 free( p_pic->p_data_orig );
311                 free( p_pic );
312             }
313             else
314             {
315                 vout_CopyPicture( p_vout, p_pic, p_src );
316                 p_sys->p_swscaler->fmt_in.video = p_vout->fmt_in;
317                 p_sys->p_swscaler->fmt_out.video = p_vout->fmt_out;
318                 p_pic = p_sys->p_swscaler->pf_video_filter( p_sys->p_swscaler, p_pic );
319                 if( p_pic )
320                 {
321                     vout_CopyPicture( p_vout, p_dest, p_pic );
322                     free( p_pic->p_data_orig );
323                     if( p_pic->pf_release ) p_pic->pf_release( p_pic );
324                 }
325             }
326         }
327 #else
328         p_sys->p_swscaler->fmt_in.video = p_vout->fmt_in;
329         p_sys->p_swscaler->fmt_out.video = p_vout->fmt_out;
330
331         msg_Dbg( p_vout, "chroma %4.4s (%d) to %4.4s (%d)",
332                  (char *)&p_vout->fmt_in.i_chroma, p_src->i_planes,
333                  (char *)&p_vout->fmt_out.i_chroma, p_dest->i_planes  );
334         p_pic = p_sys->p_swscaler->pf_video_filter( (filter_t*)p_sys->p_swscaler, p_src );
335         if( p_pic )
336         {
337             vout_CopyPicture( p_vout, p_dest, p_pic );
338             free( p_pic->p_data_orig );
339             if( p_pic->pf_release ) p_pic->pf_release( p_pic );
340         }
341 #endif
342     }
343 }
344
345 /*****************************************************************************
346  * CloseChroma: free the chroma function
347  *****************************************************************************
348  * This function frees the previously allocated chroma function
349  *****************************************************************************/
350 void E_(CloseChroma)( vlc_object_t *p_this )
351 {
352     vout_thread_t *p_vout = (vout_thread_t *)p_this;
353     chroma_sys_t  *p_sys  = (chroma_sys_t *)p_vout->chroma.p_sys;
354     if( p_sys->p_swscaler && p_sys->p_swscaler->p_module )
355     {
356         free( p_sys->p_swscaler->p_owner );
357         module_Unneed( p_sys->p_swscaler, p_sys->p_swscaler->p_module );
358         vlc_object_detach( p_sys->p_swscaler );
359         vlc_object_destroy( p_sys->p_swscaler );
360         p_sys->p_swscaler= NULL;
361     }
362     free( p_vout->chroma.p_sys );
363 }
364
365 #endif /* !defined(HAVE_FFMPEG_SWSCALE_H) && !defined(HAVE_LIBSWSCALE_TREE) */