Return VLC_ENOMEM when malloc fails.
[vlc.git] / modules / codec / ffmpeg / scale.c
1 /*****************************************************************************
2  * filter.c: video scaling module using the swscale library
3  *****************************************************************************
4  * Copyright (C) 2003 the VideoLAN team
5  * $Id$
6  *
7  * Author: Gildas Bazin <gbazin@videolan.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * Preamble
26  *****************************************************************************/
27 #ifdef HAVE_CONFIG_H
28 # include "config.h"
29 #endif
30
31 #include <vlc/vlc.h>
32 #include <vlc_codec.h>
33 #include <vlc_vout.h>
34 #include <vlc_filter.h>
35
36 /* ffmpeg headers */
37 #ifdef HAVE_LIBAVCODEC_AVCODEC_H
38 #   include <libavcodec/avcodec.h>
39 #elif defined(HAVE_FFMPEG_AVCODEC_H)
40 #   include <ffmpeg/avcodec.h>
41 #else
42 #   include <avcodec.h>
43 #endif
44
45 #ifdef HAVE_LIBSWSCALE_SWSCALE_H
46 #   include <libswscale/swscale.h>
47 #elif defined(HAVE_FFMPEG_SWSCALE_H)
48 #   include <ffmpeg/swscale.h>
49 #elif defined(HAVE_LIBSWSCALE_TREE)
50 #   include <swscale.h>
51 #endif
52
53 #include "ffmpeg.h"
54
55 /* Version checking */
56 #if ( (defined(HAVE_LIBSWSCALE_SWSCALE_H) || defined(HAVE_FFMPEG_SWSCALE_H) || defined(HAVE_LIBSWSCALE_TREE)) && (LIBSWSCALE_VERSION_INT >= ((0<<16)+(5<<8)+0)) )
57
58 /*****************************************************************************
59  * filter_sys_t : filter descriptor
60  *****************************************************************************/
61 struct filter_sys_t
62 {
63     struct SwsContext *ctx;
64     SwsFilter *p_src_filter;
65     SwsFilter *p_dst_filter;
66     int i_cpu_mask, i_sws_flags;
67
68     es_format_t fmt_in;
69     es_format_t fmt_out;
70 };
71
72 /****************************************************************************
73  * Local prototypes
74  ****************************************************************************/
75 void *( *swscale_fast_memcpy )( void *, const void *, size_t );
76 static picture_t *Filter( filter_t *, picture_t * );
77 static int CheckInit( filter_t * );
78
79 static const char *ppsz_mode_descriptions[] =
80 { N_("Fast bilinear"), N_("Bilinear"), N_("Bicubic (good quality)"),
81   N_("Experimental"), N_("Nearest neighbour (bad quality)"),
82   N_("Area"), N_("Luma bicubic / chroma bilinear"), N_("Gauss"),
83   N_("SincR"), N_("Lanczos"), N_("Bicubic spline") };
84
85 /*****************************************************************************
86  * OpenScaler: probe the filter and return score
87  *****************************************************************************/
88 int OpenScaler( vlc_object_t *p_this )
89 {
90     filter_t *p_filter = (filter_t*)p_this;
91     filter_sys_t *p_sys;
92     vlc_value_t val;
93
94     int i_fmt_in, i_fmt_out;
95     unsigned int i_cpu;
96     int i_sws_mode;
97
98     float sws_lum_gblur = 0.0, sws_chr_gblur = 0.0;
99     int sws_chr_vshift = 0, sws_chr_hshift = 0;
100     float sws_chr_sharpen = 0.0, sws_lum_sharpen = 0.0;
101
102     /* Supported Input formats: YV12, I420/IYUV, YUY2, UYVY, BGR32, BGR24,
103      * BGR16, BGR15, RGB32, RGB24, Y8/Y800, YVU9/IF09 */
104     i_fmt_in = GetFfmpegChroma(p_filter->fmt_in.video.i_chroma);
105     /* Supported output formats: YV12, I420/IYUV, YUY2, UYVY,
106      * {BGR,RGB}{1,4,8,15,16,24,32}, Y8/Y800, YVU9/IF09 */
107     i_fmt_out = GetFfmpegChroma(p_filter->fmt_out.video.i_chroma);
108     if( ( i_fmt_in < 0 ) || ( i_fmt_out < 0 ) )
109     {
110         return VLC_EGENERIC;
111     }
112
113     /* Allocate the memory needed to store the decoder's structure */
114     if( ( p_filter->p_sys = p_sys =
115           (filter_sys_t *)malloc(sizeof(filter_sys_t)) ) == NULL )
116     {
117         return VLC_ENOMEM;
118     }
119
120     swscale_fast_memcpy = vlc_memcpy;
121
122     /* Set CPU capabilities */
123     i_cpu = vlc_CPU();
124     p_sys->i_cpu_mask = 0;
125     if( i_cpu & CPU_CAPABILITY_MMX )
126     {
127         p_sys->i_cpu_mask |= SWS_CPU_CAPS_MMX;
128     }
129 #if (LIBSWSCALE_VERSION_INT >= ((0<<16)+(5<<8)+0))
130     if( i_cpu & CPU_CAPABILITY_MMXEXT )
131     {
132         p_sys->i_cpu_mask |= SWS_CPU_CAPS_MMX2;
133     }
134 #endif
135     if( i_cpu & CPU_CAPABILITY_3DNOW )
136     {
137         p_sys->i_cpu_mask |= SWS_CPU_CAPS_3DNOW;
138     }
139     if( i_cpu & CPU_CAPABILITY_ALTIVEC )
140     {
141         p_sys->i_cpu_mask |= SWS_CPU_CAPS_ALTIVEC;
142     }
143
144     var_Create( p_filter, "swscale-mode", VLC_VAR_INTEGER|VLC_VAR_DOINHERIT );
145     var_Get( p_filter, "swscale-mode", &val );
146     i_sws_mode = val.i_int;
147
148     switch( i_sws_mode )
149     {
150     case 0:  p_sys->i_sws_flags = SWS_FAST_BILINEAR; break;
151     case 1:  p_sys->i_sws_flags = SWS_BILINEAR; break;
152     case 2:  p_sys->i_sws_flags = SWS_BICUBIC; break;
153     case 3:  p_sys->i_sws_flags = SWS_X; break;
154     case 4:  p_sys->i_sws_flags = SWS_POINT; break;
155     case 5:  p_sys->i_sws_flags = SWS_AREA; break;
156     case 6:  p_sys->i_sws_flags = SWS_BICUBLIN; break;
157     case 7:  p_sys->i_sws_flags = SWS_GAUSS; break;
158     case 8:  p_sys->i_sws_flags = SWS_SINC; break;
159     case 9:  p_sys->i_sws_flags = SWS_LANCZOS; break;
160     case 10: p_sys->i_sws_flags = SWS_SPLINE; break;
161     default: p_sys->i_sws_flags = SWS_FAST_BILINEAR; i_sws_mode = 0; break;
162     }
163
164     p_sys->p_src_filter = NULL;
165     p_sys->p_dst_filter = NULL;
166     p_sys->p_src_filter =
167         sws_getDefaultFilter( sws_lum_gblur, sws_chr_gblur,
168                               sws_lum_sharpen, sws_chr_sharpen,
169                               sws_chr_hshift, sws_chr_vshift, 0 );
170
171     /* Misc init */
172     p_sys->ctx = NULL;
173     p_filter->pf_video_filter = Filter;
174     es_format_Init( &p_sys->fmt_in, 0, 0 );
175     es_format_Init( &p_sys->fmt_out, 0, 0 );
176
177     if( CheckInit( p_filter ) != VLC_SUCCESS )
178     {
179         if( p_sys->p_src_filter ) sws_freeFilter( p_sys->p_src_filter );
180         free( p_sys );
181         return VLC_EGENERIC;
182     }
183
184     msg_Dbg( p_filter, "%ix%i chroma: %4.4s -> %ix%i chroma: %4.4s",
185              p_filter->fmt_in.video.i_width, p_filter->fmt_in.video.i_height,
186              (char *)&p_filter->fmt_in.video.i_chroma,
187              p_filter->fmt_out.video.i_width, p_filter->fmt_out.video.i_height,
188              (char *)&p_filter->fmt_out.video.i_chroma );
189
190     if( p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width ||
191         p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height )
192     {
193         msg_Dbg( p_filter, "scaling mode: %s",
194                  ppsz_mode_descriptions[i_sws_mode] );
195     }
196
197     return VLC_SUCCESS;
198 }
199
200 /*****************************************************************************
201  * CloseFilter: clean up the filter
202  *****************************************************************************/
203 void CloseScaler( vlc_object_t *p_this )
204 {
205     filter_t *p_filter = (filter_t*)p_this;
206     filter_sys_t *p_sys = p_filter->p_sys;
207
208     if( p_sys->ctx ) sws_freeContext( p_sys->ctx );
209     if( p_sys->p_src_filter ) sws_freeFilter( p_sys->p_src_filter );
210     free( p_sys );
211 }
212
213 /*****************************************************************************
214  * CheckInit: Initialise filter when necessary
215  *****************************************************************************/
216 static int CheckInit( filter_t *p_filter )
217 {
218     filter_sys_t *p_sys = p_filter->p_sys;
219
220     if( ( p_filter->fmt_in.video.i_width != p_sys->fmt_in.video.i_width ) ||
221         ( p_filter->fmt_in.video.i_height != p_sys->fmt_in.video.i_height ) ||
222         ( p_filter->fmt_out.video.i_width != p_sys->fmt_out.video.i_width ) ||
223         ( p_filter->fmt_out.video.i_height != p_sys->fmt_out.video.i_height ) )
224     {
225         int i_fmt_in, i_fmt_out;
226
227         i_fmt_in = GetFfmpegChroma(p_filter->fmt_in.video.i_chroma);
228         i_fmt_out = GetFfmpegChroma(p_filter->fmt_out.video.i_chroma);
229         if( (i_fmt_in < 0) || (i_fmt_out < 0) )
230         {
231             msg_Err( p_filter, "format not supported" );
232             return VLC_EGENERIC;
233         }
234
235         if( p_sys->ctx ) sws_freeContext( p_sys->ctx );
236
237         p_sys->ctx =
238             sws_getContext( p_filter->fmt_in.video.i_width,
239                             p_filter->fmt_in.video.i_height, i_fmt_in,
240                             p_filter->fmt_out.video.i_width,
241                             p_filter->fmt_out.video.i_height, i_fmt_out,
242                             p_sys->i_sws_flags | p_sys->i_cpu_mask,
243                             p_sys->p_src_filter, p_sys->p_dst_filter, 0 );
244         if( !p_sys->ctx )
245         {
246             msg_Err( p_filter, "could not init SwScaler" );
247             return VLC_EGENERIC;
248         }
249
250         p_sys->fmt_in = p_filter->fmt_in;
251         p_sys->fmt_out = p_filter->fmt_out;
252     }
253
254     return VLC_SUCCESS;
255 }
256
257 /****************************************************************************
258  * Filter: the whole thing
259  ****************************************************************************
260  * This function is called just after the thread is launched.
261  ****************************************************************************/
262 static picture_t *Filter( filter_t *p_filter, picture_t *p_pic )
263 {
264     filter_sys_t *p_sys = p_filter->p_sys;
265     uint8_t *src[3]; int src_stride[3];
266     uint8_t *dst[3]; int dst_stride[3];
267     picture_t *p_pic_dst;
268     int i_plane;
269     int i_nb_planes = p_pic->i_planes;
270
271     /* Check if format properties changed */
272     if( CheckInit( p_filter ) != VLC_SUCCESS ) return NULL;
273
274     /* Request output picture */
275     p_pic_dst = p_filter->pf_vout_buffer_new( p_filter );
276     if( !p_pic_dst )
277     {
278         msg_Warn( p_filter, "can't get output picture" );
279         return NULL;
280     }
281
282     if( p_filter->fmt_out.video.i_chroma == VLC_FOURCC('Y','U','V','A') )
283     {
284         i_nb_planes = 3;
285         memset( p_pic_dst->p[3].p_pixels, 0xff, p_filter->fmt_out.video.i_height
286                                                  * p_pic_dst->p[3].i_pitch );
287     }
288
289     for( i_plane = 0; i_plane < __MIN(3, p_pic->i_planes); i_plane++ )
290     {
291         src[i_plane] = p_pic->p[i_plane].p_pixels;
292         src_stride[i_plane] = p_pic->p[i_plane].i_pitch;
293     }
294     for( i_plane = 0; i_plane < __MIN(3, i_nb_planes); i_plane++ )
295     {
296         dst[i_plane] = p_pic_dst->p[i_plane].p_pixels;
297         dst_stride[i_plane] = p_pic_dst->p[i_plane].i_pitch;
298     }
299
300 #if LIBSWSCALE_VERSION_INT  >= ((0<<16)+(5<<8)+0)
301     sws_scale( p_sys->ctx, src, src_stride,
302                0, p_filter->fmt_in.video.i_height,
303                dst, dst_stride );
304 #else
305     sws_scale_ordered( p_sys->ctx, src, src_stride,
306                0, p_filter->fmt_in.video.i_height,
307                dst, dst_stride );
308 #endif
309
310     p_pic_dst->date = p_pic->date;
311     p_pic_dst->b_force = p_pic->b_force;
312     p_pic_dst->i_nb_fields = p_pic->i_nb_fields;
313     p_pic_dst->b_progressive = p_pic->b_progressive;
314     p_pic_dst->b_top_field_first = p_pic->b_top_field_first;
315
316     p_pic->pf_release( p_pic );
317     return p_pic_dst;
318 }
319
320 #else /* LIBSWSCALE_VERSION_INT >= ((0<<16)+(5<<8)+0) */
321
322 int OpenScaler( vlc_object_t *p_this )
323 {
324     return VLC_EGENERIC;
325 }
326
327 void CloseScaler( vlc_object_t *p_this )
328 {
329 }
330
331 #endif /* LIBSWSCALE_VERSION_INT >= ((0<<16)+(5<<8)+0) */