vdpau/chroma: NULL dereference on error
[vlc.git] / modules / hw / vdpau / chroma.c
1 /*****************************************************************************
2  * chroma.c: VLC picture import into VDPAU
3  *****************************************************************************
4  * Copyright (C) 2013 RĂ©mi Denis-Courmont
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by
8  * the Free Software Foundation; either version 2.1 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
19  *****************************************************************************/
20
21 #ifdef HAVE_CONFIG_H
22 # include "config.h"
23 #endif
24
25 #include <stdlib.h>
26 #include <inttypes.h>
27 #include <assert.h>
28
29 #include <vlc_common.h>
30 #include <vlc_plugin.h>
31 #include <vlc_filter.h>
32 #include "vlc_vdpau.h"
33
34 /* Picture history as recommended by VDPAU documentation */
35 #define MAX_PAST   2
36 #define MAX_FUTURE 1
37
38 struct filter_sys_t
39 {
40     vdp_t *vdp;
41     VdpDevice device;
42     VdpVideoMixer mixer;
43     VdpChromaType chroma;
44     VdpYCbCrFormat format;
45     picture_t *(*import)(filter_t *, picture_t *);
46
47     struct
48     {
49         vlc_vdp_video_field_t *field;
50         mtime_t date;
51         bool force;
52     } history[MAX_PAST + 1 + MAX_FUTURE];
53
54     struct
55     {
56         float brightness;
57         float contrast;
58         float saturation;
59         float hue;
60     } procamp;
61 };
62
63 /** Initialize the colour space conversion matrix */
64 static VdpStatus MixerSetupColors(filter_t *filter, const VdpProcamp *procamp,
65                                   VdpCSCMatrix *restrict csc)
66 {
67     filter_sys_t *sys = filter->p_sys;
68     VdpStatus err;
69     VdpColorStandard std = (filter->fmt_in.video.i_height > 576)
70                          ? VDP_COLOR_STANDARD_ITUR_BT_709
71                          : VDP_COLOR_STANDARD_ITUR_BT_601;
72
73     err = vdp_generate_csc_matrix(sys->vdp, procamp, std, csc);
74     if (err != VDP_STATUS_OK)
75     {
76         msg_Err(filter, "video %s failure: %s", "color space matrix",
77                 vdp_get_error_string(sys->vdp, err));
78         return err;
79     }
80
81     if (procamp != NULL)
82     {
83         sys->procamp.brightness = procamp->brightness;
84         sys->procamp.contrast = procamp->contrast;
85         sys->procamp.saturation = procamp->saturation;
86         sys->procamp.hue = procamp->hue;
87     }
88     else
89     {
90         sys->procamp.brightness = 0.f;
91         sys->procamp.contrast = 1.f;
92         sys->procamp.saturation = 1.f;
93         sys->procamp.hue = 0.f;
94     }
95     return VDP_STATUS_OK;
96 }
97
98 /** Create VDPAU video mixer */
99 static VdpVideoMixer MixerCreate(filter_t *filter)
100 {
101     filter_sys_t *sys = filter->p_sys;
102     VdpVideoMixer mixer;
103     VdpStatus err;
104     VdpBool ok;
105
106     /* Check for potentially useful features */
107     VdpVideoMixerFeature featv[5];
108     unsigned featc = 0;
109
110     int algo = var_InheritInteger(filter, "vdpau-deinterlace");
111     bool ivtc = false;
112     if (algo == VDP_VIDEO_MIXER_FEATURE_DEINTERLACE_TEMPORAL_SPATIAL)
113     {
114         err = vdp_video_mixer_query_feature_support(sys->vdp, sys->device,
115                                                     algo, &ok);
116         if (err == VDP_STATUS_OK && ok == VDP_TRUE)
117             msg_Dbg(filter, "using video mixer %s feature",
118                     "temporal-spatial deinterlace");
119         else
120             algo = VDP_VIDEO_MIXER_FEATURE_DEINTERLACE_TEMPORAL; /* fallback */
121     }
122     if (algo == VDP_VIDEO_MIXER_FEATURE_DEINTERLACE_TEMPORAL)
123     {
124         err = vdp_video_mixer_query_feature_support(sys->vdp, sys->device,
125                                                     algo, &ok);
126         if (err == VDP_STATUS_OK && ok == VDP_TRUE)
127             msg_Dbg(filter, "using video mixer %s feature",
128                     "temporal deinterlace");
129         else
130             algo = -1;
131     }
132     if (algo >= 0)
133     {
134         featv[featc++] = algo;
135         ivtc = var_InheritBool(filter, "vdpau-ivtc");
136         if (ivtc)
137         {
138             err = vdp_video_mixer_query_feature_support(sys->vdp, sys->device,
139                                 VDP_VIDEO_MIXER_FEATURE_INVERSE_TELECINE, &ok);
140             if (err == VDP_STATUS_OK && ok == VDP_TRUE)
141                 msg_Dbg(filter, "using video mixer %s feature",
142                         "inverse telecine");
143             featv[featc++] = VDP_VIDEO_MIXER_FEATURE_INVERSE_TELECINE;
144         }
145     }
146
147     const float noise = var_InheritFloat(filter, "vdpau-noise-reduction");
148     if (noise > 0.f)
149     {
150         err = vdp_video_mixer_query_feature_support(sys->vdp, sys->device,
151                                  VDP_VIDEO_MIXER_FEATURE_NOISE_REDUCTION, &ok);
152         if (err == VDP_STATUS_OK && ok == VDP_TRUE)
153         {
154             msg_Dbg(filter, "using video mixer %s feature", "noise reduction");
155             featv[featc++] = VDP_VIDEO_MIXER_FEATURE_NOISE_REDUCTION;
156         }
157     }
158
159     err = vdp_video_mixer_query_feature_support(sys->vdp, sys->device,
160                                        VDP_VIDEO_MIXER_FEATURE_SHARPNESS, &ok);
161     if (err == VDP_STATUS_OK && ok == VDP_TRUE)
162     {
163         msg_Dbg(filter, "using video mixer %s feature", "sharpness");
164         featv[featc++] = VDP_VIDEO_MIXER_FEATURE_SHARPNESS;
165     }
166
167     const int offset = VDP_VIDEO_MIXER_FEATURE_HIGH_QUALITY_SCALING_L1 - 1;
168     unsigned level = var_InheritInteger(filter, "vdpau-scaling");
169     while (level > 0)
170     {
171
172         err = vdp_video_mixer_query_feature_support(sys->vdp, sys->device,
173                                                     offset + level, &ok);
174         if (err == VDP_STATUS_OK && ok == VDP_TRUE)
175         {
176             msg_Dbg(filter, "using video mixer high quality scaling L%u",
177                     level);
178             featv[featc++] = offset + level;
179             break;
180         }
181         level--; /* fallback to lower quality */
182     }
183
184     /* Create the mixer */
185     VdpVideoMixerParameter parms[3] = {
186         VDP_VIDEO_MIXER_PARAMETER_VIDEO_SURFACE_WIDTH,
187         VDP_VIDEO_MIXER_PARAMETER_VIDEO_SURFACE_HEIGHT,
188         VDP_VIDEO_MIXER_PARAMETER_CHROMA_TYPE,
189     };
190     uint32_t width = filter->fmt_in.video.i_width;
191     uint32_t height = filter->fmt_in.video.i_height;
192     const void *values[3] = { &width, &height, &sys->chroma, };
193
194     err = vdp_video_mixer_create(sys->vdp, sys->device, featc, featv,
195                                  3, parms, values, &mixer);
196     if (err != VDP_STATUS_OK)
197     {
198         msg_Err(filter, "video %s %s failure: %s", "mixer", "creation",
199                 vdp_get_error_string(sys->vdp, err));
200         mixer = VDP_INVALID_HANDLE;
201     }
202
203     /* Set initial features and attributes */
204     VdpVideoMixerAttribute attrv[3];
205     const void *valv[3];
206     unsigned attrc = 0;
207     VdpCSCMatrix csc;
208     uint8_t chroma_skip;
209
210     featc = 0;
211
212     if (MixerSetupColors(filter, NULL, &csc) == VDP_STATUS_OK)
213     {
214         attrv[attrc] = VDP_VIDEO_MIXER_ATTRIBUTE_CSC_MATRIX;
215         valv[attrc] = &csc;
216         attrc++;
217     }
218
219     if (algo >= 0)
220     {
221         featv[featc++] = algo;
222         if (ivtc)
223             featv[featc++] = VDP_VIDEO_MIXER_FEATURE_INVERSE_TELECINE;
224
225         chroma_skip = var_InheritBool(filter, "vdpau-chroma-skip");
226         attrv[attrc] = VDP_VIDEO_MIXER_ATTRIBUTE_SKIP_CHROMA_DEINTERLACE;
227         valv[attrc] = &chroma_skip;
228         attrc++;
229     }
230
231     if (noise > 0.f)
232     {
233         featv[featc++] = VDP_VIDEO_MIXER_FEATURE_NOISE_REDUCTION;
234
235         attrv[attrc] = VDP_VIDEO_MIXER_ATTRIBUTE_NOISE_REDUCTION_LEVEL;
236         valv[attrc] = &noise;
237         attrc++;
238     }
239
240     if (level > 0)
241         featv[featc++] = offset + level;
242
243     if (featc > 0)
244     {
245         VdpBool enablev[featc];
246
247         for (unsigned i = 0; i < featc; i++)
248             enablev[i] = VDP_TRUE;
249
250         err = vdp_video_mixer_set_feature_enables(sys->vdp, mixer,
251                                                   featc, featv, enablev);
252         if (err != VDP_STATUS_OK)
253             msg_Err(filter, "video %s %s failure: %s", "mixer", "features",
254                     vdp_get_error_string(sys->vdp, err));
255     }
256
257     if (attrc > 0)
258     {
259         err = vdp_video_mixer_set_attribute_values(sys->vdp, mixer,
260                                                    attrc, attrv, valv);
261         if (err != VDP_STATUS_OK)
262             msg_Err(filter, "video %s %s failure: %s", "mixer", "attributes",
263                     vdp_get_error_string(sys->vdp, err));
264     }
265
266     return mixer;
267 }
268
269 static void Flush(filter_t *filter)
270 {
271     filter_sys_t *sys = filter->p_sys;
272
273     for (unsigned i = 0; i < MAX_PAST + MAX_FUTURE; i++)
274         if (sys->history[i].field != NULL)
275         {
276             sys->history[i].field->destroy(sys->history[i].field);
277             sys->history[i].field = NULL;
278         }
279 }
280
281 /** Get a VLC picture for a VDPAU output surface */
282 static picture_t *OutputAllocate(filter_t *filter)
283 {
284     filter_sys_t *sys = filter->p_sys;
285
286     picture_t *pic = filter_NewPicture(filter);
287     if (pic == NULL)
288         return NULL;
289
290     picture_sys_t *psys = pic->p_sys;
291     assert(psys->vdp != NULL);
292
293     if (unlikely(sys->vdp != psys->vdp) && (sys->mixer != VDP_INVALID_HANDLE))
294     {
295         Flush(filter); /* release surfaces from the old device */
296         vdp_video_mixer_destroy(sys->vdp, sys->mixer);
297         vdp_release_x11(sys->vdp);
298         sys->mixer = VDP_INVALID_HANDLE;
299     }
300
301     if (unlikely(sys->mixer == VDP_INVALID_HANDLE))
302     {
303         sys->vdp = vdp_hold_x11(psys->vdp, NULL);
304         sys->device = psys->device;
305         sys->mixer = MixerCreate(filter);
306         if (sys->mixer == VDP_INVALID_HANDLE)
307             goto error;
308         msg_Dbg(filter, "using video mixer %"PRIu32, sys->mixer);
309     }
310     return pic;
311 error:
312     picture_Release(pic);
313     return NULL;
314 }
315
316 /** Export a VDPAU video surface picture to a normal VLC picture */
317 static picture_t *VideoExport(filter_t *filter, picture_t *src, picture_t *dst)
318 {
319     filter_sys_t *sys = filter->p_sys;
320     vlc_vdp_video_field_t *field = src->context;
321     vlc_vdp_video_frame_t *psys = field->frame;
322     VdpStatus err;
323     VdpVideoSurface surface = psys->surface;
324     void *planes[3];
325     uint32_t pitches[3];
326
327     picture_CopyProperties(dst, src);
328
329     for (int i = 0; i < dst->i_planes; i++)
330     {
331         planes[i] = dst->p[i].p_pixels;
332         pitches[i] = dst->p[i].i_pitch;
333     }
334     if (dst->format.i_chroma == VLC_CODEC_I420
335      || dst->format.i_chroma == VLC_CODEC_I422)
336     {
337         planes[1] = dst->p[2].p_pixels;
338         planes[2] = dst->p[1].p_pixels;
339         pitches[1] = dst->p[2].i_pitch;
340         pitches[2] = dst->p[1].i_pitch;
341     }
342     err = vdp_video_surface_get_bits_y_cb_cr(psys->vdp, surface, sys->format,
343                                              planes, pitches);
344     if (err != VDP_STATUS_OK)
345     {
346         msg_Err(filter, "video %s %s failure: %s", "surface", "export",
347                 vdp_get_error_string(psys->vdp, err));
348         picture_Release(dst);
349         dst = NULL;
350     }
351     picture_Release(src);
352     return dst;
353 }
354
355 /** Import VLC picture into VDPAU video surface */
356 static picture_t *VideoImport(filter_t *filter, picture_t *src)
357 {
358     filter_sys_t *sys = filter->p_sys;
359     VdpVideoSurface surface;
360     VdpStatus err;
361
362     if (sys->vdp == NULL)
363         goto drop;
364
365     /* Create surface (TODO: reuse?) */
366     err = vdp_video_surface_create(sys->vdp, sys->device, sys->chroma,
367                                    filter->fmt_in.video.i_width,
368                                    filter->fmt_in.video.i_height, &surface);
369     if (err != VDP_STATUS_OK)
370     {
371         msg_Err(filter, "video %s %s failure: %s", "surface", "creation",
372                 vdp_get_error_string(sys->vdp, err));
373         goto drop;
374     }
375
376     /* Put bits */
377     const void *planes[3];
378     uint32_t pitches[3];
379     for (int i = 0; i < src->i_planes; i++)
380     {
381         planes[i] = src->p[i].p_pixels;
382         pitches[i] = src->p[i].i_pitch;
383     }
384     if (src->format.i_chroma == VLC_CODEC_I420)
385     {
386         planes[1] = src->p[2].p_pixels;
387         planes[2] = src->p[1].p_pixels;
388         pitches[1] = src->p[2].i_pitch;
389         pitches[2] = src->p[1].i_pitch;
390     }
391     err = vdp_video_surface_put_bits_y_cb_cr(sys->vdp, surface, sys->format,
392                                              planes, pitches);
393     if (err != VDP_STATUS_OK)
394     {
395         msg_Err(filter, "video %s %s failure: %s", "surface", "import",
396                 vdp_get_error_string(sys->vdp, err));
397         goto error;
398     }
399
400     /* Wrap surface into a picture */
401     video_format_t fmt = src->format;
402     fmt.i_chroma = (sys->chroma == VDP_CHROMA_TYPE_420)
403         ? VLC_CODEC_VDPAU_VIDEO_420 : VLC_CODEC_VDPAU_VIDEO_422;
404
405     picture_t *dst = picture_NewFromFormat(&fmt);
406     if (unlikely(dst == NULL))
407         goto error;
408     picture_CopyProperties(dst, src);
409     picture_Release(src);
410
411     err = vlc_vdp_video_attach(sys->vdp, surface, dst);
412     if (unlikely(err != VDP_STATUS_OK))
413     {
414         picture_Release(dst);
415         dst = NULL;
416     }
417     return dst;
418 error:
419     vdp_video_surface_destroy(sys->vdp, surface);
420 drop:
421     picture_Release(src);
422     return NULL;
423 }
424
425 static picture_t *VideoPassthrough(filter_t *filter, picture_t *src)
426 {
427     filter_sys_t *sys = filter->p_sys;
428     vlc_vdp_video_field_t *field = src->context;
429
430     if (unlikely(field == NULL))
431     {
432         msg_Err(filter, "corrupt VDPAU video surface");
433         return NULL;
434     }
435
436     vlc_vdp_video_frame_t *psys = field->frame;
437
438     /* Corner case: different VDPAU instances decoding and rendering */
439     if (psys->vdp != sys->vdp)
440     {
441         video_format_t fmt = src->format;
442         switch (sys->chroma)
443         {
444              case VDP_CHROMA_TYPE_420: fmt.i_chroma = VLC_CODEC_NV12; break;
445              case VDP_CHROMA_TYPE_422: fmt.i_chroma = VLC_CODEC_UYVY; break;
446              case VDP_CHROMA_TYPE_444: fmt.i_chroma = VLC_CODEC_NV24; break;
447              default: assert(0);
448         }
449
450         picture_t *pic = picture_NewFromFormat(&fmt);
451         if (unlikely(pic == NULL))
452             return NULL;
453
454         pic = VideoExport(filter, src, pic);
455         if (pic == NULL)
456             return NULL;
457
458         src = VideoImport(filter, pic);
459     }
460     return src;
461 }
462
463 static inline VdpVideoSurface picture_GetVideoSurface(const picture_t *pic)
464 {
465     vlc_vdp_video_field_t *field = pic->context;
466     return field->frame->surface;
467 }
468
469 static picture_t *MixerRender(filter_t *filter, picture_t *src)
470 {
471     filter_sys_t *sys = filter->p_sys;
472     VdpStatus err;
473
474     picture_t *dst = OutputAllocate(filter);
475
476     src = sys->import(filter, src);
477     /* Update history and take "present" picture field */
478     if (likely(src != NULL))
479     {
480         sys->history[MAX_PAST + MAX_FUTURE].field =
481                                               vlc_vdp_video_copy(src->context);
482         sys->history[MAX_PAST + MAX_FUTURE].date = src->date;
483         sys->history[MAX_PAST + MAX_FUTURE].force = src->b_force;
484         picture_Release(src);
485     }
486     else
487         sys->history[MAX_PAST + MAX_FUTURE].field = NULL;
488
489     if (dst == NULL)
490         goto skip;
491
492     vlc_vdp_video_field_t *f = sys->history[MAX_PAST].field;
493     if (f == NULL)
494     {
495         picture_Release(dst);
496         dst = NULL;
497         goto skip;
498     }
499
500     dst->date = sys->history[MAX_PAST].date;
501     dst->b_force = sys->history[MAX_PAST].force;
502
503     /* Enable/Disable features */
504     const VdpVideoMixerFeature features[] = {
505         VDP_VIDEO_MIXER_FEATURE_SHARPNESS,
506     };
507     const VdpBool enables[] = {
508         f->sharpen != 0.f,
509     };
510
511     err = vdp_video_mixer_set_feature_enables(sys->vdp, sys->mixer,
512                   sizeof (features) / sizeof (features[0]), features, enables);
513     if (err != VDP_STATUS_OK)
514         msg_Err(filter, "video %s %s failure: %s", "mixer", "features",
515                 vdp_get_error_string(sys->vdp, err));
516
517     /* Configure mixer depending on upstream video filters */
518     VdpVideoMixerAttribute attrs[2] = {
519         VDP_VIDEO_MIXER_ATTRIBUTE_SHARPNESS_LEVEL,
520     };
521     const void *values[2] = {
522         &f->sharpen,
523     };
524     unsigned count = 1;
525     VdpCSCMatrix csc;
526
527     if ((sys->procamp.brightness != f->procamp.brightness
528       || sys->procamp.contrast != f->procamp.contrast
529       || sys->procamp.saturation != f->procamp.saturation
530       || sys->procamp.hue != f->procamp.hue)
531      && (MixerSetupColors(filter, &f->procamp, &csc) == VDP_STATUS_OK))
532     {
533         attrs[count] = VDP_VIDEO_MIXER_ATTRIBUTE_CSC_MATRIX;
534         values[count] = &csc;
535         count++;
536     }
537
538     err = vdp_video_mixer_set_attribute_values(sys->vdp, sys->mixer,
539                                                count, attrs, values);
540     if (err != VDP_STATUS_OK)
541         msg_Err(filter, "video %s %s failure: %s", "mixer", "attributes",
542                 vdp_get_error_string(sys->vdp, err));
543
544     /* Render video into output */
545     VdpVideoMixerPictureStructure structure = f->structure;
546     VdpVideoSurface past[MAX_PAST];
547     VdpVideoSurface surface = f->frame->surface;
548     VdpVideoSurface future[MAX_FUTURE];
549     VdpRect src_rect = {
550         filter->fmt_in.video.i_x_offset, filter->fmt_in.video.i_y_offset,
551         filter->fmt_in.video.i_visible_width + filter->fmt_in.video.i_x_offset,
552         filter->fmt_in.video.i_visible_height + filter->fmt_in.video.i_y_offset
553     };
554     VdpOutputSurface output = dst->p_sys->surface;
555     VdpRect dst_rect = {
556         0, 0,
557         filter->fmt_out.video.i_visible_width,
558         filter->fmt_out.video.i_visible_height
559     };
560
561     for (unsigned i = 0; i < MAX_PAST; i++)
562     {
563         f = sys->history[(MAX_PAST - 1) - i].field;
564         past[i] = (f != NULL) ? f->frame->surface : VDP_INVALID_HANDLE;
565     }
566     for (unsigned i = 0; i < MAX_FUTURE; i++)
567     {
568         f = sys->history[(MAX_PAST + 1) + i].field;
569         future[i] = (f != NULL) ? f->frame->surface : VDP_INVALID_HANDLE;
570     }
571
572     err = vdp_video_mixer_render(sys->vdp, sys->mixer, VDP_INVALID_HANDLE,
573                                  NULL, structure,
574                                  MAX_PAST, past, surface, MAX_FUTURE, future,
575                                  &src_rect, output, &dst_rect, NULL, 0, NULL);
576     if (err != VDP_STATUS_OK)
577     {
578         msg_Err(filter, "video %s %s failure: %s", "mixer", "rendering",
579                 vdp_get_error_string(sys->vdp, err));
580         picture_Release(dst);
581         dst = NULL;
582     }
583
584 skip:
585     f = sys->history[0].field;
586     if (f != NULL)
587         f->destroy(f); /* Release oldest field */
588     memmove(sys->history, sys->history + 1, /* Advance history */
589             sizeof (sys->history[0]) * (MAX_PAST + MAX_FUTURE));
590
591     return dst;
592 }
593
594 static int OutputOpen(vlc_object_t *obj)
595 {
596     filter_t *filter = (filter_t *)obj;
597     if (filter->fmt_out.video.i_chroma != VLC_CODEC_VDPAU_OUTPUT)
598         return VLC_EGENERIC;
599
600     filter_sys_t *sys = malloc(sizeof (*sys));
601     if (unlikely(sys == NULL))
602         return VLC_ENOMEM;
603
604     sys->vdp = NULL;
605     sys->mixer = VDP_INVALID_HANDLE;
606
607     if (filter->fmt_in.video.i_chroma == VLC_CODEC_VDPAU_VIDEO_444)
608     {
609         sys->chroma = VDP_CHROMA_TYPE_444;
610         sys->format = VDP_YCBCR_FORMAT_NV12;
611         sys->import = VideoPassthrough;
612     }
613     else
614     if (filter->fmt_in.video.i_chroma == VLC_CODEC_VDPAU_VIDEO_422)
615     {
616         sys->chroma = VDP_CHROMA_TYPE_422;
617         /* TODO: check if the drivery supports NV12 or UYVY */
618         sys->format = VDP_YCBCR_FORMAT_UYVY;
619         sys->import = VideoPassthrough;
620     }
621     else
622     if (filter->fmt_in.video.i_chroma == VLC_CODEC_VDPAU_VIDEO_420)
623     {
624         sys->chroma = VDP_CHROMA_TYPE_420;
625         sys->format = VDP_YCBCR_FORMAT_NV12;
626         sys->import = VideoPassthrough;
627     }
628     else
629     if (vlc_fourcc_to_vdp_ycc(filter->fmt_in.video.i_chroma,
630                               &sys->chroma, &sys->format))
631         sys->import = VideoImport;
632     else
633     {
634         free(sys);
635         return VLC_EGENERIC;
636     }
637
638     /* NOTE: The video mixer capabilities should be checked here, and the
639      * then video mixer set up. But:
640      * 1) The VDPAU back-end is accessible only once the first picture
641      *    gets filtered. Thus the video mixer is created later.
642      * 2) Bailing out due to insufficient capabilities would break the
643      *    video pipeline. Thus capabilities should be checked earlier. */
644
645     for (unsigned i = 0; i < MAX_PAST + MAX_FUTURE; i++)
646         sys->history[i].field = NULL;
647
648     sys->procamp.brightness = 0.f;
649     sys->procamp.contrast = 1.f;
650     sys->procamp.saturation = 1.f;
651     sys->procamp.hue = 0.f;
652
653     filter->pf_video_filter = MixerRender;
654     filter->pf_video_flush = Flush;
655     filter->p_sys = sys;
656     return VLC_SUCCESS;
657 }
658
659 static void OutputClose(vlc_object_t *obj)
660 {
661     filter_t *filter = (filter_t *)obj;
662     filter_sys_t *sys = filter->p_sys;
663
664     Flush(filter);
665     if (sys->mixer != VDP_INVALID_HANDLE)
666     {
667         vdp_video_mixer_destroy(sys->vdp, sys->mixer);
668         vdp_release_x11(sys->vdp);
669     }
670     free(sys);
671 }
672
673 static picture_t *VideoExport_Filter(filter_t *filter, picture_t *src)
674 {
675     if (unlikely(src->context == NULL))
676     {
677         msg_Err(filter, "corrupt VDPAU video surface %p", src);
678         picture_Release(src);
679         return NULL;
680     }
681
682     picture_t *dst = filter_NewPicture(filter);
683     if (dst == NULL)
684         return NULL;
685
686     return VideoExport(filter, src, dst);
687 }
688
689 static int YCbCrOpen(vlc_object_t *obj)
690 {
691     filter_t *filter = (filter_t *)obj;
692     if (filter->fmt_in.video.i_chroma != VLC_CODEC_VDPAU_VIDEO_420
693      && filter->fmt_in.video.i_chroma != VLC_CODEC_VDPAU_VIDEO_422
694      && filter->fmt_in.video.i_chroma != VLC_CODEC_VDPAU_VIDEO_444)
695         return VLC_EGENERIC;
696
697     if (filter->fmt_in.video.i_visible_width
698                                        != filter->fmt_out.video.i_visible_width
699      || filter->fmt_in.video.i_visible_height
700                                       != filter->fmt_out.video.i_visible_height
701      || filter->fmt_in.video.i_x_offset != filter->fmt_out.video.i_x_offset
702      || filter->fmt_in.video.i_y_offset != filter->fmt_out.video.i_y_offset
703      || (filter->fmt_in.video.i_sar_num * filter->fmt_out.video.i_sar_den
704           != filter->fmt_in.video.i_sar_den * filter->fmt_out.video.i_sar_num))
705         return VLC_EGENERIC;
706
707     filter_sys_t *sys = malloc(sizeof (*sys));
708     if (unlikely(sys == NULL))
709         return VLC_ENOMEM;
710
711     if (!vlc_fourcc_to_vdp_ycc(filter->fmt_out.video.i_chroma,
712                                &sys->chroma, &sys->format))
713     {
714         free(sys);
715         return VLC_EGENERIC;
716     }
717
718     filter->pf_video_filter = VideoExport_Filter;
719     filter->p_sys = sys;
720     return VLC_SUCCESS;
721 }
722
723 static void YCbCrClose(vlc_object_t *obj)
724 {
725     filter_t *filter = (filter_t *)obj;
726     filter_sys_t *sys = filter->p_sys;
727
728     free(sys);
729 }
730
731 static const int algo_values[] = {
732     -1,
733     VDP_VIDEO_MIXER_FEATURE_DEINTERLACE_TEMPORAL,
734     VDP_VIDEO_MIXER_FEATURE_DEINTERLACE_TEMPORAL_SPATIAL,
735 };
736
737 static const char *const algo_names[] = {
738     N_("Bob"), N_("Temporal"), N_("Temporal-spatial"),
739 };
740
741 vlc_module_begin()
742     set_shortname(N_("VDPAU"))
743     set_description(N_("VDPAU surface conversions"))
744     set_capability("video filter2", 10)
745     set_category(CAT_VIDEO)
746     set_subcategory(SUBCAT_VIDEO_VFILTER)
747     set_callbacks(OutputOpen, OutputClose)
748
749     add_integer("vdpau-deinterlace",
750                 VDP_VIDEO_MIXER_FEATURE_DEINTERLACE_TEMPORAL_SPATIAL,
751                 N_("Deinterlace"), N_("Deinterlacing algorithm"), true)
752         change_integer_list(algo_values, algo_names)
753     add_bool("vdpau-ivtc", false,
754              N_("Inverse telecine"), N_("Inverse telecine"), true)
755     add_bool("vdpau-chroma-skip", false,
756              N_("Deinterlace chroma skip"),
757              N_("Whether temporal deinterlacing applies to luma only"), true)
758     add_float_with_range("vdpau-noise-reduction", 0., 0., 1.,
759         N_("Noise reduction level"), N_("Noise reduction level"), true)
760     add_integer_with_range("vdpau-scaling", 0, 0, 9,
761        N_("Scaling quality"), N_("High quality scaling level"), true)
762
763     add_submodule()
764     set_callbacks(YCbCrOpen, YCbCrClose)
765 vlc_module_end()