h264: vdpau: fix crash with unsupported colorspace
authorUoti Urpala <uau@glyph.nonexistent.invalid>
Sat, 18 Aug 2012 14:11:45 +0000 (16:11 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Sat, 18 Aug 2012 14:36:09 +0000 (16:36 +0200)
The h264_vdpau decoder crashed if output colorspace was not 8-bit 420.
Add a check to error out instead (current hardware does not support
other colorspaces, so successful decoding is not possible).

Check implemented at a different place by michael, thus blame for bugs goes to michael
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/h264.c

index c07d10a..42de033 100644 (file)
@@ -2494,6 +2494,15 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         s->avctx->sample_aspect_ratio = h->sps.sar;
         av_assert0(s->avctx->sample_aspect_ratio.den);
 
+        if (s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU
+            && (h->sps.bit_depth_luma != 8 ||
+                h->sps.chroma_format_idc > 1)) {
+            av_log(s->avctx, AV_LOG_ERROR,
+                   "VDPAU decoding does not support video "
+                   "colorspace\n");
+            return -1;
+        }
+
         if (s->avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
             h->cur_chroma_format_idc != h->sps.chroma_format_idc) {
             if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 14 && h->sps.bit_depth_luma != 11 && h->sps.bit_depth_luma != 13 &&