Check more completely for invalid pix_fmts in planarRgb16ToRgb16Wrapper()
authorCarl Eugen Hoyos <cehoyos@ag.or.at>
Mon, 10 Jun 2013 23:04:49 +0000 (01:04 +0200)
committerCarl Eugen Hoyos <cehoyos@ag.or.at>
Mon, 10 Jun 2013 23:14:38 +0000 (01:14 +0200)
This makes the conversion function more robust.

libswscale/swscale_unscaled.c

index 774c655..3b07800 100644 (file)
@@ -495,7 +495,8 @@ static int planarRgb16ToRgb16Wrapper(SwsContext *c, const uint8_t *src[],
         swap += 2;
 
     if ((src_format->flags & (AV_PIX_FMT_FLAG_PLANAR | AV_PIX_FMT_FLAG_RGB)) !=
-        (AV_PIX_FMT_FLAG_PLANAR | AV_PIX_FMT_FLAG_RGB)) {
+        (AV_PIX_FMT_FLAG_PLANAR | AV_PIX_FMT_FLAG_RGB) ||
+        bits_per_sample <= 8) {
         av_log(c, AV_LOG_ERROR, "unsupported planar RGB conversion %s -> %s\n",
                src_format->name, dst_format->name);
         return srcSliceH;