avcodec/hevc_ps: Fix max_dec_buffer check
authorMichael Niedermayer <michael@niedermayer.cc>
Sat, 24 Jun 2017 22:00:13 +0000 (00:00 +0200)
committerMichael Niedermayer <michael@niedermayer.cc>
Sat, 24 Jun 2017 23:13:17 +0000 (01:13 +0200)
Fixes: runtime error: signed integer overflow: -2147483648 - 1 cannot be represented in type 'int'
Fixes: 2339/clusterfuzz-testcase-minimized-6663164320022528

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/hevc_ps.c

index 0fa50a1..6cbf40a 100644 (file)
@@ -941,9 +941,9 @@ int ff_hevc_parse_sps(HEVCSPS *sps, GetBitContext *gb, unsigned int *sps_id,
         sps->temporal_layer[i].max_dec_pic_buffering = get_ue_golomb_long(gb) + 1;
         sps->temporal_layer[i].num_reorder_pics      = get_ue_golomb_long(gb);
         sps->temporal_layer[i].max_latency_increase  = get_ue_golomb_long(gb) - 1;
-        if (sps->temporal_layer[i].max_dec_pic_buffering > HEVC_MAX_DPB_SIZE) {
+        if (sps->temporal_layer[i].max_dec_pic_buffering > (unsigned)HEVC_MAX_DPB_SIZE) {
             av_log(avctx, AV_LOG_ERROR, "sps_max_dec_pic_buffering_minus1 out of range: %d\n",
-                   sps->temporal_layer[i].max_dec_pic_buffering - 1);
+                   sps->temporal_layer[i].max_dec_pic_buffering - 1U);
             return AVERROR_INVALIDDATA;
         }
         if (sps->temporal_layer[i].num_reorder_pics > sps->temporal_layer[i].max_dec_pic_buffering - 1) {