avcodec/flacdec: reduce limit for golomb so that the max value does not overflow
authorMichael Niedermayer <michael@niedermayer.cc>
Mon, 20 Feb 2017 22:59:26 +0000 (23:59 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Mon, 20 Feb 2017 23:40:20 +0000 (00:40 +0100)
Fixes: runtime error: left shift of 32 by 26 places cannot be represented in type 'int'

Fixes: 628/clusterfuzz-testcase-6187747641393152
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/flacdec.c

index 6ea86d4..f73a32b 100644 (file)
@@ -258,8 +258,9 @@ static int decode_residuals(FLACContext *s, int32_t *decoded, int pred_order)
             for (; i < samples; i++)
                 *decoded++ = get_sbits_long(&s->gb, tmp);
         } else {
+            int real_limit = tmp ? (INT_MAX >> tmp) + 2 : INT_MAX;
             for (; i < samples; i++) {
-                int v = get_sr_golomb_flac(&s->gb, tmp, INT_MAX, 0);
+                int v = get_sr_golomb_flac(&s->gb, tmp, real_limit, 0);
                 if (v == 0x80000000){
                     av_log(s->avctx, AV_LOG_ERROR, "invalid residual\n");
                     return AVERROR_INVALIDDATA;