avcodec/dirac_vlc: Fix undefined shift
authorMichael Niedermayer <michael@niedermayer.cc>
Tue, 18 Jul 2017 23:43:24 +0000 (01:43 +0200)
committerMichael Niedermayer <michael@niedermayer.cc>
Mon, 24 Jul 2017 01:03:14 +0000 (03:03 +0200)
Fixes: runtime error: shift exponent 64 is too large for 64-bit type 'residual' (aka 'unsigned long')
Fixes: 2674/clusterfuzz-testcase-minimized-4999700518273024

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 69e7daf6ce2a5893936ba18572c58180b29d67f9)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/dirac_vlc.c

index 336d22a..773f720 100644 (file)
@@ -216,9 +216,14 @@ static void generate_offset_lut(DiracGolombLUT *lut, int off)
         INIT_RESIDUE(res);
         SET_RESIDUE(res, idx, LUT_BITS);
 
-        l->preamble      = CONVERT_TO_RESIDUE(res >> (RSIZE_BITS - off), off);
         l->preamble_bits = off;
-        l->sign = ((l->preamble >> (RSIZE_BITS - l->preamble_bits)) & 1) ? -1 : +1;
+        if (off) {
+            l->preamble  = CONVERT_TO_RESIDUE(res >> (RSIZE_BITS - off), off);
+            l->sign      = ((l->preamble >> (RSIZE_BITS - l->preamble_bits)) & 1) ? -1 : +1;
+        } else {
+            l->preamble  = 0;
+            l->sign = 1;
+        }
 
         search_for_golomb(l, res << off, LUT_BITS - off);
     }