avcodec/rv40: Fix runtime error: left shift of negative value
authorMichael Niedermayer <michael@niedermayer.cc>
Tue, 21 Feb 2017 02:05:32 +0000 (03:05 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Tue, 21 Feb 2017 14:31:06 +0000 (15:31 +0100)
Fixes: 630/clusterfuzz-testcase-6608718928019456

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/rv40.c
libavcodec/rv40dsp.c

index e5ba215..dfeebda 100644 (file)
@@ -189,7 +189,7 @@ static int rv40_decode_intra_types(RV34DecContext *r, GetBitContext *gb, int8_t
             A = ptr[-r->intra_types_stride + 1]; // it won't be used for the last coefficient in a row
             B = ptr[-r->intra_types_stride];
             C = ptr[-1];
-            pattern = A + (B << 4) + (C << 8);
+            pattern = A + B * (1 << 4) + C * (1 << 8);
             for(k = 0; k < MODE2_PATTERNS_NUM; k++)
                 if(pattern == rv40_aic_table_index[k])
                     break;
index 19b0e93..95ba0a9 100644 (file)
@@ -449,7 +449,7 @@ static av_always_inline void rv40_weak_loop_filter(uint8_t *src,
         if (u > 3 - (filter_p1 && filter_q1))
             continue;
 
-        t <<= 2;
+        t *= 1 << 2;
         if (filter_p1 && filter_q1)
             t += src[-2*step] - src[1*step];