avcodec/mss2: Fix integer overflow
authorMichael Niedermayer <michaelni@gmx.at>
Wed, 1 Jul 2015 18:29:44 +0000 (20:29 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Wed, 1 Jul 2015 18:30:48 +0000 (20:30 +0200)
This also simplifies the code
Fixes: signal_sigabrt_7ffff6ac8cc9_2943_cov_3588637614_mss2_speech.wmv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/mss2.c

index f57685d..f788143 100644 (file)
@@ -52,9 +52,9 @@ static void arith2_normalise(ArithCoder *c)
             c->value ^= 0x8000;
             c->low   ^= 0x8000;
         }
-        c->high  = c->high  << 8 & 0xFFFFFF | 0xFF;
-        c->value = c->value << 8 & 0xFFFFFF | bytestream2_get_byte(c->gbc.gB);
-        c->low   = c->low   << 8 & 0xFFFFFF;
+        c->high  = (uint16_t)c->high  << 8  | 0xFF;
+        c->value = (uint16_t)c->value << 8  | bytestream2_get_byte(c->gbc.gB);
+        c->low   = (uint16_t)c->low   << 8;
     }
 }