avcodec/adpcm: Fix undefined left shifts of negative numbers
authorAndreas Rheinhardt <andreas.rheinhardt@gmail.com>
Mon, 20 Jan 2020 15:56:55 +0000 (16:56 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Tue, 21 Jan 2020 09:01:46 +0000 (10:01 +0100)
Affected the adpcm-afc, adpcm-ea-1, adpcm-ea-2, adpcm-ea-maxis-xa,
adpcm-thp and ea-cdata FATE-tests. Also fixes ticket #8487.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/adpcm.c

index 7b5b3d9..ed16438 100644 (file)
@@ -1282,8 +1282,8 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
 
             for (count2 = 0; count2 < 28; count2++) {
                 byte = bytestream2_get_byteu(&gb);
-                next_left_sample  = sign_extend(byte >> 4, 4) << shift_left;
-                next_right_sample = sign_extend(byte,      4) << shift_right;
+                next_left_sample  = sign_extend(byte >> 4, 4) * (1 << shift_left);
+                next_right_sample = sign_extend(byte,      4) * (1 << shift_right);
 
                 next_left_sample = (next_left_sample +
                     (current_left_sample * coeff1l) +
@@ -1322,7 +1322,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
             if (st) byte[1] = bytestream2_get_byteu(&gb);
             for(i = 4; i >= 0; i-=4) { /* Pairwise samples LL RR (st) or LL LL (mono) */
                 for(channel = 0; channel < avctx->channels; channel++) {
-                    int sample = sign_extend(byte[channel] >> i, 4) << shift[channel];
+                    int sample = sign_extend(byte[channel] >> i, 4) * (1 << shift[channel]);
                     sample = (sample +
                              c->status[channel].sample1 * coeff[channel][0] +
                              c->status[channel].sample2 * coeff[channel][1] + 0x80) >> 8;
@@ -1437,11 +1437,11 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
                     int level, pred;
                     int byte = bytestream2_get_byteu(&gb);
 
-                    level = sign_extend(byte >> 4, 4) << shift[n];
+                    level = sign_extend(byte >> 4, 4) * (1 << shift[n]);
                     pred  = s[-1] * coeff[0][n] + s[-2] * coeff[1][n];
                     s[0]  = av_clip_int16((level + pred + 0x80) >> 8);
 
-                    level = sign_extend(byte, 4) << shift[n];
+                    level = sign_extend(byte, 4) * (1 << shift[n]);
                     pred  = s[0] * coeff[0][n] + s[-1] * coeff[1][n];
                     s[1]  = av_clip_int16((level + pred + 0x80) >> 8);
                 }
@@ -1598,8 +1598,8 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
                         sampledat = sign_extend(byte >> 4, 4);
                     }
 
-                    sampledat = ((prev1 * factor1 + prev2 * factor2) +
-                                 ((sampledat * scale) << 11)) >> 11;
+                    sampledat = ((prev1 * factor1 + prev2 * factor2) >> 11) +
+                                sampledat * scale;
                     *samples = av_clip_int16(sampledat);
                     prev2 = prev1;
                     prev1 = *samples++;
@@ -1676,7 +1676,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
                     }
 
                     sampledat = ((c->status[ch].sample1 * factor1
-                                + c->status[ch].sample2 * factor2) >> 11) + (sampledat << exp);
+                                + c->status[ch].sample2 * factor2) >> 11) + sampledat * (1 << exp);
                     *samples = av_clip_int16(sampledat);
                     c->status[ch].sample2 = c->status[ch].sample1;
                     c->status[ch].sample1 = *samples++;