avcodec/dirac_dwt_template: Fix Integer overflow in horizontal_compose_dd137i()
authorMichael Niedermayer <michael@niedermayer.cc>
Sat, 17 Feb 2018 20:47:09 +0000 (21:47 +0100)
committerMichael Niedermayer <michael@niedermayer.cc>
Sun, 18 Feb 2018 22:27:03 +0000 (23:27 +0100)
Fixes: 5894/clusterfuzz-testcase-minimized-5315325420634112
Fixes: runtime error: signed integer overflow: 2147483647 + 1 cannot be represented in type 'int'

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
libavcodec/dirac_dwt_template.c

index e68cc4d..8c25c1f 100644 (file)
@@ -95,8 +95,8 @@ static void RENAME(horizontal_compose_dd97i)(uint8_t *_b, uint8_t *_tmp, int w)
     tmp[w2+1] = tmp[w2] = tmp[w2-1];
 
     for (x = 0; x < w2; x++) {
-        b[2*x  ] = (tmp[x] + 1)>>1;
-        b[2*x+1] = (COMPOSE_DD97iH0(tmp[x-1], tmp[x], b[x+w2], tmp[x+1], tmp[x+2]) + 1)>>1;
+        b[2*x  ] = ((int)(tmp[x] + 1U))>>1;
+        b[2*x+1] = ((int)(COMPOSE_DD97iH0(tmp[x-1], tmp[x], b[x+w2], tmp[x+1], tmp[x+2]) + 1U))>>1;
     }
 }
 
@@ -118,8 +118,8 @@ static void RENAME(horizontal_compose_dd137i)(uint8_t *_b, uint8_t *_tmp, int w)
     tmp[w2+1] = tmp[w2] = tmp[w2-1];
 
     for (x = 0; x < w2; x++) {
-        b[2*x  ] = (tmp[x] + 1)>>1;
-        b[2*x+1] = (COMPOSE_DD97iH0(tmp[x-1], tmp[x], b[x+w2], tmp[x+1], tmp[x+2]) + 1)>>1;
+        b[2*x  ] = ((int)(tmp[x] + 1U))>>1;
+        b[2*x+1] = ((int)(COMPOSE_DD97iH0(tmp[x-1], tmp[x], b[x+w2], tmp[x+1], tmp[x+2]) + 1U))>>1;
     }
 }