swscale: fix another integer overflow at large dimensions/rescales.
authorRonald S. Bultje <rsbultje@gmail.com>
Fri, 24 Feb 2012 23:32:25 +0000 (15:32 -0800)
committerRonald S. Bultje <rsbultje@gmail.com>
Fri, 24 Feb 2012 23:39:55 +0000 (15:39 -0800)
libswscale/utils.c

index 796adb6..3b488ca 100644 (file)
@@ -274,7 +274,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
             int j;
             (*filterPos)[i]= xx;
             for (j=0; j<filterSize; j++) {
-                int64_t d= ((int64_t)FFABS((xx<<17) - xDstInSrc))<<13;
+                int64_t d= (FFABS(((int64_t)xx<<17) - xDstInSrc))<<13;
                 double floatd;
                 int64_t coeff;