avoid integer overflow in dnxhd encoder, fixes #1557
authorBaptiste Coudurier <baptiste.coudurier@gmail.com>
Fri, 20 Nov 2009 00:13:34 +0000 (00:13 +0000)
committerBaptiste Coudurier <baptiste.coudurier@gmail.com>
Fri, 20 Nov 2009 00:13:34 +0000 (00:13 +0000)
Originally committed as revision 20557 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/dnxhdenc.c

index db0f09d..a52fe86 100644 (file)
@@ -574,9 +574,11 @@ static int dnxhd_encode_rdo(AVCodecContext *avctx, DNXHDEncContext *ctx)
             last_higher = FFMAX(lambda, last_higher);
             if (last_lower != INT_MAX)
                 lambda = (lambda+last_lower)>>1;
             last_higher = FFMAX(lambda, last_higher);
             if (last_lower != INT_MAX)
                 lambda = (lambda+last_lower)>>1;
+            else if ((int64_t)lambda + up_step > INT_MAX)
+                return -1;
             else
                 lambda += up_step;
             else
                 lambda += up_step;
-            up_step *= 5;
+            up_step = FFMIN((int64_t)up_step*5, INT_MAX);
             down_step = 1<<LAMBDA_FRAC_BITS;
         }
     }
             down_step = 1<<LAMBDA_FRAC_BITS;
         }
     }