avcodec/motion_est: Fix some undefined shifts
authorMichael Niedermayer <michaelni@gmx.at>
Wed, 1 Jul 2015 13:41:29 +0000 (15:41 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Wed, 1 Jul 2015 13:42:07 +0000 (15:42 +0200)
Fixes: asan_heap-oob_1dd60fd_1049_cov_4200102444_P4250048.MOV

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/motion_est.c

index 9e184df..477a682 100644 (file)
@@ -426,13 +426,13 @@ static int sad_hpel_motion_search(MpegEncContext * s,
         my > ymin && my < ymax) {
         int dx=0, dy=0;
         int d, pen_x, pen_y;
-        const int index= (my<<ME_MAP_SHIFT) + mx;
+        const int index= my*(1<<ME_MAP_SHIFT) + mx;
         const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
         const int l= score_map[(index- 1               )&(ME_MAP_SIZE-1)];
         const int r= score_map[(index+ 1               )&(ME_MAP_SIZE-1)];
         const int b= score_map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
-        mx<<=1;
-        my<<=1;
+        mx += mx;
+        my += my;
 
 
         pen_x= pred_x + mx;
@@ -490,8 +490,8 @@ static int sad_hpel_motion_search(MpegEncContext * s,
         my+=dy;
 
     }else{
-        mx<<=1;
-        my<<=1;
+        mx += mx;
+        my += my;
     }
 
     *mx_ptr = mx;