avfilter/vf_transpose: Fix rounding error
authorMichael Niedermayer <michaelni@gmx.at>
Tue, 30 Jun 2015 14:01:15 +0000 (16:01 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Tue, 30 Jun 2015 14:02:09 +0000 (16:02 +0200)
Fixes out of array access
Fixes: asan_heap-oob_7f875d_3482_cov_1818465256_ssudec.mov

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavfilter/vf_transpose.c

index 911329c..e4de31b 100644 (file)
@@ -151,7 +151,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr,
         int hsub    = plane == 1 || plane == 2 ? trans->hsub : 0;
         int vsub    = plane == 1 || plane == 2 ? trans->vsub : 0;
         int pixstep = trans->pixsteps[plane];
-        int inh     = in->height  >> vsub;
+        int inh     = FF_CEIL_RSHIFT(in->height, vsub);
         int outw    = FF_CEIL_RSHIFT(out->width,  hsub);
         int outh    = FF_CEIL_RSHIFT(out->height, vsub);
         int start   = (outh *  jobnr   ) / nb_jobs;