avfilter/vf_edgedetect: fix undefined behaviour
authorPaul B Mahol <onemda@gmail.com>
Wed, 16 Oct 2019 16:06:47 +0000 (18:06 +0200)
committerPaul B Mahol <onemda@gmail.com>
Wed, 16 Oct 2019 16:28:32 +0000 (18:28 +0200)
Fixes #8287

libavfilter/vf_edgedetect.c

index 25424db..a5614ea 100644 (file)
@@ -216,7 +216,7 @@ static int get_rounded_direction(int gx, int gy)
 
         if (gx < 0)
             gx = -gx, gy = -gy;
-        gy <<= 16;
+        gy *= (1 << 16);
         tanpi8gx  =  27146 * gx;
         tan3pi8gx = 158218 * gx;
         if (gy > -tan3pi8gx && gy < -tanpi8gx)  return DIRECTION_45UP;