avcodec/hevcpred_template: Fix integer overflows
authorMichael Niedermayer <michaelni@gmx.at>
Sun, 27 Oct 2013 10:50:24 +0000 (11:50 +0100)
committerMichael Niedermayer <michaelni@gmx.at>
Sun, 27 Oct 2013 10:54:53 +0000 (11:54 +0100)
signed integer overflow is undefined in C

Fixes the following gcc warnings:
In file included from libavcodec/hevcpred.c:27:0:
libavcodec/hevcpred_template.c: In function ‘intra_pred_8’:
libavcodec/hevcpred_template.c:302:9: warning: assuming signed overflow does not occur when assuming that (X - c) > X is always false [-Wstrict-overflow]
In file included from libavcodec/hevcpred.c:31:0:
libavcodec/hevcpred_template.c: In function ‘intra_pred_9’:
libavcodec/hevcpred_template.c:302:9: warning: assuming signed overflow does not occur when assuming that (X - c) > X is always false [-Wstrict-overflow]
In file included from libavcodec/hevcpred.c:35:0:
libavcodec/hevcpred_template.c: In function ‘intra_pred_10’:
libavcodec/hevcpred_template.c:302:9: warning: assuming signed overflow does not occur when assuming that (X - c) > X is always false [-Wstrict-overflow]

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libavcodec/hevcpred_template.c

index 9697506..f055bb6 100644 (file)
@@ -299,8 +299,8 @@ static void FUNC(intra_pred)(HEVCContext *s, int x0, int y0, int log2_size, int
     // Filtering process
     if (c_idx == 0 && mode != INTRA_DC && size != 4) {
         int intra_hor_ver_dist_thresh[] = { 7, 1, 0 };
-        int min_dist_vert_hor           = FFMIN(FFABS((int)mode - 26),
-                                                FFABS((int)mode - 10));
+        int min_dist_vert_hor           = FFMIN(FFABS((int)(mode - 26U)),
+                                                FFABS((int)(mode - 10U)));
         if (min_dist_vert_hor > intra_hor_ver_dist_thresh[log2_size - 3]) {
             int threshold = 1 << (BIT_DEPTH - 5);
             if (s->sps->sps_strong_intra_smoothing_enable_flag &&