Safety check against malicious high bit-depth input which could cause crash
authorAnton Mitrofanov <BugMaster@narod.ru>
Sun, 12 Oct 2014 17:01:53 +0000 (21:01 +0400)
committerFiona <fiona@x264.com>
Fri, 17 Oct 2014 21:25:43 +0000 (14:25 -0700)
encoder/rdo.c

index 93fdc3a..ceb2e2c 100644 (file)
@@ -186,7 +186,7 @@ static int x264_rd_cost_mb( x264_t *h, int i_lambda2 )
     h->mb.b_transform_8x8 = b_transform_bak;
     h->mb.i_type = type_bak;
 
-    return i_ssd + i_bits;
+    return X264_MIN( i_ssd + i_bits, COST_MAX );
 }
 
 /* partition RD functions use 8 bits more precision to avoid large rounding errors at low QPs */