swr-test: fix rounding error leading to NAN
authorMichael Niedermayer <michaelni@gmx.at>
Thu, 3 May 2012 12:42:37 +0000 (14:42 +0200)
committerMichael Niedermayer <michaelni@gmx.at>
Thu, 3 May 2012 12:42:37 +0000 (14:42 +0200)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
libswresample/swresample_test.c

index 35eede2..7c6e466 100644 (file)
@@ -357,6 +357,7 @@ int main(int argc, char **argv){
                 maxdiff= FFMAX(maxdiff, FFABS(a-b));
             }
             sse= sum_aa + sum_bb - 2*sum_ab;
+            if(sse < 0 && sse > -0.00001) sse=0; //fix rounding error
 
             fprintf(stderr, "[e:%f c:%f max:%f] len:%5d\n", sqrt(sse/out_count), sum_ab/(sqrt(sum_aa*sum_bb)), maxdiff, out_count);
         }
@@ -386,6 +387,7 @@ int main(int argc, char **argv){
                     maxdiff= FFMAX(maxdiff, FFABS(a-b));
                 }
                 sse= sum_aa + sum_bb - 2*sum_ab;
+                if(sse < 0 && sse > -0.00001) sse=0; //fix rounding error
 
                 fprintf(stderr, "[e:%f c:%f max:%f] len:%5d F:%3d\n", sqrt(sse/flush_count), sum_ab/(sqrt(sum_aa*sum_bb)), maxdiff, flush_count, flush_i);
             }