Fix an integer overflow in the AAC encoder.
[ffmpeg.git] / libavcodec / aaccoder.c
index 3e90880..97a5912 100644 (file)
@@ -61,7 +61,8 @@ static const uint8_t *run_value_bits[2] = {
  */
 static av_always_inline int quant(float coef, const float Q)
 {
-    return pow(coef * Q, 0.75) + 0.4054;
+    float a = coef * Q;
+    return sqrtf(a * sqrtf(a)) + 0.4054;
 }
 
 static void quantize_bands(int (*out)[2], const float *in, const float *scaled,
@@ -71,8 +72,8 @@ static void quantize_bands(int (*out)[2], const float *in, const float *scaled,
     double qc;
     for (i = 0; i < size; i++) {
         qc = scaled[i] * Q34;
-        out[i][0] = (int)FFMIN((int)qc,            maxval);
-        out[i][1] = (int)FFMIN((int)(qc + 0.4054), maxval);
+        out[i][0] = (int)FFMIN(qc,          (double)maxval);
+        out[i][1] = (int)FFMIN(qc + 0.4054, (double)maxval);
         if (is_signed && in[i] < 0.0f) {
             out[i][0] = -out[i][0];
             out[i][1] = -out[i][1];
@@ -84,16 +85,13 @@ static void abs_pow34_v(float *out, const float *in, const int size)
 {
 #ifndef USE_REALLY_FULL_SEARCH
     int i;
-    for (i = 0; i < size; i++)
-        out[i] = pow(fabsf(in[i]), 0.75);
+    for (i = 0; i < size; i++) {
+        float a = fabsf(in[i]);
+        out[i] = sqrtf(a * sqrtf(a));
+    }
 #endif /* USE_REALLY_FULL_SEARCH */
 }
 
-static av_always_inline int quant2(float coef, const float Q)
-{
-    return pow(coef * Q, 0.75);
-}
-
 static const uint8_t aac_cb_range [12] = {0, 3, 3, 3, 3, 9, 9, 8, 8, 13, 13, 17};
 static const uint8_t aac_cb_maxval[12] = {0, 1, 1, 2, 2, 4, 4, 7, 7, 12, 12, 16};
 
@@ -115,7 +113,7 @@ static float quantize_band_cost(struct AACEncContext *s, const float *in,
     const int dim = cb < FIRST_PAIR_BT ? 4 : 2;
     int resbits = 0;
 #ifndef USE_REALLY_FULL_SEARCH
-    const float  Q34 = pow(Q, 0.75);
+    const float  Q34 = sqrtf(Q * sqrtf(Q));
     const int range  = aac_cb_range[cb];
     const int maxval = aac_cb_maxval[cb];
     int offs[4];
@@ -124,6 +122,8 @@ static float quantize_band_cost(struct AACEncContext *s, const float *in,
     if (!cb) {
         for (i = 0; i < size; i++)
             cost += in[i]*in[i]*lambda;
+        if (bits)
+            *bits = 0;
         return cost;
     }
 #ifndef USE_REALLY_FULL_SEARCH
@@ -228,7 +228,7 @@ static void quantize_and_encode_band(struct AACEncContext *s, PutBitContext *pb,
     const int dim = (cb < FIRST_PAIR_BT) ? 4 : 2;
     int i, j, k;
 #ifndef USE_REALLY_FULL_SEARCH
-    const float  Q34 = pow(Q, 0.75);
+    const float  Q34 = sqrtf(Q * sqrtf(Q));
     const int range  = aac_cb_range[cb];
     const int maxval = aac_cb_maxval[cb];
     int offs[4];
@@ -457,15 +457,6 @@ static void encode_window_bands_info(AACEncContext *s, SingleChannelElement *sce
     }
 }
 
-static void encode_window_bands_info_fixed(AACEncContext *s,
-                                           SingleChannelElement *sce,
-                                           int win, int group_len,
-                                           const float lambda)
-{
-    encode_window_bands_info(s, sce, win, group_len, 1.0f);
-}
-
-
 typedef struct TrellisPath {
     float cost;
     int prev;
@@ -520,8 +511,8 @@ static void search_for_quantizers_anmr(AVCodecContext *avctx, AACEncContext *s,
                 for (i = 0; i < sce->ics.swb_sizes[g]; i++) {
                     float t = fabsf(coefs[w2*128+i]);
                     if (t > 0.0f)
-                        qmin = fminf(qmin, t);
-                    qmax = fmaxf(qmax, t);
+                        qmin = FFMIN(qmin, t);
+                    qmax = FFMAX(qmax, t);
                 }
             }
             if (nz) {
@@ -543,8 +534,8 @@ static void search_for_quantizers_anmr(AVCodecContext *avctx, AACEncContext *s,
                     }
                     dist = dists[0];
                     for (i = 1; i <= ESC_BT; i++)
-                        dist = fminf(dist, dists[i]);
-                    minrd = fminf(minrd, dist);
+                        dist = FFMIN(dist, dists[i]);
+                    minrd = FFMIN(minrd, dist);
 
                     for (i = FFMAX(q - SCALE_MAX_DIFF, 0); i < FFMIN(q + SCALE_MAX_DIFF, 256); i++) {
                         float cost;
@@ -649,7 +640,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
             uplims[w*16+g] = uplim *512;
             sce->zeroes[w*16+g] = !nz;
             if (nz)
-                minthr = fminf(minthr, uplim);
+                minthr = FFMIN(minthr, uplim);
             allz = FFMAX(allz, nz);
         }
     }
@@ -659,7 +650,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
                 sce->sf_idx[w*16+g] = SCALE_ONE_POS;
                 continue;
             }
-            sce->sf_idx[w*16+g] = SCALE_ONE_POS + fminf(log2(uplims[w*16+g]/minthr)*4,59);
+            sce->sf_idx[w*16+g] = SCALE_ONE_POS + FFMIN(log2(uplims[w*16+g]/minthr)*4,59);
         }
     }
 
@@ -700,7 +691,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
                                                        sce->ics.swb_sizes[g],
                                                        sce->sf_idx[w*16+g],
                                                        ESC_BT,
-                                                       1.0,
+                                                       lambda,
                                                        INFINITY,
                                                        &b);
                             bb += b;
@@ -710,7 +701,7 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
                             minbits = bb;
                         }
                     }
-                    dists[w*16+g] = mindist - minbits;
+                    dists[w*16+g] = (mindist - minbits) / lambda;
                     bits = minbits;
                     if (prev != -1) {
                         bits += ff_aac_scalefactor_bits[sce->sf_idx[w*16+g] - prev + SCALE_DIFF_ZERO];
@@ -819,7 +810,7 @@ static void search_for_quantizers_faac(AVCodecContext *avctx, AACEncContext *s,
             for (w2 = 0; w2 < sce->ics.group_len[w]; w2++) {
                 for (i = 0; i < size; i++) {
                     float t = coefs[w2*128+i]*coefs[w2*128+i];
-                    maxq[w*16+g] = fmaxf(maxq[w*16+g], fabsf(coefs[w2*128 + i]));
+                    maxq[w*16+g] = FFMAX(maxq[w*16+g], fabsf(coefs[w2*128 + i]));
                     thr += t;
                     if (sce->ics.num_windows == 1 && maxval < t) {
                         maxval  = t;
@@ -870,12 +861,12 @@ static void search_for_quantizers_faac(AVCodecContext *avctx, AACEncContext *s,
                                                sce->ics.swb_sizes[g],
                                                scf,
                                                ESC_BT,
-                                               1.0,
+                                               lambda,
                                                INFINITY,
                                                &b);
                     dist -= b;
                 }
-                dist *= 1.0f/512.0f;
+                dist *= 1.0f / 512.0f / lambda;
                 quant_max = quant(maxq[w*16+g], ff_aac_pow2sf_tab[200 - scf + SCALE_ONE_POS - SCALE_DIV_512]);
                 if (quant_max >= 8191) { // too much, return to the previous quantizer
                     sce->sf_idx[w*16+g] = prev_scf;
@@ -972,8 +963,8 @@ static void search_for_ms(AACEncContext *s, ChannelElement *cpe,
                 for (w2 = 0; w2 < sce0->ics.group_len[w]; w2++) {
                     FFPsyBand *band0 = &s->psy.psy_bands[(s->cur_channel+0)*PSY_MAX_BANDS+(w+w2)*16+g];
                     FFPsyBand *band1 = &s->psy.psy_bands[(s->cur_channel+1)*PSY_MAX_BANDS+(w+w2)*16+g];
-                    float minthr = fminf(band0->threshold, band1->threshold);
-                    float maxthr = fmaxf(band0->threshold, band1->threshold);
+                    float minthr = FFMIN(band0->threshold, band1->threshold);
+                    float maxthr = FFMAX(band0->threshold, band1->threshold);
                     for (i = 0; i < sce0->ics.swb_sizes[g]; i++) {
                         M[i] = (sce0->coeffs[start+w2*128+i]
                               + sce1->coeffs[start+w2*128+i]) * 0.5;
@@ -1019,7 +1010,7 @@ static void search_for_ms(AACEncContext *s, ChannelElement *cpe,
 AACCoefficientsEncoder ff_aac_coders[] = {
     {
         search_for_quantizers_faac,
-        encode_window_bands_info_fixed,
+        encode_window_bands_info,
         quantize_and_encode_band,
 //        search_for_ms,
     },