Merge commit '4912b634b517c8acfc476c5d47f10be83fe7e18b'
[ffmpeg.git] / libavcodec / jpeg2000.c
index 14f36c4..ede1a79 100644 (file)
@@ -25,6 +25,7 @@
  * JPEG 2000 image encoder and decoder common functions
  */
 
+#include "libavutil/attributes.h"
 #include "libavutil/avassert.h"
 #include "libavutil/common.h"
 #include "libavutil/mem.h"
@@ -153,7 +154,7 @@ static int getsgnctxno(int flag, uint8_t *xorbit)
     return ctxlbltab[hcontrib][vcontrib];
 }
 
-void ff_jpeg2000_init_tier1_luts(void)
+void av_cold ff_jpeg2000_init_tier1_luts(void)
 {
     int i, j;
     for (i = 0; i < 256; i++)
@@ -200,6 +201,11 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
     int reslevelno, bandno, gbandno = 0, ret, i, j;
     uint32_t csize;
 
+    if (codsty->nreslevels2decode <= 0) {
+        av_log(avctx, AV_LOG_ERROR, "nreslevels2decode %d invalid or uninitialized\n", codsty->nreslevels2decode);
+        return AVERROR_INVALIDDATA;
+    }
+
     if (ret = ff_jpeg2000_dwt_init(&comp->dwt, comp->coord,
                                    codsty->nreslevels2decode - 1,
                                    codsty->transform))
@@ -210,16 +216,16 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
 
     if (codsty->transform == FF_DWT97) {
         comp->i_data = NULL;
-        comp->f_data = av_malloc_array(csize, sizeof(*comp->f_data));
+        comp->f_data = av_mallocz_array(csize, sizeof(*comp->f_data));
         if (!comp->f_data)
             return AVERROR(ENOMEM);
     } else {
         comp->f_data = NULL;
-        comp->i_data = av_malloc_array(csize, sizeof(*comp->i_data));
+        comp->i_data = av_mallocz_array(csize, sizeof(*comp->i_data));
         if (!comp->i_data)
             return AVERROR(ENOMEM);
     }
-    comp->reslevel = av_malloc_array(codsty->nreslevels, sizeof(*comp->reslevel));
+    comp->reslevel = av_mallocz_array(codsty->nreslevels, sizeof(*comp->reslevel));
     if (!comp->reslevel)
         return AVERROR(ENOMEM);
     /* LOOP on resolution levels */
@@ -244,7 +250,7 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
         else
             reslevel->nbands = 3;
 
-        /* Number of precincts wich span the tile for resolution level reslevelno
+        /* Number of precincts which span the tile for resolution level reslevelno
          * see B.6 in ISO/IEC 15444-1:2002 eq. B-16
          * num_precincts_x = |- trx_1 / 2 ^ log2_prec_width) -| - (trx_0 / 2 ^ log2_prec_width)
          * num_precincts_y = |- try_1 / 2 ^ log2_prec_width) -| - (try_0 / 2 ^ log2_prec_width)
@@ -267,7 +273,7 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                                         reslevel->log2_prec_height) -
                 (reslevel->coord[1][0] >> reslevel->log2_prec_height);
 
-        reslevel->band = av_malloc_array(reslevel->nbands, sizeof(*reslevel->band));
+        reslevel->band = av_mallocz_array(reslevel->nbands, sizeof(*reslevel->band));
         if (!reslevel->band)
             return AVERROR(ENOMEM);
 
@@ -290,7 +296,7 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                 numbps = cbps +
                          lut_gain[codsty->transform == FF_DWT53][bandno + (reslevelno > 0)];
                 band->f_stepsize = SHL(2048 + qntsty->mant[gbandno],
-                                            2 + numbps - qntsty->expn[gbandno]);
+                                       2 + numbps - qntsty->expn[gbandno]);
                 break;
             case JPEG2000_QSTY_SE:
                 /* Exponent quantization step.
@@ -311,11 +317,11 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                 break;
             }
             /* FIXME: In openjepg code stespize = stepsize * 0.5. Why?
-                * If not set output of entropic decoder is not correct. */
+             * If not set output of entropic decoder is not correct. */
             if (!av_codec_is_encoder(avctx->codec))
                 band->f_stepsize *= 0.5;
 
-            band->i_stepsize = band->f_stepsize * (1 << 16);
+            band->i_stepsize = band->f_stepsize * (1 << 15);
 
             /* computation of tbx_0, tbx_1, tby_0, tby_1
              * see ISO/IEC 15444-1:2002 B.5 eq. B-15 and tbl B.1
@@ -363,9 +369,9 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
             for (j = 0; j < 2; j++)
                 band->coord[1][j] = ff_jpeg2000_ceildiv(band->coord[1][j], dy);
 
-            band->prec = av_malloc_array(reslevel->num_precincts_x *
-                                         (uint64_t)reslevel->num_precincts_y,
-                                         sizeof(*band->prec));
+            band->prec = av_mallocz_array(reslevel->num_precincts_x *
+                                          (uint64_t)reslevel->num_precincts_y,
+                                          sizeof(*band->prec));
             if (!band->prec)
                 return AVERROR(ENOMEM);
 
@@ -419,9 +425,9 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                 if (!prec->zerobits)
                     return AVERROR(ENOMEM);
 
-                prec->cblk = av_malloc_array(prec->nb_codeblocks_width *
-                                             (uint64_t)prec->nb_codeblocks_height,
-                                             sizeof(*prec->cblk));
+                prec->cblk = av_mallocz_array(prec->nb_codeblocks_width *
+                                              (uint64_t)prec->nb_codeblocks_height,
+                                              sizeof(*prec->cblk));
                 if (!prec->cblk)
                     return AVERROR(ENOMEM);
                 for (cblkno = 0; cblkno < prec->nb_codeblocks_width * prec->nb_codeblocks_height; cblkno++) {
@@ -446,14 +452,18 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
                     /* Compute Cy1 */
                     cblk->coord[1][1] = FFMIN(Cy0 + (1 << band->log2_cblk_height),
                                               prec->coord[1][1]);
-
-                    if((bandno + !!reslevelno) & 1) {
-                        cblk->coord[0][0] += comp->reslevel[reslevelno-1].coord[0][1] - comp->reslevel[reslevelno-1].coord[0][0];
-                        cblk->coord[0][1] += comp->reslevel[reslevelno-1].coord[0][1] - comp->reslevel[reslevelno-1].coord[0][0];
+                    /* Update code-blocks coordinates according sub-band position */
+                    if ((bandno + !!reslevelno) & 1) {
+                        cblk->coord[0][0] += comp->reslevel[reslevelno-1].coord[0][1] -
+                                             comp->reslevel[reslevelno-1].coord[0][0];
+                        cblk->coord[0][1] += comp->reslevel[reslevelno-1].coord[0][1] -
+                                             comp->reslevel[reslevelno-1].coord[0][0];
                     }
-                    if((bandno + !!reslevelno) & 2) {
-                        cblk->coord[1][0] += comp->reslevel[reslevelno-1].coord[1][1] - comp->reslevel[reslevelno-1].coord[1][0];
-                        cblk->coord[1][1] += comp->reslevel[reslevelno-1].coord[1][1] - comp->reslevel[reslevelno-1].coord[1][0];
+                    if ((bandno + !!reslevelno) & 2) {
+                        cblk->coord[1][0] += comp->reslevel[reslevelno-1].coord[1][1] -
+                                             comp->reslevel[reslevelno-1].coord[1][0];
+                        cblk->coord[1][1] += comp->reslevel[reslevelno-1].coord[1][1] -
+                                             comp->reslevel[reslevelno-1].coord[1][0];
                     }
 
                     cblk->zero      = 0;
@@ -492,16 +502,29 @@ void ff_jpeg2000_reinit(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty)
 void ff_jpeg2000_cleanup(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty)
 {
     int reslevelno, bandno, precno;
-    for (reslevelno = 0; comp->reslevel && reslevelno < codsty->nreslevels; reslevelno++) {
-        Jpeg2000ResLevel *reslevel = comp->reslevel + reslevelno;
+    for (reslevelno = 0;
+         comp->reslevel && reslevelno < codsty->nreslevels;
+         reslevelno++) {
+        Jpeg2000ResLevel *reslevel;
+
+        if (!comp->reslevel)
+            continue;
 
+        reslevel = comp->reslevel + reslevelno;
         for (bandno = 0; bandno < reslevel->nbands; bandno++) {
-            Jpeg2000Band *band = reslevel->band + bandno;
+            Jpeg2000Band *band;
+
+            if (!reslevel->band)
+                continue;
+
+            band = reslevel->band + bandno;
             for (precno = 0; precno < reslevel->num_precincts_x * reslevel->num_precincts_y; precno++) {
-                Jpeg2000Prec *prec = band->prec + precno;
-                av_freep(&prec->zerobits);
-                av_freep(&prec->cblkincl);
-                av_freep(&prec->cblk);
+                if (band->prec) {
+                    Jpeg2000Prec *prec = band->prec + precno;
+                    av_freep(&prec->zerobits);
+                    av_freep(&prec->cblkincl);
+                    av_freep(&prec->cblk);
+                }
             }
 
             av_freep(&band->prec);