error_resilience: guess_dc: check malloc failure
[ffmpeg.git] / libavcodec / h264_mp4toannexb_bsf.c
index df49b34..2dea933 100644 (file)
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include <string.h>
+
 #include "libavutil/intreadwrite.h"
+#include "libavutil/mem.h"
 #include "avcodec.h"
 
 typedef struct H264BSFContext {
@@ -59,10 +62,12 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
                                    const uint8_t *buf, int      buf_size,
                                    int keyframe) {
     H264BSFContext *ctx = bsfc->priv_data;
+    int i;
     uint8_t unit_type;
     int32_t nal_size;
     uint32_t cumul_size = 0;
     const uint8_t *buf_end = buf + buf_size;
+    int ret = AVERROR(EINVAL);
 
     /* nothing to filter */
     if (!avctx->extradata || avctx->extradata_size < 6) {
@@ -81,17 +86,11 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
 
         /* retrieve length coded size */
         ctx->length_size = (*extradata++ & 0x3) + 1;
-        if (ctx->length_size == 3)
-            return AVERROR(EINVAL);
 
         /* retrieve sps and pps unit(s) */
         unit_nb = *extradata++ & 0x1f; /* number of sps unit(s) */
         if (!unit_nb) {
-            unit_nb = *extradata++; /* number of pps unit(s) */
-            sps_done++;
-
-            if (unit_nb)
-                pps_seen = 1;
+            goto pps;
         } else {
             sps_seen = 1;
         }
@@ -115,9 +114,12 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
             memcpy(out+total_size-unit_size-4, nalu_header, 4);
             memcpy(out+total_size-unit_size,   extradata+2, unit_size);
             extradata += 2+unit_size;
-
-            if (!unit_nb && !sps_done++)
+pps:
+            if (!unit_nb && !sps_done++) {
                 unit_nb = *extradata++; /* number of pps unit(s) */
+                if (unit_nb)
+                    pps_seen = 1;
+            }
         }
 
         if(out)
@@ -138,15 +140,12 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
     *poutbuf_size = 0;
     *poutbuf = NULL;
     do {
+        ret= AVERROR(EINVAL);
         if (buf + ctx->length_size > buf_end)
             goto fail;
 
-        if (ctx->length_size == 1) {
-            nal_size = buf[0];
-        } else if (ctx->length_size == 2) {
-            nal_size = AV_RB16(buf);
-        } else
-            nal_size = AV_RB32(buf);
+        for (nal_size = 0, i = 0; i<ctx->length_size; i++)
+            nal_size = (nal_size << 8) | buf[i];
 
         buf += ctx->length_size;
         unit_type = *buf & 0x1f;
@@ -156,15 +155,15 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
 
         /* prepend only to the first type 5 NAL unit of an IDR picture */
         if (ctx->first_idr && unit_type == 5) {
-            if (alloc_and_copy(poutbuf, poutbuf_size,
+            if ((ret=alloc_and_copy(poutbuf, poutbuf_size,
                                avctx->extradata, avctx->extradata_size,
-                               buf, nal_size) < 0)
+                               buf, nal_size)) < 0)
                 goto fail;
             ctx->first_idr = 0;
         } else {
-            if (alloc_and_copy(poutbuf, poutbuf_size,
+            if ((ret=alloc_and_copy(poutbuf, poutbuf_size,
                                NULL, 0,
-                               buf, nal_size) < 0)
+                               buf, nal_size)) < 0)
                 goto fail;
             if (!ctx->first_idr && unit_type == 1)
                 ctx->first_idr = 1;
@@ -179,7 +178,7 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
 fail:
     av_freep(poutbuf);
     *poutbuf_size = 0;
-    return AVERROR(EINVAL);
+    return ret;
 }
 
 AVBitStreamFilter ff_h264_mp4toannexb_bsf = {
@@ -187,4 +186,3 @@ AVBitStreamFilter ff_h264_mp4toannexb_bsf = {
     sizeof(H264BSFContext),
     h264_mp4toannexb_filter,
 };
-