mpeg4dec: Ensure data is not clobbered too early.
[ffmpeg.git] / libavcodec / h263dec.c
index 9714632..52ab779 100644 (file)
@@ -3,20 +3,20 @@
  * Copyright (c) 2001 Fabrice Bellard
  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
  *
- * This file is part of Libav.
+ * This file is part of FFmpeg.
  *
- * Libav is free software; you can redistribute it and/or
+ * FFmpeg is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
- * Libav is distributed in the hope that it will be useful,
+ * FFmpeg is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
- * License along with Libav; if not, write to the Free Software
+ * License along with FFmpeg; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
  * H.263 decoder.
  */
 
+#define UNCHECKED_BITSTREAM_READER 1
+
 #include "libavutil/cpu.h"
 #include "internal.h"
 #include "avcodec.h"
+#include "error_resilience.h"
 #include "mpegvideo.h"
 #include "h263.h"
 #include "h263_parser.h"
 #include "flv.h"
 #include "mpeg4video.h"
 
-//#define DEBUG
-//#define PRINT_FRAME_TIME
-
 av_cold int ff_h263_decode_init(AVCodecContext *avctx)
 {
     MpegEncContext *s = avctx->priv_data;
+    int ret;
 
     s->avctx = avctx;
     s->out_format = FMT_H263;
@@ -66,6 +67,7 @@ av_cold int ff_h263_decode_init(AVCodecContext *avctx)
     /* select sub codec */
     switch(avctx->codec->id) {
     case AV_CODEC_ID_H263:
+    case AV_CODEC_ID_H263P:
         s->unrestricted_mv= 0;
         avctx->chroma_sample_location = AVCHROMA_LOC_CENTER;
         break;
@@ -106,15 +108,18 @@ av_cold int ff_h263_decode_init(AVCodecContext *avctx)
         s->h263_flv = 1;
         break;
     default:
-        return -1;
+        return AVERROR(EINVAL);
     }
     s->codec_id= avctx->codec->id;
     avctx->hwaccel= ff_find_hwaccel(avctx->codec->id, avctx->pix_fmt);
 
+    if (avctx->stream_codec_tag == AV_RL32("l263") && avctx->extradata_size == 56 && avctx->extradata[0] == 1)
+        s->ehc_mode = 1;
+
     /* for h263, we allocate the images after having read the header */
-    if (avctx->codec->id != AV_CODEC_ID_H263 && avctx->codec->id != AV_CODEC_ID_MPEG4)
-        if (ff_MPV_common_init(s) < 0)
-            return -1;
+    if (avctx->codec->id != AV_CODEC_ID_H263 && avctx->codec->id != AV_CODEC_ID_H263P && avctx->codec->id != AV_CODEC_ID_MPEG4)
+        if ((ret = ff_MPV_common_init(s)) < 0)
+            return ret;
 
         ff_h263_decode_init_vlc();
 
@@ -152,7 +157,9 @@ static int get_consumed_bytes(MpegEncContext *s, int buf_size){
 
 static int decode_slice(MpegEncContext *s){
     const int part_mask= s->partitioned_frame ? (ER_AC_END|ER_AC_ERROR) : 0x7F;
-    const int mb_size = 16;
+    const int mb_size= 16>>s->avctx->lowres;
+    int ret;
+
     s->last_resync_gb= s->gb;
     s->first_slice_line= 1;
 
@@ -163,7 +170,7 @@ static int decode_slice(MpegEncContext *s){
 
     if (s->avctx->hwaccel) {
         const uint8_t *start= s->gb.buffer + get_bits_count(&s->gb)/8;
-        const uint8_t *end  = ff_h263_find_resync_marker(start + 1, s->gb.buffer_end);
+        const uint8_t *end  = ff_h263_find_resync_marker(s, start + 1, s->gb.buffer_end);
         skip_bits_long(&s->gb, 8*(end - start));
         return s->avctx->hwaccel->decode_slice(s->avctx, start, end - start);
     }
@@ -172,8 +179,8 @@ static int decode_slice(MpegEncContext *s){
         const int qscale= s->qscale;
 
         if(CONFIG_MPEG4_DECODER && s->codec_id==AV_CODEC_ID_MPEG4){
-            if(ff_mpeg4_decode_partitions(s) < 0)
-                return -1;
+            if ((ret = ff_mpeg4_decode_partitions(s)) < 0)
+                return ret;
         }
 
         /* restore variables which were modified */
@@ -242,12 +249,12 @@ static int decode_slice(MpegEncContext *s){
                 }else if(ret==SLICE_NOEND){
                     av_log(s->avctx, AV_LOG_ERROR, "Slice mismatch at MB: %d\n", xy);
                     ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y, s->mb_x+1, s->mb_y, ER_MB_END&part_mask);
-                    return -1;
+                    return AVERROR_INVALIDDATA;
                 }
                 av_log(s->avctx, AV_LOG_ERROR, "Error at MB: %d\n", xy);
                 ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y, s->mb_x, s->mb_y, ER_MB_ERROR&part_mask);
 
-                return -1;
+                return AVERROR_INVALIDDATA;
             }
 
             ff_MPV_decode_mb(s, s->block);
@@ -261,7 +268,7 @@ static int decode_slice(MpegEncContext *s){
         s->mb_x= 0;
     }
 
-    assert(s->mb_x==0 && s->mb_y==s->mb_height);
+    av_assert1(s->mb_x==0 && s->mb_y==s->mb_height);
 
     if(s->codec_id==AV_CODEC_ID_MPEG4
        && (s->workaround_bugs&FF_BUG_AUTODETECT)
@@ -274,7 +281,7 @@ static int decode_slice(MpegEncContext *s){
     if(      s->codec_id==AV_CODEC_ID_MPEG4
        &&   (s->workaround_bugs&FF_BUG_AUTODETECT)
        &&    get_bits_left(&s->gb) >=0
-       &&    get_bits_left(&s->gb) < 48
+       &&    get_bits_left(&s->gb) < 137
 //       &&   !s->resync_marker
        &&   !s->data_partitioning){
 
@@ -313,7 +320,7 @@ static int decode_slice(MpegEncContext *s){
             max_extra+= 17;
 
         /* buggy padding but the frame should still end approximately at the bitstream end */
-        if((s->workaround_bugs&FF_BUG_NO_PADDING) && (s->err_recognition&AV_EF_BUFFER))
+        if((s->workaround_bugs&FF_BUG_NO_PADDING) && (s->err_recognition&(AV_EF_BUFFER|AV_EF_AGGRESSIVE)))
             max_extra+= 48;
         else if((s->workaround_bugs&FF_BUG_NO_PADDING))
             max_extra+= 256*256*256*64;
@@ -335,7 +342,7 @@ static int decode_slice(MpegEncContext *s){
 
     ff_er_add_slice(&s->er, s->resync_mb_x, s->resync_mb_y, s->mb_x, s->mb_y, ER_MB_END&part_mask);
 
-    return -1;
+    return AVERROR_INVALIDDATA;
 }
 
 int ff_h263_decode_frame(AVCodecContext *avctx,
@@ -348,9 +355,6 @@ int ff_h263_decode_frame(AVCodecContext *avctx,
     int ret;
     AVFrame *pict = data;
 
-#ifdef PRINT_FRAME_TIME
-uint64_t time= rdtsc();
-#endif
     s->flags= avctx->flags;
     s->flags2= avctx->flags2;
 
@@ -358,7 +362,8 @@ uint64_t time= rdtsc();
     if (buf_size == 0) {
         /* special case for last picture */
         if (s->low_delay==0 && s->next_picture_ptr) {
-            *pict = s->next_picture_ptr->f;
+            if ((ret = av_frame_ref(pict, &s->next_picture_ptr->f)) < 0)
+                return ret;
             s->next_picture_ptr= NULL;
 
             *got_frame = 1;
@@ -374,9 +379,11 @@ uint64_t time= rdtsc();
             next= ff_mpeg4_find_frame_end(&s->parse_context, buf, buf_size);
         }else if(CONFIG_H263_DECODER && s->codec_id==AV_CODEC_ID_H263){
             next= ff_h263_find_frame_end(&s->parse_context, buf, buf_size);
+        }else if(CONFIG_H263P_DECODER && s->codec_id==AV_CODEC_ID_H263P){
+            next= ff_h263_find_frame_end(&s->parse_context, buf, buf_size);
         }else{
             av_log(s->avctx, AV_LOG_ERROR, "this codec does not support truncated bitstreams\n");
-            return -1;
+            return AVERROR(EINVAL);
         }
 
         if( ff_combine_frame(&s->parse_context, next, (const uint8_t **)&buf, &buf_size) < 0 )
@@ -385,6 +392,18 @@ uint64_t time= rdtsc();
 
 
 retry:
+    if(s->divx_packed && s->bitstream_buffer_size){
+        int i;
+        for(i=0; i<buf_size-3; i++){
+            if(buf[i]==0 && buf[i+1]==0 && buf[i+2]==1){
+                if(buf[i+3]==0xB0){
+                    av_log(s->avctx, AV_LOG_WARNING, "Discarding excessive bitstream in packed xvid\n");
+                    s->bitstream_buffer_size=0;
+                }
+                break;
+            }
+        }
+    }
 
     if(s->bitstream_buffer_size && (s->divx_packed || buf_size<20)){ //divx 5.01+/xvid frame reorder
         init_get_bits(&s->gb, s->bitstream_buffer, s->bitstream_buffer_size*8);
@@ -393,8 +412,8 @@ retry:
     s->bitstream_buffer_size=0;
 
     if (!s->context_initialized) {
-        if (ff_MPV_common_init(s) < 0) //we need the idct permutaton for reading a custom matrix
-            return -1;
+        if ((ret = ff_MPV_common_init(s)) < 0) //we need the idct permutaton for reading a custom matrix
+            return ret;
     }
 
     /* We need to set current_picture_ptr before reading the header,
@@ -427,12 +446,20 @@ retry:
         ret = ff_h263_decode_picture_header(s);
     }
 
+    if (ret < 0 || ret==FRAME_SKIPPED) {
+        if (   s->width  != avctx->coded_width
+            || s->height != avctx->coded_height) {
+                av_log(s->avctx, AV_LOG_WARNING, "Reverting picture dimensions change due to header decoding failure\n");
+                s->width = avctx->coded_width;
+                s->height= avctx->coded_height;
+        }
+    }
     if(ret==FRAME_SKIPPED) return get_consumed_bytes(s, buf_size);
 
     /* skip if the header was thrashed */
     if (ret < 0){
         av_log(s->avctx, AV_LOG_ERROR, "header damaged\n");
-        return -1;
+        return ret;
     }
 
     avctx->has_b_frames= !s->low_delay;
@@ -559,7 +586,7 @@ retry:
     if (s->codec_id == AV_CODEC_ID_MPEG4 && s->xvid_build>=0 && avctx->idct_algo == FF_IDCT_AUTO && (av_get_cpu_flags() & AV_CPU_FLAG_MMX)) {
         avctx->idct_algo= FF_IDCT_XVIDMMX;
         ff_dct_common_init(s);
-        s->picture_number=0;
+        goto retry;
     }
 #endif
 
@@ -568,7 +595,7 @@ retry:
         /* FIXME: By the way H263 decoder is evolving it should have */
         /* an H263EncContext                                         */
 
-    if (!avctx->coded_width || !avctx->coded_height) {
+    if ((!avctx->coded_width || !avctx->coded_height) && 0) {
         ParseContext pc= s->parse_context; //FIXME move these demuxng hack to avformat
 
         s->parse_context.buffer=0;
@@ -622,10 +649,11 @@ retry:
         s->me.qpel_avg= s->dsp.avg_qpel_pixels_tab;
     }
 
-    if(ff_MPV_frame_start(s, avctx) < 0)
-        return -1;
+    if ((ret = ff_MPV_frame_start(s, avctx)) < 0)
+        return ret;
 
-    if (!s->divx_packed) ff_thread_finish_setup(avctx);
+    if (!s->divx_packed && !avctx->hwaccel)
+        ff_thread_finish_setup(avctx);
 
     if (CONFIG_MPEG4_VDPAU_DECODER && (s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU)) {
         ff_vdpau_mpeg4_decode_picture(s, s->gb.buffer, s->gb.buffer_end - s->gb.buffer);
@@ -633,8 +661,8 @@ retry:
     }
 
     if (avctx->hwaccel) {
-        if (avctx->hwaccel->start_frame(avctx, s->gb.buffer, s->gb.buffer_end - s->gb.buffer) < 0)
-            return -1;
+        if ((ret = avctx->hwaccel->start_frame(avctx, s->gb.buffer, s->gb.buffer_end - s->gb.buffer)) < 0)
+            return ret;
     }
 
     ff_mpeg_er_frame_start(s);
@@ -644,7 +672,7 @@ retry:
     if (CONFIG_WMV2_DECODER && s->msmpeg4_version==5){
         ret = ff_wmv2_decode_secondary_picture_header(s);
         if(ret<0) return ret;
-        if(ret==1) goto intrax8_decoded;
+        if(ret==1) goto frame_end;
     }
 
     /* decode each macroblock */
@@ -675,26 +703,33 @@ retry:
             s->er.error_status_table[s->mb_num - 1] = ER_MB_ERROR;
         }
 
-    assert(s->bitstream_buffer_size==0);
+    av_assert1(s->bitstream_buffer_size==0);
 frame_end:
-    /* divx 5.01+ bistream reorder stuff */
+    ff_er_frame_end(&s->er);
+
+    if (avctx->hwaccel) {
+        if ((ret = avctx->hwaccel->end_frame(avctx)) < 0)
+            return ret;
+    }
+
+    ff_MPV_frame_end(s);
+
+    /* divx 5.01+ bitstream reorder stuff */
+    /* Since this clobbers the input buffer and hwaccel codecs still need the
+     * data during hwaccel->end_frame we should not do this any earlier */
     if(s->codec_id==AV_CODEC_ID_MPEG4 && s->divx_packed){
-        int current_pos= get_bits_count(&s->gb)>>3;
+        int current_pos= s->gb.buffer == s->bitstream_buffer ? 0 : (get_bits_count(&s->gb)>>3);
         int startcode_found=0;
 
-        if(buf_size - current_pos > 5){
+        if(buf_size - current_pos > 7){
             int i;
-            for(i=current_pos; i<buf_size-3; i++){
+            for(i=current_pos; i<buf_size-4; i++){
                 if(buf[i]==0 && buf[i+1]==0 && buf[i+2]==1 && buf[i+3]==0xB6){
-                    startcode_found=1;
+                    startcode_found=!(buf[i+4]&0x40);
                     break;
                 }
             }
         }
-        if(s->gb.buffer == s->bitstream_buffer && buf_size>7 && s->xvid_build>=0){ //xvid style
-            startcode_found=1;
-            current_pos=0;
-        }
 
         if(startcode_found){
             av_fast_malloc(
@@ -708,33 +743,42 @@ frame_end:
         }
     }
 
-intrax8_decoded:
-    ff_er_frame_end(&s->er);
+    if (!s->divx_packed && avctx->hwaccel)
+        ff_thread_finish_setup(avctx);
 
-    if (avctx->hwaccel) {
-        if (avctx->hwaccel->end_frame(avctx) < 0)
-            return -1;
-    }
-
-    ff_MPV_frame_end(s);
-
-    assert(s->current_picture.f.pict_type == s->current_picture_ptr->f.pict_type);
-    assert(s->current_picture.f.pict_type == s->pict_type);
+    av_assert1(s->current_picture.f.pict_type == s->current_picture_ptr->f.pict_type);
+    av_assert1(s->current_picture.f.pict_type == s->pict_type);
     if (s->pict_type == AV_PICTURE_TYPE_B || s->low_delay) {
-        *pict = s->current_picture_ptr->f;
+        if ((ret = av_frame_ref(pict, &s->current_picture_ptr->f)) < 0)
+            return ret;
+        ff_print_debug_info(s, s->current_picture_ptr, pict);
+        ff_mpv_export_qp_table(s, pict, s->current_picture_ptr, FF_QSCALE_TYPE_MPEG1);
     } else if (s->last_picture_ptr != NULL) {
-        *pict = s->last_picture_ptr->f;
+        if ((ret = av_frame_ref(pict, &s->last_picture_ptr->f)) < 0)
+            return ret;
+        ff_print_debug_info(s, s->last_picture_ptr, pict);
+        ff_mpv_export_qp_table(s, pict, s->last_picture_ptr, FF_QSCALE_TYPE_MPEG1);
     }
 
     if(s->last_picture_ptr || s->low_delay){
+        if (   pict->format == AV_PIX_FMT_YUV420P
+            && (s->codec_tag == AV_RL32("GEOV") || s->codec_tag == AV_RL32("GEOX"))) {
+            int x, y, p;
+            av_frame_make_writable(pict);
+            for (p=0; p<3; p++) {
+                int w = FF_CEIL_RSHIFT(pict-> width, !!p);
+                int h = FF_CEIL_RSHIFT(pict->height, !!p);
+                int linesize = pict->linesize[p];
+                for (y=0; y<(h>>1); y++)
+                    for (x=0; x<w; x++)
+                        FFSWAP(int,
+                               pict->data[p][x + y*linesize],
+                               pict->data[p][x + (h-1-y)*linesize]);
+            }
+        }
         *got_frame = 1;
-        ff_print_debug_info(s, pict);
     }
 
-#ifdef PRINT_FRAME_TIME
-av_log(avctx, AV_LOG_DEBUG, "%"PRId64"\n", rdtsc()-time);
-#endif
-
     return (ret && (avctx->err_recognition & AV_EF_EXPLODE))?ret:get_consumed_bytes(s, buf_size);
 }
 
@@ -760,6 +804,23 @@ AVCodec ff_h263_decoder = {
     .capabilities   = CODEC_CAP_DRAW_HORIZ_BAND | CODEC_CAP_DR1 |
                       CODEC_CAP_TRUNCATED | CODEC_CAP_DELAY,
     .flush          = ff_mpeg_flush,
+    .max_lowres     = 3,
+    .long_name      = NULL_IF_CONFIG_SMALL("H.263 / H.263-1996, H.263+ / H.263-1998 / H.263 version 2"),
+    .pix_fmts       = ff_h263_hwaccel_pixfmt_list_420,
+};
+
+AVCodec ff_h263p_decoder = {
+    .name           = "h263p",
+    .type           = AVMEDIA_TYPE_VIDEO,
+    .id             = AV_CODEC_ID_H263P,
+    .priv_data_size = sizeof(MpegEncContext),
+    .init           = ff_h263_decode_init,
+    .close          = ff_h263_decode_end,
+    .decode         = ff_h263_decode_frame,
+    .capabilities   = CODEC_CAP_DRAW_HORIZ_BAND | CODEC_CAP_DR1 |
+                      CODEC_CAP_TRUNCATED | CODEC_CAP_DELAY,
+    .flush          = ff_mpeg_flush,
+    .max_lowres     = 3,
     .long_name      = NULL_IF_CONFIG_SMALL("H.263 / H.263-1996, H.263+ / H.263-1998 / H.263 version 2"),
     .pix_fmts       = ff_h263_hwaccel_pixfmt_list_420,
 };