xan: Add some buffer checks
authorAlex Converse <alex.converse@gmail.com>
Fri, 9 Sep 2011 23:10:03 +0000 (16:10 -0700)
committerMichael Niedermayer <michaelni@gmx.at>
Wed, 21 Sep 2011 17:47:12 +0000 (19:47 +0200)
(cherry picked from commit 0872bb23b4bd2d94a8ba91070f706d1bc1c3ced8)

libavcodec/xan.c

index 1dace41..357593b 100644 (file)
@@ -108,6 +108,9 @@ static int xan_huffman_decode(unsigned char *dest, int dest_len,
     unsigned char *dest_end = dest + dest_len;
     GetBitContext gb;
 
+    if (ptr_len < 0)
+        return AVERROR_INVALIDDATA;
+
     init_get_bits(&gb, ptr, ptr_len * 8);
 
     while ( val != 0x16 ) {
@@ -247,7 +250,7 @@ static inline void xan_wc3_copy_pixel_run(XanContext *s,
     }
 }
 
-static void xan_wc3_decode_frame(XanContext *s) {
+static int xan_wc3_decode_frame(XanContext *s) {
 
     int width = s->avctx->width;
     int height = s->avctx->height;
@@ -267,14 +270,30 @@ static void xan_wc3_decode_frame(XanContext *s) {
     const unsigned char *size_segment;
     const unsigned char *vector_segment;
     const unsigned char *imagedata_segment;
+    int huffman_offset, size_offset, vector_offset, imagedata_offset;
+
+    if (s->size < 8)
+        return AVERROR_INVALIDDATA;
+
+    huffman_offset    = AV_RL16(&s->buf[0]);
+    size_offset       = AV_RL16(&s->buf[2]);
+    vector_offset     = AV_RL16(&s->buf[4]);
+    imagedata_offset  = AV_RL16(&s->buf[6]);
 
-    huffman_segment =   s->buf + AV_RL16(&s->buf[0]);
-    size_segment =      s->buf + AV_RL16(&s->buf[2]);
-    vector_segment =    s->buf + AV_RL16(&s->buf[4]);
-    imagedata_segment = s->buf + AV_RL16(&s->buf[6]);
+    if (huffman_offset   >= s->size ||
+        size_offset      >= s->size ||
+        vector_offset    >= s->size ||
+        imagedata_offset >= s->size)
+        return AVERROR_INVALIDDATA;
 
-    xan_huffman_decode(opcode_buffer, opcode_buffer_size,
-                       huffman_segment, s->size - (huffman_segment - s->buf) );
+    huffman_segment   = s->buf + huffman_offset;
+    size_segment      = s->buf + size_offset;
+    vector_segment    = s->buf + vector_offset;
+    imagedata_segment = s->buf + imagedata_offset;
+
+    if (xan_huffman_decode(opcode_buffer, opcode_buffer_size,
+                           huffman_segment, s->size - huffman_offset) < 0)
+        return AVERROR_INVALIDDATA;
 
     if (imagedata_segment[0] == 2)
         xan_unpack(s->buffer2, &imagedata_segment[1], s->buffer2_size);
@@ -360,6 +379,7 @@ static void xan_wc3_decode_frame(XanContext *s) {
         y += (x + size) / width;
         x  = (x + size) % width;
     }
+    return 0;
 }
 
 #if RUNTIME_GAMMA
@@ -521,7 +541,8 @@ static int xan_decode_frame(AVCodecContext *avctx,
     s->buf = buf;
     s->size = buf_size;
 
-    xan_wc3_decode_frame(s);
+    if (xan_wc3_decode_frame(s) < 0)
+        return AVERROR_INVALIDDATA;
 
     /* release the last frame if it is allocated */
     if (s->last_frame.data[0])