loco: take decode overflow into account.
authorNicolas George <nicolas.george@normalesup.org>
Sat, 28 Jul 2012 22:12:27 +0000 (00:12 +0200)
committerNicolas George <nicolas.george@normalesup.org>
Tue, 31 Jul 2012 15:25:36 +0000 (17:25 +0200)
Commit 2bf0982 introduced an overflow check in loco_decode_plane,
but the error code is never taken into account, leading to
completely idiotic return values.

libavcodec/loco.c

index 9f14757..cc8ce21 100644 (file)
@@ -179,51 +179,56 @@ static int decode_frame(AVCodecContext *avctx,
     }
     p->key_frame = 1;
 
+#define ADVANCE_BY_DECODED do { \
+    if (decoded < 0) goto stop; \
+    buf += decoded; buf_size -= decoded; \
+} while(0)
     switch(l->mode) {
     case LOCO_CYUY2: case LOCO_YUY2: case LOCO_UYVY:
         decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
                                     p->linesize[0], buf, buf_size, 1);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height,
                                     p->linesize[1], buf, buf_size, 1);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height,
                                     p->linesize[2], buf, buf_size, 1);
         break;
     case LOCO_CYV12: case LOCO_YV12:
         decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
                                     p->linesize[0], buf, buf_size, 1);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height / 2,
                                     p->linesize[2], buf, buf_size, 1);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height / 2,
                                     p->linesize[1], buf, buf_size, 1);
         break;
     case LOCO_CRGB: case LOCO_RGB:
         decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1), avctx->width, avctx->height,
                                     -p->linesize[0], buf, buf_size, 3);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 1, avctx->width, avctx->height,
                                     -p->linesize[0], buf, buf_size, 3);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 2, avctx->width, avctx->height,
                                     -p->linesize[0], buf, buf_size, 3);
         break;
     case LOCO_CRGBA: case LOCO_RGBA:
         decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height,
                                     p->linesize[0], buf, buf_size, 4);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[0] + 1, avctx->width, avctx->height,
                                     p->linesize[0], buf, buf_size, 4);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[0] + 2, avctx->width, avctx->height,
                                     p->linesize[0], buf, buf_size, 4);
-        buf += decoded; buf_size -= decoded;
+        ADVANCE_BY_DECODED;
         decoded = loco_decode_plane(l, p->data[0] + 3, avctx->width, avctx->height,
                                     p->linesize[0], buf, buf_size, 4);
         break;
     }
+stop:
 
     *data_size = sizeof(AVFrame);
     *(AVFrame*)data = l->pic;