fate: add more tests for hue video filter
[ffmpeg.git] / libavcodec / vmnc.c
index 49abb77..30b1414 100644 (file)
@@ -287,17 +287,24 @@ static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb,
                     return AVERROR_INVALIDDATA;
                 }
                 for (k = 0; k < rects; k++) {
+                    int rect_x, rect_y, rect_w, rect_h;
                     if (color)
                         fg = vmnc_get_pixel(gb, bpp, c->bigendian);
                     xy = bytestream2_get_byte(gb);
                     wh = bytestream2_get_byte(gb);
-                    if (   (xy >> 4) + (wh >> 4) + 1 > w - i
-                        || (xy & 0xF) + (wh & 0xF)+1 > h - j) {
+
+                    rect_x = xy >> 4;
+                    rect_y = xy & 0xF;
+                    rect_w = (wh >> 4) + 1;
+                    rect_h = (wh & 0xF) + 1;
+
+                    if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) {
                         av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n");
                         return AVERROR_INVALIDDATA;
                     }
-                    paint_rect(dst2, xy >> 4, xy & 0xF,
-                               (wh>>4)+1, (wh & 0xF)+1, fg, bpp, stride);
+
+                    paint_rect(dst2, rect_x, rect_y,
+                               rect_w, rect_h, fg, bpp, stride);
                 }
             }
         }
@@ -374,6 +381,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
         w   = bytestream2_get_be16(gb);
         h   = bytestream2_get_be16(gb);
         enc = bytestream2_get_be32(gb);
+        if ((dx + w > c->width) || (dy + h > c->height)) {
+            av_log(avctx, AV_LOG_ERROR,
+                    "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
+                    w, h, dx, dy, c->width, c->height);
+            return AVERROR_INVALIDDATA;
+        }
         outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
         size_left = bytestream2_get_bytes_left(gb);
         switch (enc) {
@@ -451,12 +464,6 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
             bytestream2_skip(gb, 2);
             break;
         case 0x00000000: // raw rectangle data
-            if ((dx + w > c->width) || (dy + h > c->height)) {
-                av_log(avctx, AV_LOG_ERROR,
-                       "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
-                       w, h, dx, dy, c->width, c->height);
-                return AVERROR_INVALIDDATA;
-            }
             if (size_left < w * h * c->bpp2) {
                 av_log(avctx, AV_LOG_ERROR,
                        "Premature end of data! (need %i got %i)\n",
@@ -467,12 +474,6 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
                       c->pic->linesize[0]);
             break;
         case 0x00000005: // HexTile encoded rectangle
-            if ((dx + w > c->width) || (dy + h > c->height)) {
-                av_log(avctx, AV_LOG_ERROR,
-                       "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
-                       w, h, dx, dy, c->width, c->height);
-                return AVERROR_INVALIDDATA;
-            }
             res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]);
             if (res < 0)
                 return res;