error_resilience: guess_dc: check malloc failure
[ffmpeg.git] / libavcodec / cinepak.c
1 /*
2  * Cinepak Video Decoder
3  * Copyright (C) 2003 the ffmpeg project
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * @file
24  * Cinepak video decoder
25  * @author Ewald Snel <ewald@rambo.its.tudelft.nl>
26  *
27  * @see For more information on the Cinepak algorithm, visit:
28  *   http://www.csse.monash.edu.au/~timf/
29  * @see For more information on the quirky data inside Sega FILM/CPK files, visit:
30  *   http://wiki.multimedia.cx/index.php?title=Sega_FILM
31  */
32
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36
37 #include "libavutil/common.h"
38 #include "libavutil/intreadwrite.h"
39 #include "avcodec.h"
40
41
42 typedef struct {
43     uint8_t  y0, y1, y2, y3;
44     uint8_t  u, v;
45 } cvid_codebook;
46
47 #define MAX_STRIPS      32
48
49 typedef struct {
50     uint16_t          id;
51     uint16_t          x1, y1;
52     uint16_t          x2, y2;
53     cvid_codebook     v4_codebook[256];
54     cvid_codebook     v1_codebook[256];
55 } cvid_strip;
56
57 typedef struct CinepakContext {
58
59     AVCodecContext *avctx;
60     AVFrame frame;
61
62     const unsigned char *data;
63     int size;
64
65     int width, height;
66
67     int palette_video;
68     cvid_strip strips[MAX_STRIPS];
69
70     int sega_film_skip_bytes;
71
72     uint32_t pal[256];
73 } CinepakContext;
74
75 static void cinepak_decode_codebook (cvid_codebook *codebook,
76                                      int chunk_id, int size, const uint8_t *data)
77 {
78     const uint8_t *eod = (data + size);
79     uint32_t flag, mask;
80     int      i, n;
81
82     /* check if this chunk contains 4- or 6-element vectors */
83     n    = (chunk_id & 0x04) ? 4 : 6;
84     flag = 0;
85     mask = 0;
86
87     for (i=0; i < 256; i++) {
88         if ((chunk_id & 0x01) && !(mask >>= 1)) {
89             if ((data + 4) > eod)
90                 break;
91
92             flag  = AV_RB32 (data);
93             data += 4;
94             mask  = 0x80000000;
95         }
96
97         if (!(chunk_id & 0x01) || (flag & mask)) {
98             if ((data + n) > eod)
99                 break;
100
101             if (n == 6) {
102                 codebook[i].y0 = *data++;
103                 codebook[i].y1 = *data++;
104                 codebook[i].y2 = *data++;
105                 codebook[i].y3 = *data++;
106                 codebook[i].u  = 128 + *data++;
107                 codebook[i].v  = 128 + *data++;
108             } else {
109                 /* this codebook type indicates either greyscale or
110                  * palettized video; if palettized, U & V components will
111                  * not be used so it is safe to set them to 128 for the
112                  * benefit of greyscale rendering in YUV420P */
113                 codebook[i].y0 = *data++;
114                 codebook[i].y1 = *data++;
115                 codebook[i].y2 = *data++;
116                 codebook[i].y3 = *data++;
117                 codebook[i].u  = 128;
118                 codebook[i].v  = 128;
119             }
120         }
121     }
122 }
123
124 static int cinepak_decode_vectors (CinepakContext *s, cvid_strip *strip,
125                                    int chunk_id, int size, const uint8_t *data)
126 {
127     const uint8_t   *eod = (data + size);
128     uint32_t         flag, mask;
129     cvid_codebook   *codebook;
130     unsigned int     x, y;
131     uint32_t         iy[4];
132     uint32_t         iu[2];
133     uint32_t         iv[2];
134
135     flag = 0;
136     mask = 0;
137
138     for (y=strip->y1; y < strip->y2; y+=4) {
139
140         iy[0] = strip->x1 + (y * s->frame.linesize[0]);
141         iy[1] = iy[0] + s->frame.linesize[0];
142         iy[2] = iy[1] + s->frame.linesize[0];
143         iy[3] = iy[2] + s->frame.linesize[0];
144         iu[0] = (strip->x1/2) + ((y/2) * s->frame.linesize[1]);
145         iu[1] = iu[0] + s->frame.linesize[1];
146         iv[0] = (strip->x1/2) + ((y/2) * s->frame.linesize[2]);
147         iv[1] = iv[0] + s->frame.linesize[2];
148
149         for (x=strip->x1; x < strip->x2; x+=4) {
150             if ((chunk_id & 0x01) && !(mask >>= 1)) {
151                 if ((data + 4) > eod)
152                     return AVERROR_INVALIDDATA;
153
154                 flag  = AV_RB32 (data);
155                 data += 4;
156                 mask  = 0x80000000;
157             }
158
159             if (!(chunk_id & 0x01) || (flag & mask)) {
160                 if (!(chunk_id & 0x02) && !(mask >>= 1)) {
161                     if ((data + 4) > eod)
162                         return AVERROR_INVALIDDATA;
163
164                     flag  = AV_RB32 (data);
165                     data += 4;
166                     mask  = 0x80000000;
167                 }
168
169                 if ((chunk_id & 0x02) || (~flag & mask)) {
170                     if (data >= eod)
171                         return AVERROR_INVALIDDATA;
172
173                     codebook = &strip->v1_codebook[*data++];
174                     s->frame.data[0][iy[0] + 0] = codebook->y0;
175                     s->frame.data[0][iy[0] + 1] = codebook->y0;
176                     s->frame.data[0][iy[1] + 0] = codebook->y0;
177                     s->frame.data[0][iy[1] + 1] = codebook->y0;
178                     if (!s->palette_video) {
179                         s->frame.data[1][iu[0]] = codebook->u;
180                         s->frame.data[2][iv[0]] = codebook->v;
181                     }
182
183                     s->frame.data[0][iy[0] + 2] = codebook->y1;
184                     s->frame.data[0][iy[0] + 3] = codebook->y1;
185                     s->frame.data[0][iy[1] + 2] = codebook->y1;
186                     s->frame.data[0][iy[1] + 3] = codebook->y1;
187                     if (!s->palette_video) {
188                         s->frame.data[1][iu[0] + 1] = codebook->u;
189                         s->frame.data[2][iv[0] + 1] = codebook->v;
190                     }
191
192                     s->frame.data[0][iy[2] + 0] = codebook->y2;
193                     s->frame.data[0][iy[2] + 1] = codebook->y2;
194                     s->frame.data[0][iy[3] + 0] = codebook->y2;
195                     s->frame.data[0][iy[3] + 1] = codebook->y2;
196                     if (!s->palette_video) {
197                         s->frame.data[1][iu[1]] = codebook->u;
198                         s->frame.data[2][iv[1]] = codebook->v;
199                     }
200
201                     s->frame.data[0][iy[2] + 2] = codebook->y3;
202                     s->frame.data[0][iy[2] + 3] = codebook->y3;
203                     s->frame.data[0][iy[3] + 2] = codebook->y3;
204                     s->frame.data[0][iy[3] + 3] = codebook->y3;
205                     if (!s->palette_video) {
206                         s->frame.data[1][iu[1] + 1] = codebook->u;
207                         s->frame.data[2][iv[1] + 1] = codebook->v;
208                     }
209
210                 } else if (flag & mask) {
211                     if ((data + 4) > eod)
212                         return AVERROR_INVALIDDATA;
213
214                     codebook = &strip->v4_codebook[*data++];
215                     s->frame.data[0][iy[0] + 0] = codebook->y0;
216                     s->frame.data[0][iy[0] + 1] = codebook->y1;
217                     s->frame.data[0][iy[1] + 0] = codebook->y2;
218                     s->frame.data[0][iy[1] + 1] = codebook->y3;
219                     if (!s->palette_video) {
220                         s->frame.data[1][iu[0]] = codebook->u;
221                         s->frame.data[2][iv[0]] = codebook->v;
222                     }
223
224                     codebook = &strip->v4_codebook[*data++];
225                     s->frame.data[0][iy[0] + 2] = codebook->y0;
226                     s->frame.data[0][iy[0] + 3] = codebook->y1;
227                     s->frame.data[0][iy[1] + 2] = codebook->y2;
228                     s->frame.data[0][iy[1] + 3] = codebook->y3;
229                     if (!s->palette_video) {
230                         s->frame.data[1][iu[0] + 1] = codebook->u;
231                         s->frame.data[2][iv[0] + 1] = codebook->v;
232                     }
233
234                     codebook = &strip->v4_codebook[*data++];
235                     s->frame.data[0][iy[2] + 0] = codebook->y0;
236                     s->frame.data[0][iy[2] + 1] = codebook->y1;
237                     s->frame.data[0][iy[3] + 0] = codebook->y2;
238                     s->frame.data[0][iy[3] + 1] = codebook->y3;
239                     if (!s->palette_video) {
240                         s->frame.data[1][iu[1]] = codebook->u;
241                         s->frame.data[2][iv[1]] = codebook->v;
242                     }
243
244                     codebook = &strip->v4_codebook[*data++];
245                     s->frame.data[0][iy[2] + 2] = codebook->y0;
246                     s->frame.data[0][iy[2] + 3] = codebook->y1;
247                     s->frame.data[0][iy[3] + 2] = codebook->y2;
248                     s->frame.data[0][iy[3] + 3] = codebook->y3;
249                     if (!s->palette_video) {
250                         s->frame.data[1][iu[1] + 1] = codebook->u;
251                         s->frame.data[2][iv[1] + 1] = codebook->v;
252                     }
253
254                 }
255             }
256
257             iy[0] += 4;  iy[1] += 4;
258             iy[2] += 4;  iy[3] += 4;
259             iu[0] += 2;  iu[1] += 2;
260             iv[0] += 2;  iv[1] += 2;
261         }
262     }
263
264     return 0;
265 }
266
267 static int cinepak_decode_strip (CinepakContext *s,
268                                  cvid_strip *strip, const uint8_t *data, int size)
269 {
270     const uint8_t *eod = (data + size);
271     int      chunk_id, chunk_size;
272
273     /* coordinate sanity checks */
274     if (strip->x2 > s->width   ||
275         strip->y2 > s->height  ||
276         strip->x1 >= strip->x2 || strip->y1 >= strip->y2)
277         return AVERROR_INVALIDDATA;
278
279     while ((data + 4) <= eod) {
280         chunk_id   = data[0];
281         chunk_size = AV_RB24 (&data[1]) - 4;
282         if(chunk_size < 0)
283             return AVERROR_INVALIDDATA;
284
285         data      += 4;
286         chunk_size = ((data + chunk_size) > eod) ? (eod - data) : chunk_size;
287
288         switch (chunk_id) {
289
290         case 0x20:
291         case 0x21:
292         case 0x24:
293         case 0x25:
294             cinepak_decode_codebook (strip->v4_codebook, chunk_id,
295                 chunk_size, data);
296             break;
297
298         case 0x22:
299         case 0x23:
300         case 0x26:
301         case 0x27:
302             cinepak_decode_codebook (strip->v1_codebook, chunk_id,
303                 chunk_size, data);
304             break;
305
306         case 0x30:
307         case 0x31:
308         case 0x32:
309             return cinepak_decode_vectors (s, strip, chunk_id,
310                 chunk_size, data);
311         }
312
313         data += chunk_size;
314     }
315
316     return AVERROR_INVALIDDATA;
317 }
318
319 static int cinepak_decode (CinepakContext *s)
320 {
321     const uint8_t  *eod = (s->data + s->size);
322     int           i, result, strip_size, frame_flags, num_strips;
323     int           y0 = 0;
324     int           encoded_buf_size;
325
326     if (s->size < 10)
327         return AVERROR_INVALIDDATA;
328
329     frame_flags = s->data[0];
330     num_strips  = AV_RB16 (&s->data[8]);
331     encoded_buf_size = AV_RB24(&s->data[1]);
332
333     /* if this is the first frame, check for deviant Sega FILM data */
334     if (s->sega_film_skip_bytes == -1) {
335         if (!encoded_buf_size) {
336             av_log_ask_for_sample(s->avctx, "encoded_buf_size is 0");
337             return AVERROR_INVALIDDATA;
338         }
339         if (encoded_buf_size != s->size && (s->size % encoded_buf_size) != 0) {
340             /* If the encoded frame size differs from the frame size as indicated
341              * by the container file, this data likely comes from a Sega FILM/CPK file.
342              * If the frame header is followed by the bytes FE 00 00 06 00 00 then
343              * this is probably one of the two known files that have 6 extra bytes
344              * after the frame header. Else, assume 2 extra bytes. The container
345              * size also cannot be a multiple of the encoded size. */
346             if (s->size >= 16 &&
347                 (s->data[10] == 0xFE) &&
348                 (s->data[11] == 0x00) &&
349                 (s->data[12] == 0x00) &&
350                 (s->data[13] == 0x06) &&
351                 (s->data[14] == 0x00) &&
352                 (s->data[15] == 0x00))
353                 s->sega_film_skip_bytes = 6;
354             else
355                 s->sega_film_skip_bytes = 2;
356         } else
357             s->sega_film_skip_bytes = 0;
358     }
359
360     s->data += 10 + s->sega_film_skip_bytes;
361
362     num_strips = FFMIN(num_strips, MAX_STRIPS);
363
364     s->frame.key_frame = 0;
365
366     for (i=0; i < num_strips; i++) {
367         if ((s->data + 12) > eod)
368             return AVERROR_INVALIDDATA;
369
370         s->strips[i].id = s->data[0];
371         s->strips[i].y1 = y0;
372         s->strips[i].x1 = 0;
373         s->strips[i].y2 = y0 + AV_RB16 (&s->data[8]);
374         s->strips[i].x2 = s->avctx->width;
375
376         if (s->strips[i].id == 0x10)
377             s->frame.key_frame = 1;
378
379         strip_size = AV_RB24 (&s->data[1]) - 12;
380         if (strip_size < 0)
381             return AVERROR_INVALIDDATA;
382         s->data   += 12;
383         strip_size = ((s->data + strip_size) > eod) ? (eod - s->data) : strip_size;
384
385         if ((i > 0) && !(frame_flags & 0x01)) {
386             memcpy (s->strips[i].v4_codebook, s->strips[i-1].v4_codebook,
387                 sizeof(s->strips[i].v4_codebook));
388             memcpy (s->strips[i].v1_codebook, s->strips[i-1].v1_codebook,
389                 sizeof(s->strips[i].v1_codebook));
390         }
391
392         result = cinepak_decode_strip (s, &s->strips[i], s->data, strip_size);
393
394         if (result != 0)
395             return result;
396
397         s->data += strip_size;
398         y0    = s->strips[i].y2;
399     }
400     return 0;
401 }
402
403 static av_cold int cinepak_decode_init(AVCodecContext *avctx)
404 {
405     CinepakContext *s = avctx->priv_data;
406
407     s->avctx = avctx;
408     s->width = (avctx->width + 3) & ~3;
409     s->height = (avctx->height + 3) & ~3;
410     s->sega_film_skip_bytes = -1;  /* uninitialized state */
411
412     // check for paletted data
413     if (avctx->bits_per_coded_sample != 8) {
414         s->palette_video = 0;
415         avctx->pix_fmt = PIX_FMT_YUV420P;
416     } else {
417         s->palette_video = 1;
418         avctx->pix_fmt = PIX_FMT_PAL8;
419     }
420
421     avcodec_get_frame_defaults(&s->frame);
422     s->frame.data[0] = NULL;
423
424     return 0;
425 }
426
427 static int cinepak_decode_frame(AVCodecContext *avctx,
428                                 void *data, int *data_size,
429                                 AVPacket *avpkt)
430 {
431     const uint8_t *buf = avpkt->data;
432     int ret = 0, buf_size = avpkt->size;
433     CinepakContext *s = avctx->priv_data;
434
435     s->data = buf;
436     s->size = buf_size;
437
438     s->frame.reference = 3;
439     s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
440                             FF_BUFFER_HINTS_REUSABLE;
441     if ((ret = avctx->reget_buffer(avctx, &s->frame))) {
442         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
443         return ret;
444     }
445
446     if (s->palette_video) {
447         const uint8_t *pal = av_packet_get_side_data(avpkt, AV_PKT_DATA_PALETTE, NULL);
448         if (pal) {
449             s->frame.palette_has_changed = 1;
450             memcpy(s->pal, pal, AVPALETTE_SIZE);
451         }
452     }
453
454     cinepak_decode(s);
455
456     if (s->palette_video)
457         memcpy (s->frame.data[1], s->pal, AVPALETTE_SIZE);
458
459     *data_size = sizeof(AVFrame);
460     *(AVFrame*)data = s->frame;
461
462     /* report that the buffer was completely consumed */
463     return buf_size;
464 }
465
466 static av_cold int cinepak_decode_end(AVCodecContext *avctx)
467 {
468     CinepakContext *s = avctx->priv_data;
469
470     if (s->frame.data[0])
471         avctx->release_buffer(avctx, &s->frame);
472
473     return 0;
474 }
475
476 AVCodec ff_cinepak_decoder = {
477     .name           = "cinepak",
478     .type           = AVMEDIA_TYPE_VIDEO,
479     .id             = AV_CODEC_ID_CINEPAK,
480     .priv_data_size = sizeof(CinepakContext),
481     .init           = cinepak_decode_init,
482     .close          = cinepak_decode_end,
483     .decode         = cinepak_decode_frame,
484     .capabilities   = CODEC_CAP_DR1,
485     .long_name      = NULL_IF_CONFIG_SMALL("Cinepak"),
486 };