cinepack: return non-generic errors
[ffmpeg.git] / libavcodec / cinepak.c
1 /*
2  * Cinepak Video Decoder
3  * Copyright (C) 2003 the ffmpeg project
4  *
5  * This file is part of Libav.
6  *
7  * Libav is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * Libav is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with Libav; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * @file
24  * Cinepak video decoder
25  * by Ewald Snel <ewald@rambo.its.tudelft.nl>
26  * For more information on the Cinepak algorithm, visit:
27  *   http://www.csse.monash.edu.au/~timf/
28  * For more information on the quirky data inside Sega FILM/CPK files, visit:
29  *   http://wiki.multimedia.cx/index.php?title=Sega_FILM
30  */
31
32 #include <stdio.h>
33 #include <stdlib.h>
34 #include <string.h>
35
36 #include "libavutil/intreadwrite.h"
37 #include "avcodec.h"
38
39
40 typedef struct {
41     uint8_t  y0, y1, y2, y3;
42     uint8_t  u, v;
43 } cvid_codebook;
44
45 #define MAX_STRIPS      32
46
47 typedef struct {
48     uint16_t          id;
49     uint16_t          x1, y1;
50     uint16_t          x2, y2;
51     cvid_codebook     v4_codebook[256];
52     cvid_codebook     v1_codebook[256];
53 } cvid_strip;
54
55 typedef struct CinepakContext {
56
57     AVCodecContext *avctx;
58     AVFrame frame;
59
60     const unsigned char *data;
61     int size;
62
63     int width, height;
64
65     int palette_video;
66     cvid_strip strips[MAX_STRIPS];
67
68     int sega_film_skip_bytes;
69
70     uint32_t pal[256];
71 } CinepakContext;
72
73 static void cinepak_decode_codebook (cvid_codebook *codebook,
74                                      int chunk_id, int size, const uint8_t *data)
75 {
76     const uint8_t *eod = (data + size);
77     uint32_t flag, mask;
78     int      i, n;
79
80     /* check if this chunk contains 4- or 6-element vectors */
81     n    = (chunk_id & 0x04) ? 4 : 6;
82     flag = 0;
83     mask = 0;
84
85     for (i=0; i < 256; i++) {
86         if ((chunk_id & 0x01) && !(mask >>= 1)) {
87             if ((data + 4) > eod)
88                 break;
89
90             flag  = AV_RB32 (data);
91             data += 4;
92             mask  = 0x80000000;
93         }
94
95         if (!(chunk_id & 0x01) || (flag & mask)) {
96             if ((data + n) > eod)
97                 break;
98
99             if (n == 6) {
100                 codebook[i].y0 = *data++;
101                 codebook[i].y1 = *data++;
102                 codebook[i].y2 = *data++;
103                 codebook[i].y3 = *data++;
104                 codebook[i].u  = 128 + *data++;
105                 codebook[i].v  = 128 + *data++;
106             } else {
107                 /* this codebook type indicates either greyscale or
108                  * palettized video; if palettized, U & V components will
109                  * not be used so it is safe to set them to 128 for the
110                  * benefit of greyscale rendering in YUV420P */
111                 codebook[i].y0 = *data++;
112                 codebook[i].y1 = *data++;
113                 codebook[i].y2 = *data++;
114                 codebook[i].y3 = *data++;
115                 codebook[i].u  = 128;
116                 codebook[i].v  = 128;
117             }
118         }
119     }
120 }
121
122 static int cinepak_decode_vectors (CinepakContext *s, cvid_strip *strip,
123                                    int chunk_id, int size, const uint8_t *data)
124 {
125     const uint8_t   *eod = (data + size);
126     uint32_t         flag, mask;
127     cvid_codebook   *codebook;
128     unsigned int     x, y;
129     uint32_t         iy[4];
130     uint32_t         iu[2];
131     uint32_t         iv[2];
132
133     flag = 0;
134     mask = 0;
135
136     for (y=strip->y1; y < strip->y2; y+=4) {
137
138         iy[0] = strip->x1 + (y * s->frame.linesize[0]);
139         iy[1] = iy[0] + s->frame.linesize[0];
140         iy[2] = iy[1] + s->frame.linesize[0];
141         iy[3] = iy[2] + s->frame.linesize[0];
142         iu[0] = (strip->x1/2) + ((y/2) * s->frame.linesize[1]);
143         iu[1] = iu[0] + s->frame.linesize[1];
144         iv[0] = (strip->x1/2) + ((y/2) * s->frame.linesize[2]);
145         iv[1] = iv[0] + s->frame.linesize[2];
146
147         for (x=strip->x1; x < strip->x2; x+=4) {
148             if ((chunk_id & 0x01) && !(mask >>= 1)) {
149                 if ((data + 4) > eod)
150                     return AVERROR_INVALIDDATA;
151
152                 flag  = AV_RB32 (data);
153                 data += 4;
154                 mask  = 0x80000000;
155             }
156
157             if (!(chunk_id & 0x01) || (flag & mask)) {
158                 if (!(chunk_id & 0x02) && !(mask >>= 1)) {
159                     if ((data + 4) > eod)
160                         return AVERROR_INVALIDDATA;
161
162                     flag  = AV_RB32 (data);
163                     data += 4;
164                     mask  = 0x80000000;
165                 }
166
167                 if ((chunk_id & 0x02) || (~flag & mask)) {
168                     if (data >= eod)
169                         return AVERROR_INVALIDDATA;
170
171                     codebook = &strip->v1_codebook[*data++];
172                     s->frame.data[0][iy[0] + 0] = codebook->y0;
173                     s->frame.data[0][iy[0] + 1] = codebook->y0;
174                     s->frame.data[0][iy[1] + 0] = codebook->y0;
175                     s->frame.data[0][iy[1] + 1] = codebook->y0;
176                     if (!s->palette_video) {
177                         s->frame.data[1][iu[0]] = codebook->u;
178                         s->frame.data[2][iv[0]] = codebook->v;
179                     }
180
181                     s->frame.data[0][iy[0] + 2] = codebook->y1;
182                     s->frame.data[0][iy[0] + 3] = codebook->y1;
183                     s->frame.data[0][iy[1] + 2] = codebook->y1;
184                     s->frame.data[0][iy[1] + 3] = codebook->y1;
185                     if (!s->palette_video) {
186                         s->frame.data[1][iu[0] + 1] = codebook->u;
187                         s->frame.data[2][iv[0] + 1] = codebook->v;
188                     }
189
190                     s->frame.data[0][iy[2] + 0] = codebook->y2;
191                     s->frame.data[0][iy[2] + 1] = codebook->y2;
192                     s->frame.data[0][iy[3] + 0] = codebook->y2;
193                     s->frame.data[0][iy[3] + 1] = codebook->y2;
194                     if (!s->palette_video) {
195                         s->frame.data[1][iu[1]] = codebook->u;
196                         s->frame.data[2][iv[1]] = codebook->v;
197                     }
198
199                     s->frame.data[0][iy[2] + 2] = codebook->y3;
200                     s->frame.data[0][iy[2] + 3] = codebook->y3;
201                     s->frame.data[0][iy[3] + 2] = codebook->y3;
202                     s->frame.data[0][iy[3] + 3] = codebook->y3;
203                     if (!s->palette_video) {
204                         s->frame.data[1][iu[1] + 1] = codebook->u;
205                         s->frame.data[2][iv[1] + 1] = codebook->v;
206                     }
207
208                 } else if (flag & mask) {
209                     if ((data + 4) > eod)
210                         return AVERROR_INVALIDDATA;
211
212                     codebook = &strip->v4_codebook[*data++];
213                     s->frame.data[0][iy[0] + 0] = codebook->y0;
214                     s->frame.data[0][iy[0] + 1] = codebook->y1;
215                     s->frame.data[0][iy[1] + 0] = codebook->y2;
216                     s->frame.data[0][iy[1] + 1] = codebook->y3;
217                     if (!s->palette_video) {
218                         s->frame.data[1][iu[0]] = codebook->u;
219                         s->frame.data[2][iv[0]] = codebook->v;
220                     }
221
222                     codebook = &strip->v4_codebook[*data++];
223                     s->frame.data[0][iy[0] + 2] = codebook->y0;
224                     s->frame.data[0][iy[0] + 3] = codebook->y1;
225                     s->frame.data[0][iy[1] + 2] = codebook->y2;
226                     s->frame.data[0][iy[1] + 3] = codebook->y3;
227                     if (!s->palette_video) {
228                         s->frame.data[1][iu[0] + 1] = codebook->u;
229                         s->frame.data[2][iv[0] + 1] = codebook->v;
230                     }
231
232                     codebook = &strip->v4_codebook[*data++];
233                     s->frame.data[0][iy[2] + 0] = codebook->y0;
234                     s->frame.data[0][iy[2] + 1] = codebook->y1;
235                     s->frame.data[0][iy[3] + 0] = codebook->y2;
236                     s->frame.data[0][iy[3] + 1] = codebook->y3;
237                     if (!s->palette_video) {
238                         s->frame.data[1][iu[1]] = codebook->u;
239                         s->frame.data[2][iv[1]] = codebook->v;
240                     }
241
242                     codebook = &strip->v4_codebook[*data++];
243                     s->frame.data[0][iy[2] + 2] = codebook->y0;
244                     s->frame.data[0][iy[2] + 3] = codebook->y1;
245                     s->frame.data[0][iy[3] + 2] = codebook->y2;
246                     s->frame.data[0][iy[3] + 3] = codebook->y3;
247                     if (!s->palette_video) {
248                         s->frame.data[1][iu[1] + 1] = codebook->u;
249                         s->frame.data[2][iv[1] + 1] = codebook->v;
250                     }
251
252                 }
253             }
254
255             iy[0] += 4;  iy[1] += 4;
256             iy[2] += 4;  iy[3] += 4;
257             iu[0] += 2;  iu[1] += 2;
258             iv[0] += 2;  iv[1] += 2;
259         }
260     }
261
262     return 0;
263 }
264
265 static int cinepak_decode_strip (CinepakContext *s,
266                                  cvid_strip *strip, const uint8_t *data, int size)
267 {
268     const uint8_t *eod = (data + size);
269     int      chunk_id, chunk_size;
270
271     /* coordinate sanity checks */
272     if (strip->x2 > s->width   ||
273         strip->y2 > s->height  ||
274         strip->x1 >= strip->x2 || strip->y1 >= strip->y2)
275         return AVERROR_INVALIDDATA;
276
277     while ((data + 4) <= eod) {
278         chunk_id   = data[0];
279         chunk_size = AV_RB24 (&data[1]) - 4;
280         if(chunk_size < 0)
281             return AVERROR_INVALIDDATA;
282
283         data      += 4;
284         chunk_size = ((data + chunk_size) > eod) ? (eod - data) : chunk_size;
285
286         switch (chunk_id) {
287
288         case 0x20:
289         case 0x21:
290         case 0x24:
291         case 0x25:
292             cinepak_decode_codebook (strip->v4_codebook, chunk_id,
293                 chunk_size, data);
294             break;
295
296         case 0x22:
297         case 0x23:
298         case 0x26:
299         case 0x27:
300             cinepak_decode_codebook (strip->v1_codebook, chunk_id,
301                 chunk_size, data);
302             break;
303
304         case 0x30:
305         case 0x31:
306         case 0x32:
307             return cinepak_decode_vectors (s, strip, chunk_id,
308                 chunk_size, data);
309         }
310
311         data += chunk_size;
312     }
313
314     return AVERROR_INVALIDDATA;
315 }
316
317 static int cinepak_decode (CinepakContext *s)
318 {
319     const uint8_t  *eod = (s->data + s->size);
320     int           i, result, strip_size, frame_flags, num_strips;
321     int           y0 = 0;
322     int           encoded_buf_size;
323
324     if (s->size < 10)
325         return AVERROR_INVALIDDATA;
326
327     frame_flags = s->data[0];
328     num_strips  = AV_RB16 (&s->data[8]);
329     encoded_buf_size = AV_RB24(&s->data[1]);
330
331     /* if this is the first frame, check for deviant Sega FILM data */
332     if (s->sega_film_skip_bytes == -1) {
333         if (!encoded_buf_size) {
334             av_log_ask_for_sample(s->avctx, "encoded_buf_size is 0");
335             return AVERROR_INVALIDDATA;
336         }
337         if (encoded_buf_size != s->size && (s->size % encoded_buf_size) != 0) {
338             /* If the encoded frame size differs from the frame size as indicated
339              * by the container file, this data likely comes from a Sega FILM/CPK file.
340              * If the frame header is followed by the bytes FE 00 00 06 00 00 then
341              * this is probably one of the two known files that have 6 extra bytes
342              * after the frame header. Else, assume 2 extra bytes. The container
343              * size also cannot be a multiple of the encoded size. */
344             if (s->size >= 16 &&
345                 (s->data[10] == 0xFE) &&
346                 (s->data[11] == 0x00) &&
347                 (s->data[12] == 0x00) &&
348                 (s->data[13] == 0x06) &&
349                 (s->data[14] == 0x00) &&
350                 (s->data[15] == 0x00))
351                 s->sega_film_skip_bytes = 6;
352             else
353                 s->sega_film_skip_bytes = 2;
354         } else
355             s->sega_film_skip_bytes = 0;
356     }
357
358     s->data += 10 + s->sega_film_skip_bytes;
359
360     num_strips = FFMIN(num_strips, MAX_STRIPS);
361
362     for (i=0; i < num_strips; i++) {
363         if ((s->data + 12) > eod)
364             return AVERROR_INVALIDDATA;
365
366         s->strips[i].id = s->data[0];
367         s->strips[i].y1 = y0;
368         s->strips[i].x1 = 0;
369         s->strips[i].y2 = y0 + AV_RB16 (&s->data[8]);
370         s->strips[i].x2 = s->avctx->width;
371
372         strip_size = AV_RB24 (&s->data[1]) - 12;
373         if (strip_size < 0)
374             return AVERROR_INVALIDDATA;
375         s->data   += 12;
376         strip_size = ((s->data + strip_size) > eod) ? (eod - s->data) : strip_size;
377
378         if ((i > 0) && !(frame_flags & 0x01)) {
379             memcpy (s->strips[i].v4_codebook, s->strips[i-1].v4_codebook,
380                 sizeof(s->strips[i].v4_codebook));
381             memcpy (s->strips[i].v1_codebook, s->strips[i-1].v1_codebook,
382                 sizeof(s->strips[i].v1_codebook));
383         }
384
385         result = cinepak_decode_strip (s, &s->strips[i], s->data, strip_size);
386
387         if (result != 0)
388             return result;
389
390         s->data += strip_size;
391         y0    = s->strips[i].y2;
392     }
393     return 0;
394 }
395
396 static av_cold int cinepak_decode_init(AVCodecContext *avctx)
397 {
398     CinepakContext *s = avctx->priv_data;
399
400     s->avctx = avctx;
401     s->width = (avctx->width + 3) & ~3;
402     s->height = (avctx->height + 3) & ~3;
403     s->sega_film_skip_bytes = -1;  /* uninitialized state */
404
405     // check for paletted data
406     if (avctx->bits_per_coded_sample != 8) {
407         s->palette_video = 0;
408         avctx->pix_fmt = PIX_FMT_YUV420P;
409     } else {
410         s->palette_video = 1;
411         avctx->pix_fmt = PIX_FMT_PAL8;
412     }
413
414     s->frame.data[0] = NULL;
415
416     return 0;
417 }
418
419 static int cinepak_decode_frame(AVCodecContext *avctx,
420                                 void *data, int *data_size,
421                                 AVPacket *avpkt)
422 {
423     const uint8_t *buf = avpkt->data;
424     int ret = 0, buf_size = avpkt->size;
425     CinepakContext *s = avctx->priv_data;
426
427     s->data = buf;
428     s->size = buf_size;
429
430     s->frame.reference = 1;
431     s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
432                             FF_BUFFER_HINTS_REUSABLE;
433     if ((ret = avctx->reget_buffer(avctx, &s->frame))) {
434         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
435         return ret;
436     }
437
438     if (s->palette_video) {
439         const uint8_t *pal = av_packet_get_side_data(avpkt, AV_PKT_DATA_PALETTE, NULL);
440         if (pal) {
441             s->frame.palette_has_changed = 1;
442             memcpy(s->pal, pal, AVPALETTE_SIZE);
443         }
444     }
445
446     cinepak_decode(s);
447
448     if (s->palette_video)
449         memcpy (s->frame.data[1], s->pal, AVPALETTE_SIZE);
450
451     *data_size = sizeof(AVFrame);
452     *(AVFrame*)data = s->frame;
453
454     /* report that the buffer was completely consumed */
455     return buf_size;
456 }
457
458 static av_cold int cinepak_decode_end(AVCodecContext *avctx)
459 {
460     CinepakContext *s = avctx->priv_data;
461
462     if (s->frame.data[0])
463         avctx->release_buffer(avctx, &s->frame);
464
465     return 0;
466 }
467
468 AVCodec ff_cinepak_decoder = {
469     .name           = "cinepak",
470     .type           = AVMEDIA_TYPE_VIDEO,
471     .id             = CODEC_ID_CINEPAK,
472     .priv_data_size = sizeof(CinepakContext),
473     .init           = cinepak_decode_init,
474     .close          = cinepak_decode_end,
475     .decode         = cinepak_decode_frame,
476     .capabilities   = CODEC_CAP_DR1,
477     .long_name = NULL_IF_CONFIG_SMALL("Cinepak"),
478 };