correctly handle very large Cinepak frames (courtesy of John Koleszar
[ffmpeg.git] / libavcodec / cinepak.c
1 /*
2  * Cinepak Video Decoder
3  * Copyright (C) 2003 the ffmpeg project
4  *
5  * This library is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU Lesser General Public
7  * License as published by the Free Software Foundation; either
8  * version 2 of the License, or (at your option) any later version.
9  *
10  * This library is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * Lesser General Public License for more details.
14  *
15  * You should have received a copy of the GNU Lesser General Public
16  * License along with this library; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  *
19  */
20
21 /**
22  * @file cinepak.c
23  * Cinepak video decoder
24  * by Ewald Snel <ewald@rambo.its.tudelft.nl>
25  * For more information on the Cinepak algorithm, visit:
26  *   http://www.csse.monash.edu.au/~timf/
27  */
28
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <unistd.h>
33
34 #include "common.h"
35 #include "avcodec.h"
36 #include "dsputil.h"
37
38
39 typedef struct {
40     uint8_t  y0, y1, y2, y3;
41     uint8_t  u, v;
42 } cvid_codebook_t;
43
44 #define MAX_STRIPS      32
45
46 typedef struct {
47     uint16_t          id;
48     uint16_t          x1, y1;
49     uint16_t          x2, y2;
50     cvid_codebook_t   v4_codebook[256];
51     cvid_codebook_t   v1_codebook[256];
52 } cvid_strip_t;
53
54 typedef struct CinepakContext {
55
56     AVCodecContext *avctx;
57     DSPContext dsp;
58     AVFrame frame;
59
60     unsigned char *data;
61     int size;
62
63     int width, height;
64
65     int palette_video;
66     cvid_strip_t strips[MAX_STRIPS];
67
68 } CinepakContext;
69
70 static void cinepak_decode_codebook (cvid_codebook_t *codebook,
71                                      int chunk_id, int size, uint8_t *data)
72 {
73     uint8_t *eod = (data + size);
74     uint32_t flag, mask;
75     int      i, n;
76
77     /* check if this chunk contains 4- or 6-element vectors */
78     n    = (chunk_id & 0x0400) ? 4 : 6;
79     flag = 0;
80     mask = 0;
81
82     for (i=0; i < 256; i++) {
83         if ((chunk_id & 0x0100) && !(mask >>= 1)) {
84             if ((data + 4) > eod)
85                 break;
86
87             flag  = BE_32 (data);
88             data += 4;
89             mask  = 0x80000000;
90         }
91
92         if (!(chunk_id & 0x0100) || (flag & mask)) {
93             if ((data + n) > eod)
94                 break;
95
96             if (n == 6) {
97                 codebook[i].y0 = *data++;
98                 codebook[i].y1 = *data++;
99                 codebook[i].y2 = *data++;
100                 codebook[i].y3 = *data++;
101                 codebook[i].u  = 128 + *data++;
102                 codebook[i].v  = 128 + *data++;
103             } else {
104                 /* this codebook type indicates either greyscale or 
105                  * palettized video; if palettized, U & V components will
106                  * not be used so it is safe to set them to 128 for the
107                  * benefit of greyscale rendering in YUV420P */
108                 codebook[i].y0 = *data++;
109                 codebook[i].y1 = *data++;
110                 codebook[i].y2 = *data++;
111                 codebook[i].y3 = *data++;
112                 codebook[i].u  = 128;
113                 codebook[i].v  = 128;
114             }
115         }
116     }
117 }
118
119 static int cinepak_decode_vectors (CinepakContext *s, cvid_strip_t *strip,
120                                    int chunk_id, int size, uint8_t *data)
121 {
122     uint8_t         *eod = (data + size);
123     uint32_t         flag, mask;
124     cvid_codebook_t *codebook;
125     unsigned int     x, y;
126     uint32_t         iy[4];
127     uint32_t         iu[2];
128     uint32_t         iv[2];
129
130     flag = 0;
131     mask = 0;
132
133     for (y=strip->y1; y < strip->y2; y+=4) {
134
135         iy[0] = strip->x1 + (y * s->frame.linesize[0]);
136         iy[1] = iy[0] + s->frame.linesize[0];
137         iy[2] = iy[1] + s->frame.linesize[0];
138         iy[3] = iy[2] + s->frame.linesize[0];
139         iu[0] = (strip->x1/2) + ((y/2) * s->frame.linesize[1]);
140         iu[1] = iu[0] + s->frame.linesize[1];
141         iv[0] = (strip->x1/2) + ((y/2) * s->frame.linesize[2]);
142         iv[1] = iv[0] + s->frame.linesize[2];
143
144         for (x=strip->x1; x < strip->x2; x+=4) {
145             if ((chunk_id & 0x0100) && !(mask >>= 1)) {
146                 if ((data + 4) > eod)
147                     return -1;
148
149                 flag  = BE_32 (data);
150                 data += 4;
151                 mask  = 0x80000000;
152             }
153
154             if (!(chunk_id & 0x0100) || (flag & mask)) {
155                 if (!(chunk_id & 0x0200) && !(mask >>= 1)) {
156                     if ((data + 4) > eod)
157                         return -1;
158
159                     flag  = BE_32 (data);
160                     data += 4;
161                     mask  = 0x80000000;
162                 }
163
164                 if ((chunk_id & 0x0200) || (~flag & mask)) {
165                     if (data >= eod)
166                         return -1;
167
168                     codebook = &strip->v1_codebook[*data++];
169                     s->frame.data[0][iy[0] + 0] = codebook->y0;
170                     s->frame.data[0][iy[0] + 1] = codebook->y0;
171                     s->frame.data[0][iy[1] + 0] = codebook->y0;
172                     s->frame.data[0][iy[1] + 1] = codebook->y0;
173                     if (!s->palette_video) {
174                         s->frame.data[1][iu[0]] = codebook->u;
175                         s->frame.data[2][iv[0]] = codebook->v;
176                     }
177
178                     s->frame.data[0][iy[0] + 2] = codebook->y1;
179                     s->frame.data[0][iy[0] + 3] = codebook->y1;
180                     s->frame.data[0][iy[1] + 2] = codebook->y1;
181                     s->frame.data[0][iy[1] + 3] = codebook->y1;
182                     if (!s->palette_video) {
183                         s->frame.data[1][iu[0] + 1] = codebook->u;
184                         s->frame.data[2][iv[0] + 1] = codebook->v;
185                     }
186
187                     s->frame.data[0][iy[2] + 0] = codebook->y2;
188                     s->frame.data[0][iy[2] + 1] = codebook->y2;
189                     s->frame.data[0][iy[3] + 0] = codebook->y2;
190                     s->frame.data[0][iy[3] + 1] = codebook->y2;
191                     if (!s->palette_video) {
192                         s->frame.data[1][iu[1]] = codebook->u;
193                         s->frame.data[2][iv[1]] = codebook->v;
194                     }
195
196                     s->frame.data[0][iy[2] + 2] = codebook->y3;
197                     s->frame.data[0][iy[2] + 3] = codebook->y3;
198                     s->frame.data[0][iy[3] + 2] = codebook->y3;
199                     s->frame.data[0][iy[3] + 3] = codebook->y3;
200                     if (!s->palette_video) {
201                         s->frame.data[1][iu[1] + 1] = codebook->u;
202                         s->frame.data[2][iv[1] + 1] = codebook->v;
203                     }
204
205                 } else if (flag & mask) {
206                     if ((data + 4) > eod)
207                         return -1;
208
209                     codebook = &strip->v4_codebook[*data++];
210                     s->frame.data[0][iy[0] + 0] = codebook->y0;
211                     s->frame.data[0][iy[0] + 1] = codebook->y1;
212                     s->frame.data[0][iy[1] + 0] = codebook->y2;
213                     s->frame.data[0][iy[1] + 1] = codebook->y3;
214                     if (!s->palette_video) {
215                         s->frame.data[1][iu[0]] = codebook->u;
216                         s->frame.data[2][iv[0]] = codebook->v;
217                     }
218
219                     codebook = &strip->v4_codebook[*data++];
220                     s->frame.data[0][iy[0] + 2] = codebook->y0;
221                     s->frame.data[0][iy[0] + 3] = codebook->y1;
222                     s->frame.data[0][iy[1] + 2] = codebook->y2;
223                     s->frame.data[0][iy[1] + 3] = codebook->y3;
224                     if (!s->palette_video) {
225                         s->frame.data[1][iu[0] + 1] = codebook->u;
226                         s->frame.data[2][iv[0] + 1] = codebook->v;
227                     }
228
229                     codebook = &strip->v4_codebook[*data++];
230                     s->frame.data[0][iy[2] + 0] = codebook->y0;
231                     s->frame.data[0][iy[2] + 1] = codebook->y1;
232                     s->frame.data[0][iy[3] + 0] = codebook->y2;
233                     s->frame.data[0][iy[3] + 1] = codebook->y3;
234                     if (!s->palette_video) {
235                         s->frame.data[1][iu[1]] = codebook->u;
236                         s->frame.data[2][iv[1]] = codebook->v;
237                     }
238
239                     codebook = &strip->v4_codebook[*data++];
240                     s->frame.data[0][iy[2] + 2] = codebook->y0;
241                     s->frame.data[0][iy[2] + 3] = codebook->y1;
242                     s->frame.data[0][iy[3] + 2] = codebook->y2;
243                     s->frame.data[0][iy[3] + 3] = codebook->y3;
244                     if (!s->palette_video) {
245                         s->frame.data[1][iu[1] + 1] = codebook->u;
246                         s->frame.data[2][iv[1] + 1] = codebook->v;
247                     }
248
249                 }
250             }
251
252             iy[0] += 4;  iy[1] += 4;
253             iy[2] += 4;  iy[3] += 4;
254             iu[0] += 2;  iu[1] += 2;
255             iv[0] += 2;  iv[1] += 2;
256         }
257     }
258
259     return 0;
260 }
261
262 static int cinepak_decode_strip (CinepakContext *s,
263                                  cvid_strip_t *strip, uint8_t *data, int size)
264 {
265     uint8_t *eod = (data + size);
266     int      chunk_id, chunk_size;
267
268     /* coordinate sanity checks */
269     if (strip->x1 >= s->width  || strip->x2 > s->width  ||
270         strip->y1 >= s->height || strip->y2 > s->height ||
271         strip->x1 >= strip->x2 || strip->y1 >= strip->y2)
272         return -1;
273
274     while ((data + 4) <= eod) {
275         chunk_id   = BE_16 (&data[0]);
276         chunk_size = BE_16 (&data[2]) - 4;
277         if(chunk_size < 0)
278             return -1;
279
280         data      += 4;
281         chunk_size = ((data + chunk_size) > eod) ? (eod - data) : chunk_size;
282
283         switch (chunk_id) {
284
285         case 0x2000:
286         case 0x2100:
287         case 0x2400:
288         case 0x2500:
289             cinepak_decode_codebook (strip->v4_codebook, chunk_id, 
290                 chunk_size, data);
291             break;
292
293         case 0x2200:
294         case 0x2300:
295         case 0x2600:
296         case 0x2700:
297             cinepak_decode_codebook (strip->v1_codebook, chunk_id, 
298                 chunk_size, data);
299             break;
300
301         case 0x3000:
302         case 0x3100:
303         case 0x3200:
304             return cinepak_decode_vectors (s, strip, chunk_id, 
305                 chunk_size, data);
306         }
307
308         data += chunk_size;
309     }
310
311     return -1;
312 }
313
314 static int cinepak_decode (CinepakContext *s)
315 {
316     uint8_t      *eod = (s->data + s->size);
317     int           i, result, strip_size, frame_flags, num_strips;
318     int           y0 = 0;
319     int           encoded_buf_size;
320     /* if true, Cinepak data is from a Sega FILM/CPK file */
321     int           sega_film_data = 0;
322
323     if (s->size < 10)
324         return -1;
325
326     frame_flags = s->data[0];
327     num_strips  = BE_16 (&s->data[8]);
328     encoded_buf_size = ((s->data[1] << 16) | BE_16 (&s->data[2]));
329     if (encoded_buf_size != s->size)
330         sega_film_data = 1;
331     if (sega_film_data)
332         s->data    += 12;
333     else
334         s->data    += 10;
335
336     if (num_strips > MAX_STRIPS)
337         num_strips = MAX_STRIPS;
338
339     for (i=0; i < num_strips; i++) {
340         if ((s->data + 12) > eod)
341             return -1;
342
343         s->strips[i].id = BE_16 (s->data);
344         s->strips[i].y1 = y0;
345         s->strips[i].x1 = 0;
346         s->strips[i].y2 = y0 + BE_16 (&s->data[8]);
347         s->strips[i].x2 = s->avctx->width;
348
349         strip_size = BE_16 (&s->data[2]) - 12;
350         s->data   += 12;
351         strip_size = ((s->data + strip_size) > eod) ? (eod - s->data) : strip_size;
352
353         if ((i > 0) && !(frame_flags & 0x01)) {
354             memcpy (s->strips[i].v4_codebook, s->strips[i-1].v4_codebook,
355                 sizeof(s->strips[i].v4_codebook));
356             memcpy (s->strips[i].v1_codebook, s->strips[i-1].v1_codebook,
357                 sizeof(s->strips[i].v1_codebook));
358         }
359
360         result = cinepak_decode_strip (s, &s->strips[i], s->data, strip_size);
361
362         if (result != 0)
363             return result;
364
365         s->data += strip_size;
366         y0    = s->strips[i].y2;
367     }
368     return 0;
369 }
370
371 static int cinepak_decode_init(AVCodecContext *avctx)
372 {
373     CinepakContext *s = (CinepakContext *)avctx->priv_data;
374
375     s->avctx = avctx;
376     s->width = (avctx->width + 3) & ~3;
377     s->height = (avctx->height + 3) & ~3;
378
379     // check for paletted data
380     if ((avctx->palctrl == NULL) || (avctx->bits_per_sample == 40)) {
381         s->palette_video = 0;
382         avctx->pix_fmt = PIX_FMT_YUV420P;
383     } else {
384         s->palette_video = 1;
385         avctx->pix_fmt = PIX_FMT_PAL8;
386     }
387
388     avctx->has_b_frames = 0;
389     dsputil_init(&s->dsp, avctx);
390
391     s->frame.data[0] = NULL;
392
393     return 0;
394 }
395
396 static int cinepak_decode_frame(AVCodecContext *avctx,
397                                 void *data, int *data_size,
398                                 uint8_t *buf, int buf_size)
399 {
400     CinepakContext *s = (CinepakContext *)avctx->priv_data;
401
402     s->data = buf;
403     s->size = buf_size;
404
405     s->frame.reference = 1;
406     s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
407                             FF_BUFFER_HINTS_REUSABLE;
408     if (avctx->reget_buffer(avctx, &s->frame)) {
409         av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n");
410         return -1;
411     }
412
413     cinepak_decode(s);
414
415     if (s->palette_video) {
416         memcpy (s->frame.data[1], avctx->palctrl->palette, AVPALETTE_SIZE);
417         if (avctx->palctrl->palette_changed) {
418             s->frame.palette_has_changed = 1;
419             avctx->palctrl->palette_changed = 0;
420         } else
421             s->frame.palette_has_changed = 0;
422     }
423
424     *data_size = sizeof(AVFrame);
425     *(AVFrame*)data = s->frame;
426
427     /* report that the buffer was completely consumed */
428     return buf_size;
429 }
430
431 static int cinepak_decode_end(AVCodecContext *avctx)
432 {
433     CinepakContext *s = (CinepakContext *)avctx->priv_data;
434
435     if (s->frame.data[0])
436         avctx->release_buffer(avctx, &s->frame);
437
438     return 0;
439 }
440
441 AVCodec cinepak_decoder = {
442     "cinepak",
443     CODEC_TYPE_VIDEO,
444     CODEC_ID_CINEPAK,
445     sizeof(CinepakContext),
446     cinepak_decode_init,
447     NULL,
448     cinepak_decode_end,
449     cinepak_decode_frame,
450     CODEC_CAP_DR1,
451 };