shorten: remove VLA and check for buffer overflow
[ffmpeg.git] / libavcodec / shorten.c
1 /*
2  * Shorten decoder
3  * Copyright (c) 2005 Jeff Muizelaar
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * @file
24  * Shorten decoder
25  * @author Jeff Muizelaar
26  *
27  */
28
29 #define DEBUG
30 #include <limits.h>
31 #include "avcodec.h"
32 #include "get_bits.h"
33 #include "golomb.h"
34
35 #define MAX_CHANNELS 8
36 #define MAX_BLOCKSIZE 65535
37
38 #define OUT_BUFFER_SIZE 16384
39
40 #define ULONGSIZE 2
41
42 #define WAVE_FORMAT_PCM 0x0001
43
44 #define DEFAULT_BLOCK_SIZE 256
45
46 #define TYPESIZE 4
47 #define CHANSIZE 0
48 #define LPCQSIZE 2
49 #define ENERGYSIZE 3
50 #define BITSHIFTSIZE 2
51
52 #define TYPE_S16HL 3
53 #define TYPE_S16LH 5
54
55 #define NWRAP 3
56 #define NSKIPSIZE 1
57
58 #define LPCQUANT 5
59 #define V2LPCQOFFSET (1 << LPCQUANT)
60
61 #define FNSIZE 2
62 #define FN_DIFF0        0
63 #define FN_DIFF1        1
64 #define FN_DIFF2        2
65 #define FN_DIFF3        3
66 #define FN_QUIT         4
67 #define FN_BLOCKSIZE    5
68 #define FN_BITSHIFT     6
69 #define FN_QLPC         7
70 #define FN_ZERO         8
71 #define FN_VERBATIM     9
72
73 #define VERBATIM_CKSIZE_SIZE 5
74 #define VERBATIM_BYTE_SIZE 8
75 #define CANONICAL_HEADER_SIZE 44
76
77 typedef struct ShortenContext {
78     AVCodecContext *avctx;
79     GetBitContext gb;
80
81     int min_framesize, max_framesize;
82     int channels;
83
84     int32_t *decoded[MAX_CHANNELS];
85     int32_t *offset[MAX_CHANNELS];
86     int *coeffs;
87     uint8_t *bitstream;
88     int bitstream_size;
89     int bitstream_index;
90     unsigned int allocated_bitstream_size;
91     int header_size;
92     uint8_t header[OUT_BUFFER_SIZE];
93     int version;
94     int cur_chan;
95     int bitshift;
96     int nmean;
97     int internal_ftype;
98     int nwrap;
99     int blocksize;
100     int bitindex;
101     int32_t lpcqoffset;
102 } ShortenContext;
103
104 static av_cold int shorten_decode_init(AVCodecContext * avctx)
105 {
106     ShortenContext *s = avctx->priv_data;
107     s->avctx = avctx;
108     avctx->sample_fmt = SAMPLE_FMT_S16;
109
110     return 0;
111 }
112
113 static int allocate_buffers(ShortenContext *s)
114 {
115     int i, chan;
116     int *coeffs;
117
118     for (chan=0; chan<s->channels; chan++) {
119         if(FFMAX(1, s->nmean) >= UINT_MAX/sizeof(int32_t)){
120             av_log(s->avctx, AV_LOG_ERROR, "nmean too large\n");
121             return -1;
122         }
123         if(s->blocksize + s->nwrap >= UINT_MAX/sizeof(int32_t) || s->blocksize + s->nwrap <= (unsigned)s->nwrap){
124             av_log(s->avctx, AV_LOG_ERROR, "s->blocksize + s->nwrap too large\n");
125             return -1;
126         }
127
128         s->offset[chan] = av_realloc(s->offset[chan], sizeof(int32_t)*FFMAX(1, s->nmean));
129
130         s->decoded[chan] = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap));
131         for (i=0; i<s->nwrap; i++)
132             s->decoded[chan][i] = 0;
133         s->decoded[chan] += s->nwrap;
134     }
135
136     coeffs = av_realloc(s->coeffs, s->nwrap * sizeof(*s->coeffs));
137     if (!coeffs)
138         return AVERROR(ENOMEM);
139     s->coeffs = coeffs;
140
141     return 0;
142 }
143
144
145 static inline unsigned int get_uint(ShortenContext *s, int k)
146 {
147     if (s->version != 0)
148         k = get_ur_golomb_shorten(&s->gb, ULONGSIZE);
149     return get_ur_golomb_shorten(&s->gb, k);
150 }
151
152
153 static void fix_bitshift(ShortenContext *s, int32_t *buffer)
154 {
155     int i;
156
157     if (s->bitshift != 0)
158         for (i = 0; i < s->blocksize; i++)
159             buffer[s->nwrap + i] <<= s->bitshift;
160 }
161
162
163 static void init_offset(ShortenContext *s)
164 {
165     int32_t mean = 0;
166     int  chan, i;
167     int nblock = FFMAX(1, s->nmean);
168     /* initialise offset */
169     switch (s->internal_ftype)
170     {
171         case TYPE_S16HL:
172         case TYPE_S16LH:
173             mean = 0;
174             break;
175         default:
176             av_log(s->avctx, AV_LOG_ERROR, "unknown audio type");
177             abort();
178     }
179
180     for (chan = 0; chan < s->channels; chan++)
181         for (i = 0; i < nblock; i++)
182             s->offset[chan][i] = mean;
183 }
184
185 static inline int get_le32(GetBitContext *gb)
186 {
187     return bswap_32(get_bits_long(gb, 32));
188 }
189
190 static inline short get_le16(GetBitContext *gb)
191 {
192     return bswap_16(get_bits_long(gb, 16));
193 }
194
195 static int decode_wave_header(AVCodecContext *avctx, uint8_t *header, int header_size)
196 {
197     GetBitContext hb;
198     int len;
199     int chunk_size;
200     short wave_format;
201
202     init_get_bits(&hb, header, header_size*8);
203     if (get_le32(&hb) != MKTAG('R','I','F','F')) {
204         av_log(avctx, AV_LOG_ERROR, "missing RIFF tag\n");
205         return -1;
206     }
207
208     chunk_size = get_le32(&hb);
209
210     if (get_le32(&hb) != MKTAG('W','A','V','E')) {
211         av_log(avctx, AV_LOG_ERROR, "missing WAVE tag\n");
212         return -1;
213     }
214
215     while (get_le32(&hb) != MKTAG('f','m','t',' ')) {
216         len = get_le32(&hb);
217         skip_bits(&hb, 8*len);
218     }
219     len = get_le32(&hb);
220
221     if (len < 16) {
222         av_log(avctx, AV_LOG_ERROR, "fmt chunk was too short\n");
223         return -1;
224     }
225
226     wave_format = get_le16(&hb);
227
228     switch (wave_format) {
229         case WAVE_FORMAT_PCM:
230             break;
231         default:
232             av_log(avctx, AV_LOG_ERROR, "unsupported wave format\n");
233             return -1;
234     }
235
236     avctx->channels = get_le16(&hb);
237     avctx->sample_rate = get_le32(&hb);
238     avctx->bit_rate = get_le32(&hb) * 8;
239     avctx->block_align = get_le16(&hb);
240     avctx->bits_per_coded_sample = get_le16(&hb);
241
242     if (avctx->bits_per_coded_sample != 16) {
243         av_log(avctx, AV_LOG_ERROR, "unsupported number of bits per sample\n");
244         return -1;
245     }
246
247     len -= 16;
248     if (len > 0)
249         av_log(avctx, AV_LOG_INFO, "%d header bytes unparsed\n", len);
250
251     return 0;
252 }
253
254 static int16_t * interleave_buffer(int16_t *samples, int nchan, int blocksize, int32_t **buffer) {
255     int i, chan;
256     for (i=0; i<blocksize; i++)
257         for (chan=0; chan < nchan; chan++)
258             *samples++ = FFMIN(buffer[chan][i], 32768);
259     return samples;
260 }
261
262 static void decode_subframe_lpc(ShortenContext *s, int channel, int residual_size, int pred_order)
263 {
264     int sum, i, j;
265     int *coeffs = s->coeffs;
266
267     for (i=0; i<pred_order; i++)
268         coeffs[i] = get_sr_golomb_shorten(&s->gb, LPCQUANT);
269
270     for (i=0; i < s->blocksize; i++) {
271         sum = s->lpcqoffset;
272         for (j=0; j<pred_order; j++)
273             sum += coeffs[j] * s->decoded[channel][i-j-1];
274         s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + (sum >> LPCQUANT);
275     }
276 }
277
278
279 static int shorten_decode_frame(AVCodecContext *avctx,
280         void *data, int *data_size,
281         AVPacket *avpkt)
282 {
283     const uint8_t *buf = avpkt->data;
284     int buf_size = avpkt->size;
285     ShortenContext *s = avctx->priv_data;
286     int i, input_buf_size = 0;
287     int16_t *samples = data;
288     if(s->max_framesize == 0){
289         s->max_framesize= 1024; // should hopefully be enough for the first header
290         s->bitstream= av_fast_realloc(s->bitstream, &s->allocated_bitstream_size, s->max_framesize);
291     }
292
293     if(1 && s->max_framesize){//FIXME truncated
294         buf_size= FFMIN(buf_size, s->max_framesize - s->bitstream_size);
295         input_buf_size= buf_size;
296
297         if(s->bitstream_index + s->bitstream_size + buf_size > s->allocated_bitstream_size){
298             //                printf("memmove\n");
299             memmove(s->bitstream, &s->bitstream[s->bitstream_index], s->bitstream_size);
300             s->bitstream_index=0;
301         }
302         memcpy(&s->bitstream[s->bitstream_index + s->bitstream_size], buf, buf_size);
303         buf= &s->bitstream[s->bitstream_index];
304         buf_size += s->bitstream_size;
305         s->bitstream_size= buf_size;
306
307         if(buf_size < s->max_framesize){
308             //dprintf(avctx, "wanna more data ... %d\n", buf_size);
309             *data_size = 0;
310             return input_buf_size;
311         }
312     }
313     init_get_bits(&s->gb, buf, buf_size*8);
314     skip_bits(&s->gb, s->bitindex);
315     if (!s->blocksize)
316     {
317         int maxnlpc = 0;
318         /* shorten signature */
319         if (get_bits_long(&s->gb, 32) != AV_RB32("ajkg")) {
320             av_log(s->avctx, AV_LOG_ERROR, "missing shorten magic 'ajkg'\n");
321             return -1;
322         }
323
324         s->lpcqoffset = 0;
325         s->blocksize = DEFAULT_BLOCK_SIZE;
326         s->channels = 1;
327         s->nmean = -1;
328         s->version = get_bits(&s->gb, 8);
329         s->internal_ftype = get_uint(s, TYPESIZE);
330
331         s->channels = get_uint(s, CHANSIZE);
332         if (s->channels > MAX_CHANNELS) {
333             av_log(s->avctx, AV_LOG_ERROR, "too many channels: %d\n", s->channels);
334             return -1;
335         }
336
337         /* get blocksize if version > 0 */
338         if (s->version > 0) {
339             int skip_bytes;
340             s->blocksize = get_uint(s, av_log2(DEFAULT_BLOCK_SIZE));
341             maxnlpc = get_uint(s, LPCQSIZE);
342             s->nmean = get_uint(s, 0);
343
344             skip_bytes = get_uint(s, NSKIPSIZE);
345             for (i=0; i<skip_bytes; i++) {
346                 skip_bits(&s->gb, 8);
347             }
348         }
349         s->nwrap = FFMAX(NWRAP, maxnlpc);
350
351         if (allocate_buffers(s))
352             return -1;
353
354         init_offset(s);
355
356         if (s->version > 1)
357             s->lpcqoffset = V2LPCQOFFSET;
358
359         if (get_ur_golomb_shorten(&s->gb, FNSIZE) != FN_VERBATIM) {
360             av_log(s->avctx, AV_LOG_ERROR, "missing verbatim section at beginning of stream\n");
361             return -1;
362         }
363
364         s->header_size = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
365         if (s->header_size >= OUT_BUFFER_SIZE || s->header_size < CANONICAL_HEADER_SIZE) {
366             av_log(s->avctx, AV_LOG_ERROR, "header is wrong size: %d\n", s->header_size);
367             return -1;
368         }
369
370         for (i=0; i<s->header_size; i++)
371             s->header[i] = (char)get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
372
373         if (decode_wave_header(avctx, s->header, s->header_size) < 0)
374             return -1;
375
376         s->cur_chan = 0;
377         s->bitshift = 0;
378     }
379     else
380     {
381         int cmd;
382         int len;
383         cmd = get_ur_golomb_shorten(&s->gb, FNSIZE);
384         switch (cmd) {
385             case FN_ZERO:
386             case FN_DIFF0:
387             case FN_DIFF1:
388             case FN_DIFF2:
389             case FN_DIFF3:
390             case FN_QLPC:
391                 {
392                     int residual_size = 0;
393                     int channel = s->cur_chan;
394                     int32_t coffset;
395                     if (cmd != FN_ZERO) {
396                         residual_size = get_ur_golomb_shorten(&s->gb, ENERGYSIZE);
397                         /* this is a hack as version 0 differed in defintion of get_sr_golomb_shorten */
398                         if (s->version == 0)
399                             residual_size--;
400                     }
401
402                     if (s->nmean == 0)
403                         coffset = s->offset[channel][0];
404                     else {
405                         int32_t sum = (s->version < 2) ? 0 : s->nmean / 2;
406                         for (i=0; i<s->nmean; i++)
407                             sum += s->offset[channel][i];
408                         coffset = sum / s->nmean;
409                         if (s->version >= 2)
410                             coffset >>= FFMIN(1, s->bitshift);
411                     }
412                     switch (cmd) {
413                         case FN_ZERO:
414                             for (i=0; i<s->blocksize; i++)
415                                 s->decoded[channel][i] = 0;
416                             break;
417                         case FN_DIFF0:
418                             for (i=0; i<s->blocksize; i++)
419                                 s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + coffset;
420                             break;
421                         case FN_DIFF1:
422                             for (i=0; i<s->blocksize; i++)
423                                 s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + s->decoded[channel][i - 1];
424                             break;
425                         case FN_DIFF2:
426                             for (i=0; i<s->blocksize; i++)
427                                 s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + 2*s->decoded[channel][i-1]
428                                                                                                       -   s->decoded[channel][i-2];
429                             break;
430                         case FN_DIFF3:
431                             for (i=0; i<s->blocksize; i++)
432                                 s->decoded[channel][i] = get_sr_golomb_shorten(&s->gb, residual_size) + 3*s->decoded[channel][i-1]
433                                                                                                       - 3*s->decoded[channel][i-2]
434                                                                                                       +   s->decoded[channel][i-3];
435                             break;
436                         case FN_QLPC:
437                             {
438                                 int pred_order = get_ur_golomb_shorten(&s->gb, LPCQSIZE);
439                                 if (pred_order > s->nwrap) {
440                                     av_log(avctx, AV_LOG_ERROR,
441                                            "invalid pred_order %d\n",
442                                            pred_order);
443                                     return -1;
444                                 }
445                                 for (i=0; i<pred_order; i++)
446                                     s->decoded[channel][i - pred_order] -= coffset;
447                                 decode_subframe_lpc(s, channel, residual_size, pred_order);
448                                 if (coffset != 0)
449                                     for (i=0; i < s->blocksize; i++)
450                                         s->decoded[channel][i] += coffset;
451                             }
452                     }
453                     if (s->nmean > 0) {
454                         int32_t sum = (s->version < 2) ? 0 : s->blocksize / 2;
455                         for (i=0; i<s->blocksize; i++)
456                             sum += s->decoded[channel][i];
457
458                         for (i=1; i<s->nmean; i++)
459                             s->offset[channel][i-1] = s->offset[channel][i];
460
461                         if (s->version < 2)
462                             s->offset[channel][s->nmean - 1] = sum / s->blocksize;
463                         else
464                             s->offset[channel][s->nmean - 1] = (sum / s->blocksize) << s->bitshift;
465                     }
466                     for (i=-s->nwrap; i<0; i++)
467                         s->decoded[channel][i] = s->decoded[channel][i + s->blocksize];
468
469                     fix_bitshift(s, s->decoded[channel]);
470
471                     s->cur_chan++;
472                     if (s->cur_chan == s->channels) {
473                         samples = interleave_buffer(samples, s->channels, s->blocksize, s->decoded);
474                         s->cur_chan = 0;
475                         goto frame_done;
476                     }
477                     break;
478                 }
479                 break;
480             case FN_VERBATIM:
481                 len = get_ur_golomb_shorten(&s->gb, VERBATIM_CKSIZE_SIZE);
482                 while (len--) {
483                     get_ur_golomb_shorten(&s->gb, VERBATIM_BYTE_SIZE);
484                 }
485                 break;
486             case FN_BITSHIFT:
487                 s->bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
488                 break;
489             case FN_BLOCKSIZE:
490                 s->blocksize = get_uint(s, av_log2(s->blocksize));
491                 break;
492             case FN_QUIT:
493                 *data_size = 0;
494                 return buf_size;
495                 break;
496             default:
497                 av_log(avctx, AV_LOG_ERROR, "unknown shorten function %d\n", cmd);
498                 return -1;
499                 break;
500         }
501     }
502 frame_done:
503     *data_size = (int8_t *)samples - (int8_t *)data;
504
505     //    s->last_blocksize = s->blocksize;
506     s->bitindex = get_bits_count(&s->gb) - 8*((get_bits_count(&s->gb))/8);
507     i= (get_bits_count(&s->gb))/8;
508     if (i > buf_size) {
509         av_log(s->avctx, AV_LOG_ERROR, "overread: %d\n", i - buf_size);
510         s->bitstream_size=0;
511         s->bitstream_index=0;
512         return -1;
513     }
514     if (s->bitstream_size) {
515         s->bitstream_index += i;
516         s->bitstream_size  -= i;
517         return input_buf_size;
518     } else
519         return i;
520 }
521
522 static av_cold int shorten_decode_close(AVCodecContext *avctx)
523 {
524     ShortenContext *s = avctx->priv_data;
525     int i;
526
527     for (i = 0; i < s->channels; i++) {
528         s->decoded[i] -= s->nwrap;
529         av_freep(&s->decoded[i]);
530         av_freep(&s->offset[i]);
531     }
532     av_freep(&s->bitstream);
533     av_freep(&s->coeffs);
534     return 0;
535 }
536
537 static void shorten_flush(AVCodecContext *avctx){
538     ShortenContext *s = avctx->priv_data;
539
540     s->bitstream_size=
541         s->bitstream_index= 0;
542 }
543
544 AVCodec shorten_decoder = {
545     "shorten",
546     AVMEDIA_TYPE_AUDIO,
547     CODEC_ID_SHORTEN,
548     sizeof(ShortenContext),
549     shorten_decode_init,
550     NULL,
551     shorten_decode_close,
552     shorten_decode_frame,
553     .flush= shorten_flush,
554     .long_name= NULL_IF_CONFIG_SMALL("Shorten"),
555 };