Correct edge MC for chroma
[ffmpeg.git] / libavcodec / cook.c
1 /*
2  * COOK compatible decoder
3  * Copyright (c) 2003 Sascha Sommer
4  * Copyright (c) 2005 Benjamin Larsson
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  *
20  */
21
22 /**
23  * @file cook.c
24  * Cook compatible decoder.
25  * This decoder handles RealNetworks, RealAudio G2 data.
26  * Cook is identified by the codec name cook in RM files.
27  *
28  * To use this decoder, a calling application must supply the extradata
29  * bytes provided from the RM container; 8+ bytes for mono streams and
30  * 16+ for stereo streams (maybe more).
31  *
32  * Codec technicalities (all this assume a buffer length of 1024):
33  * Cook works with several different techniques to achieve its compression.
34  * In the timedomain the buffer is divided into 8 pieces and quantized. If
35  * two neighboring pieces have different quantization index a smooth
36  * quantization curve is used to get a smooth overlap between the different
37  * pieces.
38  * To get to the transformdomain Cook uses a modulated lapped transform.
39  * The transform domain has 50 subbands with 20 elements each. This
40  * means only a maximum of 50*20=1000 coefficients are used out of the 1024
41  * available.
42  */
43
44 #include <math.h>
45 #include <stddef.h>
46 #include <stdio.h>
47
48 #define ALT_BITSTREAM_READER
49 #include "avcodec.h"
50 #include "bitstream.h"
51 #include "dsputil.h"
52
53 #include "cookdata.h"
54
55 /* the different Cook versions */
56 #define MONO_COOK1      0x1000001
57 #define MONO_COOK2      0x1000002
58 #define JOINT_STEREO    0x1000003
59 #define MC_COOK         0x2000000   //multichannel Cook, not supported
60
61 #define SUBBAND_SIZE    20
62 //#define COOKDEBUG
63
64 typedef struct {
65     int     size;
66     int     qidx_table1[8];
67     int     qidx_table2[8];
68 } COOKgain;
69
70 typedef struct __attribute__((__packed__)){
71     /* codec data start */
72     uint32_t cookversion;               //in network order, bigendian
73     uint16_t samples_per_frame;         //amount of samples per frame per channel, bigendian
74     uint16_t subbands;                  //amount of bands used in the frequency domain, bigendian
75     /* Mono extradata ends here. */
76     uint32_t unused;
77     uint16_t js_subband_start;          //bigendian
78     uint16_t js_vlc_bits;               //bigendian
79     /* Stereo extradata ends here. */
80 } COOKextradata;
81
82
83 typedef struct {
84     GetBitContext       gb;
85     /* stream data */
86     int                 nb_channels;
87     int                 joint_stereo;
88     int                 bit_rate;
89     int                 sample_rate;
90     int                 samples_per_channel;
91     int                 samples_per_frame;
92     int                 subbands;
93     int                 log2_numvector_size;
94     int                 numvector_size;                //1 << log2_numvector_size;
95     int                 js_subband_start;
96     int                 total_subbands;
97     int                 num_vectors;
98     int                 bits_per_subpacket;
99     /* states */
100     int                 random_state;
101
102     /* transform data */
103     FFTContext          fft_ctx;
104     FFTSample           mlt_tmp[1024] __attribute__((aligned(16))); /* temporary storage for imlt */
105     float*              mlt_window;
106     float*              mlt_precos;
107     float*              mlt_presin;
108     float*              mlt_postcos;
109     int                 fft_size;
110     int                 fft_order;
111     int                 mlt_size;       //modulated lapped transform size
112
113     /* gain buffers */
114     COOKgain*           gain_now_ptr;
115     COOKgain*           gain_previous_ptr;
116     COOKgain            gain_current;
117     COOKgain            gain_now;
118     COOKgain            gain_previous;
119     COOKgain            gain_channel1[2];
120     COOKgain            gain_channel2[2];
121
122     /* VLC data */
123     int                 js_vlc_bits;
124     VLC                 envelope_quant_index[13];
125     VLC                 sqvh[7];          //scalar quantization
126     VLC                 ccpl;             //channel coupling
127
128     /* generatable tables and related variables */
129     int                 gain_size_factor;
130     float               gain_table[23];
131     float               pow2tab[127];
132     float               rootpow2tab[127];
133
134     /* data buffers */
135
136     uint8_t*            decoded_bytes_buffer;
137     float               mono_mdct_output[2048] __attribute__((aligned(16)));
138     float*              previous_buffer_ptr[2];
139     float               mono_previous_buffer1[1024];
140     float               mono_previous_buffer2[1024];
141     float*              decode_buf_ptr[4];
142     float*              decode_buf_ptr2[2];
143     float               decode_buffer_1[1024];
144     float               decode_buffer_2[1024];
145     float               decode_buffer_3[1024];
146     float               decode_buffer_4[1024];
147 } COOKContext;
148
149 /* debug functions */
150
151 #ifdef COOKDEBUG
152 static void dump_float_table(float* table, int size, int delimiter) {
153     int i=0;
154     av_log(NULL,AV_LOG_ERROR,"\n[%d]: ",i);
155     for (i=0 ; i<size ; i++) {
156         av_log(NULL, AV_LOG_ERROR, "%5.1f, ", table[i]);
157         if ((i+1)%delimiter == 0) av_log(NULL,AV_LOG_ERROR,"\n[%d]: ",i+1);
158     }
159 }
160
161 static void dump_int_table(int* table, int size, int delimiter) {
162     int i=0;
163     av_log(NULL,AV_LOG_ERROR,"\n[%d]: ",i);
164     for (i=0 ; i<size ; i++) {
165         av_log(NULL, AV_LOG_ERROR, "%d, ", table[i]);
166         if ((i+1)%delimiter == 0) av_log(NULL,AV_LOG_ERROR,"\n[%d]: ",i+1);
167     }
168 }
169
170 static void dump_short_table(short* table, int size, int delimiter) {
171     int i=0;
172     av_log(NULL,AV_LOG_ERROR,"\n[%d]: ",i);
173     for (i=0 ; i<size ; i++) {
174         av_log(NULL, AV_LOG_ERROR, "%d, ", table[i]);
175         if ((i+1)%delimiter == 0) av_log(NULL,AV_LOG_ERROR,"\n[%d]: ",i+1);
176     }
177 }
178
179 #endif
180
181 /*************** init functions ***************/
182
183 /* table generator */
184 static void init_pow2table(COOKContext *q){
185     int i;
186     q->pow2tab[63] = 1.0;
187     for (i=1 ; i<64 ; i++){
188         q->pow2tab[63+i]=(float)((uint64_t)1<<i);
189         q->pow2tab[63-i]=1.0/(float)((uint64_t)1<<i);
190     }
191 }
192
193 /* table generator */
194 static void init_rootpow2table(COOKContext *q){
195     int i;
196     q->rootpow2tab[63] = 1.0;
197     for (i=1 ; i<64 ; i++){
198         q->rootpow2tab[63+i]=sqrt((float)((uint64_t)1<<i));
199         q->rootpow2tab[63-i]=sqrt(1.0/(float)((uint64_t)1<<i));
200     }
201 }
202
203 /* table generator */
204 static void init_gain_table(COOKContext *q) {
205     int i;
206     q->gain_size_factor = q->samples_per_channel/8;
207     for (i=0 ; i<23 ; i++) {
208         q->gain_table[i] = pow((double)q->pow2tab[i+52] ,
209                                (1.0/(double)q->gain_size_factor));
210     }
211 }
212
213
214 static int init_cook_vlc_tables(COOKContext *q) {
215     int i, result;
216
217     result = 0;
218     for (i=0 ; i<13 ; i++) {
219         result &= init_vlc (&q->envelope_quant_index[i], 9, 24,
220             envelope_quant_index_huffbits[i], 1, 1,
221             envelope_quant_index_huffcodes[i], 2, 2, 0);
222     }
223     av_log(NULL,AV_LOG_DEBUG,"sqvh VLC init\n");
224     for (i=0 ; i<7 ; i++) {
225         result &= init_vlc (&q->sqvh[i], vhvlcsize_tab[i], vhsize_tab[i],
226             cvh_huffbits[i], 1, 1,
227             cvh_huffcodes[i], 2, 2, 0);
228     }
229
230     if (q->nb_channels==2 && q->joint_stereo==1){
231         result &= init_vlc (&q->ccpl, 6, (1<<q->js_vlc_bits)-1,
232             ccpl_huffbits[q->js_vlc_bits-2], 1, 1,
233             ccpl_huffcodes[q->js_vlc_bits-2], 2, 2, 0);
234         av_log(NULL,AV_LOG_DEBUG,"Joint-stereo VLC used.\n");
235     }
236
237     av_log(NULL,AV_LOG_DEBUG,"VLC tables initialized.\n");
238     return result;
239 }
240
241 static int init_cook_mlt(COOKContext *q) {
242     int j;
243     float alpha;
244
245     /* Allocate the buffers, could be replaced with a static [512]
246        array if needed. */
247     q->mlt_size = q->samples_per_channel;
248     q->mlt_window = av_malloc(sizeof(float)*q->mlt_size);
249     q->mlt_precos = av_malloc(sizeof(float)*q->mlt_size/2);
250     q->mlt_presin = av_malloc(sizeof(float)*q->mlt_size/2);
251     q->mlt_postcos = av_malloc(sizeof(float)*q->mlt_size/2);
252
253     /* Initialize the MLT window: simple sine window. */
254     alpha = M_PI / (2.0 * (float)q->mlt_size);
255     for(j=0 ; j<q->mlt_size ; j++) {
256         q->mlt_window[j] = sin((j + 512.0/(float)q->mlt_size) * alpha);
257     }
258
259     /* pre/post twiddle factors */
260     for (j=0 ; j<q->mlt_size/2 ; j++){
261         q->mlt_precos[j] = cos( ((j+0.25)*M_PI)/q->mlt_size);
262         q->mlt_presin[j] = sin( ((j+0.25)*M_PI)/q->mlt_size);
263         q->mlt_postcos[j] = (float)sqrt(2.0/(float)q->mlt_size)*cos( ((float)j*M_PI) /q->mlt_size); //sqrt(2/MLT_size) = scalefactor
264     }
265
266     /* Initialize the FFT. */
267     ff_fft_init(&q->fft_ctx, av_log2(q->mlt_size)-1, 0);
268     av_log(NULL,AV_LOG_DEBUG,"FFT initialized, order = %d.\n",
269            av_log2(q->samples_per_channel)-1);
270
271     return (int)(q->mlt_window && q->mlt_precos && q->mlt_presin && q->mlt_postcos);
272 }
273
274 /*************** init functions end ***********/
275
276 /**
277  * Cook indata decoding, every 32 bits are XORed with 0x37c511f2.
278  * Why? No idea, some checksum/error detection method maybe.
279  * Nice way to waste CPU cycles.
280  *
281  * @param in        pointer to 32bit array of indata
282  * @param bits      amount of bits
283  * @param out       pointer to 32bit array of outdata
284  */
285
286 static inline void decode_bytes(uint8_t* inbuffer, uint8_t* out, int bytes){
287     int i;
288     uint32_t* buf = (uint32_t*) inbuffer;
289     uint32_t* obuf = (uint32_t*) out;
290     /* FIXME: 64 bit platforms would be able to do 64 bits at a time.
291      * I'm too lazy though, should be something like
292      * for(i=0 ; i<bitamount/64 ; i++)
293      *     (int64_t)out[i] = 0x37c511f237c511f2^be2me_64(int64_t)in[i]);
294      * Buffer alignment needs to be checked. */
295
296
297     for(i=0 ; i<bytes/4 ; i++){
298 #ifdef WORDS_BIGENDIAN
299         obuf[i] = 0x37c511f2^buf[i];
300 #else
301         obuf[i] = 0xf211c537^buf[i];
302 #endif
303     }
304 }
305
306 /**
307  * Cook uninit
308  */
309
310 static int cook_decode_close(AVCodecContext *avctx)
311 {
312     int i;
313     COOKContext *q = avctx->priv_data;
314     av_log(NULL,AV_LOG_DEBUG, "Deallocating memory.\n");
315
316     /* Free allocated memory buffers. */
317     av_free(q->mlt_window);
318     av_free(q->mlt_precos);
319     av_free(q->mlt_presin);
320     av_free(q->mlt_postcos);
321     av_free(q->decoded_bytes_buffer);
322
323     /* Free the transform. */
324     ff_fft_end(&q->fft_ctx);
325
326     /* Free the VLC tables. */
327     for (i=0 ; i<13 ; i++) {
328         free_vlc(&q->envelope_quant_index[i]);
329     }
330     for (i=0 ; i<7 ; i++) {
331         free_vlc(&q->sqvh[i]);
332     }
333     if(q->nb_channels==2 && q->joint_stereo==1 ){
334         free_vlc(&q->ccpl);
335     }
336
337     av_log(NULL,AV_LOG_DEBUG,"Memory deallocated.\n");
338
339     return 0;
340 }
341
342 /**
343  * Fill the COOKgain structure for the timedomain quantization.
344  *
345  * @param q                 pointer to the COOKContext
346  * @param gaininfo          pointer to the COOKgain
347  */
348
349 static void decode_gain_info(GetBitContext *gb, COOKgain* gaininfo) {
350     int i;
351
352     while (get_bits1(gb)) {}
353
354     gaininfo->size = get_bits_count(gb) - 1;     //amount of elements*2 to update
355
356     if (get_bits_count(gb) - 1 <= 0) return;
357
358     for (i=0 ; i<gaininfo->size ; i++){
359         gaininfo->qidx_table1[i] = get_bits(gb,3);
360         if (get_bits1(gb)) {
361             gaininfo->qidx_table2[i] = get_bits(gb,4) - 7;  //convert to signed
362         } else {
363             gaininfo->qidx_table2[i] = -1;
364         }
365     }
366 }
367
368 /**
369  * Create the quant index table needed for the envelope.
370  *
371  * @param q                 pointer to the COOKContext
372  * @param quant_index_table pointer to the array
373  */
374
375 static void decode_envelope(COOKContext *q, int* quant_index_table) {
376     int i,j, vlc_index;
377     int bitbias;
378
379     bitbias = get_bits_count(&q->gb);
380     quant_index_table[0]= get_bits(&q->gb,6) - 6;       //This is used later in categorize
381
382     for (i=1 ; i < q->total_subbands ; i++){
383         vlc_index=i;
384         if (i >= q->js_subband_start * 2) {
385             vlc_index-=q->js_subband_start;
386         } else {
387             vlc_index/=2;
388             if(vlc_index < 1) vlc_index = 1;
389         }
390         if (vlc_index>13) vlc_index = 13;           //the VLC tables >13 are identical to No. 13
391
392         j = get_vlc2(&q->gb, q->envelope_quant_index[vlc_index-1].table,
393                      q->envelope_quant_index[vlc_index-1].bits,2);
394         quant_index_table[i] = quant_index_table[i-1] + j - 12;    //differential encoding
395     }
396 }
397
398 /**
399  * Create the quant value table.
400  *
401  * @param q                 pointer to the COOKContext
402  * @param quant_value_table pointer to the array
403  */
404
405 static void inline dequant_envelope(COOKContext *q, int* quant_index_table,
406                                     float* quant_value_table){
407
408     int i;
409     for(i=0 ; i < q->total_subbands ; i++){
410         quant_value_table[i] = q->rootpow2tab[quant_index_table[i]+63];
411     }
412 }
413
414 /**
415  * Calculate the category and category_index vector.
416  *
417  * @param q                     pointer to the COOKContext
418  * @param quant_index_table     pointer to the array
419  * @param category              pointer to the category array
420  * @param category_index        pointer to the category_index array
421  */
422
423 static void categorize(COOKContext *q, int* quant_index_table,
424                        int* category, int* category_index){
425     int exp_idx, bias, tmpbias, bits_left, num_bits, index, v, i, j;
426     int exp_index2[102];
427     int exp_index1[102];
428
429     int tmp_categorize_array1[128];
430     int tmp_categorize_array1_idx=0;
431     int tmp_categorize_array2[128];
432     int tmp_categorize_array2_idx=0;
433     int category_index_size=0;
434
435     bits_left =  q->bits_per_subpacket - get_bits_count(&q->gb);
436
437     if(bits_left > q->samples_per_channel) {
438         bits_left = q->samples_per_channel +
439                     ((bits_left - q->samples_per_channel)*5)/8;
440         //av_log(NULL, AV_LOG_ERROR, "bits_left = %d\n",bits_left);
441     }
442
443     memset(&exp_index1,0,102*sizeof(int));
444     memset(&exp_index2,0,102*sizeof(int));
445     memset(&tmp_categorize_array1,0,128*sizeof(int));
446     memset(&tmp_categorize_array2,0,128*sizeof(int));
447
448     bias=-32;
449
450     /* Estimate bias. */
451     for (i=32 ; i>0 ; i=i/2){
452         num_bits = 0;
453         index = 0;
454         for (j=q->total_subbands ; j>0 ; j--){
455             exp_idx = (i - quant_index_table[index] + bias) / 2;
456             if (exp_idx<0){
457                 exp_idx=0;
458             } else if(exp_idx >7) {
459                 exp_idx=7;
460             }
461             index++;
462             num_bits+=expbits_tab[exp_idx];
463         }
464         if(num_bits >= bits_left - 32){
465             bias+=i;
466         }
467     }
468
469     /* Calculate total number of bits. */
470     num_bits=0;
471     for (i=0 ; i<q->total_subbands ; i++) {
472         exp_idx = (bias - quant_index_table[i]) / 2;
473         if (exp_idx<0) {
474             exp_idx=0;
475         } else if(exp_idx >7) {
476             exp_idx=7;
477         }
478         num_bits += expbits_tab[exp_idx];
479         exp_index1[i] = exp_idx;
480         exp_index2[i] = exp_idx;
481     }
482     tmpbias = bias = num_bits;
483
484     for (j = 1 ; j < q->numvector_size ; j++) {
485         if (tmpbias + bias > 2*bits_left) {  /* ---> */
486             int max = -999999;
487             index=-1;
488             for (i=0 ; i<q->total_subbands ; i++){
489                 if (exp_index1[i] < 7) {
490                     v = (-2*exp_index1[i]) - quant_index_table[i] - 32;
491                     if ( v >= max) {
492                         max = v;
493                         index = i;
494                     }
495                 }
496             }
497             if(index==-1)break;
498             tmp_categorize_array1[tmp_categorize_array1_idx++] = index;
499             tmpbias -= expbits_tab[exp_index1[index]] -
500                        expbits_tab[exp_index1[index]+1];
501             ++exp_index1[index];
502         } else {  /* <--- */
503             int min = 999999;
504             index=-1;
505             for (i=0 ; i<q->total_subbands ; i++){
506                 if(exp_index2[i] > 0){
507                     v = (-2*exp_index2[i])-quant_index_table[i];
508                     if ( v < min) {
509                         min = v;
510                         index = i;
511                     }
512                 }
513             }
514             if(index == -1)break;
515             tmp_categorize_array2[tmp_categorize_array2_idx++] = index;
516             tmpbias -= expbits_tab[exp_index2[index]] -
517                        expbits_tab[exp_index2[index]-1];
518             --exp_index2[index];
519         }
520     }
521
522     for(i=0 ; i<q->total_subbands ; i++)
523         category[i] = exp_index2[i];
524
525     /* Concatenate the two arrays. */
526     for(i=tmp_categorize_array2_idx-1 ; i >= 0; i--)
527         category_index[category_index_size++] =  tmp_categorize_array2[i];
528
529     for(i=0;i<tmp_categorize_array1_idx;i++)
530         category_index[category_index_size++ ] =  tmp_categorize_array1[i];
531
532     /* FIXME: mc_sich_ra8_20.rm triggers this, not sure with what we
533        should fill the remaining bytes. */
534     for(i=category_index_size;i<q->numvector_size;i++)
535         category_index[i]=0;
536
537 }
538
539
540 /**
541  * Expand the category vector.
542  *
543  * @param q                     pointer to the COOKContext
544  * @param category              pointer to the category array
545  * @param category_index        pointer to the category_index array
546  */
547
548 static void inline expand_category(COOKContext *q, int* category,
549                                    int* category_index){
550     int i;
551     for(i=0 ; i<q->num_vectors ; i++){
552         ++category[category_index[i]];
553     }
554 }
555
556 /**
557  * The real requantization of the mltcoefs
558  *
559  * @param q                     pointer to the COOKContext
560  * @param index                 index
561  * @param band                  current subband
562  * @param quant_value_table     pointer to the array
563  * @param subband_coef_index    array of indexes to quant_centroid_tab
564  * @param subband_coef_noise    use random noise instead of predetermined value
565  * @param mlt_buffer            pointer to the mlt buffer
566  */
567
568
569 static void scalar_dequant(COOKContext *q, int index, int band,
570                            float* quant_value_table, int* subband_coef_index,
571                            int* subband_coef_noise, float* mlt_buffer){
572     int i;
573     float f1;
574
575     for(i=0 ; i<SUBBAND_SIZE ; i++) {
576         if (subband_coef_index[i]) {
577             if (subband_coef_noise[i]) {
578                 f1 = -quant_centroid_tab[index][subband_coef_index[i]];
579             } else {
580                 f1 = quant_centroid_tab[index][subband_coef_index[i]];
581             }
582         } else {
583             /* noise coding if subband_coef_noise[i] == 0 */
584             q->random_state = q->random_state * 214013 + 2531011;    //typical RNG numbers
585             f1 = randsign[(q->random_state/0x1000000)&1] * dither_tab[index]; //>>31
586         }
587         mlt_buffer[band*20+ i] = f1 * quant_value_table[band];
588     }
589 }
590 /**
591  * Unpack the subband_coef_index and subband_coef_noise vectors.
592  *
593  * @param q                     pointer to the COOKContext
594  * @param category              pointer to the category array
595  * @param subband_coef_index    array of indexes to quant_centroid_tab
596  * @param subband_coef_noise    use random noise instead of predetermined value
597  */
598
599 static int unpack_SQVH(COOKContext *q, int category, int* subband_coef_index,
600                        int* subband_coef_noise) {
601     int i,j;
602     int vlc, vd ,tmp, result;
603     int ub;
604     int cb;
605
606     vd = vd_tab[category];
607     result = 0;
608     for(i=0 ; i<vpr_tab[category] ; i++){
609         ub = get_bits_count(&q->gb);
610         vlc = get_vlc2(&q->gb, q->sqvh[category].table, q->sqvh[category].bits, 3);
611         cb = get_bits_count(&q->gb);
612         if (q->bits_per_subpacket < get_bits_count(&q->gb)){
613             vlc = 0;
614             result = 1;
615         }
616         for(j=vd-1 ; j>=0 ; j--){
617             tmp = (vlc * invradix_tab[category])/0x100000;
618             subband_coef_index[vd*i+j] = vlc - tmp * (kmax_tab[category]+1);
619             vlc = tmp;
620         }
621         for(j=0 ; j<vd ; j++){
622             if (subband_coef_index[i*vd + j]) {
623                 if(get_bits_count(&q->gb) < q->bits_per_subpacket){
624                     subband_coef_noise[i*vd+j] = get_bits1(&q->gb);
625                 } else {
626                     result=1;
627                     subband_coef_noise[i*vd+j]=0;
628                 }
629             } else {
630                 subband_coef_noise[i*vd+j]=0;
631             }
632         }
633     }
634     return result;
635 }
636
637
638 /**
639  * Fill the mlt_buffer with mlt coefficients.
640  *
641  * @param q                 pointer to the COOKContext
642  * @param category          pointer to the category array
643  * @param quant_value_table pointer to the array
644  * @param mlt_buffer        pointer to mlt coefficients
645  */
646
647
648 static void decode_vectors(COOKContext* q, int* category,
649                            float* quant_value_table, float* mlt_buffer){
650     /* A zero in this table means that the subband coefficient is
651        random noise coded. */
652     int subband_coef_noise[SUBBAND_SIZE];
653     /* A zero in this table means that the subband coefficient is a
654        positive multiplicator. */
655     int subband_coef_index[SUBBAND_SIZE];
656     int band, j;
657     int index=0;
658
659     for(band=0 ; band<q->total_subbands ; band++){
660         index = category[band];
661         if(category[band] < 7){
662             if(unpack_SQVH(q, category[band], subband_coef_index, subband_coef_noise)){
663                 index=7;
664                 for(j=0 ; j<q->total_subbands ; j++) category[band+j]=7;
665             }
666         }
667         if(index==7) {
668             memset(subband_coef_index, 0, sizeof(subband_coef_index));
669             memset(subband_coef_noise, 0, sizeof(subband_coef_noise));
670         }
671         scalar_dequant(q, index, band, quant_value_table, subband_coef_index,
672                        subband_coef_noise, mlt_buffer);
673     }
674
675     if(q->total_subbands*SUBBAND_SIZE >= q->samples_per_channel){
676         return;
677     }
678 }
679
680
681 /**
682  * function for decoding mono data
683  *
684  * @param q                 pointer to the COOKContext
685  * @param mlt_buffer1       pointer to left channel mlt coefficients
686  * @param mlt_buffer2       pointer to right channel mlt coefficients
687  */
688
689 static void mono_decode(COOKContext *q, float* mlt_buffer) {
690
691     int category_index[128];
692     float quant_value_table[102];
693     int quant_index_table[102];
694     int category[128];
695
696     memset(&category, 0, 128*sizeof(int));
697     memset(&quant_value_table, 0, 102*sizeof(int));
698     memset(&category_index, 0, 128*sizeof(int));
699
700     decode_envelope(q, quant_index_table);
701     q->num_vectors = get_bits(&q->gb,q->log2_numvector_size);
702     dequant_envelope(q, quant_index_table, quant_value_table);
703     categorize(q, quant_index_table, category, category_index);
704     expand_category(q, category, category_index);
705     decode_vectors(q, category, quant_value_table, mlt_buffer);
706 }
707
708
709 /**
710  * The modulated lapped transform, this takes transform coefficients
711  * and transforms them into timedomain samples. This is done through
712  * an FFT-based algorithm with pre- and postrotation steps.
713  * A window and reorder step is also included.
714  *
715  * @param q                 pointer to the COOKContext
716  * @param inbuffer          pointer to the mltcoefficients
717  * @param outbuffer         pointer to the timedomain buffer
718  * @param mlt_tmp           pointer to temporary storage space
719  */
720
721 static void cook_imlt(COOKContext *q, float* inbuffer, float* outbuffer,
722                       float* mlt_tmp){
723     int i;
724
725     /* prerotation */
726     for(i=0 ; i<q->mlt_size ; i+=2){
727         outbuffer[i] = (q->mlt_presin[i/2] * inbuffer[q->mlt_size-1-i]) +
728                        (q->mlt_precos[i/2] * inbuffer[i]);
729         outbuffer[i+1] = (q->mlt_precos[i/2] * inbuffer[q->mlt_size-1-i]) -
730                          (q->mlt_presin[i/2] * inbuffer[i]);
731     }
732
733     /* FFT */
734     ff_fft_permute(&q->fft_ctx, (FFTComplex *) outbuffer);
735     ff_fft_calc (&q->fft_ctx, (FFTComplex *) outbuffer);
736
737     /* postrotation */
738     for(i=0 ; i<q->mlt_size ; i+=2){
739         mlt_tmp[i] =               (q->mlt_postcos[(q->mlt_size-1-i)/2] * outbuffer[i+1]) +
740                                    (q->mlt_postcos[i/2] * outbuffer[i]);
741         mlt_tmp[q->mlt_size-1-i] = (q->mlt_postcos[(q->mlt_size-1-i)/2] * outbuffer[i]) -
742                                    (q->mlt_postcos[i/2] * outbuffer[i+1]);
743     }
744
745     /* window and reorder */
746     for(i=0 ; i<q->mlt_size/2 ; i++){
747         outbuffer[i] = mlt_tmp[q->mlt_size/2-1-i] * q->mlt_window[i];
748         outbuffer[q->mlt_size-1-i]= mlt_tmp[q->mlt_size/2-1-i] *
749                                     q->mlt_window[q->mlt_size-1-i];
750         outbuffer[q->mlt_size+i]= mlt_tmp[q->mlt_size/2+i] *
751                                   q->mlt_window[q->mlt_size-1-i];
752         outbuffer[2*q->mlt_size-1-i]= -(mlt_tmp[q->mlt_size/2+i] *
753                                       q->mlt_window[i]);
754     }
755 }
756
757
758 /**
759  * the actual requantization of the timedomain samples
760  *
761  * @param q                 pointer to the COOKContext
762  * @param buffer            pointer to the timedomain buffer
763  * @param gain_index        index for the block multiplier
764  * @param gain_index_next   index for the next block multiplier
765  */
766
767 static void interpolate(COOKContext *q, float* buffer,
768                         int gain_index, int gain_index_next){
769     int i;
770     float fc1, fc2;
771     fc1 = q->pow2tab[gain_index+63];
772
773     if(gain_index == gain_index_next){              //static gain
774         for(i=0 ; i<q->gain_size_factor ; i++){
775             buffer[i]*=fc1;
776         }
777         return;
778     } else {                                        //smooth gain
779         fc2 = q->gain_table[11 + (gain_index_next-gain_index)];
780         for(i=0 ; i<q->gain_size_factor ; i++){
781             buffer[i]*=fc1;
782             fc1*=fc2;
783         }
784         return;
785     }
786 }
787
788 /**
789  * timedomain requantization of the timedomain samples
790  *
791  * @param q                 pointer to the COOKContext
792  * @param buffer            pointer to the timedomain buffer
793  * @param gain_now          current gain structure
794  * @param gain_previous     previous gain structure
795  */
796
797 static void gain_window(COOKContext *q, float* buffer, COOKgain* gain_now,
798                         COOKgain* gain_previous){
799     int i, index;
800     int gain_index[9];
801     int tmp_gain_index;
802
803     gain_index[8]=0;
804     index = gain_previous->size;
805     for (i=7 ; i>=0 ; i--) {
806         if(index && gain_previous->qidx_table1[index-1]==i) {
807             gain_index[i] = gain_previous->qidx_table2[index-1];
808             index--;
809         } else {
810             gain_index[i]=gain_index[i+1];
811         }
812     }
813     /* This is applied to the to be previous data buffer. */
814     for(i=0;i<8;i++){
815         interpolate(q, &buffer[q->samples_per_channel+q->gain_size_factor*i],
816                     gain_index[i], gain_index[i+1]);
817     }
818
819     tmp_gain_index = gain_index[0];
820     index = gain_now->size;
821     for (i=7 ; i>=0 ; i--) {
822         if(index && gain_now->qidx_table1[index-1]==i) {
823             gain_index[i]= gain_now->qidx_table2[index-1];
824             index--;
825         } else {
826             gain_index[i]=gain_index[i+1];
827         }
828     }
829
830     /* This is applied to the to be current block. */
831     for(i=0;i<8;i++){
832         interpolate(q, &buffer[i*q->gain_size_factor],
833                     tmp_gain_index+gain_index[i],
834                     tmp_gain_index+gain_index[i+1]);
835     }
836 }
837
838
839 /**
840  * mlt overlapping and buffer management
841  *
842  * @param q                 pointer to the COOKContext
843  * @param buffer            pointer to the timedomain buffer
844  * @param gain_now          current gain structure
845  * @param gain_previous     previous gain structure
846  * @param previous_buffer   pointer to the previous buffer to be used for overlapping
847  *
848  */
849
850 static void gain_compensate(COOKContext *q, float* buffer, COOKgain* gain_now,
851                             COOKgain* gain_previous, float* previous_buffer) {
852     int i;
853     if((gain_now->size  || gain_previous->size)) {
854         gain_window(q, buffer, gain_now, gain_previous);
855     }
856
857     /* Overlap with the previous block. */
858     for(i=0 ; i<q->samples_per_channel ; i++) buffer[i]+=previous_buffer[i];
859
860     /* Save away the current to be previous block. */
861     memcpy(previous_buffer, buffer+q->samples_per_channel,
862            sizeof(float)*q->samples_per_channel);
863 }
864
865
866 /**
867  * function for getting the jointstereo coupling information
868  *
869  * @param q                 pointer to the COOKContext
870  * @param decouple_tab      decoupling array
871  *
872  */
873
874 static void decouple_info(COOKContext *q, int* decouple_tab){
875     int length, i;
876
877     if(get_bits1(&q->gb)) {
878         if(cplband[q->js_subband_start] > cplband[q->subbands-1]) return;
879
880         length = cplband[q->subbands-1] - cplband[q->js_subband_start] + 1;
881         for (i=0 ; i<length ; i++) {
882             decouple_tab[cplband[q->js_subband_start] + i] = get_vlc2(&q->gb, q->ccpl.table, q->ccpl.bits, 2);
883         }
884         return;
885     }
886
887     if(cplband[q->js_subband_start] > cplband[q->subbands-1]) return;
888
889     length = cplband[q->subbands-1] - cplband[q->js_subband_start] + 1;
890     for (i=0 ; i<length ; i++) {
891        decouple_tab[cplband[q->js_subband_start] + i] = get_bits(&q->gb, q->js_vlc_bits);
892     }
893     return;
894 }
895
896
897 /**
898  * function for decoding joint stereo data
899  *
900  * @param q                 pointer to the COOKContext
901  * @param mlt_buffer1       pointer to left channel mlt coefficients
902  * @param mlt_buffer2       pointer to right channel mlt coefficients
903  */
904
905 static void joint_decode(COOKContext *q, float* mlt_buffer1,
906                          float* mlt_buffer2) {
907     int i,j;
908     int decouple_tab[SUBBAND_SIZE];
909     float decode_buffer[1060];
910     int idx, cpl_tmp,tmp_idx;
911     float f1,f2;
912     float* cplscale;
913
914     memset(decouple_tab, 0, sizeof(decouple_tab));
915     memset(decode_buffer, 0, sizeof(decode_buffer));
916
917     /* Make sure the buffers are zeroed out. */
918     memset(mlt_buffer1,0, 1024*sizeof(float));
919     memset(mlt_buffer2,0, 1024*sizeof(float));
920     decouple_info(q, decouple_tab);
921     mono_decode(q, decode_buffer);
922
923     /* The two channels are stored interleaved in decode_buffer. */
924     for (i=0 ; i<q->js_subband_start ; i++) {
925         for (j=0 ; j<SUBBAND_SIZE ; j++) {
926             mlt_buffer1[i*20+j] = decode_buffer[i*40+j];
927             mlt_buffer2[i*20+j] = decode_buffer[i*40+20+j];
928         }
929     }
930
931     /* When we reach js_subband_start (the higher frequencies)
932        the coefficients are stored in a coupling scheme. */
933     idx = (1 << q->js_vlc_bits) - 1;
934     for (i=q->js_subband_start ; i<q->subbands ; i++) {
935         cpl_tmp = cplband[i];
936         idx -=decouple_tab[cpl_tmp];
937         cplscale = (float*)cplscales[q->js_vlc_bits-2];  //choose decoupler table
938         f1 = cplscale[decouple_tab[cpl_tmp]];
939         f2 = cplscale[idx-1];
940         for (j=0 ; j<SUBBAND_SIZE ; j++) {
941             tmp_idx = ((q->js_subband_start + i)*20)+j;
942             mlt_buffer1[20*i + j] = f1 * decode_buffer[tmp_idx];
943             mlt_buffer2[20*i + j] = f2 * decode_buffer[tmp_idx];
944         }
945         idx = (1 << q->js_vlc_bits) - 1;
946     }
947 }
948
949 /**
950  * Cook subpacket decoding. This function returns one decoded subpacket,
951  * usually 1024 samples per channel.
952  *
953  * @param q                 pointer to the COOKContext
954  * @param inbuffer          pointer to the inbuffer
955  * @param sub_packet_size   subpacket size
956  * @param outbuffer         pointer to the outbuffer
957  */
958
959
960 static int decode_subpacket(COOKContext *q, uint8_t *inbuffer,
961                             int sub_packet_size, int16_t *outbuffer) {
962     int i,j;
963     int value;
964     float* tmp_ptr;
965
966     /* packet dump */
967 //    for (i=0 ; i<sub_packet_size ; i++) {
968 //        av_log(NULL, AV_LOG_ERROR, "%02x", inbuffer[i]);
969 //    }
970 //    av_log(NULL, AV_LOG_ERROR, "\n");
971
972     decode_bytes(inbuffer, q->decoded_bytes_buffer, sub_packet_size);
973     init_get_bits(&q->gb, q->decoded_bytes_buffer, sub_packet_size*8);
974     decode_gain_info(&q->gb, &q->gain_current);
975
976     if(q->nb_channels==2 && q->joint_stereo==1){
977         joint_decode(q, q->decode_buf_ptr[0], q->decode_buf_ptr[2]);
978
979         /* Swap buffer pointers. */
980         tmp_ptr = q->decode_buf_ptr[1];
981         q->decode_buf_ptr[1] = q->decode_buf_ptr[0];
982         q->decode_buf_ptr[0] = tmp_ptr;
983         tmp_ptr = q->decode_buf_ptr[3];
984         q->decode_buf_ptr[3] = q->decode_buf_ptr[2];
985         q->decode_buf_ptr[2] = tmp_ptr;
986
987         /* FIXME: Rethink the gainbuffer handling, maybe a rename?
988            now/previous swap */
989         q->gain_now_ptr = &q->gain_now;
990         q->gain_previous_ptr = &q->gain_previous;
991         for (i=0 ; i<q->nb_channels ; i++){
992
993             cook_imlt(q, q->decode_buf_ptr[i*2], q->mono_mdct_output, q->mlt_tmp);
994             gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
995                             q->gain_previous_ptr, q->previous_buffer_ptr[0]);
996
997             /* Swap out the previous buffer. */
998             tmp_ptr = q->previous_buffer_ptr[0];
999             q->previous_buffer_ptr[0] = q->previous_buffer_ptr[1];
1000             q->previous_buffer_ptr[1] = tmp_ptr;
1001
1002             /* Clip and convert the floats to 16 bits. */
1003             for (j=0 ; j<q->samples_per_frame ; j++){
1004                 value = lrintf(q->mono_mdct_output[j]);
1005                 if(value < -32768) value = -32768;
1006                 else if(value > 32767) value = 32767;
1007                 outbuffer[2*j+i] = value;
1008             }
1009         }
1010
1011         memcpy(&q->gain_now, &q->gain_previous, sizeof(COOKgain));
1012         memcpy(&q->gain_previous, &q->gain_current, sizeof(COOKgain));
1013
1014     } else if (q->nb_channels==2 && q->joint_stereo==0) {
1015             /* channel 0 */
1016             mono_decode(q, q->decode_buf_ptr2[0]);
1017
1018             tmp_ptr = q->decode_buf_ptr2[0];
1019             q->decode_buf_ptr2[0] = q->decode_buf_ptr2[1];
1020             q->decode_buf_ptr2[1] = tmp_ptr;
1021
1022             memcpy(&q->gain_channel1[0], &q->gain_current ,sizeof(COOKgain));
1023             q->gain_now_ptr = &q->gain_channel1[0];
1024             q->gain_previous_ptr = &q->gain_channel1[1];
1025
1026             cook_imlt(q, q->decode_buf_ptr2[0], q->mono_mdct_output,q->mlt_tmp);
1027             gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
1028                             q->gain_previous_ptr, q->mono_previous_buffer1);
1029
1030             memcpy(&q->gain_channel1[1], &q->gain_channel1[0],sizeof(COOKgain));
1031
1032
1033             for (j=0 ; j<q->samples_per_frame ; j++){
1034                 value = lrintf(q->mono_mdct_output[j]);
1035                 if(value < -32768) value = -32768;
1036                 else if(value > 32767) value = 32767;
1037                 outbuffer[2*j+1] = value;
1038             }
1039
1040             /* channel 1 */
1041             //av_log(NULL,AV_LOG_ERROR,"bits = %d\n",get_bits_count(&q->gb));
1042             init_get_bits(&q->gb, q->decoded_bytes_buffer, sub_packet_size*8+q->bits_per_subpacket);
1043
1044             q->gain_now_ptr = &q->gain_channel2[0];
1045             q->gain_previous_ptr = &q->gain_channel2[1];
1046
1047             decode_gain_info(&q->gb, &q->gain_channel2[0]);
1048             mono_decode(q, q->decode_buf_ptr[0]);
1049
1050             tmp_ptr = q->decode_buf_ptr[0];
1051             q->decode_buf_ptr[0] = q->decode_buf_ptr[1];
1052             q->decode_buf_ptr[1] = tmp_ptr;
1053
1054             cook_imlt(q, q->decode_buf_ptr[0], q->mono_mdct_output,q->mlt_tmp);
1055             gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
1056                             q->gain_previous_ptr, q->mono_previous_buffer2);
1057
1058             /* Swap out the previous buffer. */
1059             tmp_ptr = q->previous_buffer_ptr[0];
1060             q->previous_buffer_ptr[0] = q->previous_buffer_ptr[1];
1061             q->previous_buffer_ptr[1] = tmp_ptr;
1062
1063             memcpy(&q->gain_channel2[1], &q->gain_channel2[0] ,sizeof(COOKgain));
1064
1065             for (j=0 ; j<q->samples_per_frame ; j++){
1066                 value = lrintf(q->mono_mdct_output[j]);
1067                 if(value < -32768) value = -32768;
1068                 else if(value > 32767) value = 32767;
1069                 outbuffer[2*j] = value;
1070             }
1071
1072     } else {
1073         mono_decode(q, q->decode_buf_ptr[0]);
1074
1075         /* Swap buffer pointers. */
1076         tmp_ptr = q->decode_buf_ptr[1];
1077         q->decode_buf_ptr[1] = q->decode_buf_ptr[0];
1078         q->decode_buf_ptr[0] = tmp_ptr;
1079
1080         /* FIXME: Rethink the gainbuffer handling, maybe a rename?
1081            now/previous swap */
1082         q->gain_now_ptr = &q->gain_now;
1083         q->gain_previous_ptr = &q->gain_previous;
1084
1085         cook_imlt(q, q->decode_buf_ptr[0], q->mono_mdct_output,q->mlt_tmp);
1086         gain_compensate(q, q->mono_mdct_output, q->gain_now_ptr,
1087                         q->gain_previous_ptr, q->mono_previous_buffer1);
1088
1089         /* Clip and convert the floats to 16 bits */
1090         for (j=0 ; j<q->samples_per_frame ; j++){
1091             value = lrintf(q->mono_mdct_output[j]);
1092             if(value < -32768) value = -32768;
1093             else if(value > 32767) value = 32767;
1094             outbuffer[j] = value;
1095         }
1096         memcpy(&q->gain_now, &q->gain_previous, sizeof(COOKgain));
1097         memcpy(&q->gain_previous, &q->gain_current, sizeof(COOKgain));
1098     }
1099     return q->samples_per_frame * sizeof(int16_t);
1100 }
1101
1102
1103 /**
1104  * Cook frame decoding
1105  *
1106  * @param avctx     pointer to the AVCodecContext
1107  */
1108
1109 static int cook_decode_frame(AVCodecContext *avctx,
1110             void *data, int *data_size,
1111             uint8_t *buf, int buf_size) {
1112     COOKContext *q = avctx->priv_data;
1113
1114     if (buf_size < avctx->block_align)
1115         return buf_size;
1116
1117     *data_size = decode_subpacket(q, buf, avctx->block_align, data);
1118
1119     return avctx->block_align;
1120 }
1121
1122 #ifdef COOKDEBUG
1123 static void dump_cook_context(COOKContext *q, COOKextradata *e)
1124 {
1125     //int i=0;
1126 #define PRINT(a,b) av_log(NULL,AV_LOG_ERROR," %s = %d\n", a, b);
1127     av_log(NULL,AV_LOG_ERROR,"COOKextradata\n");
1128     av_log(NULL,AV_LOG_ERROR,"cookversion=%x\n",e->cookversion);
1129     if (e->cookversion > MONO_COOK2) {
1130         PRINT("js_subband_start",e->js_subband_start);
1131         PRINT("js_vlc_bits",e->js_vlc_bits);
1132     }
1133     av_log(NULL,AV_LOG_ERROR,"COOKContext\n");
1134     PRINT("nb_channels",q->nb_channels);
1135     PRINT("bit_rate",q->bit_rate);
1136     PRINT("sample_rate",q->sample_rate);
1137     PRINT("samples_per_channel",q->samples_per_channel);
1138     PRINT("samples_per_frame",q->samples_per_frame);
1139     PRINT("subbands",q->subbands);
1140     PRINT("random_state",q->random_state);
1141     PRINT("mlt_size",q->mlt_size);
1142     PRINT("js_subband_start",q->js_subband_start);
1143     PRINT("log2_numvector_size",q->log2_numvector_size);
1144     PRINT("numvector_size",q->numvector_size);
1145     PRINT("total_subbands",q->total_subbands);
1146 }
1147 #endif
1148
1149 /**
1150  * Cook initialization
1151  *
1152  * @param avctx     pointer to the AVCodecContext
1153  */
1154
1155 static int cook_decode_init(AVCodecContext *avctx)
1156 {
1157     COOKextradata *e = avctx->extradata;
1158     COOKContext *q = avctx->priv_data;
1159
1160     /* Take care of the codec specific extradata. */
1161     if (avctx->extradata_size <= 0) {
1162         av_log(NULL,AV_LOG_ERROR,"Necessary extradata missing!\n");
1163         return -1;
1164     } else {
1165         /* 8 for mono, 16 for stereo, ? for multichannel
1166            Swap to right endianness so we don't need to care later on. */
1167         av_log(NULL,AV_LOG_DEBUG,"codecdata_length=%d\n",avctx->extradata_size);
1168         if (avctx->extradata_size >= 8){
1169             e->cookversion = be2me_32(e->cookversion);
1170             e->samples_per_frame = be2me_16(e->samples_per_frame);
1171             e->subbands = be2me_16(e->subbands);
1172         }
1173         if (avctx->extradata_size >= 16){
1174             e->js_subband_start = be2me_16(e->js_subband_start);
1175             e->js_vlc_bits = be2me_16(e->js_vlc_bits);
1176         }
1177     }
1178
1179     /* Take data from the AVCodecContext (RM container). */
1180     q->sample_rate = avctx->sample_rate;
1181     q->nb_channels = avctx->channels;
1182     q->bit_rate = avctx->bit_rate;
1183
1184     /* Initialize state. */
1185     q->random_state = 1;
1186
1187     /* Initialize extradata related variables. */
1188     q->samples_per_channel = e->samples_per_frame / q->nb_channels;
1189     q->samples_per_frame = e->samples_per_frame;
1190     q->subbands = e->subbands;
1191     q->bits_per_subpacket = avctx->block_align * 8;
1192
1193     /* Initialize default data states. */
1194     q->js_subband_start = 0;
1195     q->log2_numvector_size = 5;
1196     q->total_subbands = q->subbands;
1197
1198     /* Initialize version-dependent variables */
1199     av_log(NULL,AV_LOG_DEBUG,"e->cookversion=%x\n",e->cookversion);
1200     switch (e->cookversion) {
1201         case MONO_COOK1:
1202             if (q->nb_channels != 1) {
1203                 av_log(NULL,AV_LOG_ERROR,"Container channels != 1, report sample!\n");
1204                 return -1;
1205             }
1206             av_log(NULL,AV_LOG_DEBUG,"MONO_COOK1\n");
1207             break;
1208         case MONO_COOK2:
1209             if (q->nb_channels != 1) {
1210                 q->joint_stereo = 0;
1211                 q->bits_per_subpacket = q->bits_per_subpacket/2;
1212             }
1213             av_log(NULL,AV_LOG_DEBUG,"MONO_COOK2\n");
1214             break;
1215         case JOINT_STEREO:
1216             if (q->nb_channels != 2) {
1217                 av_log(NULL,AV_LOG_ERROR,"Container channels != 2, report sample!\n");
1218                 return -1;
1219             }
1220             av_log(NULL,AV_LOG_DEBUG,"JOINT_STEREO\n");
1221             if (avctx->extradata_size >= 16){
1222                 q->total_subbands = q->subbands + e->js_subband_start;
1223                 q->js_subband_start = e->js_subband_start;
1224                 q->joint_stereo = 1;
1225                 q->js_vlc_bits = e->js_vlc_bits;
1226             }
1227             if (q->samples_per_channel > 256) {
1228                 q->log2_numvector_size  = 6;
1229             }
1230             if (q->samples_per_channel > 512) {
1231                 q->log2_numvector_size  = 7;
1232             }
1233             break;
1234         case MC_COOK:
1235             av_log(NULL,AV_LOG_ERROR,"MC_COOK not supported!\n");
1236             return -1;
1237             break;
1238         default:
1239             av_log(NULL,AV_LOG_ERROR,"Unknown Cook version, report sample!\n");
1240             return -1;
1241             break;
1242     }
1243
1244     /* Initialize variable relations */
1245     q->mlt_size = q->samples_per_channel;
1246     q->numvector_size = (1 << q->log2_numvector_size);
1247
1248     /* Generate tables */
1249     init_rootpow2table(q);
1250     init_pow2table(q);
1251     init_gain_table(q);
1252
1253     if (init_cook_vlc_tables(q) != 0)
1254         return -1;
1255
1256
1257     if(avctx->block_align >= UINT_MAX/2)
1258         return -1;
1259
1260     /* Pad the databuffer with FF_INPUT_BUFFER_PADDING_SIZE,
1261        this is for the bitstreamreader. */
1262     if ((q->decoded_bytes_buffer = av_mallocz((avctx->block_align+(4-avctx->block_align%4) + FF_INPUT_BUFFER_PADDING_SIZE)*sizeof(uint8_t)))  == NULL)
1263         return -1;
1264
1265     q->decode_buf_ptr[0] = q->decode_buffer_1;
1266     q->decode_buf_ptr[1] = q->decode_buffer_2;
1267     q->decode_buf_ptr[2] = q->decode_buffer_3;
1268     q->decode_buf_ptr[3] = q->decode_buffer_4;
1269
1270     q->decode_buf_ptr2[0] = q->decode_buffer_3;
1271     q->decode_buf_ptr2[1] = q->decode_buffer_4;
1272
1273     q->previous_buffer_ptr[0] = q->mono_previous_buffer1;
1274     q->previous_buffer_ptr[1] = q->mono_previous_buffer2;
1275
1276     /* Initialize transform. */
1277     if ( init_cook_mlt(q) == 0 )
1278         return -1;
1279
1280     /* Try to catch some obviously faulty streams, othervise it might be exploitable */
1281     if (q->total_subbands > 53) {
1282         av_log(NULL,AV_LOG_ERROR,"total_subbands > 53, report sample!\n");
1283         return -1;
1284     }
1285     if (q->subbands > 50) {
1286         av_log(NULL,AV_LOG_ERROR,"subbands > 50, report sample!\n");
1287         return -1;
1288     }
1289     if ((q->samples_per_channel == 256) || (q->samples_per_channel == 512) || (q->samples_per_channel == 1024)) {
1290     } else {
1291         av_log(NULL,AV_LOG_ERROR,"unknown amount of samples_per_channel = %d, report sample!\n",q->samples_per_channel);
1292         return -1;
1293     }
1294
1295 #ifdef COOKDEBUG
1296     dump_cook_context(q,e);
1297 #endif
1298     return 0;
1299 }
1300
1301
1302 AVCodec cook_decoder =
1303 {
1304     .name = "cook",
1305     .type = CODEC_TYPE_AUDIO,
1306     .id = CODEC_ID_COOK,
1307     .priv_data_size = sizeof(COOKContext),
1308     .init = cook_decode_init,
1309     .close = cook_decode_close,
1310     .decode = cook_decode_frame,
1311 };