avcodec/cavsdec: Propagate error codes inside decode_mb_i()
[ffmpeg.git] / libavcodec / aaccoder_trellis.h
1 /*
2  * AAC encoder trellis codebook selector
3  * Copyright (C) 2008-2009 Konstantin Shishkov
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * @file
24  * AAC encoder trellis codebook selector
25  * @author Konstantin Shishkov
26  */
27
28 /**
29  * This file contains a template for the codebook_trellis_rate selector function.
30  * It needs to be provided, externally, as an already included declaration,
31  * the following functions from aacenc_quantization/util.h. They're not included
32  * explicitly here to make it possible to provide alternative implementations:
33  *  - quantize_band_cost_bits
34  *  - abs_pow34_v
35  */
36
37 #ifndef AVCODEC_AACCODER_TRELLIS_H
38 #define AVCODEC_AACCODER_TRELLIS_H
39
40 #include <float.h>
41 #include "libavutil/mathematics.h"
42 #include "avcodec.h"
43 #include "put_bits.h"
44 #include "aac.h"
45 #include "aacenc.h"
46 #include "aactab.h"
47 #include "aacenctab.h"
48
49 /**
50  * structure used in optimal codebook search
51  */
52 typedef struct TrellisBandCodingPath {
53     int prev_idx; ///< pointer to the previous path point
54     float cost;   ///< path cost
55     int run;
56 } TrellisBandCodingPath;
57
58
59 static void codebook_trellis_rate(AACEncContext *s, SingleChannelElement *sce,
60                                   int win, int group_len, const float lambda)
61 {
62     TrellisBandCodingPath path[120][CB_TOT_ALL];
63     int w, swb, cb, start, size;
64     int i, j;
65     const int max_sfb  = sce->ics.max_sfb;
66     const int run_bits = sce->ics.num_windows == 1 ? 5 : 3;
67     const int run_esc  = (1 << run_bits) - 1;
68     int idx, ppos, count;
69     int stackrun[120], stackcb[120], stack_len;
70     float next_minbits = INFINITY;
71     int next_mincb = 0;
72
73     s->abs_pow34(s->scoefs, sce->coeffs, 1024);
74     start = win*128;
75     for (cb = 0; cb < CB_TOT_ALL; cb++) {
76         path[0][cb].cost     = run_bits+4;
77         path[0][cb].prev_idx = -1;
78         path[0][cb].run      = 0;
79     }
80     for (swb = 0; swb < max_sfb; swb++) {
81         size = sce->ics.swb_sizes[swb];
82         if (sce->zeroes[win*16 + swb]) {
83             float cost_stay_here = path[swb][0].cost;
84             float cost_get_here  = next_minbits + run_bits + 4;
85             if (   run_value_bits[sce->ics.num_windows == 8][path[swb][0].run]
86                 != run_value_bits[sce->ics.num_windows == 8][path[swb][0].run+1])
87                 cost_stay_here += run_bits;
88             if (cost_get_here < cost_stay_here) {
89                 path[swb+1][0].prev_idx = next_mincb;
90                 path[swb+1][0].cost     = cost_get_here;
91                 path[swb+1][0].run      = 1;
92             } else {
93                 path[swb+1][0].prev_idx = 0;
94                 path[swb+1][0].cost     = cost_stay_here;
95                 path[swb+1][0].run      = path[swb][0].run + 1;
96             }
97             next_minbits = path[swb+1][0].cost;
98             next_mincb = 0;
99             for (cb = 1; cb < CB_TOT_ALL; cb++) {
100                 path[swb+1][cb].cost = 61450;
101                 path[swb+1][cb].prev_idx = -1;
102                 path[swb+1][cb].run = 0;
103             }
104         } else {
105             float minbits = next_minbits;
106             int mincb = next_mincb;
107             int startcb = sce->band_type[win*16+swb];
108             startcb = aac_cb_in_map[startcb];
109             next_minbits = INFINITY;
110             next_mincb = 0;
111             for (cb = 0; cb < startcb; cb++) {
112                 path[swb+1][cb].cost = 61450;
113                 path[swb+1][cb].prev_idx = -1;
114                 path[swb+1][cb].run = 0;
115             }
116             for (cb = startcb; cb < CB_TOT_ALL; cb++) {
117                 float cost_stay_here, cost_get_here;
118                 float bits = 0.0f;
119                 if (cb >= 12 && sce->band_type[win*16+swb] != aac_cb_out_map[cb]) {
120                     path[swb+1][cb].cost = 61450;
121                     path[swb+1][cb].prev_idx = -1;
122                     path[swb+1][cb].run = 0;
123                     continue;
124                 }
125                 for (w = 0; w < group_len; w++) {
126                     bits += quantize_band_cost_bits(s, &sce->coeffs[start + w*128],
127                                                &s->scoefs[start + w*128], size,
128                                                sce->sf_idx[win*16+swb],
129                                                aac_cb_out_map[cb],
130                                                0, INFINITY, NULL, NULL, 0);
131                 }
132                 cost_stay_here = path[swb][cb].cost + bits;
133                 cost_get_here  = minbits            + bits + run_bits + 4;
134                 if (   run_value_bits[sce->ics.num_windows == 8][path[swb][cb].run]
135                     != run_value_bits[sce->ics.num_windows == 8][path[swb][cb].run+1])
136                     cost_stay_here += run_bits;
137                 if (cost_get_here < cost_stay_here) {
138                     path[swb+1][cb].prev_idx = mincb;
139                     path[swb+1][cb].cost     = cost_get_here;
140                     path[swb+1][cb].run      = 1;
141                 } else {
142                     path[swb+1][cb].prev_idx = cb;
143                     path[swb+1][cb].cost     = cost_stay_here;
144                     path[swb+1][cb].run      = path[swb][cb].run + 1;
145                 }
146                 if (path[swb+1][cb].cost < next_minbits) {
147                     next_minbits = path[swb+1][cb].cost;
148                     next_mincb = cb;
149                 }
150             }
151         }
152         start += sce->ics.swb_sizes[swb];
153     }
154
155     //convert resulting path from backward-linked list
156     stack_len = 0;
157     idx       = 0;
158     for (cb = 1; cb < CB_TOT_ALL; cb++)
159         if (path[max_sfb][cb].cost < path[max_sfb][idx].cost)
160             idx = cb;
161     ppos = max_sfb;
162     while (ppos > 0) {
163         av_assert1(idx >= 0);
164         cb = idx;
165         stackrun[stack_len] = path[ppos][cb].run;
166         stackcb [stack_len] = cb;
167         idx = path[ppos-path[ppos][cb].run+1][cb].prev_idx;
168         ppos -= path[ppos][cb].run;
169         stack_len++;
170     }
171     //perform actual band info encoding
172     start = 0;
173     for (i = stack_len - 1; i >= 0; i--) {
174         cb = aac_cb_out_map[stackcb[i]];
175         put_bits(&s->pb, 4, cb);
176         count = stackrun[i];
177         memset(sce->zeroes + win*16 + start, !cb, count);
178         //XXX: memset when band_type is also uint8_t
179         for (j = 0; j < count; j++) {
180             sce->band_type[win*16 + start] = cb;
181             start++;
182         }
183         while (count >= run_esc) {
184             put_bits(&s->pb, run_bits, run_esc);
185             count -= run_esc;
186         }
187         put_bits(&s->pb, run_bits, count);
188     }
189 }
190
191
192 #endif /* AVCODEC_AACCODER_TRELLIS_H */