Decrease returning probe score for WAV demuxer to avoid
[ffmpeg.git] / libavcodec / h264.c
1 /*
2  * H.26L/H.264/AVC/JVT/14496-10/... encoder/decoder
3  * Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * @file h264.c
24  * H.264 / AVC / MPEG4 part10 codec.
25  * @author Michael Niedermayer <michaelni@gmx.at>
26  */
27
28 #include "dsputil.h"
29 #include "avcodec.h"
30 #include "mpegvideo.h"
31 #include "h264.h"
32 #include "h264data.h"
33 #include "h264_parser.h"
34 #include "golomb.h"
35 #include "rectangle.h"
36
37 #include "cabac.h"
38 #ifdef ARCH_X86
39 #include "i386/h264_i386.h"
40 #endif
41
42 //#undef NDEBUG
43 #include <assert.h>
44
45 /**
46  * Value of Picture.reference when Picture is not a reference picture, but
47  * is held for delayed output.
48  */
49 #define DELAYED_PIC_REF 4
50
51 static VLC coeff_token_vlc[4];
52 static VLC chroma_dc_coeff_token_vlc;
53
54 static VLC total_zeros_vlc[15];
55 static VLC chroma_dc_total_zeros_vlc[3];
56
57 static VLC run_vlc[6];
58 static VLC run7_vlc;
59
60 static void svq3_luma_dc_dequant_idct_c(DCTELEM *block, int qp);
61 static void svq3_add_idct_c(uint8_t *dst, DCTELEM *block, int stride, int qp, int dc);
62 static void filter_mb( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint8_t *img_cb, uint8_t *img_cr, unsigned int linesize, unsigned int uvlinesize);
63 static void filter_mb_fast( H264Context *h, int mb_x, int mb_y, uint8_t *img_y, uint8_t *img_cb, uint8_t *img_cr, unsigned int linesize, unsigned int uvlinesize);
64
65 static av_always_inline uint32_t pack16to32(int a, int b){
66 #ifdef WORDS_BIGENDIAN
67    return (b&0xFFFF) + (a<<16);
68 #else
69    return (a&0xFFFF) + (b<<16);
70 #endif
71 }
72
73 const uint8_t ff_rem6[52]={
74 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
75 };
76
77 const uint8_t ff_div6[52]={
78 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 8, 8, 8, 8,
79 };
80
81
82 static void fill_caches(H264Context *h, int mb_type, int for_deblock){
83     MpegEncContext * const s = &h->s;
84     const int mb_xy= s->mb_x + s->mb_y*s->mb_stride;
85     int topleft_xy, top_xy, topright_xy, left_xy[2];
86     int topleft_type, top_type, topright_type, left_type[2];
87     int left_block[8];
88     int topleft_partition= -1;
89     int i;
90
91     top_xy     = mb_xy  - (s->mb_stride << FIELD_PICTURE);
92
93     //FIXME deblocking could skip the intra and nnz parts.
94     if(for_deblock && (h->slice_num == 1 || h->slice_table[mb_xy] == h->slice_table[top_xy]) && !FRAME_MBAFF)
95         return;
96
97     /* Wow, what a mess, why didn't they simplify the interlacing & intra
98      * stuff, I can't imagine that these complex rules are worth it. */
99
100     topleft_xy = top_xy - 1;
101     topright_xy= top_xy + 1;
102     left_xy[1] = left_xy[0] = mb_xy-1;
103     left_block[0]= 0;
104     left_block[1]= 1;
105     left_block[2]= 2;
106     left_block[3]= 3;
107     left_block[4]= 7;
108     left_block[5]= 10;
109     left_block[6]= 8;
110     left_block[7]= 11;
111     if(FRAME_MBAFF){
112         const int pair_xy          = s->mb_x     + (s->mb_y & ~1)*s->mb_stride;
113         const int top_pair_xy      = pair_xy     - s->mb_stride;
114         const int topleft_pair_xy  = top_pair_xy - 1;
115         const int topright_pair_xy = top_pair_xy + 1;
116         const int topleft_mb_frame_flag  = !IS_INTERLACED(s->current_picture.mb_type[topleft_pair_xy]);
117         const int top_mb_frame_flag      = !IS_INTERLACED(s->current_picture.mb_type[top_pair_xy]);
118         const int topright_mb_frame_flag = !IS_INTERLACED(s->current_picture.mb_type[topright_pair_xy]);
119         const int left_mb_frame_flag = !IS_INTERLACED(s->current_picture.mb_type[pair_xy-1]);
120         const int curr_mb_frame_flag = !IS_INTERLACED(mb_type);
121         const int bottom = (s->mb_y & 1);
122         tprintf(s->avctx, "fill_caches: curr_mb_frame_flag:%d, left_mb_frame_flag:%d, topleft_mb_frame_flag:%d, top_mb_frame_flag:%d, topright_mb_frame_flag:%d\n", curr_mb_frame_flag, left_mb_frame_flag, topleft_mb_frame_flag, top_mb_frame_flag, topright_mb_frame_flag);
123         if (bottom
124                 ? !curr_mb_frame_flag // bottom macroblock
125                 : (!curr_mb_frame_flag && !top_mb_frame_flag) // top macroblock
126                 ) {
127             top_xy -= s->mb_stride;
128         }
129         if (bottom
130                 ? !curr_mb_frame_flag // bottom macroblock
131                 : (!curr_mb_frame_flag && !topleft_mb_frame_flag) // top macroblock
132                 ) {
133             topleft_xy -= s->mb_stride;
134         } else if(bottom && curr_mb_frame_flag && !left_mb_frame_flag) {
135             topleft_xy += s->mb_stride;
136             // take topleft mv from the middle of the mb, as opposed to all other modes which use the bottom-right partition
137             topleft_partition = 0;
138         }
139         if (bottom
140                 ? !curr_mb_frame_flag // bottom macroblock
141                 : (!curr_mb_frame_flag && !topright_mb_frame_flag) // top macroblock
142                 ) {
143             topright_xy -= s->mb_stride;
144         }
145         if (left_mb_frame_flag != curr_mb_frame_flag) {
146             left_xy[1] = left_xy[0] = pair_xy - 1;
147             if (curr_mb_frame_flag) {
148                 if (bottom) {
149                     left_block[0]= 2;
150                     left_block[1]= 2;
151                     left_block[2]= 3;
152                     left_block[3]= 3;
153                     left_block[4]= 8;
154                     left_block[5]= 11;
155                     left_block[6]= 8;
156                     left_block[7]= 11;
157                 } else {
158                     left_block[0]= 0;
159                     left_block[1]= 0;
160                     left_block[2]= 1;
161                     left_block[3]= 1;
162                     left_block[4]= 7;
163                     left_block[5]= 10;
164                     left_block[6]= 7;
165                     left_block[7]= 10;
166                 }
167             } else {
168                 left_xy[1] += s->mb_stride;
169                 //left_block[0]= 0;
170                 left_block[1]= 2;
171                 left_block[2]= 0;
172                 left_block[3]= 2;
173                 //left_block[4]= 7;
174                 left_block[5]= 10;
175                 left_block[6]= 7;
176                 left_block[7]= 10;
177             }
178         }
179     }
180
181     h->top_mb_xy = top_xy;
182     h->left_mb_xy[0] = left_xy[0];
183     h->left_mb_xy[1] = left_xy[1];
184     if(for_deblock){
185         topleft_type = 0;
186         topright_type = 0;
187         top_type     = h->slice_table[top_xy     ] < 255 ? s->current_picture.mb_type[top_xy]     : 0;
188         left_type[0] = h->slice_table[left_xy[0] ] < 255 ? s->current_picture.mb_type[left_xy[0]] : 0;
189         left_type[1] = h->slice_table[left_xy[1] ] < 255 ? s->current_picture.mb_type[left_xy[1]] : 0;
190
191         if(FRAME_MBAFF && !IS_INTRA(mb_type)){
192             int list;
193             int v = *(uint16_t*)&h->non_zero_count[mb_xy][14];
194             for(i=0; i<16; i++)
195                 h->non_zero_count_cache[scan8[i]] = (v>>i)&1;
196             for(list=0; list<h->list_count; list++){
197                 if(USES_LIST(mb_type,list)){
198                     uint32_t *src = (uint32_t*)s->current_picture.motion_val[list][h->mb2b_xy[mb_xy]];
199                     uint32_t *dst = (uint32_t*)h->mv_cache[list][scan8[0]];
200                     int8_t *ref = &s->current_picture.ref_index[list][h->mb2b8_xy[mb_xy]];
201                     for(i=0; i<4; i++, dst+=8, src+=h->b_stride){
202                         dst[0] = src[0];
203                         dst[1] = src[1];
204                         dst[2] = src[2];
205                         dst[3] = src[3];
206                     }
207                     *(uint32_t*)&h->ref_cache[list][scan8[ 0]] =
208                     *(uint32_t*)&h->ref_cache[list][scan8[ 2]] = pack16to32(ref[0],ref[1])*0x0101;
209                     ref += h->b8_stride;
210                     *(uint32_t*)&h->ref_cache[list][scan8[ 8]] =
211                     *(uint32_t*)&h->ref_cache[list][scan8[10]] = pack16to32(ref[0],ref[1])*0x0101;
212                 }else{
213                     fill_rectangle(&h-> mv_cache[list][scan8[ 0]], 4, 4, 8, 0, 4);
214                     fill_rectangle(&h->ref_cache[list][scan8[ 0]], 4, 4, 8, (uint8_t)LIST_NOT_USED, 1);
215                 }
216             }
217         }
218     }else{
219         topleft_type = h->slice_table[topleft_xy ] == h->slice_num ? s->current_picture.mb_type[topleft_xy] : 0;
220         top_type     = h->slice_table[top_xy     ] == h->slice_num ? s->current_picture.mb_type[top_xy]     : 0;
221         topright_type= h->slice_table[topright_xy] == h->slice_num ? s->current_picture.mb_type[topright_xy]: 0;
222         left_type[0] = h->slice_table[left_xy[0] ] == h->slice_num ? s->current_picture.mb_type[left_xy[0]] : 0;
223         left_type[1] = h->slice_table[left_xy[1] ] == h->slice_num ? s->current_picture.mb_type[left_xy[1]] : 0;
224     }
225
226     if(IS_INTRA(mb_type)){
227         h->topleft_samples_available=
228         h->top_samples_available=
229         h->left_samples_available= 0xFFFF;
230         h->topright_samples_available= 0xEEEA;
231
232         if(!IS_INTRA(top_type) && (top_type==0 || h->pps.constrained_intra_pred)){
233             h->topleft_samples_available= 0xB3FF;
234             h->top_samples_available= 0x33FF;
235             h->topright_samples_available= 0x26EA;
236         }
237         for(i=0; i<2; i++){
238             if(!IS_INTRA(left_type[i]) && (left_type[i]==0 || h->pps.constrained_intra_pred)){
239                 h->topleft_samples_available&= 0xDF5F;
240                 h->left_samples_available&= 0x5F5F;
241             }
242         }
243
244         if(!IS_INTRA(topleft_type) && (topleft_type==0 || h->pps.constrained_intra_pred))
245             h->topleft_samples_available&= 0x7FFF;
246
247         if(!IS_INTRA(topright_type) && (topright_type==0 || h->pps.constrained_intra_pred))
248             h->topright_samples_available&= 0xFBFF;
249
250         if(IS_INTRA4x4(mb_type)){
251             if(IS_INTRA4x4(top_type)){
252                 h->intra4x4_pred_mode_cache[4+8*0]= h->intra4x4_pred_mode[top_xy][4];
253                 h->intra4x4_pred_mode_cache[5+8*0]= h->intra4x4_pred_mode[top_xy][5];
254                 h->intra4x4_pred_mode_cache[6+8*0]= h->intra4x4_pred_mode[top_xy][6];
255                 h->intra4x4_pred_mode_cache[7+8*0]= h->intra4x4_pred_mode[top_xy][3];
256             }else{
257                 int pred;
258                 if(!top_type || (IS_INTER(top_type) && h->pps.constrained_intra_pred))
259                     pred= -1;
260                 else{
261                     pred= 2;
262                 }
263                 h->intra4x4_pred_mode_cache[4+8*0]=
264                 h->intra4x4_pred_mode_cache[5+8*0]=
265                 h->intra4x4_pred_mode_cache[6+8*0]=
266                 h->intra4x4_pred_mode_cache[7+8*0]= pred;
267             }
268             for(i=0; i<2; i++){
269                 if(IS_INTRA4x4(left_type[i])){
270                     h->intra4x4_pred_mode_cache[3+8*1 + 2*8*i]= h->intra4x4_pred_mode[left_xy[i]][left_block[0+2*i]];
271                     h->intra4x4_pred_mode_cache[3+8*2 + 2*8*i]= h->intra4x4_pred_mode[left_xy[i]][left_block[1+2*i]];
272                 }else{
273                     int pred;
274                     if(!left_type[i] || (IS_INTER(left_type[i]) && h->pps.constrained_intra_pred))
275                         pred= -1;
276                     else{
277                         pred= 2;
278                     }
279                     h->intra4x4_pred_mode_cache[3+8*1 + 2*8*i]=
280                     h->intra4x4_pred_mode_cache[3+8*2 + 2*8*i]= pred;
281                 }
282             }
283         }
284     }
285
286
287 /*
288 0 . T T. T T T T
289 1 L . .L . . . .
290 2 L . .L . . . .
291 3 . T TL . . . .
292 4 L . .L . . . .
293 5 L . .. . . . .
294 */
295 //FIXME constraint_intra_pred & partitioning & nnz (lets hope this is just a typo in the spec)
296     if(top_type){
297         h->non_zero_count_cache[4+8*0]= h->non_zero_count[top_xy][4];
298         h->non_zero_count_cache[5+8*0]= h->non_zero_count[top_xy][5];
299         h->non_zero_count_cache[6+8*0]= h->non_zero_count[top_xy][6];
300         h->non_zero_count_cache[7+8*0]= h->non_zero_count[top_xy][3];
301
302         h->non_zero_count_cache[1+8*0]= h->non_zero_count[top_xy][9];
303         h->non_zero_count_cache[2+8*0]= h->non_zero_count[top_xy][8];
304
305         h->non_zero_count_cache[1+8*3]= h->non_zero_count[top_xy][12];
306         h->non_zero_count_cache[2+8*3]= h->non_zero_count[top_xy][11];
307
308     }else{
309         h->non_zero_count_cache[4+8*0]=
310         h->non_zero_count_cache[5+8*0]=
311         h->non_zero_count_cache[6+8*0]=
312         h->non_zero_count_cache[7+8*0]=
313
314         h->non_zero_count_cache[1+8*0]=
315         h->non_zero_count_cache[2+8*0]=
316
317         h->non_zero_count_cache[1+8*3]=
318         h->non_zero_count_cache[2+8*3]= h->pps.cabac && !IS_INTRA(mb_type) ? 0 : 64;
319
320     }
321
322     for (i=0; i<2; i++) {
323         if(left_type[i]){
324             h->non_zero_count_cache[3+8*1 + 2*8*i]= h->non_zero_count[left_xy[i]][left_block[0+2*i]];
325             h->non_zero_count_cache[3+8*2 + 2*8*i]= h->non_zero_count[left_xy[i]][left_block[1+2*i]];
326             h->non_zero_count_cache[0+8*1 +   8*i]= h->non_zero_count[left_xy[i]][left_block[4+2*i]];
327             h->non_zero_count_cache[0+8*4 +   8*i]= h->non_zero_count[left_xy[i]][left_block[5+2*i]];
328         }else{
329             h->non_zero_count_cache[3+8*1 + 2*8*i]=
330             h->non_zero_count_cache[3+8*2 + 2*8*i]=
331             h->non_zero_count_cache[0+8*1 +   8*i]=
332             h->non_zero_count_cache[0+8*4 +   8*i]= h->pps.cabac && !IS_INTRA(mb_type) ? 0 : 64;
333         }
334     }
335
336     if( h->pps.cabac ) {
337         // top_cbp
338         if(top_type) {
339             h->top_cbp = h->cbp_table[top_xy];
340         } else if(IS_INTRA(mb_type)) {
341             h->top_cbp = 0x1C0;
342         } else {
343             h->top_cbp = 0;
344         }
345         // left_cbp
346         if (left_type[0]) {
347             h->left_cbp = h->cbp_table[left_xy[0]] & 0x1f0;
348         } else if(IS_INTRA(mb_type)) {
349             h->left_cbp = 0x1C0;
350         } else {
351             h->left_cbp = 0;
352         }
353         if (left_type[0]) {
354             h->left_cbp |= ((h->cbp_table[left_xy[0]]>>((left_block[0]&(~1))+1))&0x1) << 1;
355         }
356         if (left_type[1]) {
357             h->left_cbp |= ((h->cbp_table[left_xy[1]]>>((left_block[2]&(~1))+1))&0x1) << 3;
358         }
359     }
360
361 #if 1
362     if(IS_INTER(mb_type) || IS_DIRECT(mb_type)){
363         int list;
364         for(list=0; list<h->list_count; list++){
365             if(!USES_LIST(mb_type, list) && !IS_DIRECT(mb_type) && !h->deblocking_filter){
366                 /*if(!h->mv_cache_clean[list]){
367                     memset(h->mv_cache [list],  0, 8*5*2*sizeof(int16_t)); //FIXME clean only input? clean at all?
368                     memset(h->ref_cache[list], PART_NOT_AVAILABLE, 8*5*sizeof(int8_t));
369                     h->mv_cache_clean[list]= 1;
370                 }*/
371                 continue;
372             }
373             h->mv_cache_clean[list]= 0;
374
375             if(USES_LIST(top_type, list)){
376                 const int b_xy= h->mb2b_xy[top_xy] + 3*h->b_stride;
377                 const int b8_xy= h->mb2b8_xy[top_xy] + h->b8_stride;
378                 *(uint32_t*)h->mv_cache[list][scan8[0] + 0 - 1*8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy + 0];
379                 *(uint32_t*)h->mv_cache[list][scan8[0] + 1 - 1*8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy + 1];
380                 *(uint32_t*)h->mv_cache[list][scan8[0] + 2 - 1*8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy + 2];
381                 *(uint32_t*)h->mv_cache[list][scan8[0] + 3 - 1*8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy + 3];
382                 h->ref_cache[list][scan8[0] + 0 - 1*8]=
383                 h->ref_cache[list][scan8[0] + 1 - 1*8]= s->current_picture.ref_index[list][b8_xy + 0];
384                 h->ref_cache[list][scan8[0] + 2 - 1*8]=
385                 h->ref_cache[list][scan8[0] + 3 - 1*8]= s->current_picture.ref_index[list][b8_xy + 1];
386             }else{
387                 *(uint32_t*)h->mv_cache [list][scan8[0] + 0 - 1*8]=
388                 *(uint32_t*)h->mv_cache [list][scan8[0] + 1 - 1*8]=
389                 *(uint32_t*)h->mv_cache [list][scan8[0] + 2 - 1*8]=
390                 *(uint32_t*)h->mv_cache [list][scan8[0] + 3 - 1*8]= 0;
391                 *(uint32_t*)&h->ref_cache[list][scan8[0] + 0 - 1*8]= ((top_type ? LIST_NOT_USED : PART_NOT_AVAILABLE)&0xFF)*0x01010101;
392             }
393
394             for(i=0; i<2; i++){
395                 int cache_idx = scan8[0] - 1 + i*2*8;
396                 if(USES_LIST(left_type[i], list)){
397                     const int b_xy= h->mb2b_xy[left_xy[i]] + 3;
398                     const int b8_xy= h->mb2b8_xy[left_xy[i]] + 1;
399                     *(uint32_t*)h->mv_cache[list][cache_idx  ]= *(uint32_t*)s->current_picture.motion_val[list][b_xy + h->b_stride*left_block[0+i*2]];
400                     *(uint32_t*)h->mv_cache[list][cache_idx+8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy + h->b_stride*left_block[1+i*2]];
401                     h->ref_cache[list][cache_idx  ]= s->current_picture.ref_index[list][b8_xy + h->b8_stride*(left_block[0+i*2]>>1)];
402                     h->ref_cache[list][cache_idx+8]= s->current_picture.ref_index[list][b8_xy + h->b8_stride*(left_block[1+i*2]>>1)];
403                 }else{
404                     *(uint32_t*)h->mv_cache [list][cache_idx  ]=
405                     *(uint32_t*)h->mv_cache [list][cache_idx+8]= 0;
406                     h->ref_cache[list][cache_idx  ]=
407                     h->ref_cache[list][cache_idx+8]= left_type[i] ? LIST_NOT_USED : PART_NOT_AVAILABLE;
408                 }
409             }
410
411             if((for_deblock || (IS_DIRECT(mb_type) && !h->direct_spatial_mv_pred)) && !FRAME_MBAFF)
412                 continue;
413
414             if(USES_LIST(topleft_type, list)){
415                 const int b_xy = h->mb2b_xy[topleft_xy] + 3 + h->b_stride + (topleft_partition & 2*h->b_stride);
416                 const int b8_xy= h->mb2b8_xy[topleft_xy] + 1 + (topleft_partition & h->b8_stride);
417                 *(uint32_t*)h->mv_cache[list][scan8[0] - 1 - 1*8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy];
418                 h->ref_cache[list][scan8[0] - 1 - 1*8]= s->current_picture.ref_index[list][b8_xy];
419             }else{
420                 *(uint32_t*)h->mv_cache[list][scan8[0] - 1 - 1*8]= 0;
421                 h->ref_cache[list][scan8[0] - 1 - 1*8]= topleft_type ? LIST_NOT_USED : PART_NOT_AVAILABLE;
422             }
423
424             if(USES_LIST(topright_type, list)){
425                 const int b_xy= h->mb2b_xy[topright_xy] + 3*h->b_stride;
426                 const int b8_xy= h->mb2b8_xy[topright_xy] + h->b8_stride;
427                 *(uint32_t*)h->mv_cache[list][scan8[0] + 4 - 1*8]= *(uint32_t*)s->current_picture.motion_val[list][b_xy];
428                 h->ref_cache[list][scan8[0] + 4 - 1*8]= s->current_picture.ref_index[list][b8_xy];
429             }else{
430                 *(uint32_t*)h->mv_cache [list][scan8[0] + 4 - 1*8]= 0;
431                 h->ref_cache[list][scan8[0] + 4 - 1*8]= topright_type ? LIST_NOT_USED : PART_NOT_AVAILABLE;
432             }
433
434             if((IS_SKIP(mb_type) || IS_DIRECT(mb_type)) && !FRAME_MBAFF)
435                 continue;
436
437             h->ref_cache[list][scan8[5 ]+1] =
438             h->ref_cache[list][scan8[7 ]+1] =
439             h->ref_cache[list][scan8[13]+1] =  //FIXME remove past 3 (init somewhere else)
440             h->ref_cache[list][scan8[4 ]] =
441             h->ref_cache[list][scan8[12]] = PART_NOT_AVAILABLE;
442             *(uint32_t*)h->mv_cache [list][scan8[5 ]+1]=
443             *(uint32_t*)h->mv_cache [list][scan8[7 ]+1]=
444             *(uint32_t*)h->mv_cache [list][scan8[13]+1]= //FIXME remove past 3 (init somewhere else)
445             *(uint32_t*)h->mv_cache [list][scan8[4 ]]=
446             *(uint32_t*)h->mv_cache [list][scan8[12]]= 0;
447
448             if( h->pps.cabac ) {
449                 /* XXX beurk, Load mvd */
450                 if(USES_LIST(top_type, list)){
451                     const int b_xy= h->mb2b_xy[top_xy] + 3*h->b_stride;
452                     *(uint32_t*)h->mvd_cache[list][scan8[0] + 0 - 1*8]= *(uint32_t*)h->mvd_table[list][b_xy + 0];
453                     *(uint32_t*)h->mvd_cache[list][scan8[0] + 1 - 1*8]= *(uint32_t*)h->mvd_table[list][b_xy + 1];
454                     *(uint32_t*)h->mvd_cache[list][scan8[0] + 2 - 1*8]= *(uint32_t*)h->mvd_table[list][b_xy + 2];
455                     *(uint32_t*)h->mvd_cache[list][scan8[0] + 3 - 1*8]= *(uint32_t*)h->mvd_table[list][b_xy + 3];
456                 }else{
457                     *(uint32_t*)h->mvd_cache [list][scan8[0] + 0 - 1*8]=
458                     *(uint32_t*)h->mvd_cache [list][scan8[0] + 1 - 1*8]=
459                     *(uint32_t*)h->mvd_cache [list][scan8[0] + 2 - 1*8]=
460                     *(uint32_t*)h->mvd_cache [list][scan8[0] + 3 - 1*8]= 0;
461                 }
462                 if(USES_LIST(left_type[0], list)){
463                     const int b_xy= h->mb2b_xy[left_xy[0]] + 3;
464                     *(uint32_t*)h->mvd_cache[list][scan8[0] - 1 + 0*8]= *(uint32_t*)h->mvd_table[list][b_xy + h->b_stride*left_block[0]];
465                     *(uint32_t*)h->mvd_cache[list][scan8[0] - 1 + 1*8]= *(uint32_t*)h->mvd_table[list][b_xy + h->b_stride*left_block[1]];
466                 }else{
467                     *(uint32_t*)h->mvd_cache [list][scan8[0] - 1 + 0*8]=
468                     *(uint32_t*)h->mvd_cache [list][scan8[0] - 1 + 1*8]= 0;
469                 }
470                 if(USES_LIST(left_type[1], list)){
471                     const int b_xy= h->mb2b_xy[left_xy[1]] + 3;
472                     *(uint32_t*)h->mvd_cache[list][scan8[0] - 1 + 2*8]= *(uint32_t*)h->mvd_table[list][b_xy + h->b_stride*left_block[2]];
473                     *(uint32_t*)h->mvd_cache[list][scan8[0] - 1 + 3*8]= *(uint32_t*)h->mvd_table[list][b_xy + h->b_stride*left_block[3]];
474                 }else{
475                     *(uint32_t*)h->mvd_cache [list][scan8[0] - 1 + 2*8]=
476                     *(uint32_t*)h->mvd_cache [list][scan8[0] - 1 + 3*8]= 0;
477                 }
478                 *(uint32_t*)h->mvd_cache [list][scan8[5 ]+1]=
479                 *(uint32_t*)h->mvd_cache [list][scan8[7 ]+1]=
480                 *(uint32_t*)h->mvd_cache [list][scan8[13]+1]= //FIXME remove past 3 (init somewhere else)
481                 *(uint32_t*)h->mvd_cache [list][scan8[4 ]]=
482                 *(uint32_t*)h->mvd_cache [list][scan8[12]]= 0;
483
484                 if(h->slice_type == FF_B_TYPE){
485                     fill_rectangle(&h->direct_cache[scan8[0]], 4, 4, 8, 0, 1);
486
487                     if(IS_DIRECT(top_type)){
488                         *(uint32_t*)&h->direct_cache[scan8[0] - 1*8]= 0x01010101;
489                     }else if(IS_8X8(top_type)){
490                         int b8_xy = h->mb2b8_xy[top_xy] + h->b8_stride;
491                         h->direct_cache[scan8[0] + 0 - 1*8]= h->direct_table[b8_xy];
492                         h->direct_cache[scan8[0] + 2 - 1*8]= h->direct_table[b8_xy + 1];
493                     }else{
494                         *(uint32_t*)&h->direct_cache[scan8[0] - 1*8]= 0;
495                     }
496
497                     if(IS_DIRECT(left_type[0]))
498                         h->direct_cache[scan8[0] - 1 + 0*8]= 1;
499                     else if(IS_8X8(left_type[0]))
500                         h->direct_cache[scan8[0] - 1 + 0*8]= h->direct_table[h->mb2b8_xy[left_xy[0]] + 1 + h->b8_stride*(left_block[0]>>1)];
501                     else
502                         h->direct_cache[scan8[0] - 1 + 0*8]= 0;
503
504                     if(IS_DIRECT(left_type[1]))
505                         h->direct_cache[scan8[0] - 1 + 2*8]= 1;
506                     else if(IS_8X8(left_type[1]))
507                         h->direct_cache[scan8[0] - 1 + 2*8]= h->direct_table[h->mb2b8_xy[left_xy[1]] + 1 + h->b8_stride*(left_block[2]>>1)];
508                     else
509                         h->direct_cache[scan8[0] - 1 + 2*8]= 0;
510                 }
511             }
512
513             if(FRAME_MBAFF){
514 #define MAP_MVS\
515                     MAP_F2F(scan8[0] - 1 - 1*8, topleft_type)\
516                     MAP_F2F(scan8[0] + 0 - 1*8, top_type)\
517                     MAP_F2F(scan8[0] + 1 - 1*8, top_type)\
518                     MAP_F2F(scan8[0] + 2 - 1*8, top_type)\
519                     MAP_F2F(scan8[0] + 3 - 1*8, top_type)\
520                     MAP_F2F(scan8[0] + 4 - 1*8, topright_type)\
521                     MAP_F2F(scan8[0] - 1 + 0*8, left_type[0])\
522                     MAP_F2F(scan8[0] - 1 + 1*8, left_type[0])\
523                     MAP_F2F(scan8[0] - 1 + 2*8, left_type[1])\
524                     MAP_F2F(scan8[0] - 1 + 3*8, left_type[1])
525                 if(MB_FIELD){
526 #define MAP_F2F(idx, mb_type)\
527                     if(!IS_INTERLACED(mb_type) && h->ref_cache[list][idx] >= 0){\
528                         h->ref_cache[list][idx] <<= 1;\
529                         h->mv_cache[list][idx][1] /= 2;\
530                         h->mvd_cache[list][idx][1] /= 2;\
531                     }
532                     MAP_MVS
533 #undef MAP_F2F
534                 }else{
535 #define MAP_F2F(idx, mb_type)\
536                     if(IS_INTERLACED(mb_type) && h->ref_cache[list][idx] >= 0){\
537                         h->ref_cache[list][idx] >>= 1;\
538                         h->mv_cache[list][idx][1] <<= 1;\
539                         h->mvd_cache[list][idx][1] <<= 1;\
540                     }
541                     MAP_MVS
542 #undef MAP_F2F
543                 }
544             }
545         }
546     }
547 #endif
548
549     h->neighbor_transform_size= !!IS_8x8DCT(top_type) + !!IS_8x8DCT(left_type[0]);
550 }
551
552 static inline void write_back_intra_pred_mode(H264Context *h){
553     MpegEncContext * const s = &h->s;
554     const int mb_xy= s->mb_x + s->mb_y*s->mb_stride;
555
556     h->intra4x4_pred_mode[mb_xy][0]= h->intra4x4_pred_mode_cache[7+8*1];
557     h->intra4x4_pred_mode[mb_xy][1]= h->intra4x4_pred_mode_cache[7+8*2];
558     h->intra4x4_pred_mode[mb_xy][2]= h->intra4x4_pred_mode_cache[7+8*3];
559     h->intra4x4_pred_mode[mb_xy][3]= h->intra4x4_pred_mode_cache[7+8*4];
560     h->intra4x4_pred_mode[mb_xy][4]= h->intra4x4_pred_mode_cache[4+8*4];
561     h->intra4x4_pred_mode[mb_xy][5]= h->intra4x4_pred_mode_cache[5+8*4];
562     h->intra4x4_pred_mode[mb_xy][6]= h->intra4x4_pred_mode_cache[6+8*4];
563 }
564
565 /**
566  * checks if the top & left blocks are available if needed & changes the dc mode so it only uses the available blocks.
567  */
568 static inline int check_intra4x4_pred_mode(H264Context *h){
569     MpegEncContext * const s = &h->s;
570     static const int8_t top [12]= {-1, 0,LEFT_DC_PRED,-1,-1,-1,-1,-1, 0};
571     static const int8_t left[12]= { 0,-1, TOP_DC_PRED, 0,-1,-1,-1, 0,-1,DC_128_PRED};
572     int i;
573
574     if(!(h->top_samples_available&0x8000)){
575         for(i=0; i<4; i++){
576             int status= top[ h->intra4x4_pred_mode_cache[scan8[0] + i] ];
577             if(status<0){
578                 av_log(h->s.avctx, AV_LOG_ERROR, "top block unavailable for requested intra4x4 mode %d at %d %d\n", status, s->mb_x, s->mb_y);
579                 return -1;
580             } else if(status){
581                 h->intra4x4_pred_mode_cache[scan8[0] + i]= status;
582             }
583         }
584     }
585
586     if(!(h->left_samples_available&0x8000)){
587         for(i=0; i<4; i++){
588             int status= left[ h->intra4x4_pred_mode_cache[scan8[0] + 8*i] ];
589             if(status<0){
590                 av_log(h->s.avctx, AV_LOG_ERROR, "left block unavailable for requested intra4x4 mode %d at %d %d\n", status, s->mb_x, s->mb_y);
591                 return -1;
592             } else if(status){
593                 h->intra4x4_pred_mode_cache[scan8[0] + 8*i]= status;
594             }
595         }
596     }
597
598     return 0;
599 } //FIXME cleanup like next
600
601 /**
602  * checks if the top & left blocks are available if needed & changes the dc mode so it only uses the available blocks.
603  */
604 static inline int check_intra_pred_mode(H264Context *h, int mode){
605     MpegEncContext * const s = &h->s;
606     static const int8_t top [7]= {LEFT_DC_PRED8x8, 1,-1,-1};
607     static const int8_t left[7]= { TOP_DC_PRED8x8,-1, 2,-1,DC_128_PRED8x8};
608
609     if(mode > 6U) {
610         av_log(h->s.avctx, AV_LOG_ERROR, "out of range intra chroma pred mode at %d %d\n", s->mb_x, s->mb_y);
611         return -1;
612     }
613
614     if(!(h->top_samples_available&0x8000)){
615         mode= top[ mode ];
616         if(mode<0){
617             av_log(h->s.avctx, AV_LOG_ERROR, "top block unavailable for requested intra mode at %d %d\n", s->mb_x, s->mb_y);
618             return -1;
619         }
620     }
621
622     if(!(h->left_samples_available&0x8000)){
623         mode= left[ mode ];
624         if(mode<0){
625             av_log(h->s.avctx, AV_LOG_ERROR, "left block unavailable for requested intra mode at %d %d\n", s->mb_x, s->mb_y);
626             return -1;
627         }
628     }
629
630     return mode;
631 }
632
633 /**
634  * gets the predicted intra4x4 prediction mode.
635  */
636 static inline int pred_intra_mode(H264Context *h, int n){
637     const int index8= scan8[n];
638     const int left= h->intra4x4_pred_mode_cache[index8 - 1];
639     const int top = h->intra4x4_pred_mode_cache[index8 - 8];
640     const int min= FFMIN(left, top);
641
642     tprintf(h->s.avctx, "mode:%d %d min:%d\n", left ,top, min);
643
644     if(min<0) return DC_PRED;
645     else      return min;
646 }
647
648 static inline void write_back_non_zero_count(H264Context *h){
649     MpegEncContext * const s = &h->s;
650     const int mb_xy= s->mb_x + s->mb_y*s->mb_stride;
651
652     h->non_zero_count[mb_xy][0]= h->non_zero_count_cache[7+8*1];
653     h->non_zero_count[mb_xy][1]= h->non_zero_count_cache[7+8*2];
654     h->non_zero_count[mb_xy][2]= h->non_zero_count_cache[7+8*3];
655     h->non_zero_count[mb_xy][3]= h->non_zero_count_cache[7+8*4];
656     h->non_zero_count[mb_xy][4]= h->non_zero_count_cache[4+8*4];
657     h->non_zero_count[mb_xy][5]= h->non_zero_count_cache[5+8*4];
658     h->non_zero_count[mb_xy][6]= h->non_zero_count_cache[6+8*4];
659
660     h->non_zero_count[mb_xy][9]= h->non_zero_count_cache[1+8*2];
661     h->non_zero_count[mb_xy][8]= h->non_zero_count_cache[2+8*2];
662     h->non_zero_count[mb_xy][7]= h->non_zero_count_cache[2+8*1];
663
664     h->non_zero_count[mb_xy][12]=h->non_zero_count_cache[1+8*5];
665     h->non_zero_count[mb_xy][11]=h->non_zero_count_cache[2+8*5];
666     h->non_zero_count[mb_xy][10]=h->non_zero_count_cache[2+8*4];
667
668     if(FRAME_MBAFF){
669         // store all luma nnzs, for deblocking
670         int v = 0, i;
671         for(i=0; i<16; i++)
672             v += (!!h->non_zero_count_cache[scan8[i]]) << i;
673         *(uint16_t*)&h->non_zero_count[mb_xy][14] = v;
674     }
675 }
676
677 /**
678  * gets the predicted number of non zero coefficients.
679  * @param n block index
680  */
681 static inline int pred_non_zero_count(H264Context *h, int n){
682     const int index8= scan8[n];
683     const int left= h->non_zero_count_cache[index8 - 1];
684     const int top = h->non_zero_count_cache[index8 - 8];
685     int i= left + top;
686
687     if(i<64) i= (i+1)>>1;
688
689     tprintf(h->s.avctx, "pred_nnz L%X T%X n%d s%d P%X\n", left, top, n, scan8[n], i&31);
690
691     return i&31;
692 }
693
694 static inline int fetch_diagonal_mv(H264Context *h, const int16_t **C, int i, int list, int part_width){
695     const int topright_ref= h->ref_cache[list][ i - 8 + part_width ];
696     MpegEncContext *s = &h->s;
697
698     /* there is no consistent mapping of mvs to neighboring locations that will
699      * make mbaff happy, so we can't move all this logic to fill_caches */
700     if(FRAME_MBAFF){
701         const uint32_t *mb_types = s->current_picture_ptr->mb_type;
702         const int16_t *mv;
703         *(uint32_t*)h->mv_cache[list][scan8[0]-2] = 0;
704         *C = h->mv_cache[list][scan8[0]-2];
705
706         if(!MB_FIELD
707            && (s->mb_y&1) && i < scan8[0]+8 && topright_ref != PART_NOT_AVAILABLE){
708             int topright_xy = s->mb_x + (s->mb_y-1)*s->mb_stride + (i == scan8[0]+3);
709             if(IS_INTERLACED(mb_types[topright_xy])){
710 #define SET_DIAG_MV(MV_OP, REF_OP, X4, Y4)\
711                 const int x4 = X4, y4 = Y4;\
712                 const int mb_type = mb_types[(x4>>2)+(y4>>2)*s->mb_stride];\
713                 if(!USES_LIST(mb_type,list))\
714                     return LIST_NOT_USED;\
715                 mv = s->current_picture_ptr->motion_val[list][x4 + y4*h->b_stride];\
716                 h->mv_cache[list][scan8[0]-2][0] = mv[0];\
717                 h->mv_cache[list][scan8[0]-2][1] = mv[1] MV_OP;\
718                 return s->current_picture_ptr->ref_index[list][(x4>>1) + (y4>>1)*h->b8_stride] REF_OP;
719
720                 SET_DIAG_MV(*2, >>1, s->mb_x*4+(i&7)-4+part_width, s->mb_y*4-1);
721             }
722         }
723         if(topright_ref == PART_NOT_AVAILABLE
724            && ((s->mb_y&1) || i >= scan8[0]+8) && (i&7)==4
725            && h->ref_cache[list][scan8[0]-1] != PART_NOT_AVAILABLE){
726             if(!MB_FIELD
727                && IS_INTERLACED(mb_types[h->left_mb_xy[0]])){
728                 SET_DIAG_MV(*2, >>1, s->mb_x*4-1, (s->mb_y|1)*4+(s->mb_y&1)*2+(i>>4)-1);
729             }
730             if(MB_FIELD
731                && !IS_INTERLACED(mb_types[h->left_mb_xy[0]])
732                && i >= scan8[0]+8){
733                 // leftshift will turn LIST_NOT_USED into PART_NOT_AVAILABLE, but that's ok.
734                 SET_DIAG_MV(/2, <<1, s->mb_x*4-1, (s->mb_y&~1)*4 - 1 + ((i-scan8[0])>>3)*2);
735             }
736         }
737 #undef SET_DIAG_MV
738     }
739
740     if(topright_ref != PART_NOT_AVAILABLE){
741         *C= h->mv_cache[list][ i - 8 + part_width ];
742         return topright_ref;
743     }else{
744         tprintf(s->avctx, "topright MV not available\n");
745
746         *C= h->mv_cache[list][ i - 8 - 1 ];
747         return h->ref_cache[list][ i - 8 - 1 ];
748     }
749 }
750
751 /**
752  * gets the predicted MV.
753  * @param n the block index
754  * @param part_width the width of the partition (4, 8,16) -> (1, 2, 4)
755  * @param mx the x component of the predicted motion vector
756  * @param my the y component of the predicted motion vector
757  */
758 static inline void pred_motion(H264Context * const h, int n, int part_width, int list, int ref, int * const mx, int * const my){
759     const int index8= scan8[n];
760     const int top_ref=      h->ref_cache[list][ index8 - 8 ];
761     const int left_ref=     h->ref_cache[list][ index8 - 1 ];
762     const int16_t * const A= h->mv_cache[list][ index8 - 1 ];
763     const int16_t * const B= h->mv_cache[list][ index8 - 8 ];
764     const int16_t * C;
765     int diagonal_ref, match_count;
766
767     assert(part_width==1 || part_width==2 || part_width==4);
768
769 /* mv_cache
770   B . . A T T T T
771   U . . L . . , .
772   U . . L . . . .
773   U . . L . . , .
774   . . . L . . . .
775 */
776
777     diagonal_ref= fetch_diagonal_mv(h, &C, index8, list, part_width);
778     match_count= (diagonal_ref==ref) + (top_ref==ref) + (left_ref==ref);
779     tprintf(h->s.avctx, "pred_motion match_count=%d\n", match_count);
780     if(match_count > 1){ //most common
781         *mx= mid_pred(A[0], B[0], C[0]);
782         *my= mid_pred(A[1], B[1], C[1]);
783     }else if(match_count==1){
784         if(left_ref==ref){
785             *mx= A[0];
786             *my= A[1];
787         }else if(top_ref==ref){
788             *mx= B[0];
789             *my= B[1];
790         }else{
791             *mx= C[0];
792             *my= C[1];
793         }
794     }else{
795         if(top_ref == PART_NOT_AVAILABLE && diagonal_ref == PART_NOT_AVAILABLE && left_ref != PART_NOT_AVAILABLE){
796             *mx= A[0];
797             *my= A[1];
798         }else{
799             *mx= mid_pred(A[0], B[0], C[0]);
800             *my= mid_pred(A[1], B[1], C[1]);
801         }
802     }
803
804     tprintf(h->s.avctx, "pred_motion (%2d %2d %2d) (%2d %2d %2d) (%2d %2d %2d) -> (%2d %2d %2d) at %2d %2d %d list %d\n", top_ref, B[0], B[1],                    diagonal_ref, C[0], C[1], left_ref, A[0], A[1], ref, *mx, *my, h->s.mb_x, h->s.mb_y, n, list);
805 }
806
807 /**
808  * gets the directionally predicted 16x8 MV.
809  * @param n the block index
810  * @param mx the x component of the predicted motion vector
811  * @param my the y component of the predicted motion vector
812  */
813 static inline void pred_16x8_motion(H264Context * const h, int n, int list, int ref, int * const mx, int * const my){
814     if(n==0){
815         const int top_ref=      h->ref_cache[list][ scan8[0] - 8 ];
816         const int16_t * const B= h->mv_cache[list][ scan8[0] - 8 ];
817
818         tprintf(h->s.avctx, "pred_16x8: (%2d %2d %2d) at %2d %2d %d list %d\n", top_ref, B[0], B[1], h->s.mb_x, h->s.mb_y, n, list);
819
820         if(top_ref == ref){
821             *mx= B[0];
822             *my= B[1];
823             return;
824         }
825     }else{
826         const int left_ref=     h->ref_cache[list][ scan8[8] - 1 ];
827         const int16_t * const A= h->mv_cache[list][ scan8[8] - 1 ];
828
829         tprintf(h->s.avctx, "pred_16x8: (%2d %2d %2d) at %2d %2d %d list %d\n", left_ref, A[0], A[1], h->s.mb_x, h->s.mb_y, n, list);
830
831         if(left_ref == ref){
832             *mx= A[0];
833             *my= A[1];
834             return;
835         }
836     }
837
838     //RARE
839     pred_motion(h, n, 4, list, ref, mx, my);
840 }
841
842 /**
843  * gets the directionally predicted 8x16 MV.
844  * @param n the block index
845  * @param mx the x component of the predicted motion vector
846  * @param my the y component of the predicted motion vector
847  */
848 static inline void pred_8x16_motion(H264Context * const h, int n, int list, int ref, int * const mx, int * const my){
849     if(n==0){
850         const int left_ref=      h->ref_cache[list][ scan8[0] - 1 ];
851         const int16_t * const A=  h->mv_cache[list][ scan8[0] - 1 ];
852
853         tprintf(h->s.avctx, "pred_8x16: (%2d %2d %2d) at %2d %2d %d list %d\n", left_ref, A[0], A[1], h->s.mb_x, h->s.mb_y, n, list);
854
855         if(left_ref == ref){
856             *mx= A[0];
857             *my= A[1];
858             return;
859         }
860     }else{
861         const int16_t * C;
862         int diagonal_ref;
863
864         diagonal_ref= fetch_diagonal_mv(h, &C, scan8[4], list, 2);
865
866         tprintf(h->s.avctx, "pred_8x16: (%2d %2d %2d) at %2d %2d %d list %d\n", diagonal_ref, C[0], C[1], h->s.mb_x, h->s.mb_y, n, list);
867
868         if(diagonal_ref == ref){
869             *mx= C[0];
870             *my= C[1];
871             return;
872         }
873     }
874
875     //RARE
876     pred_motion(h, n, 2, list, ref, mx, my);
877 }
878
879 static inline void pred_pskip_motion(H264Context * const h, int * const mx, int * const my){
880     const int top_ref = h->ref_cache[0][ scan8[0] - 8 ];
881     const int left_ref= h->ref_cache[0][ scan8[0] - 1 ];
882
883     tprintf(h->s.avctx, "pred_pskip: (%d) (%d) at %2d %2d\n", top_ref, left_ref, h->s.mb_x, h->s.mb_y);
884
885     if(top_ref == PART_NOT_AVAILABLE || left_ref == PART_NOT_AVAILABLE
886        || (top_ref == 0  && *(uint32_t*)h->mv_cache[0][ scan8[0] - 8 ] == 0)
887        || (left_ref == 0 && *(uint32_t*)h->mv_cache[0][ scan8[0] - 1 ] == 0)){
888
889         *mx = *my = 0;
890         return;
891     }
892
893     pred_motion(h, 0, 4, 0, 0, mx, my);
894
895     return;
896 }
897
898 static inline void direct_dist_scale_factor(H264Context * const h){
899     const int poc = h->s.current_picture_ptr->poc;
900     const int poc1 = h->ref_list[1][0].poc;
901     int i;
902     for(i=0; i<h->ref_count[0]; i++){
903         int poc0 = h->ref_list[0][i].poc;
904         int td = av_clip(poc1 - poc0, -128, 127);
905         if(td == 0 /* FIXME || pic0 is a long-term ref */){
906             h->dist_scale_factor[i] = 256;
907         }else{
908             int tb = av_clip(poc - poc0, -128, 127);
909             int tx = (16384 + (FFABS(td) >> 1)) / td;
910             h->dist_scale_factor[i] = av_clip((tb*tx + 32) >> 6, -1024, 1023);
911         }
912     }
913     if(FRAME_MBAFF){
914         for(i=0; i<h->ref_count[0]; i++){
915             h->dist_scale_factor_field[2*i] =
916             h->dist_scale_factor_field[2*i+1] = h->dist_scale_factor[i];
917         }
918     }
919 }
920 static inline void direct_ref_list_init(H264Context * const h){
921     MpegEncContext * const s = &h->s;
922     Picture * const ref1 = &h->ref_list[1][0];
923     Picture * const cur = s->current_picture_ptr;
924     int list, i, j;
925     if(cur->pict_type == FF_I_TYPE)
926         cur->ref_count[0] = 0;
927     if(cur->pict_type != FF_B_TYPE)
928         cur->ref_count[1] = 0;
929     for(list=0; list<2; list++){
930         cur->ref_count[list] = h->ref_count[list];
931         for(j=0; j<h->ref_count[list]; j++)
932             cur->ref_poc[list][j] = h->ref_list[list][j].poc;
933     }
934     if(cur->pict_type != FF_B_TYPE || h->direct_spatial_mv_pred)
935         return;
936     for(list=0; list<2; list++){
937         for(i=0; i<ref1->ref_count[list]; i++){
938             const int poc = ref1->ref_poc[list][i];
939             h->map_col_to_list0[list][i] = 0; /* bogus; fills in for missing frames */
940             for(j=0; j<h->ref_count[list]; j++)
941                 if(h->ref_list[list][j].poc == poc){
942                     h->map_col_to_list0[list][i] = j;
943                     break;
944                 }
945         }
946     }
947     if(FRAME_MBAFF){
948         for(list=0; list<2; list++){
949             for(i=0; i<ref1->ref_count[list]; i++){
950                 j = h->map_col_to_list0[list][i];
951                 h->map_col_to_list0_field[list][2*i] = 2*j;
952                 h->map_col_to_list0_field[list][2*i+1] = 2*j+1;
953             }
954         }
955     }
956 }
957
958 static inline void pred_direct_motion(H264Context * const h, int *mb_type){
959     MpegEncContext * const s = &h->s;
960     const int mb_xy =   s->mb_x +   s->mb_y*s->mb_stride;
961     const int b8_xy = 2*s->mb_x + 2*s->mb_y*h->b8_stride;
962     const int b4_xy = 4*s->mb_x + 4*s->mb_y*h->b_stride;
963     const int mb_type_col = h->ref_list[1][0].mb_type[mb_xy];
964     const int16_t (*l1mv0)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[0][b4_xy];
965     const int16_t (*l1mv1)[2] = (const int16_t (*)[2]) &h->ref_list[1][0].motion_val[1][b4_xy];
966     const int8_t *l1ref0 = &h->ref_list[1][0].ref_index[0][b8_xy];
967     const int8_t *l1ref1 = &h->ref_list[1][0].ref_index[1][b8_xy];
968     const int is_b8x8 = IS_8X8(*mb_type);
969     unsigned int sub_mb_type;
970     int i8, i4;
971
972 #define MB_TYPE_16x16_OR_INTRA (MB_TYPE_16x16|MB_TYPE_INTRA4x4|MB_TYPE_INTRA16x16|MB_TYPE_INTRA_PCM)
973     if(IS_8X8(mb_type_col) && !h->sps.direct_8x8_inference_flag){
974         /* FIXME save sub mb types from previous frames (or derive from MVs)
975          * so we know exactly what block size to use */
976         sub_mb_type = MB_TYPE_8x8|MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2; /* B_SUB_4x4 */
977         *mb_type =    MB_TYPE_8x8|MB_TYPE_L0L1;
978     }else if(!is_b8x8 && (mb_type_col & MB_TYPE_16x16_OR_INTRA)){
979         sub_mb_type = MB_TYPE_16x16|MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2; /* B_SUB_8x8 */
980         *mb_type =    MB_TYPE_16x16|MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2; /* B_16x16 */
981     }else{
982         sub_mb_type = MB_TYPE_16x16|MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2; /* B_SUB_8x8 */
983         *mb_type =    MB_TYPE_8x8|MB_TYPE_L0L1;
984     }
985     if(!is_b8x8)
986         *mb_type |= MB_TYPE_DIRECT2;
987     if(MB_FIELD)
988         *mb_type |= MB_TYPE_INTERLACED;
989
990     tprintf(s->avctx, "mb_type = %08x, sub_mb_type = %08x, is_b8x8 = %d, mb_type_col = %08x\n", *mb_type, sub_mb_type, is_b8x8, mb_type_col);
991
992     if(h->direct_spatial_mv_pred){
993         int ref[2];
994         int mv[2][2];
995         int list;
996
997         /* FIXME interlacing + spatial direct uses wrong colocated block positions */
998
999         /* ref = min(neighbors) */
1000         for(list=0; list<2; list++){
1001             int refa = h->ref_cache[list][scan8[0] - 1];
1002             int refb = h->ref_cache[list][scan8[0] - 8];
1003             int refc = h->ref_cache[list][scan8[0] - 8 + 4];
1004             if(refc == -2)
1005                 refc = h->ref_cache[list][scan8[0] - 8 - 1];
1006             ref[list] = refa;
1007             if(ref[list] < 0 || (refb < ref[list] && refb >= 0))
1008                 ref[list] = refb;
1009             if(ref[list] < 0 || (refc < ref[list] && refc >= 0))
1010                 ref[list] = refc;
1011             if(ref[list] < 0)
1012                 ref[list] = -1;
1013         }
1014
1015         if(ref[0] < 0 && ref[1] < 0){
1016             ref[0] = ref[1] = 0;
1017             mv[0][0] = mv[0][1] =
1018             mv[1][0] = mv[1][1] = 0;
1019         }else{
1020             for(list=0; list<2; list++){
1021                 if(ref[list] >= 0)
1022                     pred_motion(h, 0, 4, list, ref[list], &mv[list][0], &mv[list][1]);
1023                 else
1024                     mv[list][0] = mv[list][1] = 0;
1025             }
1026         }
1027
1028         if(ref[1] < 0){
1029             if(!is_b8x8)
1030                 *mb_type &= ~MB_TYPE_L1;
1031             sub_mb_type &= ~MB_TYPE_L1;
1032         }else if(ref[0] < 0){
1033             if(!is_b8x8)
1034                 *mb_type &= ~MB_TYPE_L0;
1035             sub_mb_type &= ~MB_TYPE_L0;
1036         }
1037
1038         if(IS_INTERLACED(*mb_type) != IS_INTERLACED(mb_type_col)){
1039             int pair_xy = s->mb_x + (s->mb_y&~1)*s->mb_stride;
1040             int mb_types_col[2];
1041             int b8_stride = h->b8_stride;
1042             int b4_stride = h->b_stride;
1043
1044             *mb_type = (*mb_type & ~MB_TYPE_16x16) | MB_TYPE_8x8;
1045
1046             if(IS_INTERLACED(*mb_type)){
1047                 mb_types_col[0] = h->ref_list[1][0].mb_type[pair_xy];
1048                 mb_types_col[1] = h->ref_list[1][0].mb_type[pair_xy+s->mb_stride];
1049                 if(s->mb_y&1){
1050                     l1ref0 -= 2*b8_stride;
1051                     l1ref1 -= 2*b8_stride;
1052                     l1mv0 -= 4*b4_stride;
1053                     l1mv1 -= 4*b4_stride;
1054                 }
1055                 b8_stride *= 3;
1056                 b4_stride *= 6;
1057             }else{
1058                 int cur_poc = s->current_picture_ptr->poc;
1059                 int *col_poc = h->ref_list[1]->field_poc;
1060                 int col_parity = FFABS(col_poc[0] - cur_poc) >= FFABS(col_poc[1] - cur_poc);
1061                 int dy = 2*col_parity - (s->mb_y&1);
1062                 mb_types_col[0] =
1063                 mb_types_col[1] = h->ref_list[1][0].mb_type[pair_xy + col_parity*s->mb_stride];
1064                 l1ref0 += dy*b8_stride;
1065                 l1ref1 += dy*b8_stride;
1066                 l1mv0 += 2*dy*b4_stride;
1067                 l1mv1 += 2*dy*b4_stride;
1068                 b8_stride = 0;
1069             }
1070
1071             for(i8=0; i8<4; i8++){
1072                 int x8 = i8&1;
1073                 int y8 = i8>>1;
1074                 int xy8 = x8+y8*b8_stride;
1075                 int xy4 = 3*x8+y8*b4_stride;
1076                 int a=0, b=0;
1077
1078                 if(is_b8x8 && !IS_DIRECT(h->sub_mb_type[i8]))
1079                     continue;
1080                 h->sub_mb_type[i8] = sub_mb_type;
1081
1082                 fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[0], 1);
1083                 fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[1], 1);
1084                 if(!IS_INTRA(mb_types_col[y8])
1085                    && (   (l1ref0[xy8] == 0 && FFABS(l1mv0[xy4][0]) <= 1 && FFABS(l1mv0[xy4][1]) <= 1)
1086                        || (l1ref0[xy8]  < 0 && l1ref1[xy8] == 0 && FFABS(l1mv1[xy4][0]) <= 1 && FFABS(l1mv1[xy4][1]) <= 1))){
1087                     if(ref[0] > 0)
1088                         a= pack16to32(mv[0][0],mv[0][1]);
1089                     if(ref[1] > 0)
1090                         b= pack16to32(mv[1][0],mv[1][1]);
1091                 }else{
1092                     a= pack16to32(mv[0][0],mv[0][1]);
1093                     b= pack16to32(mv[1][0],mv[1][1]);
1094                 }
1095                 fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, a, 4);
1096                 fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, b, 4);
1097             }
1098         }else if(IS_16X16(*mb_type)){
1099             int a=0, b=0;
1100
1101             fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, (uint8_t)ref[0], 1);
1102             fill_rectangle(&h->ref_cache[1][scan8[0]], 4, 4, 8, (uint8_t)ref[1], 1);
1103             if(!IS_INTRA(mb_type_col)
1104                && (   (l1ref0[0] == 0 && FFABS(l1mv0[0][0]) <= 1 && FFABS(l1mv0[0][1]) <= 1)
1105                    || (l1ref0[0]  < 0 && l1ref1[0] == 0 && FFABS(l1mv1[0][0]) <= 1 && FFABS(l1mv1[0][1]) <= 1
1106                        && (h->x264_build>33 || !h->x264_build)))){
1107                 if(ref[0] > 0)
1108                     a= pack16to32(mv[0][0],mv[0][1]);
1109                 if(ref[1] > 0)
1110                     b= pack16to32(mv[1][0],mv[1][1]);
1111             }else{
1112                 a= pack16to32(mv[0][0],mv[0][1]);
1113                 b= pack16to32(mv[1][0],mv[1][1]);
1114             }
1115             fill_rectangle(&h->mv_cache[0][scan8[0]], 4, 4, 8, a, 4);
1116             fill_rectangle(&h->mv_cache[1][scan8[0]], 4, 4, 8, b, 4);
1117         }else{
1118             for(i8=0; i8<4; i8++){
1119                 const int x8 = i8&1;
1120                 const int y8 = i8>>1;
1121
1122                 if(is_b8x8 && !IS_DIRECT(h->sub_mb_type[i8]))
1123                     continue;
1124                 h->sub_mb_type[i8] = sub_mb_type;
1125
1126                 fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, pack16to32(mv[0][0],mv[0][1]), 4);
1127                 fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, pack16to32(mv[1][0],mv[1][1]), 4);
1128                 fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[0], 1);
1129                 fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[1], 1);
1130
1131                 /* col_zero_flag */
1132                 if(!IS_INTRA(mb_type_col) && (   l1ref0[x8 + y8*h->b8_stride] == 0
1133                                               || (l1ref0[x8 + y8*h->b8_stride] < 0 && l1ref1[x8 + y8*h->b8_stride] == 0
1134                                                   && (h->x264_build>33 || !h->x264_build)))){
1135                     const int16_t (*l1mv)[2]= l1ref0[x8 + y8*h->b8_stride] == 0 ? l1mv0 : l1mv1;
1136                     if(IS_SUB_8X8(sub_mb_type)){
1137                         const int16_t *mv_col = l1mv[x8*3 + y8*3*h->b_stride];
1138                         if(FFABS(mv_col[0]) <= 1 && FFABS(mv_col[1]) <= 1){
1139                             if(ref[0] == 0)
1140                                 fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, 0, 4);
1141                             if(ref[1] == 0)
1142                                 fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, 0, 4);
1143                         }
1144                     }else
1145                     for(i4=0; i4<4; i4++){
1146                         const int16_t *mv_col = l1mv[x8*2 + (i4&1) + (y8*2 + (i4>>1))*h->b_stride];
1147                         if(FFABS(mv_col[0]) <= 1 && FFABS(mv_col[1]) <= 1){
1148                             if(ref[0] == 0)
1149                                 *(uint32_t*)h->mv_cache[0][scan8[i8*4+i4]] = 0;
1150                             if(ref[1] == 0)
1151                                 *(uint32_t*)h->mv_cache[1][scan8[i8*4+i4]] = 0;
1152                         }
1153                     }
1154                 }
1155             }
1156         }
1157     }else{ /* direct temporal mv pred */
1158         const int *map_col_to_list0[2] = {h->map_col_to_list0[0], h->map_col_to_list0[1]};
1159         const int *dist_scale_factor = h->dist_scale_factor;
1160
1161         if(FRAME_MBAFF){
1162             if(IS_INTERLACED(*mb_type)){
1163                 map_col_to_list0[0] = h->map_col_to_list0_field[0];
1164                 map_col_to_list0[1] = h->map_col_to_list0_field[1];
1165                 dist_scale_factor = h->dist_scale_factor_field;
1166             }
1167             if(IS_INTERLACED(*mb_type) != IS_INTERLACED(mb_type_col)){
1168                 /* FIXME assumes direct_8x8_inference == 1 */
1169                 const int pair_xy = s->mb_x + (s->mb_y&~1)*s->mb_stride;
1170                 int mb_types_col[2];
1171                 int y_shift;
1172
1173                 *mb_type = MB_TYPE_8x8|MB_TYPE_L0L1
1174                          | (is_b8x8 ? 0 : MB_TYPE_DIRECT2)
1175                          | (*mb_type & MB_TYPE_INTERLACED);
1176                 sub_mb_type = MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2|MB_TYPE_16x16;
1177
1178                 if(IS_INTERLACED(*mb_type)){
1179                     /* frame to field scaling */
1180                     mb_types_col[0] = h->ref_list[1][0].mb_type[pair_xy];
1181                     mb_types_col[1] = h->ref_list[1][0].mb_type[pair_xy+s->mb_stride];
1182                     if(s->mb_y&1){
1183                         l1ref0 -= 2*h->b8_stride;
1184                         l1ref1 -= 2*h->b8_stride;
1185                         l1mv0 -= 4*h->b_stride;
1186                         l1mv1 -= 4*h->b_stride;
1187                     }
1188                     y_shift = 0;
1189
1190                     if(   (mb_types_col[0] & MB_TYPE_16x16_OR_INTRA)
1191                        && (mb_types_col[1] & MB_TYPE_16x16_OR_INTRA)
1192                        && !is_b8x8)
1193                         *mb_type |= MB_TYPE_16x8;
1194                     else
1195                         *mb_type |= MB_TYPE_8x8;
1196                 }else{
1197                     /* field to frame scaling */
1198                     /* col_mb_y = (mb_y&~1) + (topAbsDiffPOC < bottomAbsDiffPOC ? 0 : 1)
1199                      * but in MBAFF, top and bottom POC are equal */
1200                     int dy = (s->mb_y&1) ? 1 : 2;
1201                     mb_types_col[0] =
1202                     mb_types_col[1] = h->ref_list[1][0].mb_type[pair_xy+s->mb_stride];
1203                     l1ref0 += dy*h->b8_stride;
1204                     l1ref1 += dy*h->b8_stride;
1205                     l1mv0 += 2*dy*h->b_stride;
1206                     l1mv1 += 2*dy*h->b_stride;
1207                     y_shift = 2;
1208
1209                     if((mb_types_col[0] & (MB_TYPE_16x16_OR_INTRA|MB_TYPE_16x8))
1210                        && !is_b8x8)
1211                         *mb_type |= MB_TYPE_16x16;
1212                     else
1213                         *mb_type |= MB_TYPE_8x8;
1214                 }
1215
1216                 for(i8=0; i8<4; i8++){
1217                     const int x8 = i8&1;
1218                     const int y8 = i8>>1;
1219                     int ref0, scale;
1220                     const int16_t (*l1mv)[2]= l1mv0;
1221
1222                     if(is_b8x8 && !IS_DIRECT(h->sub_mb_type[i8]))
1223                         continue;
1224                     h->sub_mb_type[i8] = sub_mb_type;
1225
1226                     fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, 0, 1);
1227                     if(IS_INTRA(mb_types_col[y8])){
1228                         fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, 0, 1);
1229                         fill_rectangle(&h-> mv_cache[0][scan8[i8*4]], 2, 2, 8, 0, 4);
1230                         fill_rectangle(&h-> mv_cache[1][scan8[i8*4]], 2, 2, 8, 0, 4);
1231                         continue;
1232                     }
1233
1234                     ref0 = l1ref0[x8 + (y8*2>>y_shift)*h->b8_stride];
1235                     if(ref0 >= 0)
1236                         ref0 = map_col_to_list0[0][ref0*2>>y_shift];
1237                     else{
1238                         ref0 = map_col_to_list0[1][l1ref1[x8 + (y8*2>>y_shift)*h->b8_stride]*2>>y_shift];
1239                         l1mv= l1mv1;
1240                     }
1241                     scale = dist_scale_factor[ref0];
1242                     fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, ref0, 1);
1243
1244                     {
1245                         const int16_t *mv_col = l1mv[x8*3 + (y8*6>>y_shift)*h->b_stride];
1246                         int my_col = (mv_col[1]<<y_shift)/2;
1247                         int mx = (scale * mv_col[0] + 128) >> 8;
1248                         int my = (scale * my_col + 128) >> 8;
1249                         fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, pack16to32(mx,my), 4);
1250                         fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, pack16to32(mx-mv_col[0],my-my_col), 4);
1251                     }
1252                 }
1253                 return;
1254             }
1255         }
1256
1257         /* one-to-one mv scaling */
1258
1259         if(IS_16X16(*mb_type)){
1260             int ref, mv0, mv1;
1261
1262             fill_rectangle(&h->ref_cache[1][scan8[0]], 4, 4, 8, 0, 1);
1263             if(IS_INTRA(mb_type_col)){
1264                 ref=mv0=mv1=0;
1265             }else{
1266                 const int ref0 = l1ref0[0] >= 0 ? map_col_to_list0[0][l1ref0[0]]
1267                                                 : map_col_to_list0[1][l1ref1[0]];
1268                 const int scale = dist_scale_factor[ref0];
1269                 const int16_t *mv_col = l1ref0[0] >= 0 ? l1mv0[0] : l1mv1[0];
1270                 int mv_l0[2];
1271                 mv_l0[0] = (scale * mv_col[0] + 128) >> 8;
1272                 mv_l0[1] = (scale * mv_col[1] + 128) >> 8;
1273                 ref= ref0;
1274                 mv0= pack16to32(mv_l0[0],mv_l0[1]);
1275                 mv1= pack16to32(mv_l0[0]-mv_col[0],mv_l0[1]-mv_col[1]);
1276             }
1277             fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, ref, 1);
1278             fill_rectangle(&h-> mv_cache[0][scan8[0]], 4, 4, 8, mv0, 4);
1279             fill_rectangle(&h-> mv_cache[1][scan8[0]], 4, 4, 8, mv1, 4);
1280         }else{
1281             for(i8=0; i8<4; i8++){
1282                 const int x8 = i8&1;
1283                 const int y8 = i8>>1;
1284                 int ref0, scale;
1285                 const int16_t (*l1mv)[2]= l1mv0;
1286
1287                 if(is_b8x8 && !IS_DIRECT(h->sub_mb_type[i8]))
1288                     continue;
1289                 h->sub_mb_type[i8] = sub_mb_type;
1290                 fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, 0, 1);
1291                 if(IS_INTRA(mb_type_col)){
1292                     fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, 0, 1);
1293                     fill_rectangle(&h-> mv_cache[0][scan8[i8*4]], 2, 2, 8, 0, 4);
1294                     fill_rectangle(&h-> mv_cache[1][scan8[i8*4]], 2, 2, 8, 0, 4);
1295                     continue;
1296                 }
1297
1298                 ref0 = l1ref0[x8 + y8*h->b8_stride];
1299                 if(ref0 >= 0)
1300                     ref0 = map_col_to_list0[0][ref0];
1301                 else{
1302                     ref0 = map_col_to_list0[1][l1ref1[x8 + y8*h->b8_stride]];
1303                     l1mv= l1mv1;
1304                 }
1305                 scale = dist_scale_factor[ref0];
1306
1307                 fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, ref0, 1);
1308                 if(IS_SUB_8X8(sub_mb_type)){
1309                     const int16_t *mv_col = l1mv[x8*3 + y8*3*h->b_stride];
1310                     int mx = (scale * mv_col[0] + 128) >> 8;
1311                     int my = (scale * mv_col[1] + 128) >> 8;
1312                     fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, pack16to32(mx,my), 4);
1313                     fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, pack16to32(mx-mv_col[0],my-mv_col[1]), 4);
1314                 }else
1315                 for(i4=0; i4<4; i4++){
1316                     const int16_t *mv_col = l1mv[x8*2 + (i4&1) + (y8*2 + (i4>>1))*h->b_stride];
1317                     int16_t *mv_l0 = h->mv_cache[0][scan8[i8*4+i4]];
1318                     mv_l0[0] = (scale * mv_col[0] + 128) >> 8;
1319                     mv_l0[1] = (scale * mv_col[1] + 128) >> 8;
1320                     *(uint32_t*)h->mv_cache[1][scan8[i8*4+i4]] =
1321                         pack16to32(mv_l0[0]-mv_col[0],mv_l0[1]-mv_col[1]);
1322                 }
1323             }
1324         }
1325     }
1326 }
1327
1328 static inline void write_back_motion(H264Context *h, int mb_type){
1329     MpegEncContext * const s = &h->s;
1330     const int b_xy = 4*s->mb_x + 4*s->mb_y*h->b_stride;
1331     const int b8_xy= 2*s->mb_x + 2*s->mb_y*h->b8_stride;
1332     int list;
1333
1334     if(!USES_LIST(mb_type, 0))
1335         fill_rectangle(&s->current_picture.ref_index[0][b8_xy], 2, 2, h->b8_stride, (uint8_t)LIST_NOT_USED, 1);
1336
1337     for(list=0; list<h->list_count; list++){
1338         int y;
1339         if(!USES_LIST(mb_type, list))
1340             continue;
1341
1342         for(y=0; y<4; y++){
1343             *(uint64_t*)s->current_picture.motion_val[list][b_xy + 0 + y*h->b_stride]= *(uint64_t*)h->mv_cache[list][scan8[0]+0 + 8*y];
1344             *(uint64_t*)s->current_picture.motion_val[list][b_xy + 2 + y*h->b_stride]= *(uint64_t*)h->mv_cache[list][scan8[0]+2 + 8*y];
1345         }
1346         if( h->pps.cabac ) {
1347             if(IS_SKIP(mb_type))
1348                 fill_rectangle(h->mvd_table[list][b_xy], 4, 4, h->b_stride, 0, 4);
1349             else
1350             for(y=0; y<4; y++){
1351                 *(uint64_t*)h->mvd_table[list][b_xy + 0 + y*h->b_stride]= *(uint64_t*)h->mvd_cache[list][scan8[0]+0 + 8*y];
1352                 *(uint64_t*)h->mvd_table[list][b_xy + 2 + y*h->b_stride]= *(uint64_t*)h->mvd_cache[list][scan8[0]+2 + 8*y];
1353             }
1354         }
1355
1356         {
1357             int8_t *ref_index = &s->current_picture.ref_index[list][b8_xy];
1358             ref_index[0+0*h->b8_stride]= h->ref_cache[list][scan8[0]];
1359             ref_index[1+0*h->b8_stride]= h->ref_cache[list][scan8[4]];
1360             ref_index[0+1*h->b8_stride]= h->ref_cache[list][scan8[8]];
1361             ref_index[1+1*h->b8_stride]= h->ref_cache[list][scan8[12]];
1362         }
1363     }
1364
1365     if(h->slice_type == FF_B_TYPE && h->pps.cabac){
1366         if(IS_8X8(mb_type)){
1367             uint8_t *direct_table = &h->direct_table[b8_xy];
1368             direct_table[1+0*h->b8_stride] = IS_DIRECT(h->sub_mb_type[1]) ? 1 : 0;
1369             direct_table[0+1*h->b8_stride] = IS_DIRECT(h->sub_mb_type[2]) ? 1 : 0;
1370             direct_table[1+1*h->b8_stride] = IS_DIRECT(h->sub_mb_type[3]) ? 1 : 0;
1371         }
1372     }
1373 }
1374
1375 /**
1376  * Decodes a network abstraction layer unit.
1377  * @param consumed is the number of bytes used as input
1378  * @param length is the length of the array
1379  * @param dst_length is the number of decoded bytes FIXME here or a decode rbsp tailing?
1380  * @returns decoded bytes, might be src+1 if no escapes
1381  */
1382 static const uint8_t *decode_nal(H264Context *h, const uint8_t *src, int *dst_length, int *consumed, int length){
1383     int i, si, di;
1384     uint8_t *dst;
1385     int bufidx;
1386
1387 //    src[0]&0x80;                //forbidden bit
1388     h->nal_ref_idc= src[0]>>5;
1389     h->nal_unit_type= src[0]&0x1F;
1390
1391     src++; length--;
1392 #if 0
1393     for(i=0; i<length; i++)
1394         printf("%2X ", src[i]);
1395 #endif
1396     for(i=0; i+1<length; i+=2){
1397         if(src[i]) continue;
1398         if(i>0 && src[i-1]==0) i--;
1399         if(i+2<length && src[i+1]==0 && src[i+2]<=3){
1400             if(src[i+2]!=3){
1401                 /* startcode, so we must be past the end */
1402                 length=i;
1403             }
1404             break;
1405         }
1406     }
1407
1408     if(i>=length-1){ //no escaped 0
1409         *dst_length= length;
1410         *consumed= length+1; //+1 for the header
1411         return src;
1412     }
1413
1414     bufidx = h->nal_unit_type == NAL_DPC ? 1 : 0; // use second escape buffer for inter data
1415     h->rbsp_buffer[bufidx]= av_fast_realloc(h->rbsp_buffer[bufidx], &h->rbsp_buffer_size[bufidx], length);
1416     dst= h->rbsp_buffer[bufidx];
1417
1418     if (dst == NULL){
1419         return NULL;
1420     }
1421
1422 //printf("decoding esc\n");
1423     si=di=0;
1424     while(si<length){
1425         //remove escapes (very rare 1:2^22)
1426         if(si+2<length && src[si]==0 && src[si+1]==0 && src[si+2]<=3){
1427             if(src[si+2]==3){ //escape
1428                 dst[di++]= 0;
1429                 dst[di++]= 0;
1430                 si+=3;
1431                 continue;
1432             }else //next start code
1433                 break;
1434         }
1435
1436         dst[di++]= src[si++];
1437     }
1438
1439     *dst_length= di;
1440     *consumed= si + 1;//+1 for the header
1441 //FIXME store exact number of bits in the getbitcontext (it is needed for decoding)
1442     return dst;
1443 }
1444
1445 /**
1446  * identifies the exact end of the bitstream
1447  * @return the length of the trailing, or 0 if damaged
1448  */
1449 static int decode_rbsp_trailing(H264Context *h, const uint8_t *src){
1450     int v= *src;
1451     int r;
1452
1453     tprintf(h->s.avctx, "rbsp trailing %X\n", v);
1454
1455     for(r=1; r<9; r++){
1456         if(v&1) return r;
1457         v>>=1;
1458     }
1459     return 0;
1460 }
1461
1462 /**
1463  * idct tranforms the 16 dc values and dequantize them.
1464  * @param qp quantization parameter
1465  */
1466 static void h264_luma_dc_dequant_idct_c(DCTELEM *block, int qp, int qmul){
1467 #define stride 16
1468     int i;
1469     int temp[16]; //FIXME check if this is a good idea
1470     static const int x_offset[4]={0, 1*stride, 4* stride,  5*stride};
1471     static const int y_offset[4]={0, 2*stride, 8* stride, 10*stride};
1472
1473 //memset(block, 64, 2*256);
1474 //return;
1475     for(i=0; i<4; i++){
1476         const int offset= y_offset[i];
1477         const int z0= block[offset+stride*0] + block[offset+stride*4];
1478         const int z1= block[offset+stride*0] - block[offset+stride*4];
1479         const int z2= block[offset+stride*1] - block[offset+stride*5];
1480         const int z3= block[offset+stride*1] + block[offset+stride*5];
1481
1482         temp[4*i+0]= z0+z3;
1483         temp[4*i+1]= z1+z2;
1484         temp[4*i+2]= z1-z2;
1485         temp[4*i+3]= z0-z3;
1486     }
1487
1488     for(i=0; i<4; i++){
1489         const int offset= x_offset[i];
1490         const int z0= temp[4*0+i] + temp[4*2+i];
1491         const int z1= temp[4*0+i] - temp[4*2+i];
1492         const int z2= temp[4*1+i] - temp[4*3+i];
1493         const int z3= temp[4*1+i] + temp[4*3+i];
1494
1495         block[stride*0 +offset]= ((((z0 + z3)*qmul + 128 ) >> 8)); //FIXME think about merging this into decode_resdual
1496         block[stride*2 +offset]= ((((z1 + z2)*qmul + 128 ) >> 8));
1497         block[stride*8 +offset]= ((((z1 - z2)*qmul + 128 ) >> 8));
1498         block[stride*10+offset]= ((((z0 - z3)*qmul + 128 ) >> 8));
1499     }
1500 }
1501
1502 #if 0
1503 /**
1504  * dct tranforms the 16 dc values.
1505  * @param qp quantization parameter ??? FIXME
1506  */
1507 static void h264_luma_dc_dct_c(DCTELEM *block/*, int qp*/){
1508 //    const int qmul= dequant_coeff[qp][0];
1509     int i;
1510     int temp[16]; //FIXME check if this is a good idea
1511     static const int x_offset[4]={0, 1*stride, 4* stride,  5*stride};
1512     static const int y_offset[4]={0, 2*stride, 8* stride, 10*stride};
1513
1514     for(i=0; i<4; i++){
1515         const int offset= y_offset[i];
1516         const int z0= block[offset+stride*0] + block[offset+stride*4];
1517         const int z1= block[offset+stride*0] - block[offset+stride*4];
1518         const int z2= block[offset+stride*1] - block[offset+stride*5];
1519         const int z3= block[offset+stride*1] + block[offset+stride*5];
1520
1521         temp[4*i+0]= z0+z3;
1522         temp[4*i+1]= z1+z2;
1523         temp[4*i+2]= z1-z2;
1524         temp[4*i+3]= z0-z3;
1525     }
1526
1527     for(i=0; i<4; i++){
1528         const int offset= x_offset[i];
1529         const int z0= temp[4*0+i] + temp[4*2+i];
1530         const int z1= temp[4*0+i] - temp[4*2+i];
1531         const int z2= temp[4*1+i] - temp[4*3+i];
1532         const int z3= temp[4*1+i] + temp[4*3+i];
1533
1534         block[stride*0 +offset]= (z0 + z3)>>1;
1535         block[stride*2 +offset]= (z1 + z2)>>1;
1536         block[stride*8 +offset]= (z1 - z2)>>1;
1537         block[stride*10+offset]= (z0 - z3)>>1;
1538     }
1539 }
1540 #endif
1541
1542 #undef xStride
1543 #undef stride
1544
1545 static void chroma_dc_dequant_idct_c(DCTELEM *block, int qp, int qmul){
1546     const int stride= 16*2;
1547     const int xStride= 16;
1548     int a,b,c,d,e;
1549
1550     a= block[stride*0 + xStride*0];
1551     b= block[stride*0 + xStride*1];
1552     c= block[stride*1 + xStride*0];
1553     d= block[stride*1 + xStride*1];
1554
1555     e= a-b;
1556     a= a+b;
1557     b= c-d;
1558     c= c+d;
1559
1560     block[stride*0 + xStride*0]= ((a+c)*qmul) >> 7;
1561     block[stride*0 + xStride*1]= ((e+b)*qmul) >> 7;
1562     block[stride*1 + xStride*0]= ((a-c)*qmul) >> 7;
1563     block[stride*1 + xStride*1]= ((e-b)*qmul) >> 7;
1564 }
1565
1566 #if 0
1567 static void chroma_dc_dct_c(DCTELEM *block){
1568     const int stride= 16*2;
1569     const int xStride= 16;
1570     int a,b,c,d,e;
1571
1572     a= block[stride*0 + xStride*0];
1573     b= block[stride*0 + xStride*1];
1574     c= block[stride*1 + xStride*0];
1575     d= block[stride*1 + xStride*1];
1576
1577     e= a-b;
1578     a= a+b;
1579     b= c-d;
1580     c= c+d;
1581
1582     block[stride*0 + xStride*0]= (a+c);
1583     block[stride*0 + xStride*1]= (e+b);
1584     block[stride*1 + xStride*0]= (a-c);
1585     block[stride*1 + xStride*1]= (e-b);
1586 }
1587 #endif
1588
1589 /**
1590  * gets the chroma qp.
1591  */
1592 static inline int get_chroma_qp(H264Context *h, int t, int qscale){
1593     return h->pps.chroma_qp_table[t][qscale & 0xff];
1594 }
1595
1596 //FIXME need to check that this does not overflow signed 32 bit for low qp, I am not sure, it's very close
1597 //FIXME check that gcc inlines this (and optimizes intra & separate_dc stuff away)
1598 static inline int quantize_c(DCTELEM *block, uint8_t *scantable, int qscale, int intra, int separate_dc){
1599     int i;
1600     const int * const quant_table= quant_coeff[qscale];
1601     const int bias= intra ? (1<<QUANT_SHIFT)/3 : (1<<QUANT_SHIFT)/6;
1602     const unsigned int threshold1= (1<<QUANT_SHIFT) - bias - 1;
1603     const unsigned int threshold2= (threshold1<<1);
1604     int last_non_zero;
1605
1606     if(separate_dc){
1607         if(qscale<=18){
1608             //avoid overflows
1609             const int dc_bias= intra ? (1<<(QUANT_SHIFT-2))/3 : (1<<(QUANT_SHIFT-2))/6;
1610             const unsigned int dc_threshold1= (1<<(QUANT_SHIFT-2)) - dc_bias - 1;
1611             const unsigned int dc_threshold2= (dc_threshold1<<1);
1612
1613             int level= block[0]*quant_coeff[qscale+18][0];
1614             if(((unsigned)(level+dc_threshold1))>dc_threshold2){
1615                 if(level>0){
1616                     level= (dc_bias + level)>>(QUANT_SHIFT-2);
1617                     block[0]= level;
1618                 }else{
1619                     level= (dc_bias - level)>>(QUANT_SHIFT-2);
1620                     block[0]= -level;
1621                 }
1622 //                last_non_zero = i;
1623             }else{
1624                 block[0]=0;
1625             }
1626         }else{
1627             const int dc_bias= intra ? (1<<(QUANT_SHIFT+1))/3 : (1<<(QUANT_SHIFT+1))/6;
1628             const unsigned int dc_threshold1= (1<<(QUANT_SHIFT+1)) - dc_bias - 1;
1629             const unsigned int dc_threshold2= (dc_threshold1<<1);
1630
1631             int level= block[0]*quant_table[0];
1632             if(((unsigned)(level+dc_threshold1))>dc_threshold2){
1633                 if(level>0){
1634                     level= (dc_bias + level)>>(QUANT_SHIFT+1);
1635                     block[0]= level;
1636                 }else{
1637                     level= (dc_bias - level)>>(QUANT_SHIFT+1);
1638                     block[0]= -level;
1639                 }
1640 //                last_non_zero = i;
1641             }else{
1642                 block[0]=0;
1643             }
1644         }
1645         last_non_zero= 0;
1646         i=1;
1647     }else{
1648         last_non_zero= -1;
1649         i=0;
1650     }
1651
1652     for(; i<16; i++){
1653         const int j= scantable[i];
1654         int level= block[j]*quant_table[j];
1655
1656 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
1657 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
1658         if(((unsigned)(level+threshold1))>threshold2){
1659             if(level>0){
1660                 level= (bias + level)>>QUANT_SHIFT;
1661                 block[j]= level;
1662             }else{
1663                 level= (bias - level)>>QUANT_SHIFT;
1664                 block[j]= -level;
1665             }
1666             last_non_zero = i;
1667         }else{
1668             block[j]=0;
1669         }
1670     }
1671
1672     return last_non_zero;
1673 }
1674
1675 static inline void mc_dir_part(H264Context *h, Picture *pic, int n, int square, int chroma_height, int delta, int list,
1676                            uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
1677                            int src_x_offset, int src_y_offset,
1678                            qpel_mc_func *qpix_op, h264_chroma_mc_func chroma_op){
1679     MpegEncContext * const s = &h->s;
1680     const int mx= h->mv_cache[list][ scan8[n] ][0] + src_x_offset*8;
1681     int my=       h->mv_cache[list][ scan8[n] ][1] + src_y_offset*8;
1682     const int luma_xy= (mx&3) + ((my&3)<<2);
1683     uint8_t * src_y = pic->data[0] + (mx>>2) + (my>>2)*h->mb_linesize;
1684     uint8_t * src_cb, * src_cr;
1685     int extra_width= h->emu_edge_width;
1686     int extra_height= h->emu_edge_height;
1687     int emu=0;
1688     const int full_mx= mx>>2;
1689     const int full_my= my>>2;
1690     const int pic_width  = 16*s->mb_width;
1691     const int pic_height = 16*s->mb_height >> MB_FIELD;
1692
1693     if(!pic->data[0]) //FIXME this is unacceptable, some senseable error concealment must be done for missing reference frames
1694         return;
1695
1696     if(mx&7) extra_width -= 3;
1697     if(my&7) extra_height -= 3;
1698
1699     if(   full_mx < 0-extra_width
1700        || full_my < 0-extra_height
1701        || full_mx + 16/*FIXME*/ > pic_width + extra_width
1702        || full_my + 16/*FIXME*/ > pic_height + extra_height){
1703         ff_emulated_edge_mc(s->edge_emu_buffer, src_y - 2 - 2*h->mb_linesize, h->mb_linesize, 16+5, 16+5/*FIXME*/, full_mx-2, full_my-2, pic_width, pic_height);
1704             src_y= s->edge_emu_buffer + 2 + 2*h->mb_linesize;
1705         emu=1;
1706     }
1707
1708     qpix_op[luma_xy](dest_y, src_y, h->mb_linesize); //FIXME try variable height perhaps?
1709     if(!square){
1710         qpix_op[luma_xy](dest_y + delta, src_y + delta, h->mb_linesize);
1711     }
1712
1713     if(ENABLE_GRAY && s->flags&CODEC_FLAG_GRAY) return;
1714
1715     if(MB_FIELD){
1716         // chroma offset when predicting from a field of opposite parity
1717         my += 2 * ((s->mb_y & 1) - (pic->reference - 1));
1718         emu |= (my>>3) < 0 || (my>>3) + 8 >= (pic_height>>1);
1719     }
1720     src_cb= pic->data[1] + (mx>>3) + (my>>3)*h->mb_uvlinesize;
1721     src_cr= pic->data[2] + (mx>>3) + (my>>3)*h->mb_uvlinesize;
1722
1723     if(emu){
1724         ff_emulated_edge_mc(s->edge_emu_buffer, src_cb, h->mb_uvlinesize, 9, 9/*FIXME*/, (mx>>3), (my>>3), pic_width>>1, pic_height>>1);
1725             src_cb= s->edge_emu_buffer;
1726     }
1727     chroma_op(dest_cb, src_cb, h->mb_uvlinesize, chroma_height, mx&7, my&7);
1728
1729     if(emu){
1730         ff_emulated_edge_mc(s->edge_emu_buffer, src_cr, h->mb_uvlinesize, 9, 9/*FIXME*/, (mx>>3), (my>>3), pic_width>>1, pic_height>>1);
1731             src_cr= s->edge_emu_buffer;
1732     }
1733     chroma_op(dest_cr, src_cr, h->mb_uvlinesize, chroma_height, mx&7, my&7);
1734 }
1735
1736 static inline void mc_part_std(H264Context *h, int n, int square, int chroma_height, int delta,
1737                            uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
1738                            int x_offset, int y_offset,
1739                            qpel_mc_func *qpix_put, h264_chroma_mc_func chroma_put,
1740                            qpel_mc_func *qpix_avg, h264_chroma_mc_func chroma_avg,
1741                            int list0, int list1){
1742     MpegEncContext * const s = &h->s;
1743     qpel_mc_func *qpix_op=  qpix_put;
1744     h264_chroma_mc_func chroma_op= chroma_put;
1745
1746     dest_y  += 2*x_offset + 2*y_offset*h->  mb_linesize;
1747     dest_cb +=   x_offset +   y_offset*h->mb_uvlinesize;
1748     dest_cr +=   x_offset +   y_offset*h->mb_uvlinesize;
1749     x_offset += 8*s->mb_x;
1750     y_offset += 8*(s->mb_y >> MB_FIELD);
1751
1752     if(list0){
1753         Picture *ref= &h->ref_list[0][ h->ref_cache[0][ scan8[n] ] ];
1754         mc_dir_part(h, ref, n, square, chroma_height, delta, 0,
1755                            dest_y, dest_cb, dest_cr, x_offset, y_offset,
1756                            qpix_op, chroma_op);
1757
1758         qpix_op=  qpix_avg;
1759         chroma_op= chroma_avg;
1760     }
1761
1762     if(list1){
1763         Picture *ref= &h->ref_list[1][ h->ref_cache[1][ scan8[n] ] ];
1764         mc_dir_part(h, ref, n, square, chroma_height, delta, 1,
1765                            dest_y, dest_cb, dest_cr, x_offset, y_offset,
1766                            qpix_op, chroma_op);
1767     }
1768 }
1769
1770 static inline void mc_part_weighted(H264Context *h, int n, int square, int chroma_height, int delta,
1771                            uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
1772                            int x_offset, int y_offset,
1773                            qpel_mc_func *qpix_put, h264_chroma_mc_func chroma_put,
1774                            h264_weight_func luma_weight_op, h264_weight_func chroma_weight_op,
1775                            h264_biweight_func luma_weight_avg, h264_biweight_func chroma_weight_avg,
1776                            int list0, int list1){
1777     MpegEncContext * const s = &h->s;
1778
1779     dest_y  += 2*x_offset + 2*y_offset*h->  mb_linesize;
1780     dest_cb +=   x_offset +   y_offset*h->mb_uvlinesize;
1781     dest_cr +=   x_offset +   y_offset*h->mb_uvlinesize;
1782     x_offset += 8*s->mb_x;
1783     y_offset += 8*(s->mb_y >> MB_FIELD);
1784
1785     if(list0 && list1){
1786         /* don't optimize for luma-only case, since B-frames usually
1787          * use implicit weights => chroma too. */
1788         uint8_t *tmp_cb = s->obmc_scratchpad;
1789         uint8_t *tmp_cr = s->obmc_scratchpad + 8;
1790         uint8_t *tmp_y  = s->obmc_scratchpad + 8*h->mb_uvlinesize;
1791         int refn0 = h->ref_cache[0][ scan8[n] ];
1792         int refn1 = h->ref_cache[1][ scan8[n] ];
1793
1794         mc_dir_part(h, &h->ref_list[0][refn0], n, square, chroma_height, delta, 0,
1795                     dest_y, dest_cb, dest_cr,
1796                     x_offset, y_offset, qpix_put, chroma_put);
1797         mc_dir_part(h, &h->ref_list[1][refn1], n, square, chroma_height, delta, 1,
1798                     tmp_y, tmp_cb, tmp_cr,
1799                     x_offset, y_offset, qpix_put, chroma_put);
1800
1801         if(h->use_weight == 2){
1802             int weight0 = h->implicit_weight[refn0][refn1];
1803             int weight1 = 64 - weight0;
1804             luma_weight_avg(  dest_y,  tmp_y,  h->  mb_linesize, 5, weight0, weight1, 0);
1805             chroma_weight_avg(dest_cb, tmp_cb, h->mb_uvlinesize, 5, weight0, weight1, 0);
1806             chroma_weight_avg(dest_cr, tmp_cr, h->mb_uvlinesize, 5, weight0, weight1, 0);
1807         }else{
1808             luma_weight_avg(dest_y, tmp_y, h->mb_linesize, h->luma_log2_weight_denom,
1809                             h->luma_weight[0][refn0], h->luma_weight[1][refn1],
1810                             h->luma_offset[0][refn0] + h->luma_offset[1][refn1]);
1811             chroma_weight_avg(dest_cb, tmp_cb, h->mb_uvlinesize, h->chroma_log2_weight_denom,
1812                             h->chroma_weight[0][refn0][0], h->chroma_weight[1][refn1][0],
1813                             h->chroma_offset[0][refn0][0] + h->chroma_offset[1][refn1][0]);
1814             chroma_weight_avg(dest_cr, tmp_cr, h->mb_uvlinesize, h->chroma_log2_weight_denom,
1815                             h->chroma_weight[0][refn0][1], h->chroma_weight[1][refn1][1],
1816                             h->chroma_offset[0][refn0][1] + h->chroma_offset[1][refn1][1]);
1817         }
1818     }else{
1819         int list = list1 ? 1 : 0;
1820         int refn = h->ref_cache[list][ scan8[n] ];
1821         Picture *ref= &h->ref_list[list][refn];
1822         mc_dir_part(h, ref, n, square, chroma_height, delta, list,
1823                     dest_y, dest_cb, dest_cr, x_offset, y_offset,
1824                     qpix_put, chroma_put);
1825
1826         luma_weight_op(dest_y, h->mb_linesize, h->luma_log2_weight_denom,
1827                        h->luma_weight[list][refn], h->luma_offset[list][refn]);
1828         if(h->use_weight_chroma){
1829             chroma_weight_op(dest_cb, h->mb_uvlinesize, h->chroma_log2_weight_denom,
1830                              h->chroma_weight[list][refn][0], h->chroma_offset[list][refn][0]);
1831             chroma_weight_op(dest_cr, h->mb_uvlinesize, h->chroma_log2_weight_denom,
1832                              h->chroma_weight[list][refn][1], h->chroma_offset[list][refn][1]);
1833         }
1834     }
1835 }
1836
1837 static inline void mc_part(H264Context *h, int n, int square, int chroma_height, int delta,
1838                            uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
1839                            int x_offset, int y_offset,
1840                            qpel_mc_func *qpix_put, h264_chroma_mc_func chroma_put,
1841                            qpel_mc_func *qpix_avg, h264_chroma_mc_func chroma_avg,
1842                            h264_weight_func *weight_op, h264_biweight_func *weight_avg,
1843                            int list0, int list1){
1844     if((h->use_weight==2 && list0 && list1
1845         && (h->implicit_weight[ h->ref_cache[0][scan8[n]] ][ h->ref_cache[1][scan8[n]] ] != 32))
1846        || h->use_weight==1)
1847         mc_part_weighted(h, n, square, chroma_height, delta, dest_y, dest_cb, dest_cr,
1848                          x_offset, y_offset, qpix_put, chroma_put,
1849                          weight_op[0], weight_op[3], weight_avg[0], weight_avg[3], list0, list1);
1850     else
1851         mc_part_std(h, n, square, chroma_height, delta, dest_y, dest_cb, dest_cr,
1852                     x_offset, y_offset, qpix_put, chroma_put, qpix_avg, chroma_avg, list0, list1);
1853 }
1854
1855 static inline void prefetch_motion(H264Context *h, int list){
1856     /* fetch pixels for estimated mv 4 macroblocks ahead
1857      * optimized for 64byte cache lines */
1858     MpegEncContext * const s = &h->s;
1859     const int refn = h->ref_cache[list][scan8[0]];
1860     if(refn >= 0){
1861         const int mx= (h->mv_cache[list][scan8[0]][0]>>2) + 16*s->mb_x + 8;
1862         const int my= (h->mv_cache[list][scan8[0]][1]>>2) + 16*s->mb_y;
1863         uint8_t **src= h->ref_list[list][refn].data;
1864         int off= mx + (my + (s->mb_x&3)*4)*h->mb_linesize + 64;
1865         s->dsp.prefetch(src[0]+off, s->linesize, 4);
1866         off= (mx>>1) + ((my>>1) + (s->mb_x&7))*s->uvlinesize + 64;
1867         s->dsp.prefetch(src[1]+off, src[2]-src[1], 2);
1868     }
1869 }
1870
1871 static void hl_motion(H264Context *h, uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr,
1872                       qpel_mc_func (*qpix_put)[16], h264_chroma_mc_func (*chroma_put),
1873                       qpel_mc_func (*qpix_avg)[16], h264_chroma_mc_func (*chroma_avg),
1874                       h264_weight_func *weight_op, h264_biweight_func *weight_avg){
1875     MpegEncContext * const s = &h->s;
1876     const int mb_xy= s->mb_x + s->mb_y*s->mb_stride;
1877     const int mb_type= s->current_picture.mb_type[mb_xy];
1878
1879     assert(IS_INTER(mb_type));
1880
1881     prefetch_motion(h, 0);
1882
1883     if(IS_16X16(mb_type)){
1884         mc_part(h, 0, 1, 8, 0, dest_y, dest_cb, dest_cr, 0, 0,
1885                 qpix_put[0], chroma_put[0], qpix_avg[0], chroma_avg[0],
1886                 &weight_op[0], &weight_avg[0],
1887                 IS_DIR(mb_type, 0, 0), IS_DIR(mb_type, 0, 1));
1888     }else if(IS_16X8(mb_type)){
1889         mc_part(h, 0, 0, 4, 8, dest_y, dest_cb, dest_cr, 0, 0,
1890                 qpix_put[1], chroma_put[0], qpix_avg[1], chroma_avg[0],
1891                 &weight_op[1], &weight_avg[1],
1892                 IS_DIR(mb_type, 0, 0), IS_DIR(mb_type, 0, 1));
1893         mc_part(h, 8, 0, 4, 8, dest_y, dest_cb, dest_cr, 0, 4,
1894                 qpix_put[1], chroma_put[0], qpix_avg[1], chroma_avg[0],
1895                 &weight_op[1], &weight_avg[1],
1896                 IS_DIR(mb_type, 1, 0), IS_DIR(mb_type, 1, 1));
1897     }else if(IS_8X16(mb_type)){
1898         mc_part(h, 0, 0, 8, 8*h->mb_linesize, dest_y, dest_cb, dest_cr, 0, 0,
1899                 qpix_put[1], chroma_put[1], qpix_avg[1], chroma_avg[1],
1900                 &weight_op[2], &weight_avg[2],
1901                 IS_DIR(mb_type, 0, 0), IS_DIR(mb_type, 0, 1));
1902         mc_part(h, 4, 0, 8, 8*h->mb_linesize, dest_y, dest_cb, dest_cr, 4, 0,
1903                 qpix_put[1], chroma_put[1], qpix_avg[1], chroma_avg[1],
1904                 &weight_op[2], &weight_avg[2],
1905                 IS_DIR(mb_type, 1, 0), IS_DIR(mb_type, 1, 1));
1906     }else{
1907         int i;
1908
1909         assert(IS_8X8(mb_type));
1910
1911         for(i=0; i<4; i++){
1912             const int sub_mb_type= h->sub_mb_type[i];
1913             const int n= 4*i;
1914             int x_offset= (i&1)<<2;
1915             int y_offset= (i&2)<<1;
1916
1917             if(IS_SUB_8X8(sub_mb_type)){
1918                 mc_part(h, n, 1, 4, 0, dest_y, dest_cb, dest_cr, x_offset, y_offset,
1919                     qpix_put[1], chroma_put[1], qpix_avg[1], chroma_avg[1],
1920                     &weight_op[3], &weight_avg[3],
1921                     IS_DIR(sub_mb_type, 0, 0), IS_DIR(sub_mb_type, 0, 1));
1922             }else if(IS_SUB_8X4(sub_mb_type)){
1923                 mc_part(h, n  , 0, 2, 4, dest_y, dest_cb, dest_cr, x_offset, y_offset,
1924                     qpix_put[2], chroma_put[1], qpix_avg[2], chroma_avg[1],
1925                     &weight_op[4], &weight_avg[4],
1926                     IS_DIR(sub_mb_type, 0, 0), IS_DIR(sub_mb_type, 0, 1));
1927                 mc_part(h, n+2, 0, 2, 4, dest_y, dest_cb, dest_cr, x_offset, y_offset+2,
1928                     qpix_put[2], chroma_put[1], qpix_avg[2], chroma_avg[1],
1929                     &weight_op[4], &weight_avg[4],
1930                     IS_DIR(sub_mb_type, 0, 0), IS_DIR(sub_mb_type, 0, 1));
1931             }else if(IS_SUB_4X8(sub_mb_type)){
1932                 mc_part(h, n  , 0, 4, 4*h->mb_linesize, dest_y, dest_cb, dest_cr, x_offset, y_offset,
1933                     qpix_put[2], chroma_put[2], qpix_avg[2], chroma_avg[2],
1934                     &weight_op[5], &weight_avg[5],
1935                     IS_DIR(sub_mb_type, 0, 0), IS_DIR(sub_mb_type, 0, 1));
1936                 mc_part(h, n+1, 0, 4, 4*h->mb_linesize, dest_y, dest_cb, dest_cr, x_offset+2, y_offset,
1937                     qpix_put[2], chroma_put[2], qpix_avg[2], chroma_avg[2],
1938                     &weight_op[5], &weight_avg[5],
1939                     IS_DIR(sub_mb_type, 0, 0), IS_DIR(sub_mb_type, 0, 1));
1940             }else{
1941                 int j;
1942                 assert(IS_SUB_4X4(sub_mb_type));
1943                 for(j=0; j<4; j++){
1944                     int sub_x_offset= x_offset + 2*(j&1);
1945                     int sub_y_offset= y_offset +   (j&2);
1946                     mc_part(h, n+j, 1, 2, 0, dest_y, dest_cb, dest_cr, sub_x_offset, sub_y_offset,
1947                         qpix_put[2], chroma_put[2], qpix_avg[2], chroma_avg[2],
1948                         &weight_op[6], &weight_avg[6],
1949                         IS_DIR(sub_mb_type, 0, 0), IS_DIR(sub_mb_type, 0, 1));
1950                 }
1951             }
1952         }
1953     }
1954
1955     prefetch_motion(h, 1);
1956 }
1957
1958 static av_cold void decode_init_vlc(void){
1959     static int done = 0;
1960
1961     if (!done) {
1962         int i;
1963         done = 1;
1964
1965         init_vlc(&chroma_dc_coeff_token_vlc, CHROMA_DC_COEFF_TOKEN_VLC_BITS, 4*5,
1966                  &chroma_dc_coeff_token_len [0], 1, 1,
1967                  &chroma_dc_coeff_token_bits[0], 1, 1, 1);
1968
1969         for(i=0; i<4; i++){
1970             init_vlc(&coeff_token_vlc[i], COEFF_TOKEN_VLC_BITS, 4*17,
1971                      &coeff_token_len [i][0], 1, 1,
1972                      &coeff_token_bits[i][0], 1, 1, 1);
1973         }
1974
1975         for(i=0; i<3; i++){
1976             init_vlc(&chroma_dc_total_zeros_vlc[i], CHROMA_DC_TOTAL_ZEROS_VLC_BITS, 4,
1977                      &chroma_dc_total_zeros_len [i][0], 1, 1,
1978                      &chroma_dc_total_zeros_bits[i][0], 1, 1, 1);
1979         }
1980         for(i=0; i<15; i++){
1981             init_vlc(&total_zeros_vlc[i], TOTAL_ZEROS_VLC_BITS, 16,
1982                      &total_zeros_len [i][0], 1, 1,
1983                      &total_zeros_bits[i][0], 1, 1, 1);
1984         }
1985
1986         for(i=0; i<6; i++){
1987             init_vlc(&run_vlc[i], RUN_VLC_BITS, 7,
1988                      &run_len [i][0], 1, 1,
1989                      &run_bits[i][0], 1, 1, 1);
1990         }
1991         init_vlc(&run7_vlc, RUN7_VLC_BITS, 16,
1992                  &run_len [6][0], 1, 1,
1993                  &run_bits[6][0], 1, 1, 1);
1994     }
1995 }
1996
1997 static void free_tables(H264Context *h){
1998     int i;
1999     H264Context *hx;
2000     av_freep(&h->intra4x4_pred_mode);
2001     av_freep(&h->chroma_pred_mode_table);
2002     av_freep(&h->cbp_table);
2003     av_freep(&h->mvd_table[0]);
2004     av_freep(&h->mvd_table[1]);
2005     av_freep(&h->direct_table);
2006     av_freep(&h->non_zero_count);
2007     av_freep(&h->slice_table_base);
2008     h->slice_table= NULL;
2009
2010     av_freep(&h->mb2b_xy);
2011     av_freep(&h->mb2b8_xy);
2012
2013     for(i = 0; i < MAX_SPS_COUNT; i++)
2014         av_freep(h->sps_buffers + i);
2015
2016     for(i = 0; i < MAX_PPS_COUNT; i++)
2017         av_freep(h->pps_buffers + i);
2018
2019     for(i = 0; i < h->s.avctx->thread_count; i++) {
2020         hx = h->thread_context[i];
2021         if(!hx) continue;
2022         av_freep(&hx->top_borders[1]);
2023         av_freep(&hx->top_borders[0]);
2024         av_freep(&hx->s.obmc_scratchpad);
2025     }
2026 }
2027
2028 static void init_dequant8_coeff_table(H264Context *h){
2029     int i,q,x;
2030     const int transpose = (h->s.dsp.h264_idct8_add != ff_h264_idct8_add_c); //FIXME ugly
2031     h->dequant8_coeff[0] = h->dequant8_buffer[0];
2032     h->dequant8_coeff[1] = h->dequant8_buffer[1];
2033
2034     for(i=0; i<2; i++ ){
2035         if(i && !memcmp(h->pps.scaling_matrix8[0], h->pps.scaling_matrix8[1], 64*sizeof(uint8_t))){
2036             h->dequant8_coeff[1] = h->dequant8_buffer[0];
2037             break;
2038         }
2039
2040         for(q=0; q<52; q++){
2041             int shift = ff_div6[q];
2042             int idx = ff_rem6[q];
2043             for(x=0; x<64; x++)
2044                 h->dequant8_coeff[i][q][transpose ? (x>>3)|((x&7)<<3) : x] =
2045                     ((uint32_t)dequant8_coeff_init[idx][ dequant8_coeff_init_scan[((x>>1)&12) | (x&3)] ] *
2046                     h->pps.scaling_matrix8[i][x]) << shift;
2047         }
2048     }
2049 }
2050
2051 static void init_dequant4_coeff_table(H264Context *h){
2052     int i,j,q,x;
2053     const int transpose = (h->s.dsp.h264_idct_add != ff_h264_idct_add_c); //FIXME ugly
2054     for(i=0; i<6; i++ ){
2055         h->dequant4_coeff[i] = h->dequant4_buffer[i];
2056         for(j=0; j<i; j++){
2057             if(!memcmp(h->pps.scaling_matrix4[j], h->pps.scaling_matrix4[i], 16*sizeof(uint8_t))){
2058                 h->dequant4_coeff[i] = h->dequant4_buffer[j];
2059                 break;
2060             }
2061         }
2062         if(j<i)
2063             continue;
2064
2065         for(q=0; q<52; q++){
2066             int shift = ff_div6[q] + 2;
2067             int idx = ff_rem6[q];
2068             for(x=0; x<16; x++)
2069                 h->dequant4_coeff[i][q][transpose ? (x>>2)|((x<<2)&0xF) : x] =
2070                     ((uint32_t)dequant4_coeff_init[idx][(x&1) + ((x>>2)&1)] *
2071                     h->pps.scaling_matrix4[i][x]) << shift;
2072         }
2073     }
2074 }
2075
2076 static void init_dequant_tables(H264Context *h){
2077     int i,x;
2078     init_dequant4_coeff_table(h);
2079     if(h->pps.transform_8x8_mode)
2080         init_dequant8_coeff_table(h);
2081     if(h->sps.transform_bypass){
2082         for(i=0; i<6; i++)
2083             for(x=0; x<16; x++)
2084                 h->dequant4_coeff[i][0][x] = 1<<6;
2085         if(h->pps.transform_8x8_mode)
2086             for(i=0; i<2; i++)
2087                 for(x=0; x<64; x++)
2088                     h->dequant8_coeff[i][0][x] = 1<<6;
2089     }
2090 }
2091
2092
2093 /**
2094  * allocates tables.
2095  * needs width/height
2096  */
2097 static int alloc_tables(H264Context *h){
2098     MpegEncContext * const s = &h->s;
2099     const int big_mb_num= s->mb_stride * (s->mb_height+1);
2100     int x,y;
2101
2102     CHECKED_ALLOCZ(h->intra4x4_pred_mode, big_mb_num * 8  * sizeof(uint8_t))
2103
2104     CHECKED_ALLOCZ(h->non_zero_count    , big_mb_num * 16 * sizeof(uint8_t))
2105     CHECKED_ALLOCZ(h->slice_table_base  , (big_mb_num+s->mb_stride) * sizeof(uint8_t))
2106     CHECKED_ALLOCZ(h->cbp_table, big_mb_num * sizeof(uint16_t))
2107
2108     CHECKED_ALLOCZ(h->chroma_pred_mode_table, big_mb_num * sizeof(uint8_t))
2109     CHECKED_ALLOCZ(h->mvd_table[0], 32*big_mb_num * sizeof(uint16_t));
2110     CHECKED_ALLOCZ(h->mvd_table[1], 32*big_mb_num * sizeof(uint16_t));
2111     CHECKED_ALLOCZ(h->direct_table, 32*big_mb_num * sizeof(uint8_t));
2112
2113     memset(h->slice_table_base, -1, (big_mb_num+s->mb_stride)  * sizeof(uint8_t));
2114     h->slice_table= h->slice_table_base + s->mb_stride*2 + 1;
2115
2116     CHECKED_ALLOCZ(h->mb2b_xy  , big_mb_num * sizeof(uint32_t));
2117     CHECKED_ALLOCZ(h->mb2b8_xy , big_mb_num * sizeof(uint32_t));
2118     for(y=0; y<s->mb_height; y++){
2119         for(x=0; x<s->mb_width; x++){
2120             const int mb_xy= x + y*s->mb_stride;
2121             const int b_xy = 4*x + 4*y*h->b_stride;
2122             const int b8_xy= 2*x + 2*y*h->b8_stride;
2123
2124             h->mb2b_xy [mb_xy]= b_xy;
2125             h->mb2b8_xy[mb_xy]= b8_xy;
2126         }
2127     }
2128
2129     s->obmc_scratchpad = NULL;
2130
2131     if(!h->dequant4_coeff[0])
2132         init_dequant_tables(h);
2133
2134     return 0;
2135 fail:
2136     free_tables(h);
2137     return -1;
2138 }
2139
2140 /**
2141  * Mimic alloc_tables(), but for every context thread.
2142  */
2143 static void clone_tables(H264Context *dst, H264Context *src){
2144     dst->intra4x4_pred_mode       = src->intra4x4_pred_mode;
2145     dst->non_zero_count           = src->non_zero_count;
2146     dst->slice_table              = src->slice_table;
2147     dst->cbp_table                = src->cbp_table;
2148     dst->mb2b_xy                  = src->mb2b_xy;
2149     dst->mb2b8_xy                 = src->mb2b8_xy;
2150     dst->chroma_pred_mode_table   = src->chroma_pred_mode_table;
2151     dst->mvd_table[0]             = src->mvd_table[0];
2152     dst->mvd_table[1]             = src->mvd_table[1];
2153     dst->direct_table             = src->direct_table;
2154
2155     dst->s.obmc_scratchpad = NULL;
2156     ff_h264_pred_init(&dst->hpc, src->s.codec_id);
2157 }
2158
2159 /**
2160  * Init context
2161  * Allocate buffers which are not shared amongst multiple threads.
2162  */
2163 static int context_init(H264Context *h){
2164     CHECKED_ALLOCZ(h->top_borders[0], h->s.mb_width * (16+8+8) * sizeof(uint8_t))
2165     CHECKED_ALLOCZ(h->top_borders[1], h->s.mb_width * (16+8+8) * sizeof(uint8_t))
2166
2167     return 0;
2168 fail:
2169     return -1; // free_tables will clean up for us
2170 }
2171
2172 static av_cold void common_init(H264Context *h){
2173     MpegEncContext * const s = &h->s;
2174
2175     s->width = s->avctx->width;
2176     s->height = s->avctx->height;
2177     s->codec_id= s->avctx->codec->id;
2178
2179     ff_h264_pred_init(&h->hpc, s->codec_id);
2180
2181     h->dequant_coeff_pps= -1;
2182     s->unrestricted_mv=1;
2183     s->decode=1; //FIXME
2184
2185     memset(h->pps.scaling_matrix4, 16, 6*16*sizeof(uint8_t));
2186     memset(h->pps.scaling_matrix8, 16, 2*64*sizeof(uint8_t));
2187 }
2188
2189 static av_cold int decode_init(AVCodecContext *avctx){
2190     H264Context *h= avctx->priv_data;
2191     MpegEncContext * const s = &h->s;
2192
2193     MPV_decode_defaults(s);
2194
2195     s->avctx = avctx;
2196     common_init(h);
2197
2198     s->out_format = FMT_H264;
2199     s->workaround_bugs= avctx->workaround_bugs;
2200
2201     // set defaults
2202 //    s->decode_mb= ff_h263_decode_mb;
2203     s->quarter_sample = 1;
2204     s->low_delay= 1;
2205     avctx->pix_fmt= PIX_FMT_YUV420P;
2206
2207     decode_init_vlc();
2208
2209     if(avctx->extradata_size > 0 && avctx->extradata &&
2210        *(char *)avctx->extradata == 1){
2211         h->is_avc = 1;
2212         h->got_avcC = 0;
2213     } else {
2214         h->is_avc = 0;
2215     }
2216
2217     h->thread_context[0] = h;
2218     return 0;
2219 }
2220
2221 static int frame_start(H264Context *h){
2222     MpegEncContext * const s = &h->s;
2223     int i;
2224
2225     if(MPV_frame_start(s, s->avctx) < 0)
2226         return -1;
2227     ff_er_frame_start(s);
2228     /*
2229      * MPV_frame_start uses pict_type to derive key_frame.
2230      * This is incorrect for H.264; IDR markings must be used.
2231      * Zero here; IDR markings per slice in frame or fields are OR'd in later.
2232      * See decode_nal_units().
2233      */
2234     s->current_picture_ptr->key_frame= 0;
2235
2236     assert(s->linesize && s->uvlinesize);
2237
2238     for(i=0; i<16; i++){
2239         h->block_offset[i]= 4*((scan8[i] - scan8[0])&7) + 4*s->linesize*((scan8[i] - scan8[0])>>3);
2240         h->block_offset[24+i]= 4*((scan8[i] - scan8[0])&7) + 8*s->linesize*((scan8[i] - scan8[0])>>3);
2241     }
2242     for(i=0; i<4; i++){
2243         h->block_offset[16+i]=
2244         h->block_offset[20+i]= 4*((scan8[i] - scan8[0])&7) + 4*s->uvlinesize*((scan8[i] - scan8[0])>>3);
2245         h->block_offset[24+16+i]=
2246         h->block_offset[24+20+i]= 4*((scan8[i] - scan8[0])&7) + 8*s->uvlinesize*((scan8[i] - scan8[0])>>3);
2247     }
2248
2249     /* can't be in alloc_tables because linesize isn't known there.
2250      * FIXME: redo bipred weight to not require extra buffer? */
2251     for(i = 0; i < s->avctx->thread_count; i++)
2252         if(!h->thread_context[i]->s.obmc_scratchpad)
2253             h->thread_context[i]->s.obmc_scratchpad = av_malloc(16*2*s->linesize + 8*2*s->uvlinesize);
2254
2255     /* some macroblocks will be accessed before they're available */
2256     if(FRAME_MBAFF || s->avctx->thread_count > 1)
2257         memset(h->slice_table, -1, (s->mb_height*s->mb_stride-1) * sizeof(uint8_t));
2258
2259 //    s->decode= (s->flags&CODEC_FLAG_PSNR) || !s->encoding || s->current_picture.reference /*|| h->contains_intra*/ || 1;
2260     return 0;
2261 }
2262
2263 static inline void backup_mb_border(H264Context *h, uint8_t *src_y, uint8_t *src_cb, uint8_t *src_cr, int linesize, int uvlinesize, int simple){
2264     MpegEncContext * const s = &h->s;
2265     int i;
2266
2267     src_y  -=   linesize;
2268     src_cb -= uvlinesize;
2269     src_cr -= uvlinesize;
2270
2271     // There are two lines saved, the line above the the top macroblock of a pair,
2272     // and the line above the bottom macroblock
2273     h->left_border[0]= h->top_borders[0][s->mb_x][15];
2274     for(i=1; i<17; i++){
2275         h->left_border[i]= src_y[15+i*  linesize];
2276     }
2277
2278     *(uint64_t*)(h->top_borders[0][s->mb_x]+0)= *(uint64_t*)(src_y +  16*linesize);
2279     *(uint64_t*)(h->top_borders[0][s->mb_x]+8)= *(uint64_t*)(src_y +8+16*linesize);
2280
2281     if(simple || !ENABLE_GRAY || !(s->flags&CODEC_FLAG_GRAY)){
2282         h->left_border[17  ]= h->top_borders[0][s->mb_x][16+7];
2283         h->left_border[17+9]= h->top_borders[0][s->mb_x][24+7];
2284         for(i=1; i<9; i++){
2285             h->left_border[i+17  ]= src_cb[7+i*uvlinesize];
2286             h->left_border[i+17+9]= src_cr[7+i*uvlinesize];
2287         }
2288         *(uint64_t*)(h->top_borders[0][s->mb_x]+16)= *(uint64_t*)(src_cb+8*uvlinesize);
2289         *(uint64_t*)(h->top_borders[0][s->mb_x]+24)= *(uint64_t*)(src_cr+8*uvlinesize);
2290     }
2291 }
2292
2293 static inline void xchg_mb_border(H264Context *h, uint8_t *src_y, uint8_t *src_cb, uint8_t *src_cr, int linesize, int uvlinesize, int xchg, int simple){
2294     MpegEncContext * const s = &h->s;
2295     int temp8, i;
2296     uint64_t temp64;
2297     int deblock_left;
2298     int deblock_top;
2299     int mb_xy;
2300
2301     if(h->deblocking_filter == 2) {
2302         mb_xy = s->mb_x + s->mb_y*s->mb_stride;
2303         deblock_left = h->slice_table[mb_xy] == h->slice_table[mb_xy - 1];
2304         deblock_top  = h->slice_table[mb_xy] == h->slice_table[h->top_mb_xy];
2305     } else {
2306         deblock_left = (s->mb_x > 0);
2307         deblock_top =  (s->mb_y > 0);
2308     }
2309
2310     src_y  -=   linesize + 1;
2311     src_cb -= uvlinesize + 1;
2312     src_cr -= uvlinesize + 1;
2313
2314 #define XCHG(a,b,t,xchg)\
2315 t= a;\
2316 if(xchg)\
2317     a= b;\
2318 b= t;
2319
2320     if(deblock_left){
2321         for(i = !deblock_top; i<17; i++){
2322             XCHG(h->left_border[i     ], src_y [i*  linesize], temp8, xchg);
2323         }
2324     }
2325
2326     if(deblock_top){
2327         XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+0), *(uint64_t*)(src_y +1), temp64, xchg);
2328         XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+8), *(uint64_t*)(src_y +9), temp64, 1);
2329         if(s->mb_x+1 < s->mb_width){
2330             XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x+1]), *(uint64_t*)(src_y +17), temp64, 1);
2331         }
2332     }
2333
2334     if(simple || !ENABLE_GRAY || !(s->flags&CODEC_FLAG_GRAY)){
2335         if(deblock_left){
2336             for(i = !deblock_top; i<9; i++){
2337                 XCHG(h->left_border[i+17  ], src_cb[i*uvlinesize], temp8, xchg);
2338                 XCHG(h->left_border[i+17+9], src_cr[i*uvlinesize], temp8, xchg);
2339             }
2340         }
2341         if(deblock_top){
2342             XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+16), *(uint64_t*)(src_cb+1), temp64, 1);
2343             XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+24), *(uint64_t*)(src_cr+1), temp64, 1);
2344         }
2345     }
2346 }
2347
2348 static inline void backup_pair_border(H264Context *h, uint8_t *src_y, uint8_t *src_cb, uint8_t *src_cr, int linesize, int uvlinesize){
2349     MpegEncContext * const s = &h->s;
2350     int i;
2351
2352     src_y  -= 2 *   linesize;
2353     src_cb -= 2 * uvlinesize;
2354     src_cr -= 2 * uvlinesize;
2355
2356     // There are two lines saved, the line above the the top macroblock of a pair,
2357     // and the line above the bottom macroblock
2358     h->left_border[0]= h->top_borders[0][s->mb_x][15];
2359     h->left_border[1]= h->top_borders[1][s->mb_x][15];
2360     for(i=2; i<34; i++){
2361         h->left_border[i]= src_y[15+i*  linesize];
2362     }
2363
2364     *(uint64_t*)(h->top_borders[0][s->mb_x]+0)= *(uint64_t*)(src_y +  32*linesize);
2365     *(uint64_t*)(h->top_borders[0][s->mb_x]+8)= *(uint64_t*)(src_y +8+32*linesize);
2366     *(uint64_t*)(h->top_borders[1][s->mb_x]+0)= *(uint64_t*)(src_y +  33*linesize);
2367     *(uint64_t*)(h->top_borders[1][s->mb_x]+8)= *(uint64_t*)(src_y +8+33*linesize);
2368
2369     if(!ENABLE_GRAY || !(s->flags&CODEC_FLAG_GRAY)){
2370         h->left_border[34     ]= h->top_borders[0][s->mb_x][16+7];
2371         h->left_border[34+   1]= h->top_borders[1][s->mb_x][16+7];
2372         h->left_border[34+18  ]= h->top_borders[0][s->mb_x][24+7];
2373         h->left_border[34+18+1]= h->top_borders[1][s->mb_x][24+7];
2374         for(i=2; i<18; i++){
2375             h->left_border[i+34   ]= src_cb[7+i*uvlinesize];
2376             h->left_border[i+34+18]= src_cr[7+i*uvlinesize];
2377         }
2378         *(uint64_t*)(h->top_borders[0][s->mb_x]+16)= *(uint64_t*)(src_cb+16*uvlinesize);
2379         *(uint64_t*)(h->top_borders[0][s->mb_x]+24)= *(uint64_t*)(src_cr+16*uvlinesize);
2380         *(uint64_t*)(h->top_borders[1][s->mb_x]+16)= *(uint64_t*)(src_cb+17*uvlinesize);
2381         *(uint64_t*)(h->top_borders[1][s->mb_x]+24)= *(uint64_t*)(src_cr+17*uvlinesize);
2382     }
2383 }
2384
2385 static inline void xchg_pair_border(H264Context *h, uint8_t *src_y, uint8_t *src_cb, uint8_t *src_cr, int linesize, int uvlinesize, int xchg){
2386     MpegEncContext * const s = &h->s;
2387     int temp8, i;
2388     uint64_t temp64;
2389     int deblock_left = (s->mb_x > 0);
2390     int deblock_top  = (s->mb_y > 1);
2391
2392     tprintf(s->avctx, "xchg_pair_border: src_y:%p src_cb:%p src_cr:%p ls:%d uvls:%d\n", src_y, src_cb, src_cr, linesize, uvlinesize);
2393
2394     src_y  -= 2 *   linesize + 1;
2395     src_cb -= 2 * uvlinesize + 1;
2396     src_cr -= 2 * uvlinesize + 1;
2397
2398 #define XCHG(a,b,t,xchg)\
2399 t= a;\
2400 if(xchg)\
2401     a= b;\
2402 b= t;
2403
2404     if(deblock_left){
2405         for(i = (!deblock_top)<<1; i<34; i++){
2406             XCHG(h->left_border[i     ], src_y [i*  linesize], temp8, xchg);
2407         }
2408     }
2409
2410     if(deblock_top){
2411         XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+0), *(uint64_t*)(src_y +1), temp64, xchg);
2412         XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+8), *(uint64_t*)(src_y +9), temp64, 1);
2413         XCHG(*(uint64_t*)(h->top_borders[1][s->mb_x]+0), *(uint64_t*)(src_y +1 +linesize), temp64, xchg);
2414         XCHG(*(uint64_t*)(h->top_borders[1][s->mb_x]+8), *(uint64_t*)(src_y +9 +linesize), temp64, 1);
2415         if(s->mb_x+1 < s->mb_width){
2416             XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x+1]), *(uint64_t*)(src_y +17), temp64, 1);
2417             XCHG(*(uint64_t*)(h->top_borders[1][s->mb_x+1]), *(uint64_t*)(src_y +17 +linesize), temp64, 1);
2418         }
2419     }
2420
2421     if(!ENABLE_GRAY || !(s->flags&CODEC_FLAG_GRAY)){
2422         if(deblock_left){
2423             for(i = (!deblock_top) << 1; i<18; i++){
2424                 XCHG(h->left_border[i+34   ], src_cb[i*uvlinesize], temp8, xchg);
2425                 XCHG(h->left_border[i+34+18], src_cr[i*uvlinesize], temp8, xchg);
2426             }
2427         }
2428         if(deblock_top){
2429             XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+16), *(uint64_t*)(src_cb+1), temp64, 1);
2430             XCHG(*(uint64_t*)(h->top_borders[0][s->mb_x]+24), *(uint64_t*)(src_cr+1), temp64, 1);
2431             XCHG(*(uint64_t*)(h->top_borders[1][s->mb_x]+16), *(uint64_t*)(src_cb+1 +uvlinesize), temp64, 1);
2432             XCHG(*(uint64_t*)(h->top_borders[1][s->mb_x]+24), *(uint64_t*)(src_cr+1 +uvlinesize), temp64, 1);
2433         }
2434     }
2435 }
2436
2437 static av_always_inline void hl_decode_mb_internal(H264Context *h, int simple){
2438     MpegEncContext * const s = &h->s;
2439     const int mb_x= s->mb_x;
2440     const int mb_y= s->mb_y;
2441     const int mb_xy= mb_x + mb_y*s->mb_stride;
2442     const int mb_type= s->current_picture.mb_type[mb_xy];
2443     uint8_t  *dest_y, *dest_cb, *dest_cr;
2444     int linesize, uvlinesize /*dct_offset*/;
2445     int i;
2446     int *block_offset = &h->block_offset[0];
2447     const unsigned int bottom = mb_y & 1;
2448     const int transform_bypass = (s->qscale == 0 && h->sps.transform_bypass), is_h264 = (simple || s->codec_id == CODEC_ID_H264);
2449     void (*idct_add)(uint8_t *dst, DCTELEM *block, int stride);
2450     void (*idct_dc_add)(uint8_t *dst, DCTELEM *block, int stride);
2451
2452     dest_y  = s->current_picture.data[0] + (mb_y * 16* s->linesize  ) + mb_x * 16;
2453     dest_cb = s->current_picture.data[1] + (mb_y * 8 * s->uvlinesize) + mb_x * 8;
2454     dest_cr = s->current_picture.data[2] + (mb_y * 8 * s->uvlinesize) + mb_x * 8;
2455
2456     s->dsp.prefetch(dest_y + (s->mb_x&3)*4*s->linesize + 64, s->linesize, 4);
2457     s->dsp.prefetch(dest_cb + (s->mb_x&7)*s->uvlinesize + 64, dest_cr - dest_cb, 2);
2458
2459     if (!simple && MB_FIELD) {
2460         linesize   = h->mb_linesize   = s->linesize * 2;
2461         uvlinesize = h->mb_uvlinesize = s->uvlinesize * 2;
2462         block_offset = &h->block_offset[24];
2463         if(mb_y&1){ //FIXME move out of this func?
2464             dest_y -= s->linesize*15;
2465             dest_cb-= s->uvlinesize*7;
2466             dest_cr-= s->uvlinesize*7;
2467         }
2468         if(FRAME_MBAFF) {
2469             int list;
2470             for(list=0; list<h->list_count; list++){
2471                 if(!USES_LIST(mb_type, list))
2472                     continue;
2473                 if(IS_16X16(mb_type)){
2474                     int8_t *ref = &h->ref_cache[list][scan8[0]];
2475                     fill_rectangle(ref, 4, 4, 8, (16+*ref)^(s->mb_y&1), 1);
2476                 }else{
2477                     for(i=0; i<16; i+=4){
2478                         //FIXME can refs be smaller than 8x8 when !direct_8x8_inference ?
2479                         int ref = h->ref_cache[list][scan8[i]];
2480                         if(ref >= 0)
2481                             fill_rectangle(&h->ref_cache[list][scan8[i]], 2, 2, 8, (16+ref)^(s->mb_y&1), 1);
2482                     }
2483                 }
2484             }
2485         }
2486     } else {
2487         linesize   = h->mb_linesize   = s->linesize;
2488         uvlinesize = h->mb_uvlinesize = s->uvlinesize;
2489 //        dct_offset = s->linesize * 16;
2490     }
2491
2492     if(transform_bypass){
2493         idct_dc_add =
2494         idct_add = IS_8x8DCT(mb_type) ? s->dsp.add_pixels8 : s->dsp.add_pixels4;
2495     }else if(IS_8x8DCT(mb_type)){
2496         idct_dc_add = s->dsp.h264_idct8_dc_add;
2497         idct_add = s->dsp.h264_idct8_add;
2498     }else{
2499         idct_dc_add = s->dsp.h264_idct_dc_add;
2500         idct_add = s->dsp.h264_idct_add;
2501     }
2502
2503     if(!simple && FRAME_MBAFF && h->deblocking_filter && IS_INTRA(mb_type)
2504        && (!bottom || !IS_INTRA(s->current_picture.mb_type[mb_xy-s->mb_stride]))){
2505         int mbt_y = mb_y&~1;
2506         uint8_t *top_y  = s->current_picture.data[0] + (mbt_y * 16* s->linesize  ) + mb_x * 16;
2507         uint8_t *top_cb = s->current_picture.data[1] + (mbt_y * 8 * s->uvlinesize) + mb_x * 8;
2508         uint8_t *top_cr = s->current_picture.data[2] + (mbt_y * 8 * s->uvlinesize) + mb_x * 8;
2509         xchg_pair_border(h, top_y, top_cb, top_cr, s->linesize, s->uvlinesize, 1);
2510     }
2511
2512     if (!simple && IS_INTRA_PCM(mb_type)) {
2513         unsigned int x, y;
2514
2515         // The pixels are stored in h->mb array in the same order as levels,
2516         // copy them in output in the correct order.
2517         for(i=0; i<16; i++) {
2518             for (y=0; y<4; y++) {
2519                 for (x=0; x<4; x++) {
2520                     *(dest_y + block_offset[i] + y*linesize + x) = h->mb[i*16+y*4+x];
2521                 }
2522             }
2523         }
2524         for(i=16; i<16+4; i++) {
2525             for (y=0; y<4; y++) {
2526                 for (x=0; x<4; x++) {
2527                     *(dest_cb + block_offset[i] + y*uvlinesize + x) = h->mb[i*16+y*4+x];
2528                 }
2529             }
2530         }
2531         for(i=20; i<20+4; i++) {
2532             for (y=0; y<4; y++) {
2533                 for (x=0; x<4; x++) {
2534                     *(dest_cr + block_offset[i] + y*uvlinesize + x) = h->mb[i*16+y*4+x];
2535                 }
2536             }
2537         }
2538     } else {
2539         if(IS_INTRA(mb_type)){
2540             if(h->deblocking_filter && (simple || !FRAME_MBAFF))
2541                 xchg_mb_border(h, dest_y, dest_cb, dest_cr, linesize, uvlinesize, 1, simple);
2542
2543             if(simple || !ENABLE_GRAY || !(s->flags&CODEC_FLAG_GRAY)){
2544                 h->hpc.pred8x8[ h->chroma_pred_mode ](dest_cb, uvlinesize);
2545                 h->hpc.pred8x8[ h->chroma_pred_mode ](dest_cr, uvlinesize);
2546             }
2547
2548             if(IS_INTRA4x4(mb_type)){
2549                 if(simple || !s->encoding){
2550                     if(IS_8x8DCT(mb_type)){
2551                         for(i=0; i<16; i+=4){
2552                             uint8_t * const ptr= dest_y + block_offset[i];
2553                             const int dir= h->intra4x4_pred_mode_cache[ scan8[i] ];
2554                             const int nnz = h->non_zero_count_cache[ scan8[i] ];
2555                             h->hpc.pred8x8l[ dir ](ptr, (h->topleft_samples_available<<i)&0x8000,
2556                                                    (h->topright_samples_available<<i)&0x4000, linesize);
2557                             if(nnz){
2558                                 if(nnz == 1 && h->mb[i*16])
2559                                     idct_dc_add(ptr, h->mb + i*16, linesize);
2560                                 else
2561                                     idct_add(ptr, h->mb + i*16, linesize);
2562                             }
2563                         }
2564                     }else
2565                     for(i=0; i<16; i++){
2566                         uint8_t * const ptr= dest_y + block_offset[i];
2567                         uint8_t *topright;
2568                         const int dir= h->intra4x4_pred_mode_cache[ scan8[i] ];
2569                         int nnz, tr;
2570
2571                         if(dir == DIAG_DOWN_LEFT_PRED || dir == VERT_LEFT_PRED){
2572                             const int topright_avail= (h->topright_samples_available<<i)&0x8000;
2573                             assert(mb_y || linesize <= block_offset[i]);
2574                             if(!topright_avail){
2575                                 tr= ptr[3 - linesize]*0x01010101;
2576                                 topright= (uint8_t*) &tr;
2577                             }else
2578                                 topright= ptr + 4 - linesize;
2579                         }else
2580                             topright= NULL;
2581
2582                         h->hpc.pred4x4[ dir ](ptr, topright, linesize);
2583                         nnz = h->non_zero_count_cache[ scan8[i] ];
2584                         if(nnz){
2585                             if(is_h264){
2586                                 if(nnz == 1 && h->mb[i*16])
2587                                     idct_dc_add(ptr, h->mb + i*16, linesize);
2588                                 else
2589                                     idct_add(ptr, h->mb + i*16, linesize);
2590                             }else
2591                                 svq3_add_idct_c(ptr, h->mb + i*16, linesize, s->qscale, 0);
2592                         }
2593                     }
2594                 }
2595             }else{
2596                 h->hpc.pred16x16[ h->intra16x16_pred_mode ](dest_y , linesize);
2597                 if(is_h264){
2598                     if(!transform_bypass)
2599                         h264_luma_dc_dequant_idct_c(h->mb, s->qscale, h->dequant4_coeff[0][s->qscale][0]);
2600                 }else
2601                     svq3_luma_dc_dequant_idct_c(h->mb, s->qscale);
2602             }
2603             if(h->deblocking_filter && (simple || !FRAME_MBAFF))
2604                 xchg_mb_border(h, dest_y, dest_cb, dest_cr, linesize, uvlinesize, 0, simple);
2605         }else if(is_h264){
2606             hl_motion(h, dest_y, dest_cb, dest_cr,
2607                       s->me.qpel_put, s->dsp.put_h264_chroma_pixels_tab,
2608                       s->me.qpel_avg, s->dsp.avg_h264_chroma_pixels_tab,
2609                       s->dsp.weight_h264_pixels_tab, s->dsp.biweight_h264_pixels_tab);
2610         }
2611
2612
2613         if(!IS_INTRA4x4(mb_type)){
2614             if(is_h264){
2615                 if(IS_INTRA16x16(mb_type)){
2616                     for(i=0; i<16; i++){
2617                         if(h->non_zero_count_cache[ scan8[i] ])
2618                             idct_add(dest_y + block_offset[i], h->mb + i*16, linesize);
2619                         else if(h->mb[i*16])
2620                             idct_dc_add(dest_y + block_offset[i], h->mb + i*16, linesize);
2621                     }
2622                 }else{
2623                     const int di = IS_8x8DCT(mb_type) ? 4 : 1;
2624                     for(i=0; i<16; i+=di){
2625                         int nnz = h->non_zero_count_cache[ scan8[i] ];
2626                         if(nnz){
2627                             if(nnz==1 && h->mb[i*16])
2628                                 idct_dc_add(dest_y + block_offset[i], h->mb + i*16, linesize);
2629                             else
2630                                 idct_add(dest_y + block_offset[i], h->mb + i*16, linesize);
2631                         }
2632                     }
2633                 }
2634             }else{
2635                 for(i=0; i<16; i++){
2636                     if(h->non_zero_count_cache[ scan8[i] ] || h->mb[i*16]){ //FIXME benchmark weird rule, & below
2637                         uint8_t * const ptr= dest_y + block_offset[i];
2638                         svq3_add_idct_c(ptr, h->mb + i*16, linesize, s->qscale, IS_INTRA(mb_type) ? 1 : 0);
2639                     }
2640                 }
2641             }
2642         }
2643
2644         if(simple || !ENABLE_GRAY || !(s->flags&CODEC_FLAG_GRAY)){
2645             uint8_t *dest[2] = {dest_cb, dest_cr};
2646             if(transform_bypass){
2647                 idct_add = idct_dc_add = s->dsp.add_pixels4;
2648             }else{
2649                 idct_add = s->dsp.h264_idct_add;
2650                 idct_dc_add = s->dsp.h264_idct_dc_add;
2651                 chroma_dc_dequant_idct_c(h->mb + 16*16, h->chroma_qp[0], h->dequant4_coeff[IS_INTRA(mb_type) ? 1:4][h->chroma_qp[0]][0]);
2652                 chroma_dc_dequant_idct_c(h->mb + 16*16+4*16, h->chroma_qp[1], h->dequant4_coeff[IS_INTRA(mb_type) ? 2:5][h->chroma_qp[1]][0]);
2653             }
2654             if(is_h264){
2655                 for(i=16; i<16+8; i++){
2656                     if(h->non_zero_count_cache[ scan8[i] ])
2657                         idct_add(dest[(i&4)>>2] + block_offset[i], h->mb + i*16, uvlinesize);
2658                     else if(h->mb[i*16])
2659                         idct_dc_add(dest[(i&4)>>2] + block_offset[i], h->mb + i*16, uvlinesize);
2660                 }
2661             }else{
2662                 for(i=16; i<16+8; i++){
2663                     if(h->non_zero_count_cache[ scan8[i] ] || h->mb[i*16]){
2664                         uint8_t * const ptr= dest[(i&4)>>2] + block_offset[i];
2665                         svq3_add_idct_c(ptr, h->mb + i*16, uvlinesize, chroma_qp[s->qscale + 12] - 12, 2);
2666                     }
2667                 }
2668             }
2669         }
2670     }
2671     if(h->deblocking_filter) {
2672         if (!simple && FRAME_MBAFF) {
2673             //FIXME try deblocking one mb at a time?
2674             // the reduction in load/storing mvs and such might outweigh the extra backup/xchg_border
2675             const int mb_y = s->mb_y - 1;
2676             uint8_t  *pair_dest_y, *pair_dest_cb, *pair_dest_cr;
2677             const int mb_xy= mb_x + mb_y*s->mb_stride;
2678             const int mb_type_top   = s->current_picture.mb_type[mb_xy];
2679             const int mb_type_bottom= s->current_picture.mb_type[mb_xy+s->mb_stride];
2680             if (!bottom) return;
2681             pair_dest_y  = s->current_picture.data[0] + (mb_y * 16* s->linesize  ) + mb_x * 16;
2682             pair_dest_cb = s->current_picture.data[1] + (mb_y * 8 * s->uvlinesize) + mb_x * 8;
2683             pair_dest_cr = s->current_picture.data[2] + (mb_y * 8 * s->uvlinesize) + mb_x * 8;
2684
2685             if(IS_INTRA(mb_type_top | mb_type_bottom))
2686                 xchg_pair_border(h, pair_dest_y, pair_dest_cb, pair_dest_cr, s->linesize, s->uvlinesize, 0);
2687
2688             backup_pair_border(h, pair_dest_y, pair_dest_cb, pair_dest_cr, s->linesize, s->uvlinesize);
2689             // deblock a pair
2690             // top
2691             s->mb_y--;
2692             tprintf(h->s.avctx, "call mbaff filter_mb mb_x:%d mb_y:%d pair_dest_y = %p, dest_y = %p\n", mb_x, mb_y, pair_dest_y, dest_y);
2693             fill_caches(h, mb_type_top, 1); //FIXME don't fill stuff which isn't used by filter_mb
2694             h->chroma_qp[0] = get_chroma_qp(h, 0, s->current_picture.qscale_table[mb_xy]);
2695             h->chroma_qp[1] = get_chroma_qp(h, 1, s->current_picture.qscale_table[mb_xy]);
2696             filter_mb(h, mb_x, mb_y, pair_dest_y, pair_dest_cb, pair_dest_cr, linesize, uvlinesize);
2697             // bottom
2698             s->mb_y++;
2699             tprintf(h->s.avctx, "call mbaff filter_mb\n");
2700             fill_caches(h, mb_type_bottom, 1); //FIXME don't fill stuff which isn't used by filter_mb
2701             h->chroma_qp[0] = get_chroma_qp(h, 0, s->current_picture.qscale_table[mb_xy+s->mb_stride]);
2702             h->chroma_qp[1] = get_chroma_qp(h, 1, s->current_picture.qscale_table[mb_xy+s->mb_stride]);
2703             filter_mb(h, mb_x, mb_y+1, dest_y, dest_cb, dest_cr, linesize, uvlinesize);
2704         } else {
2705             tprintf(h->s.avctx, "call filter_mb\n");
2706             backup_mb_border(h, dest_y, dest_cb, dest_cr, linesize, uvlinesize, simple);
2707             fill_caches(h, mb_type, 1); //FIXME don't fill stuff which isn't used by filter_mb
2708             filter_mb_fast(h, mb_x, mb_y, dest_y, dest_cb, dest_cr, linesize, uvlinesize);
2709         }
2710     }
2711 }
2712
2713 /**
2714  * Process a macroblock; this case avoids checks for expensive uncommon cases.
2715  */
2716 static void hl_decode_mb_simple(H264Context *h){
2717     hl_decode_mb_internal(h, 1);
2718 }
2719
2720 /**
2721  * Process a macroblock; this handles edge cases, such as interlacing.
2722  */
2723 static void av_noinline hl_decode_mb_complex(H264Context *h){
2724     hl_decode_mb_internal(h, 0);
2725 }
2726
2727 static void hl_decode_mb(H264Context *h){
2728     MpegEncContext * const s = &h->s;
2729     const int mb_x= s->mb_x;
2730     const int mb_y= s->mb_y;
2731     const int mb_xy= mb_x + mb_y*s->mb_stride;
2732     const int mb_type= s->current_picture.mb_type[mb_xy];
2733     int is_complex = FRAME_MBAFF || MB_FIELD || IS_INTRA_PCM(mb_type) || s->codec_id != CODEC_ID_H264 || (ENABLE_GRAY && (s->flags&CODEC_FLAG_GRAY)) || s->encoding;
2734
2735     if(!s->decode)
2736         return;
2737
2738     if (is_complex)
2739         hl_decode_mb_complex(h);
2740     else hl_decode_mb_simple(h);
2741 }
2742
2743 static void pic_as_field(Picture *pic, const int parity){
2744     int i;
2745     for (i = 0; i < 4; ++i) {
2746         if (parity == PICT_BOTTOM_FIELD)
2747             pic->data[i] += pic->linesize[i];
2748         pic->reference = parity;
2749         pic->linesize[i] *= 2;
2750     }
2751 }
2752
2753 static int split_field_copy(Picture *dest, Picture *src,
2754                             int parity, int id_add){
2755     int match = !!(src->reference & parity);
2756
2757     if (match) {
2758         *dest = *src;
2759         pic_as_field(dest, parity);
2760         dest->pic_id *= 2;
2761         dest->pic_id += id_add;
2762     }
2763
2764     return match;
2765 }
2766
2767 /**
2768  * Split one reference list into field parts, interleaving by parity
2769  * as per H.264 spec section 8.2.4.2.5. Output fields have their data pointers
2770  * set to look at the actual start of data for that field.
2771  *
2772  * @param dest output list
2773  * @param dest_len maximum number of fields to put in dest
2774  * @param src the source reference list containing fields and/or field pairs
2775  *            (aka short_ref/long_ref, or
2776  *             refFrameListXShortTerm/refFrameListLongTerm in spec-speak)
2777  * @param src_len number of Picture's in source (pairs and unmatched fields)
2778  * @param parity the parity of the picture being decoded/needing
2779  *        these ref pics (PICT_{TOP,BOTTOM}_FIELD)
2780  * @return number of fields placed in dest
2781  */
2782 static int split_field_half_ref_list(Picture *dest, int dest_len,
2783                                      Picture *src,  int src_len,  int parity){
2784     int same_parity   = 1;
2785     int same_i        = 0;
2786     int opp_i         = 0;
2787     int out_i;
2788     int field_output;
2789
2790     for (out_i = 0; out_i < dest_len; out_i += field_output) {
2791         if (same_parity && same_i < src_len) {
2792             field_output = split_field_copy(dest + out_i, src + same_i,
2793                                             parity, 1);
2794             same_parity = !field_output;
2795             same_i++;
2796
2797         } else if (opp_i < src_len) {
2798             field_output = split_field_copy(dest + out_i, src + opp_i,
2799                                             PICT_FRAME - parity, 0);
2800             same_parity = field_output;
2801             opp_i++;
2802
2803         } else {
2804             break;
2805         }
2806     }
2807
2808     return out_i;
2809 }
2810
2811 /**
2812  * Split the reference frame list into a reference field list.
2813  * This implements H.264 spec 8.2.4.2.5 for a combined input list.
2814  * The input list contains both reference field pairs and
2815  * unmatched reference fields; it is ordered as spec describes
2816  * RefPicListX for frames in 8.2.4.2.1 and 8.2.4.2.3, except that
2817  * unmatched field pairs are also present. Conceptually this is equivalent
2818  * to concatenation of refFrameListXShortTerm with refFrameListLongTerm.
2819  *
2820  * @param dest output reference list where ordered fields are to be placed
2821  * @param dest_len max number of fields to place at dest
2822  * @param src source reference list, as described above
2823  * @param src_len number of pictures (pairs and unmatched fields) in src
2824  * @param parity parity of field being currently decoded
2825  *        (one of PICT_{TOP,BOTTOM}_FIELD)
2826  * @param long_i index into src array that holds first long reference picture,
2827  *        or src_len if no long refs present.
2828  */
2829 static int split_field_ref_list(Picture *dest, int dest_len,
2830                                 Picture *src,  int src_len,
2831                                 int parity,    int long_i){
2832
2833     int i = split_field_half_ref_list(dest, dest_len, src, long_i, parity);
2834     dest += i;
2835     dest_len -= i;
2836
2837     i += split_field_half_ref_list(dest, dest_len, src + long_i,
2838                                    src_len - long_i, parity);
2839     return i;
2840 }
2841
2842 /**
2843  * fills the default_ref_list.
2844  */
2845 static int fill_default_ref_list(H264Context *h){
2846     MpegEncContext * const s = &h->s;
2847     int i;
2848     int smallest_poc_greater_than_current = -1;
2849     int structure_sel;
2850     Picture sorted_short_ref[32];
2851     Picture field_entry_list[2][32];
2852     Picture *frame_list[2];
2853
2854     if (FIELD_PICTURE) {
2855         structure_sel = PICT_FRAME;
2856         frame_list[0] = field_entry_list[0];
2857         frame_list[1] = field_entry_list[1];
2858     } else {
2859         structure_sel = 0;
2860         frame_list[0] = h->default_ref_list[0];
2861         frame_list[1] = h->default_ref_list[1];
2862     }
2863
2864     if(h->slice_type==FF_B_TYPE){
2865         int list;
2866         int len[2];
2867         int short_len[2];
2868         int out_i;
2869         int limit= INT_MIN;
2870
2871         /* sort frame according to poc in B slice */
2872         for(out_i=0; out_i<h->short_ref_count; out_i++){
2873             int best_i=INT_MIN;
2874             int best_poc=INT_MAX;
2875
2876             for(i=0; i<h->short_ref_count; i++){
2877                 const int poc= h->short_ref[i]->poc;
2878                 if(poc > limit && poc < best_poc){
2879                     best_poc= poc;
2880                     best_i= i;
2881                 }
2882             }
2883
2884             assert(best_i != INT_MIN);
2885
2886             limit= best_poc;
2887             sorted_short_ref[out_i]= *h->short_ref[best_i];
2888             tprintf(h->s.avctx, "sorted poc: %d->%d poc:%d fn:%d\n", best_i, out_i, sorted_short_ref[out_i].poc, sorted_short_ref[out_i].frame_num);
2889             if (-1 == smallest_poc_greater_than_current) {
2890                 if (h->short_ref[best_i]->poc >= s->current_picture_ptr->poc) {
2891                     smallest_poc_greater_than_current = out_i;
2892                 }
2893             }
2894         }
2895
2896         tprintf(h->s.avctx, "current poc: %d, smallest_poc_greater_than_current: %d\n", s->current_picture_ptr->poc, smallest_poc_greater_than_current);
2897
2898         // find the largest poc
2899         for(list=0; list<2; list++){
2900             int index = 0;
2901             int j= -99;
2902             int step= list ? -1 : 1;
2903
2904             for(i=0; i<h->short_ref_count && index < h->ref_count[list]; i++, j+=step) {
2905                 int sel;
2906                 while(j<0 || j>= h->short_ref_count){
2907                     if(j != -99 && step == (list ? -1 : 1))
2908                         return -1;
2909                     step = -step;
2910                     j= smallest_poc_greater_than_current + (step>>1);
2911                 }
2912                 sel = sorted_short_ref[j].reference | structure_sel;
2913                 if(sel != PICT_FRAME) continue;
2914                 frame_list[list][index  ]= sorted_short_ref[j];
2915                 frame_list[list][index++].pic_id= sorted_short_ref[j].frame_num;
2916             }
2917             short_len[list] = index;
2918
2919             for(i = 0; i < 16 && index < h->ref_count[ list ]; i++){
2920                 int sel;
2921                 if(h->long_ref[i] == NULL) continue;
2922                 sel = h->long_ref[i]->reference | structure_sel;
2923                 if(sel != PICT_FRAME) continue;
2924
2925                 frame_list[ list ][index  ]= *h->long_ref[i];
2926                 frame_list[ list ][index++].pic_id= i;
2927             }
2928             len[list] = index;
2929         }
2930
2931         for(list=0; list<2; list++){
2932             if (FIELD_PICTURE)
2933                 len[list] = split_field_ref_list(h->default_ref_list[list],
2934                                                  h->ref_count[list],
2935                                                  frame_list[list],
2936                                                  len[list],
2937                                                  s->picture_structure,
2938                                                  short_len[list]);
2939
2940             // swap the two first elements of L1 when L0 and L1 are identical
2941             if(list && len[0] > 1 && len[0] == len[1])
2942                 for(i=0; h->default_ref_list[0][i].data[0] == h->default_ref_list[1][i].data[0]; i++)
2943                     if(i == len[0]){
2944                         FFSWAP(Picture, h->default_ref_list[1][0], h->default_ref_list[1][1]);
2945                         break;
2946                     }
2947
2948             if(len[list] < h->ref_count[ list ])
2949                 memset(&h->default_ref_list[list][len[list]], 0, sizeof(Picture)*(h->ref_count[ list ] - len[list]));
2950         }
2951
2952
2953     }else{
2954         int index=0;
2955         int short_len;
2956         for(i=0; i<h->short_ref_count; i++){
2957             int sel;
2958             sel = h->short_ref[i]->reference | structure_sel;
2959             if(sel != PICT_FRAME) continue;
2960             frame_list[0][index  ]= *h->short_ref[i];
2961             frame_list[0][index++].pic_id= h->short_ref[i]->frame_num;
2962         }
2963         short_len = index;
2964         for(i = 0; i < 16; i++){
2965             int sel;
2966             if(h->long_ref[i] == NULL) continue;
2967             sel = h->long_ref[i]->reference | structure_sel;
2968             if(sel != PICT_FRAME) continue;
2969             frame_list[0][index  ]= *h->long_ref[i];
2970             frame_list[0][index++].pic_id= i;
2971         }
2972
2973         if (FIELD_PICTURE)
2974             index = split_field_ref_list(h->default_ref_list[0],
2975                                          h->ref_count[0], frame_list[0],
2976                                          index, s->picture_structure,
2977                                          short_len);
2978
2979         if(index < h->ref_count[0])
2980             memset(&h->default_ref_list[0][index], 0, sizeof(Picture)*(h->ref_count[0] - index));
2981     }
2982 #ifdef TRACE
2983     for (i=0; i<h->ref_count[0]; i++) {
2984         tprintf(h->s.avctx, "List0: %s fn:%d 0x%p\n", (h->default_ref_list[0][i].long_ref ? "LT" : "ST"), h->default_ref_list[0][i].pic_id, h->default_ref_list[0][i].data[0]);
2985     }
2986     if(h->slice_type==FF_B_TYPE){
2987         for (i=0; i<h->ref_count[1]; i++) {
2988             tprintf(h->s.avctx, "List1: %s fn:%d 0x%p\n", (h->default_ref_list[1][i].long_ref ? "LT" : "ST"), h->default_ref_list[1][i].pic_id, h->default_ref_list[1][i].data[0]);
2989         }
2990     }
2991 #endif
2992     return 0;
2993 }
2994
2995 static void print_short_term(H264Context *h);
2996 static void print_long_term(H264Context *h);
2997
2998 /**
2999  * Extract structure information about the picture described by pic_num in
3000  * the current decoding context (frame or field). Note that pic_num is
3001  * picture number without wrapping (so, 0<=pic_num<max_pic_num).
3002  * @param pic_num picture number for which to extract structure information
3003  * @param structure one of PICT_XXX describing structure of picture
3004  *                      with pic_num
3005  * @return frame number (short term) or long term index of picture
3006  *         described by pic_num
3007  */
3008 static int pic_num_extract(H264Context *h, int pic_num, int *structure){
3009     MpegEncContext * const s = &h->s;
3010
3011     *structure = s->picture_structure;
3012     if(FIELD_PICTURE){
3013         if (!(pic_num & 1))
3014             /* opposite field */
3015             *structure ^= PICT_FRAME;
3016         pic_num >>= 1;
3017     }
3018
3019     return pic_num;
3020 }
3021
3022 static int decode_ref_pic_list_reordering(H264Context *h){
3023     MpegEncContext * const s = &h->s;
3024     int list, index, pic_structure;
3025
3026     print_short_term(h);
3027     print_long_term(h);
3028     if(h->slice_type==FF_I_TYPE || h->slice_type==FF_SI_TYPE) return 0; //FIXME move before func
3029
3030     for(list=0; list<h->list_count; list++){
3031         memcpy(h->ref_list[list], h->default_ref_list[list], sizeof(Picture)*h->ref_count[list]);
3032
3033         if(get_bits1(&s->gb)){
3034             int pred= h->curr_pic_num;
3035
3036             for(index=0; ; index++){
3037                 unsigned int reordering_of_pic_nums_idc= get_ue_golomb(&s->gb);
3038                 unsigned int pic_id;
3039                 int i;
3040                 Picture *ref = NULL;
3041
3042                 if(reordering_of_pic_nums_idc==3)
3043                     break;
3044
3045                 if(index >= h->ref_count[list]){
3046                     av_log(h->s.avctx, AV_LOG_ERROR, "reference count overflow\n");
3047                     return -1;
3048                 }
3049
3050                 if(reordering_of_pic_nums_idc<3){
3051                     if(reordering_of_pic_nums_idc<2){
3052                         const unsigned int abs_diff_pic_num= get_ue_golomb(&s->gb) + 1;
3053                         int frame_num;
3054
3055                         if(abs_diff_pic_num > h->max_pic_num){
3056                             av_log(h->s.avctx, AV_LOG_ERROR, "abs_diff_pic_num overflow\n");
3057                             return -1;
3058                         }
3059
3060                         if(reordering_of_pic_nums_idc == 0) pred-= abs_diff_pic_num;
3061                         else                                pred+= abs_diff_pic_num;
3062                         pred &= h->max_pic_num - 1;
3063
3064                         frame_num = pic_num_extract(h, pred, &pic_structure);
3065
3066                         for(i= h->short_ref_count-1; i>=0; i--){
3067                             ref = h->short_ref[i];
3068                             assert(ref->reference);
3069                             assert(!ref->long_ref);
3070                             if(ref->data[0] != NULL &&
3071                                    ref->frame_num == frame_num &&
3072                                    (ref->reference & pic_structure) &&
3073                                    ref->long_ref == 0) // ignore non existing pictures by testing data[0] pointer
3074                                 break;
3075                         }
3076                         if(i>=0)
3077                             ref->pic_id= pred;
3078                     }else{
3079                         int long_idx;
3080                         pic_id= get_ue_golomb(&s->gb); //long_term_pic_idx
3081
3082                         long_idx= pic_num_extract(h, pic_id, &pic_structure);
3083
3084                         if(long_idx>31){
3085                             av_log(h->s.avctx, AV_LOG_ERROR, "long_term_pic_idx overflow\n");
3086                             return -1;
3087                         }
3088                         ref = h->long_ref[long_idx];
3089                         assert(!(ref && !ref->reference));
3090                         if(ref && (ref->reference & pic_structure)){
3091                             ref->pic_id= pic_id;
3092                             assert(ref->long_ref);
3093                             i=0;
3094                         }else{
3095                             i=-1;
3096                         }
3097                     }
3098
3099                     if (i < 0) {
3100                         av_log(h->s.avctx, AV_LOG_ERROR, "reference picture missing during reorder\n");
3101                         memset(&h->ref_list[list][index], 0, sizeof(Picture)); //FIXME
3102                     } else {
3103                         for(i=index; i+1<h->ref_count[list]; i++){
3104                             if(ref->long_ref == h->ref_list[list][i].long_ref && ref->pic_id == h->ref_list[list][i].pic_id)
3105                                 break;
3106                         }
3107                         for(; i > index; i--){
3108                             h->ref_list[list][i]= h->ref_list[list][i-1];
3109                         }
3110                         h->ref_list[list][index]= *ref;
3111                         if (FIELD_PICTURE){
3112                             pic_as_field(&h->ref_list[list][index], pic_structure);
3113                         }
3114                     }
3115                 }else{
3116                     av_log(h->s.avctx, AV_LOG_ERROR, "illegal reordering_of_pic_nums_idc\n");
3117                     return -1;
3118                 }
3119             }
3120         }
3121     }
3122     for(list=0; list<h->list_count; list++){
3123         for(index= 0; index < h->ref_count[list]; index++){
3124             if(!h->ref_list[list][index].data[0])
3125                 h->ref_list[list][index]= s->current_picture;
3126         }
3127     }
3128
3129     if(h->slice_type==FF_B_TYPE && !h->direct_spatial_mv_pred)
3130         direct_dist_scale_factor(h);
3131     direct_ref_list_init(h);
3132     return 0;
3133 }
3134
3135 static void fill_mbaff_ref_list(H264Context *h){
3136     int list, i, j;
3137     for(list=0; list<2; list++){ //FIXME try list_count
3138         for(i=0; i<h->ref_count[list]; i++){
3139             Picture *frame = &h->ref_list[list][i];
3140             Picture *field = &h->ref_list[list][16+2*i];
3141             field[0] = *frame;
3142             for(j=0; j<3; j++)
3143                 field[0].linesize[j] <<= 1;
3144             field[0].reference = PICT_TOP_FIELD;
3145             field[1] = field[0];
3146             for(j=0; j<3; j++)
3147                 field[1].data[j] += frame->linesize[j];
3148             field[1].reference = PICT_BOTTOM_FIELD;
3149
3150             h->luma_weight[list][16+2*i] = h->luma_weight[list][16+2*i+1] = h->luma_weight[list][i];
3151             h->luma_offset[list][16+2*i] = h->luma_offset[list][16+2*i+1] = h->luma_offset[list][i];
3152             for(j=0; j<2; j++){
3153                 h->chroma_weight[list][16+2*i][j] = h->chroma_weight[list][16+2*i+1][j] = h->chroma_weight[list][i][j];
3154                 h->chroma_offset[list][16+2*i][j] = h->chroma_offset[list][16+2*i+1][j] = h->chroma_offset[list][i][j];
3155             }
3156         }
3157     }
3158     for(j=0; j<h->ref_count[1]; j++){
3159         for(i=0; i<h->ref_count[0]; i++)
3160             h->implicit_weight[j][16+2*i] = h->implicit_weight[j][16+2*i+1] = h->implicit_weight[j][i];
3161         memcpy(h->implicit_weight[16+2*j],   h->implicit_weight[j], sizeof(*h->implicit_weight));
3162         memcpy(h->implicit_weight[16+2*j+1], h->implicit_weight[j], sizeof(*h->implicit_weight));
3163     }
3164 }
3165
3166 static int pred_weight_table(H264Context *h){
3167     MpegEncContext * const s = &h->s;
3168     int list, i;
3169     int luma_def, chroma_def;
3170
3171     h->use_weight= 0;
3172     h->use_weight_chroma= 0;
3173     h->luma_log2_weight_denom= get_ue_golomb(&s->gb);
3174     h->chroma_log2_weight_denom= get_ue_golomb(&s->gb);
3175     luma_def = 1<<h->luma_log2_weight_denom;
3176     chroma_def = 1<<h->chroma_log2_weight_denom;
3177
3178     for(list=0; list<2; list++){
3179         for(i=0; i<h->ref_count[list]; i++){
3180             int luma_weight_flag, chroma_weight_flag;
3181
3182             luma_weight_flag= get_bits1(&s->gb);
3183             if(luma_weight_flag){
3184                 h->luma_weight[list][i]= get_se_golomb(&s->gb);
3185                 h->luma_offset[list][i]= get_se_golomb(&s->gb);
3186                 if(   h->luma_weight[list][i] != luma_def
3187                    || h->luma_offset[list][i] != 0)
3188                     h->use_weight= 1;
3189             }else{
3190                 h->luma_weight[list][i]= luma_def;
3191                 h->luma_offset[list][i]= 0;
3192             }
3193
3194             chroma_weight_flag= get_bits1(&s->gb);
3195             if(chroma_weight_flag){
3196                 int j;
3197                 for(j=0; j<2; j++){
3198                     h->chroma_weight[list][i][j]= get_se_golomb(&s->gb);
3199                     h->chroma_offset[list][i][j]= get_se_golomb(&s->gb);
3200                     if(   h->chroma_weight[list][i][j] != chroma_def
3201                        || h->chroma_offset[list][i][j] != 0)
3202                         h->use_weight_chroma= 1;
3203                 }
3204             }else{
3205                 int j;
3206                 for(j=0; j<2; j++){
3207                     h->chroma_weight[list][i][j]= chroma_def;
3208                     h->chroma_offset[list][i][j]= 0;
3209                 }
3210             }
3211         }
3212         if(h->slice_type != FF_B_TYPE) break;
3213     }
3214     h->use_weight= h->use_weight || h->use_weight_chroma;
3215     return 0;
3216 }
3217
3218 static void implicit_weight_table(H264Context *h){
3219     MpegEncContext * const s = &h->s;
3220     int ref0, ref1;
3221     int cur_poc = s->current_picture_ptr->poc;
3222
3223     if(   h->ref_count[0] == 1 && h->ref_count[1] == 1
3224        && h->ref_list[0][0].poc + h->ref_list[1][0].poc == 2*cur_poc){
3225         h->use_weight= 0;
3226         h->use_weight_chroma= 0;
3227         return;
3228     }
3229
3230     h->use_weight= 2;
3231     h->use_weight_chroma= 2;
3232     h->luma_log2_weight_denom= 5;
3233     h->chroma_log2_weight_denom= 5;
3234
3235     for(ref0=0; ref0 < h->ref_count[0]; ref0++){
3236         int poc0 = h->ref_list[0][ref0].poc;
3237         for(ref1=0; ref1 < h->ref_count[1]; ref1++){
3238             int poc1 = h->ref_list[1][ref1].poc;
3239             int td = av_clip(poc1 - poc0, -128, 127);
3240             if(td){
3241                 int tb = av_clip(cur_poc - poc0, -128, 127);
3242                 int tx = (16384 + (FFABS(td) >> 1)) / td;
3243                 int dist_scale_factor = av_clip((tb*tx + 32) >> 6, -1024, 1023) >> 2;
3244                 if(dist_scale_factor < -64 || dist_scale_factor > 128)
3245                     h->implicit_weight[ref0][ref1] = 32;
3246                 else
3247                     h->implicit_weight[ref0][ref1] = 64 - dist_scale_factor;
3248             }else
3249                 h->implicit_weight[ref0][ref1] = 32;
3250         }
3251     }
3252 }
3253
3254 /**
3255  * Mark a picture as no longer needed for reference. The refmask
3256  * argument allows unreferencing of individual fields or the whole frame.
3257  * If the picture becomes entirely unreferenced, but is being held for
3258  * display purposes, it is marked as such.
3259  * @param refmask mask of fields to unreference; the mask is bitwise
3260  *                anded with the reference marking of pic
3261  * @return non-zero if pic becomes entirely unreferenced (except possibly
3262  *         for display purposes) zero if one of the fields remains in
3263  *         reference
3264  */
3265 static inline int unreference_pic(H264Context *h, Picture *pic, int refmask){
3266     int i;
3267     if (pic->reference &= refmask) {
3268         return 0;
3269     } else {
3270         if(pic == h->delayed_output_pic)
3271             pic->reference=DELAYED_PIC_REF;
3272         else{
3273             for(i = 0; h->delayed_pic[i]; i++)
3274                 if(pic == h->delayed_pic[i]){
3275                     pic->reference=DELAYED_PIC_REF;
3276                     break;
3277                 }
3278         }
3279         return 1;
3280     }
3281 }
3282
3283 /**
3284  * instantaneous decoder refresh.
3285  */
3286 static void idr(H264Context *h){
3287     int i;
3288
3289     for(i=0; i<16; i++){
3290         if (h->long_ref[i] != NULL) {
3291             unreference_pic(h, h->long_ref[i], 0);
3292             h->long_ref[i]= NULL;
3293         }
3294     }
3295     h->long_ref_count=0;
3296
3297     for(i=0; i<h->short_ref_count; i++){
3298         unreference_pic(h, h->short_ref[i], 0);
3299         h->short_ref[i]= NULL;
3300     }
3301     h->short_ref_count=0;
3302 }
3303
3304 /* forget old pics after a seek */
3305 static void flush_dpb(AVCodecContext *avctx){
3306     H264Context *h= avctx->priv_data;
3307     int i;
3308     for(i=0; i<16; i++) {
3309         if(h->delayed_pic[i])
3310             h->delayed_pic[i]->reference= 0;
3311         h->delayed_pic[i]= NULL;
3312     }
3313     if(h->delayed_output_pic)
3314         h->delayed_output_pic->reference= 0;
3315     h->delayed_output_pic= NULL;
3316     idr(h);
3317     if(h->s.current_picture_ptr)
3318         h->s.current_picture_ptr->reference= 0;
3319     h->s.first_field= 0;
3320     ff_mpeg_flush(avctx);
3321 }
3322
3323 /**
3324  * Find a Picture in the short term reference list by frame number.
3325  * @param frame_num frame number to search for
3326  * @param idx the index into h->short_ref where returned picture is found
3327  *            undefined if no picture found.
3328  * @return pointer to the found picture, or NULL if no pic with the provided
3329  *                 frame number is found
3330  */
3331 static Picture * find_short(H264Context *h, int frame_num, int *idx){
3332     MpegEncContext * const s = &h->s;
3333     int i;
3334
3335     for(i=0; i<h->short_ref_count; i++){
3336         Picture *pic= h->short_ref[i];
3337         if(s->avctx->debug&FF_DEBUG_MMCO)
3338             av_log(h->s.avctx, AV_LOG_DEBUG, "%d %d %p\n", i, pic->frame_num, pic);
3339         if(pic->frame_num == frame_num) {
3340             *idx = i;
3341             return pic;
3342         }
3343     }
3344     return NULL;
3345 }
3346
3347 /**
3348  * Remove a picture from the short term reference list by its index in
3349  * that list.  This does no checking on the provided index; it is assumed
3350  * to be valid. Other list entries are shifted down.
3351  * @param i index into h->short_ref of picture to remove.
3352  */
3353 static void remove_short_at_index(H264Context *h, int i){
3354     assert(i > 0 && i < h->short_ref_count);
3355     h->short_ref[i]= NULL;
3356     if (--h->short_ref_count)
3357         memmove(&h->short_ref[i], &h->short_ref[i+1], (h->short_ref_count - i)*sizeof(Picture*));
3358 }
3359
3360 /**
3361  *
3362  * @return the removed picture or NULL if an error occurs
3363  */
3364 static Picture * remove_short(H264Context *h, int frame_num){
3365     MpegEncContext * const s = &h->s;
3366     Picture *pic;
3367     int i;
3368
3369     if(s->avctx->debug&FF_DEBUG_MMCO)
3370         av_log(h->s.avctx, AV_LOG_DEBUG, "remove short %d count %d\n", frame_num, h->short_ref_count);
3371
3372     pic = find_short(h, frame_num, &i);
3373     if (pic)
3374         remove_short_at_index(h, i);
3375
3376     return pic;
3377 }
3378
3379 /**
3380  * Remove a picture from the long term reference list by its index in
3381  * that list.  This does no checking on the provided index; it is assumed
3382  * to be valid. The removed entry is set to NULL. Other entries are unaffected.
3383  * @param i index into h->long_ref of picture to remove.
3384  */
3385 static void remove_long_at_index(H264Context *h, int i){
3386     h->long_ref[i]= NULL;
3387     h->long_ref_count--;
3388 }
3389
3390 /**
3391  *
3392  * @return the removed picture or NULL if an error occurs
3393  */
3394 static Picture * remove_long(H264Context *h, int i){
3395     Picture *pic;
3396
3397     pic= h->long_ref[i];
3398     if (pic)
3399         remove_long_at_index(h, i);
3400
3401     return pic;
3402 }
3403
3404 /**
3405  * print short term list
3406  */
3407 static void print_short_term(H264Context *h) {
3408     uint32_t i;
3409     if(h->s.avctx->debug&FF_DEBUG_MMCO) {
3410         av_log(h->s.avctx, AV_LOG_DEBUG, "short term list:\n");
3411         for(i=0; i<h->short_ref_count; i++){
3412             Picture *pic= h->short_ref[i];
3413             av_log(h->s.avctx, AV_LOG_DEBUG, "%d fn:%d poc:%d %p\n", i, pic->frame_num, pic->poc, pic->data[0]);
3414         }
3415     }
3416 }
3417
3418 /**
3419  * print long term list
3420  */
3421 static void print_long_term(H264Context *h) {
3422     uint32_t i;
3423     if(h->s.avctx->debug&FF_DEBUG_MMCO) {
3424         av_log(h->s.avctx, AV_LOG_DEBUG, "long term list:\n");